Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 67dddaad5d8b8c5ee5b96a7e2f6cb0faad703865

Authored by Masami Hiramatsu
Committed by Linus Torvalds
1 parent 24aac480e7
Exists in master and in 7 other branches imx_3.0.35_4.1.0, imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga, smarc-l5.0.0_1.0.0-ga

kprobes: fix error checking of batch registration 

Fix error checking routine to catch an error which occurs in first
__register_*probe().

Signed-off-by: Masami Hiramatsu <mhiramat@redhat.com>
Cc: Ananth N Mavinakayanahalli <ananth@in.ibm.com>
Cc: Jim Keniston <jkenisto@us.ibm.com>
Cc: David Miller <davem@davemloft.net>
Cc: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 1 changed file with 9 additions and 6 deletions Inline Diff

1 /* 1 /*
2 * Kernel Probes (KProbes) 2 * Kernel Probes (KProbes)
3 * kernel/kprobes.c 3 * kernel/kprobes.c
4 * 4 *
5 * This program is free software; you can redistribute it and/or modify 5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by 6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or 7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version. 8 * (at your option) any later version.
9 * 9 *
10 * This program is distributed in the hope that it will be useful, 10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details. 13 * GNU General Public License for more details.
14 * 14 *
15 * You should have received a copy of the GNU General Public License 15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software 16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 * 18 *
19 * Copyright (C) IBM Corporation, 2002, 2004 19 * Copyright (C) IBM Corporation, 2002, 2004
20 * 20 *
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel 21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation (includes suggestions from 22 * Probes initial implementation (includes suggestions from
23 * Rusty Russell). 23 * Rusty Russell).
24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with 24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25 * hlists and exceptions notifier as suggested by Andi Kleen. 25 * hlists and exceptions notifier as suggested by Andi Kleen.
26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes 26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27 * interface to access function arguments. 27 * interface to access function arguments.
28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes 28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29 * exceptions notifier to be first on the priority list. 29 * exceptions notifier to be first on the priority list.
30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston 30 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi 31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32 * <prasanna@in.ibm.com> added function-return probes. 32 * <prasanna@in.ibm.com> added function-return probes.
33 */ 33 */
34 #include <linux/kprobes.h> 34 #include <linux/kprobes.h>
35 #include <linux/hash.h> 35 #include <linux/hash.h>
36 #include <linux/init.h> 36 #include <linux/init.h>
37 #include <linux/slab.h> 37 #include <linux/slab.h>
38 #include <linux/stddef.h> 38 #include <linux/stddef.h>
39 #include <linux/module.h> 39 #include <linux/module.h>
40 #include <linux/moduleloader.h> 40 #include <linux/moduleloader.h>
41 #include <linux/kallsyms.h> 41 #include <linux/kallsyms.h>
42 #include <linux/freezer.h> 42 #include <linux/freezer.h>
43 #include <linux/seq_file.h> 43 #include <linux/seq_file.h>
44 #include <linux/debugfs.h> 44 #include <linux/debugfs.h>
45 #include <linux/kdebug.h> 45 #include <linux/kdebug.h>
46 46
47 #include <asm-generic/sections.h> 47 #include <asm-generic/sections.h>
48 #include <asm/cacheflush.h> 48 #include <asm/cacheflush.h>
49 #include <asm/errno.h> 49 #include <asm/errno.h>
50 #include <asm/uaccess.h> 50 #include <asm/uaccess.h>
51 51
52 #define KPROBE_HASH_BITS 6 52 #define KPROBE_HASH_BITS 6
53 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS) 53 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
54 54
55 55
56 /* 56 /*
57 * Some oddball architectures like 64bit powerpc have function descriptors 57 * Some oddball architectures like 64bit powerpc have function descriptors
58 * so this must be overridable. 58 * so this must be overridable.
59 */ 59 */
60 #ifndef kprobe_lookup_name 60 #ifndef kprobe_lookup_name
61 #define kprobe_lookup_name(name, addr) \ 61 #define kprobe_lookup_name(name, addr) \
62 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name))) 62 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
63 #endif 63 #endif
64 64
65 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 65 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
66 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; 66 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
67 67
68 /* NOTE: change this value only with kprobe_mutex held */ 68 /* NOTE: change this value only with kprobe_mutex held */
69 static bool kprobe_enabled; 69 static bool kprobe_enabled;
70 70
71 DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ 71 DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */
72 DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */ 72 DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */
73 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; 73 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
74 74
75 /* 75 /*
76 * Normally, functions that we'd want to prohibit kprobes in, are marked 76 * Normally, functions that we'd want to prohibit kprobes in, are marked
77 * __kprobes. But, there are cases where such functions already belong to 77 * __kprobes. But, there are cases where such functions already belong to
78 * a different section (__sched for preempt_schedule) 78 * a different section (__sched for preempt_schedule)
79 * 79 *
80 * For such cases, we now have a blacklist 80 * For such cases, we now have a blacklist
81 */ 81 */
82 struct kprobe_blackpoint kprobe_blacklist[] = { 82 struct kprobe_blackpoint kprobe_blacklist[] = {
83 {"preempt_schedule",}, 83 {"preempt_schedule",},
84 {NULL} /* Terminator */ 84 {NULL} /* Terminator */
85 }; 85 };
86 86
87 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT 87 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
88 /* 88 /*
89 * kprobe->ainsn.insn points to the copy of the instruction to be 89 * kprobe->ainsn.insn points to the copy of the instruction to be
90 * single-stepped. x86_64, POWER4 and above have no-exec support and 90 * single-stepped. x86_64, POWER4 and above have no-exec support and
91 * stepping on the instruction on a vmalloced/kmalloced/data page 91 * stepping on the instruction on a vmalloced/kmalloced/data page
92 * is a recipe for disaster 92 * is a recipe for disaster
93 */ 93 */
94 #define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t))) 94 #define INSNS_PER_PAGE (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
95 95
96 struct kprobe_insn_page { 96 struct kprobe_insn_page {
97 struct hlist_node hlist; 97 struct hlist_node hlist;
98 kprobe_opcode_t *insns; /* Page of instruction slots */ 98 kprobe_opcode_t *insns; /* Page of instruction slots */
99 char slot_used[INSNS_PER_PAGE]; 99 char slot_used[INSNS_PER_PAGE];
100 int nused; 100 int nused;
101 int ngarbage; 101 int ngarbage;
102 }; 102 };
103 103
104 enum kprobe_slot_state { 104 enum kprobe_slot_state {
105 SLOT_CLEAN = 0, 105 SLOT_CLEAN = 0,
106 SLOT_DIRTY = 1, 106 SLOT_DIRTY = 1,
107 SLOT_USED = 2, 107 SLOT_USED = 2,
108 }; 108 };
109 109
110 static struct hlist_head kprobe_insn_pages; 110 static struct hlist_head kprobe_insn_pages;
111 static int kprobe_garbage_slots; 111 static int kprobe_garbage_slots;
112 static int collect_garbage_slots(void); 112 static int collect_garbage_slots(void);
113 113
114 static int __kprobes check_safety(void) 114 static int __kprobes check_safety(void)
115 { 115 {
116 int ret = 0; 116 int ret = 0;
117 #if defined(CONFIG_PREEMPT) && defined(CONFIG_PM) 117 #if defined(CONFIG_PREEMPT) && defined(CONFIG_PM)
118 ret = freeze_processes(); 118 ret = freeze_processes();
119 if (ret == 0) { 119 if (ret == 0) {
120 struct task_struct *p, *q; 120 struct task_struct *p, *q;
121 do_each_thread(p, q) { 121 do_each_thread(p, q) {
122 if (p != current && p->state == TASK_RUNNING && 122 if (p != current && p->state == TASK_RUNNING &&
123 p->pid != 0) { 123 p->pid != 0) {
124 printk("Check failed: %s is running\n",p->comm); 124 printk("Check failed: %s is running\n",p->comm);
125 ret = -1; 125 ret = -1;
126 goto loop_end; 126 goto loop_end;
127 } 127 }
128 } while_each_thread(p, q); 128 } while_each_thread(p, q);
129 } 129 }
130 loop_end: 130 loop_end:
131 thaw_processes(); 131 thaw_processes();
132 #else 132 #else
133 synchronize_sched(); 133 synchronize_sched();
134 #endif 134 #endif
135 return ret; 135 return ret;
136 } 136 }
137 137
138 /** 138 /**
139 * get_insn_slot() - Find a slot on an executable page for an instruction. 139 * get_insn_slot() - Find a slot on an executable page for an instruction.
140 * We allocate an executable page if there's no room on existing ones. 140 * We allocate an executable page if there's no room on existing ones.
141 */ 141 */
142 kprobe_opcode_t __kprobes *get_insn_slot(void) 142 kprobe_opcode_t __kprobes *get_insn_slot(void)
143 { 143 {
144 struct kprobe_insn_page *kip; 144 struct kprobe_insn_page *kip;
145 struct hlist_node *pos; 145 struct hlist_node *pos;
146 146
147 retry: 147 retry:
148 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) { 148 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
149 if (kip->nused < INSNS_PER_PAGE) { 149 if (kip->nused < INSNS_PER_PAGE) {
150 int i; 150 int i;
151 for (i = 0; i < INSNS_PER_PAGE; i++) { 151 for (i = 0; i < INSNS_PER_PAGE; i++) {
152 if (kip->slot_used[i] == SLOT_CLEAN) { 152 if (kip->slot_used[i] == SLOT_CLEAN) {
153 kip->slot_used[i] = SLOT_USED; 153 kip->slot_used[i] = SLOT_USED;
154 kip->nused++; 154 kip->nused++;
155 return kip->insns + (i * MAX_INSN_SIZE); 155 return kip->insns + (i * MAX_INSN_SIZE);
156 } 156 }
157 } 157 }
158 /* Surprise! No unused slots. Fix kip->nused. */ 158 /* Surprise! No unused slots. Fix kip->nused. */
159 kip->nused = INSNS_PER_PAGE; 159 kip->nused = INSNS_PER_PAGE;
160 } 160 }
161 } 161 }
162 162
163 /* If there are any garbage slots, collect it and try again. */ 163 /* If there are any garbage slots, collect it and try again. */
164 if (kprobe_garbage_slots && collect_garbage_slots() == 0) { 164 if (kprobe_garbage_slots && collect_garbage_slots() == 0) {
165 goto retry; 165 goto retry;
166 } 166 }
167 /* All out of space. Need to allocate a new page. Use slot 0. */ 167 /* All out of space. Need to allocate a new page. Use slot 0. */
168 kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL); 168 kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
169 if (!kip) 169 if (!kip)
170 return NULL; 170 return NULL;
171 171
172 /* 172 /*
173 * Use module_alloc so this page is within +/- 2GB of where the 173 * Use module_alloc so this page is within +/- 2GB of where the
174 * kernel image and loaded module images reside. This is required 174 * kernel image and loaded module images reside. This is required
175 * so x86_64 can correctly handle the %rip-relative fixups. 175 * so x86_64 can correctly handle the %rip-relative fixups.
176 */ 176 */
177 kip->insns = module_alloc(PAGE_SIZE); 177 kip->insns = module_alloc(PAGE_SIZE);
178 if (!kip->insns) { 178 if (!kip->insns) {
179 kfree(kip); 179 kfree(kip);
180 return NULL; 180 return NULL;
181 } 181 }
182 INIT_HLIST_NODE(&kip->hlist); 182 INIT_HLIST_NODE(&kip->hlist);
183 hlist_add_head(&kip->hlist, &kprobe_insn_pages); 183 hlist_add_head(&kip->hlist, &kprobe_insn_pages);
184 memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE); 184 memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE);
185 kip->slot_used[0] = SLOT_USED; 185 kip->slot_used[0] = SLOT_USED;
186 kip->nused = 1; 186 kip->nused = 1;
187 kip->ngarbage = 0; 187 kip->ngarbage = 0;
188 return kip->insns; 188 return kip->insns;
189 } 189 }
190 190
191 /* Return 1 if all garbages are collected, otherwise 0. */ 191 /* Return 1 if all garbages are collected, otherwise 0. */
192 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx) 192 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
193 { 193 {
194 kip->slot_used[idx] = SLOT_CLEAN; 194 kip->slot_used[idx] = SLOT_CLEAN;
195 kip->nused--; 195 kip->nused--;
196 if (kip->nused == 0) { 196 if (kip->nused == 0) {
197 /* 197 /*
198 * Page is no longer in use. Free it unless 198 * Page is no longer in use. Free it unless
199 * it's the last one. We keep the last one 199 * it's the last one. We keep the last one
200 * so as not to have to set it up again the 200 * so as not to have to set it up again the
201 * next time somebody inserts a probe. 201 * next time somebody inserts a probe.
202 */ 202 */
203 hlist_del(&kip->hlist); 203 hlist_del(&kip->hlist);
204 if (hlist_empty(&kprobe_insn_pages)) { 204 if (hlist_empty(&kprobe_insn_pages)) {
205 INIT_HLIST_NODE(&kip->hlist); 205 INIT_HLIST_NODE(&kip->hlist);
206 hlist_add_head(&kip->hlist, 206 hlist_add_head(&kip->hlist,
207 &kprobe_insn_pages); 207 &kprobe_insn_pages);
208 } else { 208 } else {
209 module_free(NULL, kip->insns); 209 module_free(NULL, kip->insns);
210 kfree(kip); 210 kfree(kip);
211 } 211 }
212 return 1; 212 return 1;
213 } 213 }
214 return 0; 214 return 0;
215 } 215 }
216 216
217 static int __kprobes collect_garbage_slots(void) 217 static int __kprobes collect_garbage_slots(void)
218 { 218 {
219 struct kprobe_insn_page *kip; 219 struct kprobe_insn_page *kip;
220 struct hlist_node *pos, *next; 220 struct hlist_node *pos, *next;
221 221
222 /* Ensure no-one is preepmted on the garbages */ 222 /* Ensure no-one is preepmted on the garbages */
223 if (check_safety() != 0) 223 if (check_safety() != 0)
224 return -EAGAIN; 224 return -EAGAIN;
225 225
226 hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) { 226 hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) {
227 int i; 227 int i;
228 if (kip->ngarbage == 0) 228 if (kip->ngarbage == 0)
229 continue; 229 continue;
230 kip->ngarbage = 0; /* we will collect all garbages */ 230 kip->ngarbage = 0; /* we will collect all garbages */
231 for (i = 0; i < INSNS_PER_PAGE; i++) { 231 for (i = 0; i < INSNS_PER_PAGE; i++) {
232 if (kip->slot_used[i] == SLOT_DIRTY && 232 if (kip->slot_used[i] == SLOT_DIRTY &&
233 collect_one_slot(kip, i)) 233 collect_one_slot(kip, i))
234 break; 234 break;
235 } 235 }
236 } 236 }
237 kprobe_garbage_slots = 0; 237 kprobe_garbage_slots = 0;
238 return 0; 238 return 0;
239 } 239 }
240 240
241 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty) 241 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
242 { 242 {
243 struct kprobe_insn_page *kip; 243 struct kprobe_insn_page *kip;
244 struct hlist_node *pos; 244 struct hlist_node *pos;
245 245
246 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) { 246 hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
247 if (kip->insns <= slot && 247 if (kip->insns <= slot &&
248 slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) { 248 slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {
249 int i = (slot - kip->insns) / MAX_INSN_SIZE; 249 int i = (slot - kip->insns) / MAX_INSN_SIZE;
250 if (dirty) { 250 if (dirty) {
251 kip->slot_used[i] = SLOT_DIRTY; 251 kip->slot_used[i] = SLOT_DIRTY;
252 kip->ngarbage++; 252 kip->ngarbage++;
253 } else { 253 } else {
254 collect_one_slot(kip, i); 254 collect_one_slot(kip, i);
255 } 255 }
256 break; 256 break;
257 } 257 }
258 } 258 }
259 259
260 if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE) 260 if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE)
261 collect_garbage_slots(); 261 collect_garbage_slots();
262 } 262 }
263 #endif 263 #endif
264 264
265 /* We have preemption disabled.. so it is safe to use __ versions */ 265 /* We have preemption disabled.. so it is safe to use __ versions */
266 static inline void set_kprobe_instance(struct kprobe *kp) 266 static inline void set_kprobe_instance(struct kprobe *kp)
267 { 267 {
268 __get_cpu_var(kprobe_instance) = kp; 268 __get_cpu_var(kprobe_instance) = kp;
269 } 269 }
270 270
271 static inline void reset_kprobe_instance(void) 271 static inline void reset_kprobe_instance(void)
272 { 272 {
273 __get_cpu_var(kprobe_instance) = NULL; 273 __get_cpu_var(kprobe_instance) = NULL;
274 } 274 }
275 275
276 /* 276 /*
277 * This routine is called either: 277 * This routine is called either:
278 * - under the kprobe_mutex - during kprobe_[un]register() 278 * - under the kprobe_mutex - during kprobe_[un]register()
279 * OR 279 * OR
280 * - with preemption disabled - from arch/xxx/kernel/kprobes.c 280 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
281 */ 281 */
282 struct kprobe __kprobes *get_kprobe(void *addr) 282 struct kprobe __kprobes *get_kprobe(void *addr)
283 { 283 {
284 struct hlist_head *head; 284 struct hlist_head *head;
285 struct hlist_node *node; 285 struct hlist_node *node;
286 struct kprobe *p; 286 struct kprobe *p;
287 287
288 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)]; 288 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
289 hlist_for_each_entry_rcu(p, node, head, hlist) { 289 hlist_for_each_entry_rcu(p, node, head, hlist) {
290 if (p->addr == addr) 290 if (p->addr == addr)
291 return p; 291 return p;
292 } 292 }
293 return NULL; 293 return NULL;
294 } 294 }
295 295
296 /* 296 /*
297 * Aggregate handlers for multiple kprobes support - these handlers 297 * Aggregate handlers for multiple kprobes support - these handlers
298 * take care of invoking the individual kprobe handlers on p->list 298 * take care of invoking the individual kprobe handlers on p->list
299 */ 299 */
300 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs) 300 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
301 { 301 {
302 struct kprobe *kp; 302 struct kprobe *kp;
303 303
304 list_for_each_entry_rcu(kp, &p->list, list) { 304 list_for_each_entry_rcu(kp, &p->list, list) {
305 if (kp->pre_handler) { 305 if (kp->pre_handler) {
306 set_kprobe_instance(kp); 306 set_kprobe_instance(kp);
307 if (kp->pre_handler(kp, regs)) 307 if (kp->pre_handler(kp, regs))
308 return 1; 308 return 1;
309 } 309 }
310 reset_kprobe_instance(); 310 reset_kprobe_instance();
311 } 311 }
312 return 0; 312 return 0;
313 } 313 }
314 314
315 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs, 315 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
316 unsigned long flags) 316 unsigned long flags)
317 { 317 {
318 struct kprobe *kp; 318 struct kprobe *kp;
319 319
320 list_for_each_entry_rcu(kp, &p->list, list) { 320 list_for_each_entry_rcu(kp, &p->list, list) {
321 if (kp->post_handler) { 321 if (kp->post_handler) {
322 set_kprobe_instance(kp); 322 set_kprobe_instance(kp);
323 kp->post_handler(kp, regs, flags); 323 kp->post_handler(kp, regs, flags);
324 reset_kprobe_instance(); 324 reset_kprobe_instance();
325 } 325 }
326 } 326 }
327 } 327 }
328 328
329 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs, 329 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
330 int trapnr) 330 int trapnr)
331 { 331 {
332 struct kprobe *cur = __get_cpu_var(kprobe_instance); 332 struct kprobe *cur = __get_cpu_var(kprobe_instance);
333 333
334 /* 334 /*
335 * if we faulted "during" the execution of a user specified 335 * if we faulted "during" the execution of a user specified
336 * probe handler, invoke just that probe's fault handler 336 * probe handler, invoke just that probe's fault handler
337 */ 337 */
338 if (cur && cur->fault_handler) { 338 if (cur && cur->fault_handler) {
339 if (cur->fault_handler(cur, regs, trapnr)) 339 if (cur->fault_handler(cur, regs, trapnr))
340 return 1; 340 return 1;
341 } 341 }
342 return 0; 342 return 0;
343 } 343 }
344 344
345 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs) 345 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
346 { 346 {
347 struct kprobe *cur = __get_cpu_var(kprobe_instance); 347 struct kprobe *cur = __get_cpu_var(kprobe_instance);
348 int ret = 0; 348 int ret = 0;
349 349
350 if (cur && cur->break_handler) { 350 if (cur && cur->break_handler) {
351 if (cur->break_handler(cur, regs)) 351 if (cur->break_handler(cur, regs))
352 ret = 1; 352 ret = 1;
353 } 353 }
354 reset_kprobe_instance(); 354 reset_kprobe_instance();
355 return ret; 355 return ret;
356 } 356 }
357 357
358 /* Walks the list and increments nmissed count for multiprobe case */ 358 /* Walks the list and increments nmissed count for multiprobe case */
359 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p) 359 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
360 { 360 {
361 struct kprobe *kp; 361 struct kprobe *kp;
362 if (p->pre_handler != aggr_pre_handler) { 362 if (p->pre_handler != aggr_pre_handler) {
363 p->nmissed++; 363 p->nmissed++;
364 } else { 364 } else {
365 list_for_each_entry_rcu(kp, &p->list, list) 365 list_for_each_entry_rcu(kp, &p->list, list)
366 kp->nmissed++; 366 kp->nmissed++;
367 } 367 }
368 return; 368 return;
369 } 369 }
370 370
371 /* Called with kretprobe_lock held */ 371 /* Called with kretprobe_lock held */
372 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri, 372 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
373 struct hlist_head *head) 373 struct hlist_head *head)
374 { 374 {
375 /* remove rp inst off the rprobe_inst_table */ 375 /* remove rp inst off the rprobe_inst_table */
376 hlist_del(&ri->hlist); 376 hlist_del(&ri->hlist);
377 if (ri->rp) { 377 if (ri->rp) {
378 /* remove rp inst off the used list */ 378 /* remove rp inst off the used list */
379 hlist_del(&ri->uflist); 379 hlist_del(&ri->uflist);
380 /* put rp inst back onto the free list */ 380 /* put rp inst back onto the free list */
381 INIT_HLIST_NODE(&ri->uflist); 381 INIT_HLIST_NODE(&ri->uflist);
382 hlist_add_head(&ri->uflist, &ri->rp->free_instances); 382 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
383 } else 383 } else
384 /* Unregistering */ 384 /* Unregistering */
385 hlist_add_head(&ri->hlist, head); 385 hlist_add_head(&ri->hlist, head);
386 } 386 }
387 387
388 struct hlist_head __kprobes *kretprobe_inst_table_head(struct task_struct *tsk) 388 struct hlist_head __kprobes *kretprobe_inst_table_head(struct task_struct *tsk)
389 { 389 {
390 return &kretprobe_inst_table[hash_ptr(tsk, KPROBE_HASH_BITS)]; 390 return &kretprobe_inst_table[hash_ptr(tsk, KPROBE_HASH_BITS)];
391 } 391 }
392 392
393 /* 393 /*
394 * This function is called from finish_task_switch when task tk becomes dead, 394 * This function is called from finish_task_switch when task tk becomes dead,
395 * so that we can recycle any function-return probe instances associated 395 * so that we can recycle any function-return probe instances associated
396 * with this task. These left over instances represent probed functions 396 * with this task. These left over instances represent probed functions
397 * that have been called but will never return. 397 * that have been called but will never return.
398 */ 398 */
399 void __kprobes kprobe_flush_task(struct task_struct *tk) 399 void __kprobes kprobe_flush_task(struct task_struct *tk)
400 { 400 {
401 struct kretprobe_instance *ri; 401 struct kretprobe_instance *ri;
402 struct hlist_head *head, empty_rp; 402 struct hlist_head *head, empty_rp;
403 struct hlist_node *node, *tmp; 403 struct hlist_node *node, *tmp;
404 unsigned long flags = 0; 404 unsigned long flags = 0;
405 405
406 INIT_HLIST_HEAD(&empty_rp); 406 INIT_HLIST_HEAD(&empty_rp);
407 spin_lock_irqsave(&kretprobe_lock, flags); 407 spin_lock_irqsave(&kretprobe_lock, flags);
408 head = kretprobe_inst_table_head(tk); 408 head = kretprobe_inst_table_head(tk);
409 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { 409 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
410 if (ri->task == tk) 410 if (ri->task == tk)
411 recycle_rp_inst(ri, &empty_rp); 411 recycle_rp_inst(ri, &empty_rp);
412 } 412 }
413 spin_unlock_irqrestore(&kretprobe_lock, flags); 413 spin_unlock_irqrestore(&kretprobe_lock, flags);
414 414
415 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { 415 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
416 hlist_del(&ri->hlist); 416 hlist_del(&ri->hlist);
417 kfree(ri); 417 kfree(ri);
418 } 418 }
419 } 419 }
420 420
421 static inline void free_rp_inst(struct kretprobe *rp) 421 static inline void free_rp_inst(struct kretprobe *rp)
422 { 422 {
423 struct kretprobe_instance *ri; 423 struct kretprobe_instance *ri;
424 struct hlist_node *pos, *next; 424 struct hlist_node *pos, *next;
425 425
426 hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, uflist) { 426 hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, uflist) {
427 hlist_del(&ri->uflist); 427 hlist_del(&ri->uflist);
428 kfree(ri); 428 kfree(ri);
429 } 429 }
430 } 430 }
431 431
432 static void __kprobes cleanup_rp_inst(struct kretprobe *rp) 432 static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
433 { 433 {
434 unsigned long flags; 434 unsigned long flags;
435 struct kretprobe_instance *ri; 435 struct kretprobe_instance *ri;
436 struct hlist_node *pos, *next; 436 struct hlist_node *pos, *next;
437 /* No race here */ 437 /* No race here */
438 spin_lock_irqsave(&kretprobe_lock, flags); 438 spin_lock_irqsave(&kretprobe_lock, flags);
439 hlist_for_each_entry_safe(ri, pos, next, &rp->used_instances, uflist) { 439 hlist_for_each_entry_safe(ri, pos, next, &rp->used_instances, uflist) {
440 ri->rp = NULL; 440 ri->rp = NULL;
441 hlist_del(&ri->uflist); 441 hlist_del(&ri->uflist);
442 } 442 }
443 spin_unlock_irqrestore(&kretprobe_lock, flags); 443 spin_unlock_irqrestore(&kretprobe_lock, flags);
444 free_rp_inst(rp); 444 free_rp_inst(rp);
445 } 445 }
446 446
447 /* 447 /*
448 * Keep all fields in the kprobe consistent 448 * Keep all fields in the kprobe consistent
449 */ 449 */
450 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p) 450 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
451 { 451 {
452 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t)); 452 memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
453 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn)); 453 memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
454 } 454 }
455 455
456 /* 456 /*
457 * Add the new probe to old_p->list. Fail if this is the 457 * Add the new probe to old_p->list. Fail if this is the
458 * second jprobe at the address - two jprobes can't coexist 458 * second jprobe at the address - two jprobes can't coexist
459 */ 459 */
460 static int __kprobes add_new_kprobe(struct kprobe *old_p, struct kprobe *p) 460 static int __kprobes add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
461 { 461 {
462 if (p->break_handler) { 462 if (p->break_handler) {
463 if (old_p->break_handler) 463 if (old_p->break_handler)
464 return -EEXIST; 464 return -EEXIST;
465 list_add_tail_rcu(&p->list, &old_p->list); 465 list_add_tail_rcu(&p->list, &old_p->list);
466 old_p->break_handler = aggr_break_handler; 466 old_p->break_handler = aggr_break_handler;
467 } else 467 } else
468 list_add_rcu(&p->list, &old_p->list); 468 list_add_rcu(&p->list, &old_p->list);
469 if (p->post_handler && !old_p->post_handler) 469 if (p->post_handler && !old_p->post_handler)
470 old_p->post_handler = aggr_post_handler; 470 old_p->post_handler = aggr_post_handler;
471 return 0; 471 return 0;
472 } 472 }
473 473
474 /* 474 /*
475 * Fill in the required fields of the "manager kprobe". Replace the 475 * Fill in the required fields of the "manager kprobe". Replace the
476 * earlier kprobe in the hlist with the manager kprobe 476 * earlier kprobe in the hlist with the manager kprobe
477 */ 477 */
478 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p) 478 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
479 { 479 {
480 copy_kprobe(p, ap); 480 copy_kprobe(p, ap);
481 flush_insn_slot(ap); 481 flush_insn_slot(ap);
482 ap->addr = p->addr; 482 ap->addr = p->addr;
483 ap->pre_handler = aggr_pre_handler; 483 ap->pre_handler = aggr_pre_handler;
484 ap->fault_handler = aggr_fault_handler; 484 ap->fault_handler = aggr_fault_handler;
485 if (p->post_handler) 485 if (p->post_handler)
486 ap->post_handler = aggr_post_handler; 486 ap->post_handler = aggr_post_handler;
487 if (p->break_handler) 487 if (p->break_handler)
488 ap->break_handler = aggr_break_handler; 488 ap->break_handler = aggr_break_handler;
489 489
490 INIT_LIST_HEAD(&ap->list); 490 INIT_LIST_HEAD(&ap->list);
491 list_add_rcu(&p->list, &ap->list); 491 list_add_rcu(&p->list, &ap->list);
492 492
493 hlist_replace_rcu(&p->hlist, &ap->hlist); 493 hlist_replace_rcu(&p->hlist, &ap->hlist);
494 } 494 }
495 495
496 /* 496 /*
497 * This is the second or subsequent kprobe at the address - handle 497 * This is the second or subsequent kprobe at the address - handle
498 * the intricacies 498 * the intricacies
499 */ 499 */
500 static int __kprobes register_aggr_kprobe(struct kprobe *old_p, 500 static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
501 struct kprobe *p) 501 struct kprobe *p)
502 { 502 {
503 int ret = 0; 503 int ret = 0;
504 struct kprobe *ap; 504 struct kprobe *ap;
505 505
506 if (old_p->pre_handler == aggr_pre_handler) { 506 if (old_p->pre_handler == aggr_pre_handler) {
507 copy_kprobe(old_p, p); 507 copy_kprobe(old_p, p);
508 ret = add_new_kprobe(old_p, p); 508 ret = add_new_kprobe(old_p, p);
509 } else { 509 } else {
510 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL); 510 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
511 if (!ap) 511 if (!ap)
512 return -ENOMEM; 512 return -ENOMEM;
513 add_aggr_kprobe(ap, old_p); 513 add_aggr_kprobe(ap, old_p);
514 copy_kprobe(ap, p); 514 copy_kprobe(ap, p);
515 ret = add_new_kprobe(ap, p); 515 ret = add_new_kprobe(ap, p);
516 } 516 }
517 return ret; 517 return ret;
518 } 518 }
519 519
520 static int __kprobes in_kprobes_functions(unsigned long addr) 520 static int __kprobes in_kprobes_functions(unsigned long addr)
521 { 521 {
522 struct kprobe_blackpoint *kb; 522 struct kprobe_blackpoint *kb;
523 523
524 if (addr >= (unsigned long)__kprobes_text_start && 524 if (addr >= (unsigned long)__kprobes_text_start &&
525 addr < (unsigned long)__kprobes_text_end) 525 addr < (unsigned long)__kprobes_text_end)
526 return -EINVAL; 526 return -EINVAL;
527 /* 527 /*
528 * If there exists a kprobe_blacklist, verify and 528 * If there exists a kprobe_blacklist, verify and
529 * fail any probe registration in the prohibited area 529 * fail any probe registration in the prohibited area
530 */ 530 */
531 for (kb = kprobe_blacklist; kb->name != NULL; kb++) { 531 for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
532 if (kb->start_addr) { 532 if (kb->start_addr) {
533 if (addr >= kb->start_addr && 533 if (addr >= kb->start_addr &&
534 addr < (kb->start_addr + kb->range)) 534 addr < (kb->start_addr + kb->range))
535 return -EINVAL; 535 return -EINVAL;
536 } 536 }
537 } 537 }
538 return 0; 538 return 0;
539 } 539 }
540 540
541 /* 541 /*
542 * If we have a symbol_name argument, look it up and add the offset field 542 * If we have a symbol_name argument, look it up and add the offset field
543 * to it. This way, we can specify a relative address to a symbol. 543 * to it. This way, we can specify a relative address to a symbol.
544 */ 544 */
545 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p) 545 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
546 { 546 {
547 kprobe_opcode_t *addr = p->addr; 547 kprobe_opcode_t *addr = p->addr;
548 if (p->symbol_name) { 548 if (p->symbol_name) {
549 if (addr) 549 if (addr)
550 return NULL; 550 return NULL;
551 kprobe_lookup_name(p->symbol_name, addr); 551 kprobe_lookup_name(p->symbol_name, addr);
552 } 552 }
553 553
554 if (!addr) 554 if (!addr)
555 return NULL; 555 return NULL;
556 return (kprobe_opcode_t *)(((char *)addr) + p->offset); 556 return (kprobe_opcode_t *)(((char *)addr) + p->offset);
557 } 557 }
558 558
559 static int __kprobes __register_kprobe(struct kprobe *p, 559 static int __kprobes __register_kprobe(struct kprobe *p,
560 unsigned long called_from) 560 unsigned long called_from)
561 { 561 {
562 int ret = 0; 562 int ret = 0;
563 struct kprobe *old_p; 563 struct kprobe *old_p;
564 struct module *probed_mod; 564 struct module *probed_mod;
565 kprobe_opcode_t *addr; 565 kprobe_opcode_t *addr;
566 566
567 addr = kprobe_addr(p); 567 addr = kprobe_addr(p);
568 if (!addr) 568 if (!addr)
569 return -EINVAL; 569 return -EINVAL;
570 p->addr = addr; 570 p->addr = addr;
571 571
572 if (!kernel_text_address((unsigned long) p->addr) || 572 if (!kernel_text_address((unsigned long) p->addr) ||
573 in_kprobes_functions((unsigned long) p->addr)) 573 in_kprobes_functions((unsigned long) p->addr))
574 return -EINVAL; 574 return -EINVAL;
575 575
576 p->mod_refcounted = 0; 576 p->mod_refcounted = 0;
577 577
578 /* 578 /*
579 * Check if are we probing a module. 579 * Check if are we probing a module.
580 */ 580 */
581 probed_mod = module_text_address((unsigned long) p->addr); 581 probed_mod = module_text_address((unsigned long) p->addr);
582 if (probed_mod) { 582 if (probed_mod) {
583 struct module *calling_mod = module_text_address(called_from); 583 struct module *calling_mod = module_text_address(called_from);
584 /* 584 /*
585 * We must allow modules to probe themself and in this case 585 * We must allow modules to probe themself and in this case
586 * avoid incrementing the module refcount, so as to allow 586 * avoid incrementing the module refcount, so as to allow
587 * unloading of self probing modules. 587 * unloading of self probing modules.
588 */ 588 */
589 if (calling_mod && calling_mod != probed_mod) { 589 if (calling_mod && calling_mod != probed_mod) {
590 if (unlikely(!try_module_get(probed_mod))) 590 if (unlikely(!try_module_get(probed_mod)))
591 return -EINVAL; 591 return -EINVAL;
592 p->mod_refcounted = 1; 592 p->mod_refcounted = 1;
593 } else 593 } else
594 probed_mod = NULL; 594 probed_mod = NULL;
595 } 595 }
596 596
597 p->nmissed = 0; 597 p->nmissed = 0;
598 INIT_LIST_HEAD(&p->list); 598 INIT_LIST_HEAD(&p->list);
599 mutex_lock(&kprobe_mutex); 599 mutex_lock(&kprobe_mutex);
600 old_p = get_kprobe(p->addr); 600 old_p = get_kprobe(p->addr);
601 if (old_p) { 601 if (old_p) {
602 ret = register_aggr_kprobe(old_p, p); 602 ret = register_aggr_kprobe(old_p, p);
603 goto out; 603 goto out;
604 } 604 }
605 605
606 ret = arch_prepare_kprobe(p); 606 ret = arch_prepare_kprobe(p);
607 if (ret) 607 if (ret)
608 goto out; 608 goto out;
609 609
610 INIT_HLIST_NODE(&p->hlist); 610 INIT_HLIST_NODE(&p->hlist);
611 hlist_add_head_rcu(&p->hlist, 611 hlist_add_head_rcu(&p->hlist,
612 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]); 612 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
613 613
614 if (kprobe_enabled) 614 if (kprobe_enabled)
615 arch_arm_kprobe(p); 615 arch_arm_kprobe(p);
616 616
617 out: 617 out:
618 mutex_unlock(&kprobe_mutex); 618 mutex_unlock(&kprobe_mutex);
619 619
620 if (ret && probed_mod) 620 if (ret && probed_mod)
621 module_put(probed_mod); 621 module_put(probed_mod);
622 return ret; 622 return ret;
623 } 623 }
624 624
625 /* 625 /*
626 * Unregister a kprobe without a scheduler synchronization. 626 * Unregister a kprobe without a scheduler synchronization.
627 */ 627 */
628 static int __kprobes __unregister_kprobe_top(struct kprobe *p) 628 static int __kprobes __unregister_kprobe_top(struct kprobe *p)
629 { 629 {
630 struct kprobe *old_p, *list_p; 630 struct kprobe *old_p, *list_p;
631 631
632 old_p = get_kprobe(p->addr); 632 old_p = get_kprobe(p->addr);
633 if (unlikely(!old_p)) 633 if (unlikely(!old_p))
634 return -EINVAL; 634 return -EINVAL;
635 635
636 if (p != old_p) { 636 if (p != old_p) {
637 list_for_each_entry_rcu(list_p, &old_p->list, list) 637 list_for_each_entry_rcu(list_p, &old_p->list, list)
638 if (list_p == p) 638 if (list_p == p)
639 /* kprobe p is a valid probe */ 639 /* kprobe p is a valid probe */
640 goto valid_p; 640 goto valid_p;
641 return -EINVAL; 641 return -EINVAL;
642 } 642 }
643 valid_p: 643 valid_p:
644 if (old_p == p || 644 if (old_p == p ||
645 (old_p->pre_handler == aggr_pre_handler && 645 (old_p->pre_handler == aggr_pre_handler &&
646 list_is_singular(&old_p->list))) { 646 list_is_singular(&old_p->list))) {
647 /* 647 /*
648 * Only probe on the hash list. Disarm only if kprobes are 648 * Only probe on the hash list. Disarm only if kprobes are
649 * enabled - otherwise, the breakpoint would already have 649 * enabled - otherwise, the breakpoint would already have
650 * been removed. We save on flushing icache. 650 * been removed. We save on flushing icache.
651 */ 651 */
652 if (kprobe_enabled) 652 if (kprobe_enabled)
653 arch_disarm_kprobe(p); 653 arch_disarm_kprobe(p);
654 hlist_del_rcu(&old_p->hlist); 654 hlist_del_rcu(&old_p->hlist);
655 } else { 655 } else {
656 if (p->break_handler) 656 if (p->break_handler)
657 old_p->break_handler = NULL; 657 old_p->break_handler = NULL;
658 if (p->post_handler) { 658 if (p->post_handler) {
659 list_for_each_entry_rcu(list_p, &old_p->list, list) { 659 list_for_each_entry_rcu(list_p, &old_p->list, list) {
660 if ((list_p != p) && (list_p->post_handler)) 660 if ((list_p != p) && (list_p->post_handler))
661 goto noclean; 661 goto noclean;
662 } 662 }
663 old_p->post_handler = NULL; 663 old_p->post_handler = NULL;
664 } 664 }
665 noclean: 665 noclean:
666 list_del_rcu(&p->list); 666 list_del_rcu(&p->list);
667 } 667 }
668 return 0; 668 return 0;
669 } 669 }
670 670
671 static void __kprobes __unregister_kprobe_bottom(struct kprobe *p) 671 static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
672 { 672 {
673 struct module *mod; 673 struct module *mod;
674 struct kprobe *old_p; 674 struct kprobe *old_p;
675 675
676 if (p->mod_refcounted) { 676 if (p->mod_refcounted) {
677 mod = module_text_address((unsigned long)p->addr); 677 mod = module_text_address((unsigned long)p->addr);
678 if (mod) 678 if (mod)
679 module_put(mod); 679 module_put(mod);
680 } 680 }
681 681
682 if (list_empty(&p->list) || list_is_singular(&p->list)) { 682 if (list_empty(&p->list) || list_is_singular(&p->list)) {
683 if (!list_empty(&p->list)) { 683 if (!list_empty(&p->list)) {
684 /* "p" is the last child of an aggr_kprobe */ 684 /* "p" is the last child of an aggr_kprobe */
685 old_p = list_entry(p->list.next, struct kprobe, list); 685 old_p = list_entry(p->list.next, struct kprobe, list);
686 list_del(&p->list); 686 list_del(&p->list);
687 kfree(old_p); 687 kfree(old_p);
688 } 688 }
689 arch_remove_kprobe(p); 689 arch_remove_kprobe(p);
690 } 690 }
691 } 691 }
692 692
693 static int __register_kprobes(struct kprobe **kps, int num, 693 static int __register_kprobes(struct kprobe **kps, int num,
694 unsigned long called_from) 694 unsigned long called_from)
695 { 695 {
696 int i, ret = 0; 696 int i, ret = 0;
697 697
698 if (num <= 0) 698 if (num <= 0)
699 return -EINVAL; 699 return -EINVAL;
700 for (i = 0; i < num; i++) { 700 for (i = 0; i < num; i++) {
701 ret = __register_kprobe(kps[i], called_from); 701 ret = __register_kprobe(kps[i], called_from);
702 if (ret < 0 && i > 0) { 702 if (ret < 0) {
703 unregister_kprobes(kps, i); 703 if (i > 0)
704 unregister_kprobes(kps, i);
704 break; 705 break;
705 } 706 }
706 } 707 }
707 return ret; 708 return ret;
708 } 709 }
709 710
710 /* 711 /*
711 * Registration and unregistration functions for kprobe. 712 * Registration and unregistration functions for kprobe.
712 */ 713 */
713 int __kprobes register_kprobe(struct kprobe *p) 714 int __kprobes register_kprobe(struct kprobe *p)
714 { 715 {
715 return __register_kprobes(&p, 1, 716 return __register_kprobes(&p, 1,
716 (unsigned long)__builtin_return_address(0)); 717 (unsigned long)__builtin_return_address(0));
717 } 718 }
718 719
719 void __kprobes unregister_kprobe(struct kprobe *p) 720 void __kprobes unregister_kprobe(struct kprobe *p)
720 { 721 {
721 unregister_kprobes(&p, 1); 722 unregister_kprobes(&p, 1);
722 } 723 }
723 724
724 int __kprobes register_kprobes(struct kprobe **kps, int num) 725 int __kprobes register_kprobes(struct kprobe **kps, int num)
725 { 726 {
726 return __register_kprobes(kps, num, 727 return __register_kprobes(kps, num,
727 (unsigned long)__builtin_return_address(0)); 728 (unsigned long)__builtin_return_address(0));
728 } 729 }
729 730
730 void __kprobes unregister_kprobes(struct kprobe **kps, int num) 731 void __kprobes unregister_kprobes(struct kprobe **kps, int num)
731 { 732 {
732 int i; 733 int i;
733 734
734 if (num <= 0) 735 if (num <= 0)
735 return; 736 return;
736 mutex_lock(&kprobe_mutex); 737 mutex_lock(&kprobe_mutex);
737 for (i = 0; i < num; i++) 738 for (i = 0; i < num; i++)
738 if (__unregister_kprobe_top(kps[i]) < 0) 739 if (__unregister_kprobe_top(kps[i]) < 0)
739 kps[i]->addr = NULL; 740 kps[i]->addr = NULL;
740 mutex_unlock(&kprobe_mutex); 741 mutex_unlock(&kprobe_mutex);
741 742
742 synchronize_sched(); 743 synchronize_sched();
743 for (i = 0; i < num; i++) 744 for (i = 0; i < num; i++)
744 if (kps[i]->addr) 745 if (kps[i]->addr)
745 __unregister_kprobe_bottom(kps[i]); 746 __unregister_kprobe_bottom(kps[i]);
746 } 747 }
747 748
748 static struct notifier_block kprobe_exceptions_nb = { 749 static struct notifier_block kprobe_exceptions_nb = {
749 .notifier_call = kprobe_exceptions_notify, 750 .notifier_call = kprobe_exceptions_notify,
750 .priority = 0x7fffffff /* we need to be notified first */ 751 .priority = 0x7fffffff /* we need to be notified first */
751 }; 752 };
752 753
753 unsigned long __weak arch_deref_entry_point(void *entry) 754 unsigned long __weak arch_deref_entry_point(void *entry)
754 { 755 {
755 return (unsigned long)entry; 756 return (unsigned long)entry;
756 } 757 }
757 758
758 static int __register_jprobes(struct jprobe **jps, int num, 759 static int __register_jprobes(struct jprobe **jps, int num,
759 unsigned long called_from) 760 unsigned long called_from)
760 { 761 {
761 struct jprobe *jp; 762 struct jprobe *jp;
762 int ret = 0, i; 763 int ret = 0, i;
763 764
764 if (num <= 0) 765 if (num <= 0)
765 return -EINVAL; 766 return -EINVAL;
766 for (i = 0; i < num; i++) { 767 for (i = 0; i < num; i++) {
767 unsigned long addr; 768 unsigned long addr;
768 jp = jps[i]; 769 jp = jps[i];
769 addr = arch_deref_entry_point(jp->entry); 770 addr = arch_deref_entry_point(jp->entry);
770 771
771 if (!kernel_text_address(addr)) 772 if (!kernel_text_address(addr))
772 ret = -EINVAL; 773 ret = -EINVAL;
773 else { 774 else {
774 /* Todo: Verify probepoint is a function entry point */ 775 /* Todo: Verify probepoint is a function entry point */
775 jp->kp.pre_handler = setjmp_pre_handler; 776 jp->kp.pre_handler = setjmp_pre_handler;
776 jp->kp.break_handler = longjmp_break_handler; 777 jp->kp.break_handler = longjmp_break_handler;
777 ret = __register_kprobe(&jp->kp, called_from); 778 ret = __register_kprobe(&jp->kp, called_from);
778 } 779 }
779 if (ret < 0 && i > 0) { 780 if (ret < 0) {
780 unregister_jprobes(jps, i); 781 if (i > 0)
782 unregister_jprobes(jps, i);
781 break; 783 break;
782 } 784 }
783 } 785 }
784 return ret; 786 return ret;
785 } 787 }
786 788
787 int __kprobes register_jprobe(struct jprobe *jp) 789 int __kprobes register_jprobe(struct jprobe *jp)
788 { 790 {
789 return __register_jprobes(&jp, 1, 791 return __register_jprobes(&jp, 1,
790 (unsigned long)__builtin_return_address(0)); 792 (unsigned long)__builtin_return_address(0));
791 } 793 }
792 794
793 void __kprobes unregister_jprobe(struct jprobe *jp) 795 void __kprobes unregister_jprobe(struct jprobe *jp)
794 { 796 {
795 unregister_jprobes(&jp, 1); 797 unregister_jprobes(&jp, 1);
796 } 798 }
797 799
798 int __kprobes register_jprobes(struct jprobe **jps, int num) 800 int __kprobes register_jprobes(struct jprobe **jps, int num)
799 { 801 {
800 return __register_jprobes(jps, num, 802 return __register_jprobes(jps, num,
801 (unsigned long)__builtin_return_address(0)); 803 (unsigned long)__builtin_return_address(0));
802 } 804 }
803 805
804 void __kprobes unregister_jprobes(struct jprobe **jps, int num) 806 void __kprobes unregister_jprobes(struct jprobe **jps, int num)
805 { 807 {
806 int i; 808 int i;
807 809
808 if (num <= 0) 810 if (num <= 0)
809 return; 811 return;
810 mutex_lock(&kprobe_mutex); 812 mutex_lock(&kprobe_mutex);
811 for (i = 0; i < num; i++) 813 for (i = 0; i < num; i++)
812 if (__unregister_kprobe_top(&jps[i]->kp) < 0) 814 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
813 jps[i]->kp.addr = NULL; 815 jps[i]->kp.addr = NULL;
814 mutex_unlock(&kprobe_mutex); 816 mutex_unlock(&kprobe_mutex);
815 817
816 synchronize_sched(); 818 synchronize_sched();
817 for (i = 0; i < num; i++) { 819 for (i = 0; i < num; i++) {
818 if (jps[i]->kp.addr) 820 if (jps[i]->kp.addr)
819 __unregister_kprobe_bottom(&jps[i]->kp); 821 __unregister_kprobe_bottom(&jps[i]->kp);
820 } 822 }
821 } 823 }
822 824
823 #ifdef CONFIG_KRETPROBES 825 #ifdef CONFIG_KRETPROBES
824 /* 826 /*
825 * This kprobe pre_handler is registered with every kretprobe. When probe 827 * This kprobe pre_handler is registered with every kretprobe. When probe
826 * hits it will set up the return probe. 828 * hits it will set up the return probe.
827 */ 829 */
828 static int __kprobes pre_handler_kretprobe(struct kprobe *p, 830 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
829 struct pt_regs *regs) 831 struct pt_regs *regs)
830 { 832 {
831 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 833 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
832 unsigned long flags = 0; 834 unsigned long flags = 0;
833 835
834 /*TODO: consider to only swap the RA after the last pre_handler fired */ 836 /*TODO: consider to only swap the RA after the last pre_handler fired */
835 spin_lock_irqsave(&kretprobe_lock, flags); 837 spin_lock_irqsave(&kretprobe_lock, flags);
836 if (!hlist_empty(&rp->free_instances)) { 838 if (!hlist_empty(&rp->free_instances)) {
837 struct kretprobe_instance *ri; 839 struct kretprobe_instance *ri;
838 840
839 ri = hlist_entry(rp->free_instances.first, 841 ri = hlist_entry(rp->free_instances.first,
840 struct kretprobe_instance, uflist); 842 struct kretprobe_instance, uflist);
841 ri->rp = rp; 843 ri->rp = rp;
842 ri->task = current; 844 ri->task = current;
843 845
844 if (rp->entry_handler && rp->entry_handler(ri, regs)) { 846 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
845 spin_unlock_irqrestore(&kretprobe_lock, flags); 847 spin_unlock_irqrestore(&kretprobe_lock, flags);
846 return 0; 848 return 0;
847 } 849 }
848 850
849 arch_prepare_kretprobe(ri, regs); 851 arch_prepare_kretprobe(ri, regs);
850 852
851 /* XXX(hch): why is there no hlist_move_head? */ 853 /* XXX(hch): why is there no hlist_move_head? */
852 hlist_del(&ri->uflist); 854 hlist_del(&ri->uflist);
853 hlist_add_head(&ri->uflist, &ri->rp->used_instances); 855 hlist_add_head(&ri->uflist, &ri->rp->used_instances);
854 hlist_add_head(&ri->hlist, kretprobe_inst_table_head(ri->task)); 856 hlist_add_head(&ri->hlist, kretprobe_inst_table_head(ri->task));
855 } else 857 } else
856 rp->nmissed++; 858 rp->nmissed++;
857 spin_unlock_irqrestore(&kretprobe_lock, flags); 859 spin_unlock_irqrestore(&kretprobe_lock, flags);
858 return 0; 860 return 0;
859 } 861 }
860 862
861 static int __kprobes __register_kretprobe(struct kretprobe *rp, 863 static int __kprobes __register_kretprobe(struct kretprobe *rp,
862 unsigned long called_from) 864 unsigned long called_from)
863 { 865 {
864 int ret = 0; 866 int ret = 0;
865 struct kretprobe_instance *inst; 867 struct kretprobe_instance *inst;
866 int i; 868 int i;
867 void *addr; 869 void *addr;
868 870
869 if (kretprobe_blacklist_size) { 871 if (kretprobe_blacklist_size) {
870 addr = kprobe_addr(&rp->kp); 872 addr = kprobe_addr(&rp->kp);
871 if (!addr) 873 if (!addr)
872 return -EINVAL; 874 return -EINVAL;
873 875
874 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 876 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
875 if (kretprobe_blacklist[i].addr == addr) 877 if (kretprobe_blacklist[i].addr == addr)
876 return -EINVAL; 878 return -EINVAL;
877 } 879 }
878 } 880 }
879 881
880 rp->kp.pre_handler = pre_handler_kretprobe; 882 rp->kp.pre_handler = pre_handler_kretprobe;
881 rp->kp.post_handler = NULL; 883 rp->kp.post_handler = NULL;
882 rp->kp.fault_handler = NULL; 884 rp->kp.fault_handler = NULL;
883 rp->kp.break_handler = NULL; 885 rp->kp.break_handler = NULL;
884 886
885 /* Pre-allocate memory for max kretprobe instances */ 887 /* Pre-allocate memory for max kretprobe instances */
886 if (rp->maxactive <= 0) { 888 if (rp->maxactive <= 0) {
887 #ifdef CONFIG_PREEMPT 889 #ifdef CONFIG_PREEMPT
888 rp->maxactive = max(10, 2 * NR_CPUS); 890 rp->maxactive = max(10, 2 * NR_CPUS);
889 #else 891 #else
890 rp->maxactive = NR_CPUS; 892 rp->maxactive = NR_CPUS;
891 #endif 893 #endif
892 } 894 }
893 INIT_HLIST_HEAD(&rp->used_instances); 895 INIT_HLIST_HEAD(&rp->used_instances);
894 INIT_HLIST_HEAD(&rp->free_instances); 896 INIT_HLIST_HEAD(&rp->free_instances);
895 for (i = 0; i < rp->maxactive; i++) { 897 for (i = 0; i < rp->maxactive; i++) {
896 inst = kmalloc(sizeof(struct kretprobe_instance) + 898 inst = kmalloc(sizeof(struct kretprobe_instance) +
897 rp->data_size, GFP_KERNEL); 899 rp->data_size, GFP_KERNEL);
898 if (inst == NULL) { 900 if (inst == NULL) {
899 free_rp_inst(rp); 901 free_rp_inst(rp);
900 return -ENOMEM; 902 return -ENOMEM;
901 } 903 }
902 INIT_HLIST_NODE(&inst->uflist); 904 INIT_HLIST_NODE(&inst->uflist);
903 hlist_add_head(&inst->uflist, &rp->free_instances); 905 hlist_add_head(&inst->uflist, &rp->free_instances);
904 } 906 }
905 907
906 rp->nmissed = 0; 908 rp->nmissed = 0;
907 /* Establish function entry probe point */ 909 /* Establish function entry probe point */
908 ret = __register_kprobe(&rp->kp, called_from); 910 ret = __register_kprobe(&rp->kp, called_from);
909 if (ret != 0) 911 if (ret != 0)
910 free_rp_inst(rp); 912 free_rp_inst(rp);
911 return ret; 913 return ret;
912 } 914 }
913 915
914 static int __register_kretprobes(struct kretprobe **rps, int num, 916 static int __register_kretprobes(struct kretprobe **rps, int num,
915 unsigned long called_from) 917 unsigned long called_from)
916 { 918 {
917 int ret = 0, i; 919 int ret = 0, i;
918 920
919 if (num <= 0) 921 if (num <= 0)
920 return -EINVAL; 922 return -EINVAL;
921 for (i = 0; i < num; i++) { 923 for (i = 0; i < num; i++) {
922 ret = __register_kretprobe(rps[i], called_from); 924 ret = __register_kretprobe(rps[i], called_from);
923 if (ret < 0 && i > 0) { 925 if (ret < 0) {
924 unregister_kretprobes(rps, i); 926 if (i > 0)
927 unregister_kretprobes(rps, i);
925 break; 928 break;
926 } 929 }
927 } 930 }
928 return ret; 931 return ret;
929 } 932 }
930 933
931 int __kprobes register_kretprobe(struct kretprobe *rp) 934 int __kprobes register_kretprobe(struct kretprobe *rp)
932 { 935 {
933 return __register_kretprobes(&rp, 1, 936 return __register_kretprobes(&rp, 1,
934 (unsigned long)__builtin_return_address(0)); 937 (unsigned long)__builtin_return_address(0));
935 } 938 }
936 939
937 void __kprobes unregister_kretprobe(struct kretprobe *rp) 940 void __kprobes unregister_kretprobe(struct kretprobe *rp)
938 { 941 {
939 unregister_kretprobes(&rp, 1); 942 unregister_kretprobes(&rp, 1);
940 } 943 }
941 944
942 int __kprobes register_kretprobes(struct kretprobe **rps, int num) 945 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
943 { 946 {
944 return __register_kretprobes(rps, num, 947 return __register_kretprobes(rps, num,
945 (unsigned long)__builtin_return_address(0)); 948 (unsigned long)__builtin_return_address(0));
946 } 949 }
947 950
948 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) 951 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
949 { 952 {
950 int i; 953 int i;
951 954
952 if (num <= 0) 955 if (num <= 0)
953 return; 956 return;
954 mutex_lock(&kprobe_mutex); 957 mutex_lock(&kprobe_mutex);
955 for (i = 0; i < num; i++) 958 for (i = 0; i < num; i++)
956 if (__unregister_kprobe_top(&rps[i]->kp) < 0) 959 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
957 rps[i]->kp.addr = NULL; 960 rps[i]->kp.addr = NULL;
958 mutex_unlock(&kprobe_mutex); 961 mutex_unlock(&kprobe_mutex);
959 962
960 synchronize_sched(); 963 synchronize_sched();
961 for (i = 0; i < num; i++) { 964 for (i = 0; i < num; i++) {
962 if (rps[i]->kp.addr) { 965 if (rps[i]->kp.addr) {
963 __unregister_kprobe_bottom(&rps[i]->kp); 966 __unregister_kprobe_bottom(&rps[i]->kp);
964 cleanup_rp_inst(rps[i]); 967 cleanup_rp_inst(rps[i]);
965 } 968 }
966 } 969 }
967 } 970 }
968 971
969 #else /* CONFIG_KRETPROBES */ 972 #else /* CONFIG_KRETPROBES */
970 int __kprobes register_kretprobe(struct kretprobe *rp) 973 int __kprobes register_kretprobe(struct kretprobe *rp)
971 { 974 {
972 return -ENOSYS; 975 return -ENOSYS;
973 } 976 }
974 977
975 int __kprobes register_kretprobes(struct kretprobe **rps, int num) 978 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
976 { 979 {
977 return -ENOSYS; 980 return -ENOSYS;
978 } 981 }
979 void __kprobes unregister_kretprobe(struct kretprobe *rp) 982 void __kprobes unregister_kretprobe(struct kretprobe *rp)
980 { 983 {
981 } 984 }
982 985
983 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num) 986 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
984 { 987 {
985 } 988 }
986 989
987 static int __kprobes pre_handler_kretprobe(struct kprobe *p, 990 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
988 struct pt_regs *regs) 991 struct pt_regs *regs)
989 { 992 {
990 return 0; 993 return 0;
991 } 994 }
992 995
993 #endif /* CONFIG_KRETPROBES */ 996 #endif /* CONFIG_KRETPROBES */
994 997
995 static int __init init_kprobes(void) 998 static int __init init_kprobes(void)
996 { 999 {
997 int i, err = 0; 1000 int i, err = 0;
998 unsigned long offset = 0, size = 0; 1001 unsigned long offset = 0, size = 0;
999 char *modname, namebuf[128]; 1002 char *modname, namebuf[128];
1000 const char *symbol_name; 1003 const char *symbol_name;
1001 void *addr; 1004 void *addr;
1002 struct kprobe_blackpoint *kb; 1005 struct kprobe_blackpoint *kb;
1003 1006
1004 /* FIXME allocate the probe table, currently defined statically */ 1007 /* FIXME allocate the probe table, currently defined statically */
1005 /* initialize all list heads */ 1008 /* initialize all list heads */
1006 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1009 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1007 INIT_HLIST_HEAD(&kprobe_table[i]); 1010 INIT_HLIST_HEAD(&kprobe_table[i]);
1008 INIT_HLIST_HEAD(&kretprobe_inst_table[i]); 1011 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1009 } 1012 }
1010 1013
1011 /* 1014 /*
1012 * Lookup and populate the kprobe_blacklist. 1015 * Lookup and populate the kprobe_blacklist.
1013 * 1016 *
1014 * Unlike the kretprobe blacklist, we'll need to determine 1017 * Unlike the kretprobe blacklist, we'll need to determine
1015 * the range of addresses that belong to the said functions, 1018 * the range of addresses that belong to the said functions,
1016 * since a kprobe need not necessarily be at the beginning 1019 * since a kprobe need not necessarily be at the beginning
1017 * of a function. 1020 * of a function.
1018 */ 1021 */
1019 for (kb = kprobe_blacklist; kb->name != NULL; kb++) { 1022 for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1020 kprobe_lookup_name(kb->name, addr); 1023 kprobe_lookup_name(kb->name, addr);
1021 if (!addr) 1024 if (!addr)
1022 continue; 1025 continue;
1023 1026
1024 kb->start_addr = (unsigned long)addr; 1027 kb->start_addr = (unsigned long)addr;
1025 symbol_name = kallsyms_lookup(kb->start_addr, 1028 symbol_name = kallsyms_lookup(kb->start_addr,
1026 &size, &offset, &modname, namebuf); 1029 &size, &offset, &modname, namebuf);
1027 if (!symbol_name) 1030 if (!symbol_name)
1028 kb->range = 0; 1031 kb->range = 0;
1029 else 1032 else
1030 kb->range = size; 1033 kb->range = size;
1031 } 1034 }
1032 1035
1033 if (kretprobe_blacklist_size) { 1036 if (kretprobe_blacklist_size) {
1034 /* lookup the function address from its name */ 1037 /* lookup the function address from its name */
1035 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) { 1038 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1036 kprobe_lookup_name(kretprobe_blacklist[i].name, 1039 kprobe_lookup_name(kretprobe_blacklist[i].name,
1037 kretprobe_blacklist[i].addr); 1040 kretprobe_blacklist[i].addr);
1038 if (!kretprobe_blacklist[i].addr) 1041 if (!kretprobe_blacklist[i].addr)
1039 printk("kretprobe: lookup failed: %s\n", 1042 printk("kretprobe: lookup failed: %s\n",
1040 kretprobe_blacklist[i].name); 1043 kretprobe_blacklist[i].name);
1041 } 1044 }
1042 } 1045 }
1043 1046
1044 /* By default, kprobes are enabled */ 1047 /* By default, kprobes are enabled */
1045 kprobe_enabled = true; 1048 kprobe_enabled = true;
1046 1049
1047 err = arch_init_kprobes(); 1050 err = arch_init_kprobes();
1048 if (!err) 1051 if (!err)
1049 err = register_die_notifier(&kprobe_exceptions_nb); 1052 err = register_die_notifier(&kprobe_exceptions_nb);
1050 1053
1051 if (!err) 1054 if (!err)
1052 init_test_probes(); 1055 init_test_probes();
1053 return err; 1056 return err;
1054 } 1057 }
1055 1058
1056 #ifdef CONFIG_DEBUG_FS 1059 #ifdef CONFIG_DEBUG_FS
1057 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p, 1060 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
1058 const char *sym, int offset,char *modname) 1061 const char *sym, int offset,char *modname)
1059 { 1062 {
1060 char *kprobe_type; 1063 char *kprobe_type;
1061 1064
1062 if (p->pre_handler == pre_handler_kretprobe) 1065 if (p->pre_handler == pre_handler_kretprobe)
1063 kprobe_type = "r"; 1066 kprobe_type = "r";
1064 else if (p->pre_handler == setjmp_pre_handler) 1067 else if (p->pre_handler == setjmp_pre_handler)
1065 kprobe_type = "j"; 1068 kprobe_type = "j";
1066 else 1069 else
1067 kprobe_type = "k"; 1070 kprobe_type = "k";
1068 if (sym) 1071 if (sym)
1069 seq_printf(pi, "%p %s %s+0x%x %s\n", p->addr, kprobe_type, 1072 seq_printf(pi, "%p %s %s+0x%x %s\n", p->addr, kprobe_type,
1070 sym, offset, (modname ? modname : " ")); 1073 sym, offset, (modname ? modname : " "));
1071 else 1074 else
1072 seq_printf(pi, "%p %s %p\n", p->addr, kprobe_type, p->addr); 1075 seq_printf(pi, "%p %s %p\n", p->addr, kprobe_type, p->addr);
1073 } 1076 }
1074 1077
1075 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos) 1078 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
1076 { 1079 {
1077 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL; 1080 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
1078 } 1081 }
1079 1082
1080 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos) 1083 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
1081 { 1084 {
1082 (*pos)++; 1085 (*pos)++;
1083 if (*pos >= KPROBE_TABLE_SIZE) 1086 if (*pos >= KPROBE_TABLE_SIZE)
1084 return NULL; 1087 return NULL;
1085 return pos; 1088 return pos;
1086 } 1089 }
1087 1090
1088 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v) 1091 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
1089 { 1092 {
1090 /* Nothing to do */ 1093 /* Nothing to do */
1091 } 1094 }
1092 1095
1093 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v) 1096 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
1094 { 1097 {
1095 struct hlist_head *head; 1098 struct hlist_head *head;
1096 struct hlist_node *node; 1099 struct hlist_node *node;
1097 struct kprobe *p, *kp; 1100 struct kprobe *p, *kp;
1098 const char *sym = NULL; 1101 const char *sym = NULL;
1099 unsigned int i = *(loff_t *) v; 1102 unsigned int i = *(loff_t *) v;
1100 unsigned long offset = 0; 1103 unsigned long offset = 0;
1101 char *modname, namebuf[128]; 1104 char *modname, namebuf[128];
1102 1105
1103 head = &kprobe_table[i]; 1106 head = &kprobe_table[i];
1104 preempt_disable(); 1107 preempt_disable();
1105 hlist_for_each_entry_rcu(p, node, head, hlist) { 1108 hlist_for_each_entry_rcu(p, node, head, hlist) {
1106 sym = kallsyms_lookup((unsigned long)p->addr, NULL, 1109 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
1107 &offset, &modname, namebuf); 1110 &offset, &modname, namebuf);
1108 if (p->pre_handler == aggr_pre_handler) { 1111 if (p->pre_handler == aggr_pre_handler) {
1109 list_for_each_entry_rcu(kp, &p->list, list) 1112 list_for_each_entry_rcu(kp, &p->list, list)
1110 report_probe(pi, kp, sym, offset, modname); 1113 report_probe(pi, kp, sym, offset, modname);
1111 } else 1114 } else
1112 report_probe(pi, p, sym, offset, modname); 1115 report_probe(pi, p, sym, offset, modname);
1113 } 1116 }
1114 preempt_enable(); 1117 preempt_enable();
1115 return 0; 1118 return 0;
1116 } 1119 }
1117 1120
1118 static struct seq_operations kprobes_seq_ops = { 1121 static struct seq_operations kprobes_seq_ops = {
1119 .start = kprobe_seq_start, 1122 .start = kprobe_seq_start,
1120 .next = kprobe_seq_next, 1123 .next = kprobe_seq_next,
1121 .stop = kprobe_seq_stop, 1124 .stop = kprobe_seq_stop,
1122 .show = show_kprobe_addr 1125 .show = show_kprobe_addr
1123 }; 1126 };
1124 1127
1125 static int __kprobes kprobes_open(struct inode *inode, struct file *filp) 1128 static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
1126 { 1129 {
1127 return seq_open(filp, &kprobes_seq_ops); 1130 return seq_open(filp, &kprobes_seq_ops);
1128 } 1131 }
1129 1132
1130 static struct file_operations debugfs_kprobes_operations = { 1133 static struct file_operations debugfs_kprobes_operations = {
1131 .open = kprobes_open, 1134 .open = kprobes_open,
1132 .read = seq_read, 1135 .read = seq_read,
1133 .llseek = seq_lseek, 1136 .llseek = seq_lseek,
1134 .release = seq_release, 1137 .release = seq_release,
1135 }; 1138 };
1136 1139
1137 static void __kprobes enable_all_kprobes(void) 1140 static void __kprobes enable_all_kprobes(void)
1138 { 1141 {
1139 struct hlist_head *head; 1142 struct hlist_head *head;
1140 struct hlist_node *node; 1143 struct hlist_node *node;
1141 struct kprobe *p; 1144 struct kprobe *p;
1142 unsigned int i; 1145 unsigned int i;
1143 1146
1144 mutex_lock(&kprobe_mutex); 1147 mutex_lock(&kprobe_mutex);
1145 1148
1146 /* If kprobes are already enabled, just return */ 1149 /* If kprobes are already enabled, just return */
1147 if (kprobe_enabled) 1150 if (kprobe_enabled)
1148 goto already_enabled; 1151 goto already_enabled;
1149 1152
1150 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1153 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1151 head = &kprobe_table[i]; 1154 head = &kprobe_table[i];
1152 hlist_for_each_entry_rcu(p, node, head, hlist) 1155 hlist_for_each_entry_rcu(p, node, head, hlist)
1153 arch_arm_kprobe(p); 1156 arch_arm_kprobe(p);
1154 } 1157 }
1155 1158
1156 kprobe_enabled = true; 1159 kprobe_enabled = true;
1157 printk(KERN_INFO "Kprobes globally enabled\n"); 1160 printk(KERN_INFO "Kprobes globally enabled\n");
1158 1161
1159 already_enabled: 1162 already_enabled:
1160 mutex_unlock(&kprobe_mutex); 1163 mutex_unlock(&kprobe_mutex);
1161 return; 1164 return;
1162 } 1165 }
1163 1166
1164 static void __kprobes disable_all_kprobes(void) 1167 static void __kprobes disable_all_kprobes(void)
1165 { 1168 {
1166 struct hlist_head *head; 1169 struct hlist_head *head;
1167 struct hlist_node *node; 1170 struct hlist_node *node;
1168 struct kprobe *p; 1171 struct kprobe *p;
1169 unsigned int i; 1172 unsigned int i;
1170 1173
1171 mutex_lock(&kprobe_mutex); 1174 mutex_lock(&kprobe_mutex);
1172 1175
1173 /* If kprobes are already disabled, just return */ 1176 /* If kprobes are already disabled, just return */
1174 if (!kprobe_enabled) 1177 if (!kprobe_enabled)
1175 goto already_disabled; 1178 goto already_disabled;
1176 1179
1177 kprobe_enabled = false; 1180 kprobe_enabled = false;
1178 printk(KERN_INFO "Kprobes globally disabled\n"); 1181 printk(KERN_INFO "Kprobes globally disabled\n");
1179 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1182 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1180 head = &kprobe_table[i]; 1183 head = &kprobe_table[i];
1181 hlist_for_each_entry_rcu(p, node, head, hlist) { 1184 hlist_for_each_entry_rcu(p, node, head, hlist) {
1182 if (!arch_trampoline_kprobe(p)) 1185 if (!arch_trampoline_kprobe(p))
1183 arch_disarm_kprobe(p); 1186 arch_disarm_kprobe(p);
1184 } 1187 }
1185 } 1188 }
1186 1189
1187 mutex_unlock(&kprobe_mutex); 1190 mutex_unlock(&kprobe_mutex);
1188 /* Allow all currently running kprobes to complete */ 1191 /* Allow all currently running kprobes to complete */
1189 synchronize_sched(); 1192 synchronize_sched();
1190 return; 1193 return;
1191 1194
1192 already_disabled: 1195 already_disabled:
1193 mutex_unlock(&kprobe_mutex); 1196 mutex_unlock(&kprobe_mutex);
1194 return; 1197 return;
1195 } 1198 }
1196 1199
1197 /* 1200 /*
1198 * XXX: The debugfs bool file interface doesn't allow for callbacks 1201 * XXX: The debugfs bool file interface doesn't allow for callbacks
1199 * when the bool state is switched. We can reuse that facility when 1202 * when the bool state is switched. We can reuse that facility when
1200 * available 1203 * available
1201 */ 1204 */
1202 static ssize_t read_enabled_file_bool(struct file *file, 1205 static ssize_t read_enabled_file_bool(struct file *file,
1203 char __user *user_buf, size_t count, loff_t *ppos) 1206 char __user *user_buf, size_t count, loff_t *ppos)
1204 { 1207 {
1205 char buf[3]; 1208 char buf[3];
1206 1209
1207 if (kprobe_enabled) 1210 if (kprobe_enabled)
1208 buf[0] = '1'; 1211 buf[0] = '1';
1209 else 1212 else
1210 buf[0] = '0'; 1213 buf[0] = '0';
1211 buf[1] = '\n'; 1214 buf[1] = '\n';
1212 buf[2] = 0x00; 1215 buf[2] = 0x00;
1213 return simple_read_from_buffer(user_buf, count, ppos, buf, 2); 1216 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1214 } 1217 }
1215 1218
1216 static ssize_t write_enabled_file_bool(struct file *file, 1219 static ssize_t write_enabled_file_bool(struct file *file,
1217 const char __user *user_buf, size_t count, loff_t *ppos) 1220 const char __user *user_buf, size_t count, loff_t *ppos)
1218 { 1221 {
1219 char buf[32]; 1222 char buf[32];
1220 int buf_size; 1223 int buf_size;
1221 1224
1222 buf_size = min(count, (sizeof(buf)-1)); 1225 buf_size = min(count, (sizeof(buf)-1));
1223 if (copy_from_user(buf, user_buf, buf_size)) 1226 if (copy_from_user(buf, user_buf, buf_size))
1224 return -EFAULT; 1227 return -EFAULT;
1225 1228
1226 switch (buf[0]) { 1229 switch (buf[0]) {
1227 case 'y': 1230 case 'y':
1228 case 'Y': 1231 case 'Y':
1229 case '1': 1232 case '1':
1230 enable_all_kprobes(); 1233 enable_all_kprobes();
1231 break; 1234 break;
1232 case 'n': 1235 case 'n':
1233 case 'N': 1236 case 'N':
1234 case '0': 1237 case '0':
1235 disable_all_kprobes(); 1238 disable_all_kprobes();
1236 break; 1239 break;
1237 } 1240 }
1238 1241
1239 return count; 1242 return count;
1240 } 1243 }
1241 1244
1242 static struct file_operations fops_kp = { 1245 static struct file_operations fops_kp = {
1243 .read = read_enabled_file_bool, 1246 .read = read_enabled_file_bool,
1244 .write = write_enabled_file_bool, 1247 .write = write_enabled_file_bool,
1245 }; 1248 };
1246 1249
1247 static int __kprobes debugfs_kprobe_init(void) 1250 static int __kprobes debugfs_kprobe_init(void)
1248 { 1251 {
1249 struct dentry *dir, *file; 1252 struct dentry *dir, *file;
1250 unsigned int value = 1; 1253 unsigned int value = 1;
1251 1254
1252 dir = debugfs_create_dir("kprobes", NULL); 1255 dir = debugfs_create_dir("kprobes", NULL);
1253 if (!dir) 1256 if (!dir)
1254 return -ENOMEM; 1257 return -ENOMEM;
1255 1258
1256 file = debugfs_create_file("list", 0444, dir, NULL, 1259 file = debugfs_create_file("list", 0444, dir, NULL,
1257 &debugfs_kprobes_operations); 1260 &debugfs_kprobes_operations);
1258 if (!file) { 1261 if (!file) {
1259 debugfs_remove(dir); 1262 debugfs_remove(dir);
1260 return -ENOMEM; 1263 return -ENOMEM;
1261 } 1264 }
1262 1265
1263 file = debugfs_create_file("enabled", 0600, dir, 1266 file = debugfs_create_file("enabled", 0600, dir,
1264 &value, &fops_kp); 1267 &value, &fops_kp);
1265 if (!file) { 1268 if (!file) {
1266 debugfs_remove(dir); 1269 debugfs_remove(dir);
1267 return -ENOMEM; 1270 return -ENOMEM;
1268 } 1271 }
1269 1272
1270 return 0; 1273 return 0;
1271 } 1274 }
1272 1275
1273 late_initcall(debugfs_kprobe_init); 1276 late_initcall(debugfs_kprobe_init);
1274 #endif /* CONFIG_DEBUG_FS */ 1277 #endif /* CONFIG_DEBUG_FS */
1275 1278
1276 module_init(init_kprobes); 1279 module_init(init_kprobes);
1277 1280
1278 EXPORT_SYMBOL_GPL(register_kprobe); 1281 EXPORT_SYMBOL_GPL(register_kprobe);
1279 EXPORT_SYMBOL_GPL(unregister_kprobe); 1282 EXPORT_SYMBOL_GPL(unregister_kprobe);
1280 EXPORT_SYMBOL_GPL(register_kprobes); 1283 EXPORT_SYMBOL_GPL(register_kprobes);
1281 EXPORT_SYMBOL_GPL(unregister_kprobes); 1284 EXPORT_SYMBOL_GPL(unregister_kprobes);
1282 EXPORT_SYMBOL_GPL(register_jprobe); 1285 EXPORT_SYMBOL_GPL(register_jprobe);
1283 EXPORT_SYMBOL_GPL(unregister_jprobe); 1286 EXPORT_SYMBOL_GPL(unregister_jprobe);
1284 EXPORT_SYMBOL_GPL(register_jprobes); 1287 EXPORT_SYMBOL_GPL(register_jprobes);
1285 EXPORT_SYMBOL_GPL(unregister_jprobes); 1288 EXPORT_SYMBOL_GPL(unregister_jprobes);
1286 #ifdef CONFIG_KPROBES 1289 #ifdef CONFIG_KPROBES
1287 EXPORT_SYMBOL_GPL(jprobe_return); 1290 EXPORT_SYMBOL_GPL(jprobe_return);
1288 #endif 1291 #endif
1289 1292
1290 #ifdef CONFIG_KPROBES 1293 #ifdef CONFIG_KPROBES
1291 EXPORT_SYMBOL_GPL(register_kretprobe); 1294 EXPORT_SYMBOL_GPL(register_kretprobe);
1292 EXPORT_SYMBOL_GPL(unregister_kretprobe); 1295 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1293 EXPORT_SYMBOL_GPL(register_kretprobes); 1296 EXPORT_SYMBOL_GPL(register_kretprobes);
1294 EXPORT_SYMBOL_GPL(unregister_kretprobes); 1297 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1295 #endif 1298 #endif
1296 1299