Blame view

kernel/tracepoint.c 19.6 KB
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
  /*
   * Copyright (C) 2008 Mathieu Desnoyers
   *
   * This program is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License as published by
   * the Free Software Foundation; either version 2 of the License, or
   * (at your option) any later version.
   *
   * This program is distributed in the hope that it will be useful,
   * but WITHOUT ANY WARRANTY; without even the implied warranty of
   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   * GNU General Public License for more details.
   *
   * You should have received a copy of the GNU General Public License
   * along with this program; if not, write to the Free Software
   * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
   */
  #include <linux/module.h>
  #include <linux/mutex.h>
  #include <linux/types.h>
  #include <linux/jhash.h>
  #include <linux/list.h>
  #include <linux/rcupdate.h>
  #include <linux/tracepoint.h>
  #include <linux/err.h>
  #include <linux/slab.h>
a871bd33a   Jason Baron   tracing: Add sysc...
27
  #include <linux/sched.h>
8f7b50c51   Jason Baron   jump label: Trace...
28
  #include <linux/jump_label.h>
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
29

654986462   Mathieu Desnoyers   tracepoints: Fix ...
30
31
  extern struct tracepoint * const __start___tracepoints_ptrs[];
  extern struct tracepoint * const __stop___tracepoints_ptrs[];
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
32
33
34
35
36
  
  /* Set to 1 to enable tracepoint debug output */
  static const int tracepoint_debug;
  
  /*
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
37
38
   * Tracepoints mutex protects the builtin and module tracepoints and the hash
   * table, as well as the local module list.
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
39
40
   */
  static DEFINE_MUTEX(tracepoints_mutex);
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
41
42
43
44
  #ifdef CONFIG_MODULES
  /* Local list of struct module */
  static LIST_HEAD(tracepoint_module_list);
  #endif /* CONFIG_MODULES */
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
45
46
47
48
49
50
  /*
   * Tracepoint hash table, containing the active tracepoints.
   * Protected by tracepoints_mutex.
   */
  #define TRACEPOINT_HASH_BITS 6
  #define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS)
19dba33c4   Lai Jiangshan   tracepoint: simpl...
51
  static struct hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE];
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
52
53
54
  
  /*
   * Note about RCU :
fd589a8f0   Anand Gadiyar   trivial: fix typo...
55
   * It is used to delay the free of multiple probes array until a quiescent
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
56
57
58
59
60
   * state is reached.
   * Tracepoint entries modifications are protected by the tracepoints_mutex.
   */
  struct tracepoint_entry {
  	struct hlist_node hlist;
38516ab59   Steven Rostedt   tracing: Let trac...
61
  	struct tracepoint_func *funcs;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
62
  	int refcount;	/* Number of times armed. 0 if disarmed. */
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
63
64
  	char name[0];
  };
19dba33c4   Lai Jiangshan   tracepoint: simpl...
65
  struct tp_probes {
127cafbb2   Lai Jiangshan   tracepoint: intro...
66
67
68
69
  	union {
  		struct rcu_head rcu;
  		struct list_head list;
  	} u;
38516ab59   Steven Rostedt   tracing: Let trac...
70
  	struct tracepoint_func probes[0];
19dba33c4   Lai Jiangshan   tracepoint: simpl...
71
  };
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
72

19dba33c4   Lai Jiangshan   tracepoint: simpl...
73
  static inline void *allocate_probes(int count)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
74
  {
38516ab59   Steven Rostedt   tracing: Let trac...
75
  	struct tp_probes *p  = kmalloc(count * sizeof(struct tracepoint_func)
19dba33c4   Lai Jiangshan   tracepoint: simpl...
76
77
  			+ sizeof(struct tp_probes), GFP_KERNEL);
  	return p == NULL ? NULL : p->probes;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
78
  }
19dba33c4   Lai Jiangshan   tracepoint: simpl...
79
  static void rcu_free_old_probes(struct rcu_head *head)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
80
  {
127cafbb2   Lai Jiangshan   tracepoint: intro...
81
  	kfree(container_of(head, struct tp_probes, u.rcu));
19dba33c4   Lai Jiangshan   tracepoint: simpl...
82
  }
38516ab59   Steven Rostedt   tracing: Let trac...
83
  static inline void release_probes(struct tracepoint_func *old)
19dba33c4   Lai Jiangshan   tracepoint: simpl...
84
85
86
87
  {
  	if (old) {
  		struct tp_probes *tp_probes = container_of(old,
  			struct tp_probes, probes[0]);
127cafbb2   Lai Jiangshan   tracepoint: intro...
88
  		call_rcu_sched(&tp_probes->u.rcu, rcu_free_old_probes);
19dba33c4   Lai Jiangshan   tracepoint: simpl...
89
  	}
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
90
91
92
93
94
  }
  
  static void debug_print_probes(struct tracepoint_entry *entry)
  {
  	int i;
19dba33c4   Lai Jiangshan   tracepoint: simpl...
95
  	if (!tracepoint_debug || !entry->funcs)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
96
  		return;
38516ab59   Steven Rostedt   tracing: Let trac...
97
98
99
  	for (i = 0; entry->funcs[i].func; i++)
  		printk(KERN_DEBUG "Probe %d : %p
  ", i, entry->funcs[i].func);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
100
  }
38516ab59   Steven Rostedt   tracing: Let trac...
101
102
103
  static struct tracepoint_func *
  tracepoint_entry_add_probe(struct tracepoint_entry *entry,
  			   void *probe, void *data)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
104
105
  {
  	int nr_probes = 0;
38516ab59   Steven Rostedt   tracing: Let trac...
106
  	struct tracepoint_func *old, *new;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
107
108
109
110
111
112
113
  
  	WARN_ON(!probe);
  
  	debug_print_probes(entry);
  	old = entry->funcs;
  	if (old) {
  		/* (N -> N+1), (N != 0, 1) probes */
38516ab59   Steven Rostedt   tracing: Let trac...
114
115
116
  		for (nr_probes = 0; old[nr_probes].func; nr_probes++)
  			if (old[nr_probes].func == probe &&
  			    old[nr_probes].data == data)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
117
118
119
  				return ERR_PTR(-EEXIST);
  	}
  	/* + 2 : one for new probe, one for NULL func */
19dba33c4   Lai Jiangshan   tracepoint: simpl...
120
  	new = allocate_probes(nr_probes + 2);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
121
122
123
  	if (new == NULL)
  		return ERR_PTR(-ENOMEM);
  	if (old)
38516ab59   Steven Rostedt   tracing: Let trac...
124
125
126
127
  		memcpy(new, old, nr_probes * sizeof(struct tracepoint_func));
  	new[nr_probes].func = probe;
  	new[nr_probes].data = data;
  	new[nr_probes + 1].func = NULL;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
128
129
130
131
132
133
134
  	entry->refcount = nr_probes + 1;
  	entry->funcs = new;
  	debug_print_probes(entry);
  	return old;
  }
  
  static void *
38516ab59   Steven Rostedt   tracing: Let trac...
135
136
  tracepoint_entry_remove_probe(struct tracepoint_entry *entry,
  			      void *probe, void *data)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
137
138
  {
  	int nr_probes = 0, nr_del = 0, i;
38516ab59   Steven Rostedt   tracing: Let trac...
139
  	struct tracepoint_func *old, *new;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
140
141
  
  	old = entry->funcs;
f66af459a   Frederic Weisbecker   tracepoint: check...
142
  	if (!old)
19dba33c4   Lai Jiangshan   tracepoint: simpl...
143
  		return ERR_PTR(-ENOENT);
f66af459a   Frederic Weisbecker   tracepoint: check...
144

97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
145
146
  	debug_print_probes(entry);
  	/* (N -> M), (N > 1, M >= 0) probes */
38516ab59   Steven Rostedt   tracing: Let trac...
147
148
149
150
  	for (nr_probes = 0; old[nr_probes].func; nr_probes++) {
  		if (!probe ||
  		    (old[nr_probes].func == probe &&
  		     old[nr_probes].data == data))
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
151
152
153
154
155
156
157
158
159
160
161
162
163
  			nr_del++;
  	}
  
  	if (nr_probes - nr_del == 0) {
  		/* N -> 0, (N > 1) */
  		entry->funcs = NULL;
  		entry->refcount = 0;
  		debug_print_probes(entry);
  		return old;
  	} else {
  		int j = 0;
  		/* N -> M, (N > 1, M > 0) */
  		/* + 1 for NULL */
19dba33c4   Lai Jiangshan   tracepoint: simpl...
164
  		new = allocate_probes(nr_probes - nr_del + 1);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
165
166
  		if (new == NULL)
  			return ERR_PTR(-ENOMEM);
38516ab59   Steven Rostedt   tracing: Let trac...
167
168
169
  		for (i = 0; old[i].func; i++)
  			if (probe &&
  			    (old[i].func != probe || old[i].data != data))
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
170
  				new[j++] = old[i];
38516ab59   Steven Rostedt   tracing: Let trac...
171
  		new[nr_probes - nr_del].func = NULL;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
  		entry->refcount = nr_probes - nr_del;
  		entry->funcs = new;
  	}
  	debug_print_probes(entry);
  	return old;
  }
  
  /*
   * Get tracepoint if the tracepoint is present in the tracepoint hash table.
   * Must be called with tracepoints_mutex held.
   * Returns NULL if not present.
   */
  static struct tracepoint_entry *get_tracepoint(const char *name)
  {
  	struct hlist_head *head;
  	struct hlist_node *node;
  	struct tracepoint_entry *e;
  	u32 hash = jhash(name, strlen(name), 0);
9795302ac   Mathieu Desnoyers   tracepoints: use ...
190
  	head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
  	hlist_for_each_entry(e, node, head, hlist) {
  		if (!strcmp(name, e->name))
  			return e;
  	}
  	return NULL;
  }
  
  /*
   * Add the tracepoint to the tracepoint hash table. Must be called with
   * tracepoints_mutex held.
   */
  static struct tracepoint_entry *add_tracepoint(const char *name)
  {
  	struct hlist_head *head;
  	struct hlist_node *node;
  	struct tracepoint_entry *e;
  	size_t name_len = strlen(name) + 1;
  	u32 hash = jhash(name, name_len-1, 0);
9795302ac   Mathieu Desnoyers   tracepoints: use ...
209
  	head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
  	hlist_for_each_entry(e, node, head, hlist) {
  		if (!strcmp(name, e->name)) {
  			printk(KERN_NOTICE
  				"tracepoint %s busy
  ", name);
  			return ERR_PTR(-EEXIST);	/* Already there */
  		}
  	}
  	/*
  	 * Using kmalloc here to allocate a variable length element. Could
  	 * cause some memory fragmentation if overused.
  	 */
  	e = kmalloc(sizeof(struct tracepoint_entry) + name_len, GFP_KERNEL);
  	if (!e)
  		return ERR_PTR(-ENOMEM);
  	memcpy(&e->name[0], name, name_len);
  	e->funcs = NULL;
  	e->refcount = 0;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
228
229
230
231
232
233
234
235
  	hlist_add_head(&e->hlist, head);
  	return e;
  }
  
  /*
   * Remove the tracepoint from the tracepoint hash table. Must be called with
   * mutex_lock held.
   */
19dba33c4   Lai Jiangshan   tracepoint: simpl...
236
  static inline void remove_tracepoint(struct tracepoint_entry *e)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
237
  {
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
238
  	hlist_del(&e->hlist);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
239
  	kfree(e);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
240
241
242
243
244
245
246
247
248
  }
  
  /*
   * Sets the probe callback corresponding to one tracepoint.
   */
  static void set_tracepoint(struct tracepoint_entry **entry,
  	struct tracepoint *elem, int active)
  {
  	WARN_ON(strcmp((*entry)->name, elem->name) != 0);
d430d3d7e   Jason Baron   jump label: Intro...
249
  	if (elem->regfunc && !jump_label_enabled(&elem->key) && active)
974198758   Josh Stone   tracing: Move tra...
250
  		elem->regfunc();
d430d3d7e   Jason Baron   jump label: Intro...
251
  	else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active)
974198758   Josh Stone   tracing: Move tra...
252
  		elem->unregfunc();
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
253
254
255
256
257
258
259
260
  	/*
  	 * rcu_assign_pointer has a smp_wmb() which makes sure that the new
  	 * probe callbacks array is consistent before setting a pointer to it.
  	 * This array is referenced by __DO_TRACE from
  	 * include/linux/tracepoints.h. A matching smp_read_barrier_depends()
  	 * is used.
  	 */
  	rcu_assign_pointer(elem->funcs, (*entry)->funcs);
d430d3d7e   Jason Baron   jump label: Intro...
261
262
263
264
  	if (active && !jump_label_enabled(&elem->key))
  		jump_label_inc(&elem->key);
  	else if (!active && jump_label_enabled(&elem->key))
  		jump_label_dec(&elem->key);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
265
266
267
268
269
270
271
272
273
274
  }
  
  /*
   * Disable a tracepoint and its probe callback.
   * Note: only waiting an RCU period after setting elem->call to the empty
   * function insures that the original callback is not used anymore. This insured
   * by preempt_disable around the call site.
   */
  static void disable_tracepoint(struct tracepoint *elem)
  {
d430d3d7e   Jason Baron   jump label: Intro...
275
  	if (elem->unregfunc && jump_label_enabled(&elem->key))
974198758   Josh Stone   tracing: Move tra...
276
  		elem->unregfunc();
d430d3d7e   Jason Baron   jump label: Intro...
277
278
  	if (jump_label_enabled(&elem->key))
  		jump_label_dec(&elem->key);
de0baf9ad   Mathieu Desnoyers   tracepoints: fix ...
279
  	rcu_assign_pointer(elem->funcs, NULL);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
280
281
282
283
284
285
286
287
  }
  
  /**
   * tracepoint_update_probe_range - Update a probe range
   * @begin: beginning of the range
   * @end: end of the range
   *
   * Updates the probe callback corresponding to a range of tracepoints.
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
288
   * Called with tracepoints_mutex held.
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
289
   */
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
290
291
  static void tracepoint_update_probe_range(struct tracepoint * const *begin,
  					  struct tracepoint * const *end)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
292
  {
654986462   Mathieu Desnoyers   tracepoints: Fix ...
293
  	struct tracepoint * const *iter;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
294
  	struct tracepoint_entry *mark_entry;
ec625cb29   Ingo Molnar   tracepoints: dont...
295
  	if (!begin)
09933a108   Jaswinder Singh Rajput   tracing: fix oops...
296
  		return;
09933a108   Jaswinder Singh Rajput   tracing: fix oops...
297

97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
298
  	for (iter = begin; iter < end; iter++) {
654986462   Mathieu Desnoyers   tracepoints: Fix ...
299
  		mark_entry = get_tracepoint((*iter)->name);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
300
  		if (mark_entry) {
654986462   Mathieu Desnoyers   tracepoints: Fix ...
301
  			set_tracepoint(&mark_entry, *iter,
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
302
303
  					!!mark_entry->refcount);
  		} else {
654986462   Mathieu Desnoyers   tracepoints: Fix ...
304
  			disable_tracepoint(*iter);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
305
306
  		}
  	}
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
307
  }
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
308
309
310
311
312
313
314
315
316
317
318
319
320
321
  #ifdef CONFIG_MODULES
  void module_update_tracepoints(void)
  {
  	struct tp_module *tp_mod;
  
  	list_for_each_entry(tp_mod, &tracepoint_module_list, list)
  		tracepoint_update_probe_range(tp_mod->tracepoints_ptrs,
  			tp_mod->tracepoints_ptrs + tp_mod->num_tracepoints);
  }
  #else /* CONFIG_MODULES */
  void module_update_tracepoints(void)
  {
  }
  #endif /* CONFIG_MODULES */
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
322
323
  /*
   * Update probes, removing the faulty probes.
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
324
   * Called with tracepoints_mutex held.
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
325
326
327
328
   */
  static void tracepoint_update_probes(void)
  {
  	/* Core kernel tracepoints */
654986462   Mathieu Desnoyers   tracepoints: Fix ...
329
330
  	tracepoint_update_probe_range(__start___tracepoints_ptrs,
  		__stop___tracepoints_ptrs);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
331
332
333
  	/* tracepoints in modules. */
  	module_update_tracepoints();
  }
38516ab59   Steven Rostedt   tracing: Let trac...
334
335
  static struct tracepoint_func *
  tracepoint_add_probe(const char *name, void *probe, void *data)
127cafbb2   Lai Jiangshan   tracepoint: intro...
336
337
  {
  	struct tracepoint_entry *entry;
38516ab59   Steven Rostedt   tracing: Let trac...
338
  	struct tracepoint_func *old;
127cafbb2   Lai Jiangshan   tracepoint: intro...
339
340
341
342
343
  
  	entry = get_tracepoint(name);
  	if (!entry) {
  		entry = add_tracepoint(name);
  		if (IS_ERR(entry))
38516ab59   Steven Rostedt   tracing: Let trac...
344
  			return (struct tracepoint_func *)entry;
127cafbb2   Lai Jiangshan   tracepoint: intro...
345
  	}
38516ab59   Steven Rostedt   tracing: Let trac...
346
  	old = tracepoint_entry_add_probe(entry, probe, data);
127cafbb2   Lai Jiangshan   tracepoint: intro...
347
348
349
350
  	if (IS_ERR(old) && !entry->refcount)
  		remove_tracepoint(entry);
  	return old;
  }
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
351
352
353
354
355
356
357
358
  /**
   * tracepoint_probe_register -  Connect a probe to a tracepoint
   * @name: tracepoint name
   * @probe: probe handler
   *
   * Returns 0 if ok, error value on error.
   * The probe address must at least be aligned on the architecture pointer size.
   */
38516ab59   Steven Rostedt   tracing: Let trac...
359
  int tracepoint_probe_register(const char *name, void *probe, void *data)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
360
  {
38516ab59   Steven Rostedt   tracing: Let trac...
361
  	struct tracepoint_func *old;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
362
363
  
  	mutex_lock(&tracepoints_mutex);
38516ab59   Steven Rostedt   tracing: Let trac...
364
  	old = tracepoint_add_probe(name, probe, data);
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
365
366
  	if (IS_ERR(old)) {
  		mutex_unlock(&tracepoints_mutex);
127cafbb2   Lai Jiangshan   tracepoint: intro...
367
  		return PTR_ERR(old);
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
368
  	}
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
369
  	tracepoint_update_probes();		/* may update entry */
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
370
  	mutex_unlock(&tracepoints_mutex);
19dba33c4   Lai Jiangshan   tracepoint: simpl...
371
372
  	release_probes(old);
  	return 0;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
373
374
  }
  EXPORT_SYMBOL_GPL(tracepoint_probe_register);
38516ab59   Steven Rostedt   tracing: Let trac...
375
376
  static struct tracepoint_func *
  tracepoint_remove_probe(const char *name, void *probe, void *data)
127cafbb2   Lai Jiangshan   tracepoint: intro...
377
378
  {
  	struct tracepoint_entry *entry;
38516ab59   Steven Rostedt   tracing: Let trac...
379
  	struct tracepoint_func *old;
127cafbb2   Lai Jiangshan   tracepoint: intro...
380
381
382
383
  
  	entry = get_tracepoint(name);
  	if (!entry)
  		return ERR_PTR(-ENOENT);
38516ab59   Steven Rostedt   tracing: Let trac...
384
  	old = tracepoint_entry_remove_probe(entry, probe, data);
127cafbb2   Lai Jiangshan   tracepoint: intro...
385
386
387
388
389
390
  	if (IS_ERR(old))
  		return old;
  	if (!entry->refcount)
  		remove_tracepoint(entry);
  	return old;
  }
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
391
392
393
394
395
396
397
398
399
400
  /**
   * tracepoint_probe_unregister -  Disconnect a probe from a tracepoint
   * @name: tracepoint name
   * @probe: probe function pointer
   *
   * We do not need to call a synchronize_sched to make sure the probes have
   * finished running before doing a module unload, because the module unload
   * itself uses stop_machine(), which insures that every preempt disabled section
   * have finished.
   */
38516ab59   Steven Rostedt   tracing: Let trac...
401
  int tracepoint_probe_unregister(const char *name, void *probe, void *data)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
402
  {
38516ab59   Steven Rostedt   tracing: Let trac...
403
  	struct tracepoint_func *old;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
404
405
  
  	mutex_lock(&tracepoints_mutex);
38516ab59   Steven Rostedt   tracing: Let trac...
406
  	old = tracepoint_remove_probe(name, probe, data);
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
407
408
  	if (IS_ERR(old)) {
  		mutex_unlock(&tracepoints_mutex);
127cafbb2   Lai Jiangshan   tracepoint: intro...
409
  		return PTR_ERR(old);
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
410
  	}
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
411
  	tracepoint_update_probes();		/* may update entry */
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
412
  	mutex_unlock(&tracepoints_mutex);
19dba33c4   Lai Jiangshan   tracepoint: simpl...
413
414
  	release_probes(old);
  	return 0;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
415
416
  }
  EXPORT_SYMBOL_GPL(tracepoint_probe_unregister);
127cafbb2   Lai Jiangshan   tracepoint: intro...
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
  static LIST_HEAD(old_probes);
  static int need_update;
  
  static void tracepoint_add_old_probes(void *old)
  {
  	need_update = 1;
  	if (old) {
  		struct tp_probes *tp_probes = container_of(old,
  			struct tp_probes, probes[0]);
  		list_add(&tp_probes->u.list, &old_probes);
  	}
  }
  
  /**
   * tracepoint_probe_register_noupdate -  register a probe but not connect
   * @name: tracepoint name
   * @probe: probe handler
   *
   * caller must call tracepoint_probe_update_all()
   */
38516ab59   Steven Rostedt   tracing: Let trac...
437
438
  int tracepoint_probe_register_noupdate(const char *name, void *probe,
  				       void *data)
127cafbb2   Lai Jiangshan   tracepoint: intro...
439
  {
38516ab59   Steven Rostedt   tracing: Let trac...
440
  	struct tracepoint_func *old;
127cafbb2   Lai Jiangshan   tracepoint: intro...
441
442
  
  	mutex_lock(&tracepoints_mutex);
38516ab59   Steven Rostedt   tracing: Let trac...
443
  	old = tracepoint_add_probe(name, probe, data);
127cafbb2   Lai Jiangshan   tracepoint: intro...
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
  	if (IS_ERR(old)) {
  		mutex_unlock(&tracepoints_mutex);
  		return PTR_ERR(old);
  	}
  	tracepoint_add_old_probes(old);
  	mutex_unlock(&tracepoints_mutex);
  	return 0;
  }
  EXPORT_SYMBOL_GPL(tracepoint_probe_register_noupdate);
  
  /**
   * tracepoint_probe_unregister_noupdate -  remove a probe but not disconnect
   * @name: tracepoint name
   * @probe: probe function pointer
   *
   * caller must call tracepoint_probe_update_all()
   */
38516ab59   Steven Rostedt   tracing: Let trac...
461
462
  int tracepoint_probe_unregister_noupdate(const char *name, void *probe,
  					 void *data)
127cafbb2   Lai Jiangshan   tracepoint: intro...
463
  {
38516ab59   Steven Rostedt   tracing: Let trac...
464
  	struct tracepoint_func *old;
127cafbb2   Lai Jiangshan   tracepoint: intro...
465
466
  
  	mutex_lock(&tracepoints_mutex);
38516ab59   Steven Rostedt   tracing: Let trac...
467
  	old = tracepoint_remove_probe(name, probe, data);
127cafbb2   Lai Jiangshan   tracepoint: intro...
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
  	if (IS_ERR(old)) {
  		mutex_unlock(&tracepoints_mutex);
  		return PTR_ERR(old);
  	}
  	tracepoint_add_old_probes(old);
  	mutex_unlock(&tracepoints_mutex);
  	return 0;
  }
  EXPORT_SYMBOL_GPL(tracepoint_probe_unregister_noupdate);
  
  /**
   * tracepoint_probe_update_all -  update tracepoints
   */
  void tracepoint_probe_update_all(void)
  {
  	LIST_HEAD(release_probes);
  	struct tp_probes *pos, *next;
  
  	mutex_lock(&tracepoints_mutex);
  	if (!need_update) {
  		mutex_unlock(&tracepoints_mutex);
  		return;
  	}
  	if (!list_empty(&old_probes))
  		list_replace_init(&old_probes, &release_probes);
  	need_update = 0;
127cafbb2   Lai Jiangshan   tracepoint: intro...
494
  	tracepoint_update_probes();
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
495
  	mutex_unlock(&tracepoints_mutex);
127cafbb2   Lai Jiangshan   tracepoint: intro...
496
497
498
499
500
501
  	list_for_each_entry_safe(pos, next, &release_probes, u.list) {
  		list_del(&pos->u.list);
  		call_rcu_sched(&pos->u.rcu, rcu_free_old_probes);
  	}
  }
  EXPORT_SYMBOL_GPL(tracepoint_probe_update_all);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
502
503
504
505
506
507
508
509
510
511
  /**
   * tracepoint_get_iter_range - Get a next tracepoint iterator given a range.
   * @tracepoint: current tracepoints (in), next tracepoint (out)
   * @begin: beginning of the range
   * @end: end of the range
   *
   * Returns whether a next tracepoint has been found (1) or not (0).
   * Will return the first tracepoint in the range if the input tracepoint is
   * NULL.
   */
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
512
  static int tracepoint_get_iter_range(struct tracepoint * const **tracepoint,
654986462   Mathieu Desnoyers   tracepoints: Fix ...
513
  	struct tracepoint * const *begin, struct tracepoint * const *end)
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
514
515
516
517
518
519
520
521
522
  {
  	if (!*tracepoint && begin != end) {
  		*tracepoint = begin;
  		return 1;
  	}
  	if (*tracepoint >= begin && *tracepoint < end)
  		return 1;
  	return 0;
  }
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
523

b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
524
  #ifdef CONFIG_MODULES
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
525
526
527
  static void tracepoint_get_iter(struct tracepoint_iter *iter)
  {
  	int found = 0;
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
528
  	struct tp_module *iter_mod;
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
529
530
531
532
  
  	/* Core kernel tracepoints */
  	if (!iter->module) {
  		found = tracepoint_get_iter_range(&iter->tracepoint,
654986462   Mathieu Desnoyers   tracepoints: Fix ...
533
534
  				__start___tracepoints_ptrs,
  				__stop___tracepoints_ptrs);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
535
536
537
  		if (found)
  			goto end;
  	}
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
  	/* Tracepoints in modules */
  	mutex_lock(&tracepoints_mutex);
  	list_for_each_entry(iter_mod, &tracepoint_module_list, list) {
  		/*
  		 * Sorted module list
  		 */
  		if (iter_mod < iter->module)
  			continue;
  		else if (iter_mod > iter->module)
  			iter->tracepoint = NULL;
  		found = tracepoint_get_iter_range(&iter->tracepoint,
  			iter_mod->tracepoints_ptrs,
  			iter_mod->tracepoints_ptrs
  				+ iter_mod->num_tracepoints);
  		if (found) {
  			iter->module = iter_mod;
  			break;
  		}
  	}
  	mutex_unlock(&tracepoints_mutex);
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
558
559
560
561
  end:
  	if (!found)
  		tracepoint_iter_reset(iter);
  }
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
562
563
564
565
566
567
568
569
570
571
572
573
574
  #else /* CONFIG_MODULES */
  static void tracepoint_get_iter(struct tracepoint_iter *iter)
  {
  	int found = 0;
  
  	/* Core kernel tracepoints */
  	found = tracepoint_get_iter_range(&iter->tracepoint,
  			__start___tracepoints_ptrs,
  			__stop___tracepoints_ptrs);
  	if (!found)
  		tracepoint_iter_reset(iter);
  }
  #endif /* CONFIG_MODULES */
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
  
  void tracepoint_iter_start(struct tracepoint_iter *iter)
  {
  	tracepoint_get_iter(iter);
  }
  EXPORT_SYMBOL_GPL(tracepoint_iter_start);
  
  void tracepoint_iter_next(struct tracepoint_iter *iter)
  {
  	iter->tracepoint++;
  	/*
  	 * iter->tracepoint may be invalid because we blindly incremented it.
  	 * Make sure it is valid by marshalling on the tracepoints, getting the
  	 * tracepoints from following modules if necessary.
  	 */
  	tracepoint_get_iter(iter);
  }
  EXPORT_SYMBOL_GPL(tracepoint_iter_next);
  
  void tracepoint_iter_stop(struct tracepoint_iter *iter)
  {
  }
  EXPORT_SYMBOL_GPL(tracepoint_iter_stop);
  
  void tracepoint_iter_reset(struct tracepoint_iter *iter)
  {
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
601
  #ifdef CONFIG_MODULES
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
602
  	iter->module = NULL;
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
603
  #endif /* CONFIG_MODULES */
97e1c18e8   Mathieu Desnoyers   tracing: Kernel T...
604
605
606
  	iter->tracepoint = NULL;
  }
  EXPORT_SYMBOL_GPL(tracepoint_iter_reset);
32f857427   Mathieu Desnoyers   tracepoints: use ...
607

227a83756   Ingo Molnar   markers/tracpoint...
608
  #ifdef CONFIG_MODULES
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
  static int tracepoint_module_coming(struct module *mod)
  {
  	struct tp_module *tp_mod, *iter;
  	int ret = 0;
  
  	/*
  	 * We skip modules that tain the kernel, especially those with different
  	 * module header (for forced load), to make sure we don't cause a crash.
  	 */
  	if (mod->taints)
  		return 0;
  	mutex_lock(&tracepoints_mutex);
  	tp_mod = kmalloc(sizeof(struct tp_module), GFP_KERNEL);
  	if (!tp_mod) {
  		ret = -ENOMEM;
  		goto end;
  	}
  	tp_mod->num_tracepoints = mod->num_tracepoints;
  	tp_mod->tracepoints_ptrs = mod->tracepoints_ptrs;
  
  	/*
  	 * tracepoint_module_list is kept sorted by struct module pointer
  	 * address for iteration on tracepoints from a seq_file that can release
  	 * the mutex between calls.
  	 */
  	list_for_each_entry_reverse(iter, &tracepoint_module_list, list) {
  		BUG_ON(iter == tp_mod);	/* Should never be in the list twice */
  		if (iter < tp_mod) {
  			/* We belong to the location right after iter. */
  			list_add(&tp_mod->list, &iter->list);
  			goto module_added;
  		}
  	}
  	/* We belong to the beginning of the list */
  	list_add(&tp_mod->list, &tracepoint_module_list);
  module_added:
  	tracepoint_update_probe_range(mod->tracepoints_ptrs,
  		mod->tracepoints_ptrs + mod->num_tracepoints);
  end:
  	mutex_unlock(&tracepoints_mutex);
  	return ret;
  }
  
  static int tracepoint_module_going(struct module *mod)
  {
  	struct tp_module *pos;
  
  	mutex_lock(&tracepoints_mutex);
  	tracepoint_update_probe_range(mod->tracepoints_ptrs,
  		mod->tracepoints_ptrs + mod->num_tracepoints);
  	list_for_each_entry(pos, &tracepoint_module_list, list) {
  		if (pos->tracepoints_ptrs == mod->tracepoints_ptrs) {
  			list_del(&pos->list);
  			kfree(pos);
  			break;
  		}
  	}
  	/*
  	 * In the case of modules that were tainted at "coming", we'll simply
  	 * walk through the list without finding it. We cannot use the "tainted"
  	 * flag on "going", in case a module taints the kernel only after being
  	 * loaded.
  	 */
  	mutex_unlock(&tracepoints_mutex);
  	return 0;
  }
227a83756   Ingo Molnar   markers/tracpoint...
675

32f857427   Mathieu Desnoyers   tracepoints: use ...
676
677
678
679
  int tracepoint_module_notify(struct notifier_block *self,
  			     unsigned long val, void *data)
  {
  	struct module *mod = data;
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
680
  	int ret = 0;
32f857427   Mathieu Desnoyers   tracepoints: use ...
681
682
683
  
  	switch (val) {
  	case MODULE_STATE_COMING:
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
684
685
686
687
  		ret = tracepoint_module_coming(mod);
  		break;
  	case MODULE_STATE_LIVE:
  		break;
32f857427   Mathieu Desnoyers   tracepoints: use ...
688
  	case MODULE_STATE_GOING:
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
689
  		ret = tracepoint_module_going(mod);
32f857427   Mathieu Desnoyers   tracepoints: use ...
690
691
  		break;
  	}
b75ef8b44   Mathieu Desnoyers   Tracepoint: Disso...
692
  	return ret;
32f857427   Mathieu Desnoyers   tracepoints: use ...
693
694
695
696
697
698
699
700
701
702
703
704
  }
  
  struct notifier_block tracepoint_module_nb = {
  	.notifier_call = tracepoint_module_notify,
  	.priority = 0,
  };
  
  static int init_tracepoints(void)
  {
  	return register_module_notifier(&tracepoint_module_nb);
  }
  __initcall(init_tracepoints);
227a83756   Ingo Molnar   markers/tracpoint...
705
  #endif /* CONFIG_MODULES */
a871bd33a   Jason Baron   tracing: Add sysc...
706

3d27d8cb3   Josh Stone   tracing: Make sys...
707
  #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS
60d970c25   Ingo Molnar   tracing: Fix sysc...
708

974198758   Josh Stone   tracing: Move tra...
709
  /* NB: reg/unreg are called while guarded with the tracepoints_mutex */
a871bd33a   Jason Baron   tracing: Add sysc...
710
711
712
713
714
715
  static int sys_tracepoint_refcount;
  
  void syscall_regfunc(void)
  {
  	unsigned long flags;
  	struct task_struct *g, *t;
a871bd33a   Jason Baron   tracing: Add sysc...
716
717
718
  	if (!sys_tracepoint_refcount) {
  		read_lock_irqsave(&tasklist_lock, flags);
  		do_each_thread(g, t) {
cc3b13c11   Hendrik Brueckner   tracing: Don't tr...
719
720
721
  			/* Skip kernel threads. */
  			if (t->mm)
  				set_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
a871bd33a   Jason Baron   tracing: Add sysc...
722
723
724
725
  		} while_each_thread(g, t);
  		read_unlock_irqrestore(&tasklist_lock, flags);
  	}
  	sys_tracepoint_refcount++;
a871bd33a   Jason Baron   tracing: Add sysc...
726
727
728
729
730
731
  }
  
  void syscall_unregfunc(void)
  {
  	unsigned long flags;
  	struct task_struct *g, *t;
a871bd33a   Jason Baron   tracing: Add sysc...
732
733
734
735
  	sys_tracepoint_refcount--;
  	if (!sys_tracepoint_refcount) {
  		read_lock_irqsave(&tasklist_lock, flags);
  		do_each_thread(g, t) {
667000011   Josh Stone   tracing: Rename F...
736
  			clear_tsk_thread_flag(t, TIF_SYSCALL_TRACEPOINT);
a871bd33a   Jason Baron   tracing: Add sysc...
737
738
739
  		} while_each_thread(g, t);
  		read_unlock_irqrestore(&tasklist_lock, flags);
  	}
a871bd33a   Jason Baron   tracing: Add sysc...
740
  }
60d970c25   Ingo Molnar   tracing: Fix sysc...
741
  #endif