Blame view

kernel/jump_label.c 19.8 KB
bf5438fca   Jason Baron   jump label: Base ...
1
2
3
4
  /*
   * jump label support
   *
   * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
90eec103b   Peter Zijlstra   treewide: Remove ...
5
   * Copyright (C) 2011 Peter Zijlstra
bf5438fca   Jason Baron   jump label: Base ...
6
7
   *
   */
bf5438fca   Jason Baron   jump label: Base ...
8
9
10
11
  #include <linux/memory.h>
  #include <linux/uaccess.h>
  #include <linux/module.h>
  #include <linux/list.h>
bf5438fca   Jason Baron   jump label: Base ...
12
13
14
  #include <linux/slab.h>
  #include <linux/sort.h>
  #include <linux/err.h>
c5905afb0   Ingo Molnar   static keys: Intr...
15
  #include <linux/static_key.h>
851cf6e7d   Andrew Jones   jump_label: Split...
16
  #include <linux/jump_label_ratelimit.h>
1f69bf9c6   Jason Baron   jump_label: remov...
17
  #include <linux/bug.h>
f2545b2d4   Thomas Gleixner   jump_label: Reord...
18
  #include <linux/cpu.h>
578ae447e   Josh Poimboeuf   jump_label: Disab...
19
  #include <asm/sections.h>
bf5438fca   Jason Baron   jump label: Base ...
20
21
  
  #ifdef HAVE_JUMP_LABEL
bf5438fca   Jason Baron   jump label: Base ...
22
23
  /* mutex to protect coming/going of the the jump_label table */
  static DEFINE_MUTEX(jump_label_mutex);
91bad2f8d   Jason Baron   jump label: Fix d...
24
25
26
27
28
29
30
31
32
  void jump_label_lock(void)
  {
  	mutex_lock(&jump_label_mutex);
  }
  
  void jump_label_unlock(void)
  {
  	mutex_unlock(&jump_label_mutex);
  }
bf5438fca   Jason Baron   jump label: Base ...
33
34
35
36
  static int jump_label_cmp(const void *a, const void *b)
  {
  	const struct jump_entry *jea = a;
  	const struct jump_entry *jeb = b;
9ae033aca   Ard Biesheuvel   jump_label: Abstr...
37
  	if (jump_entry_key(jea) < jump_entry_key(jeb))
bf5438fca   Jason Baron   jump label: Base ...
38
  		return -1;
9ae033aca   Ard Biesheuvel   jump_label: Abstr...
39
  	if (jump_entry_key(jea) > jump_entry_key(jeb))
bf5438fca   Jason Baron   jump label: Base ...
40
41
42
43
  		return 1;
  
  	return 0;
  }
50ff18ab4   Ard Biesheuvel   jump_label: Imple...
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
  static void jump_label_swap(void *a, void *b, int size)
  {
  	long delta = (unsigned long)a - (unsigned long)b;
  	struct jump_entry *jea = a;
  	struct jump_entry *jeb = b;
  	struct jump_entry tmp = *jea;
  
  	jea->code	= jeb->code - delta;
  	jea->target	= jeb->target - delta;
  	jea->key	= jeb->key - delta;
  
  	jeb->code	= tmp.code + delta;
  	jeb->target	= tmp.target + delta;
  	jeb->key	= tmp.key + delta;
  }
bf5438fca   Jason Baron   jump label: Base ...
59
  static void
d430d3d7e   Jason Baron   jump label: Intro...
60
  jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
bf5438fca   Jason Baron   jump label: Base ...
61
62
  {
  	unsigned long size;
50ff18ab4   Ard Biesheuvel   jump_label: Imple...
63
64
65
66
  	void *swapfn = NULL;
  
  	if (IS_ENABLED(CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE))
  		swapfn = jump_label_swap;
bf5438fca   Jason Baron   jump label: Base ...
67
68
69
  
  	size = (((unsigned long)stop - (unsigned long)start)
  					/ sizeof(struct jump_entry));
50ff18ab4   Ard Biesheuvel   jump_label: Imple...
70
  	sort(start, size, sizeof(struct jump_entry), jump_label_cmp, swapfn);
bf5438fca   Jason Baron   jump label: Base ...
71
  }
706249c22   Peter Zijlstra   locking/static_ke...
72
  static void jump_label_update(struct static_key *key);
a1efb01fe   Peter Zijlstra   jump_label, locki...
73

1f69bf9c6   Jason Baron   jump_label: remov...
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
  /*
   * There are similar definitions for the !HAVE_JUMP_LABEL case in jump_label.h.
   * The use of 'atomic_read()' requires atomic.h and its problematic for some
   * kernel headers such as kernel.h and others. Since static_key_count() is not
   * used in the branch statements as it is for the !HAVE_JUMP_LABEL case its ok
   * to have it be a function here. Similarly, for 'static_key_enable()' and
   * 'static_key_disable()', which require bug.h. This should allow jump_label.h
   * to be included from most/all places for HAVE_JUMP_LABEL.
   */
  int static_key_count(struct static_key *key)
  {
  	/*
  	 * -1 means the first static_key_slow_inc() is in progress.
  	 *  static_key_enabled() must return true, so return 1 here.
  	 */
  	int n = atomic_read(&key->enabled);
  
  	return n >= 0 ? n : 1;
  }
  EXPORT_SYMBOL_GPL(static_key_count);
ce48c1464   Peter Zijlstra   sched/core: Fix c...
94
  void static_key_slow_inc_cpuslocked(struct static_key *key)
bf5438fca   Jason Baron   jump label: Base ...
95
  {
4c5ea0a9c   Paolo Bonzini   locking/static_ke...
96
  	int v, v1;
5cdda5117   Borislav Petkov   locking/static_ke...
97
  	STATIC_KEY_CHECK_USE(key);
cb538267e   Peter Zijlstra   jump_label/lockde...
98
  	lockdep_assert_cpus_held();
4c5ea0a9c   Paolo Bonzini   locking/static_ke...
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
  
  	/*
  	 * Careful if we get concurrent static_key_slow_inc() calls;
  	 * later calls must wait for the first one to _finish_ the
  	 * jump_label_update() process.  At the same time, however,
  	 * the jump_label_update() call below wants to see
  	 * static_key_enabled(&key) for jumps to be updated properly.
  	 *
  	 * So give a special meaning to negative key->enabled: it sends
  	 * static_key_slow_inc() down the slow path, and it is non-zero
  	 * so it counts as "enabled" in jump_label_update().  Note that
  	 * atomic_inc_unless_negative() checks >= 0, so roll our own.
  	 */
  	for (v = atomic_read(&key->enabled); v > 0; v = v1) {
  		v1 = atomic_cmpxchg(&key->enabled, v, v + 1);
8b7b41280   Marc Zyngier   jump_label: Split...
114
  		if (likely(v1 == v))
4c5ea0a9c   Paolo Bonzini   locking/static_ke...
115
116
  			return;
  	}
bf5438fca   Jason Baron   jump label: Base ...
117

d430d3d7e   Jason Baron   jump label: Intro...
118
  	jump_label_lock();
4c5ea0a9c   Paolo Bonzini   locking/static_ke...
119
120
  	if (atomic_read(&key->enabled) == 0) {
  		atomic_set(&key->enabled, -1);
706249c22   Peter Zijlstra   locking/static_ke...
121
  		jump_label_update(key);
d0646a6f5   Peter Zijlstra   jump_label: Add R...
122
123
124
125
126
  		/*
  		 * Ensure that if the above cmpxchg loop observes our positive
  		 * value, it must also observe all the text changes.
  		 */
  		atomic_set_release(&key->enabled, 1);
4c5ea0a9c   Paolo Bonzini   locking/static_ke...
127
128
129
  	} else {
  		atomic_inc(&key->enabled);
  	}
d430d3d7e   Jason Baron   jump label: Intro...
130
  	jump_label_unlock();
8b7b41280   Marc Zyngier   jump_label: Split...
131
132
133
134
135
136
  }
  
  void static_key_slow_inc(struct static_key *key)
  {
  	cpus_read_lock();
  	static_key_slow_inc_cpuslocked(key);
f2545b2d4   Thomas Gleixner   jump_label: Reord...
137
  	cpus_read_unlock();
bf5438fca   Jason Baron   jump label: Base ...
138
  }
c5905afb0   Ingo Molnar   static keys: Intr...
139
  EXPORT_SYMBOL_GPL(static_key_slow_inc);
bf5438fca   Jason Baron   jump label: Base ...
140

5a40527f8   Marc Zyngier   jump_label: Provi...
141
  void static_key_enable_cpuslocked(struct static_key *key)
1dbb6704d   Paolo Bonzini   jump_label: Fix c...
142
  {
5cdda5117   Borislav Petkov   locking/static_ke...
143
  	STATIC_KEY_CHECK_USE(key);
cb538267e   Peter Zijlstra   jump_label/lockde...
144
  	lockdep_assert_cpus_held();
5a40527f8   Marc Zyngier   jump_label: Provi...
145

1dbb6704d   Paolo Bonzini   jump_label: Fix c...
146
147
148
149
  	if (atomic_read(&key->enabled) > 0) {
  		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
  		return;
  	}
1dbb6704d   Paolo Bonzini   jump_label: Fix c...
150
151
152
153
  	jump_label_lock();
  	if (atomic_read(&key->enabled) == 0) {
  		atomic_set(&key->enabled, -1);
  		jump_label_update(key);
d0646a6f5   Peter Zijlstra   jump_label: Add R...
154
155
156
157
  		/*
  		 * See static_key_slow_inc().
  		 */
  		atomic_set_release(&key->enabled, 1);
1dbb6704d   Paolo Bonzini   jump_label: Fix c...
158
159
  	}
  	jump_label_unlock();
5a40527f8   Marc Zyngier   jump_label: Provi...
160
161
162
163
164
165
166
  }
  EXPORT_SYMBOL_GPL(static_key_enable_cpuslocked);
  
  void static_key_enable(struct static_key *key)
  {
  	cpus_read_lock();
  	static_key_enable_cpuslocked(key);
1dbb6704d   Paolo Bonzini   jump_label: Fix c...
167
168
169
  	cpus_read_unlock();
  }
  EXPORT_SYMBOL_GPL(static_key_enable);
5a40527f8   Marc Zyngier   jump_label: Provi...
170
  void static_key_disable_cpuslocked(struct static_key *key)
1dbb6704d   Paolo Bonzini   jump_label: Fix c...
171
  {
5cdda5117   Borislav Petkov   locking/static_ke...
172
  	STATIC_KEY_CHECK_USE(key);
cb538267e   Peter Zijlstra   jump_label/lockde...
173
  	lockdep_assert_cpus_held();
5a40527f8   Marc Zyngier   jump_label: Provi...
174

1dbb6704d   Paolo Bonzini   jump_label: Fix c...
175
176
177
178
  	if (atomic_read(&key->enabled) != 1) {
  		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
  		return;
  	}
1dbb6704d   Paolo Bonzini   jump_label: Fix c...
179
180
181
182
  	jump_label_lock();
  	if (atomic_cmpxchg(&key->enabled, 1, 0))
  		jump_label_update(key);
  	jump_label_unlock();
5a40527f8   Marc Zyngier   jump_label: Provi...
183
184
185
186
187
188
189
  }
  EXPORT_SYMBOL_GPL(static_key_disable_cpuslocked);
  
  void static_key_disable(struct static_key *key)
  {
  	cpus_read_lock();
  	static_key_disable_cpuslocked(key);
1dbb6704d   Paolo Bonzini   jump_label: Fix c...
190
191
192
  	cpus_read_unlock();
  }
  EXPORT_SYMBOL_GPL(static_key_disable);
ce48c1464   Peter Zijlstra   sched/core: Fix c...
193
  static void __static_key_slow_dec_cpuslocked(struct static_key *key,
8b7b41280   Marc Zyngier   jump_label: Split...
194
195
  					   unsigned long rate_limit,
  					   struct delayed_work *work)
bf5438fca   Jason Baron   jump label: Base ...
196
  {
cb538267e   Peter Zijlstra   jump_label/lockde...
197
  	lockdep_assert_cpus_held();
4c5ea0a9c   Paolo Bonzini   locking/static_ke...
198
199
200
201
202
203
204
  	/*
  	 * The negative count check is valid even when a negative
  	 * key->enabled is in use by static_key_slow_inc(); a
  	 * __static_key_slow_dec() before the first static_key_slow_inc()
  	 * returns is unbalanced, because all other static_key_slow_inc()
  	 * instances block while the update is in progress.
  	 */
fadf0464b   Jason Baron   jump label: Add a...
205
206
207
208
  	if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex)) {
  		WARN(atomic_read(&key->enabled) < 0,
  		     "jump label: negative count!
  ");
d430d3d7e   Jason Baron   jump label: Intro...
209
  		return;
fadf0464b   Jason Baron   jump label: Add a...
210
  	}
bf5438fca   Jason Baron   jump label: Base ...
211

b20295207   Gleb Natapov   perf, core: Rate ...
212
213
214
  	if (rate_limit) {
  		atomic_inc(&key->enabled);
  		schedule_delayed_work(work, rate_limit);
c5905afb0   Ingo Molnar   static keys: Intr...
215
  	} else {
706249c22   Peter Zijlstra   locking/static_ke...
216
  		jump_label_update(key);
c5905afb0   Ingo Molnar   static keys: Intr...
217
  	}
91bad2f8d   Jason Baron   jump label: Fix d...
218
  	jump_label_unlock();
8b7b41280   Marc Zyngier   jump_label: Split...
219
220
221
222
223
224
225
  }
  
  static void __static_key_slow_dec(struct static_key *key,
  				  unsigned long rate_limit,
  				  struct delayed_work *work)
  {
  	cpus_read_lock();
ce48c1464   Peter Zijlstra   sched/core: Fix c...
226
  	__static_key_slow_dec_cpuslocked(key, rate_limit, work);
f2545b2d4   Thomas Gleixner   jump_label: Reord...
227
  	cpus_read_unlock();
bf5438fca   Jason Baron   jump label: Base ...
228
  }
b20295207   Gleb Natapov   perf, core: Rate ...
229
230
  static void jump_label_update_timeout(struct work_struct *work)
  {
c5905afb0   Ingo Molnar   static keys: Intr...
231
232
233
  	struct static_key_deferred *key =
  		container_of(work, struct static_key_deferred, work.work);
  	__static_key_slow_dec(&key->key, 0, NULL);
b20295207   Gleb Natapov   perf, core: Rate ...
234
  }
c5905afb0   Ingo Molnar   static keys: Intr...
235
  void static_key_slow_dec(struct static_key *key)
b20295207   Gleb Natapov   perf, core: Rate ...
236
  {
5cdda5117   Borislav Petkov   locking/static_ke...
237
  	STATIC_KEY_CHECK_USE(key);
c5905afb0   Ingo Molnar   static keys: Intr...
238
  	__static_key_slow_dec(key, 0, NULL);
b20295207   Gleb Natapov   perf, core: Rate ...
239
  }
c5905afb0   Ingo Molnar   static keys: Intr...
240
  EXPORT_SYMBOL_GPL(static_key_slow_dec);
b20295207   Gleb Natapov   perf, core: Rate ...
241

ce48c1464   Peter Zijlstra   sched/core: Fix c...
242
243
244
245
246
  void static_key_slow_dec_cpuslocked(struct static_key *key)
  {
  	STATIC_KEY_CHECK_USE(key);
  	__static_key_slow_dec_cpuslocked(key, 0, NULL);
  }
c5905afb0   Ingo Molnar   static keys: Intr...
247
  void static_key_slow_dec_deferred(struct static_key_deferred *key)
b20295207   Gleb Natapov   perf, core: Rate ...
248
  {
5cdda5117   Borislav Petkov   locking/static_ke...
249
  	STATIC_KEY_CHECK_USE(key);
c5905afb0   Ingo Molnar   static keys: Intr...
250
  	__static_key_slow_dec(&key->key, key->timeout, &key->work);
b20295207   Gleb Natapov   perf, core: Rate ...
251
  }
c5905afb0   Ingo Molnar   static keys: Intr...
252
  EXPORT_SYMBOL_GPL(static_key_slow_dec_deferred);
b20295207   Gleb Natapov   perf, core: Rate ...
253

b6416e610   David Matlack   jump_labels: API ...
254
255
  void static_key_deferred_flush(struct static_key_deferred *key)
  {
5cdda5117   Borislav Petkov   locking/static_ke...
256
  	STATIC_KEY_CHECK_USE(key);
b6416e610   David Matlack   jump_labels: API ...
257
258
259
  	flush_delayed_work(&key->work);
  }
  EXPORT_SYMBOL_GPL(static_key_deferred_flush);
c5905afb0   Ingo Molnar   static keys: Intr...
260
  void jump_label_rate_limit(struct static_key_deferred *key,
b20295207   Gleb Natapov   perf, core: Rate ...
261
262
  		unsigned long rl)
  {
5cdda5117   Borislav Petkov   locking/static_ke...
263
  	STATIC_KEY_CHECK_USE(key);
b20295207   Gleb Natapov   perf, core: Rate ...
264
265
266
  	key->timeout = rl;
  	INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);
  }
a181dc14e   Gleb Natapov   jump_label: Expor...
267
  EXPORT_SYMBOL_GPL(jump_label_rate_limit);
b20295207   Gleb Natapov   perf, core: Rate ...
268

4c3ef6d79   Jason Baron   jump label: Add j...
269
270
  static int addr_conflict(struct jump_entry *entry, void *start, void *end)
  {
9ae033aca   Ard Biesheuvel   jump_label: Abstr...
271
272
  	if (jump_entry_code(entry) <= (unsigned long)end &&
  	    jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE > (unsigned long)start)
4c3ef6d79   Jason Baron   jump label: Add j...
273
274
275
276
  		return 1;
  
  	return 0;
  }
d430d3d7e   Jason Baron   jump label: Intro...
277
278
  static int __jump_label_text_reserved(struct jump_entry *iter_start,
  		struct jump_entry *iter_stop, void *start, void *end)
4c3ef6d79   Jason Baron   jump label: Add j...
279
  {
4c3ef6d79   Jason Baron   jump label: Add j...
280
  	struct jump_entry *iter;
4c3ef6d79   Jason Baron   jump label: Add j...
281

4c3ef6d79   Jason Baron   jump label: Add j...
282
283
  	iter = iter_start;
  	while (iter < iter_stop) {
d430d3d7e   Jason Baron   jump label: Intro...
284
285
  		if (addr_conflict(iter, start, end))
  			return 1;
4c3ef6d79   Jason Baron   jump label: Add j...
286
287
  		iter++;
  	}
d430d3d7e   Jason Baron   jump label: Intro...
288
289
  	return 0;
  }
706249c22   Peter Zijlstra   locking/static_ke...
290
  /*
20284aa77   Jeremy Fitzhardinge   jump_label: add a...
291
292
293
294
295
   * Update code which is definitely not currently executing.
   * Architectures which need heavyweight synchronization to modify
   * running code can override this to make the non-live update case
   * cheaper.
   */
9cdbe1cba   Peter Zijlstra   jump_label, x86: ...
296
  void __weak __init_or_module arch_jump_label_transform_static(struct jump_entry *entry,
20284aa77   Jeremy Fitzhardinge   jump_label: add a...
297
298
  					    enum jump_label_type type)
  {
706249c22   Peter Zijlstra   locking/static_ke...
299
  	arch_jump_label_transform(entry, type);
20284aa77   Jeremy Fitzhardinge   jump_label: add a...
300
  }
706249c22   Peter Zijlstra   locking/static_ke...
301
  static inline struct jump_entry *static_key_entries(struct static_key *key)
d430d3d7e   Jason Baron   jump label: Intro...
302
  {
3821fd35b   Jason Baron   jump_label: Reduc...
303
304
  	WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);
  	return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);
4c3ef6d79   Jason Baron   jump label: Add j...
305
  }
706249c22   Peter Zijlstra   locking/static_ke...
306
  static inline bool static_key_type(struct static_key *key)
c5905afb0   Ingo Molnar   static keys: Intr...
307
  {
3821fd35b   Jason Baron   jump_label: Reduc...
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
  	return key->type & JUMP_TYPE_TRUE;
  }
  
  static inline bool static_key_linked(struct static_key *key)
  {
  	return key->type & JUMP_TYPE_LINKED;
  }
  
  static inline void static_key_clear_linked(struct static_key *key)
  {
  	key->type &= ~JUMP_TYPE_LINKED;
  }
  
  static inline void static_key_set_linked(struct static_key *key)
  {
  	key->type |= JUMP_TYPE_LINKED;
a1efb01fe   Peter Zijlstra   jump_label, locki...
324
  }
c5905afb0   Ingo Molnar   static keys: Intr...
325

3821fd35b   Jason Baron   jump_label: Reduc...
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
  /***
   * A 'struct static_key' uses a union such that it either points directly
   * to a table of 'struct jump_entry' or to a linked list of modules which in
   * turn point to 'struct jump_entry' tables.
   *
   * The two lower bits of the pointer are used to keep track of which pointer
   * type is in use and to store the initial branch direction, we use an access
   * function which preserves these bits.
   */
  static void static_key_set_entries(struct static_key *key,
  				   struct jump_entry *entries)
  {
  	unsigned long type;
  
  	WARN_ON_ONCE((unsigned long)entries & JUMP_TYPE_MASK);
  	type = key->type & JUMP_TYPE_MASK;
  	key->entries = entries;
  	key->type |= type;
  }
706249c22   Peter Zijlstra   locking/static_ke...
345
  static enum jump_label_type jump_label_type(struct jump_entry *entry)
a1efb01fe   Peter Zijlstra   jump_label, locki...
346
  {
706249c22   Peter Zijlstra   locking/static_ke...
347
  	struct static_key *key = jump_entry_key(entry);
a1efb01fe   Peter Zijlstra   jump_label, locki...
348
  	bool enabled = static_key_enabled(key);
9ae033aca   Ard Biesheuvel   jump_label: Abstr...
349
  	bool branch = jump_entry_is_branch(entry);
c5905afb0   Ingo Molnar   static keys: Intr...
350

11276d530   Peter Zijlstra   locking/static_ke...
351
352
  	/* See the comment in linux/jump_label.h */
  	return enabled ^ branch;
c5905afb0   Ingo Molnar   static keys: Intr...
353
  }
706249c22   Peter Zijlstra   locking/static_ke...
354
355
  static void __jump_label_update(struct static_key *key,
  				struct jump_entry *entry,
194836776   Ard Biesheuvel   jump_label: Annot...
356
357
  				struct jump_entry *stop,
  				bool init)
706249c22   Peter Zijlstra   locking/static_ke...
358
359
360
  {
  	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
  		/*
dc1dd184c   Josh Poimboeuf   jump_label: Warn ...
361
362
  		 * An entry->code of 0 indicates an entry which has been
  		 * disabled because it was in an init text area.
706249c22   Peter Zijlstra   locking/static_ke...
363
  		 */
194836776   Ard Biesheuvel   jump_label: Annot...
364
  		if (init || !jump_entry_is_init(entry)) {
9ae033aca   Ard Biesheuvel   jump_label: Abstr...
365
  			if (kernel_text_address(jump_entry_code(entry)))
dc1dd184c   Josh Poimboeuf   jump_label: Warn ...
366
367
  				arch_jump_label_transform(entry, jump_label_type(entry));
  			else
af1d830bf   Josh Poimboeuf   jump_label: Fix s...
368
  				WARN_ONCE(1, "can't patch jump_label at %pS",
9ae033aca   Ard Biesheuvel   jump_label: Abstr...
369
  					  (void *)jump_entry_code(entry));
dc1dd184c   Josh Poimboeuf   jump_label: Warn ...
370
  		}
706249c22   Peter Zijlstra   locking/static_ke...
371
372
  	}
  }
97ce2c88f   Jeremy Fitzhardinge   jump-label: initi...
373
  void __init jump_label_init(void)
bf5438fca   Jason Baron   jump label: Base ...
374
  {
bf5438fca   Jason Baron   jump label: Base ...
375
376
  	struct jump_entry *iter_start = __start___jump_table;
  	struct jump_entry *iter_stop = __stop___jump_table;
c5905afb0   Ingo Molnar   static keys: Intr...
377
  	struct static_key *key = NULL;
bf5438fca   Jason Baron   jump label: Base ...
378
  	struct jump_entry *iter;
1f69bf9c6   Jason Baron   jump_label: remov...
379
380
381
382
383
384
385
386
  	/*
  	 * Since we are initializing the static_key.enabled field with
  	 * with the 'raw' int values (to avoid pulling in atomic.h) in
  	 * jump_label.h, let's make sure that is safe. There are only two
  	 * cases to check since we initialize to 0 or 1.
  	 */
  	BUILD_BUG_ON((int)ATOMIC_INIT(0) != 0);
  	BUILD_BUG_ON((int)ATOMIC_INIT(1) != 1);
e3f91083f   Kevin Hao   jump_label: Make ...
387
388
  	if (static_key_initialized)
  		return;
f2545b2d4   Thomas Gleixner   jump_label: Reord...
389
  	cpus_read_lock();
91bad2f8d   Jason Baron   jump label: Fix d...
390
  	jump_label_lock();
d430d3d7e   Jason Baron   jump label: Intro...
391
392
393
  	jump_label_sort_entries(iter_start, iter_stop);
  
  	for (iter = iter_start; iter < iter_stop; iter++) {
c5905afb0   Ingo Molnar   static keys: Intr...
394
  		struct static_key *iterk;
37348804e   Jeremy Fitzhardinge   jump_label: if a ...
395

11276d530   Peter Zijlstra   locking/static_ke...
396
397
398
  		/* rewrite NOPs */
  		if (jump_label_type(iter) == JUMP_LABEL_NOP)
  			arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
194836776   Ard Biesheuvel   jump_label: Annot...
399
400
  		if (init_section_contains((void *)jump_entry_code(iter), 1))
  			jump_entry_set_init(iter);
7dcfd915b   Peter Zijlstra   jump_label: Add j...
401
  		iterk = jump_entry_key(iter);
37348804e   Jeremy Fitzhardinge   jump_label: if a ...
402
  		if (iterk == key)
d430d3d7e   Jason Baron   jump label: Intro...
403
  			continue;
37348804e   Jeremy Fitzhardinge   jump_label: if a ...
404
  		key = iterk;
3821fd35b   Jason Baron   jump_label: Reduc...
405
  		static_key_set_entries(key, iter);
bf5438fca   Jason Baron   jump label: Base ...
406
  	}
c4b2c0c5f   Hannes Frederic Sowa   static_key: WARN ...
407
  	static_key_initialized = true;
91bad2f8d   Jason Baron   jump label: Fix d...
408
  	jump_label_unlock();
f2545b2d4   Thomas Gleixner   jump_label: Reord...
409
  	cpus_read_unlock();
bf5438fca   Jason Baron   jump label: Base ...
410
  }
bf5438fca   Jason Baron   jump label: Base ...
411
412
  
  #ifdef CONFIG_MODULES
11276d530   Peter Zijlstra   locking/static_ke...
413
414
415
416
  static enum jump_label_type jump_label_init_type(struct jump_entry *entry)
  {
  	struct static_key *key = jump_entry_key(entry);
  	bool type = static_key_type(key);
9ae033aca   Ard Biesheuvel   jump_label: Abstr...
417
  	bool branch = jump_entry_is_branch(entry);
11276d530   Peter Zijlstra   locking/static_ke...
418
419
420
421
  
  	/* See the comment in linux/jump_label.h */
  	return type ^ branch;
  }
c5905afb0   Ingo Molnar   static keys: Intr...
422
423
  struct static_key_mod {
  	struct static_key_mod *next;
d430d3d7e   Jason Baron   jump label: Intro...
424
425
426
  	struct jump_entry *entries;
  	struct module *mod;
  };
3821fd35b   Jason Baron   jump_label: Reduc...
427
428
  static inline struct static_key_mod *static_key_mod(struct static_key *key)
  {
34e12b864   Borislav Petkov   jump_label: Use s...
429
  	WARN_ON_ONCE(!static_key_linked(key));
3821fd35b   Jason Baron   jump_label: Reduc...
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
  	return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);
  }
  
  /***
   * key->type and key->next are the same via union.
   * This sets key->next and preserves the type bits.
   *
   * See additional comments above static_key_set_entries().
   */
  static void static_key_set_mod(struct static_key *key,
  			       struct static_key_mod *mod)
  {
  	unsigned long type;
  
  	WARN_ON_ONCE((unsigned long)mod & JUMP_TYPE_MASK);
  	type = key->type & JUMP_TYPE_MASK;
  	key->next = mod;
  	key->type |= type;
  }
d430d3d7e   Jason Baron   jump label: Intro...
449
450
451
  static int __jump_label_mod_text_reserved(void *start, void *end)
  {
  	struct module *mod;
bdc9f3735   Rusty Russell   jump_label: disab...
452
  	preempt_disable();
d430d3d7e   Jason Baron   jump label: Intro...
453
  	mod = __module_text_address((unsigned long)start);
bdc9f3735   Rusty Russell   jump_label: disab...
454
455
  	WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
  	preempt_enable();
d430d3d7e   Jason Baron   jump label: Intro...
456
457
  	if (!mod)
  		return 0;
d430d3d7e   Jason Baron   jump label: Intro...
458
459
460
461
462
  
  	return __jump_label_text_reserved(mod->jump_entries,
  				mod->jump_entries + mod->num_jump_entries,
  				start, end);
  }
706249c22   Peter Zijlstra   locking/static_ke...
463
  static void __jump_label_mod_update(struct static_key *key)
d430d3d7e   Jason Baron   jump label: Intro...
464
  {
706249c22   Peter Zijlstra   locking/static_ke...
465
  	struct static_key_mod *mod;
d430d3d7e   Jason Baron   jump label: Intro...
466

3821fd35b   Jason Baron   jump_label: Reduc...
467
468
469
470
471
472
473
474
475
476
  	for (mod = static_key_mod(key); mod; mod = mod->next) {
  		struct jump_entry *stop;
  		struct module *m;
  
  		/*
  		 * NULL if the static_key is defined in a module
  		 * that does not use it
  		 */
  		if (!mod->entries)
  			continue;
7cbc5b8d4   Jiri Olsa   jump_label: Check...
477

3821fd35b   Jason Baron   jump_label: Reduc...
478
479
480
481
482
  		m = mod->mod;
  		if (!m)
  			stop = __stop___jump_table;
  		else
  			stop = m->jump_entries + m->num_jump_entries;
194836776   Ard Biesheuvel   jump_label: Annot...
483
  		__jump_label_update(key, mod->entries, stop,
77ac1c02d   Ard Biesheuvel   jump_label: Fix N...
484
  				    m && m->state == MODULE_STATE_COMING);
d430d3d7e   Jason Baron   jump label: Intro...
485
486
487
488
489
490
491
492
493
494
495
496
  	}
  }
  
  /***
   * apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
   * @mod: module to patch
   *
   * Allow for run-time selection of the optimal nops. Before the module
   * loads patch these with arch_get_jump_label_nop(), which is specified by
   * the arch specific jump label code.
   */
  void jump_label_apply_nops(struct module *mod)
bf5438fca   Jason Baron   jump label: Base ...
497
  {
d430d3d7e   Jason Baron   jump label: Intro...
498
499
500
501
502
503
504
  	struct jump_entry *iter_start = mod->jump_entries;
  	struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
  	struct jump_entry *iter;
  
  	/* if the module doesn't have jump label entries, just return */
  	if (iter_start == iter_stop)
  		return;
11276d530   Peter Zijlstra   locking/static_ke...
505
506
507
508
509
  	for (iter = iter_start; iter < iter_stop; iter++) {
  		/* Only write NOPs for arch_branch_static(). */
  		if (jump_label_init_type(iter) == JUMP_LABEL_NOP)
  			arch_jump_label_transform_static(iter, JUMP_LABEL_NOP);
  	}
bf5438fca   Jason Baron   jump label: Base ...
510
  }
d430d3d7e   Jason Baron   jump label: Intro...
511
  static int jump_label_add_module(struct module *mod)
bf5438fca   Jason Baron   jump label: Base ...
512
  {
d430d3d7e   Jason Baron   jump label: Intro...
513
514
515
  	struct jump_entry *iter_start = mod->jump_entries;
  	struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
  	struct jump_entry *iter;
c5905afb0   Ingo Molnar   static keys: Intr...
516
  	struct static_key *key = NULL;
3821fd35b   Jason Baron   jump_label: Reduc...
517
  	struct static_key_mod *jlm, *jlm2;
bf5438fca   Jason Baron   jump label: Base ...
518
519
  
  	/* if the module doesn't have jump label entries, just return */
d430d3d7e   Jason Baron   jump label: Intro...
520
  	if (iter_start == iter_stop)
bf5438fca   Jason Baron   jump label: Base ...
521
  		return 0;
d430d3d7e   Jason Baron   jump label: Intro...
522
523
524
  	jump_label_sort_entries(iter_start, iter_stop);
  
  	for (iter = iter_start; iter < iter_stop; iter++) {
c5905afb0   Ingo Molnar   static keys: Intr...
525
  		struct static_key *iterk;
d430d3d7e   Jason Baron   jump label: Intro...
526

194836776   Ard Biesheuvel   jump_label: Annot...
527
528
  		if (within_module_init(jump_entry_code(iter), mod))
  			jump_entry_set_init(iter);
7dcfd915b   Peter Zijlstra   jump_label: Add j...
529
  		iterk = jump_entry_key(iter);
c5905afb0   Ingo Molnar   static keys: Intr...
530
531
  		if (iterk == key)
  			continue;
d430d3d7e   Jason Baron   jump label: Intro...
532

c5905afb0   Ingo Molnar   static keys: Intr...
533
  		key = iterk;
9ae033aca   Ard Biesheuvel   jump_label: Abstr...
534
  		if (within_module((unsigned long)key, mod)) {
3821fd35b   Jason Baron   jump_label: Reduc...
535
  			static_key_set_entries(key, iter);
d430d3d7e   Jason Baron   jump label: Intro...
536
  			continue;
bf5438fca   Jason Baron   jump label: Base ...
537
  		}
c5905afb0   Ingo Molnar   static keys: Intr...
538
  		jlm = kzalloc(sizeof(struct static_key_mod), GFP_KERNEL);
d430d3d7e   Jason Baron   jump label: Intro...
539
540
  		if (!jlm)
  			return -ENOMEM;
3821fd35b   Jason Baron   jump_label: Reduc...
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
  		if (!static_key_linked(key)) {
  			jlm2 = kzalloc(sizeof(struct static_key_mod),
  				       GFP_KERNEL);
  			if (!jlm2) {
  				kfree(jlm);
  				return -ENOMEM;
  			}
  			preempt_disable();
  			jlm2->mod = __module_address((unsigned long)key);
  			preempt_enable();
  			jlm2->entries = static_key_entries(key);
  			jlm2->next = NULL;
  			static_key_set_mod(key, jlm2);
  			static_key_set_linked(key);
  		}
d430d3d7e   Jason Baron   jump label: Intro...
556
557
  		jlm->mod = mod;
  		jlm->entries = iter;
3821fd35b   Jason Baron   jump_label: Reduc...
558
559
560
  		jlm->next = static_key_mod(key);
  		static_key_set_mod(key, jlm);
  		static_key_set_linked(key);
d430d3d7e   Jason Baron   jump label: Intro...
561

11276d530   Peter Zijlstra   locking/static_ke...
562
563
  		/* Only update if we've changed from our initial state */
  		if (jump_label_type(iter) != jump_label_init_type(iter))
194836776   Ard Biesheuvel   jump_label: Annot...
564
  			__jump_label_update(key, iter, iter_stop, true);
bf5438fca   Jason Baron   jump label: Base ...
565
  	}
d430d3d7e   Jason Baron   jump label: Intro...
566

bf5438fca   Jason Baron   jump label: Base ...
567
568
  	return 0;
  }
d430d3d7e   Jason Baron   jump label: Intro...
569
  static void jump_label_del_module(struct module *mod)
bf5438fca   Jason Baron   jump label: Base ...
570
  {
d430d3d7e   Jason Baron   jump label: Intro...
571
572
573
  	struct jump_entry *iter_start = mod->jump_entries;
  	struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
  	struct jump_entry *iter;
c5905afb0   Ingo Molnar   static keys: Intr...
574
575
  	struct static_key *key = NULL;
  	struct static_key_mod *jlm, **prev;
bf5438fca   Jason Baron   jump label: Base ...
576

d430d3d7e   Jason Baron   jump label: Intro...
577
  	for (iter = iter_start; iter < iter_stop; iter++) {
7dcfd915b   Peter Zijlstra   jump_label: Add j...
578
  		if (jump_entry_key(iter) == key)
d430d3d7e   Jason Baron   jump label: Intro...
579
  			continue;
7dcfd915b   Peter Zijlstra   jump_label: Add j...
580
  		key = jump_entry_key(iter);
d430d3d7e   Jason Baron   jump label: Intro...
581

9ae033aca   Ard Biesheuvel   jump_label: Abstr...
582
  		if (within_module((unsigned long)key, mod))
d430d3d7e   Jason Baron   jump label: Intro...
583
  			continue;
3821fd35b   Jason Baron   jump_label: Reduc...
584
585
586
  		/* No memory during module load */
  		if (WARN_ON(!static_key_linked(key)))
  			continue;
d430d3d7e   Jason Baron   jump label: Intro...
587
  		prev = &key->next;
3821fd35b   Jason Baron   jump_label: Reduc...
588
  		jlm = static_key_mod(key);
bf5438fca   Jason Baron   jump label: Base ...
589

d430d3d7e   Jason Baron   jump label: Intro...
590
591
592
593
  		while (jlm && jlm->mod != mod) {
  			prev = &jlm->next;
  			jlm = jlm->next;
  		}
3821fd35b   Jason Baron   jump_label: Reduc...
594
595
596
597
598
599
600
  		/* No memory during module load */
  		if (WARN_ON(!jlm))
  			continue;
  
  		if (prev == &key->next)
  			static_key_set_mod(key, jlm->next);
  		else
d430d3d7e   Jason Baron   jump label: Intro...
601
  			*prev = jlm->next;
3821fd35b   Jason Baron   jump_label: Reduc...
602
603
604
605
606
607
608
609
  
  		kfree(jlm);
  
  		jlm = static_key_mod(key);
  		/* if only one etry is left, fold it back into the static_key */
  		if (jlm->next == NULL) {
  			static_key_set_entries(key, jlm->entries);
  			static_key_clear_linked(key);
d430d3d7e   Jason Baron   jump label: Intro...
610
  			kfree(jlm);
bf5438fca   Jason Baron   jump label: Base ...
611
612
613
614
615
616
617
618
619
620
  		}
  	}
  }
  
  static int
  jump_label_module_notify(struct notifier_block *self, unsigned long val,
  			 void *data)
  {
  	struct module *mod = data;
  	int ret = 0;
f2545b2d4   Thomas Gleixner   jump_label: Reord...
621
622
  	cpus_read_lock();
  	jump_label_lock();
bf5438fca   Jason Baron   jump label: Base ...
623
624
  	switch (val) {
  	case MODULE_STATE_COMING:
d430d3d7e   Jason Baron   jump label: Intro...
625
  		ret = jump_label_add_module(mod);
3821fd35b   Jason Baron   jump_label: Reduc...
626
  		if (ret) {
da260fe12   Borislav Petkov   jump_label: Fix t...
627
628
  			WARN(1, "Failed to allocate memory: jump_label may not work properly.
  ");
d430d3d7e   Jason Baron   jump label: Intro...
629
  			jump_label_del_module(mod);
3821fd35b   Jason Baron   jump_label: Reduc...
630
  		}
bf5438fca   Jason Baron   jump label: Base ...
631
632
  		break;
  	case MODULE_STATE_GOING:
d430d3d7e   Jason Baron   jump label: Intro...
633
  		jump_label_del_module(mod);
bf5438fca   Jason Baron   jump label: Base ...
634
635
  		break;
  	}
bf5438fca   Jason Baron   jump label: Base ...
636

f2545b2d4   Thomas Gleixner   jump_label: Reord...
637
638
  	jump_label_unlock();
  	cpus_read_unlock();
d430d3d7e   Jason Baron   jump label: Intro...
639
  	return notifier_from_errno(ret);
bf5438fca   Jason Baron   jump label: Base ...
640
  }
885885f6b   Wei Yongjun   locking/static_ke...
641
  static struct notifier_block jump_label_module_nb = {
bf5438fca   Jason Baron   jump label: Base ...
642
  	.notifier_call = jump_label_module_notify,
d430d3d7e   Jason Baron   jump label: Intro...
643
  	.priority = 1, /* higher than tracepoints */
bf5438fca   Jason Baron   jump label: Base ...
644
  };
d430d3d7e   Jason Baron   jump label: Intro...
645
  static __init int jump_label_init_module(void)
bf5438fca   Jason Baron   jump label: Base ...
646
647
648
  {
  	return register_module_notifier(&jump_label_module_nb);
  }
d430d3d7e   Jason Baron   jump label: Intro...
649
  early_initcall(jump_label_init_module);
bf5438fca   Jason Baron   jump label: Base ...
650
651
  
  #endif /* CONFIG_MODULES */
d430d3d7e   Jason Baron   jump label: Intro...
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
  /***
   * jump_label_text_reserved - check if addr range is reserved
   * @start: start text addr
   * @end: end text addr
   *
   * checks if the text addr located between @start and @end
   * overlaps with any of the jump label patch addresses. Code
   * that wants to modify kernel text should first verify that
   * it does not overlap with any of the jump label addresses.
   * Caller must hold jump_label_mutex.
   *
   * returns 1 if there is an overlap, 0 otherwise
   */
  int jump_label_text_reserved(void *start, void *end)
  {
  	int ret = __jump_label_text_reserved(__start___jump_table,
  			__stop___jump_table, start, end);
  
  	if (ret)
  		return ret;
  
  #ifdef CONFIG_MODULES
  	ret = __jump_label_mod_text_reserved(start, end);
  #endif
  	return ret;
  }
706249c22   Peter Zijlstra   locking/static_ke...
678
  static void jump_label_update(struct static_key *key)
d430d3d7e   Jason Baron   jump label: Intro...
679
  {
c5905afb0   Ingo Molnar   static keys: Intr...
680
  	struct jump_entry *stop = __stop___jump_table;
3821fd35b   Jason Baron   jump_label: Reduc...
681
  	struct jump_entry *entry;
d430d3d7e   Jason Baron   jump label: Intro...
682
  #ifdef CONFIG_MODULES
bed831f9a   Peter Zijlstra   module, jump_labe...
683
  	struct module *mod;
140fe3b1a   Xiao Guangrong   jump_label: Fix j...
684

3821fd35b   Jason Baron   jump_label: Reduc...
685
686
687
688
  	if (static_key_linked(key)) {
  		__jump_label_mod_update(key);
  		return;
  	}
140fe3b1a   Xiao Guangrong   jump_label: Fix j...
689

bed831f9a   Peter Zijlstra   module, jump_labe...
690
691
  	preempt_disable();
  	mod = __module_address((unsigned long)key);
140fe3b1a   Xiao Guangrong   jump_label: Fix j...
692
693
  	if (mod)
  		stop = mod->jump_entries + mod->num_jump_entries;
bed831f9a   Peter Zijlstra   module, jump_labe...
694
  	preempt_enable();
d430d3d7e   Jason Baron   jump label: Intro...
695
  #endif
3821fd35b   Jason Baron   jump_label: Reduc...
696
  	entry = static_key_entries(key);
140fe3b1a   Xiao Guangrong   jump_label: Fix j...
697
698
  	/* if there are no users, entry can be NULL */
  	if (entry)
194836776   Ard Biesheuvel   jump_label: Annot...
699
700
  		__jump_label_update(key, entry, stop,
  				    system_state < SYSTEM_RUNNING);
d430d3d7e   Jason Baron   jump label: Intro...
701
  }
1987c947d   Peter Zijlstra   locking/static_ke...
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
  #ifdef CONFIG_STATIC_KEYS_SELFTEST
  static DEFINE_STATIC_KEY_TRUE(sk_true);
  static DEFINE_STATIC_KEY_FALSE(sk_false);
  
  static __init int jump_label_test(void)
  {
  	int i;
  
  	for (i = 0; i < 2; i++) {
  		WARN_ON(static_key_enabled(&sk_true.key) != true);
  		WARN_ON(static_key_enabled(&sk_false.key) != false);
  
  		WARN_ON(!static_branch_likely(&sk_true));
  		WARN_ON(!static_branch_unlikely(&sk_true));
  		WARN_ON(static_branch_likely(&sk_false));
  		WARN_ON(static_branch_unlikely(&sk_false));
  
  		static_branch_disable(&sk_true);
  		static_branch_enable(&sk_false);
  
  		WARN_ON(static_key_enabled(&sk_true.key) == true);
  		WARN_ON(static_key_enabled(&sk_false.key) == false);
  
  		WARN_ON(static_branch_likely(&sk_true));
  		WARN_ON(static_branch_unlikely(&sk_true));
  		WARN_ON(!static_branch_likely(&sk_false));
  		WARN_ON(!static_branch_unlikely(&sk_false));
  
  		static_branch_enable(&sk_true);
  		static_branch_disable(&sk_false);
  	}
  
  	return 0;
  }
92ee46efe   Jason Baron   jump_label: Invok...
736
  early_initcall(jump_label_test);
1987c947d   Peter Zijlstra   locking/static_ke...
737
738
739
  #endif /* STATIC_KEYS_SELFTEST */
  
  #endif /* HAVE_JUMP_LABEL */