Blame view
kernel/notifier.c
16.3 KB
457c89965 treewide: Add SPD... |
1 |
// SPDX-License-Identifier: GPL-2.0-only |
fe9d4f576 Add kernel/notifi... |
2 3 |
#include <linux/kdebug.h> #include <linux/kprobes.h> |
9984de1a5 kernel: Map most ... |
4 |
#include <linux/export.h> |
fe9d4f576 Add kernel/notifi... |
5 6 7 |
#include <linux/notifier.h> #include <linux/rcupdate.h> #include <linux/vmalloc.h> |
c166f23cb kernel/notifier.c... |
8 |
#include <linux/reboot.h> |
fe9d4f576 Add kernel/notifi... |
9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 |
/* * Notifier list for kernel code which wants to be called * at shutdown. This is used to stop any idling DMA operations * and the like. */ BLOCKING_NOTIFIER_HEAD(reboot_notifier_list); /* * Notifier chain core routines. The exported routines below * are layered on top of these, with appropriate locking added. */ static int notifier_chain_register(struct notifier_block **nl, struct notifier_block *n) { while ((*nl) != NULL) { |
1a50cb80f kernel/notifier.c... |
26 27 28 29 |
if (unlikely((*nl) == n)) { WARN(1, "double register detected"); return 0; } |
fe9d4f576 Add kernel/notifi... |
30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
if (n->priority > (*nl)->priority) break; nl = &((*nl)->next); } n->next = *nl; rcu_assign_pointer(*nl, n); return 0; } static int notifier_chain_unregister(struct notifier_block **nl, struct notifier_block *n) { while ((*nl) != NULL) { if ((*nl) == n) { rcu_assign_pointer(*nl, n->next); return 0; } nl = &((*nl)->next); } return -ENOENT; } /** * notifier_call_chain - Informs the registered notifiers about an event. * @nl: Pointer to head of the blocking notifier chain * @val: Value passed unmodified to notifier function * @v: Pointer passed unmodified to notifier function * @nr_to_call: Number of notifier functions to be called. Don't care * value of this parameter is -1. * @nr_calls: Records the number of notifications sent. Don't care * value of this field is NULL. * @returns: notifier_call_chain returns the value returned by the * last notifier function called. */ |
b40a2cb6e kprobes, notifier... |
64 65 66 |
static int notifier_call_chain(struct notifier_block **nl, unsigned long val, void *v, int nr_to_call, int *nr_calls) |
fe9d4f576 Add kernel/notifi... |
67 68 69 |
{ int ret = NOTIFY_DONE; struct notifier_block *nb, *next_nb; |
d11c563dd sched: Use lockde... |
70 |
nb = rcu_dereference_raw(*nl); |
fe9d4f576 Add kernel/notifi... |
71 72 |
while (nb && nr_to_call) { |
d11c563dd sched: Use lockde... |
73 |
next_nb = rcu_dereference_raw(nb->next); |
1b2439dbb debug: add notifi... |
74 75 |
#ifdef CONFIG_DEBUG_NOTIFIERS |
ab7476cf7 debug: add notifi... |
76 |
if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) { |
1b2439dbb debug: add notifi... |
77 78 79 80 81 |
WARN(1, "Invalid notifier called!"); nb = next_nb; continue; } #endif |
fe9d4f576 Add kernel/notifi... |
82 83 84 85 |
ret = nb->notifier_call(nb, val, v); if (nr_calls) (*nr_calls)++; |
3e6daded1 kernel/notifier.c... |
86 |
if (ret & NOTIFY_STOP_MASK) |
fe9d4f576 Add kernel/notifi... |
87 88 89 90 91 92 |
break; nb = next_nb; nr_to_call--; } return ret; } |
b40a2cb6e kprobes, notifier... |
93 |
NOKPROBE_SYMBOL(notifier_call_chain); |
fe9d4f576 Add kernel/notifi... |
94 |
|
70d932985 notifier: Fix bro... |
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 |
/** * notifier_call_chain_robust - Inform the registered notifiers about an event * and rollback on error. * @nl: Pointer to head of the blocking notifier chain * @val_up: Value passed unmodified to the notifier function * @val_down: Value passed unmodified to the notifier function when recovering * from an error on @val_up * @v Pointer passed unmodified to the notifier function * * NOTE: It is important the @nl chain doesn't change between the two * invocations of notifier_call_chain() such that we visit the * exact same notifier callbacks; this rules out any RCU usage. * * Returns: the return value of the @val_up call. */ static int notifier_call_chain_robust(struct notifier_block **nl, unsigned long val_up, unsigned long val_down, void *v) { int ret, nr = 0; ret = notifier_call_chain(nl, val_up, v, -1, &nr); if (ret & NOTIFY_STOP_MASK) notifier_call_chain(nl, val_down, v, nr-1, NULL); return ret; } |
fe9d4f576 Add kernel/notifi... |
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 |
/* * Atomic notifier chain routines. Registration and unregistration * use a spinlock, and call_chain is synchronized by RCU (no locks). */ /** * atomic_notifier_chain_register - Add notifier to an atomic notifier chain * @nh: Pointer to head of the atomic notifier chain * @n: New entry in notifier chain * * Adds a notifier to an atomic notifier chain. * * Currently always returns zero. */ int atomic_notifier_chain_register(struct atomic_notifier_head *nh, struct notifier_block *n) { unsigned long flags; int ret; spin_lock_irqsave(&nh->lock, flags); ret = notifier_chain_register(&nh->head, n); spin_unlock_irqrestore(&nh->lock, flags); return ret; } EXPORT_SYMBOL_GPL(atomic_notifier_chain_register); /** * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain * @nh: Pointer to head of the atomic notifier chain * @n: Entry to remove from notifier chain * * Removes a notifier from an atomic notifier chain. * * Returns zero on success or %-ENOENT on failure. */ int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, struct notifier_block *n) { unsigned long flags; int ret; spin_lock_irqsave(&nh->lock, flags); ret = notifier_chain_unregister(&nh->head, n); spin_unlock_irqrestore(&nh->lock, flags); synchronize_rcu(); return ret; } EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister); |
70d932985 notifier: Fix bro... |
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 |
int atomic_notifier_call_chain_robust(struct atomic_notifier_head *nh, unsigned long val_up, unsigned long val_down, void *v) { unsigned long flags; int ret; /* * Musn't use RCU; because then the notifier list can * change between the up and down traversal. */ spin_lock_irqsave(&nh->lock, flags); ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v); spin_unlock_irqrestore(&nh->lock, flags); return ret; } EXPORT_SYMBOL_GPL(atomic_notifier_call_chain_robust); NOKPROBE_SYMBOL(atomic_notifier_call_chain_robust); |
fe9d4f576 Add kernel/notifi... |
189 |
/** |
70d932985 notifier: Fix bro... |
190 |
* atomic_notifier_call_chain - Call functions in an atomic notifier chain |
fe9d4f576 Add kernel/notifi... |
191 192 193 |
* @nh: Pointer to head of the atomic notifier chain * @val: Value passed unmodified to notifier function * @v: Pointer passed unmodified to notifier function |
fe9d4f576 Add kernel/notifi... |
194 195 196 197 198 199 200 201 202 203 204 205 |
* * Calls each function in a notifier chain in turn. The functions * run in an atomic context, so they must not block. * This routine uses RCU to synchronize with changes to the chain. * * If the return value of the notifier can be and'ed * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain() * will return immediately, with the return value of * the notifier function which halted execution. * Otherwise the return value is the return value * of the last notifier function called. */ |
70d932985 notifier: Fix bro... |
206 207 |
int atomic_notifier_call_chain(struct atomic_notifier_head *nh, unsigned long val, void *v) |
fe9d4f576 Add kernel/notifi... |
208 209 210 211 |
{ int ret; rcu_read_lock(); |
70d932985 notifier: Fix bro... |
212 |
ret = notifier_call_chain(&nh->head, val, v, -1, NULL); |
fe9d4f576 Add kernel/notifi... |
213 |
rcu_read_unlock(); |
fe9d4f576 Add kernel/notifi... |
214 |
|
70d932985 notifier: Fix bro... |
215 |
return ret; |
fe9d4f576 Add kernel/notifi... |
216 217 |
} EXPORT_SYMBOL_GPL(atomic_notifier_call_chain); |
b40a2cb6e kprobes, notifier... |
218 |
NOKPROBE_SYMBOL(atomic_notifier_call_chain); |
fe9d4f576 Add kernel/notifi... |
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 |
/* * Blocking notifier chain routines. All access to the chain is * synchronized by an rwsem. */ /** * blocking_notifier_chain_register - Add notifier to a blocking notifier chain * @nh: Pointer to head of the blocking notifier chain * @n: New entry in notifier chain * * Adds a notifier to a blocking notifier chain. * Must be called in process context. * * Currently always returns zero. */ int blocking_notifier_chain_register(struct blocking_notifier_head *nh, struct notifier_block *n) { int ret; /* * This code gets used during boot-up, when task switching is * not yet working and interrupts must remain disabled. At * such times we must not call down_write(). */ if (unlikely(system_state == SYSTEM_BOOTING)) return notifier_chain_register(&nh->head, n); down_write(&nh->rwsem); ret = notifier_chain_register(&nh->head, n); up_write(&nh->rwsem); return ret; } EXPORT_SYMBOL_GPL(blocking_notifier_chain_register); /** * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain * @nh: Pointer to head of the blocking notifier chain * @n: Entry to remove from notifier chain * * Removes a notifier from a blocking notifier chain. * Must be called from process context. * * Returns zero on success or %-ENOENT on failure. */ int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, struct notifier_block *n) { int ret; /* * This code gets used during boot-up, when task switching is * not yet working and interrupts must remain disabled. At * such times we must not call down_write(). */ if (unlikely(system_state == SYSTEM_BOOTING)) return notifier_chain_unregister(&nh->head, n); down_write(&nh->rwsem); ret = notifier_chain_unregister(&nh->head, n); up_write(&nh->rwsem); return ret; } EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister); |
70d932985 notifier: Fix bro... |
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 |
int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh, unsigned long val_up, unsigned long val_down, void *v) { int ret = NOTIFY_DONE; /* * We check the head outside the lock, but if this access is * racy then it does not matter what the result of the test * is, we re-check the list after having taken the lock anyway: */ if (rcu_access_pointer(nh->head)) { down_read(&nh->rwsem); ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v); up_read(&nh->rwsem); } return ret; } EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust); |
fe9d4f576 Add kernel/notifi... |
302 |
/** |
70d932985 notifier: Fix bro... |
303 |
* blocking_notifier_call_chain - Call functions in a blocking notifier chain |
fe9d4f576 Add kernel/notifi... |
304 305 306 |
* @nh: Pointer to head of the blocking notifier chain * @val: Value passed unmodified to notifier function * @v: Pointer passed unmodified to notifier function |
fe9d4f576 Add kernel/notifi... |
307 308 309 310 311 312 313 314 315 316 317 |
* * Calls each function in a notifier chain in turn. The functions * run in a process context, so they are allowed to block. * * If the return value of the notifier can be and'ed * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain() * will return immediately, with the return value of * the notifier function which halted execution. * Otherwise the return value is the return value * of the last notifier function called. */ |
70d932985 notifier: Fix bro... |
318 319 |
int blocking_notifier_call_chain(struct blocking_notifier_head *nh, unsigned long val, void *v) |
fe9d4f576 Add kernel/notifi... |
320 321 322 323 324 325 326 327 |
{ int ret = NOTIFY_DONE; /* * We check the head outside the lock, but if this access is * racy then it does not matter what the result of the test * is, we re-check the list after having taken the lock anyway: */ |
8857563b8 notifier: Substit... |
328 |
if (rcu_access_pointer(nh->head)) { |
fe9d4f576 Add kernel/notifi... |
329 |
down_read(&nh->rwsem); |
70d932985 notifier: Fix bro... |
330 |
ret = notifier_call_chain(&nh->head, val, v, -1, NULL); |
fe9d4f576 Add kernel/notifi... |
331 332 333 334 |
up_read(&nh->rwsem); } return ret; } |
fe9d4f576 Add kernel/notifi... |
335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 |
EXPORT_SYMBOL_GPL(blocking_notifier_call_chain); /* * Raw notifier chain routines. There is no protection; * the caller must provide it. Use at your own risk! */ /** * raw_notifier_chain_register - Add notifier to a raw notifier chain * @nh: Pointer to head of the raw notifier chain * @n: New entry in notifier chain * * Adds a notifier to a raw notifier chain. * All locking must be provided by the caller. * * Currently always returns zero. */ int raw_notifier_chain_register(struct raw_notifier_head *nh, struct notifier_block *n) { return notifier_chain_register(&nh->head, n); } EXPORT_SYMBOL_GPL(raw_notifier_chain_register); /** * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain * @nh: Pointer to head of the raw notifier chain * @n: Entry to remove from notifier chain * * Removes a notifier from a raw notifier chain. * All locking must be provided by the caller. * * Returns zero on success or %-ENOENT on failure. */ int raw_notifier_chain_unregister(struct raw_notifier_head *nh, struct notifier_block *n) { return notifier_chain_unregister(&nh->head, n); } EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister); |
70d932985 notifier: Fix bro... |
375 376 377 378 379 380 |
int raw_notifier_call_chain_robust(struct raw_notifier_head *nh, unsigned long val_up, unsigned long val_down, void *v) { return notifier_call_chain_robust(&nh->head, val_up, val_down, v); } EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust); |
fe9d4f576 Add kernel/notifi... |
381 |
/** |
70d932985 notifier: Fix bro... |
382 |
* raw_notifier_call_chain - Call functions in a raw notifier chain |
fe9d4f576 Add kernel/notifi... |
383 384 385 |
* @nh: Pointer to head of the raw notifier chain * @val: Value passed unmodified to notifier function * @v: Pointer passed unmodified to notifier function |
fe9d4f576 Add kernel/notifi... |
386 387 388 389 390 391 392 393 394 395 396 397 |
* * Calls each function in a notifier chain in turn. The functions * run in an undefined context. * All locking must be provided by the caller. * * If the return value of the notifier can be and'ed * with %NOTIFY_STOP_MASK then raw_notifier_call_chain() * will return immediately, with the return value of * the notifier function which halted execution. * Otherwise the return value is the return value * of the last notifier function called. */ |
fe9d4f576 Add kernel/notifi... |
398 399 400 |
int raw_notifier_call_chain(struct raw_notifier_head *nh, unsigned long val, void *v) { |
70d932985 notifier: Fix bro... |
401 |
return notifier_call_chain(&nh->head, val, v, -1, NULL); |
fe9d4f576 Add kernel/notifi... |
402 403 |
} EXPORT_SYMBOL_GPL(raw_notifier_call_chain); |
83fe27ea5 rcu: Make SRCU op... |
404 |
#ifdef CONFIG_SRCU |
fe9d4f576 Add kernel/notifi... |
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 |
/* * SRCU notifier chain routines. Registration and unregistration * use a mutex, and call_chain is synchronized by SRCU (no locks). */ /** * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain * @nh: Pointer to head of the SRCU notifier chain * @n: New entry in notifier chain * * Adds a notifier to an SRCU notifier chain. * Must be called in process context. * * Currently always returns zero. */ int srcu_notifier_chain_register(struct srcu_notifier_head *nh, struct notifier_block *n) { int ret; /* * This code gets used during boot-up, when task switching is * not yet working and interrupts must remain disabled. At * such times we must not call mutex_lock(). */ if (unlikely(system_state == SYSTEM_BOOTING)) return notifier_chain_register(&nh->head, n); mutex_lock(&nh->mutex); ret = notifier_chain_register(&nh->head, n); mutex_unlock(&nh->mutex); return ret; } EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); /** * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain * @nh: Pointer to head of the SRCU notifier chain * @n: Entry to remove from notifier chain * * Removes a notifier from an SRCU notifier chain. * Must be called from process context. * * Returns zero on success or %-ENOENT on failure. */ int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, struct notifier_block *n) { int ret; /* * This code gets used during boot-up, when task switching is * not yet working and interrupts must remain disabled. At * such times we must not call mutex_lock(). */ if (unlikely(system_state == SYSTEM_BOOTING)) return notifier_chain_unregister(&nh->head, n); mutex_lock(&nh->mutex); ret = notifier_chain_unregister(&nh->head, n); mutex_unlock(&nh->mutex); synchronize_srcu(&nh->srcu); return ret; } EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); /** |
70d932985 notifier: Fix bro... |
472 |
* srcu_notifier_call_chain - Call functions in an SRCU notifier chain |
fe9d4f576 Add kernel/notifi... |
473 474 475 |
* @nh: Pointer to head of the SRCU notifier chain * @val: Value passed unmodified to notifier function * @v: Pointer passed unmodified to notifier function |
fe9d4f576 Add kernel/notifi... |
476 477 478 479 480 481 482 483 484 485 486 |
* * Calls each function in a notifier chain in turn. The functions * run in a process context, so they are allowed to block. * * If the return value of the notifier can be and'ed * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain() * will return immediately, with the return value of * the notifier function which halted execution. * Otherwise the return value is the return value * of the last notifier function called. */ |
70d932985 notifier: Fix bro... |
487 488 |
int srcu_notifier_call_chain(struct srcu_notifier_head *nh, unsigned long val, void *v) |
fe9d4f576 Add kernel/notifi... |
489 490 491 492 493 |
{ int ret; int idx; idx = srcu_read_lock(&nh->srcu); |
70d932985 notifier: Fix bro... |
494 |
ret = notifier_call_chain(&nh->head, val, v, -1, NULL); |
fe9d4f576 Add kernel/notifi... |
495 496 497 |
srcu_read_unlock(&nh->srcu, idx); return ret; } |
fe9d4f576 Add kernel/notifi... |
498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 |
EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); /** * srcu_init_notifier_head - Initialize an SRCU notifier head * @nh: Pointer to head of the srcu notifier chain * * Unlike other sorts of notifier heads, SRCU notifier heads require * dynamic initialization. Be sure to call this routine before * calling any of the other SRCU notifier routines for this head. * * If an SRCU notifier head is deallocated, it must first be cleaned * up by calling srcu_cleanup_notifier_head(). Otherwise the head's * per-cpu data (used by the SRCU mechanism) will leak. */ void srcu_init_notifier_head(struct srcu_notifier_head *nh) { mutex_init(&nh->mutex); if (init_srcu_struct(&nh->srcu) < 0) BUG(); nh->head = NULL; } EXPORT_SYMBOL_GPL(srcu_init_notifier_head); |
83fe27ea5 rcu: Make SRCU op... |
520 |
#endif /* CONFIG_SRCU */ |
fe9d4f576 Add kernel/notifi... |
521 |
static ATOMIC_NOTIFIER_HEAD(die_chain); |
b40a2cb6e kprobes, notifier... |
522 |
int notrace notify_die(enum die_val val, const char *str, |
fe9d4f576 Add kernel/notifi... |
523 524 525 526 527 528 529 530 531 532 |
struct pt_regs *regs, long err, int trap, int sig) { struct die_args args = { .regs = regs, .str = str, .err = err, .trapnr = trap, .signr = sig, }; |
5778077d0 Merge branch 'x86... |
533 |
RCU_LOCKDEP_WARN(!rcu_is_watching(), |
e727c7d7a notifiers, RCU: A... |
534 |
"notify_die called but RCU thinks we're quiescent"); |
fe9d4f576 Add kernel/notifi... |
535 536 |
return atomic_notifier_call_chain(&die_chain, val, &args); } |
b40a2cb6e kprobes, notifier... |
537 |
NOKPROBE_SYMBOL(notify_die); |
fe9d4f576 Add kernel/notifi... |
538 539 540 |
int register_die_notifier(struct notifier_block *nb) { |
fe9d4f576 Add kernel/notifi... |
541 542 543 544 545 546 547 548 549 |
return atomic_notifier_chain_register(&die_chain, nb); } EXPORT_SYMBOL_GPL(register_die_notifier); int unregister_die_notifier(struct notifier_block *nb) { return atomic_notifier_chain_unregister(&die_chain, nb); } EXPORT_SYMBOL_GPL(unregister_die_notifier); |