Blame view
kernel/rcu/tiny.c
4.81 KB
00de9d741 rcu/tiny: Convert... |
1 |
// SPDX-License-Identifier: GPL-2.0+ |
9b1d82fa1 rcu: "Tiny RCU", ... |
2 3 4 |
/* * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. * |
9b1d82fa1 rcu: "Tiny RCU", ... |
5 6 |
* Copyright IBM Corporation, 2008 * |
00de9d741 rcu/tiny: Convert... |
7 |
* Author: Paul E. McKenney <paulmck@linux.ibm.com> |
9b1d82fa1 rcu: "Tiny RCU", ... |
8 9 |
* * For detailed explanation of Read-Copy Update mechanism see - |
4ce5b9034 rcu: Do tiny clea... |
10 |
* Documentation/RCU |
9b1d82fa1 rcu: "Tiny RCU", ... |
11 |
*/ |
4ce5b9034 rcu: Do tiny clea... |
12 13 |
#include <linux/completion.h> #include <linux/interrupt.h> |
9b1d82fa1 rcu: "Tiny RCU", ... |
14 |
#include <linux/notifier.h> |
f9411ebe3 rcu: Separate the... |
15 |
#include <linux/rcupdate_wait.h> |
4ce5b9034 rcu: Do tiny clea... |
16 |
#include <linux/kernel.h> |
9984de1a5 kernel: Map most ... |
17 |
#include <linux/export.h> |
9b1d82fa1 rcu: "Tiny RCU", ... |
18 |
#include <linux/mutex.h> |
4ce5b9034 rcu: Do tiny clea... |
19 20 21 |
#include <linux/sched.h> #include <linux/types.h> #include <linux/init.h> |
9b1d82fa1 rcu: "Tiny RCU", ... |
22 |
#include <linux/time.h> |
4ce5b9034 rcu: Do tiny clea... |
23 |
#include <linux/cpu.h> |
268bb0ce3 sanitize <linux/p... |
24 |
#include <linux/prefetch.h> |
77a40f970 rcu: Remove kfree... |
25 |
#include <linux/slab.h> |
64d1d06cc rcu/tiny: support... |
26 |
#include <linux/mm.h> |
9b1d82fa1 rcu: "Tiny RCU", ... |
27 |
|
29c00b4a1 rcu: Add event-tr... |
28 |
#include "rcu.h" |
6d48152ea rcu: Remove RCU C... |
29 30 31 32 33 34 35 36 |
/* Global control variables for rcupdate callback mechanism. */ struct rcu_ctrlblk { struct rcu_head *rcucblist; /* List of pending callbacks (CBs). */ struct rcu_head **donetail; /* ->next pointer of last "done" CB. */ struct rcu_head **curtail; /* ->next pointer of last CB. */ }; /* Definition for rcupdate control block. */ |
709fdce75 rcu: Express Tiny... |
37 38 39 |
static struct rcu_ctrlblk rcu_ctrlblk = { .donetail = &rcu_ctrlblk.rcucblist, .curtail = &rcu_ctrlblk.rcucblist, |
6d48152ea rcu: Remove RCU C... |
40 |
}; |
709fdce75 rcu: Express Tiny... |
41 |
void rcu_barrier(void) |
9b1d82fa1 rcu: "Tiny RCU", ... |
42 |
{ |
709fdce75 rcu: Express Tiny... |
43 |
wait_rcu_gp(call_rcu); |
9b1d82fa1 rcu: "Tiny RCU", ... |
44 |
} |
709fdce75 rcu: Express Tiny... |
45 |
EXPORT_SYMBOL(rcu_barrier); |
9b1d82fa1 rcu: "Tiny RCU", ... |
46 |
|
65cfe3583 rcu: Define RCU-b... |
47 |
/* Record an rcu quiescent state. */ |
709fdce75 rcu: Express Tiny... |
48 |
void rcu_qs(void) |
9b1d82fa1 rcu: "Tiny RCU", ... |
49 |
{ |
b554d7de8 rcu: optimize rcu... |
50 51 52 |
unsigned long flags; local_irq_save(flags); |
709fdce75 rcu: Express Tiny... |
53 54 |
if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { rcu_ctrlblk.donetail = rcu_ctrlblk.curtail; |
18d7e4067 rcu: rcu_qs -- Us... |
55 |
raise_softirq_irqoff(RCU_SOFTIRQ); |
65cfe3583 rcu: Define RCU-b... |
56 |
} |
b554d7de8 rcu: optimize rcu... |
57 |
local_irq_restore(flags); |
9b1d82fa1 rcu: "Tiny RCU", ... |
58 59 60 61 |
} /* * Check to see if the scheduling-clock interrupt came from an extended |
9b2e4f188 rcu: Track idlene... |
62 63 64 |
* quiescent state, and, if so, tell RCU about it. This function must * be called from hardirq context. It is normally called from the * scheduling-clock interrupt. |
9b1d82fa1 rcu: "Tiny RCU", ... |
65 |
*/ |
c98cac603 rcu: Rename rcu_c... |
66 |
void rcu_sched_clock_irq(int user) |
9b1d82fa1 rcu: "Tiny RCU", ... |
67 |
{ |
c5bacd941 rcu: Motivate Tin... |
68 |
if (user) { |
709fdce75 rcu: Express Tiny... |
69 |
rcu_qs(); |
c5bacd941 rcu: Motivate Tin... |
70 71 72 73 |
} else if (rcu_ctrlblk.donetail != rcu_ctrlblk.curtail) { set_tsk_need_resched(current); set_preempt_need_resched(); } |
9b1d82fa1 rcu: "Tiny RCU", ... |
74 |
} |
77a40f970 rcu: Remove kfree... |
75 76 77 78 79 80 81 82 83 84 |
/* * Reclaim the specified callback, either by invoking it for non-kfree cases or * freeing it directly (for kfree). Return true if kfreeing, false otherwise. */ static inline bool rcu_reclaim_tiny(struct rcu_head *head) { rcu_callback_t f; unsigned long offset = (unsigned long)head->func; rcu_lock_acquire(&rcu_callback_map); |
c408b215f rcu: Rename *_kfr... |
85 86 |
if (__is_kvfree_rcu_offset(offset)) { trace_rcu_invoke_kvfree_callback("", head, offset); |
64d1d06cc rcu/tiny: support... |
87 |
kvfree((void *)head - offset); |
77a40f970 rcu: Remove kfree... |
88 89 90 91 92 93 94 95 96 97 98 |
rcu_lock_release(&rcu_callback_map); return true; } trace_rcu_invoke_callback("", head); f = head->func; WRITE_ONCE(head->func, (rcu_callback_t)0L); f(head); rcu_lock_release(&rcu_callback_map); return false; } |
65cfe3583 rcu: Define RCU-b... |
99 100 |
/* Invoke the RCU callbacks whose grace period has elapsed. */ static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) |
9b1d82fa1 rcu: "Tiny RCU", ... |
101 |
{ |
9b1d82fa1 rcu: "Tiny RCU", ... |
102 |
struct rcu_head *next, *list; |
4ce5b9034 rcu: Do tiny clea... |
103 |
unsigned long flags; |
9b1d82fa1 rcu: "Tiny RCU", ... |
104 |
|
9b1d82fa1 rcu: "Tiny RCU", ... |
105 106 |
/* Move the ready-to-invoke callbacks to a local list. */ local_irq_save(flags); |
709fdce75 rcu: Express Tiny... |
107 |
if (rcu_ctrlblk.donetail == &rcu_ctrlblk.rcucblist) { |
6e91f8cb1 rcu: Correctly ha... |
108 109 110 111 |
/* No callbacks ready, so just leave. */ local_irq_restore(flags); return; } |
709fdce75 rcu: Express Tiny... |
112 113 114 115 116 117 |
list = rcu_ctrlblk.rcucblist; rcu_ctrlblk.rcucblist = *rcu_ctrlblk.donetail; *rcu_ctrlblk.donetail = NULL; if (rcu_ctrlblk.curtail == rcu_ctrlblk.donetail) rcu_ctrlblk.curtail = &rcu_ctrlblk.rcucblist; rcu_ctrlblk.donetail = &rcu_ctrlblk.rcucblist; |
9b1d82fa1 rcu: "Tiny RCU", ... |
118 119 120 121 122 123 |
local_irq_restore(flags); /* Invoke the callbacks on the local list. */ while (list) { next = list->next; prefetch(next); |
551d55a94 tree/tiny rcu: Ad... |
124 |
debug_rcu_head_unqueue(list); |
b2c0710c4 rcu: move TINY_RC... |
125 |
local_bh_disable(); |
77a40f970 rcu: Remove kfree... |
126 |
rcu_reclaim_tiny(list); |
b2c0710c4 rcu: move TINY_RC... |
127 |
local_bh_enable(); |
9b1d82fa1 rcu: "Tiny RCU", ... |
128 129 130 |
list = next; } } |
b2c0710c4 rcu: move TINY_RC... |
131 |
/* |
9b1d82fa1 rcu: "Tiny RCU", ... |
132 |
* Wait for a grace period to elapse. But it is illegal to invoke |
679d3f309 rcu: Clean up fla... |
133 134 135 |
* synchronize_rcu() from within an RCU read-side critical section. * Therefore, any legal call to synchronize_rcu() is a quiescent * state, and so on a UP system, synchronize_rcu() need do nothing. |
65cfe3583 rcu: Define RCU-b... |
136 137 |
* (But Lai Jiangshan points out the benefits of doing might_sleep() * to reduce latency.) |
9b1d82fa1 rcu: "Tiny RCU", ... |
138 139 |
* * Cool, huh? (Due to Josh Triplett.) |
9b1d82fa1 rcu: "Tiny RCU", ... |
140 |
*/ |
709fdce75 rcu: Express Tiny... |
141 |
void synchronize_rcu(void) |
9b1d82fa1 rcu: "Tiny RCU", ... |
142 |
{ |
f78f5b90c rcu: Rename rcu_l... |
143 144 145 |
RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || lock_is_held(&rcu_lock_map) || lock_is_held(&rcu_sched_lock_map), |
679d3f309 rcu: Clean up fla... |
146 |
"Illegal synchronize_rcu() in RCU read-side critical section"); |
9b1d82fa1 rcu: "Tiny RCU", ... |
147 |
} |
709fdce75 rcu: Express Tiny... |
148 |
EXPORT_SYMBOL_GPL(synchronize_rcu); |
9b1d82fa1 rcu: "Tiny RCU", ... |
149 |
|
9b1d82fa1 rcu: "Tiny RCU", ... |
150 |
/* |
679d3f309 rcu: Clean up fla... |
151 |
* Post an RCU callback to be invoked after the end of an RCU grace |
65cfe3583 rcu: Define RCU-b... |
152 153 |
* period. But since we have but one CPU, that would be after any * quiescent state. |
9b1d82fa1 rcu: "Tiny RCU", ... |
154 |
*/ |
709fdce75 rcu: Express Tiny... |
155 |
void call_rcu(struct rcu_head *head, rcu_callback_t func) |
9b1d82fa1 rcu: "Tiny RCU", ... |
156 157 |
{ unsigned long flags; |
551d55a94 tree/tiny rcu: Ad... |
158 |
debug_rcu_head_queue(head); |
9b1d82fa1 rcu: "Tiny RCU", ... |
159 160 |
head->func = func; head->next = NULL; |
4ce5b9034 rcu: Do tiny clea... |
161 |
|
9b1d82fa1 rcu: "Tiny RCU", ... |
162 |
local_irq_save(flags); |
709fdce75 rcu: Express Tiny... |
163 164 |
*rcu_ctrlblk.curtail = head; rcu_ctrlblk.curtail = &head->next; |
9b1d82fa1 rcu: "Tiny RCU", ... |
165 |
local_irq_restore(flags); |
5f6130fa5 tiny_rcu: Directl... |
166 167 |
if (unlikely(is_idle_task(current))) { |
709fdce75 rcu: Express Tiny... |
168 |
/* force scheduling for rcu_qs() */ |
5f6130fa5 tiny_rcu: Directl... |
169 170 |
resched_cpu(0); } |
9b1d82fa1 rcu: "Tiny RCU", ... |
171 |
} |
709fdce75 rcu: Express Tiny... |
172 |
EXPORT_SYMBOL_GPL(call_rcu); |
9dc5ad324 rcu: Simplify RCU... |
173 |
|
aa23c6fbc rcutorture: Add e... |
174 |
void __init rcu_init(void) |
9dc5ad324 rcu: Simplify RCU... |
175 176 |
{ open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); |
aa23c6fbc rcutorture: Add e... |
177 |
rcu_early_boot_tests(); |
e0fcba9ac srcu: Make call_s... |
178 |
srcu_init(); |
9dc5ad324 rcu: Simplify RCU... |
179 |
} |