Blame view
kernel/rcu/tiny.c
6.65 KB
9b1d82fa1 rcu: "Tiny RCU", ... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
/* * Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License |
87de1cfdc rcu: Stop trackin... |
15 16 |
* along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. |
9b1d82fa1 rcu: "Tiny RCU", ... |
17 18 19 20 21 22 |
* * Copyright IBM Corporation, 2008 * * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com> * * For detailed explanation of Read-Copy Update mechanism see - |
4ce5b9034 rcu: Do tiny clea... |
23 |
* Documentation/RCU |
9b1d82fa1 rcu: "Tiny RCU", ... |
24 |
*/ |
4ce5b9034 rcu: Do tiny clea... |
25 26 |
#include <linux/completion.h> #include <linux/interrupt.h> |
9b1d82fa1 rcu: "Tiny RCU", ... |
27 |
#include <linux/notifier.h> |
4ce5b9034 rcu: Do tiny clea... |
28 29 |
#include <linux/rcupdate.h> #include <linux/kernel.h> |
9984de1a5 kernel: Map most ... |
30 |
#include <linux/export.h> |
9b1d82fa1 rcu: "Tiny RCU", ... |
31 |
#include <linux/mutex.h> |
4ce5b9034 rcu: Do tiny clea... |
32 33 34 |
#include <linux/sched.h> #include <linux/types.h> #include <linux/init.h> |
9b1d82fa1 rcu: "Tiny RCU", ... |
35 |
#include <linux/time.h> |
4ce5b9034 rcu: Do tiny clea... |
36 |
#include <linux/cpu.h> |
268bb0ce3 sanitize <linux/p... |
37 |
#include <linux/prefetch.h> |
af658dca2 tracing: Rename f... |
38 |
#include <linux/trace_events.h> |
9b1d82fa1 rcu: "Tiny RCU", ... |
39 |
|
29c00b4a1 rcu: Add event-tr... |
40 |
#include "rcu.h" |
4102adab9 rcu: Move RCU-rel... |
41 |
/* Forward declarations for tiny_plugin.h. */ |
24278d148 rcu: priority boo... |
42 |
struct rcu_ctrlblk; |
965a002b4 rcu: Make TINY_RC... |
43 44 |
static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp); static void rcu_process_callbacks(struct softirq_action *unused); |
a57eb940d rcu: Add a TINY_P... |
45 |
static void __call_rcu(struct rcu_head *head, |
b6a4ae766 rcu: Use rcu_call... |
46 |
rcu_callback_t func, |
a57eb940d rcu: Add a TINY_P... |
47 |
struct rcu_ctrlblk *rcp); |
4102adab9 rcu: Move RCU-rel... |
48 |
#include "tiny_plugin.h" |
6bfc09e23 rcu: Provide RCU ... |
49 |
|
cc6783f78 rcu: Is it safe t... |
50 |
#if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) |
9b2e4f188 rcu: Track idlene... |
51 52 53 54 |
/* * Test whether RCU thinks that the current CPU is idle. */ |
9418fb208 rcu: Do not trace... |
55 |
bool notrace __rcu_is_watching(void) |
9b2e4f188 rcu: Track idlene... |
56 |
{ |
5f6130fa5 tiny_rcu: Directl... |
57 |
return true; |
9b2e4f188 rcu: Track idlene... |
58 |
} |
5c173eb8b rcu: Consistent r... |
59 |
EXPORT_SYMBOL(__rcu_is_watching); |
9b2e4f188 rcu: Track idlene... |
60 |
|
cc6783f78 rcu: Is it safe t... |
61 |
#endif /* defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) */ |
9b2e4f188 rcu: Track idlene... |
62 63 |
/* |
b554d7de8 rcu: optimize rcu... |
64 65 |
* Helper function for rcu_sched_qs() and rcu_bh_qs(). * Also irqs are disabled to avoid confusion due to interrupt handlers |
4ce5b9034 rcu: Do tiny clea... |
66 |
* invoking call_rcu(). |
9b1d82fa1 rcu: "Tiny RCU", ... |
67 68 69 |
*/ static int rcu_qsctr_help(struct rcu_ctrlblk *rcp) { |
149614446 rcu: Shrink TINY_... |
70 |
RCU_TRACE(reset_cpu_stall_ticks(rcp)); |
27153acbe rcu: Remove unnec... |
71 |
if (rcp->donetail != rcp->curtail) { |
9b1d82fa1 rcu: "Tiny RCU", ... |
72 |
rcp->donetail = rcp->curtail; |
9b1d82fa1 rcu: "Tiny RCU", ... |
73 74 |
return 1; } |
4ce5b9034 rcu: Do tiny clea... |
75 |
|
9b1d82fa1 rcu: "Tiny RCU", ... |
76 77 78 79 80 81 82 83 |
return 0; } /* * Record an rcu quiescent state. And an rcu_bh quiescent state while we * are at it, given that any rcu quiescent state is also an rcu_bh * quiescent state. Use "+" instead of "||" to defeat short circuiting. */ |
284a8c93a rcu: Per-CPU oper... |
84 |
void rcu_sched_qs(void) |
9b1d82fa1 rcu: "Tiny RCU", ... |
85 |
{ |
b554d7de8 rcu: optimize rcu... |
86 87 88 |
unsigned long flags; local_irq_save(flags); |
99652b54d rcu: rename rcuti... |
89 90 |
if (rcu_qsctr_help(&rcu_sched_ctrlblk) + rcu_qsctr_help(&rcu_bh_ctrlblk)) |
9dc5ad324 rcu: Simplify RCU... |
91 |
raise_softirq(RCU_SOFTIRQ); |
b554d7de8 rcu: optimize rcu... |
92 |
local_irq_restore(flags); |
9b1d82fa1 rcu: "Tiny RCU", ... |
93 94 95 96 97 |
} /* * Record an rcu_bh quiescent state. */ |
284a8c93a rcu: Per-CPU oper... |
98 |
void rcu_bh_qs(void) |
9b1d82fa1 rcu: "Tiny RCU", ... |
99 |
{ |
b554d7de8 rcu: optimize rcu... |
100 101 102 |
unsigned long flags; local_irq_save(flags); |
9b1d82fa1 rcu: "Tiny RCU", ... |
103 |
if (rcu_qsctr_help(&rcu_bh_ctrlblk)) |
9dc5ad324 rcu: Simplify RCU... |
104 |
raise_softirq(RCU_SOFTIRQ); |
b554d7de8 rcu: optimize rcu... |
105 |
local_irq_restore(flags); |
9b1d82fa1 rcu: "Tiny RCU", ... |
106 107 108 109 |
} /* * Check to see if the scheduling-clock interrupt came from an extended |
9b2e4f188 rcu: Track idlene... |
110 111 112 |
* quiescent state, and, if so, tell RCU about it. This function must * be called from hardirq context. It is normally called from the * scheduling-clock interrupt. |
9b1d82fa1 rcu: "Tiny RCU", ... |
113 |
*/ |
c3377c2da rcu: Remove "cpu"... |
114 |
void rcu_check_callbacks(int user) |
9b1d82fa1 rcu: "Tiny RCU", ... |
115 |
{ |
149614446 rcu: Shrink TINY_... |
116 |
RCU_TRACE(check_cpu_stalls()); |
ca9558a33 rcu: Remove redun... |
117 |
if (user) |
284a8c93a rcu: Per-CPU oper... |
118 |
rcu_sched_qs(); |
9b1d82fa1 rcu: "Tiny RCU", ... |
119 |
else if (!in_softirq()) |
284a8c93a rcu: Per-CPU oper... |
120 |
rcu_bh_qs(); |
8315f4229 rcu: Add call_rcu... |
121 122 |
if (user) rcu_note_voluntary_context_switch(current); |
9b1d82fa1 rcu: "Tiny RCU", ... |
123 124 125 |
} /* |
b2c0710c4 rcu: move TINY_RC... |
126 127 |
* Invoke the RCU callbacks on the specified rcu_ctrlkblk structure * whose grace period has elapsed. |
9b1d82fa1 rcu: "Tiny RCU", ... |
128 |
*/ |
965a002b4 rcu: Make TINY_RC... |
129 |
static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) |
9b1d82fa1 rcu: "Tiny RCU", ... |
130 |
{ |
e66c33d57 rcu: Add const an... |
131 |
const char *rn = NULL; |
9b1d82fa1 rcu: "Tiny RCU", ... |
132 |
struct rcu_head *next, *list; |
4ce5b9034 rcu: Do tiny clea... |
133 |
unsigned long flags; |
9e571a82f rcu: add tracing ... |
134 |
RCU_TRACE(int cb_count = 0); |
9b1d82fa1 rcu: "Tiny RCU", ... |
135 |
|
9b1d82fa1 rcu: "Tiny RCU", ... |
136 137 |
/* Move the ready-to-invoke callbacks to a local list. */ local_irq_save(flags); |
6e91f8cb1 rcu: Correctly ha... |
138 139 140 141 142 |
if (rcp->donetail == &rcp->rcucblist) { /* No callbacks ready, so just leave. */ local_irq_restore(flags); return; } |
486e25934 rcu: Avoid waking... |
143 |
RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, rcp->qlen, -1)); |
9b1d82fa1 rcu: "Tiny RCU", ... |
144 145 146 147 148 149 150 151 152 |
list = rcp->rcucblist; rcp->rcucblist = *rcp->donetail; *rcp->donetail = NULL; if (rcp->curtail == rcp->donetail) rcp->curtail = &rcp->rcucblist; rcp->donetail = &rcp->rcucblist; local_irq_restore(flags); /* Invoke the callbacks on the local list. */ |
d4c08f2ac rcu: Add grace-pe... |
153 |
RCU_TRACE(rn = rcp->name); |
9b1d82fa1 rcu: "Tiny RCU", ... |
154 155 156 |
while (list) { next = list->next; prefetch(next); |
551d55a94 tree/tiny rcu: Ad... |
157 |
debug_rcu_head_unqueue(list); |
b2c0710c4 rcu: move TINY_RC... |
158 |
local_bh_disable(); |
d4c08f2ac rcu: Add grace-pe... |
159 |
__rcu_reclaim(rn, list); |
b2c0710c4 rcu: move TINY_RC... |
160 |
local_bh_enable(); |
9b1d82fa1 rcu: "Tiny RCU", ... |
161 |
list = next; |
9e571a82f rcu: add tracing ... |
162 |
RCU_TRACE(cb_count++); |
9b1d82fa1 rcu: "Tiny RCU", ... |
163 |
} |
9e571a82f rcu: add tracing ... |
164 |
RCU_TRACE(rcu_trace_sub_qlen(rcp, cb_count)); |
0d7529246 rcu: Have rcutiny... |
165 166 |
RCU_TRACE(trace_rcu_batch_end(rcp->name, cb_count, 0, need_resched(), |
4968c300e rcu: Augment rcu_... |
167 |
is_idle_task(current), |
9dc5ad324 rcu: Simplify RCU... |
168 |
false)); |
9b1d82fa1 rcu: "Tiny RCU", ... |
169 |
} |
0766f788e latent_entropy: M... |
170 |
static __latent_entropy void rcu_process_callbacks(struct softirq_action *unused) |
b2c0710c4 rcu: move TINY_RC... |
171 |
{ |
965a002b4 rcu: Make TINY_RC... |
172 173 |
__rcu_process_callbacks(&rcu_sched_ctrlblk); __rcu_process_callbacks(&rcu_bh_ctrlblk); |
b2c0710c4 rcu: move TINY_RC... |
174 175 176 |
} /* |
9b1d82fa1 rcu: "Tiny RCU", ... |
177 178 179 180 181 182 183 184 |
* Wait for a grace period to elapse. But it is illegal to invoke * synchronize_sched() from within an RCU read-side critical section. * Therefore, any legal call to synchronize_sched() is a quiescent * state, and so on a UP system, synchronize_sched() need do nothing. * Ditto for synchronize_rcu_bh(). (But Lai Jiangshan points out the * benefits of doing might_sleep() to reduce latency.) * * Cool, huh? (Due to Josh Triplett.) |
9b1d82fa1 rcu: "Tiny RCU", ... |
185 186 187 |
*/ void synchronize_sched(void) { |
f78f5b90c rcu: Rename rcu_l... |
188 189 190 191 |
RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) || lock_is_held(&rcu_lock_map) || lock_is_held(&rcu_sched_lock_map), "Illegal synchronize_sched() in RCU read-side critical section"); |
9b1d82fa1 rcu: "Tiny RCU", ... |
192 193 |
} EXPORT_SYMBOL_GPL(synchronize_sched); |
9b1d82fa1 rcu: "Tiny RCU", ... |
194 195 196 197 |
/* * Helper function for call_rcu() and call_rcu_bh(). */ static void __call_rcu(struct rcu_head *head, |
b6a4ae766 rcu: Use rcu_call... |
198 |
rcu_callback_t func, |
9b1d82fa1 rcu: "Tiny RCU", ... |
199 200 201 |
struct rcu_ctrlblk *rcp) { unsigned long flags; |
551d55a94 tree/tiny rcu: Ad... |
202 |
debug_rcu_head_queue(head); |
9b1d82fa1 rcu: "Tiny RCU", ... |
203 204 |
head->func = func; head->next = NULL; |
4ce5b9034 rcu: Do tiny clea... |
205 |
|
9b1d82fa1 rcu: "Tiny RCU", ... |
206 207 208 |
local_irq_save(flags); *rcp->curtail = head; rcp->curtail = &head->next; |
9e571a82f rcu: add tracing ... |
209 |
RCU_TRACE(rcp->qlen++); |
9b1d82fa1 rcu: "Tiny RCU", ... |
210 |
local_irq_restore(flags); |
5f6130fa5 tiny_rcu: Directl... |
211 212 213 214 215 |
if (unlikely(is_idle_task(current))) { /* force scheduling for rcu_sched_qs() */ resched_cpu(0); } |
9b1d82fa1 rcu: "Tiny RCU", ... |
216 217 218 |
} /* |
a57eb940d rcu: Add a TINY_P... |
219 |
* Post an RCU callback to be invoked after the end of an RCU-sched grace |
9b1d82fa1 rcu: "Tiny RCU", ... |
220 221 222 |
* period. But since we have but one CPU, that would be after any * quiescent state. */ |
b6a4ae766 rcu: Use rcu_call... |
223 |
void call_rcu_sched(struct rcu_head *head, rcu_callback_t func) |
9b1d82fa1 rcu: "Tiny RCU", ... |
224 |
{ |
99652b54d rcu: rename rcuti... |
225 |
__call_rcu(head, func, &rcu_sched_ctrlblk); |
9b1d82fa1 rcu: "Tiny RCU", ... |
226 |
} |
a57eb940d rcu: Add a TINY_P... |
227 |
EXPORT_SYMBOL_GPL(call_rcu_sched); |
9b1d82fa1 rcu: "Tiny RCU", ... |
228 229 230 231 232 |
/* * Post an RCU bottom-half callback to be invoked after any subsequent * quiescent state. */ |
b6a4ae766 rcu: Use rcu_call... |
233 |
void call_rcu_bh(struct rcu_head *head, rcu_callback_t func) |
9b1d82fa1 rcu: "Tiny RCU", ... |
234 235 236 237 |
{ __call_rcu(head, func, &rcu_bh_ctrlblk); } EXPORT_SYMBOL_GPL(call_rcu_bh); |
9dc5ad324 rcu: Simplify RCU... |
238 |
|
aa23c6fbc rcutorture: Add e... |
239 |
void __init rcu_init(void) |
9dc5ad324 rcu: Simplify RCU... |
240 241 |
{ open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); |
630181c4a rcu: Initialize t... |
242 243 |
RCU_TRACE(reset_cpu_stall_ticks(&rcu_sched_ctrlblk)); RCU_TRACE(reset_cpu_stall_ticks(&rcu_bh_ctrlblk)); |
aa23c6fbc rcutorture: Add e... |
244 245 |
rcu_early_boot_tests(); |
9dc5ad324 rcu: Simplify RCU... |
246 |
} |