Commit 851a67b825540a8e00c0be3ee25e4627ba8b133b
1 parent
34a3d1e837
lockdep: annotate rcu_read_{,un}lock{,_bh}
lockdep annotate rcu_read_{,un}lock{,_bh} in order to catch imbalanced usage. Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl> Signed-off-by: Ingo Molnar <mingo@elte.hu> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Showing 3 changed files with 29 additions and 0 deletions Inline Diff
include/linux/lockdep.h
1 | /* | 1 | /* |
2 | * Runtime locking correctness validator | 2 | * Runtime locking correctness validator |
3 | * | 3 | * |
4 | * Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> | 4 | * Copyright (C) 2006,2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> |
5 | * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> | 5 | * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> |
6 | * | 6 | * |
7 | * see Documentation/lockdep-design.txt for more details. | 7 | * see Documentation/lockdep-design.txt for more details. |
8 | */ | 8 | */ |
9 | #ifndef __LINUX_LOCKDEP_H | 9 | #ifndef __LINUX_LOCKDEP_H |
10 | #define __LINUX_LOCKDEP_H | 10 | #define __LINUX_LOCKDEP_H |
11 | 11 | ||
12 | struct task_struct; | 12 | struct task_struct; |
13 | struct lockdep_map; | 13 | struct lockdep_map; |
14 | 14 | ||
15 | #ifdef CONFIG_LOCKDEP | 15 | #ifdef CONFIG_LOCKDEP |
16 | 16 | ||
17 | #include <linux/linkage.h> | 17 | #include <linux/linkage.h> |
18 | #include <linux/list.h> | 18 | #include <linux/list.h> |
19 | #include <linux/debug_locks.h> | 19 | #include <linux/debug_locks.h> |
20 | #include <linux/stacktrace.h> | 20 | #include <linux/stacktrace.h> |
21 | 21 | ||
22 | /* | 22 | /* |
23 | * Lock-class usage-state bits: | 23 | * Lock-class usage-state bits: |
24 | */ | 24 | */ |
25 | enum lock_usage_bit | 25 | enum lock_usage_bit |
26 | { | 26 | { |
27 | LOCK_USED = 0, | 27 | LOCK_USED = 0, |
28 | LOCK_USED_IN_HARDIRQ, | 28 | LOCK_USED_IN_HARDIRQ, |
29 | LOCK_USED_IN_SOFTIRQ, | 29 | LOCK_USED_IN_SOFTIRQ, |
30 | LOCK_ENABLED_SOFTIRQS, | 30 | LOCK_ENABLED_SOFTIRQS, |
31 | LOCK_ENABLED_HARDIRQS, | 31 | LOCK_ENABLED_HARDIRQS, |
32 | LOCK_USED_IN_HARDIRQ_READ, | 32 | LOCK_USED_IN_HARDIRQ_READ, |
33 | LOCK_USED_IN_SOFTIRQ_READ, | 33 | LOCK_USED_IN_SOFTIRQ_READ, |
34 | LOCK_ENABLED_SOFTIRQS_READ, | 34 | LOCK_ENABLED_SOFTIRQS_READ, |
35 | LOCK_ENABLED_HARDIRQS_READ, | 35 | LOCK_ENABLED_HARDIRQS_READ, |
36 | LOCK_USAGE_STATES | 36 | LOCK_USAGE_STATES |
37 | }; | 37 | }; |
38 | 38 | ||
39 | /* | 39 | /* |
40 | * Usage-state bitmasks: | 40 | * Usage-state bitmasks: |
41 | */ | 41 | */ |
42 | #define LOCKF_USED (1 << LOCK_USED) | 42 | #define LOCKF_USED (1 << LOCK_USED) |
43 | #define LOCKF_USED_IN_HARDIRQ (1 << LOCK_USED_IN_HARDIRQ) | 43 | #define LOCKF_USED_IN_HARDIRQ (1 << LOCK_USED_IN_HARDIRQ) |
44 | #define LOCKF_USED_IN_SOFTIRQ (1 << LOCK_USED_IN_SOFTIRQ) | 44 | #define LOCKF_USED_IN_SOFTIRQ (1 << LOCK_USED_IN_SOFTIRQ) |
45 | #define LOCKF_ENABLED_HARDIRQS (1 << LOCK_ENABLED_HARDIRQS) | 45 | #define LOCKF_ENABLED_HARDIRQS (1 << LOCK_ENABLED_HARDIRQS) |
46 | #define LOCKF_ENABLED_SOFTIRQS (1 << LOCK_ENABLED_SOFTIRQS) | 46 | #define LOCKF_ENABLED_SOFTIRQS (1 << LOCK_ENABLED_SOFTIRQS) |
47 | 47 | ||
48 | #define LOCKF_ENABLED_IRQS (LOCKF_ENABLED_HARDIRQS | LOCKF_ENABLED_SOFTIRQS) | 48 | #define LOCKF_ENABLED_IRQS (LOCKF_ENABLED_HARDIRQS | LOCKF_ENABLED_SOFTIRQS) |
49 | #define LOCKF_USED_IN_IRQ (LOCKF_USED_IN_HARDIRQ | LOCKF_USED_IN_SOFTIRQ) | 49 | #define LOCKF_USED_IN_IRQ (LOCKF_USED_IN_HARDIRQ | LOCKF_USED_IN_SOFTIRQ) |
50 | 50 | ||
51 | #define LOCKF_USED_IN_HARDIRQ_READ (1 << LOCK_USED_IN_HARDIRQ_READ) | 51 | #define LOCKF_USED_IN_HARDIRQ_READ (1 << LOCK_USED_IN_HARDIRQ_READ) |
52 | #define LOCKF_USED_IN_SOFTIRQ_READ (1 << LOCK_USED_IN_SOFTIRQ_READ) | 52 | #define LOCKF_USED_IN_SOFTIRQ_READ (1 << LOCK_USED_IN_SOFTIRQ_READ) |
53 | #define LOCKF_ENABLED_HARDIRQS_READ (1 << LOCK_ENABLED_HARDIRQS_READ) | 53 | #define LOCKF_ENABLED_HARDIRQS_READ (1 << LOCK_ENABLED_HARDIRQS_READ) |
54 | #define LOCKF_ENABLED_SOFTIRQS_READ (1 << LOCK_ENABLED_SOFTIRQS_READ) | 54 | #define LOCKF_ENABLED_SOFTIRQS_READ (1 << LOCK_ENABLED_SOFTIRQS_READ) |
55 | 55 | ||
56 | #define LOCKF_ENABLED_IRQS_READ \ | 56 | #define LOCKF_ENABLED_IRQS_READ \ |
57 | (LOCKF_ENABLED_HARDIRQS_READ | LOCKF_ENABLED_SOFTIRQS_READ) | 57 | (LOCKF_ENABLED_HARDIRQS_READ | LOCKF_ENABLED_SOFTIRQS_READ) |
58 | #define LOCKF_USED_IN_IRQ_READ \ | 58 | #define LOCKF_USED_IN_IRQ_READ \ |
59 | (LOCKF_USED_IN_HARDIRQ_READ | LOCKF_USED_IN_SOFTIRQ_READ) | 59 | (LOCKF_USED_IN_HARDIRQ_READ | LOCKF_USED_IN_SOFTIRQ_READ) |
60 | 60 | ||
61 | #define MAX_LOCKDEP_SUBCLASSES 8UL | 61 | #define MAX_LOCKDEP_SUBCLASSES 8UL |
62 | 62 | ||
63 | /* | 63 | /* |
64 | * Lock-classes are keyed via unique addresses, by embedding the | 64 | * Lock-classes are keyed via unique addresses, by embedding the |
65 | * lockclass-key into the kernel (or module) .data section. (For | 65 | * lockclass-key into the kernel (or module) .data section. (For |
66 | * static locks we use the lock address itself as the key.) | 66 | * static locks we use the lock address itself as the key.) |
67 | */ | 67 | */ |
68 | struct lockdep_subclass_key { | 68 | struct lockdep_subclass_key { |
69 | char __one_byte; | 69 | char __one_byte; |
70 | } __attribute__ ((__packed__)); | 70 | } __attribute__ ((__packed__)); |
71 | 71 | ||
72 | struct lock_class_key { | 72 | struct lock_class_key { |
73 | struct lockdep_subclass_key subkeys[MAX_LOCKDEP_SUBCLASSES]; | 73 | struct lockdep_subclass_key subkeys[MAX_LOCKDEP_SUBCLASSES]; |
74 | }; | 74 | }; |
75 | 75 | ||
76 | /* | 76 | /* |
77 | * The lock-class itself: | 77 | * The lock-class itself: |
78 | */ | 78 | */ |
79 | struct lock_class { | 79 | struct lock_class { |
80 | /* | 80 | /* |
81 | * class-hash: | 81 | * class-hash: |
82 | */ | 82 | */ |
83 | struct list_head hash_entry; | 83 | struct list_head hash_entry; |
84 | 84 | ||
85 | /* | 85 | /* |
86 | * global list of all lock-classes: | 86 | * global list of all lock-classes: |
87 | */ | 87 | */ |
88 | struct list_head lock_entry; | 88 | struct list_head lock_entry; |
89 | 89 | ||
90 | struct lockdep_subclass_key *key; | 90 | struct lockdep_subclass_key *key; |
91 | unsigned int subclass; | 91 | unsigned int subclass; |
92 | 92 | ||
93 | /* | 93 | /* |
94 | * IRQ/softirq usage tracking bits: | 94 | * IRQ/softirq usage tracking bits: |
95 | */ | 95 | */ |
96 | unsigned long usage_mask; | 96 | unsigned long usage_mask; |
97 | struct stack_trace usage_traces[LOCK_USAGE_STATES]; | 97 | struct stack_trace usage_traces[LOCK_USAGE_STATES]; |
98 | 98 | ||
99 | /* | 99 | /* |
100 | * These fields represent a directed graph of lock dependencies, | 100 | * These fields represent a directed graph of lock dependencies, |
101 | * to every node we attach a list of "forward" and a list of | 101 | * to every node we attach a list of "forward" and a list of |
102 | * "backward" graph nodes. | 102 | * "backward" graph nodes. |
103 | */ | 103 | */ |
104 | struct list_head locks_after, locks_before; | 104 | struct list_head locks_after, locks_before; |
105 | 105 | ||
106 | /* | 106 | /* |
107 | * Generation counter, when doing certain classes of graph walking, | 107 | * Generation counter, when doing certain classes of graph walking, |
108 | * to ensure that we check one node only once: | 108 | * to ensure that we check one node only once: |
109 | */ | 109 | */ |
110 | unsigned int version; | 110 | unsigned int version; |
111 | 111 | ||
112 | /* | 112 | /* |
113 | * Statistics counter: | 113 | * Statistics counter: |
114 | */ | 114 | */ |
115 | unsigned long ops; | 115 | unsigned long ops; |
116 | 116 | ||
117 | const char *name; | 117 | const char *name; |
118 | int name_version; | 118 | int name_version; |
119 | 119 | ||
120 | #ifdef CONFIG_LOCK_STAT | 120 | #ifdef CONFIG_LOCK_STAT |
121 | unsigned long contention_point[4]; | 121 | unsigned long contention_point[4]; |
122 | #endif | 122 | #endif |
123 | }; | 123 | }; |
124 | 124 | ||
125 | #ifdef CONFIG_LOCK_STAT | 125 | #ifdef CONFIG_LOCK_STAT |
126 | struct lock_time { | 126 | struct lock_time { |
127 | s64 min; | 127 | s64 min; |
128 | s64 max; | 128 | s64 max; |
129 | s64 total; | 129 | s64 total; |
130 | unsigned long nr; | 130 | unsigned long nr; |
131 | }; | 131 | }; |
132 | 132 | ||
133 | enum bounce_type { | 133 | enum bounce_type { |
134 | bounce_acquired_write, | 134 | bounce_acquired_write, |
135 | bounce_acquired_read, | 135 | bounce_acquired_read, |
136 | bounce_contended_write, | 136 | bounce_contended_write, |
137 | bounce_contended_read, | 137 | bounce_contended_read, |
138 | nr_bounce_types, | 138 | nr_bounce_types, |
139 | 139 | ||
140 | bounce_acquired = bounce_acquired_write, | 140 | bounce_acquired = bounce_acquired_write, |
141 | bounce_contended = bounce_contended_write, | 141 | bounce_contended = bounce_contended_write, |
142 | }; | 142 | }; |
143 | 143 | ||
144 | struct lock_class_stats { | 144 | struct lock_class_stats { |
145 | unsigned long contention_point[4]; | 145 | unsigned long contention_point[4]; |
146 | struct lock_time read_waittime; | 146 | struct lock_time read_waittime; |
147 | struct lock_time write_waittime; | 147 | struct lock_time write_waittime; |
148 | struct lock_time read_holdtime; | 148 | struct lock_time read_holdtime; |
149 | struct lock_time write_holdtime; | 149 | struct lock_time write_holdtime; |
150 | unsigned long bounces[nr_bounce_types]; | 150 | unsigned long bounces[nr_bounce_types]; |
151 | }; | 151 | }; |
152 | 152 | ||
153 | struct lock_class_stats lock_stats(struct lock_class *class); | 153 | struct lock_class_stats lock_stats(struct lock_class *class); |
154 | void clear_lock_stats(struct lock_class *class); | 154 | void clear_lock_stats(struct lock_class *class); |
155 | #endif | 155 | #endif |
156 | 156 | ||
157 | /* | 157 | /* |
158 | * Map the lock object (the lock instance) to the lock-class object. | 158 | * Map the lock object (the lock instance) to the lock-class object. |
159 | * This is embedded into specific lock instances: | 159 | * This is embedded into specific lock instances: |
160 | */ | 160 | */ |
161 | struct lockdep_map { | 161 | struct lockdep_map { |
162 | struct lock_class_key *key; | 162 | struct lock_class_key *key; |
163 | struct lock_class *class_cache; | 163 | struct lock_class *class_cache; |
164 | const char *name; | 164 | const char *name; |
165 | #ifdef CONFIG_LOCK_STAT | 165 | #ifdef CONFIG_LOCK_STAT |
166 | int cpu; | 166 | int cpu; |
167 | #endif | 167 | #endif |
168 | }; | 168 | }; |
169 | 169 | ||
170 | /* | 170 | /* |
171 | * Every lock has a list of other locks that were taken after it. | 171 | * Every lock has a list of other locks that were taken after it. |
172 | * We only grow the list, never remove from it: | 172 | * We only grow the list, never remove from it: |
173 | */ | 173 | */ |
174 | struct lock_list { | 174 | struct lock_list { |
175 | struct list_head entry; | 175 | struct list_head entry; |
176 | struct lock_class *class; | 176 | struct lock_class *class; |
177 | struct stack_trace trace; | 177 | struct stack_trace trace; |
178 | int distance; | 178 | int distance; |
179 | }; | 179 | }; |
180 | 180 | ||
181 | /* | 181 | /* |
182 | * We record lock dependency chains, so that we can cache them: | 182 | * We record lock dependency chains, so that we can cache them: |
183 | */ | 183 | */ |
184 | struct lock_chain { | 184 | struct lock_chain { |
185 | struct list_head entry; | 185 | struct list_head entry; |
186 | u64 chain_key; | 186 | u64 chain_key; |
187 | }; | 187 | }; |
188 | 188 | ||
189 | struct held_lock { | 189 | struct held_lock { |
190 | /* | 190 | /* |
191 | * One-way hash of the dependency chain up to this point. We | 191 | * One-way hash of the dependency chain up to this point. We |
192 | * hash the hashes step by step as the dependency chain grows. | 192 | * hash the hashes step by step as the dependency chain grows. |
193 | * | 193 | * |
194 | * We use it for dependency-caching and we skip detection | 194 | * We use it for dependency-caching and we skip detection |
195 | * passes and dependency-updates if there is a cache-hit, so | 195 | * passes and dependency-updates if there is a cache-hit, so |
196 | * it is absolutely critical for 100% coverage of the validator | 196 | * it is absolutely critical for 100% coverage of the validator |
197 | * to have a unique key value for every unique dependency path | 197 | * to have a unique key value for every unique dependency path |
198 | * that can occur in the system, to make a unique hash value | 198 | * that can occur in the system, to make a unique hash value |
199 | * as likely as possible - hence the 64-bit width. | 199 | * as likely as possible - hence the 64-bit width. |
200 | * | 200 | * |
201 | * The task struct holds the current hash value (initialized | 201 | * The task struct holds the current hash value (initialized |
202 | * with zero), here we store the previous hash value: | 202 | * with zero), here we store the previous hash value: |
203 | */ | 203 | */ |
204 | u64 prev_chain_key; | 204 | u64 prev_chain_key; |
205 | struct lock_class *class; | 205 | struct lock_class *class; |
206 | unsigned long acquire_ip; | 206 | unsigned long acquire_ip; |
207 | struct lockdep_map *instance; | 207 | struct lockdep_map *instance; |
208 | 208 | ||
209 | #ifdef CONFIG_LOCK_STAT | 209 | #ifdef CONFIG_LOCK_STAT |
210 | u64 waittime_stamp; | 210 | u64 waittime_stamp; |
211 | u64 holdtime_stamp; | 211 | u64 holdtime_stamp; |
212 | #endif | 212 | #endif |
213 | /* | 213 | /* |
214 | * The lock-stack is unified in that the lock chains of interrupt | 214 | * The lock-stack is unified in that the lock chains of interrupt |
215 | * contexts nest ontop of process context chains, but we 'separate' | 215 | * contexts nest ontop of process context chains, but we 'separate' |
216 | * the hashes by starting with 0 if we cross into an interrupt | 216 | * the hashes by starting with 0 if we cross into an interrupt |
217 | * context, and we also keep do not add cross-context lock | 217 | * context, and we also keep do not add cross-context lock |
218 | * dependencies - the lock usage graph walking covers that area | 218 | * dependencies - the lock usage graph walking covers that area |
219 | * anyway, and we'd just unnecessarily increase the number of | 219 | * anyway, and we'd just unnecessarily increase the number of |
220 | * dependencies otherwise. [Note: hardirq and softirq contexts | 220 | * dependencies otherwise. [Note: hardirq and softirq contexts |
221 | * are separated from each other too.] | 221 | * are separated from each other too.] |
222 | * | 222 | * |
223 | * The following field is used to detect when we cross into an | 223 | * The following field is used to detect when we cross into an |
224 | * interrupt context: | 224 | * interrupt context: |
225 | */ | 225 | */ |
226 | int irq_context; | 226 | int irq_context; |
227 | int trylock; | 227 | int trylock; |
228 | int read; | 228 | int read; |
229 | int check; | 229 | int check; |
230 | int hardirqs_off; | 230 | int hardirqs_off; |
231 | }; | 231 | }; |
232 | 232 | ||
233 | /* | 233 | /* |
234 | * Initialization, self-test and debugging-output methods: | 234 | * Initialization, self-test and debugging-output methods: |
235 | */ | 235 | */ |
236 | extern void lockdep_init(void); | 236 | extern void lockdep_init(void); |
237 | extern void lockdep_info(void); | 237 | extern void lockdep_info(void); |
238 | extern void lockdep_reset(void); | 238 | extern void lockdep_reset(void); |
239 | extern void lockdep_reset_lock(struct lockdep_map *lock); | 239 | extern void lockdep_reset_lock(struct lockdep_map *lock); |
240 | extern void lockdep_free_key_range(void *start, unsigned long size); | 240 | extern void lockdep_free_key_range(void *start, unsigned long size); |
241 | extern void lockdep_sys_exit(void); | 241 | extern void lockdep_sys_exit(void); |
242 | 242 | ||
243 | extern void lockdep_off(void); | 243 | extern void lockdep_off(void); |
244 | extern void lockdep_on(void); | 244 | extern void lockdep_on(void); |
245 | 245 | ||
246 | /* | 246 | /* |
247 | * These methods are used by specific locking variants (spinlocks, | 247 | * These methods are used by specific locking variants (spinlocks, |
248 | * rwlocks, mutexes and rwsems) to pass init/acquire/release events | 248 | * rwlocks, mutexes and rwsems) to pass init/acquire/release events |
249 | * to lockdep: | 249 | * to lockdep: |
250 | */ | 250 | */ |
251 | 251 | ||
252 | extern void lockdep_init_map(struct lockdep_map *lock, const char *name, | 252 | extern void lockdep_init_map(struct lockdep_map *lock, const char *name, |
253 | struct lock_class_key *key, int subclass); | 253 | struct lock_class_key *key, int subclass); |
254 | 254 | ||
255 | /* | 255 | /* |
256 | * To initialize a lockdep_map statically use this macro. | ||
257 | * Note that _name must not be NULL. | ||
258 | */ | ||
259 | #define STATIC_LOCKDEP_MAP_INIT(_name, _key) \ | ||
260 | { .name = (_name), .key = (void *)(_key), } | ||
261 | |||
262 | /* | ||
256 | * Reinitialize a lock key - for cases where there is special locking or | 263 | * Reinitialize a lock key - for cases where there is special locking or |
257 | * special initialization of locks so that the validator gets the scope | 264 | * special initialization of locks so that the validator gets the scope |
258 | * of dependencies wrong: they are either too broad (they need a class-split) | 265 | * of dependencies wrong: they are either too broad (they need a class-split) |
259 | * or they are too narrow (they suffer from a false class-split): | 266 | * or they are too narrow (they suffer from a false class-split): |
260 | */ | 267 | */ |
261 | #define lockdep_set_class(lock, key) \ | 268 | #define lockdep_set_class(lock, key) \ |
262 | lockdep_init_map(&(lock)->dep_map, #key, key, 0) | 269 | lockdep_init_map(&(lock)->dep_map, #key, key, 0) |
263 | #define lockdep_set_class_and_name(lock, key, name) \ | 270 | #define lockdep_set_class_and_name(lock, key, name) \ |
264 | lockdep_init_map(&(lock)->dep_map, name, key, 0) | 271 | lockdep_init_map(&(lock)->dep_map, name, key, 0) |
265 | #define lockdep_set_class_and_subclass(lock, key, sub) \ | 272 | #define lockdep_set_class_and_subclass(lock, key, sub) \ |
266 | lockdep_init_map(&(lock)->dep_map, #key, key, sub) | 273 | lockdep_init_map(&(lock)->dep_map, #key, key, sub) |
267 | #define lockdep_set_subclass(lock, sub) \ | 274 | #define lockdep_set_subclass(lock, sub) \ |
268 | lockdep_init_map(&(lock)->dep_map, #lock, \ | 275 | lockdep_init_map(&(lock)->dep_map, #lock, \ |
269 | (lock)->dep_map.key, sub) | 276 | (lock)->dep_map.key, sub) |
270 | 277 | ||
271 | /* | 278 | /* |
272 | * Acquire a lock. | 279 | * Acquire a lock. |
273 | * | 280 | * |
274 | * Values for "read": | 281 | * Values for "read": |
275 | * | 282 | * |
276 | * 0: exclusive (write) acquire | 283 | * 0: exclusive (write) acquire |
277 | * 1: read-acquire (no recursion allowed) | 284 | * 1: read-acquire (no recursion allowed) |
278 | * 2: read-acquire with same-instance recursion allowed | 285 | * 2: read-acquire with same-instance recursion allowed |
279 | * | 286 | * |
280 | * Values for check: | 287 | * Values for check: |
281 | * | 288 | * |
282 | * 0: disabled | 289 | * 0: disabled |
283 | * 1: simple checks (freeing, held-at-exit-time, etc.) | 290 | * 1: simple checks (freeing, held-at-exit-time, etc.) |
284 | * 2: full validation | 291 | * 2: full validation |
285 | */ | 292 | */ |
286 | extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass, | 293 | extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass, |
287 | int trylock, int read, int check, unsigned long ip); | 294 | int trylock, int read, int check, unsigned long ip); |
288 | 295 | ||
289 | extern void lock_release(struct lockdep_map *lock, int nested, | 296 | extern void lock_release(struct lockdep_map *lock, int nested, |
290 | unsigned long ip); | 297 | unsigned long ip); |
291 | 298 | ||
292 | # define INIT_LOCKDEP .lockdep_recursion = 0, | 299 | # define INIT_LOCKDEP .lockdep_recursion = 0, |
293 | 300 | ||
294 | #define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0) | 301 | #define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0) |
295 | 302 | ||
296 | #else /* !LOCKDEP */ | 303 | #else /* !LOCKDEP */ |
297 | 304 | ||
298 | static inline void lockdep_off(void) | 305 | static inline void lockdep_off(void) |
299 | { | 306 | { |
300 | } | 307 | } |
301 | 308 | ||
302 | static inline void lockdep_on(void) | 309 | static inline void lockdep_on(void) |
303 | { | 310 | { |
304 | } | 311 | } |
305 | 312 | ||
306 | # define lock_acquire(l, s, t, r, c, i) do { } while (0) | 313 | # define lock_acquire(l, s, t, r, c, i) do { } while (0) |
307 | # define lock_release(l, n, i) do { } while (0) | 314 | # define lock_release(l, n, i) do { } while (0) |
308 | # define lockdep_init() do { } while (0) | 315 | # define lockdep_init() do { } while (0) |
309 | # define lockdep_info() do { } while (0) | 316 | # define lockdep_info() do { } while (0) |
310 | # define lockdep_init_map(lock, name, key, sub) do { (void)(key); } while (0) | 317 | # define lockdep_init_map(lock, name, key, sub) do { (void)(key); } while (0) |
311 | # define lockdep_set_class(lock, key) do { (void)(key); } while (0) | 318 | # define lockdep_set_class(lock, key) do { (void)(key); } while (0) |
312 | # define lockdep_set_class_and_name(lock, key, name) \ | 319 | # define lockdep_set_class_and_name(lock, key, name) \ |
313 | do { (void)(key); } while (0) | 320 | do { (void)(key); } while (0) |
314 | #define lockdep_set_class_and_subclass(lock, key, sub) \ | 321 | #define lockdep_set_class_and_subclass(lock, key, sub) \ |
315 | do { (void)(key); } while (0) | 322 | do { (void)(key); } while (0) |
316 | #define lockdep_set_subclass(lock, sub) do { } while (0) | 323 | #define lockdep_set_subclass(lock, sub) do { } while (0) |
317 | 324 | ||
318 | # define INIT_LOCKDEP | 325 | # define INIT_LOCKDEP |
319 | # define lockdep_reset() do { debug_locks = 1; } while (0) | 326 | # define lockdep_reset() do { debug_locks = 1; } while (0) |
320 | # define lockdep_free_key_range(start, size) do { } while (0) | 327 | # define lockdep_free_key_range(start, size) do { } while (0) |
321 | # define lockdep_sys_exit() do { } while (0) | 328 | # define lockdep_sys_exit() do { } while (0) |
322 | /* | 329 | /* |
323 | * The class key takes no space if lockdep is disabled: | 330 | * The class key takes no space if lockdep is disabled: |
324 | */ | 331 | */ |
325 | struct lock_class_key { }; | 332 | struct lock_class_key { }; |
326 | 333 | ||
327 | #define lockdep_depth(tsk) (0) | 334 | #define lockdep_depth(tsk) (0) |
328 | 335 | ||
329 | #endif /* !LOCKDEP */ | 336 | #endif /* !LOCKDEP */ |
330 | 337 | ||
331 | #ifdef CONFIG_LOCK_STAT | 338 | #ifdef CONFIG_LOCK_STAT |
332 | 339 | ||
333 | extern void lock_contended(struct lockdep_map *lock, unsigned long ip); | 340 | extern void lock_contended(struct lockdep_map *lock, unsigned long ip); |
334 | extern void lock_acquired(struct lockdep_map *lock); | 341 | extern void lock_acquired(struct lockdep_map *lock); |
335 | 342 | ||
336 | #define LOCK_CONTENDED(_lock, try, lock) \ | 343 | #define LOCK_CONTENDED(_lock, try, lock) \ |
337 | do { \ | 344 | do { \ |
338 | if (!try(_lock)) { \ | 345 | if (!try(_lock)) { \ |
339 | lock_contended(&(_lock)->dep_map, _RET_IP_); \ | 346 | lock_contended(&(_lock)->dep_map, _RET_IP_); \ |
340 | lock(_lock); \ | 347 | lock(_lock); \ |
341 | } \ | 348 | } \ |
342 | lock_acquired(&(_lock)->dep_map); \ | 349 | lock_acquired(&(_lock)->dep_map); \ |
343 | } while (0) | 350 | } while (0) |
344 | 351 | ||
345 | #else /* CONFIG_LOCK_STAT */ | 352 | #else /* CONFIG_LOCK_STAT */ |
346 | 353 | ||
347 | #define lock_contended(lockdep_map, ip) do {} while (0) | 354 | #define lock_contended(lockdep_map, ip) do {} while (0) |
348 | #define lock_acquired(lockdep_map) do {} while (0) | 355 | #define lock_acquired(lockdep_map) do {} while (0) |
349 | 356 | ||
350 | #define LOCK_CONTENDED(_lock, try, lock) \ | 357 | #define LOCK_CONTENDED(_lock, try, lock) \ |
351 | lock(_lock) | 358 | lock(_lock) |
352 | 359 | ||
353 | #endif /* CONFIG_LOCK_STAT */ | 360 | #endif /* CONFIG_LOCK_STAT */ |
354 | 361 | ||
355 | #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_GENERIC_HARDIRQS) | 362 | #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_GENERIC_HARDIRQS) |
356 | extern void early_init_irq_lock_class(void); | 363 | extern void early_init_irq_lock_class(void); |
357 | #else | 364 | #else |
358 | static inline void early_init_irq_lock_class(void) | 365 | static inline void early_init_irq_lock_class(void) |
359 | { | 366 | { |
360 | } | 367 | } |
361 | #endif | 368 | #endif |
362 | 369 | ||
363 | #ifdef CONFIG_TRACE_IRQFLAGS | 370 | #ifdef CONFIG_TRACE_IRQFLAGS |
364 | extern void early_boot_irqs_off(void); | 371 | extern void early_boot_irqs_off(void); |
365 | extern void early_boot_irqs_on(void); | 372 | extern void early_boot_irqs_on(void); |
366 | extern void print_irqtrace_events(struct task_struct *curr); | 373 | extern void print_irqtrace_events(struct task_struct *curr); |
367 | #else | 374 | #else |
368 | static inline void early_boot_irqs_off(void) | 375 | static inline void early_boot_irqs_off(void) |
369 | { | 376 | { |
370 | } | 377 | } |
371 | static inline void early_boot_irqs_on(void) | 378 | static inline void early_boot_irqs_on(void) |
372 | { | 379 | { |
373 | } | 380 | } |
374 | static inline void print_irqtrace_events(struct task_struct *curr) | 381 | static inline void print_irqtrace_events(struct task_struct *curr) |
375 | { | 382 | { |
376 | } | 383 | } |
377 | #endif | 384 | #endif |
378 | 385 | ||
379 | /* | 386 | /* |
380 | * For trivial one-depth nesting of a lock-class, the following | 387 | * For trivial one-depth nesting of a lock-class, the following |
381 | * global define can be used. (Subsystems with multiple levels | 388 | * global define can be used. (Subsystems with multiple levels |
382 | * of nesting should define their own lock-nesting subclasses.) | 389 | * of nesting should define their own lock-nesting subclasses.) |
383 | */ | 390 | */ |
384 | #define SINGLE_DEPTH_NESTING 1 | 391 | #define SINGLE_DEPTH_NESTING 1 |
385 | 392 | ||
386 | /* | 393 | /* |
387 | * Map the dependency ops to NOP or to real lockdep ops, depending | 394 | * Map the dependency ops to NOP or to real lockdep ops, depending |
388 | * on the per lock-class debug mode: | 395 | * on the per lock-class debug mode: |
389 | */ | 396 | */ |
390 | 397 | ||
391 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 398 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
392 | # ifdef CONFIG_PROVE_LOCKING | 399 | # ifdef CONFIG_PROVE_LOCKING |
393 | # define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) | 400 | # define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) |
394 | # else | 401 | # else |
395 | # define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) | 402 | # define spin_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) |
396 | # endif | 403 | # endif |
397 | # define spin_release(l, n, i) lock_release(l, n, i) | 404 | # define spin_release(l, n, i) lock_release(l, n, i) |
398 | #else | 405 | #else |
399 | # define spin_acquire(l, s, t, i) do { } while (0) | 406 | # define spin_acquire(l, s, t, i) do { } while (0) |
400 | # define spin_release(l, n, i) do { } while (0) | 407 | # define spin_release(l, n, i) do { } while (0) |
401 | #endif | 408 | #endif |
402 | 409 | ||
403 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 410 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
404 | # ifdef CONFIG_PROVE_LOCKING | 411 | # ifdef CONFIG_PROVE_LOCKING |
405 | # define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) | 412 | # define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) |
406 | # define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 2, i) | 413 | # define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 2, i) |
407 | # else | 414 | # else |
408 | # define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) | 415 | # define rwlock_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) |
409 | # define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 1, i) | 416 | # define rwlock_acquire_read(l, s, t, i) lock_acquire(l, s, t, 2, 1, i) |
410 | # endif | 417 | # endif |
411 | # define rwlock_release(l, n, i) lock_release(l, n, i) | 418 | # define rwlock_release(l, n, i) lock_release(l, n, i) |
412 | #else | 419 | #else |
413 | # define rwlock_acquire(l, s, t, i) do { } while (0) | 420 | # define rwlock_acquire(l, s, t, i) do { } while (0) |
414 | # define rwlock_acquire_read(l, s, t, i) do { } while (0) | 421 | # define rwlock_acquire_read(l, s, t, i) do { } while (0) |
415 | # define rwlock_release(l, n, i) do { } while (0) | 422 | # define rwlock_release(l, n, i) do { } while (0) |
416 | #endif | 423 | #endif |
417 | 424 | ||
418 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 425 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
419 | # ifdef CONFIG_PROVE_LOCKING | 426 | # ifdef CONFIG_PROVE_LOCKING |
420 | # define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) | 427 | # define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) |
421 | # else | 428 | # else |
422 | # define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) | 429 | # define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) |
423 | # endif | 430 | # endif |
424 | # define mutex_release(l, n, i) lock_release(l, n, i) | 431 | # define mutex_release(l, n, i) lock_release(l, n, i) |
425 | #else | 432 | #else |
426 | # define mutex_acquire(l, s, t, i) do { } while (0) | 433 | # define mutex_acquire(l, s, t, i) do { } while (0) |
427 | # define mutex_release(l, n, i) do { } while (0) | 434 | # define mutex_release(l, n, i) do { } while (0) |
428 | #endif | 435 | #endif |
429 | 436 | ||
430 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | 437 | #ifdef CONFIG_DEBUG_LOCK_ALLOC |
431 | # ifdef CONFIG_PROVE_LOCKING | 438 | # ifdef CONFIG_PROVE_LOCKING |
432 | # define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) | 439 | # define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, i) |
433 | # define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 2, i) | 440 | # define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 2, i) |
434 | # else | 441 | # else |
435 | # define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) | 442 | # define rwsem_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, i) |
436 | # define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 1, i) | 443 | # define rwsem_acquire_read(l, s, t, i) lock_acquire(l, s, t, 1, 1, i) |
437 | # endif | 444 | # endif |
438 | # define rwsem_release(l, n, i) lock_release(l, n, i) | 445 | # define rwsem_release(l, n, i) lock_release(l, n, i) |
439 | #else | 446 | #else |
440 | # define rwsem_acquire(l, s, t, i) do { } while (0) | 447 | # define rwsem_acquire(l, s, t, i) do { } while (0) |
441 | # define rwsem_acquire_read(l, s, t, i) do { } while (0) | 448 | # define rwsem_acquire_read(l, s, t, i) do { } while (0) |
442 | # define rwsem_release(l, n, i) do { } while (0) | 449 | # define rwsem_release(l, n, i) do { } while (0) |
443 | #endif | 450 | #endif |
444 | 451 | ||
445 | #endif /* __LINUX_LOCKDEP_H */ | 452 | #endif /* __LINUX_LOCKDEP_H */ |
446 | 453 |
include/linux/rcupdate.h
1 | /* | 1 | /* |
2 | * Read-Copy Update mechanism for mutual exclusion | 2 | * Read-Copy Update mechanism for mutual exclusion |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify | 4 | * This program is free software; you can redistribute it and/or modify |
5 | * it under the terms of the GNU General Public License as published by | 5 | * it under the terms of the GNU General Public License as published by |
6 | * the Free Software Foundation; either version 2 of the License, or | 6 | * the Free Software Foundation; either version 2 of the License, or |
7 | * (at your option) any later version. | 7 | * (at your option) any later version. |
8 | * | 8 | * |
9 | * This program is distributed in the hope that it will be useful, | 9 | * This program is distributed in the hope that it will be useful, |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 | * GNU General Public License for more details. | 12 | * GNU General Public License for more details. |
13 | * | 13 | * |
14 | * You should have received a copy of the GNU General Public License | 14 | * You should have received a copy of the GNU General Public License |
15 | * along with this program; if not, write to the Free Software | 15 | * along with this program; if not, write to the Free Software |
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | 16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
17 | * | 17 | * |
18 | * Copyright (C) IBM Corporation, 2001 | 18 | * Copyright (C) IBM Corporation, 2001 |
19 | * | 19 | * |
20 | * Author: Dipankar Sarma <dipankar@in.ibm.com> | 20 | * Author: Dipankar Sarma <dipankar@in.ibm.com> |
21 | * | 21 | * |
22 | * Based on the original work by Paul McKenney <paulmck@us.ibm.com> | 22 | * Based on the original work by Paul McKenney <paulmck@us.ibm.com> |
23 | * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. | 23 | * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. |
24 | * Papers: | 24 | * Papers: |
25 | * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf | 25 | * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf |
26 | * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001) | 26 | * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001) |
27 | * | 27 | * |
28 | * For detailed explanation of Read-Copy Update mechanism see - | 28 | * For detailed explanation of Read-Copy Update mechanism see - |
29 | * http://lse.sourceforge.net/locking/rcupdate.html | 29 | * http://lse.sourceforge.net/locking/rcupdate.html |
30 | * | 30 | * |
31 | */ | 31 | */ |
32 | 32 | ||
33 | #ifndef __LINUX_RCUPDATE_H | 33 | #ifndef __LINUX_RCUPDATE_H |
34 | #define __LINUX_RCUPDATE_H | 34 | #define __LINUX_RCUPDATE_H |
35 | 35 | ||
36 | #ifdef __KERNEL__ | 36 | #ifdef __KERNEL__ |
37 | 37 | ||
38 | #include <linux/cache.h> | 38 | #include <linux/cache.h> |
39 | #include <linux/spinlock.h> | 39 | #include <linux/spinlock.h> |
40 | #include <linux/threads.h> | 40 | #include <linux/threads.h> |
41 | #include <linux/percpu.h> | 41 | #include <linux/percpu.h> |
42 | #include <linux/cpumask.h> | 42 | #include <linux/cpumask.h> |
43 | #include <linux/seqlock.h> | 43 | #include <linux/seqlock.h> |
44 | #include <linux/lockdep.h> | ||
44 | 45 | ||
45 | /** | 46 | /** |
46 | * struct rcu_head - callback structure for use with RCU | 47 | * struct rcu_head - callback structure for use with RCU |
47 | * @next: next update requests in a list | 48 | * @next: next update requests in a list |
48 | * @func: actual update function to call after the grace period. | 49 | * @func: actual update function to call after the grace period. |
49 | */ | 50 | */ |
50 | struct rcu_head { | 51 | struct rcu_head { |
51 | struct rcu_head *next; | 52 | struct rcu_head *next; |
52 | void (*func)(struct rcu_head *head); | 53 | void (*func)(struct rcu_head *head); |
53 | }; | 54 | }; |
54 | 55 | ||
55 | #define RCU_HEAD_INIT { .next = NULL, .func = NULL } | 56 | #define RCU_HEAD_INIT { .next = NULL, .func = NULL } |
56 | #define RCU_HEAD(head) struct rcu_head head = RCU_HEAD_INIT | 57 | #define RCU_HEAD(head) struct rcu_head head = RCU_HEAD_INIT |
57 | #define INIT_RCU_HEAD(ptr) do { \ | 58 | #define INIT_RCU_HEAD(ptr) do { \ |
58 | (ptr)->next = NULL; (ptr)->func = NULL; \ | 59 | (ptr)->next = NULL; (ptr)->func = NULL; \ |
59 | } while (0) | 60 | } while (0) |
60 | 61 | ||
61 | 62 | ||
62 | 63 | ||
63 | /* Global control variables for rcupdate callback mechanism. */ | 64 | /* Global control variables for rcupdate callback mechanism. */ |
64 | struct rcu_ctrlblk { | 65 | struct rcu_ctrlblk { |
65 | long cur; /* Current batch number. */ | 66 | long cur; /* Current batch number. */ |
66 | long completed; /* Number of the last completed batch */ | 67 | long completed; /* Number of the last completed batch */ |
67 | int next_pending; /* Is the next batch already waiting? */ | 68 | int next_pending; /* Is the next batch already waiting? */ |
68 | 69 | ||
69 | int signaled; | 70 | int signaled; |
70 | 71 | ||
71 | spinlock_t lock ____cacheline_internodealigned_in_smp; | 72 | spinlock_t lock ____cacheline_internodealigned_in_smp; |
72 | cpumask_t cpumask; /* CPUs that need to switch in order */ | 73 | cpumask_t cpumask; /* CPUs that need to switch in order */ |
73 | /* for current batch to proceed. */ | 74 | /* for current batch to proceed. */ |
74 | } ____cacheline_internodealigned_in_smp; | 75 | } ____cacheline_internodealigned_in_smp; |
75 | 76 | ||
76 | /* Is batch a before batch b ? */ | 77 | /* Is batch a before batch b ? */ |
77 | static inline int rcu_batch_before(long a, long b) | 78 | static inline int rcu_batch_before(long a, long b) |
78 | { | 79 | { |
79 | return (a - b) < 0; | 80 | return (a - b) < 0; |
80 | } | 81 | } |
81 | 82 | ||
82 | /* Is batch a after batch b ? */ | 83 | /* Is batch a after batch b ? */ |
83 | static inline int rcu_batch_after(long a, long b) | 84 | static inline int rcu_batch_after(long a, long b) |
84 | { | 85 | { |
85 | return (a - b) > 0; | 86 | return (a - b) > 0; |
86 | } | 87 | } |
87 | 88 | ||
88 | /* | 89 | /* |
89 | * Per-CPU data for Read-Copy UPdate. | 90 | * Per-CPU data for Read-Copy UPdate. |
90 | * nxtlist - new callbacks are added here | 91 | * nxtlist - new callbacks are added here |
91 | * curlist - current batch for which quiescent cycle started if any | 92 | * curlist - current batch for which quiescent cycle started if any |
92 | */ | 93 | */ |
93 | struct rcu_data { | 94 | struct rcu_data { |
94 | /* 1) quiescent state handling : */ | 95 | /* 1) quiescent state handling : */ |
95 | long quiescbatch; /* Batch # for grace period */ | 96 | long quiescbatch; /* Batch # for grace period */ |
96 | int passed_quiesc; /* User-mode/idle loop etc. */ | 97 | int passed_quiesc; /* User-mode/idle loop etc. */ |
97 | int qs_pending; /* core waits for quiesc state */ | 98 | int qs_pending; /* core waits for quiesc state */ |
98 | 99 | ||
99 | /* 2) batch handling */ | 100 | /* 2) batch handling */ |
100 | long batch; /* Batch # for current RCU batch */ | 101 | long batch; /* Batch # for current RCU batch */ |
101 | struct rcu_head *nxtlist; | 102 | struct rcu_head *nxtlist; |
102 | struct rcu_head **nxttail; | 103 | struct rcu_head **nxttail; |
103 | long qlen; /* # of queued callbacks */ | 104 | long qlen; /* # of queued callbacks */ |
104 | struct rcu_head *curlist; | 105 | struct rcu_head *curlist; |
105 | struct rcu_head **curtail; | 106 | struct rcu_head **curtail; |
106 | struct rcu_head *donelist; | 107 | struct rcu_head *donelist; |
107 | struct rcu_head **donetail; | 108 | struct rcu_head **donetail; |
108 | long blimit; /* Upper limit on a processed batch */ | 109 | long blimit; /* Upper limit on a processed batch */ |
109 | int cpu; | 110 | int cpu; |
110 | struct rcu_head barrier; | 111 | struct rcu_head barrier; |
111 | }; | 112 | }; |
112 | 113 | ||
113 | DECLARE_PER_CPU(struct rcu_data, rcu_data); | 114 | DECLARE_PER_CPU(struct rcu_data, rcu_data); |
114 | DECLARE_PER_CPU(struct rcu_data, rcu_bh_data); | 115 | DECLARE_PER_CPU(struct rcu_data, rcu_bh_data); |
115 | 116 | ||
116 | /* | 117 | /* |
117 | * Increment the quiescent state counter. | 118 | * Increment the quiescent state counter. |
118 | * The counter is a bit degenerated: We do not need to know | 119 | * The counter is a bit degenerated: We do not need to know |
119 | * how many quiescent states passed, just if there was at least | 120 | * how many quiescent states passed, just if there was at least |
120 | * one since the start of the grace period. Thus just a flag. | 121 | * one since the start of the grace period. Thus just a flag. |
121 | */ | 122 | */ |
122 | static inline void rcu_qsctr_inc(int cpu) | 123 | static inline void rcu_qsctr_inc(int cpu) |
123 | { | 124 | { |
124 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); | 125 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); |
125 | rdp->passed_quiesc = 1; | 126 | rdp->passed_quiesc = 1; |
126 | } | 127 | } |
127 | static inline void rcu_bh_qsctr_inc(int cpu) | 128 | static inline void rcu_bh_qsctr_inc(int cpu) |
128 | { | 129 | { |
129 | struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); | 130 | struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu); |
130 | rdp->passed_quiesc = 1; | 131 | rdp->passed_quiesc = 1; |
131 | } | 132 | } |
132 | 133 | ||
133 | extern int rcu_pending(int cpu); | 134 | extern int rcu_pending(int cpu); |
134 | extern int rcu_needs_cpu(int cpu); | 135 | extern int rcu_needs_cpu(int cpu); |
135 | 136 | ||
137 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
138 | extern struct lockdep_map rcu_lock_map; | ||
139 | # define rcu_read_acquire() lock_acquire(&rcu_lock_map, 0, 0, 2, 1, _THIS_IP_) | ||
140 | # define rcu_read_release() lock_release(&rcu_lock_map, 1, _THIS_IP_) | ||
141 | #else | ||
142 | # define rcu_read_acquire() do { } while (0) | ||
143 | # define rcu_read_release() do { } while (0) | ||
144 | #endif | ||
145 | |||
136 | /** | 146 | /** |
137 | * rcu_read_lock - mark the beginning of an RCU read-side critical section. | 147 | * rcu_read_lock - mark the beginning of an RCU read-side critical section. |
138 | * | 148 | * |
139 | * When synchronize_rcu() is invoked on one CPU while other CPUs | 149 | * When synchronize_rcu() is invoked on one CPU while other CPUs |
140 | * are within RCU read-side critical sections, then the | 150 | * are within RCU read-side critical sections, then the |
141 | * synchronize_rcu() is guaranteed to block until after all the other | 151 | * synchronize_rcu() is guaranteed to block until after all the other |
142 | * CPUs exit their critical sections. Similarly, if call_rcu() is invoked | 152 | * CPUs exit their critical sections. Similarly, if call_rcu() is invoked |
143 | * on one CPU while other CPUs are within RCU read-side critical | 153 | * on one CPU while other CPUs are within RCU read-side critical |
144 | * sections, invocation of the corresponding RCU callback is deferred | 154 | * sections, invocation of the corresponding RCU callback is deferred |
145 | * until after the all the other CPUs exit their critical sections. | 155 | * until after the all the other CPUs exit their critical sections. |
146 | * | 156 | * |
147 | * Note, however, that RCU callbacks are permitted to run concurrently | 157 | * Note, however, that RCU callbacks are permitted to run concurrently |
148 | * with RCU read-side critical sections. One way that this can happen | 158 | * with RCU read-side critical sections. One way that this can happen |
149 | * is via the following sequence of events: (1) CPU 0 enters an RCU | 159 | * is via the following sequence of events: (1) CPU 0 enters an RCU |
150 | * read-side critical section, (2) CPU 1 invokes call_rcu() to register | 160 | * read-side critical section, (2) CPU 1 invokes call_rcu() to register |
151 | * an RCU callback, (3) CPU 0 exits the RCU read-side critical section, | 161 | * an RCU callback, (3) CPU 0 exits the RCU read-side critical section, |
152 | * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU | 162 | * (4) CPU 2 enters a RCU read-side critical section, (5) the RCU |
153 | * callback is invoked. This is legal, because the RCU read-side critical | 163 | * callback is invoked. This is legal, because the RCU read-side critical |
154 | * section that was running concurrently with the call_rcu() (and which | 164 | * section that was running concurrently with the call_rcu() (and which |
155 | * therefore might be referencing something that the corresponding RCU | 165 | * therefore might be referencing something that the corresponding RCU |
156 | * callback would free up) has completed before the corresponding | 166 | * callback would free up) has completed before the corresponding |
157 | * RCU callback is invoked. | 167 | * RCU callback is invoked. |
158 | * | 168 | * |
159 | * RCU read-side critical sections may be nested. Any deferred actions | 169 | * RCU read-side critical sections may be nested. Any deferred actions |
160 | * will be deferred until the outermost RCU read-side critical section | 170 | * will be deferred until the outermost RCU read-side critical section |
161 | * completes. | 171 | * completes. |
162 | * | 172 | * |
163 | * It is illegal to block while in an RCU read-side critical section. | 173 | * It is illegal to block while in an RCU read-side critical section. |
164 | */ | 174 | */ |
165 | #define rcu_read_lock() \ | 175 | #define rcu_read_lock() \ |
166 | do { \ | 176 | do { \ |
167 | preempt_disable(); \ | 177 | preempt_disable(); \ |
168 | __acquire(RCU); \ | 178 | __acquire(RCU); \ |
179 | rcu_read_acquire(); \ | ||
169 | } while(0) | 180 | } while(0) |
170 | 181 | ||
171 | /** | 182 | /** |
172 | * rcu_read_unlock - marks the end of an RCU read-side critical section. | 183 | * rcu_read_unlock - marks the end of an RCU read-side critical section. |
173 | * | 184 | * |
174 | * See rcu_read_lock() for more information. | 185 | * See rcu_read_lock() for more information. |
175 | */ | 186 | */ |
176 | #define rcu_read_unlock() \ | 187 | #define rcu_read_unlock() \ |
177 | do { \ | 188 | do { \ |
189 | rcu_read_release(); \ | ||
178 | __release(RCU); \ | 190 | __release(RCU); \ |
179 | preempt_enable(); \ | 191 | preempt_enable(); \ |
180 | } while(0) | 192 | } while(0) |
181 | 193 | ||
182 | /* | 194 | /* |
183 | * So where is rcu_write_lock()? It does not exist, as there is no | 195 | * So where is rcu_write_lock()? It does not exist, as there is no |
184 | * way for writers to lock out RCU readers. This is a feature, not | 196 | * way for writers to lock out RCU readers. This is a feature, not |
185 | * a bug -- this property is what provides RCU's performance benefits. | 197 | * a bug -- this property is what provides RCU's performance benefits. |
186 | * Of course, writers must coordinate with each other. The normal | 198 | * Of course, writers must coordinate with each other. The normal |
187 | * spinlock primitives work well for this, but any other technique may be | 199 | * spinlock primitives work well for this, but any other technique may be |
188 | * used as well. RCU does not care how the writers keep out of each | 200 | * used as well. RCU does not care how the writers keep out of each |
189 | * others' way, as long as they do so. | 201 | * others' way, as long as they do so. |
190 | */ | 202 | */ |
191 | 203 | ||
192 | /** | 204 | /** |
193 | * rcu_read_lock_bh - mark the beginning of a softirq-only RCU critical section | 205 | * rcu_read_lock_bh - mark the beginning of a softirq-only RCU critical section |
194 | * | 206 | * |
195 | * This is equivalent of rcu_read_lock(), but to be used when updates | 207 | * This is equivalent of rcu_read_lock(), but to be used when updates |
196 | * are being done using call_rcu_bh(). Since call_rcu_bh() callbacks | 208 | * are being done using call_rcu_bh(). Since call_rcu_bh() callbacks |
197 | * consider completion of a softirq handler to be a quiescent state, | 209 | * consider completion of a softirq handler to be a quiescent state, |
198 | * a process in RCU read-side critical section must be protected by | 210 | * a process in RCU read-side critical section must be protected by |
199 | * disabling softirqs. Read-side critical sections in interrupt context | 211 | * disabling softirqs. Read-side critical sections in interrupt context |
200 | * can use just rcu_read_lock(). | 212 | * can use just rcu_read_lock(). |
201 | * | 213 | * |
202 | */ | 214 | */ |
203 | #define rcu_read_lock_bh() \ | 215 | #define rcu_read_lock_bh() \ |
204 | do { \ | 216 | do { \ |
205 | local_bh_disable(); \ | 217 | local_bh_disable(); \ |
206 | __acquire(RCU_BH); \ | 218 | __acquire(RCU_BH); \ |
219 | rcu_read_acquire(); \ | ||
207 | } while(0) | 220 | } while(0) |
208 | 221 | ||
209 | /* | 222 | /* |
210 | * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section | 223 | * rcu_read_unlock_bh - marks the end of a softirq-only RCU critical section |
211 | * | 224 | * |
212 | * See rcu_read_lock_bh() for more information. | 225 | * See rcu_read_lock_bh() for more information. |
213 | */ | 226 | */ |
214 | #define rcu_read_unlock_bh() \ | 227 | #define rcu_read_unlock_bh() \ |
215 | do { \ | 228 | do { \ |
229 | rcu_read_release(); \ | ||
216 | __release(RCU_BH); \ | 230 | __release(RCU_BH); \ |
217 | local_bh_enable(); \ | 231 | local_bh_enable(); \ |
218 | } while(0) | 232 | } while(0) |
219 | 233 | ||
220 | /** | 234 | /** |
221 | * rcu_dereference - fetch an RCU-protected pointer in an | 235 | * rcu_dereference - fetch an RCU-protected pointer in an |
222 | * RCU read-side critical section. This pointer may later | 236 | * RCU read-side critical section. This pointer may later |
223 | * be safely dereferenced. | 237 | * be safely dereferenced. |
224 | * | 238 | * |
225 | * Inserts memory barriers on architectures that require them | 239 | * Inserts memory barriers on architectures that require them |
226 | * (currently only the Alpha), and, more importantly, documents | 240 | * (currently only the Alpha), and, more importantly, documents |
227 | * exactly which pointers are protected by RCU. | 241 | * exactly which pointers are protected by RCU. |
228 | */ | 242 | */ |
229 | 243 | ||
230 | #define rcu_dereference(p) ({ \ | 244 | #define rcu_dereference(p) ({ \ |
231 | typeof(p) _________p1 = p; \ | 245 | typeof(p) _________p1 = p; \ |
232 | smp_read_barrier_depends(); \ | 246 | smp_read_barrier_depends(); \ |
233 | (_________p1); \ | 247 | (_________p1); \ |
234 | }) | 248 | }) |
235 | 249 | ||
236 | /** | 250 | /** |
237 | * rcu_assign_pointer - assign (publicize) a pointer to a newly | 251 | * rcu_assign_pointer - assign (publicize) a pointer to a newly |
238 | * initialized structure that will be dereferenced by RCU read-side | 252 | * initialized structure that will be dereferenced by RCU read-side |
239 | * critical sections. Returns the value assigned. | 253 | * critical sections. Returns the value assigned. |
240 | * | 254 | * |
241 | * Inserts memory barriers on architectures that require them | 255 | * Inserts memory barriers on architectures that require them |
242 | * (pretty much all of them other than x86), and also prevents | 256 | * (pretty much all of them other than x86), and also prevents |
243 | * the compiler from reordering the code that initializes the | 257 | * the compiler from reordering the code that initializes the |
244 | * structure after the pointer assignment. More importantly, this | 258 | * structure after the pointer assignment. More importantly, this |
245 | * call documents which pointers will be dereferenced by RCU read-side | 259 | * call documents which pointers will be dereferenced by RCU read-side |
246 | * code. | 260 | * code. |
247 | */ | 261 | */ |
248 | 262 | ||
249 | #define rcu_assign_pointer(p, v) ({ \ | 263 | #define rcu_assign_pointer(p, v) ({ \ |
250 | smp_wmb(); \ | 264 | smp_wmb(); \ |
251 | (p) = (v); \ | 265 | (p) = (v); \ |
252 | }) | 266 | }) |
253 | 267 | ||
254 | /** | 268 | /** |
255 | * synchronize_sched - block until all CPUs have exited any non-preemptive | 269 | * synchronize_sched - block until all CPUs have exited any non-preemptive |
256 | * kernel code sequences. | 270 | * kernel code sequences. |
257 | * | 271 | * |
258 | * This means that all preempt_disable code sequences, including NMI and | 272 | * This means that all preempt_disable code sequences, including NMI and |
259 | * hardware-interrupt handlers, in progress on entry will have completed | 273 | * hardware-interrupt handlers, in progress on entry will have completed |
260 | * before this primitive returns. However, this does not guarantee that | 274 | * before this primitive returns. However, this does not guarantee that |
261 | * softirq handlers will have completed, since in some kernels, these | 275 | * softirq handlers will have completed, since in some kernels, these |
262 | * handlers can run in process context, and can block. | 276 | * handlers can run in process context, and can block. |
263 | * | 277 | * |
264 | * This primitive provides the guarantees made by the (now removed) | 278 | * This primitive provides the guarantees made by the (now removed) |
265 | * synchronize_kernel() API. In contrast, synchronize_rcu() only | 279 | * synchronize_kernel() API. In contrast, synchronize_rcu() only |
266 | * guarantees that rcu_read_lock() sections will have completed. | 280 | * guarantees that rcu_read_lock() sections will have completed. |
267 | * In "classic RCU", these two guarantees happen to be one and | 281 | * In "classic RCU", these two guarantees happen to be one and |
268 | * the same, but can differ in realtime RCU implementations. | 282 | * the same, but can differ in realtime RCU implementations. |
269 | */ | 283 | */ |
270 | #define synchronize_sched() synchronize_rcu() | 284 | #define synchronize_sched() synchronize_rcu() |
271 | 285 | ||
272 | extern void rcu_init(void); | 286 | extern void rcu_init(void); |
273 | extern void rcu_check_callbacks(int cpu, int user); | 287 | extern void rcu_check_callbacks(int cpu, int user); |
274 | extern void rcu_restart_cpu(int cpu); | 288 | extern void rcu_restart_cpu(int cpu); |
275 | extern long rcu_batches_completed(void); | 289 | extern long rcu_batches_completed(void); |
276 | extern long rcu_batches_completed_bh(void); | 290 | extern long rcu_batches_completed_bh(void); |
277 | 291 | ||
278 | /* Exported interfaces */ | 292 | /* Exported interfaces */ |
279 | extern void FASTCALL(call_rcu(struct rcu_head *head, | 293 | extern void FASTCALL(call_rcu(struct rcu_head *head, |
280 | void (*func)(struct rcu_head *head))); | 294 | void (*func)(struct rcu_head *head))); |
281 | extern void FASTCALL(call_rcu_bh(struct rcu_head *head, | 295 | extern void FASTCALL(call_rcu_bh(struct rcu_head *head, |
282 | void (*func)(struct rcu_head *head))); | 296 | void (*func)(struct rcu_head *head))); |
283 | extern void synchronize_rcu(void); | 297 | extern void synchronize_rcu(void); |
284 | extern void rcu_barrier(void); | 298 | extern void rcu_barrier(void); |
285 | 299 | ||
286 | #endif /* __KERNEL__ */ | 300 | #endif /* __KERNEL__ */ |
287 | #endif /* __LINUX_RCUPDATE_H */ | 301 | #endif /* __LINUX_RCUPDATE_H */ |
288 | 302 |
kernel/rcupdate.c
1 | /* | 1 | /* |
2 | * Read-Copy Update mechanism for mutual exclusion | 2 | * Read-Copy Update mechanism for mutual exclusion |
3 | * | 3 | * |
4 | * This program is free software; you can redistribute it and/or modify | 4 | * This program is free software; you can redistribute it and/or modify |
5 | * it under the terms of the GNU General Public License as published by | 5 | * it under the terms of the GNU General Public License as published by |
6 | * the Free Software Foundation; either version 2 of the License, or | 6 | * the Free Software Foundation; either version 2 of the License, or |
7 | * (at your option) any later version. | 7 | * (at your option) any later version. |
8 | * | 8 | * |
9 | * This program is distributed in the hope that it will be useful, | 9 | * This program is distributed in the hope that it will be useful, |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
12 | * GNU General Public License for more details. | 12 | * GNU General Public License for more details. |
13 | * | 13 | * |
14 | * You should have received a copy of the GNU General Public License | 14 | * You should have received a copy of the GNU General Public License |
15 | * along with this program; if not, write to the Free Software | 15 | * along with this program; if not, write to the Free Software |
16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | 16 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. |
17 | * | 17 | * |
18 | * Copyright (C) IBM Corporation, 2001 | 18 | * Copyright (C) IBM Corporation, 2001 |
19 | * | 19 | * |
20 | * Authors: Dipankar Sarma <dipankar@in.ibm.com> | 20 | * Authors: Dipankar Sarma <dipankar@in.ibm.com> |
21 | * Manfred Spraul <manfred@colorfullife.com> | 21 | * Manfred Spraul <manfred@colorfullife.com> |
22 | * | 22 | * |
23 | * Based on the original work by Paul McKenney <paulmck@us.ibm.com> | 23 | * Based on the original work by Paul McKenney <paulmck@us.ibm.com> |
24 | * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. | 24 | * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen. |
25 | * Papers: | 25 | * Papers: |
26 | * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf | 26 | * http://www.rdrop.com/users/paulmck/paper/rclockpdcsproof.pdf |
27 | * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001) | 27 | * http://lse.sourceforge.net/locking/rclock_OLS.2001.05.01c.sc.pdf (OLS2001) |
28 | * | 28 | * |
29 | * For detailed explanation of Read-Copy Update mechanism see - | 29 | * For detailed explanation of Read-Copy Update mechanism see - |
30 | * http://lse.sourceforge.net/locking/rcupdate.html | 30 | * http://lse.sourceforge.net/locking/rcupdate.html |
31 | * | 31 | * |
32 | */ | 32 | */ |
33 | #include <linux/types.h> | 33 | #include <linux/types.h> |
34 | #include <linux/kernel.h> | 34 | #include <linux/kernel.h> |
35 | #include <linux/init.h> | 35 | #include <linux/init.h> |
36 | #include <linux/spinlock.h> | 36 | #include <linux/spinlock.h> |
37 | #include <linux/smp.h> | 37 | #include <linux/smp.h> |
38 | #include <linux/rcupdate.h> | 38 | #include <linux/rcupdate.h> |
39 | #include <linux/interrupt.h> | 39 | #include <linux/interrupt.h> |
40 | #include <linux/sched.h> | 40 | #include <linux/sched.h> |
41 | #include <asm/atomic.h> | 41 | #include <asm/atomic.h> |
42 | #include <linux/bitops.h> | 42 | #include <linux/bitops.h> |
43 | #include <linux/module.h> | 43 | #include <linux/module.h> |
44 | #include <linux/completion.h> | 44 | #include <linux/completion.h> |
45 | #include <linux/moduleparam.h> | 45 | #include <linux/moduleparam.h> |
46 | #include <linux/percpu.h> | 46 | #include <linux/percpu.h> |
47 | #include <linux/notifier.h> | 47 | #include <linux/notifier.h> |
48 | #include <linux/rcupdate.h> | 48 | #include <linux/rcupdate.h> |
49 | #include <linux/cpu.h> | 49 | #include <linux/cpu.h> |
50 | #include <linux/mutex.h> | 50 | #include <linux/mutex.h> |
51 | 51 | ||
52 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | ||
53 | static struct lock_class_key rcu_lock_key; | ||
54 | struct lockdep_map rcu_lock_map = | ||
55 | STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key); | ||
56 | |||
57 | EXPORT_SYMBOL_GPL(rcu_lock_map); | ||
58 | #endif | ||
59 | |||
52 | /* Definition for rcupdate control block. */ | 60 | /* Definition for rcupdate control block. */ |
53 | static struct rcu_ctrlblk rcu_ctrlblk = { | 61 | static struct rcu_ctrlblk rcu_ctrlblk = { |
54 | .cur = -300, | 62 | .cur = -300, |
55 | .completed = -300, | 63 | .completed = -300, |
56 | .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock), | 64 | .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock), |
57 | .cpumask = CPU_MASK_NONE, | 65 | .cpumask = CPU_MASK_NONE, |
58 | }; | 66 | }; |
59 | static struct rcu_ctrlblk rcu_bh_ctrlblk = { | 67 | static struct rcu_ctrlblk rcu_bh_ctrlblk = { |
60 | .cur = -300, | 68 | .cur = -300, |
61 | .completed = -300, | 69 | .completed = -300, |
62 | .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock), | 70 | .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock), |
63 | .cpumask = CPU_MASK_NONE, | 71 | .cpumask = CPU_MASK_NONE, |
64 | }; | 72 | }; |
65 | 73 | ||
66 | DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L }; | 74 | DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L }; |
67 | DEFINE_PER_CPU(struct rcu_data, rcu_bh_data) = { 0L }; | 75 | DEFINE_PER_CPU(struct rcu_data, rcu_bh_data) = { 0L }; |
68 | 76 | ||
69 | /* Fake initialization required by compiler */ | 77 | /* Fake initialization required by compiler */ |
70 | static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL}; | 78 | static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL}; |
71 | static int blimit = 10; | 79 | static int blimit = 10; |
72 | static int qhimark = 10000; | 80 | static int qhimark = 10000; |
73 | static int qlowmark = 100; | 81 | static int qlowmark = 100; |
74 | 82 | ||
75 | static atomic_t rcu_barrier_cpu_count; | 83 | static atomic_t rcu_barrier_cpu_count; |
76 | static DEFINE_MUTEX(rcu_barrier_mutex); | 84 | static DEFINE_MUTEX(rcu_barrier_mutex); |
77 | static struct completion rcu_barrier_completion; | 85 | static struct completion rcu_barrier_completion; |
78 | 86 | ||
79 | #ifdef CONFIG_SMP | 87 | #ifdef CONFIG_SMP |
80 | static void force_quiescent_state(struct rcu_data *rdp, | 88 | static void force_quiescent_state(struct rcu_data *rdp, |
81 | struct rcu_ctrlblk *rcp) | 89 | struct rcu_ctrlblk *rcp) |
82 | { | 90 | { |
83 | int cpu; | 91 | int cpu; |
84 | cpumask_t cpumask; | 92 | cpumask_t cpumask; |
85 | set_need_resched(); | 93 | set_need_resched(); |
86 | if (unlikely(!rcp->signaled)) { | 94 | if (unlikely(!rcp->signaled)) { |
87 | rcp->signaled = 1; | 95 | rcp->signaled = 1; |
88 | /* | 96 | /* |
89 | * Don't send IPI to itself. With irqs disabled, | 97 | * Don't send IPI to itself. With irqs disabled, |
90 | * rdp->cpu is the current cpu. | 98 | * rdp->cpu is the current cpu. |
91 | */ | 99 | */ |
92 | cpumask = rcp->cpumask; | 100 | cpumask = rcp->cpumask; |
93 | cpu_clear(rdp->cpu, cpumask); | 101 | cpu_clear(rdp->cpu, cpumask); |
94 | for_each_cpu_mask(cpu, cpumask) | 102 | for_each_cpu_mask(cpu, cpumask) |
95 | smp_send_reschedule(cpu); | 103 | smp_send_reschedule(cpu); |
96 | } | 104 | } |
97 | } | 105 | } |
98 | #else | 106 | #else |
99 | static inline void force_quiescent_state(struct rcu_data *rdp, | 107 | static inline void force_quiescent_state(struct rcu_data *rdp, |
100 | struct rcu_ctrlblk *rcp) | 108 | struct rcu_ctrlblk *rcp) |
101 | { | 109 | { |
102 | set_need_resched(); | 110 | set_need_resched(); |
103 | } | 111 | } |
104 | #endif | 112 | #endif |
105 | 113 | ||
106 | /** | 114 | /** |
107 | * call_rcu - Queue an RCU callback for invocation after a grace period. | 115 | * call_rcu - Queue an RCU callback for invocation after a grace period. |
108 | * @head: structure to be used for queueing the RCU updates. | 116 | * @head: structure to be used for queueing the RCU updates. |
109 | * @func: actual update function to be invoked after the grace period | 117 | * @func: actual update function to be invoked after the grace period |
110 | * | 118 | * |
111 | * The update function will be invoked some time after a full grace | 119 | * The update function will be invoked some time after a full grace |
112 | * period elapses, in other words after all currently executing RCU | 120 | * period elapses, in other words after all currently executing RCU |
113 | * read-side critical sections have completed. RCU read-side critical | 121 | * read-side critical sections have completed. RCU read-side critical |
114 | * sections are delimited by rcu_read_lock() and rcu_read_unlock(), | 122 | * sections are delimited by rcu_read_lock() and rcu_read_unlock(), |
115 | * and may be nested. | 123 | * and may be nested. |
116 | */ | 124 | */ |
117 | void fastcall call_rcu(struct rcu_head *head, | 125 | void fastcall call_rcu(struct rcu_head *head, |
118 | void (*func)(struct rcu_head *rcu)) | 126 | void (*func)(struct rcu_head *rcu)) |
119 | { | 127 | { |
120 | unsigned long flags; | 128 | unsigned long flags; |
121 | struct rcu_data *rdp; | 129 | struct rcu_data *rdp; |
122 | 130 | ||
123 | head->func = func; | 131 | head->func = func; |
124 | head->next = NULL; | 132 | head->next = NULL; |
125 | local_irq_save(flags); | 133 | local_irq_save(flags); |
126 | rdp = &__get_cpu_var(rcu_data); | 134 | rdp = &__get_cpu_var(rcu_data); |
127 | *rdp->nxttail = head; | 135 | *rdp->nxttail = head; |
128 | rdp->nxttail = &head->next; | 136 | rdp->nxttail = &head->next; |
129 | if (unlikely(++rdp->qlen > qhimark)) { | 137 | if (unlikely(++rdp->qlen > qhimark)) { |
130 | rdp->blimit = INT_MAX; | 138 | rdp->blimit = INT_MAX; |
131 | force_quiescent_state(rdp, &rcu_ctrlblk); | 139 | force_quiescent_state(rdp, &rcu_ctrlblk); |
132 | } | 140 | } |
133 | local_irq_restore(flags); | 141 | local_irq_restore(flags); |
134 | } | 142 | } |
135 | 143 | ||
136 | /** | 144 | /** |
137 | * call_rcu_bh - Queue an RCU for invocation after a quicker grace period. | 145 | * call_rcu_bh - Queue an RCU for invocation after a quicker grace period. |
138 | * @head: structure to be used for queueing the RCU updates. | 146 | * @head: structure to be used for queueing the RCU updates. |
139 | * @func: actual update function to be invoked after the grace period | 147 | * @func: actual update function to be invoked after the grace period |
140 | * | 148 | * |
141 | * The update function will be invoked some time after a full grace | 149 | * The update function will be invoked some time after a full grace |
142 | * period elapses, in other words after all currently executing RCU | 150 | * period elapses, in other words after all currently executing RCU |
143 | * read-side critical sections have completed. call_rcu_bh() assumes | 151 | * read-side critical sections have completed. call_rcu_bh() assumes |
144 | * that the read-side critical sections end on completion of a softirq | 152 | * that the read-side critical sections end on completion of a softirq |
145 | * handler. This means that read-side critical sections in process | 153 | * handler. This means that read-side critical sections in process |
146 | * context must not be interrupted by softirqs. This interface is to be | 154 | * context must not be interrupted by softirqs. This interface is to be |
147 | * used when most of the read-side critical sections are in softirq context. | 155 | * used when most of the read-side critical sections are in softirq context. |
148 | * RCU read-side critical sections are delimited by rcu_read_lock() and | 156 | * RCU read-side critical sections are delimited by rcu_read_lock() and |
149 | * rcu_read_unlock(), * if in interrupt context or rcu_read_lock_bh() | 157 | * rcu_read_unlock(), * if in interrupt context or rcu_read_lock_bh() |
150 | * and rcu_read_unlock_bh(), if in process context. These may be nested. | 158 | * and rcu_read_unlock_bh(), if in process context. These may be nested. |
151 | */ | 159 | */ |
152 | void fastcall call_rcu_bh(struct rcu_head *head, | 160 | void fastcall call_rcu_bh(struct rcu_head *head, |
153 | void (*func)(struct rcu_head *rcu)) | 161 | void (*func)(struct rcu_head *rcu)) |
154 | { | 162 | { |
155 | unsigned long flags; | 163 | unsigned long flags; |
156 | struct rcu_data *rdp; | 164 | struct rcu_data *rdp; |
157 | 165 | ||
158 | head->func = func; | 166 | head->func = func; |
159 | head->next = NULL; | 167 | head->next = NULL; |
160 | local_irq_save(flags); | 168 | local_irq_save(flags); |
161 | rdp = &__get_cpu_var(rcu_bh_data); | 169 | rdp = &__get_cpu_var(rcu_bh_data); |
162 | *rdp->nxttail = head; | 170 | *rdp->nxttail = head; |
163 | rdp->nxttail = &head->next; | 171 | rdp->nxttail = &head->next; |
164 | 172 | ||
165 | if (unlikely(++rdp->qlen > qhimark)) { | 173 | if (unlikely(++rdp->qlen > qhimark)) { |
166 | rdp->blimit = INT_MAX; | 174 | rdp->blimit = INT_MAX; |
167 | force_quiescent_state(rdp, &rcu_bh_ctrlblk); | 175 | force_quiescent_state(rdp, &rcu_bh_ctrlblk); |
168 | } | 176 | } |
169 | 177 | ||
170 | local_irq_restore(flags); | 178 | local_irq_restore(flags); |
171 | } | 179 | } |
172 | 180 | ||
173 | /* | 181 | /* |
174 | * Return the number of RCU batches processed thus far. Useful | 182 | * Return the number of RCU batches processed thus far. Useful |
175 | * for debug and statistics. | 183 | * for debug and statistics. |
176 | */ | 184 | */ |
177 | long rcu_batches_completed(void) | 185 | long rcu_batches_completed(void) |
178 | { | 186 | { |
179 | return rcu_ctrlblk.completed; | 187 | return rcu_ctrlblk.completed; |
180 | } | 188 | } |
181 | 189 | ||
182 | /* | 190 | /* |
183 | * Return the number of RCU batches processed thus far. Useful | 191 | * Return the number of RCU batches processed thus far. Useful |
184 | * for debug and statistics. | 192 | * for debug and statistics. |
185 | */ | 193 | */ |
186 | long rcu_batches_completed_bh(void) | 194 | long rcu_batches_completed_bh(void) |
187 | { | 195 | { |
188 | return rcu_bh_ctrlblk.completed; | 196 | return rcu_bh_ctrlblk.completed; |
189 | } | 197 | } |
190 | 198 | ||
191 | static void rcu_barrier_callback(struct rcu_head *notused) | 199 | static void rcu_barrier_callback(struct rcu_head *notused) |
192 | { | 200 | { |
193 | if (atomic_dec_and_test(&rcu_barrier_cpu_count)) | 201 | if (atomic_dec_and_test(&rcu_barrier_cpu_count)) |
194 | complete(&rcu_barrier_completion); | 202 | complete(&rcu_barrier_completion); |
195 | } | 203 | } |
196 | 204 | ||
197 | /* | 205 | /* |
198 | * Called with preemption disabled, and from cross-cpu IRQ context. | 206 | * Called with preemption disabled, and from cross-cpu IRQ context. |
199 | */ | 207 | */ |
200 | static void rcu_barrier_func(void *notused) | 208 | static void rcu_barrier_func(void *notused) |
201 | { | 209 | { |
202 | int cpu = smp_processor_id(); | 210 | int cpu = smp_processor_id(); |
203 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); | 211 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); |
204 | struct rcu_head *head; | 212 | struct rcu_head *head; |
205 | 213 | ||
206 | head = &rdp->barrier; | 214 | head = &rdp->barrier; |
207 | atomic_inc(&rcu_barrier_cpu_count); | 215 | atomic_inc(&rcu_barrier_cpu_count); |
208 | call_rcu(head, rcu_barrier_callback); | 216 | call_rcu(head, rcu_barrier_callback); |
209 | } | 217 | } |
210 | 218 | ||
211 | /** | 219 | /** |
212 | * rcu_barrier - Wait until all the in-flight RCUs are complete. | 220 | * rcu_barrier - Wait until all the in-flight RCUs are complete. |
213 | */ | 221 | */ |
214 | void rcu_barrier(void) | 222 | void rcu_barrier(void) |
215 | { | 223 | { |
216 | BUG_ON(in_interrupt()); | 224 | BUG_ON(in_interrupt()); |
217 | /* Take cpucontrol mutex to protect against CPU hotplug */ | 225 | /* Take cpucontrol mutex to protect against CPU hotplug */ |
218 | mutex_lock(&rcu_barrier_mutex); | 226 | mutex_lock(&rcu_barrier_mutex); |
219 | init_completion(&rcu_barrier_completion); | 227 | init_completion(&rcu_barrier_completion); |
220 | atomic_set(&rcu_barrier_cpu_count, 0); | 228 | atomic_set(&rcu_barrier_cpu_count, 0); |
221 | on_each_cpu(rcu_barrier_func, NULL, 0, 1); | 229 | on_each_cpu(rcu_barrier_func, NULL, 0, 1); |
222 | wait_for_completion(&rcu_barrier_completion); | 230 | wait_for_completion(&rcu_barrier_completion); |
223 | mutex_unlock(&rcu_barrier_mutex); | 231 | mutex_unlock(&rcu_barrier_mutex); |
224 | } | 232 | } |
225 | EXPORT_SYMBOL_GPL(rcu_barrier); | 233 | EXPORT_SYMBOL_GPL(rcu_barrier); |
226 | 234 | ||
227 | /* | 235 | /* |
228 | * Invoke the completed RCU callbacks. They are expected to be in | 236 | * Invoke the completed RCU callbacks. They are expected to be in |
229 | * a per-cpu list. | 237 | * a per-cpu list. |
230 | */ | 238 | */ |
231 | static void rcu_do_batch(struct rcu_data *rdp) | 239 | static void rcu_do_batch(struct rcu_data *rdp) |
232 | { | 240 | { |
233 | struct rcu_head *next, *list; | 241 | struct rcu_head *next, *list; |
234 | int count = 0; | 242 | int count = 0; |
235 | 243 | ||
236 | list = rdp->donelist; | 244 | list = rdp->donelist; |
237 | while (list) { | 245 | while (list) { |
238 | next = list->next; | 246 | next = list->next; |
239 | prefetch(next); | 247 | prefetch(next); |
240 | list->func(list); | 248 | list->func(list); |
241 | list = next; | 249 | list = next; |
242 | if (++count >= rdp->blimit) | 250 | if (++count >= rdp->blimit) |
243 | break; | 251 | break; |
244 | } | 252 | } |
245 | rdp->donelist = list; | 253 | rdp->donelist = list; |
246 | 254 | ||
247 | local_irq_disable(); | 255 | local_irq_disable(); |
248 | rdp->qlen -= count; | 256 | rdp->qlen -= count; |
249 | local_irq_enable(); | 257 | local_irq_enable(); |
250 | if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark) | 258 | if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark) |
251 | rdp->blimit = blimit; | 259 | rdp->blimit = blimit; |
252 | 260 | ||
253 | if (!rdp->donelist) | 261 | if (!rdp->donelist) |
254 | rdp->donetail = &rdp->donelist; | 262 | rdp->donetail = &rdp->donelist; |
255 | else | 263 | else |
256 | tasklet_schedule(&per_cpu(rcu_tasklet, rdp->cpu)); | 264 | tasklet_schedule(&per_cpu(rcu_tasklet, rdp->cpu)); |
257 | } | 265 | } |
258 | 266 | ||
259 | /* | 267 | /* |
260 | * Grace period handling: | 268 | * Grace period handling: |
261 | * The grace period handling consists out of two steps: | 269 | * The grace period handling consists out of two steps: |
262 | * - A new grace period is started. | 270 | * - A new grace period is started. |
263 | * This is done by rcu_start_batch. The start is not broadcasted to | 271 | * This is done by rcu_start_batch. The start is not broadcasted to |
264 | * all cpus, they must pick this up by comparing rcp->cur with | 272 | * all cpus, they must pick this up by comparing rcp->cur with |
265 | * rdp->quiescbatch. All cpus are recorded in the | 273 | * rdp->quiescbatch. All cpus are recorded in the |
266 | * rcu_ctrlblk.cpumask bitmap. | 274 | * rcu_ctrlblk.cpumask bitmap. |
267 | * - All cpus must go through a quiescent state. | 275 | * - All cpus must go through a quiescent state. |
268 | * Since the start of the grace period is not broadcasted, at least two | 276 | * Since the start of the grace period is not broadcasted, at least two |
269 | * calls to rcu_check_quiescent_state are required: | 277 | * calls to rcu_check_quiescent_state are required: |
270 | * The first call just notices that a new grace period is running. The | 278 | * The first call just notices that a new grace period is running. The |
271 | * following calls check if there was a quiescent state since the beginning | 279 | * following calls check if there was a quiescent state since the beginning |
272 | * of the grace period. If so, it updates rcu_ctrlblk.cpumask. If | 280 | * of the grace period. If so, it updates rcu_ctrlblk.cpumask. If |
273 | * the bitmap is empty, then the grace period is completed. | 281 | * the bitmap is empty, then the grace period is completed. |
274 | * rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace | 282 | * rcu_check_quiescent_state calls rcu_start_batch(0) to start the next grace |
275 | * period (if necessary). | 283 | * period (if necessary). |
276 | */ | 284 | */ |
277 | /* | 285 | /* |
278 | * Register a new batch of callbacks, and start it up if there is currently no | 286 | * Register a new batch of callbacks, and start it up if there is currently no |
279 | * active batch and the batch to be registered has not already occurred. | 287 | * active batch and the batch to be registered has not already occurred. |
280 | * Caller must hold rcu_ctrlblk.lock. | 288 | * Caller must hold rcu_ctrlblk.lock. |
281 | */ | 289 | */ |
282 | static void rcu_start_batch(struct rcu_ctrlblk *rcp) | 290 | static void rcu_start_batch(struct rcu_ctrlblk *rcp) |
283 | { | 291 | { |
284 | if (rcp->next_pending && | 292 | if (rcp->next_pending && |
285 | rcp->completed == rcp->cur) { | 293 | rcp->completed == rcp->cur) { |
286 | rcp->next_pending = 0; | 294 | rcp->next_pending = 0; |
287 | /* | 295 | /* |
288 | * next_pending == 0 must be visible in | 296 | * next_pending == 0 must be visible in |
289 | * __rcu_process_callbacks() before it can see new value of cur. | 297 | * __rcu_process_callbacks() before it can see new value of cur. |
290 | */ | 298 | */ |
291 | smp_wmb(); | 299 | smp_wmb(); |
292 | rcp->cur++; | 300 | rcp->cur++; |
293 | 301 | ||
294 | /* | 302 | /* |
295 | * Accessing nohz_cpu_mask before incrementing rcp->cur needs a | 303 | * Accessing nohz_cpu_mask before incrementing rcp->cur needs a |
296 | * Barrier Otherwise it can cause tickless idle CPUs to be | 304 | * Barrier Otherwise it can cause tickless idle CPUs to be |
297 | * included in rcp->cpumask, which will extend graceperiods | 305 | * included in rcp->cpumask, which will extend graceperiods |
298 | * unnecessarily. | 306 | * unnecessarily. |
299 | */ | 307 | */ |
300 | smp_mb(); | 308 | smp_mb(); |
301 | cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask); | 309 | cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask); |
302 | 310 | ||
303 | rcp->signaled = 0; | 311 | rcp->signaled = 0; |
304 | } | 312 | } |
305 | } | 313 | } |
306 | 314 | ||
307 | /* | 315 | /* |
308 | * cpu went through a quiescent state since the beginning of the grace period. | 316 | * cpu went through a quiescent state since the beginning of the grace period. |
309 | * Clear it from the cpu mask and complete the grace period if it was the last | 317 | * Clear it from the cpu mask and complete the grace period if it was the last |
310 | * cpu. Start another grace period if someone has further entries pending | 318 | * cpu. Start another grace period if someone has further entries pending |
311 | */ | 319 | */ |
312 | static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp) | 320 | static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp) |
313 | { | 321 | { |
314 | cpu_clear(cpu, rcp->cpumask); | 322 | cpu_clear(cpu, rcp->cpumask); |
315 | if (cpus_empty(rcp->cpumask)) { | 323 | if (cpus_empty(rcp->cpumask)) { |
316 | /* batch completed ! */ | 324 | /* batch completed ! */ |
317 | rcp->completed = rcp->cur; | 325 | rcp->completed = rcp->cur; |
318 | rcu_start_batch(rcp); | 326 | rcu_start_batch(rcp); |
319 | } | 327 | } |
320 | } | 328 | } |
321 | 329 | ||
322 | /* | 330 | /* |
323 | * Check if the cpu has gone through a quiescent state (say context | 331 | * Check if the cpu has gone through a quiescent state (say context |
324 | * switch). If so and if it already hasn't done so in this RCU | 332 | * switch). If so and if it already hasn't done so in this RCU |
325 | * quiescent cycle, then indicate that it has done so. | 333 | * quiescent cycle, then indicate that it has done so. |
326 | */ | 334 | */ |
327 | static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, | 335 | static void rcu_check_quiescent_state(struct rcu_ctrlblk *rcp, |
328 | struct rcu_data *rdp) | 336 | struct rcu_data *rdp) |
329 | { | 337 | { |
330 | if (rdp->quiescbatch != rcp->cur) { | 338 | if (rdp->quiescbatch != rcp->cur) { |
331 | /* start new grace period: */ | 339 | /* start new grace period: */ |
332 | rdp->qs_pending = 1; | 340 | rdp->qs_pending = 1; |
333 | rdp->passed_quiesc = 0; | 341 | rdp->passed_quiesc = 0; |
334 | rdp->quiescbatch = rcp->cur; | 342 | rdp->quiescbatch = rcp->cur; |
335 | return; | 343 | return; |
336 | } | 344 | } |
337 | 345 | ||
338 | /* Grace period already completed for this cpu? | 346 | /* Grace period already completed for this cpu? |
339 | * qs_pending is checked instead of the actual bitmap to avoid | 347 | * qs_pending is checked instead of the actual bitmap to avoid |
340 | * cacheline trashing. | 348 | * cacheline trashing. |
341 | */ | 349 | */ |
342 | if (!rdp->qs_pending) | 350 | if (!rdp->qs_pending) |
343 | return; | 351 | return; |
344 | 352 | ||
345 | /* | 353 | /* |
346 | * Was there a quiescent state since the beginning of the grace | 354 | * Was there a quiescent state since the beginning of the grace |
347 | * period? If no, then exit and wait for the next call. | 355 | * period? If no, then exit and wait for the next call. |
348 | */ | 356 | */ |
349 | if (!rdp->passed_quiesc) | 357 | if (!rdp->passed_quiesc) |
350 | return; | 358 | return; |
351 | rdp->qs_pending = 0; | 359 | rdp->qs_pending = 0; |
352 | 360 | ||
353 | spin_lock(&rcp->lock); | 361 | spin_lock(&rcp->lock); |
354 | /* | 362 | /* |
355 | * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync | 363 | * rdp->quiescbatch/rcp->cur and the cpu bitmap can come out of sync |
356 | * during cpu startup. Ignore the quiescent state. | 364 | * during cpu startup. Ignore the quiescent state. |
357 | */ | 365 | */ |
358 | if (likely(rdp->quiescbatch == rcp->cur)) | 366 | if (likely(rdp->quiescbatch == rcp->cur)) |
359 | cpu_quiet(rdp->cpu, rcp); | 367 | cpu_quiet(rdp->cpu, rcp); |
360 | 368 | ||
361 | spin_unlock(&rcp->lock); | 369 | spin_unlock(&rcp->lock); |
362 | } | 370 | } |
363 | 371 | ||
364 | 372 | ||
365 | #ifdef CONFIG_HOTPLUG_CPU | 373 | #ifdef CONFIG_HOTPLUG_CPU |
366 | 374 | ||
367 | /* warning! helper for rcu_offline_cpu. do not use elsewhere without reviewing | 375 | /* warning! helper for rcu_offline_cpu. do not use elsewhere without reviewing |
368 | * locking requirements, the list it's pulling from has to belong to a cpu | 376 | * locking requirements, the list it's pulling from has to belong to a cpu |
369 | * which is dead and hence not processing interrupts. | 377 | * which is dead and hence not processing interrupts. |
370 | */ | 378 | */ |
371 | static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list, | 379 | static void rcu_move_batch(struct rcu_data *this_rdp, struct rcu_head *list, |
372 | struct rcu_head **tail) | 380 | struct rcu_head **tail) |
373 | { | 381 | { |
374 | local_irq_disable(); | 382 | local_irq_disable(); |
375 | *this_rdp->nxttail = list; | 383 | *this_rdp->nxttail = list; |
376 | if (list) | 384 | if (list) |
377 | this_rdp->nxttail = tail; | 385 | this_rdp->nxttail = tail; |
378 | local_irq_enable(); | 386 | local_irq_enable(); |
379 | } | 387 | } |
380 | 388 | ||
381 | static void __rcu_offline_cpu(struct rcu_data *this_rdp, | 389 | static void __rcu_offline_cpu(struct rcu_data *this_rdp, |
382 | struct rcu_ctrlblk *rcp, struct rcu_data *rdp) | 390 | struct rcu_ctrlblk *rcp, struct rcu_data *rdp) |
383 | { | 391 | { |
384 | /* if the cpu going offline owns the grace period | 392 | /* if the cpu going offline owns the grace period |
385 | * we can block indefinitely waiting for it, so flush | 393 | * we can block indefinitely waiting for it, so flush |
386 | * it here | 394 | * it here |
387 | */ | 395 | */ |
388 | spin_lock_bh(&rcp->lock); | 396 | spin_lock_bh(&rcp->lock); |
389 | if (rcp->cur != rcp->completed) | 397 | if (rcp->cur != rcp->completed) |
390 | cpu_quiet(rdp->cpu, rcp); | 398 | cpu_quiet(rdp->cpu, rcp); |
391 | spin_unlock_bh(&rcp->lock); | 399 | spin_unlock_bh(&rcp->lock); |
392 | rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); | 400 | rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); |
393 | rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); | 401 | rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); |
394 | rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); | 402 | rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); |
395 | } | 403 | } |
396 | 404 | ||
397 | static void rcu_offline_cpu(int cpu) | 405 | static void rcu_offline_cpu(int cpu) |
398 | { | 406 | { |
399 | struct rcu_data *this_rdp = &get_cpu_var(rcu_data); | 407 | struct rcu_data *this_rdp = &get_cpu_var(rcu_data); |
400 | struct rcu_data *this_bh_rdp = &get_cpu_var(rcu_bh_data); | 408 | struct rcu_data *this_bh_rdp = &get_cpu_var(rcu_bh_data); |
401 | 409 | ||
402 | __rcu_offline_cpu(this_rdp, &rcu_ctrlblk, | 410 | __rcu_offline_cpu(this_rdp, &rcu_ctrlblk, |
403 | &per_cpu(rcu_data, cpu)); | 411 | &per_cpu(rcu_data, cpu)); |
404 | __rcu_offline_cpu(this_bh_rdp, &rcu_bh_ctrlblk, | 412 | __rcu_offline_cpu(this_bh_rdp, &rcu_bh_ctrlblk, |
405 | &per_cpu(rcu_bh_data, cpu)); | 413 | &per_cpu(rcu_bh_data, cpu)); |
406 | put_cpu_var(rcu_data); | 414 | put_cpu_var(rcu_data); |
407 | put_cpu_var(rcu_bh_data); | 415 | put_cpu_var(rcu_bh_data); |
408 | tasklet_kill_immediate(&per_cpu(rcu_tasklet, cpu), cpu); | 416 | tasklet_kill_immediate(&per_cpu(rcu_tasklet, cpu), cpu); |
409 | } | 417 | } |
410 | 418 | ||
411 | #else | 419 | #else |
412 | 420 | ||
413 | static void rcu_offline_cpu(int cpu) | 421 | static void rcu_offline_cpu(int cpu) |
414 | { | 422 | { |
415 | } | 423 | } |
416 | 424 | ||
417 | #endif | 425 | #endif |
418 | 426 | ||
419 | /* | 427 | /* |
420 | * This does the RCU processing work from tasklet context. | 428 | * This does the RCU processing work from tasklet context. |
421 | */ | 429 | */ |
422 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, | 430 | static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp, |
423 | struct rcu_data *rdp) | 431 | struct rcu_data *rdp) |
424 | { | 432 | { |
425 | if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) { | 433 | if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) { |
426 | *rdp->donetail = rdp->curlist; | 434 | *rdp->donetail = rdp->curlist; |
427 | rdp->donetail = rdp->curtail; | 435 | rdp->donetail = rdp->curtail; |
428 | rdp->curlist = NULL; | 436 | rdp->curlist = NULL; |
429 | rdp->curtail = &rdp->curlist; | 437 | rdp->curtail = &rdp->curlist; |
430 | } | 438 | } |
431 | 439 | ||
432 | if (rdp->nxtlist && !rdp->curlist) { | 440 | if (rdp->nxtlist && !rdp->curlist) { |
433 | local_irq_disable(); | 441 | local_irq_disable(); |
434 | rdp->curlist = rdp->nxtlist; | 442 | rdp->curlist = rdp->nxtlist; |
435 | rdp->curtail = rdp->nxttail; | 443 | rdp->curtail = rdp->nxttail; |
436 | rdp->nxtlist = NULL; | 444 | rdp->nxtlist = NULL; |
437 | rdp->nxttail = &rdp->nxtlist; | 445 | rdp->nxttail = &rdp->nxtlist; |
438 | local_irq_enable(); | 446 | local_irq_enable(); |
439 | 447 | ||
440 | /* | 448 | /* |
441 | * start the next batch of callbacks | 449 | * start the next batch of callbacks |
442 | */ | 450 | */ |
443 | 451 | ||
444 | /* determine batch number */ | 452 | /* determine batch number */ |
445 | rdp->batch = rcp->cur + 1; | 453 | rdp->batch = rcp->cur + 1; |
446 | /* see the comment and corresponding wmb() in | 454 | /* see the comment and corresponding wmb() in |
447 | * the rcu_start_batch() | 455 | * the rcu_start_batch() |
448 | */ | 456 | */ |
449 | smp_rmb(); | 457 | smp_rmb(); |
450 | 458 | ||
451 | if (!rcp->next_pending) { | 459 | if (!rcp->next_pending) { |
452 | /* and start it/schedule start if it's a new batch */ | 460 | /* and start it/schedule start if it's a new batch */ |
453 | spin_lock(&rcp->lock); | 461 | spin_lock(&rcp->lock); |
454 | rcp->next_pending = 1; | 462 | rcp->next_pending = 1; |
455 | rcu_start_batch(rcp); | 463 | rcu_start_batch(rcp); |
456 | spin_unlock(&rcp->lock); | 464 | spin_unlock(&rcp->lock); |
457 | } | 465 | } |
458 | } | 466 | } |
459 | 467 | ||
460 | rcu_check_quiescent_state(rcp, rdp); | 468 | rcu_check_quiescent_state(rcp, rdp); |
461 | if (rdp->donelist) | 469 | if (rdp->donelist) |
462 | rcu_do_batch(rdp); | 470 | rcu_do_batch(rdp); |
463 | } | 471 | } |
464 | 472 | ||
465 | static void rcu_process_callbacks(unsigned long unused) | 473 | static void rcu_process_callbacks(unsigned long unused) |
466 | { | 474 | { |
467 | __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data)); | 475 | __rcu_process_callbacks(&rcu_ctrlblk, &__get_cpu_var(rcu_data)); |
468 | __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data)); | 476 | __rcu_process_callbacks(&rcu_bh_ctrlblk, &__get_cpu_var(rcu_bh_data)); |
469 | } | 477 | } |
470 | 478 | ||
471 | static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp) | 479 | static int __rcu_pending(struct rcu_ctrlblk *rcp, struct rcu_data *rdp) |
472 | { | 480 | { |
473 | /* This cpu has pending rcu entries and the grace period | 481 | /* This cpu has pending rcu entries and the grace period |
474 | * for them has completed. | 482 | * for them has completed. |
475 | */ | 483 | */ |
476 | if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) | 484 | if (rdp->curlist && !rcu_batch_before(rcp->completed, rdp->batch)) |
477 | return 1; | 485 | return 1; |
478 | 486 | ||
479 | /* This cpu has no pending entries, but there are new entries */ | 487 | /* This cpu has no pending entries, but there are new entries */ |
480 | if (!rdp->curlist && rdp->nxtlist) | 488 | if (!rdp->curlist && rdp->nxtlist) |
481 | return 1; | 489 | return 1; |
482 | 490 | ||
483 | /* This cpu has finished callbacks to invoke */ | 491 | /* This cpu has finished callbacks to invoke */ |
484 | if (rdp->donelist) | 492 | if (rdp->donelist) |
485 | return 1; | 493 | return 1; |
486 | 494 | ||
487 | /* The rcu core waits for a quiescent state from the cpu */ | 495 | /* The rcu core waits for a quiescent state from the cpu */ |
488 | if (rdp->quiescbatch != rcp->cur || rdp->qs_pending) | 496 | if (rdp->quiescbatch != rcp->cur || rdp->qs_pending) |
489 | return 1; | 497 | return 1; |
490 | 498 | ||
491 | /* nothing to do */ | 499 | /* nothing to do */ |
492 | return 0; | 500 | return 0; |
493 | } | 501 | } |
494 | 502 | ||
495 | /* | 503 | /* |
496 | * Check to see if there is any immediate RCU-related work to be done | 504 | * Check to see if there is any immediate RCU-related work to be done |
497 | * by the current CPU, returning 1 if so. This function is part of the | 505 | * by the current CPU, returning 1 if so. This function is part of the |
498 | * RCU implementation; it is -not- an exported member of the RCU API. | 506 | * RCU implementation; it is -not- an exported member of the RCU API. |
499 | */ | 507 | */ |
500 | int rcu_pending(int cpu) | 508 | int rcu_pending(int cpu) |
501 | { | 509 | { |
502 | return __rcu_pending(&rcu_ctrlblk, &per_cpu(rcu_data, cpu)) || | 510 | return __rcu_pending(&rcu_ctrlblk, &per_cpu(rcu_data, cpu)) || |
503 | __rcu_pending(&rcu_bh_ctrlblk, &per_cpu(rcu_bh_data, cpu)); | 511 | __rcu_pending(&rcu_bh_ctrlblk, &per_cpu(rcu_bh_data, cpu)); |
504 | } | 512 | } |
505 | 513 | ||
506 | /* | 514 | /* |
507 | * Check to see if any future RCU-related work will need to be done | 515 | * Check to see if any future RCU-related work will need to be done |
508 | * by the current CPU, even if none need be done immediately, returning | 516 | * by the current CPU, even if none need be done immediately, returning |
509 | * 1 if so. This function is part of the RCU implementation; it is -not- | 517 | * 1 if so. This function is part of the RCU implementation; it is -not- |
510 | * an exported member of the RCU API. | 518 | * an exported member of the RCU API. |
511 | */ | 519 | */ |
512 | int rcu_needs_cpu(int cpu) | 520 | int rcu_needs_cpu(int cpu) |
513 | { | 521 | { |
514 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); | 522 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); |
515 | struct rcu_data *rdp_bh = &per_cpu(rcu_bh_data, cpu); | 523 | struct rcu_data *rdp_bh = &per_cpu(rcu_bh_data, cpu); |
516 | 524 | ||
517 | return (!!rdp->curlist || !!rdp_bh->curlist || rcu_pending(cpu)); | 525 | return (!!rdp->curlist || !!rdp_bh->curlist || rcu_pending(cpu)); |
518 | } | 526 | } |
519 | 527 | ||
520 | void rcu_check_callbacks(int cpu, int user) | 528 | void rcu_check_callbacks(int cpu, int user) |
521 | { | 529 | { |
522 | if (user || | 530 | if (user || |
523 | (idle_cpu(cpu) && !in_softirq() && | 531 | (idle_cpu(cpu) && !in_softirq() && |
524 | hardirq_count() <= (1 << HARDIRQ_SHIFT))) { | 532 | hardirq_count() <= (1 << HARDIRQ_SHIFT))) { |
525 | rcu_qsctr_inc(cpu); | 533 | rcu_qsctr_inc(cpu); |
526 | rcu_bh_qsctr_inc(cpu); | 534 | rcu_bh_qsctr_inc(cpu); |
527 | } else if (!in_softirq()) | 535 | } else if (!in_softirq()) |
528 | rcu_bh_qsctr_inc(cpu); | 536 | rcu_bh_qsctr_inc(cpu); |
529 | tasklet_schedule(&per_cpu(rcu_tasklet, cpu)); | 537 | tasklet_schedule(&per_cpu(rcu_tasklet, cpu)); |
530 | } | 538 | } |
531 | 539 | ||
532 | static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, | 540 | static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp, |
533 | struct rcu_data *rdp) | 541 | struct rcu_data *rdp) |
534 | { | 542 | { |
535 | memset(rdp, 0, sizeof(*rdp)); | 543 | memset(rdp, 0, sizeof(*rdp)); |
536 | rdp->curtail = &rdp->curlist; | 544 | rdp->curtail = &rdp->curlist; |
537 | rdp->nxttail = &rdp->nxtlist; | 545 | rdp->nxttail = &rdp->nxtlist; |
538 | rdp->donetail = &rdp->donelist; | 546 | rdp->donetail = &rdp->donelist; |
539 | rdp->quiescbatch = rcp->completed; | 547 | rdp->quiescbatch = rcp->completed; |
540 | rdp->qs_pending = 0; | 548 | rdp->qs_pending = 0; |
541 | rdp->cpu = cpu; | 549 | rdp->cpu = cpu; |
542 | rdp->blimit = blimit; | 550 | rdp->blimit = blimit; |
543 | } | 551 | } |
544 | 552 | ||
545 | static void __devinit rcu_online_cpu(int cpu) | 553 | static void __devinit rcu_online_cpu(int cpu) |
546 | { | 554 | { |
547 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); | 555 | struct rcu_data *rdp = &per_cpu(rcu_data, cpu); |
548 | struct rcu_data *bh_rdp = &per_cpu(rcu_bh_data, cpu); | 556 | struct rcu_data *bh_rdp = &per_cpu(rcu_bh_data, cpu); |
549 | 557 | ||
550 | rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); | 558 | rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); |
551 | rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp); | 559 | rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp); |
552 | tasklet_init(&per_cpu(rcu_tasklet, cpu), rcu_process_callbacks, 0UL); | 560 | tasklet_init(&per_cpu(rcu_tasklet, cpu), rcu_process_callbacks, 0UL); |
553 | } | 561 | } |
554 | 562 | ||
555 | static int __cpuinit rcu_cpu_notify(struct notifier_block *self, | 563 | static int __cpuinit rcu_cpu_notify(struct notifier_block *self, |
556 | unsigned long action, void *hcpu) | 564 | unsigned long action, void *hcpu) |
557 | { | 565 | { |
558 | long cpu = (long)hcpu; | 566 | long cpu = (long)hcpu; |
559 | switch (action) { | 567 | switch (action) { |
560 | case CPU_UP_PREPARE: | 568 | case CPU_UP_PREPARE: |
561 | case CPU_UP_PREPARE_FROZEN: | 569 | case CPU_UP_PREPARE_FROZEN: |
562 | rcu_online_cpu(cpu); | 570 | rcu_online_cpu(cpu); |
563 | break; | 571 | break; |
564 | case CPU_DEAD: | 572 | case CPU_DEAD: |
565 | case CPU_DEAD_FROZEN: | 573 | case CPU_DEAD_FROZEN: |
566 | rcu_offline_cpu(cpu); | 574 | rcu_offline_cpu(cpu); |
567 | break; | 575 | break; |
568 | default: | 576 | default: |
569 | break; | 577 | break; |
570 | } | 578 | } |
571 | return NOTIFY_OK; | 579 | return NOTIFY_OK; |
572 | } | 580 | } |
573 | 581 | ||
574 | static struct notifier_block __cpuinitdata rcu_nb = { | 582 | static struct notifier_block __cpuinitdata rcu_nb = { |
575 | .notifier_call = rcu_cpu_notify, | 583 | .notifier_call = rcu_cpu_notify, |
576 | }; | 584 | }; |
577 | 585 | ||
578 | /* | 586 | /* |
579 | * Initializes rcu mechanism. Assumed to be called early. | 587 | * Initializes rcu mechanism. Assumed to be called early. |
580 | * That is before local timer(SMP) or jiffie timer (uniproc) is setup. | 588 | * That is before local timer(SMP) or jiffie timer (uniproc) is setup. |
581 | * Note that rcu_qsctr and friends are implicitly | 589 | * Note that rcu_qsctr and friends are implicitly |
582 | * initialized due to the choice of ``0'' for RCU_CTR_INVALID. | 590 | * initialized due to the choice of ``0'' for RCU_CTR_INVALID. |
583 | */ | 591 | */ |
584 | void __init rcu_init(void) | 592 | void __init rcu_init(void) |
585 | { | 593 | { |
586 | rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, | 594 | rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, |
587 | (void *)(long)smp_processor_id()); | 595 | (void *)(long)smp_processor_id()); |
588 | /* Register notifier for non-boot CPUs */ | 596 | /* Register notifier for non-boot CPUs */ |
589 | register_cpu_notifier(&rcu_nb); | 597 | register_cpu_notifier(&rcu_nb); |
590 | } | 598 | } |
591 | 599 | ||
592 | struct rcu_synchronize { | 600 | struct rcu_synchronize { |
593 | struct rcu_head head; | 601 | struct rcu_head head; |
594 | struct completion completion; | 602 | struct completion completion; |
595 | }; | 603 | }; |
596 | 604 | ||
597 | /* Because of FASTCALL declaration of complete, we use this wrapper */ | 605 | /* Because of FASTCALL declaration of complete, we use this wrapper */ |
598 | static void wakeme_after_rcu(struct rcu_head *head) | 606 | static void wakeme_after_rcu(struct rcu_head *head) |
599 | { | 607 | { |
600 | struct rcu_synchronize *rcu; | 608 | struct rcu_synchronize *rcu; |
601 | 609 | ||
602 | rcu = container_of(head, struct rcu_synchronize, head); | 610 | rcu = container_of(head, struct rcu_synchronize, head); |
603 | complete(&rcu->completion); | 611 | complete(&rcu->completion); |
604 | } | 612 | } |
605 | 613 | ||
606 | /** | 614 | /** |
607 | * synchronize_rcu - wait until a grace period has elapsed. | 615 | * synchronize_rcu - wait until a grace period has elapsed. |
608 | * | 616 | * |
609 | * Control will return to the caller some time after a full grace | 617 | * Control will return to the caller some time after a full grace |
610 | * period has elapsed, in other words after all currently executing RCU | 618 | * period has elapsed, in other words after all currently executing RCU |
611 | * read-side critical sections have completed. RCU read-side critical | 619 | * read-side critical sections have completed. RCU read-side critical |
612 | * sections are delimited by rcu_read_lock() and rcu_read_unlock(), | 620 | * sections are delimited by rcu_read_lock() and rcu_read_unlock(), |
613 | * and may be nested. | 621 | * and may be nested. |
614 | * | 622 | * |
615 | * If your read-side code is not protected by rcu_read_lock(), do -not- | 623 | * If your read-side code is not protected by rcu_read_lock(), do -not- |
616 | * use synchronize_rcu(). | 624 | * use synchronize_rcu(). |
617 | */ | 625 | */ |
618 | void synchronize_rcu(void) | 626 | void synchronize_rcu(void) |
619 | { | 627 | { |
620 | struct rcu_synchronize rcu; | 628 | struct rcu_synchronize rcu; |
621 | 629 | ||
622 | init_completion(&rcu.completion); | 630 | init_completion(&rcu.completion); |
623 | /* Will wake me after RCU finished */ | 631 | /* Will wake me after RCU finished */ |
624 | call_rcu(&rcu.head, wakeme_after_rcu); | 632 | call_rcu(&rcu.head, wakeme_after_rcu); |
625 | 633 | ||
626 | /* Wait for it */ | 634 | /* Wait for it */ |
627 | wait_for_completion(&rcu.completion); | 635 | wait_for_completion(&rcu.completion); |
628 | } | 636 | } |
629 | 637 | ||
630 | module_param(blimit, int, 0); | 638 | module_param(blimit, int, 0); |
631 | module_param(qhimark, int, 0); | 639 | module_param(qhimark, int, 0); |
632 | module_param(qlowmark, int, 0); | 640 | module_param(qlowmark, int, 0); |
633 | EXPORT_SYMBOL_GPL(rcu_batches_completed); | 641 | EXPORT_SYMBOL_GPL(rcu_batches_completed); |
634 | EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); | 642 | EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); |
635 | EXPORT_SYMBOL_GPL(call_rcu); | 643 | EXPORT_SYMBOL_GPL(call_rcu); |
636 | EXPORT_SYMBOL_GPL(call_rcu_bh); | 644 | EXPORT_SYMBOL_GPL(call_rcu_bh); |
637 | EXPORT_SYMBOL_GPL(synchronize_rcu); | 645 | EXPORT_SYMBOL_GPL(synchronize_rcu); |
638 | 646 |