Blame view
include/linux/srcu.h
8.18 KB
621934ee7
|
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 |
/* * Sleepable Read-Copy Update mechanism for mutual exclusion * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright (C) IBM Corporation, 2006 |
4e87b2d7e
|
19 |
* Copyright (C) Fujitsu, 2012 |
621934ee7
|
20 21 |
* * Author: Paul McKenney <paulmck@us.ibm.com> |
4e87b2d7e
|
22 |
* Lai Jiangshan <laijs@cn.fujitsu.com> |
621934ee7
|
23 24 25 26 27 |
* * For detailed explanation of Read-Copy Update mechanism see - * Documentation/RCU/ *.txt * */ |
eabc06940
|
28 29 |
#ifndef _LINUX_SRCU_H #define _LINUX_SRCU_H |
d14aada8e
|
30 |
#include <linux/mutex.h> |
ff195cb69
|
31 |
#include <linux/rcupdate.h> |
931ea9d1a
|
32 |
#include <linux/workqueue.h> |
d14aada8e
|
33 |
|
621934ee7
|
34 |
struct srcu_struct_array { |
cef50120b
|
35 |
unsigned long c[2]; |
b52ce066c
|
36 |
unsigned long seq[2]; |
621934ee7
|
37 |
}; |
931ea9d1a
|
38 39 40 |
struct rcu_batch { struct rcu_head *head, **tail; }; |
55c6659af
|
41 |
#define RCU_BATCH_INIT(name) { NULL, &(name.head) } |
621934ee7
|
42 |
struct srcu_struct { |
cef50120b
|
43 |
unsigned completed; |
43cf38eb5
|
44 |
struct srcu_struct_array __percpu *per_cpu_ref; |
931ea9d1a
|
45 46 47 48 49 50 51 52 53 54 |
spinlock_t queue_lock; /* protect ->batch_queue, ->running */ bool running; /* callbacks just queued */ struct rcu_batch batch_queue; /* callbacks try to do the first check_zero */ struct rcu_batch batch_check0; /* callbacks done with the first check_zero and the flip */ struct rcu_batch batch_check1; struct rcu_batch batch_done; struct delayed_work work; |
632ee2001
|
55 56 57 |
#ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
621934ee7
|
58 |
}; |
632ee2001
|
59 60 61 62 63 64 65 66 67 68 69 |
#ifdef CONFIG_DEBUG_LOCK_ALLOC int __init_srcu_struct(struct srcu_struct *sp, const char *name, struct lock_class_key *key); #define init_srcu_struct(sp) \ ({ \ static struct lock_class_key __srcu_key; \ \ __init_srcu_struct((sp), #sp, &__srcu_key); \ }) |
55c6659af
|
70 |
#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, |
632ee2001
|
71 |
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
e6a92013b
|
72 |
int init_srcu_struct(struct srcu_struct *sp); |
632ee2001
|
73 |
|
55c6659af
|
74 |
#define __SRCU_DEP_MAP_INIT(srcu_name) |
632ee2001
|
75 |
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
f2ebfbc99
|
76 |
void process_srcu(struct work_struct *work); |
55c6659af
|
77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 |
#define __SRCU_STRUCT_INIT(name) \ { \ .completed = -300, \ .per_cpu_ref = &name##_srcu_array, \ .queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock), \ .running = false, \ .batch_queue = RCU_BATCH_INIT(name.batch_queue), \ .batch_check0 = RCU_BATCH_INIT(name.batch_check0), \ .batch_check1 = RCU_BATCH_INIT(name.batch_check1), \ .batch_done = RCU_BATCH_INIT(name.batch_done), \ .work = __DELAYED_WORK_INITIALIZER(name.work, process_srcu, 0),\ __SRCU_DEP_MAP_INIT(name) \ } /* * define and init a srcu struct at build time. * dont't call init_srcu_struct() nor cleanup_srcu_struct() on it. */ #define DEFINE_SRCU(name) \ static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\ struct srcu_struct name = __SRCU_STRUCT_INIT(name); #define DEFINE_STATIC_SRCU(name) \ static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\ static struct srcu_struct name = __SRCU_STRUCT_INIT(name); |
931ea9d1a
|
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 |
/** * call_srcu() - Queue a callback for invocation after an SRCU grace period * @sp: srcu_struct in queue the callback * @head: structure to be used for queueing the SRCU callback. * @func: function to be invoked after the SRCU grace period * * The callback function will be invoked some time after a full SRCU * grace period elapses, in other words after all pre-existing SRCU * read-side critical sections have completed. However, the callback * function might well execute concurrently with other SRCU read-side * critical sections that started after call_srcu() was invoked. SRCU * read-side critical sections are delimited by srcu_read_lock() and * srcu_read_unlock(), and may be nested. * * The callback will be invoked from process context, but must nevertheless * be fast and must not block. */ void call_srcu(struct srcu_struct *sp, struct rcu_head *head, void (*func)(struct rcu_head *head)); |
621934ee7
|
121 |
void cleanup_srcu_struct(struct srcu_struct *sp); |
632ee2001
|
122 123 |
int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp); void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); |
621934ee7
|
124 |
void synchronize_srcu(struct srcu_struct *sp); |
0cd397d33
|
125 |
void synchronize_srcu_expedited(struct srcu_struct *sp); |
621934ee7
|
126 |
long srcu_batches_completed(struct srcu_struct *sp); |
931ea9d1a
|
127 |
void srcu_barrier(struct srcu_struct *sp); |
eabc06940
|
128 |
|
632ee2001
|
129 130 131 132 133 |
#ifdef CONFIG_DEBUG_LOCK_ALLOC /** * srcu_read_lock_held - might we be in SRCU read-side critical section? * |
d20200b59
|
134 135 |
* If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, |
632ee2001
|
136 137 |
* this assumes we are in an SRCU read-side critical section unless it can * prove otherwise. |
ff195cb69
|
138 |
* |
867f236bd
|
139 140 141 |
* Checks debug_lockdep_rcu_enabled() to prevent false positives during boot * and while lockdep is disabled. * |
511a0868b
|
142 143 144 |
* Note that SRCU is based on its own statemachine and it doesn't * relies on normal RCU, it can be called from the CPU which * is in the idle loop from an RCU point of view or offline. |
632ee2001
|
145 146 147 |
*/ static inline int srcu_read_lock_held(struct srcu_struct *sp) { |
867f236bd
|
148 |
if (!debug_lockdep_rcu_enabled()) |
ff195cb69
|
149 |
return 1; |
ff195cb69
|
150 |
return lock_is_held(&sp->dep_map); |
632ee2001
|
151 152 153 154 155 156 157 158 159 160 161 162 |
} #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ static inline int srcu_read_lock_held(struct srcu_struct *sp) { return 1; } #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** |
ca5ecddfa
|
163 164 165 166 167 |
* srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing * @p: the pointer to fetch and protect for later dereferencing * @sp: pointer to the srcu_struct, which is used to check that we * really are in an SRCU read-side critical section. * @c: condition to check for update-side use |
c26d34a58
|
168 |
* |
ca5ecddfa
|
169 170 171 172 |
* If PROVE_RCU is enabled, invoking this outside of an RCU read-side * critical section will result in an RCU-lockdep splat, unless @c evaluates * to 1. The @c argument will normally be a logical expression containing * lockdep_is_held() calls. |
c26d34a58
|
173 |
*/ |
ca5ecddfa
|
174 175 176 177 178 179 180 181 182 183 184 185 186 187 |
#define srcu_dereference_check(p, sp, c) \ __rcu_dereference_check((p), srcu_read_lock_held(sp) || (c), __rcu) /** * srcu_dereference - fetch SRCU-protected pointer for later dereferencing * @p: the pointer to fetch and protect for later dereferencing * @sp: pointer to the srcu_struct, which is used to check that we * really are in an SRCU read-side critical section. * * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU * is enabled, invoking this outside of an RCU read-side critical * section will result in an RCU-lockdep splat. */ #define srcu_dereference(p, sp) srcu_dereference_check((p), (sp), 0) |
c26d34a58
|
188 189 |
/** |
632ee2001
|
190 191 192 193 |
* srcu_read_lock - register a new reader for an SRCU-protected structure. * @sp: srcu_struct in which to register the new reader. * * Enter an SRCU read-side critical section. Note that SRCU read-side |
73d4da4d3
|
194 195 196 197 198 199 |
* critical sections may be nested. However, it is illegal to * call anything that waits on an SRCU grace period for the same * srcu_struct, whether directly or indirectly. Please note that * one way to indirectly wait on an SRCU grace period is to acquire * a mutex that is held elsewhere while calling synchronize_srcu() or * synchronize_srcu_expedited(). |
3842a0832
|
200 201 202 203 204 |
* * Note that srcu_read_lock() and the matching srcu_read_unlock() must * occur in the same context, for example, it is illegal to invoke * srcu_read_unlock() in an irq handler if the matching srcu_read_lock() * was invoked in process context. |
632ee2001
|
205 206 207 208 |
*/ static inline int srcu_read_lock(struct srcu_struct *sp) __acquires(sp) { int retval = __srcu_read_lock(sp); |
ff195cb69
|
209 |
rcu_lock_acquire(&(sp)->dep_map); |
632ee2001
|
210 211 212 213 214 215 216 217 218 219 220 221 222 |
return retval; } /** * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. * @sp: srcu_struct in which to unregister the old reader. * @idx: return value from corresponding srcu_read_lock(). * * Exit an SRCU read-side critical section. */ static inline void srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp) { |
ff195cb69
|
223 |
rcu_lock_release(&(sp)->dep_map); |
632ee2001
|
224 225 |
__srcu_read_unlock(sp, idx); } |
eabc06940
|
226 |
#endif |