Blame view
include/linux/srcu.h
6.57 KB
8c366db05 linux/srcu: Conve... |
1 |
/* SPDX-License-Identifier: GPL-2.0+ */ |
621934ee7 [PATCH] srcu-3: R... |
2 3 4 |
/* * Sleepable Read-Copy Update mechanism for mutual exclusion * |
621934ee7 [PATCH] srcu-3: R... |
5 |
* Copyright (C) IBM Corporation, 2006 |
4e87b2d7e srcu: Credit Lai ... |
6 |
* Copyright (C) Fujitsu, 2012 |
621934ee7 [PATCH] srcu-3: R... |
7 |
* |
8c366db05 linux/srcu: Conve... |
8 |
* Author: Paul McKenney <paulmck@linux.ibm.com> |
4e87b2d7e srcu: Credit Lai ... |
9 |
* Lai Jiangshan <laijs@cn.fujitsu.com> |
621934ee7 [PATCH] srcu-3: R... |
10 11 |
* * For detailed explanation of Read-Copy Update mechanism see - |
8660b7d8a srcu: Use rcu_seg... |
12 |
* Documentation/RCU/ *.txt |
621934ee7 [PATCH] srcu-3: R... |
13 14 |
* */ |
eabc06940 [PATCH] Add SRCU-... |
15 16 |
#ifndef _LINUX_SRCU_H #define _LINUX_SRCU_H |
d14aada8e rcu: make SRCU us... |
17 |
#include <linux/mutex.h> |
ff195cb69 rcu: Warn when sr... |
18 |
#include <linux/rcupdate.h> |
931ea9d1a rcu: Implement pe... |
19 |
#include <linux/workqueue.h> |
8660b7d8a srcu: Use rcu_seg... |
20 |
#include <linux/rcu_segcblist.h> |
d14aada8e rcu: make SRCU us... |
21 |
|
d8be81735 srcu: Create a ti... |
22 |
struct srcu_struct; |
c2a8ec077 srcu: Move to sta... |
23 |
|
632ee2001 rcu: Introduce lo... |
24 |
#ifdef CONFIG_DEBUG_LOCK_ALLOC |
aacb5d91a srcu: Use "ssp" i... |
25 |
int __init_srcu_struct(struct srcu_struct *ssp, const char *name, |
632ee2001 rcu: Introduce lo... |
26 |
struct lock_class_key *key); |
aacb5d91a srcu: Use "ssp" i... |
27 |
#define init_srcu_struct(ssp) \ |
632ee2001 rcu: Introduce lo... |
28 29 30 |
({ \ static struct lock_class_key __srcu_key; \ \ |
aacb5d91a srcu: Use "ssp" i... |
31 |
__init_srcu_struct((ssp), #ssp, &__srcu_key); \ |
632ee2001 rcu: Introduce lo... |
32 |
}) |
55c6659af srcu: Add DEFINE_... |
33 |
#define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, |
632ee2001 rcu: Introduce lo... |
34 |
#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
aacb5d91a srcu: Use "ssp" i... |
35 |
int init_srcu_struct(struct srcu_struct *ssp); |
632ee2001 rcu: Introduce lo... |
36 |
|
55c6659af srcu: Add DEFINE_... |
37 |
#define __SRCU_DEP_MAP_INIT(srcu_name) |
632ee2001 rcu: Introduce lo... |
38 |
#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
d8be81735 srcu: Create a ti... |
39 40 41 42 |
#ifdef CONFIG_TINY_SRCU #include <linux/srcutiny.h> #elif defined(CONFIG_TREE_SRCU) #include <linux/srcutree.h> |
07f6e64bf srcu: Make SRCU b... |
43 |
#elif defined(CONFIG_SRCU) |
d8be81735 srcu: Create a ti... |
44 |
#error "Unknown SRCU implementation specified to kernel configuration" |
07f6e64bf srcu: Make SRCU b... |
45 |
#else |
07f6e64bf srcu: Make SRCU b... |
46 47 |
/* Dummy definition for things like notifiers. Actual use gets link error. */ struct srcu_struct { }; |
d8be81735 srcu: Create a ti... |
48 |
#endif |
55c6659af srcu: Add DEFINE_... |
49 |
|
aacb5d91a srcu: Use "ssp" i... |
50 |
void call_srcu(struct srcu_struct *ssp, struct rcu_head *head, |
931ea9d1a rcu: Implement pe... |
51 |
void (*func)(struct rcu_head *head)); |
f5ad39914 srcu: Remove clea... |
52 |
void cleanup_srcu_struct(struct srcu_struct *ssp); |
aacb5d91a srcu: Use "ssp" i... |
53 54 55 |
int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp); void synchronize_srcu(struct srcu_struct *ssp); |
eabc06940 [PATCH] Add SRCU-... |
56 |
|
632ee2001 rcu: Introduce lo... |
57 58 59 60 |
#ifdef CONFIG_DEBUG_LOCK_ALLOC /** * srcu_read_lock_held - might we be in SRCU read-side critical section? |
aacb5d91a srcu: Use "ssp" i... |
61 |
* @ssp: The srcu_struct structure to check |
632ee2001 rcu: Introduce lo... |
62 |
* |
d20200b59 rcu: Fix bogus CO... |
63 64 |
* If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an SRCU * read-side critical section. In absence of CONFIG_DEBUG_LOCK_ALLOC, |
632ee2001 rcu: Introduce lo... |
65 66 |
* this assumes we are in an SRCU read-side critical section unless it can * prove otherwise. |
ff195cb69 rcu: Warn when sr... |
67 |
* |
867f236bd rcu: Make srcu_re... |
68 69 70 |
* Checks debug_lockdep_rcu_enabled() to prevent false positives during boot * and while lockdep is disabled. * |
511a0868b srcu: Remove chec... |
71 72 73 |
* Note that SRCU is based on its own statemachine and it doesn't * relies on normal RCU, it can be called from the CPU which * is in the idle loop from an RCU point of view or offline. |
632ee2001 rcu: Introduce lo... |
74 |
*/ |
aacb5d91a srcu: Use "ssp" i... |
75 |
static inline int srcu_read_lock_held(const struct srcu_struct *ssp) |
632ee2001 rcu: Introduce lo... |
76 |
{ |
867f236bd rcu: Make srcu_re... |
77 |
if (!debug_lockdep_rcu_enabled()) |
ff195cb69 rcu: Warn when sr... |
78 |
return 1; |
aacb5d91a srcu: Use "ssp" i... |
79 |
return lock_is_held(&ssp->dep_map); |
632ee2001 rcu: Introduce lo... |
80 81 82 |
} #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ |
aacb5d91a srcu: Use "ssp" i... |
83 |
static inline int srcu_read_lock_held(const struct srcu_struct *ssp) |
632ee2001 rcu: Introduce lo... |
84 85 86 87 88 89 90 |
{ return 1; } #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ /** |
ca5ecddfa rcu: define __rcu... |
91 92 |
* srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing * @p: the pointer to fetch and protect for later dereferencing |
aacb5d91a srcu: Use "ssp" i... |
93 |
* @ssp: pointer to the srcu_struct, which is used to check that we |
ca5ecddfa rcu: define __rcu... |
94 95 |
* really are in an SRCU read-side critical section. * @c: condition to check for update-side use |
c26d34a58 rcu: Add lockdep-... |
96 |
* |
ca5ecddfa rcu: define __rcu... |
97 98 99 100 |
* If PROVE_RCU is enabled, invoking this outside of an RCU read-side * critical section will result in an RCU-lockdep splat, unless @c evaluates * to 1. The @c argument will normally be a logical expression containing * lockdep_is_held() calls. |
c26d34a58 rcu: Add lockdep-... |
101 |
*/ |
aacb5d91a srcu: Use "ssp" i... |
102 103 |
#define srcu_dereference_check(p, ssp, c) \ __rcu_dereference_check((p), (c) || srcu_read_lock_held(ssp), __rcu) |
ca5ecddfa rcu: define __rcu... |
104 105 106 107 |
/** * srcu_dereference - fetch SRCU-protected pointer for later dereferencing * @p: the pointer to fetch and protect for later dereferencing |
aacb5d91a srcu: Use "ssp" i... |
108 |
* @ssp: pointer to the srcu_struct, which is used to check that we |
ca5ecddfa rcu: define __rcu... |
109 110 111 112 113 114 |
* really are in an SRCU read-side critical section. * * Makes rcu_dereference_check() do the dirty work. If PROVE_RCU * is enabled, invoking this outside of an RCU read-side critical * section will result in an RCU-lockdep splat. */ |
aacb5d91a srcu: Use "ssp" i... |
115 |
#define srcu_dereference(p, ssp) srcu_dereference_check((p), (ssp), 0) |
c26d34a58 rcu: Add lockdep-... |
116 117 |
/** |
0b764a6e4 srcu: Add notrace... |
118 |
* srcu_dereference_notrace - no tracing and no lockdep calls from here |
f3e763c3e srcu: Fix kernel-... |
119 |
* @p: the pointer to fetch and protect for later dereferencing |
aacb5d91a srcu: Use "ssp" i... |
120 |
* @ssp: pointer to the srcu_struct, which is used to check that we |
f3e763c3e srcu: Fix kernel-... |
121 |
* really are in an SRCU read-side critical section. |
0b764a6e4 srcu: Add notrace... |
122 |
*/ |
aacb5d91a srcu: Use "ssp" i... |
123 |
#define srcu_dereference_notrace(p, ssp) srcu_dereference_check((p), (ssp), 1) |
0b764a6e4 srcu: Add notrace... |
124 125 |
/** |
632ee2001 rcu: Introduce lo... |
126 |
* srcu_read_lock - register a new reader for an SRCU-protected structure. |
aacb5d91a srcu: Use "ssp" i... |
127 |
* @ssp: srcu_struct in which to register the new reader. |
632ee2001 rcu: Introduce lo... |
128 129 |
* * Enter an SRCU read-side critical section. Note that SRCU read-side |
73d4da4d3 rcu: Upgrade srcu... |
130 131 132 133 134 135 |
* critical sections may be nested. However, it is illegal to * call anything that waits on an SRCU grace period for the same * srcu_struct, whether directly or indirectly. Please note that * one way to indirectly wait on an SRCU grace period is to acquire * a mutex that is held elsewhere while calling synchronize_srcu() or * synchronize_srcu_expedited(). |
3842a0832 rcu: Document sam... |
136 137 138 139 140 |
* * Note that srcu_read_lock() and the matching srcu_read_unlock() must * occur in the same context, for example, it is illegal to invoke * srcu_read_unlock() in an irq handler if the matching srcu_read_lock() * was invoked in process context. |
632ee2001 rcu: Introduce lo... |
141 |
*/ |
aacb5d91a srcu: Use "ssp" i... |
142 |
static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp) |
632ee2001 rcu: Introduce lo... |
143 |
{ |
49f5903b4 rcu: Move preempt... |
144 |
int retval; |
632ee2001 rcu: Introduce lo... |
145 |
|
aacb5d91a srcu: Use "ssp" i... |
146 147 |
retval = __srcu_read_lock(ssp); rcu_lock_acquire(&(ssp)->dep_map); |
632ee2001 rcu: Introduce lo... |
148 149 |
return retval; } |
1f45a4db3 srcu: Add notrace... |
150 151 |
/* Used by tracing, cannot be traced and cannot invoke lockdep. */ static inline notrace int |
aacb5d91a srcu: Use "ssp" i... |
152 |
srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp) |
1f45a4db3 srcu: Add notrace... |
153 154 |
{ int retval; |
aacb5d91a srcu: Use "ssp" i... |
155 |
retval = __srcu_read_lock(ssp); |
1f45a4db3 srcu: Add notrace... |
156 157 |
return retval; } |
632ee2001 rcu: Introduce lo... |
158 159 |
/** * srcu_read_unlock - unregister a old reader from an SRCU-protected structure. |
aacb5d91a srcu: Use "ssp" i... |
160 |
* @ssp: srcu_struct in which to unregister the old reader. |
632ee2001 rcu: Introduce lo... |
161 162 163 164 |
* @idx: return value from corresponding srcu_read_lock(). * * Exit an SRCU read-side critical section. */ |
aacb5d91a srcu: Use "ssp" i... |
165 166 |
static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp) |
632ee2001 rcu: Introduce lo... |
167 |
{ |
c8ca1aa77 srcu: Check for i... |
168 |
WARN_ON_ONCE(idx & ~0x1); |
aacb5d91a srcu: Use "ssp" i... |
169 170 |
rcu_lock_release(&(ssp)->dep_map); __srcu_read_unlock(ssp, idx); |
632ee2001 rcu: Introduce lo... |
171 |
} |
1f45a4db3 srcu: Add notrace... |
172 173 |
/* Used by tracing, cannot be traced and cannot call lockdep. */ static inline notrace void |
aacb5d91a srcu: Use "ssp" i... |
174 |
srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp) |
1f45a4db3 srcu: Add notrace... |
175 |
{ |
aacb5d91a srcu: Use "ssp" i... |
176 |
__srcu_read_unlock(ssp, idx); |
1f45a4db3 srcu: Add notrace... |
177 |
} |
ce332f662 srcu: API for bar... |
178 179 180 181 182 183 184 185 186 187 188 189 190 |
/** * smp_mb__after_srcu_read_unlock - ensure full ordering after srcu_read_unlock * * Converts the preceding srcu_read_unlock into a two-way memory barrier. * * Call this after srcu_read_unlock, to guarantee that all memory operations * that occur after smp_mb__after_srcu_read_unlock will appear to happen after * the preceding srcu_read_unlock. */ static inline void smp_mb__after_srcu_read_unlock(void) { /* __srcu_read_unlock has smp_mb() internally so nothing to do here. */ } |
eabc06940 [PATCH] Add SRCU-... |
191 |
#endif |