Blame view
lib/percpu-refcount.c
15.2 KB
457c89965 treewide: Add SPD... |
1 |
// SPDX-License-Identifier: GPL-2.0-only |
a818e526c lib/percpu-refcou... |
2 |
#define pr_fmt(fmt) "%s: " fmt, __func__ |
215e262f2 percpu: implement... |
3 4 |
#include <linux/kernel.h> |
490c79a65 percpu_ref: decou... |
5 6 |
#include <linux/sched.h> #include <linux/wait.h> |
2b0d3d3e4 percpu_ref: reduc... |
7 |
#include <linux/slab.h> |
215e262f2 percpu: implement... |
8 9 10 11 12 13 14 15 16 |
#include <linux/percpu-refcount.h> /* * Initially, a percpu refcount is just a set of percpu counters. Initially, we * don't try to detect the ref hitting 0 - which means that get/put can just * increment or decrement the local counter. Note that the counter on a * particular cpu can (and will) wrap - this is fine, when we go to shutdown the * percpu counters will all sum to the correct value * |
bdb428c82 lib+mm: fix few s... |
17 |
* (More precisely: because modular arithmetic is commutative the sum of all the |
eecc16ba9 percpu_ref: repla... |
18 19 |
* percpu_count vars will be equal to what it would have been if all the gets * and puts were done to a single integer, even if some of the percpu integers |
215e262f2 percpu: implement... |
20 21 22 23 24 25 26 27 28 29 30 31 |
* overflow or underflow). * * The real trick to implementing percpu refcounts is shutdown. We can't detect * the ref hitting 0 on every put - this would require global synchronization * and defeat the whole purpose of using percpu refs. * * What we do is require the user to keep track of the initial refcount; we know * the ref can't hit 0 before the user drops the initial ref, so as long as we * convert to non percpu mode before the initial ref is dropped everything * works. * * Converting to non percpu mode is done with some RCUish stuff in |
e625305b3 percpu-refcount: ... |
32 33 |
* percpu_ref_kill. Additionally, we need a bias value so that the * atomic_long_t can't hit 0 before we've added up all the percpu refs. |
215e262f2 percpu: implement... |
34 |
*/ |
eecc16ba9 percpu_ref: repla... |
35 |
#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1)) |
215e262f2 percpu: implement... |
36 |
|
33e465ce7 percpu_ref: allow... |
37 |
static DEFINE_SPINLOCK(percpu_ref_switch_lock); |
490c79a65 percpu_ref: decou... |
38 |
static DECLARE_WAIT_QUEUE_HEAD(percpu_ref_switch_waitq); |
eecc16ba9 percpu_ref: repla... |
39 |
static unsigned long __percpu *percpu_count_ptr(struct percpu_ref *ref) |
eae7975dd percpu-refcount: ... |
40 |
{ |
eecc16ba9 percpu_ref: repla... |
41 |
return (unsigned long __percpu *) |
27344a901 percpu_ref: add P... |
42 |
(ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC_DEAD); |
eae7975dd percpu-refcount: ... |
43 |
} |
215e262f2 percpu: implement... |
44 45 |
/** * percpu_ref_init - initialize a percpu refcount |
ac899061a percpu-refcount: ... |
46 47 |
* @ref: percpu_ref to initialize * @release: function which will be called when refcount hits 0 |
2aad2a86f percpu_ref: add P... |
48 |
* @flags: PERCPU_REF_INIT_* flags |
a34375ef9 percpu-refcount: ... |
49 |
* @gfp: allocation mask to use |
215e262f2 percpu: implement... |
50 |
* |
15617dffa percpu_ref: Fix c... |
51 52 53 54 |
* Initializes @ref. @ref starts out in percpu mode with a refcount of 1 unless * @flags contains PERCPU_REF_INIT_ATOMIC or PERCPU_REF_INIT_DEAD. These flags * change the start state to atomic with the latter setting the initial refcount * to 0. See the definitions of PERCPU_REF_INIT_* flags for flag behaviors. |
215e262f2 percpu: implement... |
55 56 57 58 |
* * Note that @release must not sleep - it may potentially be called from RCU * callback context by percpu_ref_kill(). */ |
a34375ef9 percpu-refcount: ... |
59 |
int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, |
2aad2a86f percpu_ref: add P... |
60 |
unsigned int flags, gfp_t gfp) |
215e262f2 percpu: implement... |
61 |
{ |
27344a901 percpu_ref: add P... |
62 63 |
size_t align = max_t(size_t, 1 << __PERCPU_REF_FLAG_BITS, __alignof__(unsigned long)); |
2aad2a86f percpu_ref: add P... |
64 |
unsigned long start_count = 0; |
2b0d3d3e4 percpu_ref: reduc... |
65 |
struct percpu_ref_data *data; |
215e262f2 percpu: implement... |
66 |
|
27344a901 percpu_ref: add P... |
67 68 |
ref->percpu_count_ptr = (unsigned long) __alloc_percpu_gfp(sizeof(unsigned long), align, gfp); |
eecc16ba9 percpu_ref: repla... |
69 |
if (!ref->percpu_count_ptr) |
215e262f2 percpu: implement... |
70 |
return -ENOMEM; |
2b0d3d3e4 percpu_ref: reduc... |
71 72 73 74 75 76 77 78 |
data = kzalloc(sizeof(*ref->data), gfp); if (!data) { free_percpu((void __percpu *)ref->percpu_count_ptr); return -ENOMEM; } data->force_atomic = flags & PERCPU_REF_INIT_ATOMIC; data->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT; |
1cae13e75 percpu_ref: make ... |
79 |
|
7d9ab9b6a percpu_ref: relea... |
80 |
if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) { |
2aad2a86f percpu_ref: add P... |
81 |
ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC; |
2b0d3d3e4 percpu_ref: reduc... |
82 |
data->allow_reinit = true; |
7d9ab9b6a percpu_ref: relea... |
83 |
} else { |
2aad2a86f percpu_ref: add P... |
84 |
start_count += PERCPU_COUNT_BIAS; |
7d9ab9b6a percpu_ref: relea... |
85 |
} |
2aad2a86f percpu_ref: add P... |
86 87 88 89 90 |
if (flags & PERCPU_REF_INIT_DEAD) ref->percpu_count_ptr |= __PERCPU_REF_DEAD; else start_count++; |
2b0d3d3e4 percpu_ref: reduc... |
91 |
atomic_long_set(&data->count, start_count); |
2aad2a86f percpu_ref: add P... |
92 |
|
2b0d3d3e4 percpu_ref: reduc... |
93 94 95 96 |
data->release = release; data->confirm_switch = NULL; data->ref = ref; ref->data = data; |
215e262f2 percpu: implement... |
97 98 |
return 0; } |
5e9dd373d percpu_refcount: ... |
99 |
EXPORT_SYMBOL_GPL(percpu_ref_init); |
215e262f2 percpu: implement... |
100 |
|
2b0d3d3e4 percpu_ref: reduc... |
101 102 103 104 105 106 |
static void __percpu_ref_exit(struct percpu_ref *ref) { unsigned long __percpu *percpu_count = percpu_count_ptr(ref); if (percpu_count) { /* non-NULL confirm_switch indicates switching in progress */ |
7ea6bf2e6 percpu_ref: don't... |
107 |
WARN_ON_ONCE(ref->data && ref->data->confirm_switch); |
2b0d3d3e4 percpu_ref: reduc... |
108 109 110 111 |
free_percpu(percpu_count); ref->percpu_count_ptr = __PERCPU_REF_ATOMIC_DEAD; } } |
bc497bd33 percpu-refcount: ... |
112 |
/** |
9a1049da9 percpu-refcount: ... |
113 114 |
* percpu_ref_exit - undo percpu_ref_init() * @ref: percpu_ref to exit |
bc497bd33 percpu-refcount: ... |
115 |
* |
9a1049da9 percpu-refcount: ... |
116 117 118 119 120 |
* This function exits @ref. The caller is responsible for ensuring that * @ref is no longer in active use. The usual places to invoke this * function from are the @ref->release() callback or in init failure path * where percpu_ref_init() succeeded but other parts of the initialization * of the embedding object failed. |
bc497bd33 percpu-refcount: ... |
121 |
*/ |
9a1049da9 percpu-refcount: ... |
122 |
void percpu_ref_exit(struct percpu_ref *ref) |
bc497bd33 percpu-refcount: ... |
123 |
{ |
2b0d3d3e4 percpu_ref: reduc... |
124 125 |
struct percpu_ref_data *data = ref->data; unsigned long flags; |
bc497bd33 percpu-refcount: ... |
126 |
|
2b0d3d3e4 percpu_ref: reduc... |
127 128 129 130 131 132 133 134 135 136 137 138 |
__percpu_ref_exit(ref); if (!data) return; spin_lock_irqsave(&percpu_ref_switch_lock, flags); ref->percpu_count_ptr |= atomic_long_read(&ref->data->count) << __PERCPU_REF_FLAG_BITS; ref->data = NULL; spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); kfree(data); |
bc497bd33 percpu-refcount: ... |
139 |
} |
9a1049da9 percpu-refcount: ... |
140 |
EXPORT_SYMBOL_GPL(percpu_ref_exit); |
bc497bd33 percpu-refcount: ... |
141 |
|
490c79a65 percpu_ref: decou... |
142 143 |
static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu) { |
2b0d3d3e4 percpu_ref: reduc... |
144 145 146 |
struct percpu_ref_data *data = container_of(rcu, struct percpu_ref_data, rcu); struct percpu_ref *ref = data->ref; |
490c79a65 percpu_ref: decou... |
147 |
|
2b0d3d3e4 percpu_ref: reduc... |
148 149 |
data->confirm_switch(ref); data->confirm_switch = NULL; |
490c79a65 percpu_ref: decou... |
150 |
wake_up_all(&percpu_ref_switch_waitq); |
2b0d3d3e4 percpu_ref: reduc... |
151 152 |
if (!data->allow_reinit) __percpu_ref_exit(ref); |
7d9ab9b6a percpu_ref: relea... |
153 |
|
490c79a65 percpu_ref: decou... |
154 155 156 157 158 |
/* drop ref from percpu_ref_switch_to_atomic() */ percpu_ref_put(ref); } static void percpu_ref_switch_to_atomic_rcu(struct rcu_head *rcu) |
215e262f2 percpu: implement... |
159 |
{ |
2b0d3d3e4 percpu_ref: reduc... |
160 161 162 |
struct percpu_ref_data *data = container_of(rcu, struct percpu_ref_data, rcu); struct percpu_ref *ref = data->ref; |
eecc16ba9 percpu_ref: repla... |
163 |
unsigned long __percpu *percpu_count = percpu_count_ptr(ref); |
e625305b3 percpu-refcount: ... |
164 |
unsigned long count = 0; |
215e262f2 percpu: implement... |
165 |
int cpu; |
215e262f2 percpu: implement... |
166 |
for_each_possible_cpu(cpu) |
eecc16ba9 percpu_ref: repla... |
167 |
count += *per_cpu_ptr(percpu_count, cpu); |
215e262f2 percpu: implement... |
168 |
|
a818e526c lib/percpu-refcou... |
169 170 |
pr_debug("global %lu percpu %lu ", |
2b0d3d3e4 percpu_ref: reduc... |
171 |
atomic_long_read(&data->count), count); |
215e262f2 percpu: implement... |
172 173 174 175 176 177 178 179 180 181 182 183 184 |
/* * It's crucial that we sum the percpu counters _before_ adding the sum * to &ref->count; since gets could be happening on one cpu while puts * happen on another, adding a single cpu's count could cause * @ref->count to hit 0 before we've got a consistent value - but the * sum of all the counts will be consistent and correct. * * Subtracting the bias value then has to happen _after_ adding count to * &ref->count; we need the bias value to prevent &ref->count from * reaching 0 before we add the percpu counts. But doing it at the same * time is equivalent and saves us atomic operations: */ |
2b0d3d3e4 percpu_ref: reduc... |
185 |
atomic_long_add((long)count - PERCPU_COUNT_BIAS, &data->count); |
215e262f2 percpu: implement... |
186 |
|
2b0d3d3e4 percpu_ref: reduc... |
187 |
WARN_ONCE(atomic_long_read(&data->count) <= 0, |
d75f773c8 treewide: Switch ... |
188 |
"percpu ref (%ps) <= 0 (%ld) after switching to atomic", |
2b0d3d3e4 percpu_ref: reduc... |
189 |
data->release, atomic_long_read(&data->count)); |
687b0ad27 percpu-refcount: ... |
190 |
|
490c79a65 percpu_ref: decou... |
191 192 193 |
/* @ref is viewed as dead on all CPUs, send out switch confirmation */ percpu_ref_call_confirm_rcu(rcu); } |
dbece3a0f percpu-refcount: ... |
194 |
|
490c79a65 percpu_ref: decou... |
195 196 197 198 199 200 201 |
static void percpu_ref_noop_confirm_switch(struct percpu_ref *ref) { } static void __percpu_ref_switch_to_atomic(struct percpu_ref *ref, percpu_ref_func_t *confirm_switch) { |
b2302c7fd percpu_ref: reorg... |
202 |
if (ref->percpu_count_ptr & __PERCPU_REF_ATOMIC) { |
18808354b percpu_ref: unify... |
203 |
if (confirm_switch) |
b2302c7fd percpu_ref: reorg... |
204 |
confirm_switch(ref); |
b2302c7fd percpu_ref: reorg... |
205 |
return; |
490c79a65 percpu_ref: decou... |
206 |
} |
215e262f2 percpu: implement... |
207 |
|
b2302c7fd percpu_ref: reorg... |
208 209 210 211 212 213 214 |
/* switching from percpu to atomic */ ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC; /* * Non-NULL ->confirm_switch is used to indicate that switching is * in progress. Use noop one if unspecified. */ |
2b0d3d3e4 percpu_ref: reduc... |
215 216 |
ref->data->confirm_switch = confirm_switch ?: percpu_ref_noop_confirm_switch; |
b2302c7fd percpu_ref: reorg... |
217 218 |
percpu_ref_get(ref); /* put after confirmation */ |
2b0d3d3e4 percpu_ref: reduc... |
219 |
call_rcu(&ref->data->rcu, percpu_ref_switch_to_atomic_rcu); |
215e262f2 percpu: implement... |
220 |
} |
a22373701 percpu_ref: reloc... |
221 |
|
f47ad4578 percpu_ref: decou... |
222 |
static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) |
a22373701 percpu_ref: reloc... |
223 |
{ |
eecc16ba9 percpu_ref: repla... |
224 |
unsigned long __percpu *percpu_count = percpu_count_ptr(ref); |
a22373701 percpu_ref: reloc... |
225 |
int cpu; |
eecc16ba9 percpu_ref: repla... |
226 |
BUG_ON(!percpu_count); |
a22373701 percpu_ref: reloc... |
227 |
|
f47ad4578 percpu_ref: decou... |
228 229 |
if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC)) return; |
2b0d3d3e4 percpu_ref: reduc... |
230 |
if (WARN_ON_ONCE(!ref->data->allow_reinit)) |
7d9ab9b6a percpu_ref: relea... |
231 |
return; |
2b0d3d3e4 percpu_ref: reduc... |
232 |
atomic_long_add(PERCPU_COUNT_BIAS, &ref->data->count); |
a22373701 percpu_ref: reloc... |
233 234 |
/* |
b393e8b33 percpu: READ_ONCE... |
235 236 237 238 |
* Restore per-cpu operation. smp_store_release() is paired * with READ_ONCE() in __ref_is_percpu() and guarantees that the * zeroing is visible to all percpu accesses which can see the * following __PERCPU_REF_ATOMIC clearing. |
a22373701 percpu_ref: reloc... |
239 240 |
*/ for_each_possible_cpu(cpu) |
eecc16ba9 percpu_ref: repla... |
241 |
*per_cpu_ptr(percpu_count, cpu) = 0; |
a22373701 percpu_ref: reloc... |
242 |
|
eecc16ba9 percpu_ref: repla... |
243 |
smp_store_release(&ref->percpu_count_ptr, |
f47ad4578 percpu_ref: decou... |
244 245 |
ref->percpu_count_ptr & ~__PERCPU_REF_ATOMIC); } |
3f49bdd95 percpu_ref: restr... |
246 247 248 |
static void __percpu_ref_switch_mode(struct percpu_ref *ref, percpu_ref_func_t *confirm_switch) { |
2b0d3d3e4 percpu_ref: reduc... |
249 |
struct percpu_ref_data *data = ref->data; |
33e465ce7 percpu_ref: allow... |
250 |
lockdep_assert_held(&percpu_ref_switch_lock); |
3f49bdd95 percpu_ref: restr... |
251 252 253 254 |
/* * If the previous ATOMIC switching hasn't finished yet, wait for * its completion. If the caller ensures that ATOMIC switching * isn't in progress, this function can be called from any context. |
3f49bdd95 percpu_ref: restr... |
255 |
*/ |
2b0d3d3e4 percpu_ref: reduc... |
256 |
wait_event_lock_irq(percpu_ref_switch_waitq, !data->confirm_switch, |
33e465ce7 percpu_ref: allow... |
257 |
percpu_ref_switch_lock); |
3f49bdd95 percpu_ref: restr... |
258 |
|
2b0d3d3e4 percpu_ref: reduc... |
259 |
if (data->force_atomic || (ref->percpu_count_ptr & __PERCPU_REF_DEAD)) |
3f49bdd95 percpu_ref: restr... |
260 261 262 263 |
__percpu_ref_switch_to_atomic(ref, confirm_switch); else __percpu_ref_switch_to_percpu(ref); } |
f47ad4578 percpu_ref: decou... |
264 |
/** |
b2302c7fd percpu_ref: reorg... |
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 |
* percpu_ref_switch_to_atomic - switch a percpu_ref to atomic mode * @ref: percpu_ref to switch to atomic mode * @confirm_switch: optional confirmation callback * * There's no reason to use this function for the usual reference counting. * Use percpu_ref_kill[_and_confirm](). * * Schedule switching of @ref to atomic mode. All its percpu counts will * be collected to the main atomic counter. On completion, when all CPUs * are guaraneed to be in atomic mode, @confirm_switch, which may not * block, is invoked. This function may be invoked concurrently with all * the get/put operations and can safely be mixed with kill and reinit * operations. Note that @ref will stay in atomic mode across kill/reinit * cycles until percpu_ref_switch_to_percpu() is called. * |
3f49bdd95 percpu_ref: restr... |
280 281 282 |
* This function may block if @ref is in the process of switching to atomic * mode. If the caller ensures that @ref is not in the process of * switching to atomic mode, this function can be called from any context. |
b2302c7fd percpu_ref: reorg... |
283 284 285 286 |
*/ void percpu_ref_switch_to_atomic(struct percpu_ref *ref, percpu_ref_func_t *confirm_switch) { |
33e465ce7 percpu_ref: allow... |
287 288 289 |
unsigned long flags; spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
2b0d3d3e4 percpu_ref: reduc... |
290 |
ref->data->force_atomic = true; |
3f49bdd95 percpu_ref: restr... |
291 |
__percpu_ref_switch_mode(ref, confirm_switch); |
33e465ce7 percpu_ref: allow... |
292 293 |
spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
b2302c7fd percpu_ref: reorg... |
294 |
} |
210f7cdcf percpu-refcount: ... |
295 296 297 298 299 300 301 302 303 304 305 306 307 |
EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic); /** * percpu_ref_switch_to_atomic_sync - switch a percpu_ref to atomic mode * @ref: percpu_ref to switch to atomic mode * * Schedule switching the ref to atomic mode, and wait for the * switch to complete. Caller must ensure that no other thread * will switch back to percpu mode. */ void percpu_ref_switch_to_atomic_sync(struct percpu_ref *ref) { percpu_ref_switch_to_atomic(ref, NULL); |
2b0d3d3e4 percpu_ref: reduc... |
308 |
wait_event(percpu_ref_switch_waitq, !ref->data->confirm_switch); |
210f7cdcf percpu-refcount: ... |
309 310 |
} EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic_sync); |
b2302c7fd percpu_ref: reorg... |
311 312 |
/** |
f47ad4578 percpu_ref: decou... |
313 314 315 316 317 318 319 320 |
* percpu_ref_switch_to_percpu - switch a percpu_ref to percpu mode * @ref: percpu_ref to switch to percpu mode * * There's no reason to use this function for the usual reference counting. * To re-use an expired ref, use percpu_ref_reinit(). * * Switch @ref to percpu mode. This function may be invoked concurrently * with all the get/put operations and can safely be mixed with kill and |
1cae13e75 percpu_ref: make ... |
321 322 323 324 |
* reinit operations. This function reverses the sticky atomic state set * by PERCPU_REF_INIT_ATOMIC or percpu_ref_switch_to_atomic(). If @ref is * dying or dead, the actual switching takes place on the following * percpu_ref_reinit(). |
f47ad4578 percpu_ref: decou... |
325 |
* |
3f49bdd95 percpu_ref: restr... |
326 327 328 |
* This function may block if @ref is in the process of switching to atomic * mode. If the caller ensures that @ref is not in the process of * switching to atomic mode, this function can be called from any context. |
f47ad4578 percpu_ref: decou... |
329 330 331 |
*/ void percpu_ref_switch_to_percpu(struct percpu_ref *ref) { |
33e465ce7 percpu_ref: allow... |
332 333 334 |
unsigned long flags; spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
2b0d3d3e4 percpu_ref: reduc... |
335 |
ref->data->force_atomic = false; |
3f49bdd95 percpu_ref: restr... |
336 |
__percpu_ref_switch_mode(ref, NULL); |
33e465ce7 percpu_ref: allow... |
337 338 |
spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
a22373701 percpu_ref: reloc... |
339 |
} |
210f7cdcf percpu-refcount: ... |
340 |
EXPORT_SYMBOL_GPL(percpu_ref_switch_to_percpu); |
490c79a65 percpu_ref: decou... |
341 342 343 344 345 346 347 348 349 350 351 352 353 |
/** * percpu_ref_kill_and_confirm - drop the initial ref and schedule confirmation * @ref: percpu_ref to kill * @confirm_kill: optional confirmation callback * * Equivalent to percpu_ref_kill() but also schedules kill confirmation if * @confirm_kill is not NULL. @confirm_kill, which may not block, will be * called after @ref is seen as dead from all CPUs at which point all * further invocations of percpu_ref_tryget_live() will fail. See * percpu_ref_tryget_live() for details. * * This function normally doesn't block and can be called from any context |
f47ad4578 percpu_ref: decou... |
354 |
* but it may block if @confirm_kill is specified and @ref is in the |
a2f5630cb percpu_ref: remov... |
355 |
* process of switching to atomic mode by percpu_ref_switch_to_atomic(). |
b3a5d1119 percpu_ref: Updat... |
356 357 |
* * There are no implied RCU grace periods between kill and release. |
490c79a65 percpu_ref: decou... |
358 359 360 361 |
*/ void percpu_ref_kill_and_confirm(struct percpu_ref *ref, percpu_ref_func_t *confirm_kill) { |
33e465ce7 percpu_ref: allow... |
362 363 364 |
unsigned long flags; spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
490c79a65 percpu_ref: decou... |
365 |
WARN_ONCE(ref->percpu_count_ptr & __PERCPU_REF_DEAD, |
2b0d3d3e4 percpu_ref: reduc... |
366 367 |
"%s called more than once on %ps!", __func__, ref->data->release); |
490c79a65 percpu_ref: decou... |
368 369 |
ref->percpu_count_ptr |= __PERCPU_REF_DEAD; |
3f49bdd95 percpu_ref: restr... |
370 |
__percpu_ref_switch_mode(ref, confirm_kill); |
490c79a65 percpu_ref: decou... |
371 |
percpu_ref_put(ref); |
33e465ce7 percpu_ref: allow... |
372 373 |
spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
490c79a65 percpu_ref: decou... |
374 375 |
} EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm); |
f47ad4578 percpu_ref: decou... |
376 377 |
/** |
2b0d3d3e4 percpu_ref: reduc... |
378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 |
* percpu_ref_is_zero - test whether a percpu refcount reached zero * @ref: percpu_ref to test * * Returns %true if @ref reached zero. * * This function is safe to call as long as @ref is between init and exit. */ bool percpu_ref_is_zero(struct percpu_ref *ref) { unsigned long __percpu *percpu_count; unsigned long count, flags; if (__ref_is_percpu(ref, &percpu_count)) return false; /* protect us from being destroyed */ spin_lock_irqsave(&percpu_ref_switch_lock, flags); if (ref->data) count = atomic_long_read(&ref->data->count); else count = ref->percpu_count_ptr >> __PERCPU_REF_FLAG_BITS; spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); return count == 0; } EXPORT_SYMBOL_GPL(percpu_ref_is_zero); /** |
f47ad4578 percpu_ref: decou... |
406 407 408 409 |
* percpu_ref_reinit - re-initialize a percpu refcount * @ref: perpcu_ref to re-initialize * * Re-initialize @ref so that it's in the same state as when it finished |
1cae13e75 percpu_ref: make ... |
410 411 |
* percpu_ref_init() ignoring %PERCPU_REF_INIT_DEAD. @ref must have been * initialized successfully and reached 0 but not exited. |
f47ad4578 percpu_ref: decou... |
412 413 414 415 416 417 |
* * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while * this function is in progress. */ void percpu_ref_reinit(struct percpu_ref *ref) { |
18c9a6bbe percpu-refcount: ... |
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 |
WARN_ON_ONCE(!percpu_ref_is_zero(ref)); percpu_ref_resurrect(ref); } EXPORT_SYMBOL_GPL(percpu_ref_reinit); /** * percpu_ref_resurrect - modify a percpu refcount from dead to live * @ref: perpcu_ref to resurrect * * Modify @ref so that it's in the same state as before percpu_ref_kill() was * called. @ref must be dead but must not yet have exited. * * If @ref->release() frees @ref then the caller is responsible for * guaranteeing that @ref->release() does not get called while this * function is in progress. * * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while * this function is in progress. */ void percpu_ref_resurrect(struct percpu_ref *ref) { unsigned long __percpu *percpu_count; |
33e465ce7 percpu_ref: allow... |
441 442 443 |
unsigned long flags; spin_lock_irqsave(&percpu_ref_switch_lock, flags); |
18c9a6bbe percpu-refcount: ... |
444 445 |
WARN_ON_ONCE(!(ref->percpu_count_ptr & __PERCPU_REF_DEAD)); WARN_ON_ONCE(__ref_is_percpu(ref, &percpu_count)); |
f47ad4578 percpu_ref: decou... |
446 447 448 |
ref->percpu_count_ptr &= ~__PERCPU_REF_DEAD; percpu_ref_get(ref); |
3f49bdd95 percpu_ref: restr... |
449 |
__percpu_ref_switch_mode(ref, NULL); |
33e465ce7 percpu_ref: allow... |
450 451 |
spin_unlock_irqrestore(&percpu_ref_switch_lock, flags); |
f47ad4578 percpu_ref: decou... |
452 |
} |
18c9a6bbe percpu-refcount: ... |
453 |
EXPORT_SYMBOL_GPL(percpu_ref_resurrect); |