Blame view
drivers/dma-buf/dma-resv.c
16.4 KB
786d7257e reservation: cros... |
1 |
/* |
04a5faa8c reservation: upda... |
2 |
* Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst) |
786d7257e reservation: cros... |
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
* * Based on bo.c which bears the following copyright notice, * but is dual licensed: * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA * All Rights Reserved. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sub license, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice (including the * next paragraph) shall be included in all copies or substantial portions * of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE * USE OR OTHER DEALINGS IN THE SOFTWARE. * **************************************************************************/ /* * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> */ |
52791eeec dma-buf: rename r... |
34 |
#include <linux/dma-resv.h> |
786d7257e reservation: cros... |
35 |
#include <linux/export.h> |
0adf65f53 DMA reservations:... |
36 |
#include <linux/mm.h> |
b2a8116e2 dma_resv: prime l... |
37 |
#include <linux/sched/mm.h> |
d0b9a9aef dma-fence: prime ... |
38 |
#include <linux/mmu_notifier.h> |
786d7257e reservation: cros... |
39 |
|
dad6c3945 reservation: add ... |
40 41 42 43 44 45 46 47 48 49 |
/** * DOC: Reservation Object Overview * * The reservation object provides a mechanism to manage shared and * exclusive fences associated with a buffer. A reservation object * can have attached one exclusive fence (normally associated with * write operations) or N shared fences (read operations). The RCU * mechanism is used to protect read access to fences from locked * write-side updates. */ |
08295b3b5 locking: Implemen... |
50 |
DEFINE_WD_CLASS(reservation_ww_class); |
786d7257e reservation: cros... |
51 |
EXPORT_SYMBOL(reservation_ww_class); |
04a5faa8c reservation: upda... |
52 |
|
dad6c3945 reservation: add ... |
53 |
/** |
52791eeec dma-buf: rename r... |
54 |
* dma_resv_list_alloc - allocate fence list |
96e95496b dma-buf: fix shar... |
55 56 |
* @shared_max: number of fences we need space for * |
52791eeec dma-buf: rename r... |
57 |
* Allocate a new dma_resv_list and make sure to correctly initialize |
96e95496b dma-buf: fix shar... |
58 59 |
* shared_max. */ |
52791eeec dma-buf: rename r... |
60 |
static struct dma_resv_list *dma_resv_list_alloc(unsigned int shared_max) |
96e95496b dma-buf: fix shar... |
61 |
{ |
52791eeec dma-buf: rename r... |
62 |
struct dma_resv_list *list; |
96e95496b dma-buf: fix shar... |
63 64 65 66 67 68 69 70 71 72 73 74 |
list = kmalloc(offsetof(typeof(*list), shared[shared_max]), GFP_KERNEL); if (!list) return NULL; list->shared_max = (ksize(list) - offsetof(typeof(*list), shared)) / sizeof(*list->shared); return list; } /** |
52791eeec dma-buf: rename r... |
75 |
* dma_resv_list_free - free fence list |
96e95496b dma-buf: fix shar... |
76 77 |
* @list: list to free * |
52791eeec dma-buf: rename r... |
78 |
* Free a dma_resv_list and make sure to drop all references. |
96e95496b dma-buf: fix shar... |
79 |
*/ |
52791eeec dma-buf: rename r... |
80 |
static void dma_resv_list_free(struct dma_resv_list *list) |
96e95496b dma-buf: fix shar... |
81 82 83 84 85 86 87 88 89 90 91 |
{ unsigned int i; if (!list) return; for (i = 0; i < list->shared_count; ++i) dma_fence_put(rcu_dereference_protected(list->shared[i], true)); kfree_rcu(list, rcu); } |
b2a8116e2 dma_resv: prime l... |
92 |
#if IS_ENABLED(CONFIG_LOCKDEP) |
ffbbaa742 dma_resv: prime l... |
93 |
static int __init dma_resv_lockdep(void) |
b2a8116e2 dma_resv: prime l... |
94 95 |
{ struct mm_struct *mm = mm_alloc(); |
fedf7a441 dma-resv: Also pr... |
96 |
struct ww_acquire_ctx ctx; |
b2a8116e2 dma_resv: prime l... |
97 |
struct dma_resv obj; |
7dd1b884f dma-resv: lockdep... |
98 |
struct address_space mapping; |
fedf7a441 dma-resv: Also pr... |
99 |
int ret; |
b2a8116e2 dma_resv: prime l... |
100 101 |
if (!mm) |
ffbbaa742 dma_resv: prime l... |
102 |
return -ENOMEM; |
b2a8116e2 dma_resv: prime l... |
103 104 |
dma_resv_init(&obj); |
7dd1b884f dma-resv: lockdep... |
105 |
address_space_init_once(&mapping); |
b2a8116e2 dma_resv: prime l... |
106 |
|
0adf65f53 DMA reservations:... |
107 |
mmap_read_lock(mm); |
fedf7a441 dma-resv: Also pr... |
108 109 110 111 |
ww_acquire_init(&ctx, &reservation_ww_class); ret = dma_resv_lock(&obj, &ctx); if (ret == -EDEADLK) dma_resv_lock_slow(&obj, &ctx); |
b2a8116e2 dma_resv: prime l... |
112 |
fs_reclaim_acquire(GFP_KERNEL); |
7dd1b884f dma-resv: lockdep... |
113 114 115 |
/* for unmap_mapping_range on trylocked buffer objects in shrinkers */ i_mmap_lock_write(&mapping); i_mmap_unlock_write(&mapping); |
d0b9a9aef dma-fence: prime ... |
116 117 118 119 120 121 122 |
#ifdef CONFIG_MMU_NOTIFIER lock_map_acquire(&__mmu_notifier_invalidate_range_start_map); __dma_fence_might_wait(); lock_map_release(&__mmu_notifier_invalidate_range_start_map); #else __dma_fence_might_wait(); #endif |
b2a8116e2 dma_resv: prime l... |
123 124 |
fs_reclaim_release(GFP_KERNEL); ww_mutex_unlock(&obj.lock); |
fedf7a441 dma-resv: Also pr... |
125 |
ww_acquire_fini(&ctx); |
0adf65f53 DMA reservations:... |
126 |
mmap_read_unlock(mm); |
b2a8116e2 dma_resv: prime l... |
127 128 |
mmput(mm); |
ffbbaa742 dma_resv: prime l... |
129 130 |
return 0; |
b2a8116e2 dma_resv: prime l... |
131 132 133 |
} subsys_initcall(dma_resv_lockdep); #endif |
96e95496b dma-buf: fix shar... |
134 |
/** |
52791eeec dma-buf: rename r... |
135 |
* dma_resv_init - initialize a reservation object |
8735f1680 dma-buf: cleanup ... |
136 137 |
* @obj: the reservation object */ |
52791eeec dma-buf: rename r... |
138 |
void dma_resv_init(struct dma_resv *obj) |
8735f1680 dma-buf: cleanup ... |
139 140 |
{ ww_mutex_init(&obj->lock, &reservation_ww_class); |
cd29f2201 dma-buf: Use sequ... |
141 |
seqcount_ww_mutex_init(&obj->seq, &obj->lock); |
b016cd6ed dma-buf: Restore ... |
142 |
|
8735f1680 dma-buf: cleanup ... |
143 144 145 |
RCU_INIT_POINTER(obj->fence, NULL); RCU_INIT_POINTER(obj->fence_excl, NULL); } |
52791eeec dma-buf: rename r... |
146 |
EXPORT_SYMBOL(dma_resv_init); |
8735f1680 dma-buf: cleanup ... |
147 148 |
/** |
52791eeec dma-buf: rename r... |
149 |
* dma_resv_fini - destroys a reservation object |
8735f1680 dma-buf: cleanup ... |
150 151 |
* @obj: the reservation object */ |
52791eeec dma-buf: rename r... |
152 |
void dma_resv_fini(struct dma_resv *obj) |
8735f1680 dma-buf: cleanup ... |
153 |
{ |
52791eeec dma-buf: rename r... |
154 |
struct dma_resv_list *fobj; |
8735f1680 dma-buf: cleanup ... |
155 156 157 158 159 160 161 162 163 164 165 |
struct dma_fence *excl; /* * This object should be dead and all references must have * been released to it, so no need to be protected with rcu. */ excl = rcu_dereference_protected(obj->fence_excl, 1); if (excl) dma_fence_put(excl); fobj = rcu_dereference_protected(obj->fence, 1); |
52791eeec dma-buf: rename r... |
166 |
dma_resv_list_free(fobj); |
8735f1680 dma-buf: cleanup ... |
167 168 |
ww_mutex_destroy(&obj->lock); } |
52791eeec dma-buf: rename r... |
169 |
EXPORT_SYMBOL(dma_resv_fini); |
8735f1680 dma-buf: cleanup ... |
170 171 |
/** |
52791eeec dma-buf: rename r... |
172 173 |
* dma_resv_reserve_shared - Reserve space to add shared fences to * a dma_resv. |
dad6c3945 reservation: add ... |
174 |
* @obj: reservation object |
ca05359f1 dma-buf: allow re... |
175 |
* @num_fences: number of fences we want to add |
dad6c3945 reservation: add ... |
176 |
* |
52791eeec dma-buf: rename r... |
177 |
* Should be called before dma_resv_add_shared_fence(). Must |
dad6c3945 reservation: add ... |
178 179 180 181 |
* be called with obj->lock held. * * RETURNS * Zero for success, or -errno |
04a5faa8c reservation: upda... |
182 |
*/ |
52791eeec dma-buf: rename r... |
183 |
int dma_resv_reserve_shared(struct dma_resv *obj, unsigned int num_fences) |
04a5faa8c reservation: upda... |
184 |
{ |
52791eeec dma-buf: rename r... |
185 |
struct dma_resv_list *old, *new; |
27836b641 dma-buf: remove s... |
186 |
unsigned int i, j, k, max; |
04a5faa8c reservation: upda... |
187 |
|
52791eeec dma-buf: rename r... |
188 |
dma_resv_assert_held(obj); |
547c7138b dma-buf: add some... |
189 |
|
52791eeec dma-buf: rename r... |
190 |
old = dma_resv_get_list(obj); |
04a5faa8c reservation: upda... |
191 192 |
if (old && old->shared_max) { |
ca05359f1 dma-buf: allow re... |
193 |
if ((old->shared_count + num_fences) <= old->shared_max) |
04a5faa8c reservation: upda... |
194 |
return 0; |
27836b641 dma-buf: remove s... |
195 |
else |
ca05359f1 dma-buf: allow re... |
196 197 |
max = max(old->shared_count + num_fences, old->shared_max * 2); |
ca25fe5ef dma-buf: try to r... |
198 |
} else { |
0c500d6a5 dma-buf/dma-resv:... |
199 |
max = max(4ul, roundup_pow_of_two(num_fences)); |
ca25fe5ef dma-buf: try to r... |
200 |
} |
3c3b177a9 reservation: add ... |
201 |
|
52791eeec dma-buf: rename r... |
202 |
new = dma_resv_list_alloc(max); |
27836b641 dma-buf: remove s... |
203 204 |
if (!new) return -ENOMEM; |
04a5faa8c reservation: upda... |
205 206 207 208 209 210 211 |
/* * no need to bump fence refcounts, rcu_read access * requires the use of kref_get_unless_zero, and the * references from the old struct are carried over to * the new. */ |
27836b641 dma-buf: remove s... |
212 213 |
for (i = 0, j = 0, k = max; i < (old ? old->shared_count : 0); ++i) { struct dma_fence *fence; |
3c3b177a9 reservation: add ... |
214 |
|
27836b641 dma-buf: remove s... |
215 |
fence = rcu_dereference_protected(old->shared[i], |
52791eeec dma-buf: rename r... |
216 |
dma_resv_held(obj)); |
27836b641 dma-buf: remove s... |
217 218 |
if (dma_fence_is_signaled(fence)) RCU_INIT_POINTER(new->shared[--k], fence); |
4d9c62e8c dma-buf: keep onl... |
219 |
else |
27836b641 dma-buf: remove s... |
220 |
RCU_INIT_POINTER(new->shared[j++], fence); |
04a5faa8c reservation: upda... |
221 |
} |
27836b641 dma-buf: remove s... |
222 |
new->shared_count = j; |
04a5faa8c reservation: upda... |
223 |
|
3c3b177a9 reservation: add ... |
224 |
/* |
30fe7b07f dma-buf: Relax th... |
225 226 227 228 229 230 |
* We are not changing the effective set of fences here so can * merely update the pointer to the new array; both existing * readers and new readers will see exactly the same set of * active (unsignaled) shared fences. Individual fences and the * old array are protected by RCU and so will not vanish under * the gaze of the rcu_read_lock() readers. |
3c3b177a9 reservation: add ... |
231 |
*/ |
30fe7b07f dma-buf: Relax th... |
232 |
rcu_assign_pointer(obj->fence, new); |
3c3b177a9 reservation: add ... |
233 |
|
4d9c62e8c dma-buf: keep onl... |
234 |
if (!old) |
27836b641 dma-buf: remove s... |
235 |
return 0; |
3c3b177a9 reservation: add ... |
236 |
|
4d9c62e8c dma-buf: keep onl... |
237 |
/* Drop the references to the signaled fences */ |
94eb1e10a dma-buf: Expand r... |
238 |
for (i = k; i < max; ++i) { |
27836b641 dma-buf: remove s... |
239 |
struct dma_fence *fence; |
4d9c62e8c dma-buf: keep onl... |
240 |
|
27836b641 dma-buf: remove s... |
241 |
fence = rcu_dereference_protected(new->shared[i], |
52791eeec dma-buf: rename r... |
242 |
dma_resv_held(obj)); |
27836b641 dma-buf: remove s... |
243 |
dma_fence_put(fence); |
4d9c62e8c dma-buf: keep onl... |
244 245 |
} kfree_rcu(old, rcu); |
27836b641 dma-buf: remove s... |
246 247 |
return 0; |
04a5faa8c reservation: upda... |
248 |
} |
52791eeec dma-buf: rename r... |
249 |
EXPORT_SYMBOL(dma_resv_reserve_shared); |
04a5faa8c reservation: upda... |
250 |
|
dad6c3945 reservation: add ... |
251 |
/** |
52791eeec dma-buf: rename r... |
252 |
* dma_resv_add_shared_fence - Add a fence to a shared slot |
dad6c3945 reservation: add ... |
253 254 255 |
* @obj: the reservation object * @fence: the shared fence to add * |
04a5faa8c reservation: upda... |
256 |
* Add a fence to a shared slot, obj->lock must be held, and |
52791eeec dma-buf: rename r... |
257 |
* dma_resv_reserve_shared() has been called. |
04a5faa8c reservation: upda... |
258 |
*/ |
52791eeec dma-buf: rename r... |
259 |
void dma_resv_add_shared_fence(struct dma_resv *obj, struct dma_fence *fence) |
04a5faa8c reservation: upda... |
260 |
{ |
52791eeec dma-buf: rename r... |
261 |
struct dma_resv_list *fobj; |
93505ee7d dma-buf: fix busy... |
262 |
struct dma_fence *old; |
a590d0fdb dma-buf: Update r... |
263 |
unsigned int i, count; |
04a5faa8c reservation: upda... |
264 |
|
27836b641 dma-buf: remove s... |
265 |
dma_fence_get(fence); |
52791eeec dma-buf: rename r... |
266 |
dma_resv_assert_held(obj); |
547c7138b dma-buf: add some... |
267 |
|
52791eeec dma-buf: rename r... |
268 |
fobj = dma_resv_get_list(obj); |
a590d0fdb dma-buf: Update r... |
269 |
count = fobj->shared_count; |
04a5faa8c reservation: upda... |
270 |
|
b016cd6ed dma-buf: Restore ... |
271 |
write_seqcount_begin(&obj->seq); |
a590d0fdb dma-buf: Update r... |
272 |
for (i = 0; i < count; ++i) { |
27836b641 dma-buf: remove s... |
273 |
|
93505ee7d dma-buf: fix busy... |
274 |
old = rcu_dereference_protected(fobj->shared[i], |
52791eeec dma-buf: rename r... |
275 |
dma_resv_held(obj)); |
93505ee7d dma-buf: fix busy... |
276 277 |
if (old->context == fence->context || dma_fence_is_signaled(old)) |
27836b641 dma-buf: remove s... |
278 |
goto replace; |
27836b641 dma-buf: remove s... |
279 280 281 |
} BUG_ON(fobj->shared_count >= fobj->shared_max); |
93505ee7d dma-buf: fix busy... |
282 |
old = NULL; |
a590d0fdb dma-buf: Update r... |
283 |
count++; |
27836b641 dma-buf: remove s... |
284 285 |
replace: |
27836b641 dma-buf: remove s... |
286 |
RCU_INIT_POINTER(fobj->shared[i], fence); |
a590d0fdb dma-buf: Update r... |
287 288 |
/* pointer update must be visible before we extend the shared_count */ smp_store_mb(fobj->shared_count, count); |
b016cd6ed dma-buf: Restore ... |
289 290 |
write_seqcount_end(&obj->seq); |
93505ee7d dma-buf: fix busy... |
291 |
dma_fence_put(old); |
04a5faa8c reservation: upda... |
292 |
} |
52791eeec dma-buf: rename r... |
293 |
EXPORT_SYMBOL(dma_resv_add_shared_fence); |
04a5faa8c reservation: upda... |
294 |
|
dad6c3945 reservation: add ... |
295 |
/** |
52791eeec dma-buf: rename r... |
296 |
* dma_resv_add_excl_fence - Add an exclusive fence. |
dad6c3945 reservation: add ... |
297 298 299 300 301 |
* @obj: the reservation object * @fence: the shared fence to add * * Add a fence to the exclusive slot. The obj->lock must be held. */ |
52791eeec dma-buf: rename r... |
302 |
void dma_resv_add_excl_fence(struct dma_resv *obj, struct dma_fence *fence) |
04a5faa8c reservation: upda... |
303 |
{ |
52791eeec dma-buf: rename r... |
304 305 |
struct dma_fence *old_fence = dma_resv_get_excl(obj); struct dma_resv_list *old; |
04a5faa8c reservation: upda... |
306 |
u32 i = 0; |
52791eeec dma-buf: rename r... |
307 |
dma_resv_assert_held(obj); |
547c7138b dma-buf: add some... |
308 |
|
52791eeec dma-buf: rename r... |
309 |
old = dma_resv_get_list(obj); |
3c3b177a9 reservation: add ... |
310 |
if (old) |
04a5faa8c reservation: upda... |
311 |
i = old->shared_count; |
04a5faa8c reservation: upda... |
312 313 |
if (fence) |
f54d18670 dma-buf: Rename s... |
314 |
dma_fence_get(fence); |
04a5faa8c reservation: upda... |
315 |
|
b016cd6ed dma-buf: Restore ... |
316 317 318 |
write_seqcount_begin(&obj->seq); /* write_seqcount_begin provides the necessary memory barrier */ RCU_INIT_POINTER(obj->fence_excl, fence); |
3c3b177a9 reservation: add ... |
319 |
if (old) |
b016cd6ed dma-buf: Restore ... |
320 321 |
old->shared_count = 0; write_seqcount_end(&obj->seq); |
04a5faa8c reservation: upda... |
322 323 324 |
/* inplace update, no shared fences */ while (i--) |
f54d18670 dma-buf: Rename s... |
325 |
dma_fence_put(rcu_dereference_protected(old->shared[i], |
52791eeec dma-buf: rename r... |
326 |
dma_resv_held(obj))); |
04a5faa8c reservation: upda... |
327 |
|
f3e31b73a dma-buf: dma_fenc... |
328 |
dma_fence_put(old_fence); |
04a5faa8c reservation: upda... |
329 |
} |
52791eeec dma-buf: rename r... |
330 |
EXPORT_SYMBOL(dma_resv_add_excl_fence); |
3c3b177a9 reservation: add ... |
331 |
|
dad6c3945 reservation: add ... |
332 |
/** |
52791eeec dma-buf: rename r... |
333 |
* dma_resv_copy_fences - Copy all fences from src to dst. |
7faf952a3 dma-buf: add rese... |
334 335 336 |
* @dst: the destination reservation object * @src: the source reservation object * |
39e16ba16 dma-buf: make res... |
337 |
* Copy all fences from src to dst. dst-lock must be held. |
7faf952a3 dma-buf: add rese... |
338 |
*/ |
52791eeec dma-buf: rename r... |
339 |
int dma_resv_copy_fences(struct dma_resv *dst, struct dma_resv *src) |
7faf952a3 dma-buf: add rese... |
340 |
{ |
52791eeec dma-buf: rename r... |
341 |
struct dma_resv_list *src_list, *dst_list; |
7faf952a3 dma-buf: add rese... |
342 |
struct dma_fence *old, *new; |
b016cd6ed dma-buf: Restore ... |
343 |
unsigned i; |
7faf952a3 dma-buf: add rese... |
344 |
|
52791eeec dma-buf: rename r... |
345 |
dma_resv_assert_held(dst); |
547c7138b dma-buf: add some... |
346 |
|
39e16ba16 dma-buf: make res... |
347 |
rcu_read_lock(); |
b016cd6ed dma-buf: Restore ... |
348 |
src_list = rcu_dereference(src->fence); |
7faf952a3 dma-buf: add rese... |
349 |
|
39e16ba16 dma-buf: make res... |
350 |
retry: |
b016cd6ed dma-buf: Restore ... |
351 352 |
if (src_list) { unsigned shared_count = src_list->shared_count; |
39e16ba16 dma-buf: make res... |
353 |
rcu_read_unlock(); |
52791eeec dma-buf: rename r... |
354 |
dst_list = dma_resv_list_alloc(shared_count); |
7faf952a3 dma-buf: add rese... |
355 356 |
if (!dst_list) return -ENOMEM; |
39e16ba16 dma-buf: make res... |
357 |
rcu_read_lock(); |
b016cd6ed dma-buf: Restore ... |
358 359 |
src_list = rcu_dereference(src->fence); if (!src_list || src_list->shared_count > shared_count) { |
39e16ba16 dma-buf: make res... |
360 361 362 363 364 |
kfree(dst_list); goto retry; } dst_list->shared_count = 0; |
b016cd6ed dma-buf: Restore ... |
365 |
for (i = 0; i < src_list->shared_count; ++i) { |
39e16ba16 dma-buf: make res... |
366 367 368 369 370 371 372 373 |
struct dma_fence *fence; fence = rcu_dereference(src_list->shared[i]); if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) continue; if (!dma_fence_get_rcu(fence)) { |
52791eeec dma-buf: rename r... |
374 |
dma_resv_list_free(dst_list); |
b016cd6ed dma-buf: Restore ... |
375 |
src_list = rcu_dereference(src->fence); |
39e16ba16 dma-buf: make res... |
376 377 378 379 380 381 382 |
goto retry; } if (dma_fence_is_signaled(fence)) { dma_fence_put(fence); continue; } |
ad46d7b89 dma-buf: Use rcu_... |
383 |
rcu_assign_pointer(dst_list->shared[dst_list->shared_count++], fence); |
39e16ba16 dma-buf: make res... |
384 |
} |
7faf952a3 dma-buf: add rese... |
385 386 387 |
} else { dst_list = NULL; } |
b016cd6ed dma-buf: Restore ... |
388 |
new = dma_fence_get_rcu_safe(&src->fence_excl); |
39e16ba16 dma-buf: make res... |
389 |
rcu_read_unlock(); |
52791eeec dma-buf: rename r... |
390 391 |
src_list = dma_resv_get_list(dst); old = dma_resv_get_excl(dst); |
7faf952a3 dma-buf: add rese... |
392 |
|
b016cd6ed dma-buf: Restore ... |
393 394 395 396 397 |
write_seqcount_begin(&dst->seq); /* write_seqcount_begin provides the necessary memory barrier */ RCU_INIT_POINTER(dst->fence_excl, new); RCU_INIT_POINTER(dst->fence, dst_list); write_seqcount_end(&dst->seq); |
7faf952a3 dma-buf: add rese... |
398 |
|
52791eeec dma-buf: rename r... |
399 |
dma_resv_list_free(src_list); |
7faf952a3 dma-buf: add rese... |
400 401 402 403 |
dma_fence_put(old); return 0; } |
52791eeec dma-buf: rename r... |
404 |
EXPORT_SYMBOL(dma_resv_copy_fences); |
7faf952a3 dma-buf: add rese... |
405 406 |
/** |
52791eeec dma-buf: rename r... |
407 |
* dma_resv_get_fences_rcu - Get an object's shared and exclusive |
dad6c3945 reservation: add ... |
408 409 410 411 412 413 414 |
* fences without update side lock held * @obj: the reservation object * @pfence_excl: the returned exclusive fence (or NULL) * @pshared_count: the number of shared fences returned * @pshared: the array of shared fence ptrs returned (array is krealloc'd to * the required size, and must be freed by caller) * |
a35f2f34b dma-buf: make ret... |
415 416 417 |
* Retrieve all fences from the reservation object. If the pointer for the * exclusive fence is not specified the fence is put into the array of the * shared fences as well. Returns either zero or -ENOMEM. |
dad6c3945 reservation: add ... |
418 |
*/ |
52791eeec dma-buf: rename r... |
419 420 421 422 |
int dma_resv_get_fences_rcu(struct dma_resv *obj, struct dma_fence **pfence_excl, unsigned *pshared_count, struct dma_fence ***pshared) |
3c3b177a9 reservation: add ... |
423 |
{ |
f54d18670 dma-buf: Rename s... |
424 425 |
struct dma_fence **shared = NULL; struct dma_fence *fence_excl; |
fedf54132 dma-buf: Restart ... |
426 427 |
unsigned int shared_count; int ret = 1; |
3c3b177a9 reservation: add ... |
428 |
|
fedf54132 dma-buf: Restart ... |
429 |
do { |
52791eeec dma-buf: rename r... |
430 |
struct dma_resv_list *fobj; |
b016cd6ed dma-buf: Restore ... |
431 |
unsigned int i, seq; |
a35f2f34b dma-buf: make ret... |
432 |
size_t sz = 0; |
3c3b177a9 reservation: add ... |
433 |
|
b016cd6ed dma-buf: Restore ... |
434 |
shared_count = i = 0; |
3c3b177a9 reservation: add ... |
435 436 |
rcu_read_lock(); |
b016cd6ed dma-buf: Restore ... |
437 |
seq = read_seqcount_begin(&obj->seq); |
fedf54132 dma-buf: Restart ... |
438 |
|
b016cd6ed dma-buf: Restore ... |
439 |
fence_excl = rcu_dereference(obj->fence_excl); |
f54d18670 dma-buf: Rename s... |
440 |
if (fence_excl && !dma_fence_get_rcu(fence_excl)) |
fedf54132 dma-buf: Restart ... |
441 |
goto unlock; |
3c3b177a9 reservation: add ... |
442 |
|
b016cd6ed dma-buf: Restore ... |
443 |
fobj = rcu_dereference(obj->fence); |
a35f2f34b dma-buf: make ret... |
444 445 446 447 448 449 450 |
if (fobj) sz += sizeof(*shared) * fobj->shared_max; if (!pfence_excl && fence_excl) sz += sizeof(*shared); if (sz) { |
f54d18670 dma-buf: Rename s... |
451 |
struct dma_fence **nshared; |
3c3b177a9 reservation: add ... |
452 453 454 455 456 |
nshared = krealloc(shared, sz, GFP_NOWAIT | __GFP_NOWARN); if (!nshared) { rcu_read_unlock(); |
f5b07b04e dma-buf: Discard ... |
457 458 459 |
dma_fence_put(fence_excl); fence_excl = NULL; |
3c3b177a9 reservation: add ... |
460 461 462 463 464 465 466 |
nshared = krealloc(shared, sz, GFP_KERNEL); if (nshared) { shared = nshared; continue; } ret = -ENOMEM; |
3c3b177a9 reservation: add ... |
467 468 469 |
break; } shared = nshared; |
b016cd6ed dma-buf: Restore ... |
470 |
shared_count = fobj ? fobj->shared_count : 0; |
3c3b177a9 reservation: add ... |
471 |
for (i = 0; i < shared_count; ++i) { |
fedf54132 dma-buf: Restart ... |
472 |
shared[i] = rcu_dereference(fobj->shared[i]); |
f54d18670 dma-buf: Rename s... |
473 |
if (!dma_fence_get_rcu(shared[i])) |
fedf54132 dma-buf: Restart ... |
474 |
break; |
3c3b177a9 reservation: add ... |
475 |
} |
fedf54132 dma-buf: Restart ... |
476 |
} |
3c3b177a9 reservation: add ... |
477 |
|
b016cd6ed dma-buf: Restore ... |
478 |
if (i != shared_count || read_seqcount_retry(&obj->seq, seq)) { |
fedf54132 dma-buf: Restart ... |
479 |
while (i--) |
f54d18670 dma-buf: Rename s... |
480 481 |
dma_fence_put(shared[i]); dma_fence_put(fence_excl); |
fedf54132 dma-buf: Restart ... |
482 483 484 485 |
goto unlock; } ret = 0; |
3c3b177a9 reservation: add ... |
486 487 |
unlock: rcu_read_unlock(); |
fedf54132 dma-buf: Restart ... |
488 |
} while (ret); |
b8c036dfc dma-buf: simplify... |
489 490 491 |
if (pfence_excl) *pfence_excl = fence_excl; else if (fence_excl) |
7fbd0782b dma-buf/resv: fix... |
492 |
shared[shared_count++] = fence_excl; |
b8c036dfc dma-buf: simplify... |
493 |
|
fedf54132 dma-buf: Restart ... |
494 |
if (!shared_count) { |
3c3b177a9 reservation: add ... |
495 |
kfree(shared); |
fedf54132 dma-buf: Restart ... |
496 |
shared = NULL; |
3c3b177a9 reservation: add ... |
497 |
} |
fedf54132 dma-buf: Restart ... |
498 499 500 |
*pshared_count = shared_count; *pshared = shared; |
3c3b177a9 reservation: add ... |
501 502 |
return ret; } |
52791eeec dma-buf: rename r... |
503 |
EXPORT_SYMBOL_GPL(dma_resv_get_fences_rcu); |
3c3b177a9 reservation: add ... |
504 |
|
dad6c3945 reservation: add ... |
505 |
/** |
52791eeec dma-buf: rename r... |
506 |
* dma_resv_wait_timeout_rcu - Wait on reservation's objects |
dad6c3945 reservation: add ... |
507 508 509 510 511 512 513 514 515 516 |
* shared and/or exclusive fences. * @obj: the reservation object * @wait_all: if true, wait on all fences, else wait on just exclusive fence * @intr: if true, do interruptible wait * @timeout: timeout value in jiffies or zero to return immediately * * RETURNS * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or * greater than zer on success. */ |
52791eeec dma-buf: rename r... |
517 518 519 |
long dma_resv_wait_timeout_rcu(struct dma_resv *obj, bool wait_all, bool intr, unsigned long timeout) |
3c3b177a9 reservation: add ... |
520 |
{ |
f54d18670 dma-buf: Rename s... |
521 |
struct dma_fence *fence; |
b016cd6ed dma-buf: Restore ... |
522 |
unsigned seq, shared_count; |
06a66b5c7 reservation: reve... |
523 |
long ret = timeout ? timeout : 1; |
5bffee867 dma-buf: fix rese... |
524 |
int i; |
fb8b7d2b9 reservation: wait... |
525 |
|
3c3b177a9 reservation: add ... |
526 |
retry: |
b016cd6ed dma-buf: Restore ... |
527 528 |
shared_count = 0; seq = read_seqcount_begin(&obj->seq); |
3c3b177a9 reservation: add ... |
529 |
rcu_read_lock(); |
5bffee867 dma-buf: fix rese... |
530 |
i = -1; |
3c3b177a9 reservation: add ... |
531 |
|
b016cd6ed dma-buf: Restore ... |
532 |
fence = rcu_dereference(obj->fence_excl); |
b88fa004e dma-buf: fix rese... |
533 534 535 536 537 538 539 540 541 542 543 544 |
if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { if (!dma_fence_get_rcu(fence)) goto unlock_retry; if (dma_fence_is_signaled(fence)) { dma_fence_put(fence); fence = NULL; } } else { fence = NULL; } |
5bffee867 dma-buf: fix rese... |
545 |
if (wait_all) { |
b016cd6ed dma-buf: Restore ... |
546 547 548 549 |
struct dma_resv_list *fobj = rcu_dereference(obj->fence); if (fobj) shared_count = fobj->shared_count; |
5bffee867 dma-buf: fix rese... |
550 |
for (i = 0; !fence && i < shared_count; ++i) { |
f54d18670 dma-buf: Rename s... |
551 |
struct dma_fence *lfence = rcu_dereference(fobj->shared[i]); |
3c3b177a9 reservation: add ... |
552 |
|
f54d18670 dma-buf: Rename s... |
553 554 |
if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) |
3c3b177a9 reservation: add ... |
555 |
continue; |
f54d18670 dma-buf: Rename s... |
556 |
if (!dma_fence_get_rcu(lfence)) |
3c3b177a9 reservation: add ... |
557 |
goto unlock_retry; |
f54d18670 dma-buf: Rename s... |
558 559 |
if (dma_fence_is_signaled(lfence)) { dma_fence_put(lfence); |
3c3b177a9 reservation: add ... |
560 561 562 563 564 565 566 |
continue; } fence = lfence; break; } } |
3c3b177a9 reservation: add ... |
567 568 |
rcu_read_unlock(); if (fence) { |
b016cd6ed dma-buf: Restore ... |
569 570 571 572 |
if (read_seqcount_retry(&obj->seq, seq)) { dma_fence_put(fence); goto retry; } |
f54d18670 dma-buf: Rename s... |
573 574 |
ret = dma_fence_wait_timeout(fence, intr, ret); dma_fence_put(fence); |
3c3b177a9 reservation: add ... |
575 576 577 578 579 580 581 582 583 |
if (ret > 0 && wait_all && (i + 1 < shared_count)) goto retry; } return ret; unlock_retry: rcu_read_unlock(); goto retry; } |
52791eeec dma-buf: rename r... |
584 |
EXPORT_SYMBOL_GPL(dma_resv_wait_timeout_rcu); |
3c3b177a9 reservation: add ... |
585 |
|
52791eeec dma-buf: rename r... |
586 |
static inline int dma_resv_test_signaled_single(struct dma_fence *passed_fence) |
3c3b177a9 reservation: add ... |
587 |
{ |
f54d18670 dma-buf: Rename s... |
588 |
struct dma_fence *fence, *lfence = passed_fence; |
3c3b177a9 reservation: add ... |
589 |
int ret = 1; |
f54d18670 dma-buf: Rename s... |
590 591 |
if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) { fence = dma_fence_get_rcu(lfence); |
3c3b177a9 reservation: add ... |
592 593 |
if (!fence) return -1; |
f54d18670 dma-buf: Rename s... |
594 595 |
ret = !!dma_fence_is_signaled(fence); dma_fence_put(fence); |
3c3b177a9 reservation: add ... |
596 597 598 |
} return ret; } |
dad6c3945 reservation: add ... |
599 |
/** |
52791eeec dma-buf: rename r... |
600 |
* dma_resv_test_signaled_rcu - Test if a reservation object's |
dad6c3945 reservation: add ... |
601 602 603 604 605 606 607 608 |
* fences have been signaled. * @obj: the reservation object * @test_all: if true, test all fences, otherwise only test the exclusive * fence * * RETURNS * true if all fences signaled, else false */ |
52791eeec dma-buf: rename r... |
609 |
bool dma_resv_test_signaled_rcu(struct dma_resv *obj, bool test_all) |
3c3b177a9 reservation: add ... |
610 |
{ |
b016cd6ed dma-buf: Restore ... |
611 |
unsigned seq, shared_count; |
b68d8379c dma-buf: Restart ... |
612 |
int ret; |
3c3b177a9 reservation: add ... |
613 |
|
b68d8379c dma-buf: Restart ... |
614 |
rcu_read_lock(); |
3c3b177a9 reservation: add ... |
615 |
retry: |
b68d8379c dma-buf: Restart ... |
616 |
ret = true; |
b016cd6ed dma-buf: Restore ... |
617 618 |
shared_count = 0; seq = read_seqcount_begin(&obj->seq); |
3c3b177a9 reservation: add ... |
619 620 621 |
if (test_all) { unsigned i; |
b016cd6ed dma-buf: Restore ... |
622 623 624 625 |
struct dma_resv_list *fobj = rcu_dereference(obj->fence); if (fobj) shared_count = fobj->shared_count; |
3c3b177a9 reservation: add ... |
626 |
for (i = 0; i < shared_count; ++i) { |
f54d18670 dma-buf: Rename s... |
627 |
struct dma_fence *fence = rcu_dereference(fobj->shared[i]); |
3c3b177a9 reservation: add ... |
628 |
|
52791eeec dma-buf: rename r... |
629 |
ret = dma_resv_test_signaled_single(fence); |
3c3b177a9 reservation: add ... |
630 |
if (ret < 0) |
b68d8379c dma-buf: Restart ... |
631 |
goto retry; |
3c3b177a9 reservation: add ... |
632 633 634 |
else if (!ret) break; } |
3c3b177a9 reservation: add ... |
635 |
|
b016cd6ed dma-buf: Restore ... |
636 |
if (read_seqcount_retry(&obj->seq, seq)) |
67c97fb79 dma-buf: add rese... |
637 |
goto retry; |
3c3b177a9 reservation: add ... |
638 |
} |
b016cd6ed dma-buf: Restore ... |
639 640 641 642 643 644 645 646 647 648 649 650 |
if (!shared_count) { struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl); if (fence_excl) { ret = dma_resv_test_signaled_single(fence_excl); if (ret < 0) goto retry; if (read_seqcount_retry(&obj->seq, seq)) goto retry; } } |
3c3b177a9 reservation: add ... |
651 652 |
rcu_read_unlock(); return ret; |
3c3b177a9 reservation: add ... |
653 |
} |
52791eeec dma-buf: rename r... |
654 |
EXPORT_SYMBOL_GPL(dma_resv_test_signaled_rcu); |