Commit eee2acbae95555006307395d8a6c91452d62851d
Committed by
Linus Torvalds
Linus Torvalds
1 parent
bb4aa39676
Exists in
master
and in
7 other branches
mm: avoid repeated anon_vma lock/unlock sequences in unlink_anon_vmas()
This matches the anon_vma_clone() case, and uses the same lock helper functions. Because of the need to potentially release the anon_vma's, it's a bit more complex, though. We traverse the 'vma->anon_vma_chain' in two phases: the first loop gets the anon_vma lock (with the helper function that only takes the lock once for the whole loop), and removes any entries that don't need any more processing. The second phase just traverses the remaining list entries (without holding the anon_vma lock), and does any actual freeing of the anon_vma's that is required. Signed-off-by: Peter Zijlstra <peterz@infradead.org> Tested-by: Hugh Dickins <hughd@google.com> Tested-by: Tim Chen <tim.c.chen@linux.intel.com> Cc: Andi Kleen <ak@linux.intel.com> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Showing 1 changed file with 28 additions and 21 deletions Side-by-side Diff
mm/rmap.c
| ... | ... | @@ -324,36 +324,43 @@ |
| 324 | 324 | return -ENOMEM; |
| 325 | 325 | } |
| 326 | 326 | |
| 327 | -static void anon_vma_unlink(struct anon_vma_chain *anon_vma_chain) | |
| 328 | -{ | |
| 329 | - struct anon_vma *anon_vma = anon_vma_chain->anon_vma; | |
| 330 | - int empty; | |
| 331 | - | |
| 332 | - /* If anon_vma_fork fails, we can get an empty anon_vma_chain. */ | |
| 333 | - if (!anon_vma) | |
| 334 | - return; | |
| 335 | - | |
| 336 | - anon_vma_lock(anon_vma); | |
| 337 | - list_del(&anon_vma_chain->same_anon_vma); | |
| 338 | - | |
| 339 | - /* We must garbage collect the anon_vma if it's empty */ | |
| 340 | - empty = list_empty(&anon_vma->head); | |
| 341 | - anon_vma_unlock(anon_vma); | |
| 342 | - | |
| 343 | - if (empty) | |
| 344 | - put_anon_vma(anon_vma); | |
| 345 | -} | |
| 346 | - | |
| 347 | 327 | void unlink_anon_vmas(struct vm_area_struct *vma) |
| 348 | 328 | { |
| 349 | 329 | struct anon_vma_chain *avc, *next; |
| 330 | + struct anon_vma *root = NULL; | |
| 350 | 331 | |
| 351 | 332 | /* |
| 352 | 333 | * Unlink each anon_vma chained to the VMA. This list is ordered |
| 353 | 334 | * from newest to oldest, ensuring the root anon_vma gets freed last. |
| 354 | 335 | */ |
| 355 | 336 | list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { |
| 356 | - anon_vma_unlink(avc); | |
| 337 | + struct anon_vma *anon_vma = avc->anon_vma; | |
| 338 | + | |
| 339 | + root = lock_anon_vma_root(root, anon_vma); | |
| 340 | + list_del(&avc->same_anon_vma); | |
| 341 | + | |
| 342 | + /* | |
| 343 | + * Leave empty anon_vmas on the list - we'll need | |
| 344 | + * to free them outside the lock. | |
| 345 | + */ | |
| 346 | + if (list_empty(&anon_vma->head)) | |
| 347 | + continue; | |
| 348 | + | |
| 349 | + list_del(&avc->same_vma); | |
| 350 | + anon_vma_chain_free(avc); | |
| 351 | + } | |
| 352 | + unlock_anon_vma_root(root); | |
| 353 | + | |
| 354 | + /* | |
| 355 | + * Iterate the list once more, it now only contains empty and unlinked | |
| 356 | + * anon_vmas, destroy them. Could not do before due to __put_anon_vma() | |
| 357 | + * needing to acquire the anon_vma->root->mutex. | |
| 358 | + */ | |
| 359 | + list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { | |
| 360 | + struct anon_vma *anon_vma = avc->anon_vma; | |
| 361 | + | |
| 362 | + put_anon_vma(anon_vma); | |
| 363 | + | |
| 357 | 364 | list_del(&avc->same_vma); |
| 358 | 365 | anon_vma_chain_free(avc); |
| 359 | 366 | } |