Blame view
mm/rmap.c
51.2 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
/* * mm/rmap.c - physical to virtual reverse mappings * * Copyright 2001, Rik van Riel <riel@conectiva.com.br> * Released under the General Public License (GPL). * * Simple, low overhead reverse mapping scheme. * Please try to keep this thing as modular as possible. * * Provides methods for unmapping each kind of mapped page: * the anon methods track anonymous pages, and * the file methods track pages belonging to an inode. * * Original design by Rik van Riel <riel@conectiva.com.br> 2001 * File methods by Dave McCracken <dmccr@us.ibm.com> 2003, 2004 * Anonymous methods by Andrea Arcangeli <andrea@suse.de> 2004 |
98f32602d hugh: update emai... |
17 |
* Contributions by Hugh Dickins 2003, 2004 |
1da177e4c Linux-2.6.12-rc2 |
18 19 20 21 22 |
*/ /* * Lock ordering in mm: * |
1b1dcc1b5 [PATCH] mutex sub... |
23 |
* inode->i_mutex (while writing or truncating, not reading or faulting) |
82591e6ea [PATCH] mm: more ... |
24 25 |
* mm->mmap_sem * page->flags PG_locked (lock_page) |
88f306b68 mm: fix locking o... |
26 27 28 29 |
* hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share) * mapping->i_mmap_rwsem * anon_vma->rwsem * mm->page_table_lock or pte_lock |
a52633d8e mm, vmscan: move ... |
30 |
* zone_lru_lock (in mark_page_accessed, isolate_lru_page) |
88f306b68 mm: fix locking o... |
31 32 33 34 35 36 37 38 39 40 41 |
* swap_lock (in swap_duplicate, swap_info_get) * mmlist_lock (in mmput, drain_mmlist and others) * mapping->private_lock (in __set_page_dirty_buffers) * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) * mapping->tree_lock (widely used) * inode->i_lock (in set_page_dirty's __mark_inode_dirty) * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) * sb_lock (within inode_lock in fs/fs-writeback.c) * mapping->tree_lock (widely used, in set_page_dirty, * in arch-dependent flush_dcache_mmap_lock, * within bdi.wb->list_lock in __sync_single_inode) |
6a46079cf HWPOISON: The hig... |
42 |
* |
5a505085f mm/rmap: Convert ... |
43 |
* anon_vma->rwsem,mapping->i_mutex (memory_failure, collect_procs_anon) |
9b679320a mm/memory-failure... |
44 |
* ->tasklist_lock |
6a46079cf HWPOISON: The hig... |
45 |
* pte map lock |
1da177e4c Linux-2.6.12-rc2 |
46 47 48 49 50 51 52 53 |
*/ #include <linux/mm.h> #include <linux/pagemap.h> #include <linux/swap.h> #include <linux/swapops.h> #include <linux/slab.h> #include <linux/init.h> |
5ad646880 ksm: let shared p... |
54 |
#include <linux/ksm.h> |
1da177e4c Linux-2.6.12-rc2 |
55 56 |
#include <linux/rmap.h> #include <linux/rcupdate.h> |
b95f1b31b mm: Map most file... |
57 |
#include <linux/export.h> |
8a9f3ccd2 Memory controller... |
58 |
#include <linux/memcontrol.h> |
cddb8a5c1 mmu-notifiers: core |
59 |
#include <linux/mmu_notifier.h> |
64cdd548f mm: cleanup: remo... |
60 |
#include <linux/migrate.h> |
0fe6e20b9 hugetlb, rmap: ad... |
61 |
#include <linux/hugetlb.h> |
ef5d437f7 mm: fix XFS oops ... |
62 |
#include <linux/backing-dev.h> |
33c3fc71c mm: introduce idl... |
63 |
#include <linux/page_idle.h> |
1da177e4c Linux-2.6.12-rc2 |
64 65 |
#include <asm/tlbflush.h> |
72b252aed mm: send one IPI ... |
66 |
#include <trace/events/tlb.h> |
b291f0003 mlock: mlocked pa... |
67 |
#include "internal.h" |
fdd2e5f88 make mm/rmap.c:an... |
68 |
static struct kmem_cache *anon_vma_cachep; |
5beb49305 mm: change anon_v... |
69 |
static struct kmem_cache *anon_vma_chain_cachep; |
fdd2e5f88 make mm/rmap.c:an... |
70 71 72 |
static inline struct anon_vma *anon_vma_alloc(void) { |
01d8b20de mm: simplify anon... |
73 74 75 76 77 |
struct anon_vma *anon_vma; anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL); if (anon_vma) { atomic_set(&anon_vma->refcount, 1); |
7a3ef208e mm: prevent endle... |
78 79 |
anon_vma->degree = 1; /* Reference for first vma */ anon_vma->parent = anon_vma; |
01d8b20de mm: simplify anon... |
80 81 82 83 84 85 86 87 |
/* * Initialise the anon_vma root to point to itself. If called * from fork, the root will be reset to the parents anon_vma. */ anon_vma->root = anon_vma; } return anon_vma; |
fdd2e5f88 make mm/rmap.c:an... |
88 |
} |
01d8b20de mm: simplify anon... |
89 |
static inline void anon_vma_free(struct anon_vma *anon_vma) |
fdd2e5f88 make mm/rmap.c:an... |
90 |
{ |
01d8b20de mm: simplify anon... |
91 |
VM_BUG_ON(atomic_read(&anon_vma->refcount)); |
88c22088b mm: optimize page... |
92 93 |
/* |
4fc3f1d66 mm/rmap, migratio... |
94 |
* Synchronize against page_lock_anon_vma_read() such that |
88c22088b mm: optimize page... |
95 96 97 98 99 |
* we can safely hold the lock without the anon_vma getting * freed. * * Relies on the full mb implied by the atomic_dec_and_test() from * put_anon_vma() against the acquire barrier implied by |
4fc3f1d66 mm/rmap, migratio... |
100 |
* down_read_trylock() from page_lock_anon_vma_read(). This orders: |
88c22088b mm: optimize page... |
101 |
* |
4fc3f1d66 mm/rmap, migratio... |
102 103 |
* page_lock_anon_vma_read() VS put_anon_vma() * down_read_trylock() atomic_dec_and_test() |
88c22088b mm: optimize page... |
104 |
* LOCK MB |
4fc3f1d66 mm/rmap, migratio... |
105 |
* atomic_read() rwsem_is_locked() |
88c22088b mm: optimize page... |
106 107 108 109 |
* * LOCK should suffice since the actual taking of the lock must * happen _before_ what follows. */ |
7f39dda9d mm: fix sleeping ... |
110 |
might_sleep(); |
5a505085f mm/rmap: Convert ... |
111 |
if (rwsem_is_locked(&anon_vma->root->rwsem)) { |
4fc3f1d66 mm/rmap, migratio... |
112 |
anon_vma_lock_write(anon_vma); |
08b52706d mm/rmap: rename a... |
113 |
anon_vma_unlock_write(anon_vma); |
88c22088b mm: optimize page... |
114 |
} |
fdd2e5f88 make mm/rmap.c:an... |
115 116 |
kmem_cache_free(anon_vma_cachep, anon_vma); } |
1da177e4c Linux-2.6.12-rc2 |
117 |
|
dd34739c0 mm: avoid anon_vm... |
118 |
static inline struct anon_vma_chain *anon_vma_chain_alloc(gfp_t gfp) |
5beb49305 mm: change anon_v... |
119 |
{ |
dd34739c0 mm: avoid anon_vm... |
120 |
return kmem_cache_alloc(anon_vma_chain_cachep, gfp); |
5beb49305 mm: change anon_v... |
121 |
} |
e574b5fd2 rmap: make anon_v... |
122 |
static void anon_vma_chain_free(struct anon_vma_chain *anon_vma_chain) |
5beb49305 mm: change anon_v... |
123 124 125 |
{ kmem_cache_free(anon_vma_chain_cachep, anon_vma_chain); } |
6583a8430 rmap: anon_vma_pr... |
126 127 128 129 130 131 132 |
static void anon_vma_chain_link(struct vm_area_struct *vma, struct anon_vma_chain *avc, struct anon_vma *anon_vma) { avc->vma = vma; avc->anon_vma = anon_vma; list_add(&avc->same_vma, &vma->anon_vma_chain); |
bf181b9f9 mm anon rmap: rep... |
133 |
anon_vma_interval_tree_insert(avc, &anon_vma->rb_root); |
6583a8430 rmap: anon_vma_pr... |
134 |
} |
d9d332e08 anon_vma_prepare:... |
135 136 137 138 139 140 141 142 143 |
/** * anon_vma_prepare - attach an anon_vma to a memory region * @vma: the memory region in question * * This makes sure the memory mapping described by 'vma' has * an 'anon_vma' attached to it, so that we can associate the * anonymous pages mapped into it with that anon_vma. * * The common case will be that we already have one, but if |
23a0790af mm/rmap.c: fix co... |
144 |
* not we either need to find an adjacent mapping that we |
d9d332e08 anon_vma_prepare:... |
145 146 147 148 149 |
* can re-use the anon_vma from (very common when the only * reason for splitting a vma has been mprotect()), or we * allocate a new one. * * Anon-vma allocations are very subtle, because we may have |
4fc3f1d66 mm/rmap, migratio... |
150 |
* optimistically looked up an anon_vma in page_lock_anon_vma_read() |
d9d332e08 anon_vma_prepare:... |
151 152 153 154 155 156 157 158 159 160 161 |
* and that may actually touch the spinlock even in the newly * allocated vma (it depends on RCU to make sure that the * anon_vma isn't actually destroyed). * * As a result, we need to do proper anon_vma locking even * for the new allocation. At the same time, we do not want * to do any locking for the common case of already having * an anon_vma. * * This must be called with the mmap_sem held for reading. */ |
1da177e4c Linux-2.6.12-rc2 |
162 163 164 |
int anon_vma_prepare(struct vm_area_struct *vma) { struct anon_vma *anon_vma = vma->anon_vma; |
5beb49305 mm: change anon_v... |
165 |
struct anon_vma_chain *avc; |
1da177e4c Linux-2.6.12-rc2 |
166 167 168 169 |
might_sleep(); if (unlikely(!anon_vma)) { struct mm_struct *mm = vma->vm_mm; |
d9d332e08 anon_vma_prepare:... |
170 |
struct anon_vma *allocated; |
1da177e4c Linux-2.6.12-rc2 |
171 |
|
dd34739c0 mm: avoid anon_vm... |
172 |
avc = anon_vma_chain_alloc(GFP_KERNEL); |
5beb49305 mm: change anon_v... |
173 174 |
if (!avc) goto out_enomem; |
1da177e4c Linux-2.6.12-rc2 |
175 |
anon_vma = find_mergeable_anon_vma(vma); |
d9d332e08 anon_vma_prepare:... |
176 177 |
allocated = NULL; if (!anon_vma) { |
1da177e4c Linux-2.6.12-rc2 |
178 179 |
anon_vma = anon_vma_alloc(); if (unlikely(!anon_vma)) |
5beb49305 mm: change anon_v... |
180 |
goto out_enomem_free_avc; |
1da177e4c Linux-2.6.12-rc2 |
181 |
allocated = anon_vma; |
1da177e4c Linux-2.6.12-rc2 |
182 |
} |
4fc3f1d66 mm/rmap, migratio... |
183 |
anon_vma_lock_write(anon_vma); |
1da177e4c Linux-2.6.12-rc2 |
184 185 186 187 |
/* page_table_lock to protect against threads */ spin_lock(&mm->page_table_lock); if (likely(!vma->anon_vma)) { vma->anon_vma = anon_vma; |
6583a8430 rmap: anon_vma_pr... |
188 |
anon_vma_chain_link(vma, avc, anon_vma); |
7a3ef208e mm: prevent endle... |
189 190 |
/* vma reference or self-parent link for new root */ anon_vma->degree++; |
1da177e4c Linux-2.6.12-rc2 |
191 |
allocated = NULL; |
31f2b0ebc rmap: anon_vma_pr... |
192 |
avc = NULL; |
1da177e4c Linux-2.6.12-rc2 |
193 194 |
} spin_unlock(&mm->page_table_lock); |
08b52706d mm/rmap: rename a... |
195 |
anon_vma_unlock_write(anon_vma); |
31f2b0ebc rmap: anon_vma_pr... |
196 197 |
if (unlikely(allocated)) |
01d8b20de mm: simplify anon... |
198 |
put_anon_vma(allocated); |
31f2b0ebc rmap: anon_vma_pr... |
199 |
if (unlikely(avc)) |
5beb49305 mm: change anon_v... |
200 |
anon_vma_chain_free(avc); |
1da177e4c Linux-2.6.12-rc2 |
201 202 |
} return 0; |
5beb49305 mm: change anon_v... |
203 204 205 206 207 |
out_enomem_free_avc: anon_vma_chain_free(avc); out_enomem: return -ENOMEM; |
1da177e4c Linux-2.6.12-rc2 |
208 |
} |
bb4aa3967 mm: avoid repeate... |
209 210 211 212 213 214 215 216 217 218 219 220 221 |
/* * This is a useful helper function for locking the anon_vma root as * we traverse the vma->anon_vma_chain, looping over anon_vma's that * have the same vma. * * Such anon_vma's should have the same root, so you'd expect to see * just a single mutex_lock for the whole traversal. */ static inline struct anon_vma *lock_anon_vma_root(struct anon_vma *root, struct anon_vma *anon_vma) { struct anon_vma *new_root = anon_vma->root; if (new_root != root) { if (WARN_ON_ONCE(root)) |
5a505085f mm/rmap: Convert ... |
222 |
up_write(&root->rwsem); |
bb4aa3967 mm: avoid repeate... |
223 |
root = new_root; |
5a505085f mm/rmap: Convert ... |
224 |
down_write(&root->rwsem); |
bb4aa3967 mm: avoid repeate... |
225 226 227 228 229 230 231 |
} return root; } static inline void unlock_anon_vma_root(struct anon_vma *root) { if (root) |
5a505085f mm/rmap: Convert ... |
232 |
up_write(&root->rwsem); |
bb4aa3967 mm: avoid repeate... |
233 |
} |
5beb49305 mm: change anon_v... |
234 235 236 |
/* * Attach the anon_vmas from src to dst. * Returns 0 on success, -ENOMEM on failure. |
7a3ef208e mm: prevent endle... |
237 238 239 240 241 242 243 244 |
* * If dst->anon_vma is NULL this function tries to find and reuse existing * anon_vma which has no vmas and only one child anon_vma. This prevents * degradation of anon_vma hierarchy to endless linear chain in case of * constantly forking task. On the other hand, an anon_vma with more than one * child isn't reused even if there was no alive vma, thus rmap walker has a * good chance of avoiding scanning the whole hierarchy when it searches where * page is mapped. |
5beb49305 mm: change anon_v... |
245 246 |
*/ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src) |
1da177e4c Linux-2.6.12-rc2 |
247 |
{ |
5beb49305 mm: change anon_v... |
248 |
struct anon_vma_chain *avc, *pavc; |
bb4aa3967 mm: avoid repeate... |
249 |
struct anon_vma *root = NULL; |
5beb49305 mm: change anon_v... |
250 |
|
646d87b48 anon_vma: clone t... |
251 |
list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) { |
bb4aa3967 mm: avoid repeate... |
252 |
struct anon_vma *anon_vma; |
dd34739c0 mm: avoid anon_vm... |
253 254 255 256 257 258 259 260 |
avc = anon_vma_chain_alloc(GFP_NOWAIT | __GFP_NOWARN); if (unlikely(!avc)) { unlock_anon_vma_root(root); root = NULL; avc = anon_vma_chain_alloc(GFP_KERNEL); if (!avc) goto enomem_failure; } |
bb4aa3967 mm: avoid repeate... |
261 262 263 |
anon_vma = pavc->anon_vma; root = lock_anon_vma_root(root, anon_vma); anon_vma_chain_link(dst, avc, anon_vma); |
7a3ef208e mm: prevent endle... |
264 265 266 267 268 269 270 271 272 273 274 275 |
/* * Reuse existing anon_vma if its degree lower than two, * that means it has no vma and only one anon_vma child. * * Do not chose parent anon_vma, otherwise first child * will always reuse it. Root anon_vma is never reused: * it has self-parent reference and at least one child. */ if (!dst->anon_vma && anon_vma != src->anon_vma && anon_vma->degree < 2) dst->anon_vma = anon_vma; |
5beb49305 mm: change anon_v... |
276 |
} |
7a3ef208e mm: prevent endle... |
277 278 |
if (dst->anon_vma) dst->anon_vma->degree++; |
bb4aa3967 mm: avoid repeate... |
279 |
unlock_anon_vma_root(root); |
5beb49305 mm: change anon_v... |
280 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
281 |
|
5beb49305 mm: change anon_v... |
282 |
enomem_failure: |
3fe89b3e2 mm: fix anon_vma-... |
283 284 285 286 287 288 289 |
/* * dst->anon_vma is dropped here otherwise its degree can be incorrectly * decremented in unlink_anon_vmas(). * We can safely do this because callers of anon_vma_clone() don't care * about dst->anon_vma if anon_vma_clone() failed. */ dst->anon_vma = NULL; |
5beb49305 mm: change anon_v... |
290 291 |
unlink_anon_vmas(dst); return -ENOMEM; |
1da177e4c Linux-2.6.12-rc2 |
292 |
} |
5beb49305 mm: change anon_v... |
293 294 295 296 297 298 |
/* * Attach vma to its own anon_vma, as well as to the anon_vmas that * the corresponding VMA in the parent process is attached to. * Returns 0 on success, non-zero on failure. */ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma) |
1da177e4c Linux-2.6.12-rc2 |
299 |
{ |
5beb49305 mm: change anon_v... |
300 301 |
struct anon_vma_chain *avc; struct anon_vma *anon_vma; |
c4ea95d7c mm: fix anon_vma_... |
302 |
int error; |
1da177e4c Linux-2.6.12-rc2 |
303 |
|
5beb49305 mm: change anon_v... |
304 305 306 |
/* Don't bother if the parent process has no anon_vma here. */ if (!pvma->anon_vma) return 0; |
7a3ef208e mm: prevent endle... |
307 308 |
/* Drop inherited anon_vma, we'll reuse existing or allocate new. */ vma->anon_vma = NULL; |
5beb49305 mm: change anon_v... |
309 310 311 312 |
/* * First, attach the new VMA to the parent VMA's anon_vmas, * so rmap can find non-COWed pages in child processes. */ |
c4ea95d7c mm: fix anon_vma_... |
313 314 315 |
error = anon_vma_clone(vma, pvma); if (error) return error; |
5beb49305 mm: change anon_v... |
316 |
|
7a3ef208e mm: prevent endle... |
317 318 319 |
/* An existing anon_vma has been reused, all done then. */ if (vma->anon_vma) return 0; |
5beb49305 mm: change anon_v... |
320 321 322 323 |
/* Then add our own anon_vma. */ anon_vma = anon_vma_alloc(); if (!anon_vma) goto out_error; |
dd34739c0 mm: avoid anon_vm... |
324 |
avc = anon_vma_chain_alloc(GFP_KERNEL); |
5beb49305 mm: change anon_v... |
325 326 |
if (!avc) goto out_error_free_anon_vma; |
5c341ee1d mm: track the roo... |
327 328 329 330 331 332 |
/* * The root anon_vma's spinlock is the lock actually used when we * lock any of the anon_vmas in this anon_vma tree. */ anon_vma->root = pvma->anon_vma->root; |
7a3ef208e mm: prevent endle... |
333 |
anon_vma->parent = pvma->anon_vma; |
76545066c mm: extend KSM re... |
334 |
/* |
01d8b20de mm: simplify anon... |
335 336 337 |
* With refcounts, an anon_vma can stay around longer than the * process it belongs to. The root anon_vma needs to be pinned until * this anon_vma is freed, because the lock lives in the root. |
76545066c mm: extend KSM re... |
338 339 |
*/ get_anon_vma(anon_vma->root); |
5beb49305 mm: change anon_v... |
340 341 |
/* Mark this anon_vma as the one where our new (COWed) pages go. */ vma->anon_vma = anon_vma; |
4fc3f1d66 mm/rmap, migratio... |
342 |
anon_vma_lock_write(anon_vma); |
5c341ee1d mm: track the roo... |
343 |
anon_vma_chain_link(vma, avc, anon_vma); |
7a3ef208e mm: prevent endle... |
344 |
anon_vma->parent->degree++; |
08b52706d mm/rmap: rename a... |
345 |
anon_vma_unlock_write(anon_vma); |
5beb49305 mm: change anon_v... |
346 347 348 349 |
return 0; out_error_free_anon_vma: |
01d8b20de mm: simplify anon... |
350 |
put_anon_vma(anon_vma); |
5beb49305 mm: change anon_v... |
351 |
out_error: |
4946d54cb rmap: fix anon_vm... |
352 |
unlink_anon_vmas(vma); |
5beb49305 mm: change anon_v... |
353 |
return -ENOMEM; |
1da177e4c Linux-2.6.12-rc2 |
354 |
} |
5beb49305 mm: change anon_v... |
355 356 357 |
void unlink_anon_vmas(struct vm_area_struct *vma) { struct anon_vma_chain *avc, *next; |
eee2acbae mm: avoid repeate... |
358 |
struct anon_vma *root = NULL; |
5beb49305 mm: change anon_v... |
359 |
|
5c341ee1d mm: track the roo... |
360 361 362 363 |
/* * Unlink each anon_vma chained to the VMA. This list is ordered * from newest to oldest, ensuring the root anon_vma gets freed last. */ |
5beb49305 mm: change anon_v... |
364 |
list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { |
eee2acbae mm: avoid repeate... |
365 366 367 |
struct anon_vma *anon_vma = avc->anon_vma; root = lock_anon_vma_root(root, anon_vma); |
bf181b9f9 mm anon rmap: rep... |
368 |
anon_vma_interval_tree_remove(avc, &anon_vma->rb_root); |
eee2acbae mm: avoid repeate... |
369 370 371 372 373 |
/* * Leave empty anon_vmas on the list - we'll need * to free them outside the lock. */ |
7a3ef208e mm: prevent endle... |
374 375 |
if (RB_EMPTY_ROOT(&anon_vma->rb_root)) { anon_vma->parent->degree--; |
eee2acbae mm: avoid repeate... |
376 |
continue; |
7a3ef208e mm: prevent endle... |
377 |
} |
eee2acbae mm: avoid repeate... |
378 379 380 381 |
list_del(&avc->same_vma); anon_vma_chain_free(avc); } |
7a3ef208e mm: prevent endle... |
382 383 |
if (vma->anon_vma) vma->anon_vma->degree--; |
eee2acbae mm: avoid repeate... |
384 385 386 387 388 |
unlock_anon_vma_root(root); /* * Iterate the list once more, it now only contains empty and unlinked * anon_vmas, destroy them. Could not do before due to __put_anon_vma() |
5a505085f mm/rmap: Convert ... |
389 |
* needing to write-acquire the anon_vma->root->rwsem. |
eee2acbae mm: avoid repeate... |
390 391 392 |
*/ list_for_each_entry_safe(avc, next, &vma->anon_vma_chain, same_vma) { struct anon_vma *anon_vma = avc->anon_vma; |
e4c5800a3 mm/rmap: replace ... |
393 |
VM_WARN_ON(anon_vma->degree); |
eee2acbae mm: avoid repeate... |
394 |
put_anon_vma(anon_vma); |
5beb49305 mm: change anon_v... |
395 396 397 398 |
list_del(&avc->same_vma); anon_vma_chain_free(avc); } } |
51cc50685 SL*B: drop kmem c... |
399 |
static void anon_vma_ctor(void *data) |
1da177e4c Linux-2.6.12-rc2 |
400 |
{ |
a35afb830 Remove SLAB_CTOR_... |
401 |
struct anon_vma *anon_vma = data; |
1da177e4c Linux-2.6.12-rc2 |
402 |
|
5a505085f mm/rmap: Convert ... |
403 |
init_rwsem(&anon_vma->rwsem); |
83813267c mm: move anon_vma... |
404 |
atomic_set(&anon_vma->refcount, 0); |
bf181b9f9 mm anon rmap: rep... |
405 |
anon_vma->rb_root = RB_ROOT; |
1da177e4c Linux-2.6.12-rc2 |
406 407 408 409 410 |
} void __init anon_vma_init(void) { anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma), |
5d097056c kmemcg: account c... |
411 412 413 414 |
0, SLAB_DESTROY_BY_RCU|SLAB_PANIC|SLAB_ACCOUNT, anon_vma_ctor); anon_vma_chain_cachep = KMEM_CACHE(anon_vma_chain, SLAB_PANIC|SLAB_ACCOUNT); |
1da177e4c Linux-2.6.12-rc2 |
415 416 417 |
} /* |
6111e4ca6 mm: improve page_... |
418 419 420 421 422 423 424 425 426 |
* Getting a lock on a stable anon_vma from a page off the LRU is tricky! * * Since there is no serialization what so ever against page_remove_rmap() * the best this function can do is return a locked anon_vma that might * have been relevant to this page. * * The page might have been remapped to a different anon_vma or the anon_vma * returned may already be freed (and even reused). * |
bc658c960 mm, rmap: Add yet... |
427 428 429 430 431 |
* In case it was remapped to a different anon_vma, the new anon_vma will be a * child of the old anon_vma, and the anon_vma lifetime rules will therefore * ensure that any anon_vma obtained from the page will still be valid for as * long as we observe page_mapped() [ hence all those page_mapped() tests ]. * |
6111e4ca6 mm: improve page_... |
432 433 434 435 436 437 438 |
* All users of this function must be very careful when walking the anon_vma * chain and verify that the page in question is indeed mapped in it * [ something equivalent to page_mapped_in_vma() ]. * * Since anon_vma's slab is DESTROY_BY_RCU and we know from page_remove_rmap() * that the anon_vma pointer from page->mapping is valid if there is a * mapcount, we can dereference the anon_vma after observing those. |
1da177e4c Linux-2.6.12-rc2 |
439 |
*/ |
746b18d42 mm: use refcounts... |
440 |
struct anon_vma *page_get_anon_vma(struct page *page) |
1da177e4c Linux-2.6.12-rc2 |
441 |
{ |
746b18d42 mm: use refcounts... |
442 |
struct anon_vma *anon_vma = NULL; |
1da177e4c Linux-2.6.12-rc2 |
443 444 445 |
unsigned long anon_mapping; rcu_read_lock(); |
4db0c3c29 mm: remove rest o... |
446 |
anon_mapping = (unsigned long)READ_ONCE(page->mapping); |
3ca7b3c5b mm: define PAGE_M... |
447 |
if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) |
1da177e4c Linux-2.6.12-rc2 |
448 449 450 451 452 |
goto out; if (!page_mapped(page)) goto out; anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); |
746b18d42 mm: use refcounts... |
453 454 455 456 |
if (!atomic_inc_not_zero(&anon_vma->refcount)) { anon_vma = NULL; goto out; } |
f18194275 mm: fix hang on a... |
457 458 459 |
/* * If this page is still mapped, then its anon_vma cannot have been |
746b18d42 mm: use refcounts... |
460 461 462 463 |
* freed. But if it has been unmapped, we have no security against the * anon_vma structure being freed and reused (for another anon_vma: * SLAB_DESTROY_BY_RCU guarantees that - so the atomic_inc_not_zero() * above cannot corrupt). |
f18194275 mm: fix hang on a... |
464 |
*/ |
746b18d42 mm: use refcounts... |
465 |
if (!page_mapped(page)) { |
7f39dda9d mm: fix sleeping ... |
466 |
rcu_read_unlock(); |
746b18d42 mm: use refcounts... |
467 |
put_anon_vma(anon_vma); |
7f39dda9d mm: fix sleeping ... |
468 |
return NULL; |
746b18d42 mm: use refcounts... |
469 |
} |
1da177e4c Linux-2.6.12-rc2 |
470 471 |
out: rcu_read_unlock(); |
746b18d42 mm: use refcounts... |
472 473 474 |
return anon_vma; } |
88c22088b mm: optimize page... |
475 476 477 478 479 480 481 |
/* * Similar to page_get_anon_vma() except it locks the anon_vma. * * Its a little more complex as it tries to keep the fast path to a single * atomic op -- the trylock. If we fail the trylock, we fall back to getting a * reference like with page_get_anon_vma() and then block on the mutex. */ |
4fc3f1d66 mm/rmap, migratio... |
482 |
struct anon_vma *page_lock_anon_vma_read(struct page *page) |
746b18d42 mm: use refcounts... |
483 |
{ |
88c22088b mm: optimize page... |
484 |
struct anon_vma *anon_vma = NULL; |
eee0f252c mm: fix page_lock... |
485 |
struct anon_vma *root_anon_vma; |
88c22088b mm: optimize page... |
486 |
unsigned long anon_mapping; |
746b18d42 mm: use refcounts... |
487 |
|
88c22088b mm: optimize page... |
488 |
rcu_read_lock(); |
4db0c3c29 mm: remove rest o... |
489 |
anon_mapping = (unsigned long)READ_ONCE(page->mapping); |
88c22088b mm: optimize page... |
490 491 492 493 494 495 |
if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) goto out; if (!page_mapped(page)) goto out; anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); |
4db0c3c29 mm: remove rest o... |
496 |
root_anon_vma = READ_ONCE(anon_vma->root); |
4fc3f1d66 mm/rmap, migratio... |
497 |
if (down_read_trylock(&root_anon_vma->rwsem)) { |
88c22088b mm: optimize page... |
498 |
/* |
eee0f252c mm: fix page_lock... |
499 500 |
* If the page is still mapped, then this anon_vma is still * its anon_vma, and holding the mutex ensures that it will |
bc658c960 mm, rmap: Add yet... |
501 |
* not go away, see anon_vma_free(). |
88c22088b mm: optimize page... |
502 |
*/ |
eee0f252c mm: fix page_lock... |
503 |
if (!page_mapped(page)) { |
4fc3f1d66 mm/rmap, migratio... |
504 |
up_read(&root_anon_vma->rwsem); |
88c22088b mm: optimize page... |
505 506 507 508 |
anon_vma = NULL; } goto out; } |
746b18d42 mm: use refcounts... |
509 |
|
88c22088b mm: optimize page... |
510 511 512 513 514 515 516 |
/* trylock failed, we got to sleep */ if (!atomic_inc_not_zero(&anon_vma->refcount)) { anon_vma = NULL; goto out; } if (!page_mapped(page)) { |
7f39dda9d mm: fix sleeping ... |
517 |
rcu_read_unlock(); |
88c22088b mm: optimize page... |
518 |
put_anon_vma(anon_vma); |
7f39dda9d mm: fix sleeping ... |
519 |
return NULL; |
88c22088b mm: optimize page... |
520 521 522 523 |
} /* we pinned the anon_vma, its safe to sleep */ rcu_read_unlock(); |
4fc3f1d66 mm/rmap, migratio... |
524 |
anon_vma_lock_read(anon_vma); |
88c22088b mm: optimize page... |
525 526 527 528 529 |
if (atomic_dec_and_test(&anon_vma->refcount)) { /* * Oops, we held the last refcount, release the lock * and bail -- can't simply use put_anon_vma() because |
4fc3f1d66 mm/rmap, migratio... |
530 |
* we'll deadlock on the anon_vma_lock_write() recursion. |
88c22088b mm: optimize page... |
531 |
*/ |
4fc3f1d66 mm/rmap, migratio... |
532 |
anon_vma_unlock_read(anon_vma); |
88c22088b mm: optimize page... |
533 534 535 536 537 538 539 540 |
__put_anon_vma(anon_vma); anon_vma = NULL; } return anon_vma; out: rcu_read_unlock(); |
746b18d42 mm: use refcounts... |
541 |
return anon_vma; |
34bbd7040 [PATCH] adapt pag... |
542 |
} |
4fc3f1d66 mm/rmap, migratio... |
543 |
void page_unlock_anon_vma_read(struct anon_vma *anon_vma) |
34bbd7040 [PATCH] adapt pag... |
544 |
{ |
4fc3f1d66 mm/rmap, migratio... |
545 |
anon_vma_unlock_read(anon_vma); |
1da177e4c Linux-2.6.12-rc2 |
546 |
} |
72b252aed mm: send one IPI ... |
547 |
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH |
72b252aed mm: send one IPI ... |
548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 |
/* * Flush TLB entries for recently unmapped pages from remote CPUs. It is * important if a PTE was dirty when it was unmapped that it's flushed * before any IO is initiated on the page to prevent lost writes. Similarly, * it must be flushed before freeing to prevent data leakage. */ void try_to_unmap_flush(void) { struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; int cpu; if (!tlb_ubc->flush_required) return; cpu = get_cpu(); |
858eaaa71 mm/rmap: batched ... |
563 564 565 566 |
if (cpumask_test_cpu(cpu, &tlb_ubc->cpumask)) { count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL); local_flush_tlb(); trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL); |
72b252aed mm: send one IPI ... |
567 |
} |
858eaaa71 mm/rmap: batched ... |
568 569 570 |
if (cpumask_any_but(&tlb_ubc->cpumask, cpu) < nr_cpu_ids) flush_tlb_others(&tlb_ubc->cpumask, NULL, 0, TLB_FLUSH_ALL); |
72b252aed mm: send one IPI ... |
571 572 |
cpumask_clear(&tlb_ubc->cpumask); tlb_ubc->flush_required = false; |
d950c9477 mm: defer flush o... |
573 |
tlb_ubc->writable = false; |
72b252aed mm: send one IPI ... |
574 575 |
put_cpu(); } |
d950c9477 mm: defer flush o... |
576 577 578 579 580 581 582 583 |
/* Flush iff there are potentially writable TLB entries that can race with IO */ void try_to_unmap_flush_dirty(void) { struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; if (tlb_ubc->writable) try_to_unmap_flush(); } |
72b252aed mm: send one IPI ... |
584 |
static void set_tlb_ubc_flush_pending(struct mm_struct *mm, |
d950c9477 mm: defer flush o... |
585 |
struct page *page, bool writable) |
72b252aed mm: send one IPI ... |
586 587 588 589 590 |
{ struct tlbflush_unmap_batch *tlb_ubc = ¤t->tlb_ubc; cpumask_or(&tlb_ubc->cpumask, &tlb_ubc->cpumask, mm_cpumask(mm)); tlb_ubc->flush_required = true; |
d950c9477 mm: defer flush o... |
591 592 593 594 595 596 597 598 |
/* * If the PTE was dirty then it's best to assume it's writable. The * caller must use try_to_unmap_flush_dirty() or try_to_unmap_flush() * before the page is queued for IO. */ if (writable) tlb_ubc->writable = true; |
72b252aed mm: send one IPI ... |
599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 |
} /* * Returns true if the TLB flush should be deferred to the end of a batch of * unmap operations to reduce IPIs. */ static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) { bool should_defer = false; if (!(flags & TTU_BATCH_FLUSH)) return false; /* If remote CPUs need to be flushed then defer batch the flush */ if (cpumask_any_but(mm_cpumask(mm), get_cpu()) < nr_cpu_ids) should_defer = true; put_cpu(); return should_defer; } #else static void set_tlb_ubc_flush_pending(struct mm_struct *mm, |
d950c9477 mm: defer flush o... |
621 |
struct page *page, bool writable) |
72b252aed mm: send one IPI ... |
622 623 624 625 626 627 628 629 |
{ } static bool should_defer_flush(struct mm_struct *mm, enum ttu_flags flags) { return false; } #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */ |
1da177e4c Linux-2.6.12-rc2 |
630 |
/* |
bf89c8c86 mm/rmap.c: fix co... |
631 |
* At what user virtual address is page expected in vma? |
ab941e0ff rmap: remove anon... |
632 |
* Caller should check the page is actually part of the vma. |
1da177e4c Linux-2.6.12-rc2 |
633 634 635 |
*/ unsigned long page_address_in_vma(struct page *page, struct vm_area_struct *vma) { |
86c2ad199 mm rmap: remove v... |
636 |
unsigned long address; |
21d0d443c rmap: resurrect p... |
637 |
if (PageAnon(page)) { |
4829b906c ksm: fix page_add... |
638 639 640 641 642 643 644 |
struct anon_vma *page__anon_vma = page_anon_vma(page); /* * Note: swapoff's unuse_vma() is more efficient with this * check, and needs it to match anon_vma when KSM is active. */ if (!vma->anon_vma || !page__anon_vma || vma->anon_vma->root != page__anon_vma->root) |
21d0d443c rmap: resurrect p... |
645 |
return -EFAULT; |
27ba0644e rmap: drop suppor... |
646 647 |
} else if (page->mapping) { if (!vma->vm_file || vma->vm_file->f_mapping != page->mapping) |
1da177e4c Linux-2.6.12-rc2 |
648 649 650 |
return -EFAULT; } else return -EFAULT; |
86c2ad199 mm rmap: remove v... |
651 652 653 654 |
address = __vma_address(page, vma); if (unlikely(address < vma->vm_start || address >= vma->vm_end)) return -EFAULT; return address; |
1da177e4c Linux-2.6.12-rc2 |
655 |
} |
6219049ae mm: introduce mm_... |
656 657 658 659 660 |
pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) { pgd_t *pgd; pud_t *pud; pmd_t *pmd = NULL; |
f72e7dcdd mm: let mm_find_p... |
661 |
pmd_t pmde; |
6219049ae mm: introduce mm_... |
662 663 664 665 666 667 668 669 670 671 |
pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) goto out; pud = pud_offset(pgd, address); if (!pud_present(*pud)) goto out; pmd = pmd_offset(pud, address); |
f72e7dcdd mm: let mm_find_p... |
672 |
/* |
8809aa2d2 mm: clarify that ... |
673 |
* Some THP functions use the sequence pmdp_huge_clear_flush(), set_pmd_at() |
f72e7dcdd mm: let mm_find_p... |
674 675 676 |
* without holding anon_vma lock for write. So when looking for a * genuine pmde (in which to find pte), test present and !THP together. */ |
e37c69827 mm: replace ACCES... |
677 678 |
pmde = *pmd; barrier(); |
f72e7dcdd mm: let mm_find_p... |
679 |
if (!pmd_present(pmde) || pmd_trans_huge(pmde)) |
6219049ae mm: introduce mm_... |
680 681 682 683 |
pmd = NULL; out: return pmd; } |
1da177e4c Linux-2.6.12-rc2 |
684 |
/* |
81b4082dc [PATCH] mm: rmap.... |
685 686 |
* Check that @page is mapped at @address into @mm. * |
479db0bf4 mm: dirty page tr... |
687 688 689 690 |
* If @sync is false, page_check_address may perform a racy check to avoid * the page table lock when the pte is not present (helpful when reclaiming * highly shared pages). * |
b8072f099 [PATCH] mm: updat... |
691 |
* On success returns with pte mapped and locked. |
81b4082dc [PATCH] mm: rmap.... |
692 |
*/ |
e9a81a821 rmap: wrap page_c... |
693 |
pte_t *__page_check_address(struct page *page, struct mm_struct *mm, |
479db0bf4 mm: dirty page tr... |
694 |
unsigned long address, spinlock_t **ptlp, int sync) |
81b4082dc [PATCH] mm: rmap.... |
695 |
{ |
81b4082dc [PATCH] mm: rmap.... |
696 697 |
pmd_t *pmd; pte_t *pte; |
c0718806c [PATCH] mm: rmap ... |
698 |
spinlock_t *ptl; |
81b4082dc [PATCH] mm: rmap.... |
699 |
|
0fe6e20b9 hugetlb, rmap: ad... |
700 |
if (unlikely(PageHuge(page))) { |
98398c32f mm/hugetlb: check... |
701 |
/* when pud is not present, pte will be NULL */ |
0fe6e20b9 hugetlb, rmap: ad... |
702 |
pte = huge_pte_offset(mm, address); |
98398c32f mm/hugetlb: check... |
703 704 |
if (!pte) return NULL; |
cb900f412 mm, hugetlb: conv... |
705 |
ptl = huge_pte_lockptr(page_hstate(page), mm, pte); |
0fe6e20b9 hugetlb, rmap: ad... |
706 707 |
goto check; } |
6219049ae mm: introduce mm_... |
708 709 |
pmd = mm_find_pmd(mm, address); if (!pmd) |
c0718806c [PATCH] mm: rmap ... |
710 |
return NULL; |
c0718806c [PATCH] mm: rmap ... |
711 712 |
pte = pte_offset_map(pmd, address); /* Make a quick check before getting the lock */ |
479db0bf4 mm: dirty page tr... |
713 |
if (!sync && !pte_present(*pte)) { |
c0718806c [PATCH] mm: rmap ... |
714 715 716 |
pte_unmap(pte); return NULL; } |
4c21e2f24 [PATCH] mm: split... |
717 |
ptl = pte_lockptr(mm, pmd); |
0fe6e20b9 hugetlb, rmap: ad... |
718 |
check: |
c0718806c [PATCH] mm: rmap ... |
719 720 721 722 |
spin_lock(ptl); if (pte_present(*pte) && page_to_pfn(page) == pte_pfn(*pte)) { *ptlp = ptl; return pte; |
81b4082dc [PATCH] mm: rmap.... |
723 |
} |
c0718806c [PATCH] mm: rmap ... |
724 725 |
pte_unmap_unlock(pte, ptl); return NULL; |
81b4082dc [PATCH] mm: rmap.... |
726 |
} |
b291f0003 mlock: mlocked pa... |
727 728 729 730 731 732 733 734 735 |
/** * page_mapped_in_vma - check whether a page is really mapped in a VMA * @page: the page to test * @vma: the VMA to test * * Returns 1 if the page is mapped into the page tables of the VMA, 0 * if the page is not mapped into the page tables of this VMA. Only * valid for normal file or anonymous VMAs. */ |
6a46079cf HWPOISON: The hig... |
736 |
int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) |
b291f0003 mlock: mlocked pa... |
737 738 739 740 |
{ unsigned long address; pte_t *pte; spinlock_t *ptl; |
86c2ad199 mm rmap: remove v... |
741 742 |
address = __vma_address(page, vma); if (unlikely(address < vma->vm_start || address >= vma->vm_end)) |
b291f0003 mlock: mlocked pa... |
743 744 745 746 747 748 749 750 |
return 0; pte = page_check_address(page, vma->vm_mm, address, &ptl, 1); if (!pte) /* the page is not in this mm */ return 0; pte_unmap_unlock(pte, ptl); return 1; } |
8749cfea1 mm: add page_chec... |
751 |
#ifdef CONFIG_TRANSPARENT_HUGEPAGE |
81b4082dc [PATCH] mm: rmap.... |
752 |
/* |
8749cfea1 mm: add page_chec... |
753 754 755 756 757 |
* Check that @page is mapped at @address into @mm. In contrast to * page_check_address(), this function can handle transparent huge pages. * * On success returns true with pte mapped and locked. For PMD-mapped * transparent huge pages *@ptep is set to NULL. |
1da177e4c Linux-2.6.12-rc2 |
758 |
*/ |
8749cfea1 mm: add page_chec... |
759 760 761 |
bool page_check_address_transhuge(struct page *page, struct mm_struct *mm, unsigned long address, pmd_t **pmdp, pte_t **ptep, spinlock_t **ptlp) |
1da177e4c Linux-2.6.12-rc2 |
762 |
{ |
b20ce5e03 mm: prepare page_... |
763 764 765 766 |
pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; |
8749cfea1 mm: add page_chec... |
767 |
spinlock_t *ptl; |
1da177e4c Linux-2.6.12-rc2 |
768 |
|
b20ce5e03 mm: prepare page_... |
769 770 771 772 |
if (unlikely(PageHuge(page))) { /* when pud is not present, pte will be NULL */ pte = huge_pte_offset(mm, address); if (!pte) |
8749cfea1 mm: add page_chec... |
773 |
return false; |
2da28bfd9 thp: fix page_ref... |
774 |
|
b20ce5e03 mm: prepare page_... |
775 |
ptl = huge_pte_lockptr(page_hstate(page), mm, pte); |
8749cfea1 mm: add page_chec... |
776 |
pmd = NULL; |
b20ce5e03 mm: prepare page_... |
777 778 779 780 781 |
goto check_pte; } pgd = pgd_offset(mm, address); if (!pgd_present(*pgd)) |
8749cfea1 mm: add page_chec... |
782 |
return false; |
b20ce5e03 mm: prepare page_... |
783 784 |
pud = pud_offset(pgd, address); if (!pud_present(*pud)) |
8749cfea1 mm: add page_chec... |
785 |
return false; |
b20ce5e03 mm: prepare page_... |
786 787 788 |
pmd = pmd_offset(pud, address); if (pmd_trans_huge(*pmd)) { |
b20ce5e03 mm: prepare page_... |
789 790 791 792 |
ptl = pmd_lock(mm, pmd); if (!pmd_present(*pmd)) goto unlock_pmd; if (unlikely(!pmd_trans_huge(*pmd))) { |
117b0791a mm, thp: move ptl... |
793 |
spin_unlock(ptl); |
b20ce5e03 mm: prepare page_... |
794 795 796 797 798 |
goto map_pte; } if (pmd_page(*pmd) != page) goto unlock_pmd; |
8749cfea1 mm: add page_chec... |
799 |
pte = NULL; |
b20ce5e03 mm: prepare page_... |
800 801 802 |
goto found; unlock_pmd: spin_unlock(ptl); |
8749cfea1 mm: add page_chec... |
803 |
return false; |
71e3aac07 thp: transparent ... |
804 |
} else { |
b20ce5e03 mm: prepare page_... |
805 |
pmd_t pmde = *pmd; |
71e3aac07 thp: transparent ... |
806 |
|
b20ce5e03 mm: prepare page_... |
807 808 |
barrier(); if (!pmd_present(pmde) || pmd_trans_huge(pmde)) |
8749cfea1 mm: add page_chec... |
809 |
return false; |
b20ce5e03 mm: prepare page_... |
810 811 812 813 814 |
} map_pte: pte = pte_offset_map(pmd, address); if (!pte_present(*pte)) { pte_unmap(pte); |
8749cfea1 mm: add page_chec... |
815 |
return false; |
b20ce5e03 mm: prepare page_... |
816 |
} |
71e3aac07 thp: transparent ... |
817 |
|
b20ce5e03 mm: prepare page_... |
818 819 820 |
ptl = pte_lockptr(mm, pmd); check_pte: spin_lock(ptl); |
2da28bfd9 thp: fix page_ref... |
821 |
|
b20ce5e03 mm: prepare page_... |
822 823 |
if (!pte_present(*pte)) { pte_unmap_unlock(pte, ptl); |
8749cfea1 mm: add page_chec... |
824 |
return false; |
b20ce5e03 mm: prepare page_... |
825 826 827 828 829 |
} /* THP can be referenced by any subpage */ if (pte_pfn(*pte) - page_to_pfn(page) >= hpage_nr_pages(page)) { pte_unmap_unlock(pte, ptl); |
8749cfea1 mm: add page_chec... |
830 |
return false; |
b20ce5e03 mm: prepare page_... |
831 |
} |
8749cfea1 mm: add page_chec... |
832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 |
found: *ptep = pte; *pmdp = pmd; *ptlp = ptl; return true; } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ struct page_referenced_arg { int mapcount; int referenced; unsigned long vm_flags; struct mem_cgroup *memcg; }; /* * arg: page_referenced_arg will be passed */ static int page_referenced_one(struct page *page, struct vm_area_struct *vma, unsigned long address, void *arg) { struct mm_struct *mm = vma->vm_mm; struct page_referenced_arg *pra = arg; pmd_t *pmd; pte_t *pte; spinlock_t *ptl; int referenced = 0; if (!page_check_address_transhuge(page, mm, address, &pmd, &pte, &ptl)) return SWAP_AGAIN; |
b20ce5e03 mm: prepare page_... |
861 862 |
if (vma->vm_flags & VM_LOCKED) { |
8749cfea1 mm: add page_chec... |
863 864 865 |
if (pte) pte_unmap(pte); spin_unlock(ptl); |
b20ce5e03 mm: prepare page_... |
866 867 |
pra->vm_flags |= VM_LOCKED; return SWAP_FAIL; /* To break the loop */ |
71e3aac07 thp: transparent ... |
868 |
} |
8749cfea1 mm: add page_chec... |
869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 |
if (pte) { if (ptep_clear_flush_young_notify(vma, address, pte)) { /* * Don't treat a reference through a sequentially read * mapping as such. If the page has been used in * another mapping, we will catch it; if this other * mapping is already gone, the unmap path will have * set PG_referenced or activated the page. */ if (likely(!(vma->vm_flags & VM_SEQ_READ))) referenced++; } pte_unmap(pte); } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { if (pmdp_clear_flush_young_notify(vma, address, pmd)) |
b20ce5e03 mm: prepare page_... |
884 |
referenced++; |
8749cfea1 mm: add page_chec... |
885 886 887 |
} else { /* unexpected pmd-mapped page? */ WARN_ON_ONCE(1); |
b20ce5e03 mm: prepare page_... |
888 |
} |
8749cfea1 mm: add page_chec... |
889 |
spin_unlock(ptl); |
b20ce5e03 mm: prepare page_... |
890 |
|
33c3fc71c mm: introduce idl... |
891 892 893 894 |
if (referenced) clear_page_idle(page); if (test_and_clear_page_young(page)) referenced++; |
9f32624be mm/rmap: use rmap... |
895 896 897 |
if (referenced) { pra->referenced++; pra->vm_flags |= vma->vm_flags; |
1da177e4c Linux-2.6.12-rc2 |
898 |
} |
34bbd7040 [PATCH] adapt pag... |
899 |
|
9f32624be mm/rmap: use rmap... |
900 901 902 903 904 |
pra->mapcount--; if (!pra->mapcount) return SWAP_SUCCESS; /* To break the loop */ return SWAP_AGAIN; |
1da177e4c Linux-2.6.12-rc2 |
905 |
} |
9f32624be mm/rmap: use rmap... |
906 |
static bool invalid_page_referenced_vma(struct vm_area_struct *vma, void *arg) |
1da177e4c Linux-2.6.12-rc2 |
907 |
{ |
9f32624be mm/rmap: use rmap... |
908 909 |
struct page_referenced_arg *pra = arg; struct mem_cgroup *memcg = pra->memcg; |
1da177e4c Linux-2.6.12-rc2 |
910 |
|
9f32624be mm/rmap: use rmap... |
911 912 |
if (!mm_match_cgroup(vma->vm_mm, memcg)) return true; |
1da177e4c Linux-2.6.12-rc2 |
913 |
|
9f32624be mm/rmap: use rmap... |
914 |
return false; |
1da177e4c Linux-2.6.12-rc2 |
915 916 917 918 919 920 |
} /** * page_referenced - test if the page was referenced * @page: the page to test * @is_locked: caller holds lock on the page |
72835c86c mm: unify remaini... |
921 |
* @memcg: target memory cgroup |
6fe6b7e35 vmscan: report vm... |
922 |
* @vm_flags: collect encountered vma->vm_flags who actually referenced the page |
1da177e4c Linux-2.6.12-rc2 |
923 924 925 926 |
* * Quick test_and_clear_referenced for all mappings to a page, * returns the number of ptes which referenced the page. */ |
6fe6b7e35 vmscan: report vm... |
927 928 |
int page_referenced(struct page *page, int is_locked, |
72835c86c mm: unify remaini... |
929 |
struct mem_cgroup *memcg, |
6fe6b7e35 vmscan: report vm... |
930 |
unsigned long *vm_flags) |
1da177e4c Linux-2.6.12-rc2 |
931 |
{ |
9f32624be mm/rmap: use rmap... |
932 |
int ret; |
5ad646880 ksm: let shared p... |
933 |
int we_locked = 0; |
9f32624be mm/rmap: use rmap... |
934 |
struct page_referenced_arg pra = { |
b20ce5e03 mm: prepare page_... |
935 |
.mapcount = total_mapcount(page), |
9f32624be mm/rmap: use rmap... |
936 937 938 939 940 941 942 |
.memcg = memcg, }; struct rmap_walk_control rwc = { .rmap_one = page_referenced_one, .arg = (void *)&pra, .anon_lock = page_lock_anon_vma_read, }; |
1da177e4c Linux-2.6.12-rc2 |
943 |
|
6fe6b7e35 vmscan: report vm... |
944 |
*vm_flags = 0; |
9f32624be mm/rmap: use rmap... |
945 946 947 948 949 950 951 952 953 954 |
if (!page_mapped(page)) return 0; if (!page_rmapping(page)) return 0; if (!is_locked && (!PageAnon(page) || PageKsm(page))) { we_locked = trylock_page(page); if (!we_locked) return 1; |
1da177e4c Linux-2.6.12-rc2 |
955 |
} |
9f32624be mm/rmap: use rmap... |
956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 |
/* * If we are reclaiming on behalf of a cgroup, skip * counting on behalf of references from different * cgroups */ if (memcg) { rwc.invalid_vma = invalid_page_referenced_vma; } ret = rmap_walk(page, &rwc); *vm_flags = pra.vm_flags; if (we_locked) unlock_page(page); return pra.referenced; |
1da177e4c Linux-2.6.12-rc2 |
973 |
} |
1cb1729b1 mm: pass address ... |
974 |
static int page_mkclean_one(struct page *page, struct vm_area_struct *vma, |
9853a407b mm/rmap: use rmap... |
975 |
unsigned long address, void *arg) |
d08b3851d [PATCH] mm: track... |
976 977 |
{ struct mm_struct *mm = vma->vm_mm; |
c2fda5fed [PATCH] Fix up pa... |
978 |
pte_t *pte; |
d08b3851d [PATCH] mm: track... |
979 980 |
spinlock_t *ptl; int ret = 0; |
9853a407b mm/rmap: use rmap... |
981 |
int *cleaned = arg; |
d08b3851d [PATCH] mm: track... |
982 |
|
479db0bf4 mm: dirty page tr... |
983 |
pte = page_check_address(page, mm, address, &ptl, 1); |
d08b3851d [PATCH] mm: track... |
984 985 |
if (!pte) goto out; |
c2fda5fed [PATCH] Fix up pa... |
986 987 |
if (pte_dirty(*pte) || pte_write(*pte)) { pte_t entry; |
d08b3851d [PATCH] mm: track... |
988 |
|
c2fda5fed [PATCH] Fix up pa... |
989 |
flush_cache_page(vma, address, pte_pfn(*pte)); |
2ec74c3ef mm: move all mmu ... |
990 |
entry = ptep_clear_flush(vma, address, pte); |
c2fda5fed [PATCH] Fix up pa... |
991 992 |
entry = pte_wrprotect(entry); entry = pte_mkclean(entry); |
d6e88e671 [PATCH] page_mkcl... |
993 |
set_pte_at(mm, address, pte, entry); |
c2fda5fed [PATCH] Fix up pa... |
994 995 |
ret = 1; } |
d08b3851d [PATCH] mm: track... |
996 |
|
d08b3851d [PATCH] mm: track... |
997 |
pte_unmap_unlock(pte, ptl); |
2ec74c3ef mm: move all mmu ... |
998 |
|
9853a407b mm/rmap: use rmap... |
999 |
if (ret) { |
2ec74c3ef mm: move all mmu ... |
1000 |
mmu_notifier_invalidate_page(mm, address); |
9853a407b mm/rmap: use rmap... |
1001 1002 |
(*cleaned)++; } |
d08b3851d [PATCH] mm: track... |
1003 |
out: |
9853a407b mm/rmap: use rmap... |
1004 |
return SWAP_AGAIN; |
d08b3851d [PATCH] mm: track... |
1005 |
} |
9853a407b mm/rmap: use rmap... |
1006 |
static bool invalid_mkclean_vma(struct vm_area_struct *vma, void *arg) |
d08b3851d [PATCH] mm: track... |
1007 |
{ |
9853a407b mm/rmap: use rmap... |
1008 |
if (vma->vm_flags & VM_SHARED) |
871beb8c3 mm/rmap: fix cocc... |
1009 |
return false; |
d08b3851d [PATCH] mm: track... |
1010 |
|
871beb8c3 mm/rmap: fix cocc... |
1011 |
return true; |
d08b3851d [PATCH] mm: track... |
1012 1013 1014 1015 |
} int page_mkclean(struct page *page) { |
9853a407b mm/rmap: use rmap... |
1016 1017 1018 1019 1020 1021 1022 |
int cleaned = 0; struct address_space *mapping; struct rmap_walk_control rwc = { .arg = (void *)&cleaned, .rmap_one = page_mkclean_one, .invalid_vma = invalid_mkclean_vma, }; |
d08b3851d [PATCH] mm: track... |
1023 1024 |
BUG_ON(!PageLocked(page)); |
9853a407b mm/rmap: use rmap... |
1025 1026 1027 1028 1029 1030 1031 1032 |
if (!page_mapped(page)) return 0; mapping = page_mapping(page); if (!mapping) return 0; rmap_walk(page, &rwc); |
d08b3851d [PATCH] mm: track... |
1033 |
|
9853a407b mm/rmap: use rmap... |
1034 |
return cleaned; |
d08b3851d [PATCH] mm: track... |
1035 |
} |
60b59beaf fbdev: mm: Deferr... |
1036 |
EXPORT_SYMBOL_GPL(page_mkclean); |
d08b3851d [PATCH] mm: track... |
1037 |
|
1da177e4c Linux-2.6.12-rc2 |
1038 |
/** |
c44b67432 rmap: move exclus... |
1039 1040 1041 |
* page_move_anon_rmap - move a page to our anon_vma * @page: the page to move to our anon_vma * @vma: the vma the page belongs to |
c44b67432 rmap: move exclus... |
1042 1043 1044 1045 1046 1047 |
* * When a page belongs exclusively to one process after a COW event, * that page can be moved into the anon_vma that belongs to just that * process, so the rmap code will not search the parent or sibling * processes. */ |
5a49973d7 mm: thp: refix fa... |
1048 |
void page_move_anon_rmap(struct page *page, struct vm_area_struct *vma) |
c44b67432 rmap: move exclus... |
1049 1050 |
{ struct anon_vma *anon_vma = vma->anon_vma; |
5a49973d7 mm: thp: refix fa... |
1051 |
page = compound_head(page); |
309381fea mm: dump page whe... |
1052 |
VM_BUG_ON_PAGE(!PageLocked(page), page); |
81d1b09c6 mm: convert a few... |
1053 |
VM_BUG_ON_VMA(!anon_vma, vma); |
c44b67432 rmap: move exclus... |
1054 1055 |
anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; |
414e2fb8c rmap: fix theoret... |
1056 1057 1058 1059 1060 1061 |
/* * Ensure that anon_vma and the PAGE_MAPPING_ANON bit are written * simultaneously, so a concurrent reader (eg page_referenced()'s * PageAnon()) will not see one without the other. */ WRITE_ONCE(page->mapping, (struct address_space *) anon_vma); |
c44b67432 rmap: move exclus... |
1062 1063 1064 |
} /** |
4e1c19750 Clean up __page_s... |
1065 1066 1067 1068 |
* __page_set_anon_rmap - set up new anonymous rmap * @page: Page to add to rmap * @vma: VM area to add page to. * @address: User virtual address of the mapping |
e8a03feb5 rmap: add exclusi... |
1069 |
* @exclusive: the page is exclusively owned by the current process |
9617d95e6 [PATCH] mm: rmap ... |
1070 1071 |
*/ static void __page_set_anon_rmap(struct page *page, |
e8a03feb5 rmap: add exclusi... |
1072 |
struct vm_area_struct *vma, unsigned long address, int exclusive) |
9617d95e6 [PATCH] mm: rmap ... |
1073 |
{ |
e8a03feb5 rmap: add exclusi... |
1074 |
struct anon_vma *anon_vma = vma->anon_vma; |
ea90002b0 anonvma: when set... |
1075 |
|
e8a03feb5 rmap: add exclusi... |
1076 |
BUG_ON(!anon_vma); |
ea90002b0 anonvma: when set... |
1077 |
|
4e1c19750 Clean up __page_s... |
1078 1079 |
if (PageAnon(page)) return; |
ea90002b0 anonvma: when set... |
1080 |
/* |
e8a03feb5 rmap: add exclusi... |
1081 1082 1083 |
* If the page isn't exclusively mapped into this vma, * we must use the _oldest_ possible anon_vma for the * page mapping! |
ea90002b0 anonvma: when set... |
1084 |
*/ |
4e1c19750 Clean up __page_s... |
1085 |
if (!exclusive) |
288468c33 rmap: always use ... |
1086 |
anon_vma = anon_vma->root; |
9617d95e6 [PATCH] mm: rmap ... |
1087 |
|
9617d95e6 [PATCH] mm: rmap ... |
1088 1089 |
anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; page->mapping = (struct address_space *) anon_vma; |
9617d95e6 [PATCH] mm: rmap ... |
1090 |
page->index = linear_page_index(vma, address); |
9617d95e6 [PATCH] mm: rmap ... |
1091 1092 1093 |
} /** |
43d8eac44 mm: rmap kernel-d... |
1094 |
* __page_check_anon_rmap - sanity check anonymous rmap addition |
c97a9e10e mm: more rmap che... |
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 |
* @page: the page to add the mapping to * @vma: the vm area in which the mapping is added * @address: the user virtual address mapped */ static void __page_check_anon_rmap(struct page *page, struct vm_area_struct *vma, unsigned long address) { #ifdef CONFIG_DEBUG_VM /* * The page's anon-rmap details (mapping and index) are guaranteed to * be set up correctly at this point. * * We have exclusion against page_add_anon_rmap because the caller * always holds the page locked, except if called from page_dup_rmap, * in which case the page is already known to be setup. * * We have exclusion against page_add_new_anon_rmap because those pages * are initially only visible via the pagetables, and the pte is locked * over the call to page_add_new_anon_rmap. */ |
44ab57a06 rmap: add anon_vm... |
1115 |
BUG_ON(page_anon_vma(page)->root != vma->anon_vma->root); |
53f9263ba mm: rework mapcou... |
1116 |
BUG_ON(page_to_pgoff(page) != linear_page_index(vma, address)); |
c97a9e10e mm: more rmap che... |
1117 1118 1119 1120 |
#endif } /** |
1da177e4c Linux-2.6.12-rc2 |
1121 1122 1123 1124 |
* page_add_anon_rmap - add pte mapping to an anonymous page * @page: the page to add the mapping to * @vma: the vm area in which the mapping is added * @address: the user virtual address mapped |
d281ee614 rmap: add argumen... |
1125 |
* @compound: charge the page as compound or small page |
1da177e4c Linux-2.6.12-rc2 |
1126 |
* |
5ad646880 ksm: let shared p... |
1127 |
* The caller needs to hold the pte lock, and the page must be locked in |
80e148226 ksm: share anon p... |
1128 1129 1130 |
* the anon_vma case: to serialize mapping,index checking after setting, * and to ensure that PageAnon is not being upgraded racily to PageKsm * (but PageKsm is never downgraded to PageAnon). |
1da177e4c Linux-2.6.12-rc2 |
1131 1132 |
*/ void page_add_anon_rmap(struct page *page, |
d281ee614 rmap: add argumen... |
1133 |
struct vm_area_struct *vma, unsigned long address, bool compound) |
1da177e4c Linux-2.6.12-rc2 |
1134 |
{ |
d281ee614 rmap: add argumen... |
1135 |
do_page_add_anon_rmap(page, vma, address, compound ? RMAP_COMPOUND : 0); |
ad8c2ee80 rmap: add exclusi... |
1136 1137 1138 1139 1140 1141 1142 1143 |
} /* * Special version of the above for do_swap_page, which often runs * into pages that are exclusively owned by the current process. * Everybody else should continue to use page_add_anon_rmap above. */ void do_page_add_anon_rmap(struct page *page, |
d281ee614 rmap: add argumen... |
1144 |
struct vm_area_struct *vma, unsigned long address, int flags) |
ad8c2ee80 rmap: add exclusi... |
1145 |
{ |
53f9263ba mm: rework mapcou... |
1146 1147 |
bool compound = flags & RMAP_COMPOUND; bool first; |
e9b61f198 thp: reintroduce ... |
1148 1149 |
if (compound) { atomic_t *mapcount; |
53f9263ba mm: rework mapcou... |
1150 |
VM_BUG_ON_PAGE(!PageLocked(page), page); |
e9b61f198 thp: reintroduce ... |
1151 1152 1153 |
VM_BUG_ON_PAGE(!PageTransHuge(page), page); mapcount = compound_mapcount_ptr(page); first = atomic_inc_and_test(mapcount); |
53f9263ba mm: rework mapcou... |
1154 1155 1156 |
} else { first = atomic_inc_and_test(&page->_mapcount); } |
79134171d thp: transparent ... |
1157 |
if (first) { |
d281ee614 rmap: add argumen... |
1158 |
int nr = compound ? hpage_nr_pages(page) : 1; |
bea04b073 mm: use the light... |
1159 1160 1161 1162 1163 1164 |
/* * We use the irq-unsafe __{inc|mod}_zone_page_stat because * these counters are not modified in interrupt context, and * pte lock(a spinlock) is held, which implies preemption * disabled. */ |
65c453778 mm, rmap: account... |
1165 |
if (compound) |
11fb99898 mm: move most fil... |
1166 |
__inc_node_page_state(page, NR_ANON_THPS); |
4b9d0fab7 mm: rename NR_ANO... |
1167 |
__mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr); |
79134171d thp: transparent ... |
1168 |
} |
5ad646880 ksm: let shared p... |
1169 1170 |
if (unlikely(PageKsm(page))) return; |
309381fea mm: dump page whe... |
1171 |
VM_BUG_ON_PAGE(!PageLocked(page), page); |
53f9263ba mm: rework mapcou... |
1172 |
|
5dbe0af47 mm: fix kernel BU... |
1173 |
/* address might be in next vma when migration races vma_adjust */ |
5ad646880 ksm: let shared p... |
1174 |
if (first) |
d281ee614 rmap: add argumen... |
1175 1176 |
__page_set_anon_rmap(page, vma, address, flags & RMAP_EXCLUSIVE); |
69029cd55 memcg: remove ref... |
1177 |
else |
c97a9e10e mm: more rmap che... |
1178 |
__page_check_anon_rmap(page, vma, address); |
1da177e4c Linux-2.6.12-rc2 |
1179 |
} |
43d8eac44 mm: rmap kernel-d... |
1180 |
/** |
9617d95e6 [PATCH] mm: rmap ... |
1181 1182 1183 1184 |
* page_add_new_anon_rmap - add pte mapping to a new anonymous page * @page: the page to add the mapping to * @vma: the vm area in which the mapping is added * @address: the user virtual address mapped |
d281ee614 rmap: add argumen... |
1185 |
* @compound: charge the page as compound or small page |
9617d95e6 [PATCH] mm: rmap ... |
1186 1187 1188 |
* * Same as page_add_anon_rmap but must only be called on *new* pages. * This means the inc-and-test can be bypassed. |
c97a9e10e mm: more rmap che... |
1189 |
* Page does not have to be locked. |
9617d95e6 [PATCH] mm: rmap ... |
1190 1191 |
*/ void page_add_new_anon_rmap(struct page *page, |
d281ee614 rmap: add argumen... |
1192 |
struct vm_area_struct *vma, unsigned long address, bool compound) |
9617d95e6 [PATCH] mm: rmap ... |
1193 |
{ |
d281ee614 rmap: add argumen... |
1194 |
int nr = compound ? hpage_nr_pages(page) : 1; |
81d1b09c6 mm: convert a few... |
1195 |
VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma); |
fa9949da5 mm: use __SetPage... |
1196 |
__SetPageSwapBacked(page); |
d281ee614 rmap: add argumen... |
1197 1198 |
if (compound) { VM_BUG_ON_PAGE(!PageTransHuge(page), page); |
53f9263ba mm: rework mapcou... |
1199 1200 |
/* increment count (starts at -1) */ atomic_set(compound_mapcount_ptr(page), 0); |
11fb99898 mm: move most fil... |
1201 |
__inc_node_page_state(page, NR_ANON_THPS); |
53f9263ba mm: rework mapcou... |
1202 1203 1204 1205 1206 |
} else { /* Anon THP always mapped first with PMD */ VM_BUG_ON_PAGE(PageTransCompound(page), page); /* increment count (starts at -1) */ atomic_set(&page->_mapcount, 0); |
d281ee614 rmap: add argumen... |
1207 |
} |
4b9d0fab7 mm: rename NR_ANO... |
1208 |
__mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, nr); |
e8a03feb5 rmap: add exclusi... |
1209 |
__page_set_anon_rmap(page, vma, address, 1); |
9617d95e6 [PATCH] mm: rmap ... |
1210 |
} |
1da177e4c Linux-2.6.12-rc2 |
1211 1212 1213 1214 |
/** * page_add_file_rmap - add pte mapping to a file page * @page: the page to add the mapping to * |
b8072f099 [PATCH] mm: updat... |
1215 |
* The caller needs to hold the pte lock. |
1da177e4c Linux-2.6.12-rc2 |
1216 |
*/ |
dd78fedde rmap: support fil... |
1217 |
void page_add_file_rmap(struct page *page, bool compound) |
1da177e4c Linux-2.6.12-rc2 |
1218 |
{ |
dd78fedde rmap: support fil... |
1219 1220 1221 |
int i, nr = 1; VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page); |
62cccb8c8 mm: simplify lock... |
1222 |
lock_page_memcg(page); |
dd78fedde rmap: support fil... |
1223 1224 1225 1226 1227 1228 1229 |
if (compound && PageTransHuge(page)) { for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { if (atomic_inc_and_test(&page[i]._mapcount)) nr++; } if (!atomic_inc_and_test(compound_mapcount_ptr(page))) goto out; |
65c453778 mm, rmap: account... |
1230 |
VM_BUG_ON_PAGE(!PageSwapBacked(page), page); |
11fb99898 mm: move most fil... |
1231 |
__inc_node_page_state(page, NR_SHMEM_PMDMAPPED); |
dd78fedde rmap: support fil... |
1232 |
} else { |
c8efc390c mm, rmap: fix fal... |
1233 1234 |
if (PageTransCompound(page) && page_mapping(page)) { VM_WARN_ON_ONCE(!PageLocked(page)); |
9a73f61bd thp, mlock: do no... |
1235 1236 1237 1238 |
SetPageDoubleMap(compound_head(page)); if (PageMlocked(page)) clear_page_mlock(compound_head(page)); } |
dd78fedde rmap: support fil... |
1239 1240 |
if (!atomic_inc_and_test(&page->_mapcount)) goto out; |
d69b042f3 memcg: add file-b... |
1241 |
} |
50658e2e0 mm: move page map... |
1242 |
__mod_node_page_state(page_pgdat(page), NR_FILE_MAPPED, nr); |
dd78fedde rmap: support fil... |
1243 1244 |
mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED); out: |
62cccb8c8 mm: simplify lock... |
1245 |
unlock_page_memcg(page); |
1da177e4c Linux-2.6.12-rc2 |
1246 |
} |
dd78fedde rmap: support fil... |
1247 |
static void page_remove_file_rmap(struct page *page, bool compound) |
8186eb6a7 mm: rmap: split o... |
1248 |
{ |
dd78fedde rmap: support fil... |
1249 |
int i, nr = 1; |
57dea93ac rmap: fix compoun... |
1250 |
VM_BUG_ON_PAGE(compound && !PageHead(page), page); |
62cccb8c8 mm: simplify lock... |
1251 |
lock_page_memcg(page); |
8186eb6a7 mm: rmap: split o... |
1252 |
|
53f9263ba mm: rework mapcou... |
1253 1254 1255 1256 |
/* Hugepages are not counted in NR_FILE_MAPPED for now. */ if (unlikely(PageHuge(page))) { /* hugetlb pages are always mapped with pmds */ atomic_dec(compound_mapcount_ptr(page)); |
8186eb6a7 mm: rmap: split o... |
1257 |
goto out; |
53f9263ba mm: rework mapcou... |
1258 |
} |
8186eb6a7 mm: rmap: split o... |
1259 |
|
53f9263ba mm: rework mapcou... |
1260 |
/* page still mapped by someone else? */ |
dd78fedde rmap: support fil... |
1261 1262 1263 1264 1265 1266 1267 |
if (compound && PageTransHuge(page)) { for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { if (atomic_add_negative(-1, &page[i]._mapcount)) nr++; } if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) goto out; |
65c453778 mm, rmap: account... |
1268 |
VM_BUG_ON_PAGE(!PageSwapBacked(page), page); |
11fb99898 mm: move most fil... |
1269 |
__dec_node_page_state(page, NR_SHMEM_PMDMAPPED); |
dd78fedde rmap: support fil... |
1270 1271 1272 1273 |
} else { if (!atomic_add_negative(-1, &page->_mapcount)) goto out; } |
8186eb6a7 mm: rmap: split o... |
1274 1275 |
/* |
50658e2e0 mm: move page map... |
1276 |
* We use the irq-unsafe __{inc|mod}_zone_page_state because |
8186eb6a7 mm: rmap: split o... |
1277 1278 1279 |
* these counters are not modified in interrupt context, and * pte lock(a spinlock) is held, which implies preemption disabled. */ |
50658e2e0 mm: move page map... |
1280 |
__mod_node_page_state(page_pgdat(page), NR_FILE_MAPPED, -nr); |
62cccb8c8 mm: simplify lock... |
1281 |
mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED); |
8186eb6a7 mm: rmap: split o... |
1282 1283 1284 1285 |
if (unlikely(PageMlocked(page))) clear_page_mlock(page); out: |
62cccb8c8 mm: simplify lock... |
1286 |
unlock_page_memcg(page); |
8186eb6a7 mm: rmap: split o... |
1287 |
} |
53f9263ba mm: rework mapcou... |
1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 |
static void page_remove_anon_compound_rmap(struct page *page) { int i, nr; if (!atomic_add_negative(-1, compound_mapcount_ptr(page))) return; /* Hugepages are not counted in NR_ANON_PAGES for now. */ if (unlikely(PageHuge(page))) return; if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) return; |
11fb99898 mm: move most fil... |
1301 |
__dec_node_page_state(page, NR_ANON_THPS); |
53f9263ba mm: rework mapcou... |
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 |
if (TestClearPageDoubleMap(page)) { /* * Subpages can be mapped with PTEs too. Check how many of * themi are still mapped. */ for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) { if (atomic_add_negative(-1, &page[i]._mapcount)) nr++; } } else { nr = HPAGE_PMD_NR; } |
e90309c9f thp: allow mlocke... |
1315 1316 |
if (unlikely(PageMlocked(page))) clear_page_mlock(page); |
9a982250f thp: introduce de... |
1317 |
if (nr) { |
4b9d0fab7 mm: rename NR_ANO... |
1318 |
__mod_node_page_state(page_pgdat(page), NR_ANON_MAPPED, -nr); |
9a982250f thp: introduce de... |
1319 1320 |
deferred_split_huge_page(page); } |
53f9263ba mm: rework mapcou... |
1321 |
} |
1da177e4c Linux-2.6.12-rc2 |
1322 1323 |
/** * page_remove_rmap - take down pte mapping from a page |
d281ee614 rmap: add argumen... |
1324 1325 |
* @page: page to remove mapping from * @compound: uncharge the page as compound or small page |
1da177e4c Linux-2.6.12-rc2 |
1326 |
* |
b8072f099 [PATCH] mm: updat... |
1327 |
* The caller needs to hold the pte lock. |
1da177e4c Linux-2.6.12-rc2 |
1328 |
*/ |
d281ee614 rmap: add argumen... |
1329 |
void page_remove_rmap(struct page *page, bool compound) |
1da177e4c Linux-2.6.12-rc2 |
1330 |
{ |
dd78fedde rmap: support fil... |
1331 1332 |
if (!PageAnon(page)) return page_remove_file_rmap(page, compound); |
89c06bd52 memcg: use new lo... |
1333 |
|
53f9263ba mm: rework mapcou... |
1334 1335 |
if (compound) return page_remove_anon_compound_rmap(page); |
b904dcfed mm: clean up page... |
1336 1337 |
/* page still mapped by someone else? */ if (!atomic_add_negative(-1, &page->_mapcount)) |
8186eb6a7 mm: rmap: split o... |
1338 |
return; |
b904dcfed mm: clean up page... |
1339 |
/* |
bea04b073 mm: use the light... |
1340 1341 |
* We use the irq-unsafe __{inc|mod}_zone_page_stat because * these counters are not modified in interrupt context, and |
bea04b073 mm: use the light... |
1342 |
* pte lock(a spinlock) is held, which implies preemption disabled. |
0fe6e20b9 hugetlb, rmap: ad... |
1343 |
*/ |
4b9d0fab7 mm: rename NR_ANO... |
1344 |
__dec_node_page_state(page, NR_ANON_MAPPED); |
8186eb6a7 mm: rmap: split o... |
1345 |
|
e6c509f85 mm: use clear_pag... |
1346 1347 |
if (unlikely(PageMlocked(page))) clear_page_mlock(page); |
8186eb6a7 mm: rmap: split o... |
1348 |
|
9a982250f thp: introduce de... |
1349 1350 |
if (PageTransCompound(page)) deferred_split_huge_page(compound_head(page)); |
b904dcfed mm: clean up page... |
1351 1352 1353 1354 1355 1356 1357 1358 1359 |
/* * It would be tidy to reset the PageAnon mapping here, * but that might overwrite a racing page_add_anon_rmap * which increments mapcount after us but sets mapping * before us: so leave the reset to free_hot_cold_page, * and remember that it's only reliable while mapped. * Leaving it set also helps swapoff to reinstate ptes * faster for those pages still in swapcache. */ |
1da177e4c Linux-2.6.12-rc2 |
1360 |
} |
854e9ed09 mm: support madvi... |
1361 1362 1363 1364 |
struct rmap_private { enum ttu_flags flags; int lazyfreed; }; |
1da177e4c Linux-2.6.12-rc2 |
1365 |
/* |
526295064 mm/rmap: use rmap... |
1366 |
* @arg: enum ttu_flags will be passed to this argument |
1da177e4c Linux-2.6.12-rc2 |
1367 |
*/ |
ac7695012 mm/rmap.c: make p... |
1368 |
static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, |
526295064 mm/rmap: use rmap... |
1369 |
unsigned long address, void *arg) |
1da177e4c Linux-2.6.12-rc2 |
1370 1371 |
{ struct mm_struct *mm = vma->vm_mm; |
1da177e4c Linux-2.6.12-rc2 |
1372 1373 |
pte_t *pte; pte_t pteval; |
c0718806c [PATCH] mm: rmap ... |
1374 |
spinlock_t *ptl; |
1da177e4c Linux-2.6.12-rc2 |
1375 |
int ret = SWAP_AGAIN; |
854e9ed09 mm: support madvi... |
1376 1377 |
struct rmap_private *rp = arg; enum ttu_flags flags = rp->flags; |
1da177e4c Linux-2.6.12-rc2 |
1378 |
|
b87537d9e mm: rmap use pte ... |
1379 1380 1381 |
/* munlock has nothing to gain from examining un-locked vmas */ if ((flags & TTU_MUNLOCK) && !(vma->vm_flags & VM_LOCKED)) goto out; |
fec89c109 thp: rewrite free... |
1382 1383 1384 1385 1386 1387 1388 |
if (flags & TTU_SPLIT_HUGE_PMD) { split_huge_pmd_address(vma, address, flags & TTU_MIGRATION, page); /* check if we have anything to do after split */ if (page_mapcount(page) == 0) goto out; } |
55bda43bb mm: rmap: call pa... |
1389 1390 |
pte = page_check_address(page, mm, address, &ptl, PageTransCompound(page)); |
c0718806c [PATCH] mm: rmap ... |
1391 |
if (!pte) |
81b4082dc [PATCH] mm: rmap.... |
1392 |
goto out; |
1da177e4c Linux-2.6.12-rc2 |
1393 1394 1395 1396 1397 1398 |
/* * If the page is mlock()d, we cannot swap it out. * If it's recently referenced (perhaps page_referenced * skipped over this mm) then we should reactivate it. */ |
14fa31b89 HWPOISON: Use bit... |
1399 |
if (!(flags & TTU_IGNORE_MLOCK)) { |
b87537d9e mm: rmap use pte ... |
1400 |
if (vma->vm_flags & VM_LOCKED) { |
9a73f61bd thp, mlock: do no... |
1401 1402 1403 1404 1405 1406 1407 1408 |
/* PTE-mapped THP are never mlocked */ if (!PageTransCompound(page)) { /* * Holding pte lock, we do *not* need * mmap_sem here */ mlock_vma_page(page); } |
b87537d9e mm: rmap use pte ... |
1409 1410 1411 |
ret = SWAP_MLOCK; goto out_unmap; } |
daa5ba768 mm/rmap.c: cleanu... |
1412 |
if (flags & TTU_MUNLOCK) |
53f79acb6 mm: mlocking in t... |
1413 |
goto out_unmap; |
14fa31b89 HWPOISON: Use bit... |
1414 1415 |
} if (!(flags & TTU_IGNORE_ACCESS)) { |
b291f0003 mlock: mlocked pa... |
1416 1417 1418 1419 1420 |
if (ptep_clear_flush_young_notify(vma, address, pte)) { ret = SWAP_FAIL; goto out_unmap; } } |
1da177e4c Linux-2.6.12-rc2 |
1421 |
|
1da177e4c Linux-2.6.12-rc2 |
1422 1423 |
/* Nuke the page table entry. */ flush_cache_page(vma, address, page_to_pfn(page)); |
72b252aed mm: send one IPI ... |
1424 1425 1426 1427 1428 1429 1430 1431 1432 |
if (should_defer_flush(mm, flags)) { /* * We clear the PTE but do not flush so potentially a remote * CPU could still be writing to the page. If the entry was * previously clean then the architecture must guarantee that * a clear->dirty transition on a cached TLB entry is written * through and traps if the PTE is unmapped. */ pteval = ptep_get_and_clear(mm, address, pte); |
d950c9477 mm: defer flush o... |
1433 |
set_tlb_ubc_flush_pending(mm, page, pte_dirty(pteval)); |
72b252aed mm: send one IPI ... |
1434 1435 1436 |
} else { pteval = ptep_clear_flush(vma, address, pte); } |
1da177e4c Linux-2.6.12-rc2 |
1437 1438 1439 1440 |
/* Move the dirty bit to the physical page now the pte is gone. */ if (pte_dirty(pteval)) set_page_dirty(page); |
365e9c87a [PATCH] mm: updat... |
1441 1442 |
/* Update high watermark before we lower rss */ update_hiwater_rss(mm); |
888b9f7c5 HWPOISON: Handle ... |
1443 |
if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { |
5d317b2b6 mm: hugetlb: proc... |
1444 1445 1446 |
if (PageHuge(page)) { hugetlb_count_sub(1 << compound_order(page), mm); } else { |
eca56ff90 mm, shmem: add in... |
1447 |
dec_mm_counter(mm, mm_counter(page)); |
5f24ae585 hwpoison, hugetlb... |
1448 |
} |
888b9f7c5 HWPOISON: Handle ... |
1449 |
set_pte_at(mm, address, pte, |
5f24ae585 hwpoison, hugetlb... |
1450 |
swp_entry_to_pte(make_hwpoison_entry(page))); |
45961722f mm: add support f... |
1451 1452 1453 1454 1455 1456 |
} else if (pte_unused(pteval)) { /* * The guest indicated that the page content is of no * interest anymore. Simply discard the pte, vmscan * will take care of the rest. */ |
eca56ff90 mm, shmem: add in... |
1457 |
dec_mm_counter(mm, mm_counter(page)); |
470f119f0 mm: page migratio... |
1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 |
} else if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION)) { swp_entry_t entry; pte_t swp_pte; /* * Store the pfn of the page in a special migration * pte. do_swap_page() will wait until the migration * pte is removed and then restart fault handling. */ entry = make_migration_entry(page, pte_write(pteval)); swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); set_pte_at(mm, address, pte, swp_pte); |
888b9f7c5 HWPOISON: Handle ... |
1471 |
} else if (PageAnon(page)) { |
4c21e2f24 [PATCH] mm: split... |
1472 |
swp_entry_t entry = { .val = page_private(page) }; |
179ef71cb mm: save soft-dir... |
1473 |
pte_t swp_pte; |
470f119f0 mm: page migratio... |
1474 1475 1476 1477 1478 |
/* * Store the swap location in the pte. * See handle_pte_fault() ... */ VM_BUG_ON_PAGE(!PageSwapCache(page), page); |
854e9ed09 mm: support madvi... |
1479 1480 1481 1482 1483 1484 1485 |
if (!PageDirty(page) && (flags & TTU_LZFREE)) { /* It's a freeable page by MADV_FREE */ dec_mm_counter(mm, MM_ANONPAGES); rp->lazyfreed++; goto discard; } |
470f119f0 mm: page migratio... |
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 |
if (swap_duplicate(entry) < 0) { set_pte_at(mm, address, pte, pteval); ret = SWAP_FAIL; goto out_unmap; } if (list_empty(&mm->mmlist)) { spin_lock(&mmlist_lock); if (list_empty(&mm->mmlist)) list_add(&mm->mmlist, &init_mm.mmlist); spin_unlock(&mmlist_lock); |
1da177e4c Linux-2.6.12-rc2 |
1496 |
} |
470f119f0 mm: page migratio... |
1497 1498 |
dec_mm_counter(mm, MM_ANONPAGES); inc_mm_counter(mm, MM_SWAPENTS); |
179ef71cb mm: save soft-dir... |
1499 1500 1501 1502 |
swp_pte = swp_entry_to_pte(entry); if (pte_soft_dirty(pteval)) swp_pte = pte_swp_mksoft_dirty(swp_pte); set_pte_at(mm, address, pte, swp_pte); |
04e62a29b [PATCH] More page... |
1503 |
} else |
eca56ff90 mm, shmem: add in... |
1504 |
dec_mm_counter(mm, mm_counter_file(page)); |
1da177e4c Linux-2.6.12-rc2 |
1505 |
|
854e9ed09 mm: support madvi... |
1506 |
discard: |
d281ee614 rmap: add argumen... |
1507 |
page_remove_rmap(page, PageHuge(page)); |
09cbfeaf1 mm, fs: get rid o... |
1508 |
put_page(page); |
1da177e4c Linux-2.6.12-rc2 |
1509 1510 |
out_unmap: |
c0718806c [PATCH] mm: rmap ... |
1511 |
pte_unmap_unlock(pte, ptl); |
b87537d9e mm: rmap use pte ... |
1512 |
if (ret != SWAP_FAIL && ret != SWAP_MLOCK && !(flags & TTU_MUNLOCK)) |
2ec74c3ef mm: move all mmu ... |
1513 |
mmu_notifier_invalidate_page(mm, address); |
caed0f486 mm: simplify try_... |
1514 1515 |
out: return ret; |
1da177e4c Linux-2.6.12-rc2 |
1516 |
} |
71e3aac07 thp: transparent ... |
1517 |
bool is_vma_temporary_stack(struct vm_area_struct *vma) |
a8bef8ff6 mm: migration: av... |
1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 |
{ int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); if (!maybe_stack) return false; if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == VM_STACK_INCOMPLETE_SETUP) return true; return false; } |
526295064 mm/rmap: use rmap... |
1530 1531 1532 1533 |
static bool invalid_migration_vma(struct vm_area_struct *vma, void *arg) { return is_vma_temporary_stack(vma); } |
2a52bcbcc rmap: extend try_... |
1534 |
static int page_mapcount_is_zero(struct page *page) |
526295064 mm/rmap: use rmap... |
1535 |
{ |
2a52bcbcc rmap: extend try_... |
1536 1537 |
return !page_mapcount(page); } |
526295064 mm/rmap: use rmap... |
1538 |
|
1da177e4c Linux-2.6.12-rc2 |
1539 1540 1541 |
/** * try_to_unmap - try to remove all page table mappings to a page * @page: the page to get unmapped |
14fa31b89 HWPOISON: Use bit... |
1542 |
* @flags: action and flags |
1da177e4c Linux-2.6.12-rc2 |
1543 1544 1545 1546 1547 1548 1549 1550 |
* * Tries to remove all the page table entries which are mapping this * page, used in the pageout path. Caller must hold the page lock. * Return values are: * * SWAP_SUCCESS - we succeeded in removing all mappings * SWAP_AGAIN - we missed a mapping, try again later * SWAP_FAIL - the page is unswappable |
b291f0003 mlock: mlocked pa... |
1551 |
* SWAP_MLOCK - page is mlocked. |
1da177e4c Linux-2.6.12-rc2 |
1552 |
*/ |
14fa31b89 HWPOISON: Use bit... |
1553 |
int try_to_unmap(struct page *page, enum ttu_flags flags) |
1da177e4c Linux-2.6.12-rc2 |
1554 1555 |
{ int ret; |
854e9ed09 mm: support madvi... |
1556 1557 1558 1559 |
struct rmap_private rp = { .flags = flags, .lazyfreed = 0, }; |
526295064 mm/rmap: use rmap... |
1560 1561 |
struct rmap_walk_control rwc = { .rmap_one = try_to_unmap_one, |
854e9ed09 mm: support madvi... |
1562 |
.arg = &rp, |
2a52bcbcc rmap: extend try_... |
1563 |
.done = page_mapcount_is_zero, |
526295064 mm/rmap: use rmap... |
1564 1565 |
.anon_lock = page_lock_anon_vma_read, }; |
1da177e4c Linux-2.6.12-rc2 |
1566 |
|
526295064 mm/rmap: use rmap... |
1567 1568 1569 1570 1571 1572 1573 1574 |
/* * During exec, a temporary VMA is setup and later moved. * The VMA is moved under the anon_vma lock but not the * page tables leading to a race where migration cannot * find the migration ptes. Rather than increasing the * locking requirements of exec(), migration skips * temporary VMAs until after exec() completes. */ |
daa5ba768 mm/rmap.c: cleanu... |
1575 |
if ((flags & TTU_MIGRATION) && !PageKsm(page) && PageAnon(page)) |
526295064 mm/rmap: use rmap... |
1576 |
rwc.invalid_vma = invalid_migration_vma; |
2a52bcbcc rmap: extend try_... |
1577 1578 1579 1580 |
if (flags & TTU_RMAP_LOCKED) ret = rmap_walk_locked(page, &rwc); else ret = rmap_walk(page, &rwc); |
526295064 mm/rmap: use rmap... |
1581 |
|
2a52bcbcc rmap: extend try_... |
1582 |
if (ret != SWAP_MLOCK && !page_mapcount(page)) { |
1da177e4c Linux-2.6.12-rc2 |
1583 |
ret = SWAP_SUCCESS; |
854e9ed09 mm: support madvi... |
1584 1585 1586 |
if (rp.lazyfreed && !PageDirty(page)) ret = SWAP_LZFREE; } |
1da177e4c Linux-2.6.12-rc2 |
1587 1588 |
return ret; } |
81b4082dc [PATCH] mm: rmap.... |
1589 |
|
2a52bcbcc rmap: extend try_... |
1590 1591 1592 1593 |
static int page_not_mapped(struct page *page) { return !page_mapped(page); }; |
b291f0003 mlock: mlocked pa... |
1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 |
/** * try_to_munlock - try to munlock a page * @page: the page to be munlocked * * Called from munlock code. Checks all of the VMAs mapping the page * to make sure nobody else has this page mlocked. The page will be * returned with PG_mlocked cleared if no other vmas have it mlocked. * * Return values are: * |
53f79acb6 mm: mlocking in t... |
1604 |
* SWAP_AGAIN - no vma is holding page mlocked, or, |
b291f0003 mlock: mlocked pa... |
1605 |
* SWAP_AGAIN - page mapped in mlocked vma -- couldn't acquire mmap sem |
5ad646880 ksm: let shared p... |
1606 |
* SWAP_FAIL - page cannot be located at present |
b291f0003 mlock: mlocked pa... |
1607 1608 1609 1610 |
* SWAP_MLOCK - page is now mlocked. */ int try_to_munlock(struct page *page) { |
e8351ac9b mm/rmap: use rmap... |
1611 |
int ret; |
854e9ed09 mm: support madvi... |
1612 1613 1614 1615 |
struct rmap_private rp = { .flags = TTU_MUNLOCK, .lazyfreed = 0, }; |
e8351ac9b mm/rmap: use rmap... |
1616 1617 |
struct rmap_walk_control rwc = { .rmap_one = try_to_unmap_one, |
854e9ed09 mm: support madvi... |
1618 |
.arg = &rp, |
e8351ac9b mm/rmap: use rmap... |
1619 |
.done = page_not_mapped, |
e8351ac9b mm/rmap: use rmap... |
1620 1621 1622 |
.anon_lock = page_lock_anon_vma_read, }; |
309381fea mm: dump page whe... |
1623 |
VM_BUG_ON_PAGE(!PageLocked(page) || PageLRU(page), page); |
b291f0003 mlock: mlocked pa... |
1624 |
|
e8351ac9b mm/rmap: use rmap... |
1625 1626 |
ret = rmap_walk(page, &rwc); return ret; |
b291f0003 mlock: mlocked pa... |
1627 |
} |
e9995ef97 ksm: rmap_walk to... |
1628 |
|
01d8b20de mm: simplify anon... |
1629 |
void __put_anon_vma(struct anon_vma *anon_vma) |
76545066c mm: extend KSM re... |
1630 |
{ |
01d8b20de mm: simplify anon... |
1631 |
struct anon_vma *root = anon_vma->root; |
76545066c mm: extend KSM re... |
1632 |
|
624483f3e mm: rmap: fix use... |
1633 |
anon_vma_free(anon_vma); |
01d8b20de mm: simplify anon... |
1634 1635 |
if (root != anon_vma && atomic_dec_and_test(&root->refcount)) anon_vma_free(root); |
76545066c mm: extend KSM re... |
1636 |
} |
76545066c mm: extend KSM re... |
1637 |
|
0dd1c7bbc mm/rmap: extend r... |
1638 1639 |
static struct anon_vma *rmap_walk_anon_lock(struct page *page, struct rmap_walk_control *rwc) |
faecd8dd8 mm/rmap: factor l... |
1640 1641 |
{ struct anon_vma *anon_vma; |
0dd1c7bbc mm/rmap: extend r... |
1642 1643 |
if (rwc->anon_lock) return rwc->anon_lock(page); |
faecd8dd8 mm/rmap: factor l... |
1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 |
/* * Note: remove_migration_ptes() cannot use page_lock_anon_vma_read() * because that depends on page_mapped(); but not all its usages * are holding mmap_sem. Users without mmap_sem are required to * take a reference count to prevent the anon_vma disappearing */ anon_vma = page_anon_vma(page); if (!anon_vma) return NULL; anon_vma_lock_read(anon_vma); return anon_vma; } |
e9995ef97 ksm: rmap_walk to... |
1657 |
/* |
e8351ac9b mm/rmap: use rmap... |
1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 |
* rmap_walk_anon - do something to anonymous page using the object-based * rmap method * @page: the page to be handled * @rwc: control variable according to each walk type * * Find all the mappings of a page using the mapping pointer and the vma chains * contained in the anon_vma struct it points to. * * When called from try_to_munlock(), the mmap_sem of the mm containing the vma * where the page was found will be held for write. So, we won't recheck * vm_flags for that VMA. That should be OK, because that vma shouldn't be * LOCKED. |
e9995ef97 ksm: rmap_walk to... |
1670 |
*/ |
b97731992 rmap: introduce r... |
1671 1672 |
static int rmap_walk_anon(struct page *page, struct rmap_walk_control *rwc, bool locked) |
e9995ef97 ksm: rmap_walk to... |
1673 1674 |
{ struct anon_vma *anon_vma; |
b258d8606 mm/rmap: calculat... |
1675 |
pgoff_t pgoff; |
5beb49305 mm: change anon_v... |
1676 |
struct anon_vma_chain *avc; |
e9995ef97 ksm: rmap_walk to... |
1677 |
int ret = SWAP_AGAIN; |
b97731992 rmap: introduce r... |
1678 1679 1680 1681 1682 1683 1684 |
if (locked) { anon_vma = page_anon_vma(page); /* anon_vma disappear under us? */ VM_BUG_ON_PAGE(!anon_vma, page); } else { anon_vma = rmap_walk_anon_lock(page, rwc); } |
e9995ef97 ksm: rmap_walk to... |
1685 1686 |
if (!anon_vma) return ret; |
faecd8dd8 mm/rmap: factor l... |
1687 |
|
b258d8606 mm/rmap: calculat... |
1688 |
pgoff = page_to_pgoff(page); |
bf181b9f9 mm anon rmap: rep... |
1689 |
anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff, pgoff) { |
5beb49305 mm: change anon_v... |
1690 |
struct vm_area_struct *vma = avc->vma; |
e9995ef97 ksm: rmap_walk to... |
1691 |
unsigned long address = vma_address(page, vma); |
0dd1c7bbc mm/rmap: extend r... |
1692 |
|
ad12695f1 ksm: add cond_res... |
1693 |
cond_resched(); |
0dd1c7bbc mm/rmap: extend r... |
1694 1695 |
if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; |
051ac83ad mm/rmap: make rma... |
1696 |
ret = rwc->rmap_one(page, vma, address, rwc->arg); |
e9995ef97 ksm: rmap_walk to... |
1697 1698 |
if (ret != SWAP_AGAIN) break; |
0dd1c7bbc mm/rmap: extend r... |
1699 1700 |
if (rwc->done && rwc->done(page)) break; |
e9995ef97 ksm: rmap_walk to... |
1701 |
} |
b97731992 rmap: introduce r... |
1702 1703 1704 |
if (!locked) anon_vma_unlock_read(anon_vma); |
e9995ef97 ksm: rmap_walk to... |
1705 1706 |
return ret; } |
e8351ac9b mm/rmap: use rmap... |
1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 |
/* * rmap_walk_file - do something to file page using the object-based rmap method * @page: the page to be handled * @rwc: control variable according to each walk type * * Find all the mappings of a page using the mapping pointer and the vma chains * contained in the address_space struct it points to. * * When called from try_to_munlock(), the mmap_sem of the mm containing the vma * where the page was found will be held for write. So, we won't recheck * vm_flags for that VMA. That should be OK, because that vma shouldn't be * LOCKED. */ |
b97731992 rmap: introduce r... |
1720 1721 |
static int rmap_walk_file(struct page *page, struct rmap_walk_control *rwc, bool locked) |
e9995ef97 ksm: rmap_walk to... |
1722 |
{ |
b97731992 rmap: introduce r... |
1723 |
struct address_space *mapping = page_mapping(page); |
b258d8606 mm/rmap: calculat... |
1724 |
pgoff_t pgoff; |
e9995ef97 ksm: rmap_walk to... |
1725 |
struct vm_area_struct *vma; |
e9995ef97 ksm: rmap_walk to... |
1726 |
int ret = SWAP_AGAIN; |
9f32624be mm/rmap: use rmap... |
1727 1728 1729 1730 |
/* * The page lock not only makes sure that page->mapping cannot * suddenly be NULLified by truncation, it makes sure that the * structure at mapping cannot be freed and reused yet, |
c8c06efa8 mm: convert i_mma... |
1731 |
* so we can safely take mapping->i_mmap_rwsem. |
9f32624be mm/rmap: use rmap... |
1732 |
*/ |
81d1b09c6 mm: convert a few... |
1733 |
VM_BUG_ON_PAGE(!PageLocked(page), page); |
9f32624be mm/rmap: use rmap... |
1734 |
|
e9995ef97 ksm: rmap_walk to... |
1735 1736 |
if (!mapping) return ret; |
3dec0ba0b mm/rmap: share th... |
1737 |
|
b258d8606 mm/rmap: calculat... |
1738 |
pgoff = page_to_pgoff(page); |
b97731992 rmap: introduce r... |
1739 1740 |
if (!locked) i_mmap_lock_read(mapping); |
6b2dbba8b mm: replace vma p... |
1741 |
vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) { |
e9995ef97 ksm: rmap_walk to... |
1742 |
unsigned long address = vma_address(page, vma); |
0dd1c7bbc mm/rmap: extend r... |
1743 |
|
ad12695f1 ksm: add cond_res... |
1744 |
cond_resched(); |
0dd1c7bbc mm/rmap: extend r... |
1745 1746 |
if (rwc->invalid_vma && rwc->invalid_vma(vma, rwc->arg)) continue; |
051ac83ad mm/rmap: make rma... |
1747 |
ret = rwc->rmap_one(page, vma, address, rwc->arg); |
e9995ef97 ksm: rmap_walk to... |
1748 |
if (ret != SWAP_AGAIN) |
0dd1c7bbc mm/rmap: extend r... |
1749 1750 1751 |
goto done; if (rwc->done && rwc->done(page)) goto done; |
e9995ef97 ksm: rmap_walk to... |
1752 |
} |
0dd1c7bbc mm/rmap: extend r... |
1753 |
|
0dd1c7bbc mm/rmap: extend r... |
1754 |
done: |
b97731992 rmap: introduce r... |
1755 1756 |
if (!locked) i_mmap_unlock_read(mapping); |
e9995ef97 ksm: rmap_walk to... |
1757 1758 |
return ret; } |
051ac83ad mm/rmap: make rma... |
1759 |
int rmap_walk(struct page *page, struct rmap_walk_control *rwc) |
e9995ef97 ksm: rmap_walk to... |
1760 |
{ |
e9995ef97 ksm: rmap_walk to... |
1761 |
if (unlikely(PageKsm(page))) |
051ac83ad mm/rmap: make rma... |
1762 |
return rmap_walk_ksm(page, rwc); |
e9995ef97 ksm: rmap_walk to... |
1763 |
else if (PageAnon(page)) |
b97731992 rmap: introduce r... |
1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 |
return rmap_walk_anon(page, rwc, false); else return rmap_walk_file(page, rwc, false); } /* Like rmap_walk, but caller holds relevant rmap lock */ int rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc) { /* no ksm support for now */ VM_BUG_ON_PAGE(PageKsm(page), page); if (PageAnon(page)) return rmap_walk_anon(page, rwc, true); |
e9995ef97 ksm: rmap_walk to... |
1776 |
else |
b97731992 rmap: introduce r... |
1777 |
return rmap_walk_file(page, rwc, true); |
e9995ef97 ksm: rmap_walk to... |
1778 |
} |
0fe6e20b9 hugetlb, rmap: ad... |
1779 |
|
e3390f67a hwpoison: rename ... |
1780 |
#ifdef CONFIG_HUGETLB_PAGE |
0fe6e20b9 hugetlb, rmap: ad... |
1781 1782 1783 1784 1785 1786 1787 1788 1789 |
/* * The following three functions are for anonymous (private mapped) hugepages. * Unlike common anonymous pages, anonymous hugepages have no accounting code * and no lru code, because we handle hugepages differently from common pages. */ static void __hugepage_set_anon_rmap(struct page *page, struct vm_area_struct *vma, unsigned long address, int exclusive) { struct anon_vma *anon_vma = vma->anon_vma; |
433abed6c hugetlb, rmap: al... |
1790 |
|
0fe6e20b9 hugetlb, rmap: ad... |
1791 |
BUG_ON(!anon_vma); |
433abed6c hugetlb, rmap: al... |
1792 1793 1794 1795 1796 |
if (PageAnon(page)) return; if (!exclusive) anon_vma = anon_vma->root; |
0fe6e20b9 hugetlb, rmap: ad... |
1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 |
anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON; page->mapping = (struct address_space *) anon_vma; page->index = linear_page_index(vma, address); } void hugepage_add_anon_rmap(struct page *page, struct vm_area_struct *vma, unsigned long address) { struct anon_vma *anon_vma = vma->anon_vma; int first; |
a850ea303 hugetlb, rmap: ad... |
1807 1808 |
BUG_ON(!PageLocked(page)); |
0fe6e20b9 hugetlb, rmap: ad... |
1809 |
BUG_ON(!anon_vma); |
5dbe0af47 mm: fix kernel BU... |
1810 |
/* address might be in next vma when migration races vma_adjust */ |
53f9263ba mm: rework mapcou... |
1811 |
first = atomic_inc_and_test(compound_mapcount_ptr(page)); |
0fe6e20b9 hugetlb, rmap: ad... |
1812 1813 1814 1815 1816 1817 1818 1819 |
if (first) __hugepage_set_anon_rmap(page, vma, address, 0); } void hugepage_add_new_anon_rmap(struct page *page, struct vm_area_struct *vma, unsigned long address) { BUG_ON(address < vma->vm_start || address >= vma->vm_end); |
53f9263ba mm: rework mapcou... |
1820 |
atomic_set(compound_mapcount_ptr(page), 0); |
0fe6e20b9 hugetlb, rmap: ad... |
1821 1822 |
__hugepage_set_anon_rmap(page, vma, address, 1); } |
e3390f67a hwpoison: rename ... |
1823 |
#endif /* CONFIG_HUGETLB_PAGE */ |