Blame view
mm/page_vma_mapped.c
8.74 KB
b24413180
|
1 |
// SPDX-License-Identifier: GPL-2.0 |
ace71a19c
|
2 3 4 5 6 7 8 |
#include <linux/mm.h> #include <linux/rmap.h> #include <linux/hugetlb.h> #include <linux/swap.h> #include <linux/swapops.h> #include "internal.h" |
ace71a19c
|
9 10 11 12 13 14 15 16 17 18 19 20 21 22 |
static inline bool not_found(struct page_vma_mapped_walk *pvmw) { page_vma_mapped_walk_done(pvmw); return false; } static bool map_pte(struct page_vma_mapped_walk *pvmw) { pvmw->pte = pte_offset_map(pvmw->pmd, pvmw->address); if (!(pvmw->flags & PVMW_SYNC)) { if (pvmw->flags & PVMW_MIGRATION) { if (!is_swap_pte(*pvmw->pte)) return false; } else { |
aab8d0520
|
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
/* * We get here when we are trying to unmap a private * device page from the process address space. Such * page is not CPU accessible and thus is mapped as * a special swap entry, nonetheless it still does * count as a valid regular mapping for the page (and * is accounted as such in page maps count). * * So handle this special case as if it was a normal * page mapping ie lock CPU page table and returns * true. * * For more details on device private memory see HMM * (include/linux/hmm.h or mm/hmm.c). */ if (is_swap_pte(*pvmw->pte)) { swp_entry_t entry; /* Handle un-addressable ZONE_DEVICE memory */ entry = pte_to_swp_entry(*pvmw->pte); |
b756a3b5e
|
43 44 |
if (!is_device_private_entry(entry) && !is_device_exclusive_entry(entry)) |
aab8d0520
|
45 46 |
return false; } else if (!pte_present(*pvmw->pte)) |
ace71a19c
|
47 48 49 50 51 52 53 |
return false; } } pvmw->ptl = pte_lockptr(pvmw->vma->vm_mm, pvmw->pmd); spin_lock(pvmw->ptl); return true; } |
5b8d6e37b
|
54 |
static inline bool pfn_is_match(struct page *page, unsigned long pfn) |
7222708e8
|
55 |
{ |
5b8d6e37b
|
56 57 58 59 60 |
unsigned long page_pfn = page_to_pfn(page); /* normal page and hugetlbfs page */ if (!PageTransCompound(page) || PageHuge(page)) return page_pfn == pfn; |
7222708e8
|
61 62 |
/* THP can be referenced by any subpage */ |
6c357848b
|
63 |
return pfn >= page_pfn && pfn - page_pfn < thp_nr_pages(page); |
7222708e8
|
64 |
} |
0d665e7b1
|
65 66 |
/** * check_pte - check if @pvmw->page is mapped at the @pvmw->pte |
777f303c0
|
67 |
* @pvmw: page_vma_mapped_walk struct, includes a pair pte and page for checking |
0d665e7b1
|
68 69 70 71 |
* * page_vma_mapped_walk() found a place where @pvmw->page is *potentially* * mapped. check_pte() has to validate this. * |
777f303c0
|
72 73 |
* pvmw->pte may point to empty PTE, swap PTE or PTE pointing to * arbitrary page. |
0d665e7b1
|
74 75 76 77 |
* * If PVMW_MIGRATION flag is set, returns true if @pvmw->pte contains migration * entry that points to @pvmw->page or any subpage in case of THP. * |
777f303c0
|
78 79 |
* If PVMW_MIGRATION flag is not set, returns true if pvmw->pte points to * pvmw->page or any subpage in case of THP. |
0d665e7b1
|
80 81 82 83 |
* * Otherwise, return false. * */ |
ace71a19c
|
84 85 |
static bool check_pte(struct page_vma_mapped_walk *pvmw) { |
0d665e7b1
|
86 |
unsigned long pfn; |
ace71a19c
|
87 |
if (pvmw->flags & PVMW_MIGRATION) { |
ace71a19c
|
88 89 90 91 |
swp_entry_t entry; if (!is_swap_pte(*pvmw->pte)) return false; entry = pte_to_swp_entry(*pvmw->pte); |
a5430dda8
|
92 |
|
b756a3b5e
|
93 94 |
if (!is_migration_entry(entry) && !is_device_exclusive_entry(entry)) |
ace71a19c
|
95 |
return false; |
a5430dda8
|
96 |
|
af5cdaf82
|
97 |
pfn = swp_offset(entry); |
0d665e7b1
|
98 99 |
} else if (is_swap_pte(*pvmw->pte)) { swp_entry_t entry; |
a5430dda8
|
100 |
|
0d665e7b1
|
101 102 |
/* Handle un-addressable ZONE_DEVICE memory */ entry = pte_to_swp_entry(*pvmw->pte); |
b756a3b5e
|
103 104 |
if (!is_device_private_entry(entry) && !is_device_exclusive_entry(entry)) |
ace71a19c
|
105 |
return false; |
af5cdaf82
|
106 |
pfn = swp_offset(entry); |
0d665e7b1
|
107 108 |
} else { if (!pte_present(*pvmw->pte)) |
ace71a19c
|
109 |
return false; |
0d665e7b1
|
110 111 |
pfn = pte_pfn(*pvmw->pte); |
ace71a19c
|
112 |
} |
5b8d6e37b
|
113 |
return pfn_is_match(pvmw->page, pfn); |
ace71a19c
|
114 |
} |
a9a7504d9
|
115 116 117 118 119 120 |
static void step_forward(struct page_vma_mapped_walk *pvmw, unsigned long size) { pvmw->address = (pvmw->address + size) & ~(size - 1); if (!pvmw->address) pvmw->address = ULONG_MAX; } |
ace71a19c
|
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
/** * page_vma_mapped_walk - check if @pvmw->page is mapped in @pvmw->vma at * @pvmw->address * @pvmw: pointer to struct page_vma_mapped_walk. page, vma, address and flags * must be set. pmd, pte and ptl must be NULL. * * Returns true if the page is mapped in the vma. @pvmw->pmd and @pvmw->pte point * to relevant page table entries. @pvmw->ptl is locked. @pvmw->address is * adjusted if needed (for PTE-mapped THPs). * * If @pvmw->pmd is set but @pvmw->pte is not, you have found PMD-mapped page * (usually THP). For PTE-mapped THP, you should run page_vma_mapped_walk() in * a loop to find all PTEs that map the THP. * * For HugeTLB pages, @pvmw->pte is set to the relevant page table entry * regardless of which page table level the page is mapped at. @pvmw->pmd is * NULL. * |
baf2f90ba
|
139 |
* Returns false if there are no more page table entries for the page in |
ace71a19c
|
140 141 142 143 144 145 146 147 148 |
* the vma. @pvmw->ptl is unlocked and @pvmw->pte is unmapped. * * If you need to stop the walk before page_vma_mapped_walk() returned false, * use page_vma_mapped_walk_done(). It will do the housekeeping. */ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) { struct mm_struct *mm = pvmw->vma->vm_mm; struct page *page = pvmw->page; |
474466301
|
149 |
unsigned long end; |
ace71a19c
|
150 |
pgd_t *pgd; |
c2febafc6
|
151 |
p4d_t *p4d; |
ace71a19c
|
152 |
pud_t *pud; |
a7b100953
|
153 |
pmd_t pmde; |
ace71a19c
|
154 155 156 157 |
/* The only possible pmd mapping has been handled on last iteration */ if (pvmw->pmd && !pvmw->pte) return not_found(pvmw); |
f003c03bd
|
158 |
if (unlikely(PageHuge(page))) { |
6d0fd5987
|
159 160 161 |
/* The only possible mapping was handled on last iteration */ if (pvmw->pte) return not_found(pvmw); |
ace71a19c
|
162 |
/* when pud is not present, pte will be NULL */ |
a50b854e0
|
163 |
pvmw->pte = huge_pte_offset(mm, pvmw->address, page_size(page)); |
ace71a19c
|
164 165 166 167 168 169 170 171 172 |
if (!pvmw->pte) return false; pvmw->ptl = huge_pte_lockptr(page_hstate(page), mm, pvmw->pte); spin_lock(pvmw->ptl); if (!check_pte(pvmw)) return not_found(pvmw); return true; } |
6d0fd5987
|
173 |
|
a765c417d
|
174 175 176 177 178 179 180 181 182 |
/* * Seek to next pte only makes sense for THP. * But more important than that optimization, is to filter out * any PageKsm page: whose page->index misleads vma_address() * and vma_address_end() to disaster. */ end = PageTransCompound(page) ? vma_address_end(page, pvmw->vma) : pvmw->address + PAGE_SIZE; |
6d0fd5987
|
183 184 |
if (pvmw->pte) goto next_pte; |
ace71a19c
|
185 |
restart: |
a9a7504d9
|
186 |
do { |
b3807a91a
|
187 |
pgd = pgd_offset(mm, pvmw->address); |
a9a7504d9
|
188 189 190 191 |
if (!pgd_present(*pgd)) { step_forward(pvmw, PGDIR_SIZE); continue; } |
b3807a91a
|
192 |
p4d = p4d_offset(pgd, pvmw->address); |
a9a7504d9
|
193 194 195 196 |
if (!p4d_present(*p4d)) { step_forward(pvmw, P4D_SIZE); continue; } |
b3807a91a
|
197 |
pud = pud_offset(p4d, pvmw->address); |
a9a7504d9
|
198 199 200 201 |
if (!pud_present(*pud)) { step_forward(pvmw, PUD_SIZE); continue; } |
e2e1d4076
|
202 |
|
b3807a91a
|
203 |
pvmw->pmd = pmd_offset(pud, pvmw->address); |
732ed5582
|
204 |
/* |
b3807a91a
|
205 206 207 |
* Make sure the pmd value isn't cached in a register by the * compiler and used as a stale value after we've observed a * subsequent update. |
732ed5582
|
208 |
*/ |
b3807a91a
|
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 |
pmde = READ_ONCE(*pvmw->pmd); if (pmd_trans_huge(pmde) || is_pmd_migration_entry(pmde)) { pvmw->ptl = pmd_lock(mm, pvmw->pmd); pmde = *pvmw->pmd; if (likely(pmd_trans_huge(pmde))) { if (pvmw->flags & PVMW_MIGRATION) return not_found(pvmw); if (pmd_page(pmde) != page) return not_found(pvmw); return true; } if (!pmd_present(pmde)) { swp_entry_t entry; if (!thp_migration_supported() || !(pvmw->flags & PVMW_MIGRATION)) return not_found(pvmw); entry = pmd_to_swp_entry(pmde); if (!is_migration_entry(entry) || |
af5cdaf82
|
229 |
pfn_swap_entry_to_page(entry) != page) |
b3807a91a
|
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 |
return not_found(pvmw); return true; } /* THP pmd was split under us: handle on pte level */ spin_unlock(pvmw->ptl); pvmw->ptl = NULL; } else if (!pmd_present(pmde)) { /* * If PVMW_SYNC, take and drop THP pmd lock so that we * cannot return prematurely, while zap_huge_pmd() has * cleared *pmd but not decremented compound_mapcount(). */ if ((pvmw->flags & PVMW_SYNC) && PageTransCompound(page)) { spinlock_t *ptl = pmd_lock(mm, pvmw->pmd); |
732ed5582
|
245 |
|
b3807a91a
|
246 247 |
spin_unlock(ptl); } |
a9a7504d9
|
248 249 |
step_forward(pvmw, PMD_SIZE); continue; |
732ed5582
|
250 |
} |
b3807a91a
|
251 252 |
if (!map_pte(pvmw)) goto next_pte; |
474466301
|
253 |
this_pte: |
ace71a19c
|
254 255 |
if (check_pte(pvmw)) return true; |
d75450ff4
|
256 |
next_pte: |
d75450ff4
|
257 |
do { |
ace71a19c
|
258 |
pvmw->address += PAGE_SIZE; |
494334e43
|
259 |
if (pvmw->address >= end) |
ace71a19c
|
260 261 |
return not_found(pvmw); /* Did we cross page table boundary? */ |
448282487
|
262 |
if ((pvmw->address & (PMD_SIZE - PAGE_SIZE)) == 0) { |
ace71a19c
|
263 264 265 266 |
if (pvmw->ptl) { spin_unlock(pvmw->ptl); pvmw->ptl = NULL; } |
448282487
|
267 268 |
pte_unmap(pvmw->pte); pvmw->pte = NULL; |
ace71a19c
|
269 |
goto restart; |
ace71a19c
|
270 |
} |
448282487
|
271 |
pvmw->pte++; |
a7a69d8ba
|
272 273 274 275 |
if ((pvmw->flags & PVMW_SYNC) && !pvmw->ptl) { pvmw->ptl = pte_lockptr(mm, pvmw->pmd); spin_lock(pvmw->ptl); } |
ace71a19c
|
276 277 278 279 280 281 |
} while (pte_none(*pvmw->pte)); if (!pvmw->ptl) { pvmw->ptl = pte_lockptr(mm, pvmw->pmd); spin_lock(pvmw->ptl); } |
474466301
|
282 |
goto this_pte; |
a9a7504d9
|
283 284 285 |
} while (pvmw->address < end); return false; |
ace71a19c
|
286 |
} |
6a328a626
|
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 |
/** * page_mapped_in_vma - check whether a page is really mapped in a VMA * @page: the page to test * @vma: the VMA to test * * Returns 1 if the page is mapped into the page tables of the VMA, 0 * if the page is not mapped into the page tables of this VMA. Only * valid for normal file or anonymous VMAs. */ int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma) { struct page_vma_mapped_walk pvmw = { .page = page, .vma = vma, .flags = PVMW_SYNC, }; |
6a328a626
|
304 |
|
494334e43
|
305 306 |
pvmw.address = vma_address(page, vma); if (pvmw.address == -EFAULT) |
6a328a626
|
307 |
return 0; |
6a328a626
|
308 309 310 311 312 |
if (!page_vma_mapped_walk(&pvmw)) return 0; page_vma_mapped_walk_done(&pvmw); return 1; } |