Blame view
mm/pagewalk.c
5.89 KB
e6473092b maps4: introduce ... |
1 2 3 |
#include <linux/mm.h> #include <linux/highmem.h> #include <linux/sched.h> |
d33b9f45b mm: hugetlb: fix ... |
4 |
#include <linux/hugetlb.h> |
e6473092b maps4: introduce ... |
5 6 |
static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, |
2165009bd pagemap: pass mm ... |
7 |
struct mm_walk *walk) |
e6473092b maps4: introduce ... |
8 9 10 11 12 |
{ pte_t *pte; int err = 0; pte = pte_offset_map(pmd, addr); |
556637cda mm: fix possible ... |
13 |
for (;;) { |
2165009bd pagemap: pass mm ... |
14 |
err = walk->pte_entry(pte, addr, addr + PAGE_SIZE, walk); |
e6473092b maps4: introduce ... |
15 16 |
if (err) break; |
556637cda mm: fix possible ... |
17 18 19 20 21 |
addr += PAGE_SIZE; if (addr == end) break; pte++; } |
e6473092b maps4: introduce ... |
22 23 24 25 26 27 |
pte_unmap(pte); return err; } static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, |
2165009bd pagemap: pass mm ... |
28 |
struct mm_walk *walk) |
e6473092b maps4: introduce ... |
29 30 31 32 33 34 35 |
{ pmd_t *pmd; unsigned long next; int err = 0; pmd = pmd_offset(pud, addr); do { |
033193275 pagewalk: only sp... |
36 |
again: |
e6473092b maps4: introduce ... |
37 |
next = pmd_addr_end(addr, end); |
033193275 pagewalk: only sp... |
38 |
if (pmd_none(*pmd)) { |
e6473092b maps4: introduce ... |
39 |
if (walk->pte_hole) |
2165009bd pagemap: pass mm ... |
40 |
err = walk->pte_hole(addr, next, walk); |
e6473092b maps4: introduce ... |
41 42 43 44 |
if (err) break; continue; } |
033193275 pagewalk: only sp... |
45 46 47 48 |
/* * This implies that each ->pmd_entry() handler * needs to know about pmd_trans_huge() pmds */ |
e6473092b maps4: introduce ... |
49 |
if (walk->pmd_entry) |
2165009bd pagemap: pass mm ... |
50 |
err = walk->pmd_entry(pmd, addr, next, walk); |
033193275 pagewalk: only sp... |
51 52 53 54 55 56 57 58 59 |
if (err) break; /* * Check this here so we only break down trans_huge * pages when we _need_ to */ if (!walk->pte_entry) continue; |
e180377f1 thp: change split... |
60 |
split_huge_page_pmd_mm(walk->mm, addr, pmd); |
1a5a9906d mm: thp: fix pmd_... |
61 |
if (pmd_none_or_trans_huge_or_clear_bad(pmd)) |
033193275 pagewalk: only sp... |
62 63 |
goto again; err = walk_pte_range(pmd, addr, next, walk); |
e6473092b maps4: introduce ... |
64 65 66 67 68 69 70 71 |
if (err) break; } while (pmd++, addr = next, addr != end); return err; } static int walk_pud_range(pgd_t *pgd, unsigned long addr, unsigned long end, |
2165009bd pagemap: pass mm ... |
72 |
struct mm_walk *walk) |
e6473092b maps4: introduce ... |
73 74 75 76 77 78 79 80 81 82 |
{ pud_t *pud; unsigned long next; int err = 0; pud = pud_offset(pgd, addr); do { next = pud_addr_end(addr, end); if (pud_none_or_clear_bad(pud)) { if (walk->pte_hole) |
2165009bd pagemap: pass mm ... |
83 |
err = walk->pte_hole(addr, next, walk); |
e6473092b maps4: introduce ... |
84 85 86 87 88 |
if (err) break; continue; } if (walk->pud_entry) |
2165009bd pagemap: pass mm ... |
89 |
err = walk->pud_entry(pud, addr, next, walk); |
e6473092b maps4: introduce ... |
90 |
if (!err && (walk->pmd_entry || walk->pte_entry)) |
2165009bd pagemap: pass mm ... |
91 |
err = walk_pmd_range(pud, addr, next, walk); |
e6473092b maps4: introduce ... |
92 93 94 95 96 97 |
if (err) break; } while (pud++, addr = next, addr != end); return err; } |
116354d17 pagemap: fix pfn ... |
98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
#ifdef CONFIG_HUGETLB_PAGE static unsigned long hugetlb_entry_end(struct hstate *h, unsigned long addr, unsigned long end) { unsigned long boundary = (addr & huge_page_mask(h)) + huge_page_size(h); return boundary < end ? boundary : end; } static int walk_hugetlb_range(struct vm_area_struct *vma, unsigned long addr, unsigned long end, struct mm_walk *walk) { struct hstate *h = hstate_vma(vma); unsigned long next; unsigned long hmask = huge_page_mask(h); pte_t *pte; int err = 0; do { next = hugetlb_entry_end(h, addr, end); pte = huge_pte_offset(walk->mm, addr & hmask); if (pte && walk->hugetlb_entry) err = walk->hugetlb_entry(pte, hmask, addr, next, walk); if (err) return err; } while (addr = next, addr != end); return 0; } |
6c6d52804 pagewalk: don't l... |
127 |
|
6c6d52804 pagewalk: don't l... |
128 |
#else /* CONFIG_HUGETLB_PAGE */ |
6c6d52804 pagewalk: don't l... |
129 130 131 132 133 134 135 136 |
static int walk_hugetlb_range(struct vm_area_struct *vma, unsigned long addr, unsigned long end, struct mm_walk *walk) { return 0; } #endif /* CONFIG_HUGETLB_PAGE */ |
116354d17 pagemap: fix pfn ... |
137 |
|
e6473092b maps4: introduce ... |
138 139 |
/** * walk_page_range - walk a memory map's page tables with a callback |
7682486b3 mm: fix various k... |
140 141 142 |
* @addr: starting address * @end: ending address * @walk: set of callbacks to invoke for each level of the tree |
e6473092b maps4: introduce ... |
143 144 145 146 147 148 |
* * Recursively walk the page table for the memory area in a VMA, * calling supplied callbacks. Callbacks are called in-order (first * PGD, first PUD, first PMD, first PTE, second PTE... second PMD, * etc.). If lower-level callbacks are omitted, walking depth is reduced. * |
2165009bd pagemap: pass mm ... |
149 150 151 |
* Each callback receives an entry pointer and the start and end of the * associated range, and a copy of the original mm_walk for access to * the ->private or ->mm fields. |
e6473092b maps4: introduce ... |
152 |
* |
dd78553b5 pagewalk: fix cod... |
153 154 |
* Usually no locks are taken, but splitting transparent huge page may * take page table lock. And the bottom level iterator will map PTE |
e6473092b maps4: introduce ... |
155 156 157 158 |
* directories from highmem if necessary. * * If any callback returns a non-zero value, the walk is aborted and * the return value is propagated back to the caller. Otherwise 0 is returned. |
c27fe4c89 pagewalk: add loc... |
159 160 161 |
* * walk->mm->mmap_sem must be held for at least read if walk->hugetlb_entry * is !NULL. |
e6473092b maps4: introduce ... |
162 |
*/ |
2165009bd pagemap: pass mm ... |
163 164 |
int walk_page_range(unsigned long addr, unsigned long end, struct mm_walk *walk) |
e6473092b maps4: introduce ... |
165 166 167 168 169 170 171 |
{ pgd_t *pgd; unsigned long next; int err = 0; if (addr >= end) return err; |
2165009bd pagemap: pass mm ... |
172 173 |
if (!walk->mm) return -EINVAL; |
96dad67ff mm: use VM_BUG_ON... |
174 |
VM_BUG_ON_MM(!rwsem_is_locked(&walk->mm->mmap_sem), walk->mm); |
a9ff785e4 mm/pagewalk.c: wa... |
175 |
|
2165009bd pagemap: pass mm ... |
176 |
pgd = pgd_offset(walk->mm, addr); |
e6473092b maps4: introduce ... |
177 |
do { |
a9ff785e4 mm/pagewalk.c: wa... |
178 |
struct vm_area_struct *vma = NULL; |
5f0af70a2 mm: remove call t... |
179 |
|
e6473092b maps4: introduce ... |
180 |
next = pgd_addr_end(addr, end); |
d33b9f45b mm: hugetlb: fix ... |
181 |
|
5dc37642c mm hugetlb: add h... |
182 |
/* |
a9ff785e4 mm/pagewalk.c: wa... |
183 184 185 186 |
* This function was not intended to be vma based. * But there are vma special cases to be handled: * - hugetlb vma's * - VM_PFNMAP vma's |
5dc37642c mm hugetlb: add h... |
187 |
*/ |
a9ff785e4 mm/pagewalk.c: wa... |
188 |
vma = find_vma(walk->mm, addr); |
6c6d52804 pagewalk: don't l... |
189 |
if (vma) { |
a9ff785e4 mm/pagewalk.c: wa... |
190 191 192 193 194 195 |
/* * There are no page structures backing a VM_PFNMAP * range, so do not allow split_huge_page_pmd(). */ if ((vma->vm_start <= addr) && (vma->vm_flags & VM_PFNMAP)) { |
d33b9f45b mm: hugetlb: fix ... |
196 |
next = vma->vm_end; |
a9ff785e4 mm/pagewalk.c: wa... |
197 198 199 |
pgd = pgd_offset(walk->mm, next); continue; } |
116354d17 pagemap: fix pfn ... |
200 |
/* |
a9ff785e4 mm/pagewalk.c: wa... |
201 202 203 204 |
* Handle hugetlb vma individually because pagetable * walk for the hugetlb page is dependent on the * architecture and we can't handled it in the same * manner as non-huge pages. |
116354d17 pagemap: fix pfn ... |
205 |
*/ |
a9ff785e4 mm/pagewalk.c: wa... |
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 |
if (walk->hugetlb_entry && (vma->vm_start <= addr) && is_vm_hugetlb_page(vma)) { if (vma->vm_end < next) next = vma->vm_end; /* * Hugepage is very tightly coupled with vma, * so walk through hugetlb entries within a * given vma. */ err = walk_hugetlb_range(vma, addr, next, walk); if (err) break; pgd = pgd_offset(walk->mm, next); continue; } |
d33b9f45b mm: hugetlb: fix ... |
221 |
} |
6c6d52804 pagewalk: don't l... |
222 |
|
e6473092b maps4: introduce ... |
223 224 |
if (pgd_none_or_clear_bad(pgd)) { if (walk->pte_hole) |
2165009bd pagemap: pass mm ... |
225 |
err = walk->pte_hole(addr, next, walk); |
e6473092b maps4: introduce ... |
226 227 |
if (err) break; |
d33b9f45b mm: hugetlb: fix ... |
228 |
pgd++; |
e6473092b maps4: introduce ... |
229 230 231 |
continue; } if (walk->pgd_entry) |
2165009bd pagemap: pass mm ... |
232 |
err = walk->pgd_entry(pgd, addr, next, walk); |
e6473092b maps4: introduce ... |
233 234 |
if (!err && (walk->pud_entry || walk->pmd_entry || walk->pte_entry)) |
2165009bd pagemap: pass mm ... |
235 |
err = walk_pud_range(pgd, addr, next, walk); |
e6473092b maps4: introduce ... |
236 237 |
if (err) break; |
d33b9f45b mm: hugetlb: fix ... |
238 |
pgd++; |
3017f079e mm/pagewalk.c: fi... |
239 |
} while (addr = next, addr < end); |
e6473092b maps4: introduce ... |
240 241 242 |
return err; } |