Commit e48293fd75b3aa67f43ad6e3d2ff397caa55d58b
Committed by
Linus Torvalds
Linus Torvalds
1 parent
25ef0e50cc
Exists in
master
and in
4 other branches
mincore: do nested page table walks
Do page table walks with the well-known nested loops we use in several other places already. This avoids doing full page table walks after every pte range and also allows to handle unmapped areas bigger than one pte range in one go. Signed-off-by: Johannes Weiner <hannes@cmpxchg.org> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Showing 1 changed file with 58 additions and 17 deletions Side-by-side Diff
mm/mincore.c
| ... | ... | @@ -144,6 +144,60 @@ |
| 144 | 144 | pte_unmap_unlock(ptep - 1, ptl); |
| 145 | 145 | } |
| 146 | 146 | |
| 147 | +static void mincore_pmd_range(struct vm_area_struct *vma, pud_t *pud, | |
| 148 | + unsigned long addr, unsigned long end, | |
| 149 | + unsigned char *vec) | |
| 150 | +{ | |
| 151 | + unsigned long next; | |
| 152 | + pmd_t *pmd; | |
| 153 | + | |
| 154 | + pmd = pmd_offset(pud, addr); | |
| 155 | + do { | |
| 156 | + next = pmd_addr_end(addr, end); | |
| 157 | + if (pmd_none_or_clear_bad(pmd)) | |
| 158 | + mincore_unmapped_range(vma, addr, next, vec); | |
| 159 | + else | |
| 160 | + mincore_pte_range(vma, pmd, addr, next, vec); | |
| 161 | + vec += (next - addr) >> PAGE_SHIFT; | |
| 162 | + } while (pmd++, addr = next, addr != end); | |
| 163 | +} | |
| 164 | + | |
| 165 | +static void mincore_pud_range(struct vm_area_struct *vma, pgd_t *pgd, | |
| 166 | + unsigned long addr, unsigned long end, | |
| 167 | + unsigned char *vec) | |
| 168 | +{ | |
| 169 | + unsigned long next; | |
| 170 | + pud_t *pud; | |
| 171 | + | |
| 172 | + pud = pud_offset(pgd, addr); | |
| 173 | + do { | |
| 174 | + next = pud_addr_end(addr, end); | |
| 175 | + if (pud_none_or_clear_bad(pud)) | |
| 176 | + mincore_unmapped_range(vma, addr, next, vec); | |
| 177 | + else | |
| 178 | + mincore_pmd_range(vma, pud, addr, next, vec); | |
| 179 | + vec += (next - addr) >> PAGE_SHIFT; | |
| 180 | + } while (pud++, addr = next, addr != end); | |
| 181 | +} | |
| 182 | + | |
| 183 | +static void mincore_page_range(struct vm_area_struct *vma, | |
| 184 | + unsigned long addr, unsigned long end, | |
| 185 | + unsigned char *vec) | |
| 186 | +{ | |
| 187 | + unsigned long next; | |
| 188 | + pgd_t *pgd; | |
| 189 | + | |
| 190 | + pgd = pgd_offset(vma->vm_mm, addr); | |
| 191 | + do { | |
| 192 | + next = pgd_addr_end(addr, end); | |
| 193 | + if (pgd_none_or_clear_bad(pgd)) | |
| 194 | + mincore_unmapped_range(vma, addr, next, vec); | |
| 195 | + else | |
| 196 | + mincore_pud_range(vma, pgd, addr, next, vec); | |
| 197 | + vec += (next - addr) >> PAGE_SHIFT; | |
| 198 | + } while (pgd++, addr = next, addr != end); | |
| 199 | +} | |
| 200 | + | |
| 147 | 201 | /* |
| 148 | 202 | * Do a chunk of "sys_mincore()". We've already checked |
| 149 | 203 | * all the arguments, we hold the mmap semaphore: we should |
| ... | ... | @@ -151,9 +205,6 @@ |
| 151 | 205 | */ |
| 152 | 206 | static long do_mincore(unsigned long addr, unsigned long pages, unsigned char *vec) |
| 153 | 207 | { |
| 154 | - pgd_t *pgd; | |
| 155 | - pud_t *pud; | |
| 156 | - pmd_t *pmd; | |
| 157 | 208 | struct vm_area_struct *vma; |
| 158 | 209 | unsigned long end; |
| 159 | 210 | |
| 160 | 211 | |
| ... | ... | @@ -170,21 +221,11 @@ |
| 170 | 221 | |
| 171 | 222 | end = pmd_addr_end(addr, end); |
| 172 | 223 | |
| 173 | - pgd = pgd_offset(vma->vm_mm, addr); | |
| 174 | - if (pgd_none_or_clear_bad(pgd)) | |
| 175 | - goto none_mapped; | |
| 176 | - pud = pud_offset(pgd, addr); | |
| 177 | - if (pud_none_or_clear_bad(pud)) | |
| 178 | - goto none_mapped; | |
| 179 | - pmd = pmd_offset(pud, addr); | |
| 180 | - if (pmd_none_or_clear_bad(pmd)) | |
| 181 | - goto none_mapped; | |
| 224 | + if (is_vm_hugetlb_page(vma)) | |
| 225 | + mincore_hugetlb_page_range(vma, addr, end, vec); | |
| 226 | + else | |
| 227 | + mincore_page_range(vma, addr, end, vec); | |
| 182 | 228 | |
| 183 | - mincore_pte_range(vma, pmd, addr, end, vec); | |
| 184 | - return (end - addr) >> PAGE_SHIFT; | |
| 185 | - | |
| 186 | -none_mapped: | |
| 187 | - mincore_unmapped_range(vma, addr, end, vec); | |
| 188 | 229 | return (end - addr) >> PAGE_SHIFT; |
| 189 | 230 | } |
| 190 | 231 |