Blame view
mm/mprotect.c
17.1 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
1da177e4c Linux-2.6.12-rc2 |
2 3 4 5 6 7 |
/* * mm/mprotect.c * * (C) Copyright 1994 Linus Torvalds * (C) Copyright 2002 Christoph Hellwig * |
046c68842 mm: update my add... |
8 |
* Address space accounting code <alan@lxorguk.ukuu.org.uk> |
1da177e4c Linux-2.6.12-rc2 |
9 10 |
* (C) Copyright 2002 Red Hat Inc, All Rights Reserved */ |
a520110e4 mm: split out a n... |
11 |
#include <linux/pagewalk.h> |
1da177e4c Linux-2.6.12-rc2 |
12 |
#include <linux/hugetlb.h> |
1da177e4c Linux-2.6.12-rc2 |
13 14 15 16 17 18 19 20 |
#include <linux/shm.h> #include <linux/mman.h> #include <linux/fs.h> #include <linux/highmem.h> #include <linux/security.h> #include <linux/mempolicy.h> #include <linux/personality.h> #include <linux/syscalls.h> |
0697212a4 [PATCH] Swapless ... |
21 22 |
#include <linux/swap.h> #include <linux/swapops.h> |
cddb8a5c1 mmu-notifiers: core |
23 |
#include <linux/mmu_notifier.h> |
64cdd548f mm: cleanup: remo... |
24 |
#include <linux/migrate.h> |
cdd6c482c perf: Do the big ... |
25 |
#include <linux/perf_event.h> |
e8c24d3a2 x86/pkeys: Alloca... |
26 |
#include <linux/pkeys.h> |
64a9a34e2 mm: numa: do not ... |
27 |
#include <linux/ksm.h> |
7c0f6ba68 Replace <asm/uacc... |
28 |
#include <linux/uaccess.h> |
09a913a7a sched/numa: avoid... |
29 |
#include <linux/mm_inline.h> |
ca5999fde mm: introduce inc... |
30 |
#include <linux/pgtable.h> |
1da177e4c Linux-2.6.12-rc2 |
31 |
#include <asm/cacheflush.h> |
e8c24d3a2 x86/pkeys: Alloca... |
32 |
#include <asm/mmu_context.h> |
1da177e4c Linux-2.6.12-rc2 |
33 |
#include <asm/tlbflush.h> |
36f881883 mm: fix mprotect(... |
34 |
#include "internal.h" |
4b10e7d56 mm: mempolicy: Im... |
35 |
static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, |
c1e6098b2 [PATCH] mm: optim... |
36 |
unsigned long addr, unsigned long end, pgprot_t newprot, |
58705444c mm: merge paramet... |
37 |
unsigned long cp_flags) |
1da177e4c Linux-2.6.12-rc2 |
38 |
{ |
0697212a4 [PATCH] Swapless ... |
39 |
pte_t *pte, oldpte; |
705e87c0c [PATCH] mm: pte_o... |
40 |
spinlock_t *ptl; |
7da4d641c mm: Count the num... |
41 |
unsigned long pages = 0; |
3e3215876 mm/mprotect.c: do... |
42 |
int target_node = NUMA_NO_NODE; |
58705444c mm: merge paramet... |
43 44 |
bool dirty_accountable = cp_flags & MM_CP_DIRTY_ACCT; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; |
292924b26 userfaultfd: wp: ... |
45 46 |
bool uffd_wp = cp_flags & MM_CP_UFFD_WP; bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; |
1da177e4c Linux-2.6.12-rc2 |
47 |
|
175ad4f1e mm: mprotect: use... |
48 |
/* |
c1e8d7c6a mmap locking API:... |
49 |
* Can be called with only the mmap_lock for reading by |
175ad4f1e mm: mprotect: use... |
50 51 52 53 54 55 56 57 58 |
* prot_numa so we must check the pmd isn't constantly * changing from under us from pmd_none to pmd_trans_huge * and/or the other way around. */ if (pmd_trans_unstable(pmd)) return 0; /* * The pmd points to a regular pte so the pmd can't change |
c1e8d7c6a mmap locking API:... |
59 |
* from under us even if the mmap_lock is only hold for |
175ad4f1e mm: mprotect: use... |
60 61 62 |
* reading. */ pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); |
1ad9f620c mm: numa: recheck... |
63 |
|
3e3215876 mm/mprotect.c: do... |
64 65 66 67 |
/* Get target node for single threaded private VMAs */ if (prot_numa && !(vma->vm_flags & VM_SHARED) && atomic_read(&vma->vm_mm->mm_users) == 1) target_node = numa_node_id(); |
3ea277194 mm, mprotect: flu... |
68 |
flush_tlb_batched_pending(vma->vm_mm); |
6606c3e0d [PATCH] paravirt:... |
69 |
arch_enter_lazy_mmu_mode(); |
1da177e4c Linux-2.6.12-rc2 |
70 |
do { |
0697212a4 [PATCH] Swapless ... |
71 72 |
oldpte = *pte; if (pte_present(oldpte)) { |
1da177e4c Linux-2.6.12-rc2 |
73 |
pte_t ptent; |
b191f9b10 mm: numa: preserv... |
74 |
bool preserve_write = prot_numa && pte_write(oldpte); |
1da177e4c Linux-2.6.12-rc2 |
75 |
|
e944fd67b mm: numa: do not ... |
76 77 78 79 80 81 |
/* * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. */ if (prot_numa) { struct page *page; |
a818f5363 autonuma: reduce ... |
82 83 84 |
/* Avoid TLB flush if possible */ if (pte_protnone(oldpte)) continue; |
e944fd67b mm: numa: do not ... |
85 86 87 |
page = vm_normal_page(vma, addr, oldpte); if (!page || PageKsm(page)) continue; |
10c1045f2 mm: numa: avoid u... |
88 |
|
859d4adc3 mm: numa: do not ... |
89 90 91 92 |
/* Also skip shared copy-on-write pages */ if (is_cow_mapping(vma->vm_flags) && page_mapcount(page) != 1) continue; |
09a913a7a sched/numa: avoid... |
93 94 95 96 97 |
/* * While migration can move some dirty pages, * it cannot move them all from MIGRATE_ASYNC * context. */ |
9de4f22a6 mm: code cleanup ... |
98 |
if (page_is_file_lru(page) && PageDirty(page)) |
09a913a7a sched/numa: avoid... |
99 |
continue; |
3e3215876 mm/mprotect.c: do... |
100 101 102 103 104 105 |
/* * Don't mess with PTEs if page is already on the node * a single-threaded process is running on. */ if (target_node == page_to_nid(page)) continue; |
e944fd67b mm: numa: do not ... |
106 |
} |
04a864530 mm: update ptep_m... |
107 108 |
oldpte = ptep_modify_prot_start(vma, addr, pte); ptent = pte_modify(oldpte, newprot); |
b191f9b10 mm: numa: preserv... |
109 |
if (preserve_write) |
288bc5494 mm/autonuma: let ... |
110 |
ptent = pte_mk_savedwrite(ptent); |
4b10e7d56 mm: mempolicy: Im... |
111 |
|
292924b26 userfaultfd: wp: ... |
112 113 114 115 116 117 118 119 120 121 122 123 |
if (uffd_wp) { ptent = pte_wrprotect(ptent); ptent = pte_mkuffd_wp(ptent); } else if (uffd_wp_resolve) { /* * Leave the write bit to be handled * by PF interrupt handler, then * things like COW could be properly * handled. */ ptent = pte_clear_uffd_wp(ptent); } |
8a0516ed8 mm: convert p[te|... |
124 125 126 127 128 |
/* Avoid taking write faults for known dirty pages */ if (dirty_accountable && pte_dirty(ptent) && (pte_soft_dirty(ptent) || !(vma->vm_flags & VM_SOFTDIRTY))) { ptent = pte_mkwrite(ptent); |
4b10e7d56 mm: mempolicy: Im... |
129 |
} |
04a864530 mm: update ptep_m... |
130 |
ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent); |
8a0516ed8 mm: convert p[te|... |
131 |
pages++; |
f45ec5ff1 userfaultfd: wp: ... |
132 |
} else if (is_swap_pte(oldpte)) { |
0697212a4 [PATCH] Swapless ... |
133 |
swp_entry_t entry = pte_to_swp_entry(oldpte); |
f45ec5ff1 userfaultfd: wp: ... |
134 |
pte_t newpte; |
0697212a4 [PATCH] Swapless ... |
135 136 137 138 139 140 141 |
if (is_write_migration_entry(entry)) { /* * A protection check is difficult so * just be safe and disable write */ make_migration_entry_read(&entry); |
c3d16e165 mm: migration: do... |
142 143 144 |
newpte = swp_entry_to_pte(entry); if (pte_swp_soft_dirty(oldpte)) newpte = pte_swp_mksoft_dirty(newpte); |
f45ec5ff1 userfaultfd: wp: ... |
145 146 147 |
if (pte_swp_uffd_wp(oldpte)) newpte = pte_swp_mkuffd_wp(newpte); } else if (is_write_device_private_entry(entry)) { |
5042db43c mm/ZONE_DEVICE: n... |
148 149 150 151 152 153 |
/* * We do not preserve soft-dirtiness. See * copy_one_pte() for explanation. */ make_device_private_entry_read(&entry); newpte = swp_entry_to_pte(entry); |
f45ec5ff1 userfaultfd: wp: ... |
154 155 156 157 158 |
if (pte_swp_uffd_wp(oldpte)) newpte = pte_swp_mkuffd_wp(newpte); } else { newpte = oldpte; } |
5042db43c mm/ZONE_DEVICE: n... |
159 |
|
f45ec5ff1 userfaultfd: wp: ... |
160 161 162 163 164 165 166 |
if (uffd_wp) newpte = pte_swp_mkuffd_wp(newpte); else if (uffd_wp_resolve) newpte = pte_swp_clear_uffd_wp(newpte); if (!pte_same(oldpte, newpte)) { set_pte_at(vma->vm_mm, addr, pte, newpte); |
5042db43c mm/ZONE_DEVICE: n... |
167 168 |
pages++; } |
1da177e4c Linux-2.6.12-rc2 |
169 170 |
} } while (pte++, addr += PAGE_SIZE, addr != end); |
6606c3e0d [PATCH] paravirt:... |
171 |
arch_leave_lazy_mmu_mode(); |
705e87c0c [PATCH] mm: pte_o... |
172 |
pte_unmap_unlock(pte - 1, ptl); |
7da4d641c mm: Count the num... |
173 174 |
return pages; |
1da177e4c Linux-2.6.12-rc2 |
175 |
} |
8b272b3cb mm, numa: fix bad... |
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 |
/* * Used when setting automatic NUMA hinting protection where it is * critical that a numa hinting PMD is not confused with a bad PMD. */ static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t *pmd) { pmd_t pmdval = pmd_read_atomic(pmd); /* See pmd_none_or_trans_huge_or_clear_bad for info on barrier */ #ifdef CONFIG_TRANSPARENT_HUGEPAGE barrier(); #endif if (pmd_none(pmdval)) return 1; if (pmd_trans_huge(pmdval)) return 0; if (unlikely(pmd_bad(pmdval))) { pmd_clear_bad(pmd); return 1; } return 0; } |
7d12efaea mm/mprotect.c: co... |
200 201 |
static inline unsigned long change_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long addr, unsigned long end, |
58705444c mm: merge paramet... |
202 |
pgprot_t newprot, unsigned long cp_flags) |
1da177e4c Linux-2.6.12-rc2 |
203 204 205 |
{ pmd_t *pmd; unsigned long next; |
7da4d641c mm: Count the num... |
206 |
unsigned long pages = 0; |
72403b4a0 mm: numa: return ... |
207 |
unsigned long nr_huge_updates = 0; |
ac46d4f3c mm/mmu_notifier: ... |
208 209 210 |
struct mmu_notifier_range range; range.start = 0; |
1da177e4c Linux-2.6.12-rc2 |
211 212 213 |
pmd = pmd_offset(pud, addr); do { |
25cbbef19 mm: numa: Trap pm... |
214 |
unsigned long this_pages; |
1da177e4c Linux-2.6.12-rc2 |
215 |
next = pmd_addr_end(addr, end); |
8b272b3cb mm, numa: fix bad... |
216 217 |
/* |
c1e8d7c6a mmap locking API:... |
218 |
* Automatic NUMA balancing walks the tables with mmap_lock |
8b272b3cb mm, numa: fix bad... |
219 220 221 222 223 224 225 226 |
* held for read. It's possible a parallel update to occur * between pmd_trans_huge() and a pmd_none_or_clear_bad() * check leading to a false positive and clearing. * Hence, it's necessary to atomically read the PMD value * for all the checks. */ if (!is_swap_pmd(*pmd) && !pmd_devmap(*pmd) && pmd_none_or_clear_bad_unless_trans_huge(pmd)) |
4991c09c7 mm/mprotect: add ... |
227 |
goto next; |
a5338093b mm: move mmu noti... |
228 229 |
/* invoke the mmu notifier if the pmd is populated */ |
ac46d4f3c mm/mmu_notifier: ... |
230 |
if (!range.start) { |
7269f9999 mm/mmu_notifier: ... |
231 232 233 |
mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0, vma, vma->vm_mm, addr, end); |
ac46d4f3c mm/mmu_notifier: ... |
234 |
mmu_notifier_invalidate_range_start(&range); |
a5338093b mm: move mmu noti... |
235 |
} |
84c3fc4e9 mm: thp: check pm... |
236 |
if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { |
6b9116a65 mm, dax: check fo... |
237 |
if (next - addr != HPAGE_PMD_SIZE) { |
fd60775ae mm, thp: avoid un... |
238 |
__split_huge_pmd(vma, pmd, addr, false, NULL); |
6b9116a65 mm, dax: check fo... |
239 |
} else { |
f123d74ab mm: Only flush TL... |
240 |
int nr_ptes = change_huge_pmd(vma, pmd, addr, |
58705444c mm: merge paramet... |
241 |
newprot, cp_flags); |
f123d74ab mm: Only flush TL... |
242 243 |
if (nr_ptes) { |
72403b4a0 mm: numa: return ... |
244 245 246 247 |
if (nr_ptes == HPAGE_PMD_NR) { pages += HPAGE_PMD_NR; nr_huge_updates++; } |
1ad9f620c mm: numa: recheck... |
248 249 |
/* huge pmd was handled */ |
4991c09c7 mm/mprotect: add ... |
250 |
goto next; |
f123d74ab mm: Only flush TL... |
251 |
} |
7da4d641c mm: Count the num... |
252 |
} |
88a9ab6e3 mm,numa: reorgani... |
253 |
/* fall through, the trans huge pmd just split */ |
cd7548ab3 thp: mprotect: tr... |
254 |
} |
25cbbef19 mm: numa: Trap pm... |
255 |
this_pages = change_pte_range(vma, pmd, addr, next, newprot, |
58705444c mm: merge paramet... |
256 |
cp_flags); |
25cbbef19 mm: numa: Trap pm... |
257 |
pages += this_pages; |
4991c09c7 mm/mprotect: add ... |
258 259 |
next: cond_resched(); |
1da177e4c Linux-2.6.12-rc2 |
260 |
} while (pmd++, addr = next, addr != end); |
7da4d641c mm: Count the num... |
261 |
|
ac46d4f3c mm/mmu_notifier: ... |
262 263 |
if (range.start) mmu_notifier_invalidate_range_end(&range); |
a5338093b mm: move mmu noti... |
264 |
|
72403b4a0 mm: numa: return ... |
265 266 |
if (nr_huge_updates) count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); |
7da4d641c mm: Count the num... |
267 |
return pages; |
1da177e4c Linux-2.6.12-rc2 |
268 |
} |
7d12efaea mm/mprotect.c: co... |
269 |
static inline unsigned long change_pud_range(struct vm_area_struct *vma, |
c2febafc6 mm: convert gener... |
270 |
p4d_t *p4d, unsigned long addr, unsigned long end, |
58705444c mm: merge paramet... |
271 |
pgprot_t newprot, unsigned long cp_flags) |
1da177e4c Linux-2.6.12-rc2 |
272 273 274 |
{ pud_t *pud; unsigned long next; |
7da4d641c mm: Count the num... |
275 |
unsigned long pages = 0; |
1da177e4c Linux-2.6.12-rc2 |
276 |
|
c2febafc6 mm: convert gener... |
277 |
pud = pud_offset(p4d, addr); |
1da177e4c Linux-2.6.12-rc2 |
278 279 280 281 |
do { next = pud_addr_end(addr, end); if (pud_none_or_clear_bad(pud)) continue; |
7da4d641c mm: Count the num... |
282 |
pages += change_pmd_range(vma, pud, addr, next, newprot, |
58705444c mm: merge paramet... |
283 |
cp_flags); |
1da177e4c Linux-2.6.12-rc2 |
284 |
} while (pud++, addr = next, addr != end); |
7da4d641c mm: Count the num... |
285 286 |
return pages; |
1da177e4c Linux-2.6.12-rc2 |
287 |
} |
c2febafc6 mm: convert gener... |
288 289 |
static inline unsigned long change_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr, unsigned long end, |
58705444c mm: merge paramet... |
290 |
pgprot_t newprot, unsigned long cp_flags) |
c2febafc6 mm: convert gener... |
291 292 293 294 295 296 297 298 299 300 301 |
{ p4d_t *p4d; unsigned long next; unsigned long pages = 0; p4d = p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end); if (p4d_none_or_clear_bad(p4d)) continue; pages += change_pud_range(vma, p4d, addr, next, newprot, |
58705444c mm: merge paramet... |
302 |
cp_flags); |
c2febafc6 mm: convert gener... |
303 304 305 306 |
} while (p4d++, addr = next, addr != end); return pages; } |
7da4d641c mm: Count the num... |
307 |
static unsigned long change_protection_range(struct vm_area_struct *vma, |
c1e6098b2 [PATCH] mm: optim... |
308 |
unsigned long addr, unsigned long end, pgprot_t newprot, |
58705444c mm: merge paramet... |
309 |
unsigned long cp_flags) |
1da177e4c Linux-2.6.12-rc2 |
310 311 312 313 314 |
{ struct mm_struct *mm = vma->vm_mm; pgd_t *pgd; unsigned long next; unsigned long start = addr; |
7da4d641c mm: Count the num... |
315 |
unsigned long pages = 0; |
1da177e4c Linux-2.6.12-rc2 |
316 317 318 319 |
BUG_ON(addr >= end); pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); |
16af97dc5 mm: migrate: prev... |
320 |
inc_tlb_flush_pending(mm); |
1da177e4c Linux-2.6.12-rc2 |
321 322 323 324 |
do { next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(pgd)) continue; |
c2febafc6 mm: convert gener... |
325 |
pages += change_p4d_range(vma, pgd, addr, next, newprot, |
58705444c mm: merge paramet... |
326 |
cp_flags); |
1da177e4c Linux-2.6.12-rc2 |
327 |
} while (pgd++, addr = next, addr != end); |
7da4d641c mm: Count the num... |
328 |
|
1233d5882 mm: Optimize the ... |
329 330 331 |
/* Only flush the TLB if we actually modified any entries: */ if (pages) flush_tlb_range(vma, start, end); |
16af97dc5 mm: migrate: prev... |
332 |
dec_tlb_flush_pending(mm); |
7da4d641c mm: Count the num... |
333 334 335 336 337 338 |
return pages; } unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, unsigned long end, pgprot_t newprot, |
58705444c mm: merge paramet... |
339 |
unsigned long cp_flags) |
7da4d641c mm: Count the num... |
340 |
{ |
7da4d641c mm: Count the num... |
341 |
unsigned long pages; |
292924b26 userfaultfd: wp: ... |
342 |
BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL); |
7da4d641c mm: Count the num... |
343 344 345 |
if (is_vm_hugetlb_page(vma)) pages = hugetlb_change_protection(vma, start, end, newprot); else |
58705444c mm: merge paramet... |
346 347 |
pages = change_protection_range(vma, start, end, newprot, cp_flags); |
7da4d641c mm: Count the num... |
348 349 |
return pages; |
1da177e4c Linux-2.6.12-rc2 |
350 |
} |
42e4089c7 x86/speculation/l... |
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 |
static int prot_none_pte_entry(pte_t *pte, unsigned long addr, unsigned long next, struct mm_walk *walk) { return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ? 0 : -EACCES; } static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long next, struct mm_walk *walk) { return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ? 0 : -EACCES; } static int prot_none_test(unsigned long addr, unsigned long next, struct mm_walk *walk) { return 0; } |
7b86ac337 pagewalk: separat... |
371 372 373 374 375 |
static const struct mm_walk_ops prot_none_walk_ops = { .pte_entry = prot_none_pte_entry, .hugetlb_entry = prot_none_hugetlb_entry, .test_walk = prot_none_test, }; |
42e4089c7 x86/speculation/l... |
376 |
|
b6a2fea39 mm: variable leng... |
377 |
int |
1da177e4c Linux-2.6.12-rc2 |
378 379 380 381 382 383 384 |
mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, unsigned long start, unsigned long end, unsigned long newflags) { struct mm_struct *mm = vma->vm_mm; unsigned long oldflags = vma->vm_flags; long nrpages = (end - start) >> PAGE_SHIFT; unsigned long charged = 0; |
1da177e4c Linux-2.6.12-rc2 |
385 386 |
pgoff_t pgoff; int error; |
c1e6098b2 [PATCH] mm: optim... |
387 |
int dirty_accountable = 0; |
1da177e4c Linux-2.6.12-rc2 |
388 389 390 391 392 393 394 |
if (newflags == oldflags) { *pprev = vma; return 0; } /* |
42e4089c7 x86/speculation/l... |
395 396 397 398 399 400 |
* Do PROT_NONE PFN permission checks here when we can still * bail out without undoing a lot of state. This is a rather * uncommon case, so doesn't need to be very optimized. */ if (arch_has_pfn_modify_check() && (vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && |
6cb4d9a28 mm/vma: introduce... |
401 |
(newflags & VM_ACCESS_FLAGS) == 0) { |
7b86ac337 pagewalk: separat... |
402 403 404 405 |
pgprot_t new_pgprot = vm_get_page_prot(newflags); error = walk_page_range(current->mm, start, end, &prot_none_walk_ops, &new_pgprot); |
42e4089c7 x86/speculation/l... |
406 407 408 409 410 |
if (error) return error; } /* |
1da177e4c Linux-2.6.12-rc2 |
411 412 |
* If we make a private mapping writable we increase our commit; * but (without finer accounting) cannot reduce our commit if we |
5a6fe1259 Do not account fo... |
413 414 |
* make it unwritable again. hugetlb mapping were accounted for * even if read-only so there is no need to account for them here |
1da177e4c Linux-2.6.12-rc2 |
415 416 |
*/ if (newflags & VM_WRITE) { |
846383359 mm: rework virtua... |
417 418 419 420 |
/* Check space limits when area turns into data. */ if (!may_expand_vm(mm, newflags, nrpages) && may_expand_vm(mm, oldflags, nrpages)) return -ENOMEM; |
5a6fe1259 Do not account fo... |
421 |
if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| |
cdfd4325c mm: record MAP_NO... |
422 |
VM_SHARED|VM_NORESERVE))) { |
1da177e4c Linux-2.6.12-rc2 |
423 |
charged = nrpages; |
191c54244 mm: collapse secu... |
424 |
if (security_vm_enough_memory_mm(mm, charged)) |
1da177e4c Linux-2.6.12-rc2 |
425 426 427 428 |
return -ENOMEM; newflags |= VM_ACCOUNT; } } |
1da177e4c Linux-2.6.12-rc2 |
429 430 431 432 433 |
/* * First try to merge with previous and/or next vma. */ pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); *pprev = vma_merge(mm, *pprev, start, end, newflags, |
19a809afe userfaultfd: teac... |
434 435 |
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), vma->vm_userfaultfd_ctx); |
1da177e4c Linux-2.6.12-rc2 |
436 437 |
if (*pprev) { vma = *pprev; |
e86f15ee6 mm: vma_merge: fi... |
438 |
VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY); |
1da177e4c Linux-2.6.12-rc2 |
439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 |
goto success; } *pprev = vma; if (start != vma->vm_start) { error = split_vma(mm, vma, start, 1); if (error) goto fail; } if (end != vma->vm_end) { error = split_vma(mm, vma, end, 0); if (error) goto fail; } success: /* |
c1e8d7c6a mmap locking API:... |
458 |
* vm_flags and vm_page_prot are protected by the mmap_lock |
1da177e4c Linux-2.6.12-rc2 |
459 460 461 |
* held in write mode. */ vma->vm_flags = newflags; |
6d2329f88 mm: vm_page_prot:... |
462 |
dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot); |
64e455079 mm: softdirty: en... |
463 |
vma_set_page_prot(vma); |
d08b3851d [PATCH] mm: track... |
464 |
|
7d12efaea mm/mprotect.c: co... |
465 |
change_protection(vma, start, end, vma->vm_page_prot, |
58705444c mm: merge paramet... |
466 |
dirty_accountable ? MM_CP_DIRTY_ACCT : 0); |
7da4d641c mm: Count the num... |
467 |
|
36f881883 mm: fix mprotect(... |
468 469 470 471 472 473 474 475 |
/* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major * fault on access. */ if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED && (newflags & VM_WRITE)) { populate_vma_page_range(vma, start, end, NULL); } |
846383359 mm: rework virtua... |
476 477 |
vm_stat_account(mm, oldflags, -nrpages); vm_stat_account(mm, newflags, nrpages); |
63bfd7384 perf_events: Fix ... |
478 |
perf_event_mmap(vma); |
1da177e4c Linux-2.6.12-rc2 |
479 480 481 482 483 484 |
return 0; fail: vm_unacct_memory(charged); return error; } |
7d06d9c9b mm: Implement new... |
485 486 487 488 489 |
/* * pkey==-1 when doing a legacy mprotect() */ static int do_mprotect_pkey(unsigned long start, size_t len, unsigned long prot, int pkey) |
1da177e4c Linux-2.6.12-rc2 |
490 |
{ |
62b5f7d01 mm/core, x86/mm/p... |
491 |
unsigned long nstart, end, tmp, reqprot; |
1da177e4c Linux-2.6.12-rc2 |
492 493 494 |
struct vm_area_struct *vma, *prev; int error = -EINVAL; const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); |
f138556da mm/mprotect.c: do... |
495 496 |
const bool rier = (current->personality & READ_IMPLIES_EXEC) && (prot & PROT_READ); |
057d33891 mm: untag user po... |
497 |
start = untagged_addr(start); |
1da177e4c Linux-2.6.12-rc2 |
498 499 500 501 502 503 504 505 506 507 508 509 |
prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ return -EINVAL; if (start & ~PAGE_MASK) return -EINVAL; if (!len) return 0; len = PAGE_ALIGN(len); end = start + len; if (end <= start) return -ENOMEM; |
9035cf9a9 mm: Add address p... |
510 |
if (!arch_validate_prot(prot, start)) |
1da177e4c Linux-2.6.12-rc2 |
511 512 513 |
return -EINVAL; reqprot = prot; |
1da177e4c Linux-2.6.12-rc2 |
514 |
|
d8ed45c5d mmap locking API:... |
515 |
if (mmap_write_lock_killable(current->mm)) |
dc0ef0df7 mm: make mmap_sem... |
516 |
return -EINTR; |
1da177e4c Linux-2.6.12-rc2 |
517 |
|
e8c24d3a2 x86/pkeys: Alloca... |
518 519 520 521 522 523 524 |
/* * If userspace did not allocate the pkey, do not let * them use it here. */ error = -EINVAL; if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) goto out; |
097d59106 vm: avoid using f... |
525 |
vma = find_vma(current->mm, start); |
1da177e4c Linux-2.6.12-rc2 |
526 527 528 |
error = -ENOMEM; if (!vma) goto out; |
097d59106 vm: avoid using f... |
529 |
prev = vma->vm_prev; |
1da177e4c Linux-2.6.12-rc2 |
530 531 532 533 534 535 536 |
if (unlikely(grows & PROT_GROWSDOWN)) { if (vma->vm_start >= end) goto out; start = vma->vm_start; error = -EINVAL; if (!(vma->vm_flags & VM_GROWSDOWN)) goto out; |
7d12efaea mm/mprotect.c: co... |
537 |
} else { |
1da177e4c Linux-2.6.12-rc2 |
538 539 540 541 542 543 544 545 546 547 548 549 550 |
if (vma->vm_start > start) goto out; if (unlikely(grows & PROT_GROWSUP)) { end = vma->vm_end; error = -EINVAL; if (!(vma->vm_flags & VM_GROWSUP)) goto out; } } if (start > vma->vm_start) prev = vma; for (nstart = start ; ; ) { |
a8502b67d x86/pkeys: Make m... |
551 |
unsigned long mask_off_old_flags; |
1da177e4c Linux-2.6.12-rc2 |
552 |
unsigned long newflags; |
7d06d9c9b mm: Implement new... |
553 |
int new_vma_pkey; |
1da177e4c Linux-2.6.12-rc2 |
554 |
|
7d12efaea mm/mprotect.c: co... |
555 |
/* Here we know that vma->vm_start <= nstart < vma->vm_end. */ |
1da177e4c Linux-2.6.12-rc2 |
556 |
|
f138556da mm/mprotect.c: do... |
557 558 559 |
/* Does the application expect PROT_READ to imply PROT_EXEC */ if (rier && (vma->vm_flags & VM_MAYEXEC)) prot |= PROT_EXEC; |
a8502b67d x86/pkeys: Make m... |
560 561 562 563 564 565 |
/* * Each mprotect() call explicitly passes r/w/x permissions. * If a permission is not passed to mprotect(), it must be * cleared from the VMA. */ mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC | |
2c2d57b5e mm: Clear arch sp... |
566 |
VM_FLAGS_CLEAR; |
a8502b67d x86/pkeys: Make m... |
567 |
|
7d06d9c9b mm: Implement new... |
568 569 |
new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey); newflags = calc_vm_prot_bits(prot, new_vma_pkey); |
a8502b67d x86/pkeys: Make m... |
570 |
newflags |= (vma->vm_flags & ~mask_off_old_flags); |
1da177e4c Linux-2.6.12-rc2 |
571 |
|
7e2cff42c [PATCH] mm: add a... |
572 |
/* newflags >> 4 shift VM_MAY% in place of VM_% */ |
6cb4d9a28 mm/vma: introduce... |
573 |
if ((newflags & ~(newflags >> 4)) & VM_ACCESS_FLAGS) { |
1da177e4c Linux-2.6.12-rc2 |
574 575 576 |
error = -EACCES; goto out; } |
c462ac288 mm: Introduce arc... |
577 578 579 580 581 |
/* Allow architectures to sanity-check the new flags */ if (!arch_validate_flags(newflags)) { error = -EINVAL; goto out; } |
1da177e4c Linux-2.6.12-rc2 |
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 |
error = security_file_mprotect(vma, reqprot, prot); if (error) goto out; tmp = vma->vm_end; if (tmp > end) tmp = end; error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); if (error) goto out; nstart = tmp; if (nstart < prev->vm_end) nstart = prev->vm_end; if (nstart >= end) goto out; vma = prev->vm_next; if (!vma || vma->vm_start != nstart) { error = -ENOMEM; goto out; } |
f138556da mm/mprotect.c: do... |
604 |
prot = reqprot; |
1da177e4c Linux-2.6.12-rc2 |
605 606 |
} out: |
d8ed45c5d mmap locking API:... |
607 |
mmap_write_unlock(current->mm); |
1da177e4c Linux-2.6.12-rc2 |
608 609 |
return error; } |
7d06d9c9b mm: Implement new... |
610 611 612 613 614 615 |
SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, unsigned long, prot) { return do_mprotect_pkey(start, len, prot, -1); } |
c7142aead mm/pkeys: generat... |
616 |
#ifdef CONFIG_ARCH_HAS_PKEYS |
7d06d9c9b mm: Implement new... |
617 618 619 620 621 |
SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len, unsigned long, prot, int, pkey) { return do_mprotect_pkey(start, len, prot, pkey); } |
e8c24d3a2 x86/pkeys: Alloca... |
622 623 624 625 626 627 628 629 630 631 632 633 |
SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val) { int pkey; int ret; /* No flags supported yet. */ if (flags) return -EINVAL; /* check for unsupported init values */ if (init_val & ~PKEY_ACCESS_MASK) return -EINVAL; |
d8ed45c5d mmap locking API:... |
634 |
mmap_write_lock(current->mm); |
e8c24d3a2 x86/pkeys: Alloca... |
635 636 637 638 639 640 641 642 643 644 645 646 647 |
pkey = mm_pkey_alloc(current->mm); ret = -ENOSPC; if (pkey == -1) goto out; ret = arch_set_user_pkey_access(current, pkey, init_val); if (ret) { mm_pkey_free(current->mm, pkey); goto out; } ret = pkey; out: |
d8ed45c5d mmap locking API:... |
648 |
mmap_write_unlock(current->mm); |
e8c24d3a2 x86/pkeys: Alloca... |
649 650 651 652 653 654 |
return ret; } SYSCALL_DEFINE1(pkey_free, int, pkey) { int ret; |
d8ed45c5d mmap locking API:... |
655 |
mmap_write_lock(current->mm); |
e8c24d3a2 x86/pkeys: Alloca... |
656 |
ret = mm_pkey_free(current->mm, pkey); |
d8ed45c5d mmap locking API:... |
657 |
mmap_write_unlock(current->mm); |
e8c24d3a2 x86/pkeys: Alloca... |
658 659 660 661 662 663 664 |
/* * We could provie warnings or errors if any VMA still * has the pkey set here. */ return ret; } |
c7142aead mm/pkeys: generat... |
665 666 |
#endif /* CONFIG_ARCH_HAS_PKEYS */ |