Blame view
mm/mprotect.c
17.1 KB
b24413180
|
1 |
// SPDX-License-Identifier: GPL-2.0 |
1da177e4c
|
2 3 4 5 6 7 |
/* * mm/mprotect.c * * (C) Copyright 1994 Linus Torvalds * (C) Copyright 2002 Christoph Hellwig * |
046c68842
|
8 |
* Address space accounting code <alan@lxorguk.ukuu.org.uk> |
1da177e4c
|
9 10 |
* (C) Copyright 2002 Red Hat Inc, All Rights Reserved */ |
a520110e4
|
11 |
#include <linux/pagewalk.h> |
1da177e4c
|
12 |
#include <linux/hugetlb.h> |
1da177e4c
|
13 14 15 16 17 18 19 20 |
#include <linux/shm.h> #include <linux/mman.h> #include <linux/fs.h> #include <linux/highmem.h> #include <linux/security.h> #include <linux/mempolicy.h> #include <linux/personality.h> #include <linux/syscalls.h> |
0697212a4
|
21 22 |
#include <linux/swap.h> #include <linux/swapops.h> |
cddb8a5c1
|
23 |
#include <linux/mmu_notifier.h> |
64cdd548f
|
24 |
#include <linux/migrate.h> |
cdd6c482c
|
25 |
#include <linux/perf_event.h> |
e8c24d3a2
|
26 |
#include <linux/pkeys.h> |
64a9a34e2
|
27 |
#include <linux/ksm.h> |
7c0f6ba68
|
28 |
#include <linux/uaccess.h> |
09a913a7a
|
29 |
#include <linux/mm_inline.h> |
ca5999fde
|
30 |
#include <linux/pgtable.h> |
1da177e4c
|
31 |
#include <asm/cacheflush.h> |
e8c24d3a2
|
32 |
#include <asm/mmu_context.h> |
1da177e4c
|
33 |
#include <asm/tlbflush.h> |
36f881883
|
34 |
#include "internal.h" |
4b10e7d56
|
35 |
static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, |
c1e6098b2
|
36 |
unsigned long addr, unsigned long end, pgprot_t newprot, |
58705444c
|
37 |
unsigned long cp_flags) |
1da177e4c
|
38 |
{ |
0697212a4
|
39 |
pte_t *pte, oldpte; |
705e87c0c
|
40 |
spinlock_t *ptl; |
7da4d641c
|
41 |
unsigned long pages = 0; |
3e3215876
|
42 |
int target_node = NUMA_NO_NODE; |
58705444c
|
43 44 |
bool dirty_accountable = cp_flags & MM_CP_DIRTY_ACCT; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; |
292924b26
|
45 46 |
bool uffd_wp = cp_flags & MM_CP_UFFD_WP; bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; |
1da177e4c
|
47 |
|
175ad4f1e
|
48 |
/* |
c1e8d7c6a
|
49 |
* Can be called with only the mmap_lock for reading by |
175ad4f1e
|
50 51 52 53 54 55 56 57 58 |
* prot_numa so we must check the pmd isn't constantly * changing from under us from pmd_none to pmd_trans_huge * and/or the other way around. */ if (pmd_trans_unstable(pmd)) return 0; /* * The pmd points to a regular pte so the pmd can't change |
c1e8d7c6a
|
59 |
* from under us even if the mmap_lock is only hold for |
175ad4f1e
|
60 61 62 |
* reading. */ pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); |
1ad9f620c
|
63 |
|
3e3215876
|
64 65 66 67 |
/* Get target node for single threaded private VMAs */ if (prot_numa && !(vma->vm_flags & VM_SHARED) && atomic_read(&vma->vm_mm->mm_users) == 1) target_node = numa_node_id(); |
3ea277194
|
68 |
flush_tlb_batched_pending(vma->vm_mm); |
6606c3e0d
|
69 |
arch_enter_lazy_mmu_mode(); |
1da177e4c
|
70 |
do { |
0697212a4
|
71 72 |
oldpte = *pte; if (pte_present(oldpte)) { |
1da177e4c
|
73 |
pte_t ptent; |
b191f9b10
|
74 |
bool preserve_write = prot_numa && pte_write(oldpte); |
1da177e4c
|
75 |
|
e944fd67b
|
76 77 78 79 80 81 |
/* * Avoid trapping faults against the zero or KSM * pages. See similar comment in change_huge_pmd. */ if (prot_numa) { struct page *page; |
a818f5363
|
82 83 84 |
/* Avoid TLB flush if possible */ if (pte_protnone(oldpte)) continue; |
e944fd67b
|
85 86 87 |
page = vm_normal_page(vma, addr, oldpte); if (!page || PageKsm(page)) continue; |
10c1045f2
|
88 |
|
859d4adc3
|
89 90 91 92 |
/* Also skip shared copy-on-write pages */ if (is_cow_mapping(vma->vm_flags) && page_mapcount(page) != 1) continue; |
09a913a7a
|
93 94 95 96 97 |
/* * While migration can move some dirty pages, * it cannot move them all from MIGRATE_ASYNC * context. */ |
9de4f22a6
|
98 |
if (page_is_file_lru(page) && PageDirty(page)) |
09a913a7a
|
99 |
continue; |
3e3215876
|
100 101 102 103 104 105 |
/* * Don't mess with PTEs if page is already on the node * a single-threaded process is running on. */ if (target_node == page_to_nid(page)) continue; |
e944fd67b
|
106 |
} |
04a864530
|
107 108 |
oldpte = ptep_modify_prot_start(vma, addr, pte); ptent = pte_modify(oldpte, newprot); |
b191f9b10
|
109 |
if (preserve_write) |
288bc5494
|
110 |
ptent = pte_mk_savedwrite(ptent); |
4b10e7d56
|
111 |
|
292924b26
|
112 113 114 115 116 117 118 119 120 121 122 123 |
if (uffd_wp) { ptent = pte_wrprotect(ptent); ptent = pte_mkuffd_wp(ptent); } else if (uffd_wp_resolve) { /* * Leave the write bit to be handled * by PF interrupt handler, then * things like COW could be properly * handled. */ ptent = pte_clear_uffd_wp(ptent); } |
8a0516ed8
|
124 125 126 127 128 |
/* Avoid taking write faults for known dirty pages */ if (dirty_accountable && pte_dirty(ptent) && (pte_soft_dirty(ptent) || !(vma->vm_flags & VM_SOFTDIRTY))) { ptent = pte_mkwrite(ptent); |
4b10e7d56
|
129 |
} |
04a864530
|
130 |
ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent); |
8a0516ed8
|
131 |
pages++; |
f45ec5ff1
|
132 |
} else if (is_swap_pte(oldpte)) { |
0697212a4
|
133 |
swp_entry_t entry = pte_to_swp_entry(oldpte); |
f45ec5ff1
|
134 |
pte_t newpte; |
0697212a4
|
135 136 137 138 139 140 141 |
if (is_write_migration_entry(entry)) { /* * A protection check is difficult so * just be safe and disable write */ make_migration_entry_read(&entry); |
c3d16e165
|
142 143 144 |
newpte = swp_entry_to_pte(entry); if (pte_swp_soft_dirty(oldpte)) newpte = pte_swp_mksoft_dirty(newpte); |
f45ec5ff1
|
145 146 147 |
if (pte_swp_uffd_wp(oldpte)) newpte = pte_swp_mkuffd_wp(newpte); } else if (is_write_device_private_entry(entry)) { |
5042db43c
|
148 149 150 151 152 153 |
/* * We do not preserve soft-dirtiness. See * copy_one_pte() for explanation. */ make_device_private_entry_read(&entry); newpte = swp_entry_to_pte(entry); |
f45ec5ff1
|
154 155 156 157 158 |
if (pte_swp_uffd_wp(oldpte)) newpte = pte_swp_mkuffd_wp(newpte); } else { newpte = oldpte; } |
5042db43c
|
159 |
|
f45ec5ff1
|
160 161 162 163 164 165 166 |
if (uffd_wp) newpte = pte_swp_mkuffd_wp(newpte); else if (uffd_wp_resolve) newpte = pte_swp_clear_uffd_wp(newpte); if (!pte_same(oldpte, newpte)) { set_pte_at(vma->vm_mm, addr, pte, newpte); |
5042db43c
|
167 168 |
pages++; } |
1da177e4c
|
169 170 |
} } while (pte++, addr += PAGE_SIZE, addr != end); |
6606c3e0d
|
171 |
arch_leave_lazy_mmu_mode(); |
705e87c0c
|
172 |
pte_unmap_unlock(pte - 1, ptl); |
7da4d641c
|
173 174 |
return pages; |
1da177e4c
|
175 |
} |
8b272b3cb
|
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 |
/* * Used when setting automatic NUMA hinting protection where it is * critical that a numa hinting PMD is not confused with a bad PMD. */ static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t *pmd) { pmd_t pmdval = pmd_read_atomic(pmd); /* See pmd_none_or_trans_huge_or_clear_bad for info on barrier */ #ifdef CONFIG_TRANSPARENT_HUGEPAGE barrier(); #endif if (pmd_none(pmdval)) return 1; if (pmd_trans_huge(pmdval)) return 0; if (unlikely(pmd_bad(pmdval))) { pmd_clear_bad(pmd); return 1; } return 0; } |
7d12efaea
|
200 201 |
static inline unsigned long change_pmd_range(struct vm_area_struct *vma, pud_t *pud, unsigned long addr, unsigned long end, |
58705444c
|
202 |
pgprot_t newprot, unsigned long cp_flags) |
1da177e4c
|
203 204 205 |
{ pmd_t *pmd; unsigned long next; |
7da4d641c
|
206 |
unsigned long pages = 0; |
72403b4a0
|
207 |
unsigned long nr_huge_updates = 0; |
ac46d4f3c
|
208 209 210 |
struct mmu_notifier_range range; range.start = 0; |
1da177e4c
|
211 212 213 |
pmd = pmd_offset(pud, addr); do { |
25cbbef19
|
214 |
unsigned long this_pages; |
1da177e4c
|
215 |
next = pmd_addr_end(addr, end); |
8b272b3cb
|
216 217 |
/* |
c1e8d7c6a
|
218 |
* Automatic NUMA balancing walks the tables with mmap_lock |
8b272b3cb
|
219 220 221 222 223 224 225 226 |
* held for read. It's possible a parallel update to occur * between pmd_trans_huge() and a pmd_none_or_clear_bad() * check leading to a false positive and clearing. * Hence, it's necessary to atomically read the PMD value * for all the checks. */ if (!is_swap_pmd(*pmd) && !pmd_devmap(*pmd) && pmd_none_or_clear_bad_unless_trans_huge(pmd)) |
4991c09c7
|
227 |
goto next; |
a5338093b
|
228 229 |
/* invoke the mmu notifier if the pmd is populated */ |
ac46d4f3c
|
230 |
if (!range.start) { |
7269f9999
|
231 232 233 |
mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0, vma, vma->vm_mm, addr, end); |
ac46d4f3c
|
234 |
mmu_notifier_invalidate_range_start(&range); |
a5338093b
|
235 |
} |
84c3fc4e9
|
236 |
if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) { |
6b9116a65
|
237 |
if (next - addr != HPAGE_PMD_SIZE) { |
fd60775ae
|
238 |
__split_huge_pmd(vma, pmd, addr, false, NULL); |
6b9116a65
|
239 |
} else { |
f123d74ab
|
240 |
int nr_ptes = change_huge_pmd(vma, pmd, addr, |
58705444c
|
241 |
newprot, cp_flags); |
f123d74ab
|
242 243 |
if (nr_ptes) { |
72403b4a0
|
244 245 246 247 |
if (nr_ptes == HPAGE_PMD_NR) { pages += HPAGE_PMD_NR; nr_huge_updates++; } |
1ad9f620c
|
248 249 |
/* huge pmd was handled */ |
4991c09c7
|
250 |
goto next; |
f123d74ab
|
251 |
} |
7da4d641c
|
252 |
} |
88a9ab6e3
|
253 |
/* fall through, the trans huge pmd just split */ |
cd7548ab3
|
254 |
} |
25cbbef19
|
255 |
this_pages = change_pte_range(vma, pmd, addr, next, newprot, |
58705444c
|
256 |
cp_flags); |
25cbbef19
|
257 |
pages += this_pages; |
4991c09c7
|
258 259 |
next: cond_resched(); |
1da177e4c
|
260 |
} while (pmd++, addr = next, addr != end); |
7da4d641c
|
261 |
|
ac46d4f3c
|
262 263 |
if (range.start) mmu_notifier_invalidate_range_end(&range); |
a5338093b
|
264 |
|
72403b4a0
|
265 266 |
if (nr_huge_updates) count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates); |
7da4d641c
|
267 |
return pages; |
1da177e4c
|
268 |
} |
7d12efaea
|
269 |
static inline unsigned long change_pud_range(struct vm_area_struct *vma, |
c2febafc6
|
270 |
p4d_t *p4d, unsigned long addr, unsigned long end, |
58705444c
|
271 |
pgprot_t newprot, unsigned long cp_flags) |
1da177e4c
|
272 273 274 |
{ pud_t *pud; unsigned long next; |
7da4d641c
|
275 |
unsigned long pages = 0; |
1da177e4c
|
276 |
|
c2febafc6
|
277 |
pud = pud_offset(p4d, addr); |
1da177e4c
|
278 279 280 281 |
do { next = pud_addr_end(addr, end); if (pud_none_or_clear_bad(pud)) continue; |
7da4d641c
|
282 |
pages += change_pmd_range(vma, pud, addr, next, newprot, |
58705444c
|
283 |
cp_flags); |
1da177e4c
|
284 |
} while (pud++, addr = next, addr != end); |
7da4d641c
|
285 286 |
return pages; |
1da177e4c
|
287 |
} |
c2febafc6
|
288 289 |
static inline unsigned long change_p4d_range(struct vm_area_struct *vma, pgd_t *pgd, unsigned long addr, unsigned long end, |
58705444c
|
290 |
pgprot_t newprot, unsigned long cp_flags) |
c2febafc6
|
291 292 293 294 295 296 297 298 299 300 301 |
{ p4d_t *p4d; unsigned long next; unsigned long pages = 0; p4d = p4d_offset(pgd, addr); do { next = p4d_addr_end(addr, end); if (p4d_none_or_clear_bad(p4d)) continue; pages += change_pud_range(vma, p4d, addr, next, newprot, |
58705444c
|
302 |
cp_flags); |
c2febafc6
|
303 304 305 306 |
} while (p4d++, addr = next, addr != end); return pages; } |
7da4d641c
|
307 |
static unsigned long change_protection_range(struct vm_area_struct *vma, |
c1e6098b2
|
308 |
unsigned long addr, unsigned long end, pgprot_t newprot, |
58705444c
|
309 |
unsigned long cp_flags) |
1da177e4c
|
310 311 312 313 314 |
{ struct mm_struct *mm = vma->vm_mm; pgd_t *pgd; unsigned long next; unsigned long start = addr; |
7da4d641c
|
315 |
unsigned long pages = 0; |
1da177e4c
|
316 317 318 319 |
BUG_ON(addr >= end); pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); |
16af97dc5
|
320 |
inc_tlb_flush_pending(mm); |
1da177e4c
|
321 322 323 324 |
do { next = pgd_addr_end(addr, end); if (pgd_none_or_clear_bad(pgd)) continue; |
c2febafc6
|
325 |
pages += change_p4d_range(vma, pgd, addr, next, newprot, |
58705444c
|
326 |
cp_flags); |
1da177e4c
|
327 |
} while (pgd++, addr = next, addr != end); |
7da4d641c
|
328 |
|
1233d5882
|
329 330 331 |
/* Only flush the TLB if we actually modified any entries: */ if (pages) flush_tlb_range(vma, start, end); |
16af97dc5
|
332 |
dec_tlb_flush_pending(mm); |
7da4d641c
|
333 334 335 336 337 338 |
return pages; } unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, unsigned long end, pgprot_t newprot, |
58705444c
|
339 |
unsigned long cp_flags) |
7da4d641c
|
340 |
{ |
7da4d641c
|
341 |
unsigned long pages; |
292924b26
|
342 |
BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL); |
7da4d641c
|
343 344 345 |
if (is_vm_hugetlb_page(vma)) pages = hugetlb_change_protection(vma, start, end, newprot); else |
58705444c
|
346 347 |
pages = change_protection_range(vma, start, end, newprot, cp_flags); |
7da4d641c
|
348 349 |
return pages; |
1da177e4c
|
350 |
} |
42e4089c7
|
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 |
static int prot_none_pte_entry(pte_t *pte, unsigned long addr, unsigned long next, struct mm_walk *walk) { return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ? 0 : -EACCES; } static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask, unsigned long addr, unsigned long next, struct mm_walk *walk) { return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ? 0 : -EACCES; } static int prot_none_test(unsigned long addr, unsigned long next, struct mm_walk *walk) { return 0; } |
7b86ac337
|
371 372 373 374 375 |
static const struct mm_walk_ops prot_none_walk_ops = { .pte_entry = prot_none_pte_entry, .hugetlb_entry = prot_none_hugetlb_entry, .test_walk = prot_none_test, }; |
42e4089c7
|
376 |
|
b6a2fea39
|
377 |
int |
1da177e4c
|
378 379 380 381 382 383 384 |
mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, unsigned long start, unsigned long end, unsigned long newflags) { struct mm_struct *mm = vma->vm_mm; unsigned long oldflags = vma->vm_flags; long nrpages = (end - start) >> PAGE_SHIFT; unsigned long charged = 0; |
1da177e4c
|
385 386 |
pgoff_t pgoff; int error; |
c1e6098b2
|
387 |
int dirty_accountable = 0; |
1da177e4c
|
388 389 390 391 392 393 394 |
if (newflags == oldflags) { *pprev = vma; return 0; } /* |
42e4089c7
|
395 396 397 398 399 400 |
* Do PROT_NONE PFN permission checks here when we can still * bail out without undoing a lot of state. This is a rather * uncommon case, so doesn't need to be very optimized. */ if (arch_has_pfn_modify_check() && (vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) && |
6cb4d9a28
|
401 |
(newflags & VM_ACCESS_FLAGS) == 0) { |
7b86ac337
|
402 403 404 405 |
pgprot_t new_pgprot = vm_get_page_prot(newflags); error = walk_page_range(current->mm, start, end, &prot_none_walk_ops, &new_pgprot); |
42e4089c7
|
406 407 408 409 410 |
if (error) return error; } /* |
1da177e4c
|
411 412 |
* If we make a private mapping writable we increase our commit; * but (without finer accounting) cannot reduce our commit if we |
5a6fe1259
|
413 414 |
* make it unwritable again. hugetlb mapping were accounted for * even if read-only so there is no need to account for them here |
1da177e4c
|
415 416 |
*/ if (newflags & VM_WRITE) { |
846383359
|
417 418 419 420 |
/* Check space limits when area turns into data. */ if (!may_expand_vm(mm, newflags, nrpages) && may_expand_vm(mm, oldflags, nrpages)) return -ENOMEM; |
5a6fe1259
|
421 |
if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB| |
cdfd4325c
|
422 |
VM_SHARED|VM_NORESERVE))) { |
1da177e4c
|
423 |
charged = nrpages; |
191c54244
|
424 |
if (security_vm_enough_memory_mm(mm, charged)) |
1da177e4c
|
425 426 427 428 |
return -ENOMEM; newflags |= VM_ACCOUNT; } } |
1da177e4c
|
429 430 431 432 433 |
/* * First try to merge with previous and/or next vma. */ pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); *pprev = vma_merge(mm, *pprev, start, end, newflags, |
19a809afe
|
434 435 |
vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma), vma->vm_userfaultfd_ctx); |
1da177e4c
|
436 437 |
if (*pprev) { vma = *pprev; |
e86f15ee6
|
438 |
VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY); |
1da177e4c
|
439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 |
goto success; } *pprev = vma; if (start != vma->vm_start) { error = split_vma(mm, vma, start, 1); if (error) goto fail; } if (end != vma->vm_end) { error = split_vma(mm, vma, end, 0); if (error) goto fail; } success: /* |
c1e8d7c6a
|
458 |
* vm_flags and vm_page_prot are protected by the mmap_lock |
1da177e4c
|
459 460 461 |
* held in write mode. */ vma->vm_flags = newflags; |
6d2329f88
|
462 |
dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot); |
64e455079
|
463 |
vma_set_page_prot(vma); |
d08b3851d
|
464 |
|
7d12efaea
|
465 |
change_protection(vma, start, end, vma->vm_page_prot, |
58705444c
|
466 |
dirty_accountable ? MM_CP_DIRTY_ACCT : 0); |
7da4d641c
|
467 |
|
36f881883
|
468 469 470 471 472 473 474 475 |
/* * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major * fault on access. */ if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED && (newflags & VM_WRITE)) { populate_vma_page_range(vma, start, end, NULL); } |
846383359
|
476 477 |
vm_stat_account(mm, oldflags, -nrpages); vm_stat_account(mm, newflags, nrpages); |
63bfd7384
|
478 |
perf_event_mmap(vma); |
1da177e4c
|
479 480 481 482 483 484 |
return 0; fail: vm_unacct_memory(charged); return error; } |
7d06d9c9b
|
485 486 487 488 489 |
/* * pkey==-1 when doing a legacy mprotect() */ static int do_mprotect_pkey(unsigned long start, size_t len, unsigned long prot, int pkey) |
1da177e4c
|
490 |
{ |
62b5f7d01
|
491 |
unsigned long nstart, end, tmp, reqprot; |
1da177e4c
|
492 493 494 |
struct vm_area_struct *vma, *prev; int error = -EINVAL; const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP); |
f138556da
|
495 496 |
const bool rier = (current->personality & READ_IMPLIES_EXEC) && (prot & PROT_READ); |
057d33891
|
497 |
start = untagged_addr(start); |
1da177e4c
|
498 499 500 501 502 503 504 505 506 507 508 509 |
prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP); if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */ return -EINVAL; if (start & ~PAGE_MASK) return -EINVAL; if (!len) return 0; len = PAGE_ALIGN(len); end = start + len; if (end <= start) return -ENOMEM; |
9035cf9a9
|
510 |
if (!arch_validate_prot(prot, start)) |
1da177e4c
|
511 512 513 |
return -EINVAL; reqprot = prot; |
1da177e4c
|
514 |
|
d8ed45c5d
|
515 |
if (mmap_write_lock_killable(current->mm)) |
dc0ef0df7
|
516 |
return -EINTR; |
1da177e4c
|
517 |
|
e8c24d3a2
|
518 519 520 521 522 523 524 |
/* * If userspace did not allocate the pkey, do not let * them use it here. */ error = -EINVAL; if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey)) goto out; |
097d59106
|
525 |
vma = find_vma(current->mm, start); |
1da177e4c
|
526 527 528 |
error = -ENOMEM; if (!vma) goto out; |
097d59106
|
529 |
prev = vma->vm_prev; |
1da177e4c
|
530 531 532 533 534 535 536 |
if (unlikely(grows & PROT_GROWSDOWN)) { if (vma->vm_start >= end) goto out; start = vma->vm_start; error = -EINVAL; if (!(vma->vm_flags & VM_GROWSDOWN)) goto out; |
7d12efaea
|
537 |
} else { |
1da177e4c
|
538 539 540 541 542 543 544 545 546 547 548 549 550 |
if (vma->vm_start > start) goto out; if (unlikely(grows & PROT_GROWSUP)) { end = vma->vm_end; error = -EINVAL; if (!(vma->vm_flags & VM_GROWSUP)) goto out; } } if (start > vma->vm_start) prev = vma; for (nstart = start ; ; ) { |
a8502b67d
|
551 |
unsigned long mask_off_old_flags; |
1da177e4c
|
552 |
unsigned long newflags; |
7d06d9c9b
|
553 |
int new_vma_pkey; |
1da177e4c
|
554 |
|
7d12efaea
|
555 |
/* Here we know that vma->vm_start <= nstart < vma->vm_end. */ |
1da177e4c
|
556 |
|
f138556da
|
557 558 559 |
/* Does the application expect PROT_READ to imply PROT_EXEC */ if (rier && (vma->vm_flags & VM_MAYEXEC)) prot |= PROT_EXEC; |
a8502b67d
|
560 561 562 563 564 565 |
/* * Each mprotect() call explicitly passes r/w/x permissions. * If a permission is not passed to mprotect(), it must be * cleared from the VMA. */ mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC | |
2c2d57b5e
|
566 |
VM_FLAGS_CLEAR; |
a8502b67d
|
567 |
|
7d06d9c9b
|
568 569 |
new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey); newflags = calc_vm_prot_bits(prot, new_vma_pkey); |
a8502b67d
|
570 |
newflags |= (vma->vm_flags & ~mask_off_old_flags); |
1da177e4c
|
571 |
|
7e2cff42c
|
572 |
/* newflags >> 4 shift VM_MAY% in place of VM_% */ |
6cb4d9a28
|
573 |
if ((newflags & ~(newflags >> 4)) & VM_ACCESS_FLAGS) { |
1da177e4c
|
574 575 576 |
error = -EACCES; goto out; } |
c462ac288
|
577 578 579 580 581 |
/* Allow architectures to sanity-check the new flags */ if (!arch_validate_flags(newflags)) { error = -EINVAL; goto out; } |
1da177e4c
|
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 |
error = security_file_mprotect(vma, reqprot, prot); if (error) goto out; tmp = vma->vm_end; if (tmp > end) tmp = end; error = mprotect_fixup(vma, &prev, nstart, tmp, newflags); if (error) goto out; nstart = tmp; if (nstart < prev->vm_end) nstart = prev->vm_end; if (nstart >= end) goto out; vma = prev->vm_next; if (!vma || vma->vm_start != nstart) { error = -ENOMEM; goto out; } |
f138556da
|
604 |
prot = reqprot; |
1da177e4c
|
605 606 |
} out: |
d8ed45c5d
|
607 |
mmap_write_unlock(current->mm); |
1da177e4c
|
608 609 |
return error; } |
7d06d9c9b
|
610 611 612 613 614 615 |
SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len, unsigned long, prot) { return do_mprotect_pkey(start, len, prot, -1); } |
c7142aead
|
616 |
#ifdef CONFIG_ARCH_HAS_PKEYS |
7d06d9c9b
|
617 618 619 620 621 |
SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len, unsigned long, prot, int, pkey) { return do_mprotect_pkey(start, len, prot, pkey); } |
e8c24d3a2
|
622 623 624 625 626 627 628 629 630 631 632 633 |
SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val) { int pkey; int ret; /* No flags supported yet. */ if (flags) return -EINVAL; /* check for unsupported init values */ if (init_val & ~PKEY_ACCESS_MASK) return -EINVAL; |
d8ed45c5d
|
634 |
mmap_write_lock(current->mm); |
e8c24d3a2
|
635 636 637 638 639 640 641 642 643 644 645 646 647 |
pkey = mm_pkey_alloc(current->mm); ret = -ENOSPC; if (pkey == -1) goto out; ret = arch_set_user_pkey_access(current, pkey, init_val); if (ret) { mm_pkey_free(current->mm, pkey); goto out; } ret = pkey; out: |
d8ed45c5d
|
648 |
mmap_write_unlock(current->mm); |
e8c24d3a2
|
649 650 651 652 653 654 |
return ret; } SYSCALL_DEFINE1(pkey_free, int, pkey) { int ret; |
d8ed45c5d
|
655 |
mmap_write_lock(current->mm); |
e8c24d3a2
|
656 |
ret = mm_pkey_free(current->mm, pkey); |
d8ed45c5d
|
657 |
mmap_write_unlock(current->mm); |
e8c24d3a2
|
658 659 660 661 662 663 664 |
/* * We could provie warnings or errors if any VMA still * has the pkey set here. */ return ret; } |
c7142aead
|
665 666 |
#endif /* CONFIG_ARCH_HAS_PKEYS */ |