Blame view
mm/vmacache.c
2.74 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
615d6e875 mm: per-thread vm... |
2 3 4 |
/* * Copyright (C) 2014 Davidlohr Bueso. */ |
3f07c0144 sched/headers: Pr... |
5 |
#include <linux/sched/signal.h> |
9164bb4a1 sched/headers: Pr... |
6 |
#include <linux/sched/task.h> |
615d6e875 mm: per-thread vm... |
7 8 |
#include <linux/mm.h> #include <linux/vmacache.h> |
ddbf369c0 mm, vmacache: has... |
9 10 11 12 13 14 15 16 17 18 19 20 |
#include <asm/pgtable.h> /* * Hash based on the pmd of addr if configured with MMU, which provides a good * hit rate for workloads with spatial locality. Otherwise, use pages. */ #ifdef CONFIG_MMU #define VMACACHE_SHIFT PMD_SHIFT #else #define VMACACHE_SHIFT PAGE_SHIFT #endif #define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK) |
615d6e875 mm: per-thread vm... |
21 22 |
/* |
615d6e875 mm: per-thread vm... |
23 24 25 26 27 28 29 30 |
* This task may be accessing a foreign mm via (for example) * get_user_pages()->find_vma(). The vmacache is task-local and this * task's vmacache pertains to a different mm (ie, its own). There is * nothing we can do here. * * Also handle the case where a kernel thread has adopted this mm via use_mm(). * That kernel thread's vmacache is not applicable to this mm. */ |
a2c1aad3b mm/vmacache: inli... |
31 |
static inline bool vmacache_valid_mm(struct mm_struct *mm) |
615d6e875 mm: per-thread vm... |
32 33 34 35 36 37 38 |
{ return current->mm == mm && !(current->flags & PF_KTHREAD); } void vmacache_update(unsigned long addr, struct vm_area_struct *newvma) { if (vmacache_valid_mm(newvma->vm_mm)) |
314ff7851 mm/vmacache, sche... |
39 |
current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma; |
615d6e875 mm: per-thread vm... |
40 41 42 43 44 45 46 47 48 49 |
} static bool vmacache_valid(struct mm_struct *mm) { struct task_struct *curr; if (!vmacache_valid_mm(mm)) return false; curr = current; |
314ff7851 mm/vmacache, sche... |
50 |
if (mm->vmacache_seqnum != curr->vmacache.seqnum) { |
615d6e875 mm: per-thread vm... |
51 52 53 54 |
/* * First attempt will always be invalid, initialize * the new cache for this task here. */ |
314ff7851 mm/vmacache, sche... |
55 |
curr->vmacache.seqnum = mm->vmacache_seqnum; |
615d6e875 mm: per-thread vm... |
56 57 58 59 60 61 62 63 |
vmacache_flush(curr); return false; } return true; } struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr) { |
ddbf369c0 mm, vmacache: has... |
64 |
int idx = VMACACHE_HASH(addr); |
615d6e875 mm: per-thread vm... |
65 |
int i; |
131ddc5c7 mm: unrig VMA cac... |
66 |
count_vm_vmacache_event(VMACACHE_FIND_CALLS); |
615d6e875 mm: per-thread vm... |
67 68 69 70 |
if (!vmacache_valid(mm)) return NULL; for (i = 0; i < VMACACHE_SIZE; i++) { |
ddbf369c0 mm, vmacache: has... |
71 |
struct vm_area_struct *vma = current->vmacache.vmas[idx]; |
615d6e875 mm: per-thread vm... |
72 |
|
ddbf369c0 mm, vmacache: has... |
73 74 75 76 77 78 79 80 81 |
if (vma) { #ifdef CONFIG_DEBUG_VM_VMACACHE if (WARN_ON_ONCE(vma->vm_mm != mm)) break; #endif if (vma->vm_start <= addr && vma->vm_end > addr) { count_vm_vmacache_event(VMACACHE_FIND_HITS); return vma; } |
4f115147f mm,vmacache: add ... |
82 |
} |
ddbf369c0 mm, vmacache: has... |
83 84 |
if (++idx == VMACACHE_SIZE) idx = 0; |
615d6e875 mm: per-thread vm... |
85 86 87 88 89 90 91 92 93 94 |
} return NULL; } #ifndef CONFIG_MMU struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, unsigned long start, unsigned long end) { |
ddbf369c0 mm, vmacache: has... |
95 |
int idx = VMACACHE_HASH(start); |
615d6e875 mm: per-thread vm... |
96 |
int i; |
131ddc5c7 mm: unrig VMA cac... |
97 |
count_vm_vmacache_event(VMACACHE_FIND_CALLS); |
615d6e875 mm: per-thread vm... |
98 99 100 101 |
if (!vmacache_valid(mm)) return NULL; for (i = 0; i < VMACACHE_SIZE; i++) { |
ddbf369c0 mm, vmacache: has... |
102 |
struct vm_area_struct *vma = current->vmacache.vmas[idx]; |
615d6e875 mm: per-thread vm... |
103 |
|
4f115147f mm,vmacache: add ... |
104 105 |
if (vma && vma->vm_start == start && vma->vm_end == end) { count_vm_vmacache_event(VMACACHE_FIND_HITS); |
615d6e875 mm: per-thread vm... |
106 |
return vma; |
4f115147f mm,vmacache: add ... |
107 |
} |
ddbf369c0 mm, vmacache: has... |
108 109 |
if (++idx == VMACACHE_SIZE) idx = 0; |
615d6e875 mm: per-thread vm... |
110 111 112 113 114 |
} return NULL; } #endif |