Blame view
mm/vmacache.c
2.73 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
615d6e875 mm: per-thread vm... |
2 3 4 |
/* * Copyright (C) 2014 Davidlohr Bueso. */ |
3f07c0144 sched/headers: Pr... |
5 |
#include <linux/sched/signal.h> |
9164bb4a1 sched/headers: Pr... |
6 |
#include <linux/sched/task.h> |
615d6e875 mm: per-thread vm... |
7 8 |
#include <linux/mm.h> #include <linux/vmacache.h> |
ddbf369c0 mm, vmacache: has... |
9 10 11 12 13 14 15 16 17 18 19 |
/* * Hash based on the pmd of addr if configured with MMU, which provides a good * hit rate for workloads with spatial locality. Otherwise, use pages. */ #ifdef CONFIG_MMU #define VMACACHE_SHIFT PMD_SHIFT #else #define VMACACHE_SHIFT PAGE_SHIFT #endif #define VMACACHE_HASH(addr) ((addr >> VMACACHE_SHIFT) & VMACACHE_MASK) |
615d6e875 mm: per-thread vm... |
20 21 |
/* |
615d6e875 mm: per-thread vm... |
22 23 24 25 26 |
* This task may be accessing a foreign mm via (for example) * get_user_pages()->find_vma(). The vmacache is task-local and this * task's vmacache pertains to a different mm (ie, its own). There is * nothing we can do here. * |
f5678e7f2 kernel: better do... |
27 28 |
* Also handle the case where a kernel thread has adopted this mm via * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm. |
615d6e875 mm: per-thread vm... |
29 |
*/ |
a2c1aad3b mm/vmacache: inli... |
30 |
static inline bool vmacache_valid_mm(struct mm_struct *mm) |
615d6e875 mm: per-thread vm... |
31 32 33 34 35 36 37 |
{ return current->mm == mm && !(current->flags & PF_KTHREAD); } void vmacache_update(unsigned long addr, struct vm_area_struct *newvma) { if (vmacache_valid_mm(newvma->vm_mm)) |
314ff7851 mm/vmacache, sche... |
38 |
current->vmacache.vmas[VMACACHE_HASH(addr)] = newvma; |
615d6e875 mm: per-thread vm... |
39 40 41 42 43 44 45 46 47 48 |
} static bool vmacache_valid(struct mm_struct *mm) { struct task_struct *curr; if (!vmacache_valid_mm(mm)) return false; curr = current; |
314ff7851 mm/vmacache, sche... |
49 |
if (mm->vmacache_seqnum != curr->vmacache.seqnum) { |
615d6e875 mm: per-thread vm... |
50 51 52 53 |
/* * First attempt will always be invalid, initialize * the new cache for this task here. */ |
314ff7851 mm/vmacache, sche... |
54 |
curr->vmacache.seqnum = mm->vmacache_seqnum; |
615d6e875 mm: per-thread vm... |
55 56 57 58 59 60 61 62 |
vmacache_flush(curr); return false; } return true; } struct vm_area_struct *vmacache_find(struct mm_struct *mm, unsigned long addr) { |
ddbf369c0 mm, vmacache: has... |
63 |
int idx = VMACACHE_HASH(addr); |
615d6e875 mm: per-thread vm... |
64 |
int i; |
131ddc5c7 mm: unrig VMA cac... |
65 |
count_vm_vmacache_event(VMACACHE_FIND_CALLS); |
615d6e875 mm: per-thread vm... |
66 67 68 69 |
if (!vmacache_valid(mm)) return NULL; for (i = 0; i < VMACACHE_SIZE; i++) { |
ddbf369c0 mm, vmacache: has... |
70 |
struct vm_area_struct *vma = current->vmacache.vmas[idx]; |
615d6e875 mm: per-thread vm... |
71 |
|
ddbf369c0 mm, vmacache: has... |
72 73 74 75 76 77 78 79 80 |
if (vma) { #ifdef CONFIG_DEBUG_VM_VMACACHE if (WARN_ON_ONCE(vma->vm_mm != mm)) break; #endif if (vma->vm_start <= addr && vma->vm_end > addr) { count_vm_vmacache_event(VMACACHE_FIND_HITS); return vma; } |
4f115147f mm,vmacache: add ... |
81 |
} |
ddbf369c0 mm, vmacache: has... |
82 83 |
if (++idx == VMACACHE_SIZE) idx = 0; |
615d6e875 mm: per-thread vm... |
84 85 86 87 88 89 90 91 92 93 |
} return NULL; } #ifndef CONFIG_MMU struct vm_area_struct *vmacache_find_exact(struct mm_struct *mm, unsigned long start, unsigned long end) { |
ddbf369c0 mm, vmacache: has... |
94 |
int idx = VMACACHE_HASH(start); |
615d6e875 mm: per-thread vm... |
95 |
int i; |
131ddc5c7 mm: unrig VMA cac... |
96 |
count_vm_vmacache_event(VMACACHE_FIND_CALLS); |
615d6e875 mm: per-thread vm... |
97 98 99 100 |
if (!vmacache_valid(mm)) return NULL; for (i = 0; i < VMACACHE_SIZE; i++) { |
ddbf369c0 mm, vmacache: has... |
101 |
struct vm_area_struct *vma = current->vmacache.vmas[idx]; |
615d6e875 mm: per-thread vm... |
102 |
|
4f115147f mm,vmacache: add ... |
103 104 |
if (vma && vma->vm_start == start && vma->vm_end == end) { count_vm_vmacache_event(VMACACHE_FIND_HITS); |
615d6e875 mm: per-thread vm... |
105 |
return vma; |
4f115147f mm,vmacache: add ... |
106 |
} |
ddbf369c0 mm, vmacache: has... |
107 108 |
if (++idx == VMACACHE_SIZE) idx = 0; |
615d6e875 mm: per-thread vm... |
109 110 111 112 113 |
} return NULL; } #endif |