Blame view
include/linux/huge_mm.h
6.94 KB
71e3aac07 thp: transparent ... |
1 2 3 4 5 6 7 8 9 10 |
#ifndef _LINUX_HUGE_MM_H #define _LINUX_HUGE_MM_H extern int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, unsigned int flags); extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, struct vm_area_struct *vma); |
a1dd450bc mm: thp: set the ... |
11 12 13 14 |
extern void huge_pmd_set_accessed(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, pmd_t orig_pmd, int dirty); |
71e3aac07 thp: transparent ... |
15 16 17 |
extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, pmd_t orig_pmd); |
b676b293f mm, thp: fix mapp... |
18 |
extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, |
71e3aac07 thp: transparent ... |
19 20 21 22 23 |
unsigned long addr, pmd_t *pmd, unsigned int flags); extern int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, |
f21760b15 thp: add tlb_remo... |
24 |
pmd_t *pmd, unsigned long addr); |
0ca1634d4 thp: mincore tran... |
25 26 27 |
extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, unsigned char *vec); |
37a1c49a9 thp: mremap suppo... |
28 29 30 31 32 |
extern int move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma, unsigned long old_addr, unsigned long new_addr, unsigned long old_end, pmd_t *old_pmd, pmd_t *new_pmd); |
cd7548ab3 thp: mprotect: tr... |
33 |
extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, |
4b10e7d56 mm: mempolicy: Im... |
34 35 |
unsigned long addr, pgprot_t newprot, int prot_numa); |
71e3aac07 thp: transparent ... |
36 37 38 39 40 41 |
enum transparent_hugepage_flag { TRANSPARENT_HUGEPAGE_FLAG, TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, |
ba76149f4 thp: khugepaged |
42 |
TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG, |
79da5407e thp: introduce sy... |
43 |
TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG, |
71e3aac07 thp: transparent ... |
44 45 46 47 48 49 50 51 52 53 54 55 56 |
#ifdef CONFIG_DEBUG_VM TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG, #endif }; enum page_check_address_pmd_flag { PAGE_CHECK_ADDRESS_PMD_FLAG, PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG, }; extern pmd_t *page_check_address_pmd(struct page *page, struct mm_struct *mm, unsigned long address, |
117b0791a mm, thp: move ptl... |
57 58 |
enum page_check_address_pmd_flag flag, spinlock_t **ptl); |
71e3aac07 thp: transparent ... |
59 |
|
d8c37c480 thp: add HPAGE_PM... |
60 61 |
#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT) #define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER) |
71e3aac07 thp: transparent ... |
62 |
#ifdef CONFIG_TRANSPARENT_HUGEPAGE |
fde52796d mm/THP: don't use... |
63 64 65 |
#define HPAGE_PMD_SHIFT PMD_SHIFT #define HPAGE_PMD_SIZE ((1UL) << HPAGE_PMD_SHIFT) #define HPAGE_PMD_MASK (~(HPAGE_PMD_SIZE - 1)) |
71e3aac07 thp: transparent ... |
66 |
|
209959740 mm: move is_vma_t... |
67 |
extern bool is_vma_temporary_stack(struct vm_area_struct *vma); |
71e3aac07 thp: transparent ... |
68 |
#define transparent_hugepage_enabled(__vma) \ |
a664b2d85 thp: madvise(MADV... |
69 70 71 72 73 |
((transparent_hugepage_flags & \ (1<<TRANSPARENT_HUGEPAGE_FLAG) || \ (transparent_hugepage_flags & \ (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \ ((__vma)->vm_flags & VM_HUGEPAGE))) && \ |
a7d6e4ecd thp: prevent huge... |
74 75 |
!((__vma)->vm_flags & VM_NOHUGEPAGE) && \ !is_vma_temporary_stack(__vma)) |
71e3aac07 thp: transparent ... |
76 77 78 79 80 81 |
#define transparent_hugepage_defrag(__vma) \ ((transparent_hugepage_flags & \ (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) || \ (transparent_hugepage_flags & \ (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) && \ (__vma)->vm_flags & VM_HUGEPAGE)) |
79da5407e thp: introduce sy... |
82 83 84 |
#define transparent_hugepage_use_zero_page() \ (transparent_hugepage_flags & \ (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG)) |
71e3aac07 thp: transparent ... |
85 86 87 88 89 90 91 92 93 94 95 96 97 |
#ifdef CONFIG_DEBUG_VM #define transparent_hugepage_debug_cow() \ (transparent_hugepage_flags & \ (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG)) #else /* CONFIG_DEBUG_VM */ #define transparent_hugepage_debug_cow() 0 #endif /* CONFIG_DEBUG_VM */ extern unsigned long transparent_hugepage_flags; extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, unsigned long addr, unsigned long end); |
5bc7b8aca mm: thp: add spli... |
98 99 100 101 102 |
extern int split_huge_page_to_list(struct page *page, struct list_head *list); static inline int split_huge_page(struct page *page) { return split_huge_page_to_list(page, NULL); } |
e180377f1 thp: change split... |
103 104 105 |
extern void __split_huge_page_pmd(struct vm_area_struct *vma, unsigned long address, pmd_t *pmd); #define split_huge_page_pmd(__vma, __address, __pmd) \ |
71e3aac07 thp: transparent ... |
106 107 108 |
do { \ pmd_t *____pmd = (__pmd); \ if (unlikely(pmd_trans_huge(*____pmd))) \ |
e180377f1 thp: change split... |
109 110 |
__split_huge_page_pmd(__vma, __address, \ ____pmd); \ |
71e3aac07 thp: transparent ... |
111 112 113 114 |
} while (0) #define wait_split_huge_page(__anon_vma, __pmd) \ do { \ pmd_t *____pmd = (__pmd); \ |
4fc3f1d66 mm/rmap, migratio... |
115 |
anon_vma_lock_write(__anon_vma); \ |
08b52706d mm/rmap: rename a... |
116 |
anon_vma_unlock_write(__anon_vma); \ |
71e3aac07 thp: transparent ... |
117 118 119 |
BUG_ON(pmd_trans_splitting(*____pmd) || \ pmd_trans_huge(*____pmd)); \ } while (0) |
e180377f1 thp: change split... |
120 121 |
extern void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address, pmd_t *pmd); |
dfa5e237e mm: thp: Correct ... |
122 |
#if HPAGE_PMD_ORDER >= MAX_ORDER |
71e3aac07 thp: transparent ... |
123 124 |
#error "hugepages can't be allocated by the buddy allocator" #endif |
60ab3244e thp: khugepaged: ... |
125 126 |
extern int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, int advice); |
94fcc585f thp: avoid breaki... |
127 128 129 130 |
extern void __vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, long adjust_next); |
bf929152e mm, thp: change p... |
131 132 |
extern int __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma, spinlock_t **ptl); |
025c5b245 thp: optimize awa... |
133 |
/* mmap_sem must be held on entry */ |
bf929152e mm, thp: change p... |
134 135 |
static inline int pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma, spinlock_t **ptl) |
025c5b245 thp: optimize awa... |
136 137 138 |
{ VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem)); if (pmd_trans_huge(*pmd)) |
bf929152e mm, thp: change p... |
139 |
return __pmd_trans_huge_lock(pmd, vma, ptl); |
025c5b245 thp: optimize awa... |
140 141 142 |
else return 0; } |
94fcc585f thp: avoid breaki... |
143 144 145 146 147 |
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, long adjust_next) { |
78f11a255 mm: thp: fix /dev... |
148 |
if (!vma->anon_vma || vma->vm_ops) |
94fcc585f thp: avoid breaki... |
149 150 151 |
return; __vma_adjust_trans_huge(vma, start, end, adjust_next); } |
2c888cfbc thp: fix anon mem... |
152 153 154 155 156 157 |
static inline int hpage_nr_pages(struct page *page) { if (unlikely(PageTransHuge(page))) return HPAGE_PMD_NR; return 1; } |
d10e63f29 mm: numa: Create ... |
158 |
|
4daae3b4b mm: mempolicy: Us... |
159 160 |
extern int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pmd_t pmd, pmd_t *pmdp); |
d10e63f29 mm: numa: Create ... |
161 |
|
71e3aac07 thp: transparent ... |
162 |
#else /* CONFIG_TRANSPARENT_HUGEPAGE */ |
d8c37c480 thp: add HPAGE_PM... |
163 164 165 |
#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; }) #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; }) #define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; }) |
71e3aac07 thp: transparent ... |
166 |
|
2c888cfbc thp: fix anon mem... |
167 |
#define hpage_nr_pages(x) 1 |
71e3aac07 thp: transparent ... |
168 169 170 |
#define transparent_hugepage_enabled(__vma) 0 #define transparent_hugepage_flags 0UL |
5bc7b8aca mm: thp: add spli... |
171 172 173 174 175 |
static inline int split_huge_page_to_list(struct page *page, struct list_head *list) { return 0; } |
71e3aac07 thp: transparent ... |
176 177 178 179 |
static inline int split_huge_page(struct page *page) { return 0; } |
e180377f1 thp: change split... |
180 |
#define split_huge_page_pmd(__vma, __address, __pmd) \ |
71e3aac07 thp: transparent ... |
181 182 183 |
do { } while (0) #define wait_split_huge_page(__anon_vma, __pmd) \ do { } while (0) |
e180377f1 thp: change split... |
184 185 |
#define split_huge_page_pmd_mm(__mm, __address, __pmd) \ do { } while (0) |
60ab3244e thp: khugepaged: ... |
186 187 |
static inline int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, int advice) |
0af4e98b6 thp: madvise(MADV... |
188 189 190 191 |
{ BUG(); return 0; } |
94fcc585f thp: avoid breaki... |
192 193 194 195 196 197 |
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, long adjust_next) { } |
bf929152e mm, thp: change p... |
198 199 |
static inline int pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma, spinlock_t **ptl) |
025c5b245 thp: optimize awa... |
200 201 202 |
{ return 0; } |
d10e63f29 mm: numa: Create ... |
203 |
|
4daae3b4b mm: mempolicy: Us... |
204 205 |
static inline int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, pmd_t pmd, pmd_t *pmdp) |
d10e63f29 mm: numa: Create ... |
206 |
{ |
4daae3b4b mm: mempolicy: Us... |
207 |
return 0; |
d10e63f29 mm: numa: Create ... |
208 |
} |
71e3aac07 thp: transparent ... |
209 210 211 |
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* _LINUX_HUGE_MM_H */ |