Blame view

include/linux/huge_mm.h 6.94 KB
71e3aac07   Andrea Arcangeli   thp: transparent ...
1
2
3
4
5
6
7
8
9
10
  #ifndef _LINUX_HUGE_MM_H
  #define _LINUX_HUGE_MM_H
  
  extern int do_huge_pmd_anonymous_page(struct mm_struct *mm,
  				      struct vm_area_struct *vma,
  				      unsigned long address, pmd_t *pmd,
  				      unsigned int flags);
  extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  			 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
  			 struct vm_area_struct *vma);
a1dd450bc   Will Deacon   mm: thp: set the ...
11
12
13
14
  extern void huge_pmd_set_accessed(struct mm_struct *mm,
  				  struct vm_area_struct *vma,
  				  unsigned long address, pmd_t *pmd,
  				  pmd_t orig_pmd, int dirty);
71e3aac07   Andrea Arcangeli   thp: transparent ...
15
16
17
  extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
  			       unsigned long address, pmd_t *pmd,
  			       pmd_t orig_pmd);
b676b293f   David Rientjes   mm, thp: fix mapp...
18
  extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
71e3aac07   Andrea Arcangeli   thp: transparent ...
19
20
21
22
23
  					  unsigned long addr,
  					  pmd_t *pmd,
  					  unsigned int flags);
  extern int zap_huge_pmd(struct mmu_gather *tlb,
  			struct vm_area_struct *vma,
f21760b15   Shaohua Li   thp: add tlb_remo...
24
  			pmd_t *pmd, unsigned long addr);
0ca1634d4   Johannes Weiner   thp: mincore tran...
25
26
27
  extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
  			unsigned long addr, unsigned long end,
  			unsigned char *vec);
37a1c49a9   Andrea Arcangeli   thp: mremap suppo...
28
29
30
31
32
  extern int move_huge_pmd(struct vm_area_struct *vma,
  			 struct vm_area_struct *new_vma,
  			 unsigned long old_addr,
  			 unsigned long new_addr, unsigned long old_end,
  			 pmd_t *old_pmd, pmd_t *new_pmd);
cd7548ab3   Johannes Weiner   thp: mprotect: tr...
33
  extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
4b10e7d56   Mel Gorman   mm: mempolicy: Im...
34
35
  			unsigned long addr, pgprot_t newprot,
  			int prot_numa);
71e3aac07   Andrea Arcangeli   thp: transparent ...
36
37
38
39
40
41
  
  enum transparent_hugepage_flag {
  	TRANSPARENT_HUGEPAGE_FLAG,
  	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
  	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
  	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
ba76149f4   Andrea Arcangeli   thp: khugepaged
42
  	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
79da5407e   Kirill A. Shutemov   thp: introduce sy...
43
  	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
71e3aac07   Andrea Arcangeli   thp: transparent ...
44
45
46
47
48
49
50
51
52
53
54
55
56
  #ifdef CONFIG_DEBUG_VM
  	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
  #endif
  };
  
  enum page_check_address_pmd_flag {
  	PAGE_CHECK_ADDRESS_PMD_FLAG,
  	PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG,
  	PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG,
  };
  extern pmd_t *page_check_address_pmd(struct page *page,
  				     struct mm_struct *mm,
  				     unsigned long address,
117b0791a   Kirill A. Shutemov   mm, thp: move ptl...
57
58
  				     enum page_check_address_pmd_flag flag,
  				     spinlock_t **ptl);
71e3aac07   Andrea Arcangeli   thp: transparent ...
59

d8c37c480   Naoya Horiguchi   thp: add HPAGE_PM...
60
61
  #define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
  #define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
71e3aac07   Andrea Arcangeli   thp: transparent ...
62
  #ifdef CONFIG_TRANSPARENT_HUGEPAGE
fde52796d   Aneesh Kumar K.V   mm/THP: don't use...
63
64
65
  #define HPAGE_PMD_SHIFT PMD_SHIFT
  #define HPAGE_PMD_SIZE	((1UL) << HPAGE_PMD_SHIFT)
  #define HPAGE_PMD_MASK	(~(HPAGE_PMD_SIZE - 1))
71e3aac07   Andrea Arcangeli   thp: transparent ...
66

209959740   Alex Shi   mm: move is_vma_t...
67
  extern bool is_vma_temporary_stack(struct vm_area_struct *vma);
71e3aac07   Andrea Arcangeli   thp: transparent ...
68
  #define transparent_hugepage_enabled(__vma)				\
a664b2d85   Andrea Arcangeli   thp: madvise(MADV...
69
70
71
72
73
  	((transparent_hugepage_flags &					\
  	  (1<<TRANSPARENT_HUGEPAGE_FLAG) ||				\
  	  (transparent_hugepage_flags &					\
  	   (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) &&			\
  	   ((__vma)->vm_flags & VM_HUGEPAGE))) &&			\
a7d6e4ecd   Andrea Arcangeli   thp: prevent huge...
74
75
  	 !((__vma)->vm_flags & VM_NOHUGEPAGE) &&			\
  	 !is_vma_temporary_stack(__vma))
71e3aac07   Andrea Arcangeli   thp: transparent ...
76
77
78
79
80
81
  #define transparent_hugepage_defrag(__vma)				\
  	((transparent_hugepage_flags &					\
  	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) ||			\
  	 (transparent_hugepage_flags &					\
  	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) &&		\
  	  (__vma)->vm_flags & VM_HUGEPAGE))
79da5407e   Kirill A. Shutemov   thp: introduce sy...
82
83
84
  #define transparent_hugepage_use_zero_page()				\
  	(transparent_hugepage_flags &					\
  	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
71e3aac07   Andrea Arcangeli   thp: transparent ...
85
86
87
88
89
90
91
92
93
94
95
96
97
  #ifdef CONFIG_DEBUG_VM
  #define transparent_hugepage_debug_cow()				\
  	(transparent_hugepage_flags &					\
  	 (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
  #else /* CONFIG_DEBUG_VM */
  #define transparent_hugepage_debug_cow() 0
  #endif /* CONFIG_DEBUG_VM */
  
  extern unsigned long transparent_hugepage_flags;
  extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
  			  pmd_t *dst_pmd, pmd_t *src_pmd,
  			  struct vm_area_struct *vma,
  			  unsigned long addr, unsigned long end);
5bc7b8aca   Shaohua Li   mm: thp: add spli...
98
99
100
101
102
  extern int split_huge_page_to_list(struct page *page, struct list_head *list);
  static inline int split_huge_page(struct page *page)
  {
  	return split_huge_page_to_list(page, NULL);
  }
e180377f1   Kirill A. Shutemov   thp: change split...
103
104
105
  extern void __split_huge_page_pmd(struct vm_area_struct *vma,
  		unsigned long address, pmd_t *pmd);
  #define split_huge_page_pmd(__vma, __address, __pmd)			\
71e3aac07   Andrea Arcangeli   thp: transparent ...
106
107
108
  	do {								\
  		pmd_t *____pmd = (__pmd);				\
  		if (unlikely(pmd_trans_huge(*____pmd)))			\
e180377f1   Kirill A. Shutemov   thp: change split...
109
110
  			__split_huge_page_pmd(__vma, __address,		\
  					____pmd);			\
71e3aac07   Andrea Arcangeli   thp: transparent ...
111
112
113
114
  	}  while (0)
  #define wait_split_huge_page(__anon_vma, __pmd)				\
  	do {								\
  		pmd_t *____pmd = (__pmd);				\
4fc3f1d66   Ingo Molnar   mm/rmap, migratio...
115
  		anon_vma_lock_write(__anon_vma);			\
08b52706d   Konstantin Khlebnikov   mm/rmap: rename a...
116
  		anon_vma_unlock_write(__anon_vma);			\
71e3aac07   Andrea Arcangeli   thp: transparent ...
117
118
119
  		BUG_ON(pmd_trans_splitting(*____pmd) ||			\
  		       pmd_trans_huge(*____pmd));			\
  	} while (0)
e180377f1   Kirill A. Shutemov   thp: change split...
120
121
  extern void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address,
  		pmd_t *pmd);
dfa5e237e   Steve Capper   mm: thp: Correct ...
122
  #if HPAGE_PMD_ORDER >= MAX_ORDER
71e3aac07   Andrea Arcangeli   thp: transparent ...
123
124
  #error "hugepages can't be allocated by the buddy allocator"
  #endif
60ab3244e   Andrea Arcangeli   thp: khugepaged: ...
125
126
  extern int hugepage_madvise(struct vm_area_struct *vma,
  			    unsigned long *vm_flags, int advice);
94fcc585f   Andrea Arcangeli   thp: avoid breaki...
127
128
129
130
  extern void __vma_adjust_trans_huge(struct vm_area_struct *vma,
  				    unsigned long start,
  				    unsigned long end,
  				    long adjust_next);
bf929152e   Kirill A. Shutemov   mm, thp: change p...
131
132
  extern int __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
  		spinlock_t **ptl);
025c5b245   Naoya Horiguchi   thp: optimize awa...
133
  /* mmap_sem must be held on entry */
bf929152e   Kirill A. Shutemov   mm, thp: change p...
134
135
  static inline int pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
  		spinlock_t **ptl)
025c5b245   Naoya Horiguchi   thp: optimize awa...
136
137
138
  {
  	VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem));
  	if (pmd_trans_huge(*pmd))
bf929152e   Kirill A. Shutemov   mm, thp: change p...
139
  		return __pmd_trans_huge_lock(pmd, vma, ptl);
025c5b245   Naoya Horiguchi   thp: optimize awa...
140
141
142
  	else
  		return 0;
  }
94fcc585f   Andrea Arcangeli   thp: avoid breaki...
143
144
145
146
147
  static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
  					 unsigned long start,
  					 unsigned long end,
  					 long adjust_next)
  {
78f11a255   Andrea Arcangeli   mm: thp: fix /dev...
148
  	if (!vma->anon_vma || vma->vm_ops)
94fcc585f   Andrea Arcangeli   thp: avoid breaki...
149
150
151
  		return;
  	__vma_adjust_trans_huge(vma, start, end, adjust_next);
  }
2c888cfbc   Rik van Riel   thp: fix anon mem...
152
153
154
155
156
157
  static inline int hpage_nr_pages(struct page *page)
  {
  	if (unlikely(PageTransHuge(page)))
  		return HPAGE_PMD_NR;
  	return 1;
  }
d10e63f29   Mel Gorman   mm: numa: Create ...
158

4daae3b4b   Mel Gorman   mm: mempolicy: Us...
159
160
  extern int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
  				unsigned long addr, pmd_t pmd, pmd_t *pmdp);
d10e63f29   Mel Gorman   mm: numa: Create ...
161

71e3aac07   Andrea Arcangeli   thp: transparent ...
162
  #else /* CONFIG_TRANSPARENT_HUGEPAGE */
d8c37c480   Naoya Horiguchi   thp: add HPAGE_PM...
163
164
165
  #define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
  #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
  #define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
71e3aac07   Andrea Arcangeli   thp: transparent ...
166

2c888cfbc   Rik van Riel   thp: fix anon mem...
167
  #define hpage_nr_pages(x) 1
71e3aac07   Andrea Arcangeli   thp: transparent ...
168
169
170
  #define transparent_hugepage_enabled(__vma) 0
  
  #define transparent_hugepage_flags 0UL
5bc7b8aca   Shaohua Li   mm: thp: add spli...
171
172
173
174
175
  static inline int
  split_huge_page_to_list(struct page *page, struct list_head *list)
  {
  	return 0;
  }
71e3aac07   Andrea Arcangeli   thp: transparent ...
176
177
178
179
  static inline int split_huge_page(struct page *page)
  {
  	return 0;
  }
e180377f1   Kirill A. Shutemov   thp: change split...
180
  #define split_huge_page_pmd(__vma, __address, __pmd)	\
71e3aac07   Andrea Arcangeli   thp: transparent ...
181
182
183
  	do { } while (0)
  #define wait_split_huge_page(__anon_vma, __pmd)	\
  	do { } while (0)
e180377f1   Kirill A. Shutemov   thp: change split...
184
185
  #define split_huge_page_pmd_mm(__mm, __address, __pmd)	\
  	do { } while (0)
60ab3244e   Andrea Arcangeli   thp: khugepaged: ...
186
187
  static inline int hugepage_madvise(struct vm_area_struct *vma,
  				   unsigned long *vm_flags, int advice)
0af4e98b6   Andrea Arcangeli   thp: madvise(MADV...
188
189
190
191
  {
  	BUG();
  	return 0;
  }
94fcc585f   Andrea Arcangeli   thp: avoid breaki...
192
193
194
195
196
197
  static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
  					 unsigned long start,
  					 unsigned long end,
  					 long adjust_next)
  {
  }
bf929152e   Kirill A. Shutemov   mm, thp: change p...
198
199
  static inline int pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
  		spinlock_t **ptl)
025c5b245   Naoya Horiguchi   thp: optimize awa...
200
201
202
  {
  	return 0;
  }
d10e63f29   Mel Gorman   mm: numa: Create ...
203

4daae3b4b   Mel Gorman   mm: mempolicy: Us...
204
205
  static inline int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
  					unsigned long addr, pmd_t pmd, pmd_t *pmdp)
d10e63f29   Mel Gorman   mm: numa: Create ...
206
  {
4daae3b4b   Mel Gorman   mm: mempolicy: Us...
207
  	return 0;
d10e63f29   Mel Gorman   mm: numa: Create ...
208
  }
71e3aac07   Andrea Arcangeli   thp: transparent ...
209
210
211
  #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
  
  #endif /* _LINUX_HUGE_MM_H */