Doug / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 1998cc048901109a29924380b8e91bc049b32951

Authored by Shaohua Li
Committed by Linus Torvalds
1 parent a394cb8ee6
Exists in smarc-l5.0.0_1.0.0-ga and in 5 other branches imx_3.10.17_1.0.1_ga, imx_3.10.53_1.1.0_ga, imx_3.14.28_1.0.0_ga, smarc-imx_3.10.53_1.1.0_ga, smarc-imx_3.14.28_1.0.0_ga

mm: make madvise(MADV_WILLNEED) support swap file prefetch 

Make madvise(MADV_WILLNEED) support swap file prefetch.  If memory is
swapout, this syscall can do swapin prefetch.  It has no impact if the
memory isn't swapout.

[akpm@linux-foundation.org: fix CONFIG_SWAP=n build]
[sasha.levin@oracle.com: fix BUG on madvise early failure]
Signed-off-by: Shaohua Li <shli@fusionio.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Rik van Riel <riel@redhat.com>
Signed-off-by: Sasha Levin <sasha.levin@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

Showing 1 changed file with 101 additions and 4 deletions Inline Diff

1 /* 1 /*
2 * linux/mm/madvise.c 2 * linux/mm/madvise.c
3 * 3 *
4 * Copyright (C) 1999 Linus Torvalds 4 * Copyright (C) 1999 Linus Torvalds
5 * Copyright (C) 2002 Christoph Hellwig 5 * Copyright (C) 2002 Christoph Hellwig
6 */ 6 */
7 7
8 #include <linux/mman.h> 8 #include <linux/mman.h>
9 #include <linux/pagemap.h> 9 #include <linux/pagemap.h>
10 #include <linux/syscalls.h> 10 #include <linux/syscalls.h>
11 #include <linux/mempolicy.h> 11 #include <linux/mempolicy.h>
12 #include <linux/page-isolation.h> 12 #include <linux/page-isolation.h>
13 #include <linux/hugetlb.h> 13 #include <linux/hugetlb.h>
14 #include <linux/falloc.h> 14 #include <linux/falloc.h>
15 #include <linux/sched.h> 15 #include <linux/sched.h>
16 #include <linux/ksm.h> 16 #include <linux/ksm.h>
17 #include <linux/fs.h> 17 #include <linux/fs.h>
18 #include <linux/file.h> 18 #include <linux/file.h>
19 #include <linux/blkdev.h>
20 #include <linux/swap.h>
21 #include <linux/swapops.h>
19 22
20 /* 23 /*
21 * Any behaviour which results in changes to the vma->vm_flags needs to 24 * Any behaviour which results in changes to the vma->vm_flags needs to
22 * take mmap_sem for writing. Others, which simply traverse vmas, need 25 * take mmap_sem for writing. Others, which simply traverse vmas, need
23 * to only take it for reading. 26 * to only take it for reading.
24 */ 27 */
25 static int madvise_need_mmap_write(int behavior) 28 static int madvise_need_mmap_write(int behavior)
26 { 29 {
27 switch (behavior) { 30 switch (behavior) {
28 case MADV_REMOVE: 31 case MADV_REMOVE:
29 case MADV_WILLNEED: 32 case MADV_WILLNEED:
30 case MADV_DONTNEED: 33 case MADV_DONTNEED:
31 return 0; 34 return 0;
32 default: 35 default:
33 /* be safe, default to 1. list exceptions explicitly */ 36 /* be safe, default to 1. list exceptions explicitly */
34 return 1; 37 return 1;
35 } 38 }
36 } 39 }
37 40
38 /* 41 /*
39 * We can potentially split a vm area into separate 42 * We can potentially split a vm area into separate
40 * areas, each area with its own behavior. 43 * areas, each area with its own behavior.
41 */ 44 */
42 static long madvise_behavior(struct vm_area_struct * vma, 45 static long madvise_behavior(struct vm_area_struct * vma,
43 struct vm_area_struct **prev, 46 struct vm_area_struct **prev,
44 unsigned long start, unsigned long end, int behavior) 47 unsigned long start, unsigned long end, int behavior)
45 { 48 {
46 struct mm_struct * mm = vma->vm_mm; 49 struct mm_struct * mm = vma->vm_mm;
47 int error = 0; 50 int error = 0;
48 pgoff_t pgoff; 51 pgoff_t pgoff;
49 unsigned long new_flags = vma->vm_flags; 52 unsigned long new_flags = vma->vm_flags;
50 53
51 switch (behavior) { 54 switch (behavior) {
52 case MADV_NORMAL: 55 case MADV_NORMAL:
53 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ; 56 new_flags = new_flags & ~VM_RAND_READ & ~VM_SEQ_READ;
54 break; 57 break;
55 case MADV_SEQUENTIAL: 58 case MADV_SEQUENTIAL:
56 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ; 59 new_flags = (new_flags & ~VM_RAND_READ) | VM_SEQ_READ;
57 break; 60 break;
58 case MADV_RANDOM: 61 case MADV_RANDOM:
59 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ; 62 new_flags = (new_flags & ~VM_SEQ_READ) | VM_RAND_READ;
60 break; 63 break;
61 case MADV_DONTFORK: 64 case MADV_DONTFORK:
62 new_flags |= VM_DONTCOPY; 65 new_flags |= VM_DONTCOPY;
63 break; 66 break;
64 case MADV_DOFORK: 67 case MADV_DOFORK:
65 if (vma->vm_flags & VM_IO) { 68 if (vma->vm_flags & VM_IO) {
66 error = -EINVAL; 69 error = -EINVAL;
67 goto out; 70 goto out;
68 } 71 }
69 new_flags &= ~VM_DONTCOPY; 72 new_flags &= ~VM_DONTCOPY;
70 break; 73 break;
71 case MADV_DONTDUMP: 74 case MADV_DONTDUMP:
72 new_flags |= VM_DONTDUMP; 75 new_flags |= VM_DONTDUMP;
73 break; 76 break;
74 case MADV_DODUMP: 77 case MADV_DODUMP:
75 if (new_flags & VM_SPECIAL) { 78 if (new_flags & VM_SPECIAL) {
76 error = -EINVAL; 79 error = -EINVAL;
77 goto out; 80 goto out;
78 } 81 }
79 new_flags &= ~VM_DONTDUMP; 82 new_flags &= ~VM_DONTDUMP;
80 break; 83 break;
81 case MADV_MERGEABLE: 84 case MADV_MERGEABLE:
82 case MADV_UNMERGEABLE: 85 case MADV_UNMERGEABLE:
83 error = ksm_madvise(vma, start, end, behavior, &new_flags); 86 error = ksm_madvise(vma, start, end, behavior, &new_flags);
84 if (error) 87 if (error)
85 goto out; 88 goto out;
86 break; 89 break;
87 case MADV_HUGEPAGE: 90 case MADV_HUGEPAGE:
88 case MADV_NOHUGEPAGE: 91 case MADV_NOHUGEPAGE:
89 error = hugepage_madvise(vma, &new_flags, behavior); 92 error = hugepage_madvise(vma, &new_flags, behavior);
90 if (error) 93 if (error)
91 goto out; 94 goto out;
92 break; 95 break;
93 } 96 }
94 97
95 if (new_flags == vma->vm_flags) { 98 if (new_flags == vma->vm_flags) {
96 *prev = vma; 99 *prev = vma;
97 goto out; 100 goto out;
98 } 101 }
99 102
100 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT); 103 pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
101 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma, 104 *prev = vma_merge(mm, *prev, start, end, new_flags, vma->anon_vma,
102 vma->vm_file, pgoff, vma_policy(vma)); 105 vma->vm_file, pgoff, vma_policy(vma));
103 if (*prev) { 106 if (*prev) {
104 vma = *prev; 107 vma = *prev;
105 goto success; 108 goto success;
106 } 109 }
107 110
108 *prev = vma; 111 *prev = vma;
109 112
110 if (start != vma->vm_start) { 113 if (start != vma->vm_start) {
111 error = split_vma(mm, vma, start, 1); 114 error = split_vma(mm, vma, start, 1);
112 if (error) 115 if (error)
113 goto out; 116 goto out;
114 } 117 }
115 118
116 if (end != vma->vm_end) { 119 if (end != vma->vm_end) {
117 error = split_vma(mm, vma, end, 0); 120 error = split_vma(mm, vma, end, 0);
118 if (error) 121 if (error)
119 goto out; 122 goto out;
120 } 123 }
121 124
122 success: 125 success:
123 /* 126 /*
124 * vm_flags is protected by the mmap_sem held in write mode. 127 * vm_flags is protected by the mmap_sem held in write mode.
125 */ 128 */
126 vma->vm_flags = new_flags; 129 vma->vm_flags = new_flags;
127 130
128 out: 131 out:
129 if (error == -ENOMEM) 132 if (error == -ENOMEM)
130 error = -EAGAIN; 133 error = -EAGAIN;
131 return error; 134 return error;
132 } 135 }
133 136
137 #ifdef CONFIG_SWAP
138 static int swapin_walk_pmd_entry(pmd_t *pmd, unsigned long start,
139 unsigned long end, struct mm_walk *walk)
140 {
141 pte_t *orig_pte;
142 struct vm_area_struct *vma = walk->private;
143 unsigned long index;
144
145 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
146 return 0;
147
148 for (index = start; index != end; index += PAGE_SIZE) {
149 pte_t pte;
150 swp_entry_t entry;
151 struct page *page;
152 spinlock_t *ptl;
153
154 orig_pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl);
155 pte = *(orig_pte + ((index - start) / PAGE_SIZE));
156 pte_unmap_unlock(orig_pte, ptl);
157
158 if (pte_present(pte) || pte_none(pte) || pte_file(pte))
159 continue;
160 entry = pte_to_swp_entry(pte);
161 if (unlikely(non_swap_entry(entry)))
162 continue;
163
164 page = read_swap_cache_async(entry, GFP_HIGHUSER_MOVABLE,
165 vma, index);
166 if (page)
167 page_cache_release(page);
168 }
169
170 return 0;
171 }
172
173 static void force_swapin_readahead(struct vm_area_struct *vma,
174 unsigned long start, unsigned long end)
175 {
176 struct mm_walk walk = {
177 .mm = vma->vm_mm,
178 .pmd_entry = swapin_walk_pmd_entry,
179 .private = vma,
180 };
181
182 walk_page_range(start, end, &walk);
183
184 lru_add_drain(); /* Push any new pages onto the LRU now */
185 }
186
187 static void force_shm_swapin_readahead(struct vm_area_struct *vma,
188 unsigned long start, unsigned long end,
189 struct address_space *mapping)
190 {
191 pgoff_t index;
192 struct page *page;
193 swp_entry_t swap;
194
195 for (; start < end; start += PAGE_SIZE) {
196 index = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
197
198 page = find_get_page(mapping, index);
199 if (!radix_tree_exceptional_entry(page)) {
200 if (page)
201 page_cache_release(page);
202 continue;
203 }
204 swap = radix_to_swp_entry(page);
205 page = read_swap_cache_async(swap, GFP_HIGHUSER_MOVABLE,
206 NULL, 0);
207 if (page)
208 page_cache_release(page);
209 }
210
211 lru_add_drain(); /* Push any new pages onto the LRU now */
212 }
213 #endif /* CONFIG_SWAP */
214
134 /* 215 /*
135 * Schedule all required I/O operations. Do not wait for completion. 216 * Schedule all required I/O operations. Do not wait for completion.
136 */ 217 */
137 static long madvise_willneed(struct vm_area_struct * vma, 218 static long madvise_willneed(struct vm_area_struct * vma,
138 struct vm_area_struct ** prev, 219 struct vm_area_struct ** prev,
139 unsigned long start, unsigned long end) 220 unsigned long start, unsigned long end)
140 { 221 {
141 struct file *file = vma->vm_file; 222 struct file *file = vma->vm_file;
142 223
224 #ifdef CONFIG_SWAP
225 if (!file || mapping_cap_swap_backed(file->f_mapping)) {
226 *prev = vma;
227 if (!file)
228 force_swapin_readahead(vma, start, end);
229 else
230 force_shm_swapin_readahead(vma, start, end,
231 file->f_mapping);
232 return 0;
233 }
234 #endif
235
143 if (!file) 236 if (!file)
144 return -EBADF; 237 return -EBADF;
145 238
146 if (file->f_mapping->a_ops->get_xip_mem) { 239 if (file->f_mapping->a_ops->get_xip_mem) {
147 /* no bad return value, but ignore advice */ 240 /* no bad return value, but ignore advice */
148 return 0; 241 return 0;
149 } 242 }
150 243
151 *prev = vma; 244 *prev = vma;
152 start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 245 start = ((start - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
153 if (end > vma->vm_end) 246 if (end > vma->vm_end)
154 end = vma->vm_end; 247 end = vma->vm_end;
155 end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 248 end = ((end - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
156 249
157 force_page_cache_readahead(file->f_mapping, file, start, end - start); 250 force_page_cache_readahead(file->f_mapping, file, start, end - start);
158 return 0; 251 return 0;
159 } 252 }
160 253
161 /* 254 /*
162 * Application no longer needs these pages. If the pages are dirty, 255 * Application no longer needs these pages. If the pages are dirty,
163 * it's OK to just throw them away. The app will be more careful about 256 * it's OK to just throw them away. The app will be more careful about
164 * data it wants to keep. Be sure to free swap resources too. The 257 * data it wants to keep. Be sure to free swap resources too. The
165 * zap_page_range call sets things up for shrink_active_list to actually free 258 * zap_page_range call sets things up for shrink_active_list to actually free
166 * these pages later if no one else has touched them in the meantime, 259 * these pages later if no one else has touched them in the meantime,
167 * although we could add these pages to a global reuse list for 260 * although we could add these pages to a global reuse list for
168 * shrink_active_list to pick up before reclaiming other pages. 261 * shrink_active_list to pick up before reclaiming other pages.
169 * 262 *
170 * NB: This interface discards data rather than pushes it out to swap, 263 * NB: This interface discards data rather than pushes it out to swap,
171 * as some implementations do. This has performance implications for 264 * as some implementations do. This has performance implications for
172 * applications like large transactional databases which want to discard 265 * applications like large transactional databases which want to discard
173 * pages in anonymous maps after committing to backing store the data 266 * pages in anonymous maps after committing to backing store the data
174 * that was kept in them. There is no reason to write this data out to 267 * that was kept in them. There is no reason to write this data out to
175 * the swap area if the application is discarding it. 268 * the swap area if the application is discarding it.
176 * 269 *
177 * An interface that causes the system to free clean pages and flush 270 * An interface that causes the system to free clean pages and flush
178 * dirty pages is already available as msync(MS_INVALIDATE). 271 * dirty pages is already available as msync(MS_INVALIDATE).
179 */ 272 */
180 static long madvise_dontneed(struct vm_area_struct * vma, 273 static long madvise_dontneed(struct vm_area_struct * vma,
181 struct vm_area_struct ** prev, 274 struct vm_area_struct ** prev,
182 unsigned long start, unsigned long end) 275 unsigned long start, unsigned long end)
183 { 276 {
184 *prev = vma; 277 *prev = vma;
185 if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP)) 278 if (vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP))
186 return -EINVAL; 279 return -EINVAL;
187 280
188 if (unlikely(vma->vm_flags & VM_NONLINEAR)) { 281 if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
189 struct zap_details details = { 282 struct zap_details details = {
190 .nonlinear_vma = vma, 283 .nonlinear_vma = vma,
191 .last_index = ULONG_MAX, 284 .last_index = ULONG_MAX,
192 }; 285 };
193 zap_page_range(vma, start, end - start, &details); 286 zap_page_range(vma, start, end - start, &details);
194 } else 287 } else
195 zap_page_range(vma, start, end - start, NULL); 288 zap_page_range(vma, start, end - start, NULL);
196 return 0; 289 return 0;
197 } 290 }
198 291
199 /* 292 /*
200 * Application wants to free up the pages and associated backing store. 293 * Application wants to free up the pages and associated backing store.
201 * This is effectively punching a hole into the middle of a file. 294 * This is effectively punching a hole into the middle of a file.
202 * 295 *
203 * NOTE: Currently, only shmfs/tmpfs is supported for this operation. 296 * NOTE: Currently, only shmfs/tmpfs is supported for this operation.
204 * Other filesystems return -ENOSYS. 297 * Other filesystems return -ENOSYS.
205 */ 298 */
206 static long madvise_remove(struct vm_area_struct *vma, 299 static long madvise_remove(struct vm_area_struct *vma,
207 struct vm_area_struct **prev, 300 struct vm_area_struct **prev,
208 unsigned long start, unsigned long end) 301 unsigned long start, unsigned long end)
209 { 302 {
210 loff_t offset; 303 loff_t offset;
211 int error; 304 int error;
212 struct file *f; 305 struct file *f;
213 306
214 *prev = NULL; /* tell sys_madvise we drop mmap_sem */ 307 *prev = NULL; /* tell sys_madvise we drop mmap_sem */
215 308
216 if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB)) 309 if (vma->vm_flags & (VM_LOCKED|VM_NONLINEAR|VM_HUGETLB))
217 return -EINVAL; 310 return -EINVAL;
218 311
219 f = vma->vm_file; 312 f = vma->vm_file;
220 313
221 if (!f || !f->f_mapping || !f->f_mapping->host) { 314 if (!f || !f->f_mapping || !f->f_mapping->host) {
222 return -EINVAL; 315 return -EINVAL;
223 } 316 }
224 317
225 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE)) 318 if ((vma->vm_flags & (VM_SHARED|VM_WRITE)) != (VM_SHARED|VM_WRITE))
226 return -EACCES; 319 return -EACCES;
227 320
228 offset = (loff_t)(start - vma->vm_start) 321 offset = (loff_t)(start - vma->vm_start)
229 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT); 322 + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
230 323
231 /* 324 /*
232 * Filesystem's fallocate may need to take i_mutex. We need to 325 * Filesystem's fallocate may need to take i_mutex. We need to
233 * explicitly grab a reference because the vma (and hence the 326 * explicitly grab a reference because the vma (and hence the
234 * vma's reference to the file) can go away as soon as we drop 327 * vma's reference to the file) can go away as soon as we drop
235 * mmap_sem. 328 * mmap_sem.
236 */ 329 */
237 get_file(f); 330 get_file(f);
238 up_read(&current->mm->mmap_sem); 331 up_read(&current->mm->mmap_sem);
239 error = do_fallocate(f, 332 error = do_fallocate(f,
240 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 333 FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
241 offset, end - start); 334 offset, end - start);
242 fput(f); 335 fput(f);
243 down_read(&current->mm->mmap_sem); 336 down_read(&current->mm->mmap_sem);
244 return error; 337 return error;
245 } 338 }
246 339
247 #ifdef CONFIG_MEMORY_FAILURE 340 #ifdef CONFIG_MEMORY_FAILURE
248 /* 341 /*
249 * Error injection support for memory error handling. 342 * Error injection support for memory error handling.
250 */ 343 */
251 static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end) 344 static int madvise_hwpoison(int bhv, unsigned long start, unsigned long end)
252 { 345 {
253 int ret = 0; 346 int ret = 0;
254 347
255 if (!capable(CAP_SYS_ADMIN)) 348 if (!capable(CAP_SYS_ADMIN))
256 return -EPERM; 349 return -EPERM;
257 for (; start < end; start += PAGE_SIZE) { 350 for (; start < end; start += PAGE_SIZE) {
258 struct page *p; 351 struct page *p;
259 int ret = get_user_pages_fast(start, 1, 0, &p); 352 int ret = get_user_pages_fast(start, 1, 0, &p);
260 if (ret != 1) 353 if (ret != 1)
261 return ret; 354 return ret;
262 if (bhv == MADV_SOFT_OFFLINE) { 355 if (bhv == MADV_SOFT_OFFLINE) {
263 printk(KERN_INFO "Soft offlining page %lx at %lx\n", 356 printk(KERN_INFO "Soft offlining page %lx at %lx\n",
264 page_to_pfn(p), start); 357 page_to_pfn(p), start);
265 ret = soft_offline_page(p, MF_COUNT_INCREASED); 358 ret = soft_offline_page(p, MF_COUNT_INCREASED);
266 if (ret) 359 if (ret)
267 break; 360 break;
268 continue; 361 continue;
269 } 362 }
270 printk(KERN_INFO "Injecting memory failure for page %lx at %lx\n", 363 printk(KERN_INFO "Injecting memory failure for page %lx at %lx\n",
271 page_to_pfn(p), start); 364 page_to_pfn(p), start);
272 /* Ignore return value for now */ 365 /* Ignore return value for now */
273 memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED); 366 memory_failure(page_to_pfn(p), 0, MF_COUNT_INCREASED);
274 } 367 }
275 return ret; 368 return ret;
276 } 369 }
277 #endif 370 #endif
278 371
279 static long 372 static long
280 madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev, 373 madvise_vma(struct vm_area_struct *vma, struct vm_area_struct **prev,
281 unsigned long start, unsigned long end, int behavior) 374 unsigned long start, unsigned long end, int behavior)
282 { 375 {
283 switch (behavior) { 376 switch (behavior) {
284 case MADV_REMOVE: 377 case MADV_REMOVE:
285 return madvise_remove(vma, prev, start, end); 378 return madvise_remove(vma, prev, start, end);
286 case MADV_WILLNEED: 379 case MADV_WILLNEED:
287 return madvise_willneed(vma, prev, start, end); 380 return madvise_willneed(vma, prev, start, end);
288 case MADV_DONTNEED: 381 case MADV_DONTNEED:
289 return madvise_dontneed(vma, prev, start, end); 382 return madvise_dontneed(vma, prev, start, end);
290 default: 383 default:
291 return madvise_behavior(vma, prev, start, end, behavior); 384 return madvise_behavior(vma, prev, start, end, behavior);
292 } 385 }
293 } 386 }
294 387
295 static int 388 static int
296 madvise_behavior_valid(int behavior) 389 madvise_behavior_valid(int behavior)
297 { 390 {
298 switch (behavior) { 391 switch (behavior) {
299 case MADV_DOFORK: 392 case MADV_DOFORK:
300 case MADV_DONTFORK: 393 case MADV_DONTFORK:
301 case MADV_NORMAL: 394 case MADV_NORMAL:
302 case MADV_SEQUENTIAL: 395 case MADV_SEQUENTIAL:
303 case MADV_RANDOM: 396 case MADV_RANDOM:
304 case MADV_REMOVE: 397 case MADV_REMOVE:
305 case MADV_WILLNEED: 398 case MADV_WILLNEED:
306 case MADV_DONTNEED: 399 case MADV_DONTNEED:
307 #ifdef CONFIG_KSM 400 #ifdef CONFIG_KSM
308 case MADV_MERGEABLE: 401 case MADV_MERGEABLE:
309 case MADV_UNMERGEABLE: 402 case MADV_UNMERGEABLE:
310 #endif 403 #endif
311 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 404 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
312 case MADV_HUGEPAGE: 405 case MADV_HUGEPAGE:
313 case MADV_NOHUGEPAGE: 406 case MADV_NOHUGEPAGE:
314 #endif 407 #endif
315 case MADV_DONTDUMP: 408 case MADV_DONTDUMP:
316 case MADV_DODUMP: 409 case MADV_DODUMP:
317 return 1; 410 return 1;
318 411
319 default: 412 default:
320 return 0; 413 return 0;
321 } 414 }
322 } 415 }
323 416
324 /* 417 /*
325 * The madvise(2) system call. 418 * The madvise(2) system call.
326 * 419 *
327 * Applications can use madvise() to advise the kernel how it should 420 * Applications can use madvise() to advise the kernel how it should
328 * handle paging I/O in this VM area. The idea is to help the kernel 421 * handle paging I/O in this VM area. The idea is to help the kernel
329 * use appropriate read-ahead and caching techniques. The information 422 * use appropriate read-ahead and caching techniques. The information
330 * provided is advisory only, and can be safely disregarded by the 423 * provided is advisory only, and can be safely disregarded by the
331 * kernel without affecting the correct operation of the application. 424 * kernel without affecting the correct operation of the application.
332 * 425 *
333 * behavior values: 426 * behavior values:
334 * MADV_NORMAL - the default behavior is to read clusters. This 427 * MADV_NORMAL - the default behavior is to read clusters. This
335 * results in some read-ahead and read-behind. 428 * results in some read-ahead and read-behind.
336 * MADV_RANDOM - the system should read the minimum amount of data 429 * MADV_RANDOM - the system should read the minimum amount of data
337 * on any access, since it is unlikely that the appli- 430 * on any access, since it is unlikely that the appli-
338 * cation will need more than what it asks for. 431 * cation will need more than what it asks for.
339 * MADV_SEQUENTIAL - pages in the given range will probably be accessed 432 * MADV_SEQUENTIAL - pages in the given range will probably be accessed
340 * once, so they can be aggressively read ahead, and 433 * once, so they can be aggressively read ahead, and
341 * can be freed soon after they are accessed. 434 * can be freed soon after they are accessed.
342 * MADV_WILLNEED - the application is notifying the system to read 435 * MADV_WILLNEED - the application is notifying the system to read
343 * some pages ahead. 436 * some pages ahead.
344 * MADV_DONTNEED - the application is finished with the given range, 437 * MADV_DONTNEED - the application is finished with the given range,
345 * so the kernel can free resources associated with it. 438 * so the kernel can free resources associated with it.
346 * MADV_REMOVE - the application wants to free up the given range of 439 * MADV_REMOVE - the application wants to free up the given range of
347 * pages and associated backing store. 440 * pages and associated backing store.
348 * MADV_DONTFORK - omit this area from child's address space when forking: 441 * MADV_DONTFORK - omit this area from child's address space when forking:
349 * typically, to avoid COWing pages pinned by get_user_pages(). 442 * typically, to avoid COWing pages pinned by get_user_pages().
350 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking. 443 * MADV_DOFORK - cancel MADV_DONTFORK: no longer omit this area when forking.
351 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in 444 * MADV_MERGEABLE - the application recommends that KSM try to merge pages in
352 * this area with pages of identical content from other such areas. 445 * this area with pages of identical content from other such areas.
353 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others. 446 * MADV_UNMERGEABLE- cancel MADV_MERGEABLE: no longer merge pages with others.
354 * 447 *
355 * return values: 448 * return values:
356 * zero - success 449 * zero - success
357 * -EINVAL - start + len < 0, start is not page-aligned, 450 * -EINVAL - start + len < 0, start is not page-aligned,
358 * "behavior" is not a valid value, or application 451 * "behavior" is not a valid value, or application
359 * is attempting to release locked or shared pages. 452 * is attempting to release locked or shared pages.
360 * -ENOMEM - addresses in the specified range are not currently 453 * -ENOMEM - addresses in the specified range are not currently
361 * mapped, or are outside the AS of the process. 454 * mapped, or are outside the AS of the process.
362 * -EIO - an I/O error occurred while paging in data. 455 * -EIO - an I/O error occurred while paging in data.
363 * -EBADF - map exists, but area maps something that isn't a file. 456 * -EBADF - map exists, but area maps something that isn't a file.
364 * -EAGAIN - a kernel resource was temporarily unavailable. 457 * -EAGAIN - a kernel resource was temporarily unavailable.
365 */ 458 */
366 SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior) 459 SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
367 { 460 {
368 unsigned long end, tmp; 461 unsigned long end, tmp;
369 struct vm_area_struct * vma, *prev; 462 struct vm_area_struct * vma, *prev;
370 int unmapped_error = 0; 463 int unmapped_error = 0;
371 int error = -EINVAL; 464 int error = -EINVAL;
372 int write; 465 int write;
373 size_t len; 466 size_t len;
467 struct blk_plug plug;
374 468
375 #ifdef CONFIG_MEMORY_FAILURE 469 #ifdef CONFIG_MEMORY_FAILURE
376 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE) 470 if (behavior == MADV_HWPOISON || behavior == MADV_SOFT_OFFLINE)
377 return madvise_hwpoison(behavior, start, start+len_in); 471 return madvise_hwpoison(behavior, start, start+len_in);
378 #endif 472 #endif
379 if (!madvise_behavior_valid(behavior)) 473 if (!madvise_behavior_valid(behavior))
380 return error; 474 return error;
381 475
382 write = madvise_need_mmap_write(behavior); 476 write = madvise_need_mmap_write(behavior);
383 if (write) 477 if (write)
384 down_write(&current->mm->mmap_sem); 478 down_write(&current->mm->mmap_sem);
385 else 479 else
386 down_read(&current->mm->mmap_sem); 480 down_read(&current->mm->mmap_sem);
387 481
388 if (start & ~PAGE_MASK) 482 if (start & ~PAGE_MASK)
389 goto out; 483 goto out;
390 len = (len_in + ~PAGE_MASK) & PAGE_MASK; 484 len = (len_in + ~PAGE_MASK) & PAGE_MASK;
391 485
392 /* Check to see whether len was rounded up from small -ve to zero */ 486 /* Check to see whether len was rounded up from small -ve to zero */
393 if (len_in && !len) 487 if (len_in && !len)
394 goto out; 488 goto out;
395 489
396 end = start + len; 490 end = start + len;
397 if (end < start) 491 if (end < start)
398 goto out; 492 goto out;
399 493
400 error = 0; 494 error = 0;
401 if (end == start) 495 if (end == start)
402 goto out; 496 goto out;
403 497
404 /* 498 /*
405 * If the interval [start,end) covers some unmapped address 499 * If the interval [start,end) covers some unmapped address
406 * ranges, just ignore them, but return -ENOMEM at the end. 500 * ranges, just ignore them, but return -ENOMEM at the end.
407 * - different from the way of handling in mlock etc. 501 * - different from the way of handling in mlock etc.
408 */ 502 */
409 vma = find_vma_prev(current->mm, start, &prev); 503 vma = find_vma_prev(current->mm, start, &prev);
410 if (vma && start > vma->vm_start) 504 if (vma && start > vma->vm_start)
411 prev = vma; 505 prev = vma;
412 506
507 blk_start_plug(&plug);
413 for (;;) { 508 for (;;) {
414 /* Still start < end. */ 509 /* Still start < end. */
415 error = -ENOMEM; 510 error = -ENOMEM;
416 if (!vma) 511 if (!vma)
417 goto out; 512 goto out_plug;
418 513
419 /* Here start < (end|vma->vm_end). */ 514 /* Here start < (end|vma->vm_end). */
420 if (start < vma->vm_start) { 515 if (start < vma->vm_start) {
421 unmapped_error = -ENOMEM; 516 unmapped_error = -ENOMEM;
422 start = vma->vm_start; 517 start = vma->vm_start;
423 if (start >= end) 518 if (start >= end)
424 goto out; 519 goto out_plug;
425 } 520 }
426 521
427 /* Here vma->vm_start <= start < (end|vma->vm_end) */ 522 /* Here vma->vm_start <= start < (end|vma->vm_end) */
428 tmp = vma->vm_end; 523 tmp = vma->vm_end;
429 if (end < tmp) 524 if (end < tmp)
430 tmp = end; 525 tmp = end;
431 526
432 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */ 527 /* Here vma->vm_start <= start < tmp <= (end|vma->vm_end). */
433 error = madvise_vma(vma, &prev, start, tmp, behavior); 528 error = madvise_vma(vma, &prev, start, tmp, behavior);
434 if (error) 529 if (error)
435 goto out; 530 goto out_plug;
436 start = tmp; 531 start = tmp;
437 if (prev && start < prev->vm_end) 532 if (prev && start < prev->vm_end)
438 start = prev->vm_end; 533 start = prev->vm_end;
439 error = unmapped_error; 534 error = unmapped_error;
440 if (start >= end) 535 if (start >= end)
441 goto out; 536 goto out_plug;
442 if (prev) 537 if (prev)
443 vma = prev->vm_next; 538 vma = prev->vm_next;
444 else /* madvise_remove dropped mmap_sem */ 539 else /* madvise_remove dropped mmap_sem */
445 vma = find_vma(current->mm, start); 540 vma = find_vma(current->mm, start);
446 } 541 }
542 out_plug:
543 blk_finish_plug(&plug);
447 out: 544 out:
448 if (write) 545 if (write)
449 up_write(&current->mm->mmap_sem); 546 up_write(&current->mm->mmap_sem);
450 else 547 else
451 up_read(&current->mm->mmap_sem); 548 up_read(&current->mm->mmap_sem);
452 549
453 return error; 550 return error;
454 } 551 }
455 552