Blame view
mm/truncate.c
23.7 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 5 |
/* * mm/truncate.c - code for taking down pages from address_spaces * * Copyright (C) 2002, Linus Torvalds * |
e1f8e8744 Remove Andrew Mor... |
6 |
* 10Sep2002 Andrew Morton |
1da177e4c Linux-2.6.12-rc2 |
7 8 9 10 |
* Initial version. */ #include <linux/kernel.h> |
4af3c9cc4 Drop some headers... |
11 |
#include <linux/backing-dev.h> |
5a0e3ad6a include cleanup: ... |
12 |
#include <linux/gfp.h> |
1da177e4c Linux-2.6.12-rc2 |
13 |
#include <linux/mm.h> |
0fd0e6b05 [PATCH] page inva... |
14 |
#include <linux/swap.h> |
b95f1b31b mm: Map most file... |
15 |
#include <linux/export.h> |
1da177e4c Linux-2.6.12-rc2 |
16 |
#include <linux/pagemap.h> |
01f2705da fs: convert core ... |
17 |
#include <linux/highmem.h> |
1da177e4c Linux-2.6.12-rc2 |
18 |
#include <linux/pagevec.h> |
e08748ce0 [PATCH] io-accoun... |
19 |
#include <linux/task_io_accounting_ops.h> |
1da177e4c Linux-2.6.12-rc2 |
20 |
#include <linux/buffer_head.h> /* grr. try_to_release_page, |
aaa4059bc [PATCH] ext3: Fix... |
21 |
do_invalidatepage */ |
c515e1fd3 mm/fs: add hooks ... |
22 |
#include <linux/cleancache.h> |
90a802027 vfs: fix data cor... |
23 |
#include <linux/rmap.h> |
ba470de43 mmap: handle mloc... |
24 |
#include "internal.h" |
1da177e4c Linux-2.6.12-rc2 |
25 |
|
0cd6144aa mm + fs: prepare ... |
26 27 28 |
static void clear_exceptional_entry(struct address_space *mapping, pgoff_t index, void *entry) { |
449dd6984 mm: keep page cac... |
29 30 |
struct radix_tree_node *node; void **slot; |
0cd6144aa mm + fs: prepare ... |
31 32 33 34 35 36 37 38 39 40 |
/* Handled by shmem itself */ if (shmem_mapping(mapping)) return; spin_lock_irq(&mapping->tree_lock); /* * Regular page slots are stabilized by the page lock even * without the tree itself locked. These unlocked entries * need verification under the tree lock. */ |
449dd6984 mm: keep page cac... |
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
if (!__radix_tree_lookup(&mapping->page_tree, index, &node, &slot)) goto unlock; if (*slot != entry) goto unlock; radix_tree_replace_slot(slot, NULL); mapping->nrshadows--; if (!node) goto unlock; workingset_node_shadows_dec(node); /* * Don't track node without shadow entries. * * Avoid acquiring the list_lru lock if already untracked. * The list_empty() test is safe as node->private_list is * protected by mapping->tree_lock. */ if (!workingset_node_shadows(node) && !list_empty(&node->private_list)) list_lru_del(&workingset_shadow_nodes, &node->private_list); __radix_tree_delete_node(&mapping->page_tree, node); unlock: |
0cd6144aa mm + fs: prepare ... |
62 63 |
spin_unlock_irq(&mapping->tree_lock); } |
1da177e4c Linux-2.6.12-rc2 |
64 |
|
cf9a2ae8d [PATCH] BLOCK: Mo... |
65 |
/** |
28bc44d7d do_invalidatepage... |
66 |
* do_invalidatepage - invalidate part or all of a page |
cf9a2ae8d [PATCH] BLOCK: Mo... |
67 |
* @page: the page which is affected |
d47992f86 mm: change invali... |
68 69 |
* @offset: start of the range to invalidate * @length: length of the range to invalidate |
cf9a2ae8d [PATCH] BLOCK: Mo... |
70 71 72 73 74 75 76 77 78 79 |
* * do_invalidatepage() is called when all or part of the page has become * invalidated by a truncate operation. * * do_invalidatepage() does not have to release all buffers, but it must * ensure that no dirty buffer is left outside @offset and that no I/O * is underway against any of the blocks which are outside the truncation * point. Because the caller is about to free (and possibly reuse) those * blocks on-disk. */ |
d47992f86 mm: change invali... |
80 81 |
void do_invalidatepage(struct page *page, unsigned int offset, unsigned int length) |
cf9a2ae8d [PATCH] BLOCK: Mo... |
82 |
{ |
d47992f86 mm: change invali... |
83 |
void (*invalidatepage)(struct page *, unsigned int, unsigned int); |
cf9a2ae8d [PATCH] BLOCK: Mo... |
84 |
invalidatepage = page->mapping->a_ops->invalidatepage; |
9361401eb [PATCH] BLOCK: Ma... |
85 |
#ifdef CONFIG_BLOCK |
cf9a2ae8d [PATCH] BLOCK: Mo... |
86 87 |
if (!invalidatepage) invalidatepage = block_invalidatepage; |
9361401eb [PATCH] BLOCK: Ma... |
88 |
#endif |
cf9a2ae8d [PATCH] BLOCK: Mo... |
89 |
if (invalidatepage) |
d47992f86 mm: change invali... |
90 |
(*invalidatepage)(page, offset, length); |
cf9a2ae8d [PATCH] BLOCK: Mo... |
91 |
} |
ecdfc9787 Resurrect 'try_to... |
92 |
/* |
1da177e4c Linux-2.6.12-rc2 |
93 |
* If truncate cannot remove the fs-private metadata from the page, the page |
62e1c5530 page migraton: ha... |
94 |
* becomes orphaned. It will be left on the LRU and may even be mapped into |
54cb8821d mm: merge populat... |
95 |
* user pagetables if we're racing with filemap_fault(). |
1da177e4c Linux-2.6.12-rc2 |
96 97 98 |
* * We need to bale out if page->mapping is no longer equal to the original * mapping. This happens a) when the VM reclaimed the page while we waited on |
fc0ecff69 [PATCH] remove in... |
99 |
* its lock, b) when a concurrent invalidate_mapping_pages got there first and |
1da177e4c Linux-2.6.12-rc2 |
100 101 |
* c) when tmpfs swizzles a page between a tmpfs inode and swapper_space. */ |
750b4987b HWPOISON: Refacto... |
102 |
static int |
1da177e4c Linux-2.6.12-rc2 |
103 104 105 |
truncate_complete_page(struct address_space *mapping, struct page *page) { if (page->mapping != mapping) |
750b4987b HWPOISON: Refacto... |
106 |
return -EIO; |
1da177e4c Linux-2.6.12-rc2 |
107 |
|
266cf658e FS-Cache: Recruit... |
108 |
if (page_has_private(page)) |
d47992f86 mm: change invali... |
109 |
do_invalidatepage(page, 0, PAGE_CACHE_SIZE); |
1da177e4c Linux-2.6.12-rc2 |
110 |
|
b9ea25152 page_writeback: c... |
111 112 113 114 115 |
/* * Some filesystems seem to re-dirty the page even after * the VM has canceled the dirty bit (eg ext3 journaling). * Hence dirty accounting check is placed after invalidation. */ |
11f81becc page_writeback: r... |
116 |
cancel_dirty_page(page); |
1da177e4c Linux-2.6.12-rc2 |
117 |
ClearPageMappedToDisk(page); |
5adc7b518 mm: truncate: cha... |
118 |
delete_from_page_cache(page); |
750b4987b HWPOISON: Refacto... |
119 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
120 121 122 |
} /* |
fc0ecff69 [PATCH] remove in... |
123 |
* This is for invalidate_mapping_pages(). That function can be called at |
1da177e4c Linux-2.6.12-rc2 |
124 |
* any time, and is not supposed to throw away dirty pages. But pages can |
0fd0e6b05 [PATCH] page inva... |
125 126 |
* be marked dirty at any time too, so use remove_mapping which safely * discards clean, unused pages. |
1da177e4c Linux-2.6.12-rc2 |
127 128 129 130 131 132 |
* * Returns non-zero if the page was successfully invalidated. */ static int invalidate_complete_page(struct address_space *mapping, struct page *page) { |
0fd0e6b05 [PATCH] page inva... |
133 |
int ret; |
1da177e4c Linux-2.6.12-rc2 |
134 135 |
if (page->mapping != mapping) return 0; |
266cf658e FS-Cache: Recruit... |
136 |
if (page_has_private(page) && !try_to_release_page(page, 0)) |
1da177e4c Linux-2.6.12-rc2 |
137 |
return 0; |
0fd0e6b05 [PATCH] page inva... |
138 |
ret = remove_mapping(mapping, page); |
0fd0e6b05 [PATCH] page inva... |
139 140 |
return ret; |
1da177e4c Linux-2.6.12-rc2 |
141 |
} |
750b4987b HWPOISON: Refacto... |
142 143 144 145 146 147 148 149 150 |
int truncate_inode_page(struct address_space *mapping, struct page *page) { if (page_mapped(page)) { unmap_mapping_range(mapping, (loff_t)page->index << PAGE_CACHE_SHIFT, PAGE_CACHE_SIZE, 0); } return truncate_complete_page(mapping, page); } |
83f786680 HWPOISON: Add inv... |
151 |
/* |
257187362 HWPOISON: Define ... |
152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 |
* Used to get rid of pages on hardware memory corruption. */ int generic_error_remove_page(struct address_space *mapping, struct page *page) { if (!mapping) return -EINVAL; /* * Only punch for normal data pages for now. * Handling other types like directories would need more auditing. */ if (!S_ISREG(mapping->host->i_mode)) return -EIO; return truncate_inode_page(mapping, page); } EXPORT_SYMBOL(generic_error_remove_page); /* |
83f786680 HWPOISON: Add inv... |
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 |
* Safely invalidate one page from its pagecache mapping. * It only drops clean, unused pages. The page must be locked. * * Returns 1 if the page is successfully invalidated, otherwise 0. */ int invalidate_inode_page(struct page *page) { struct address_space *mapping = page_mapping(page); if (!mapping) return 0; if (PageDirty(page) || PageWriteback(page)) return 0; if (page_mapped(page)) return 0; return invalidate_complete_page(mapping, page); } |
1da177e4c Linux-2.6.12-rc2 |
185 |
/** |
73c1e2043 mm: fix comment t... |
186 |
* truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets |
1da177e4c Linux-2.6.12-rc2 |
187 188 |
* @mapping: mapping to truncate * @lstart: offset from which to truncate |
5a7203947 mm: teach truncat... |
189 |
* @lend: offset to which to truncate (inclusive) |
1da177e4c Linux-2.6.12-rc2 |
190 |
* |
d7339071f [PATCH] reiser4: ... |
191 |
* Truncate the page cache, removing the pages that are between |
5a7203947 mm: teach truncat... |
192 193 |
* specified offsets (and zeroing out partial pages * if lstart or lend + 1 is not page aligned). |
1da177e4c Linux-2.6.12-rc2 |
194 195 196 197 198 199 200 |
* * Truncate takes two passes - the first pass is nonblocking. It will not * block on page locks and it will not block on writeback. The second pass * will wait. This is to prevent as much IO as possible in the affected region. * The first pass will remove most pages, so the search cost of the second pass * is low. * |
1da177e4c Linux-2.6.12-rc2 |
201 202 203 |
* We pass down the cache-hot hint to the page freeing code. Even if the * mapping is large, it is probably the case that the final pages are the most * recently touched, and freeing happens in ascending file offset order. |
5a7203947 mm: teach truncat... |
204 205 206 207 |
* * Note that since ->invalidatepage() accepts range to invalidate * truncate_inode_pages_range is able to handle cases where lend + 1 is not * page aligned properly. |
1da177e4c Linux-2.6.12-rc2 |
208 |
*/ |
d7339071f [PATCH] reiser4: ... |
209 210 |
void truncate_inode_pages_range(struct address_space *mapping, loff_t lstart, loff_t lend) |
1da177e4c Linux-2.6.12-rc2 |
211 |
{ |
5a7203947 mm: teach truncat... |
212 213 214 215 216 |
pgoff_t start; /* inclusive */ pgoff_t end; /* exclusive */ unsigned int partial_start; /* inclusive */ unsigned int partial_end; /* exclusive */ struct pagevec pvec; |
0cd6144aa mm + fs: prepare ... |
217 |
pgoff_t indices[PAGEVEC_SIZE]; |
5a7203947 mm: teach truncat... |
218 219 |
pgoff_t index; int i; |
1da177e4c Linux-2.6.12-rc2 |
220 |
|
3167760f8 mm: cleancache: s... |
221 |
cleancache_invalidate_inode(mapping); |
91b0abe36 mm + fs: store sh... |
222 |
if (mapping->nrpages == 0 && mapping->nrshadows == 0) |
1da177e4c Linux-2.6.12-rc2 |
223 |
return; |
5a7203947 mm: teach truncat... |
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 |
/* Offsets within partial pages */ partial_start = lstart & (PAGE_CACHE_SIZE - 1); partial_end = (lend + 1) & (PAGE_CACHE_SIZE - 1); /* * 'start' and 'end' always covers the range of pages to be fully * truncated. Partial pages are covered with 'partial_start' at the * start of the range and 'partial_end' at the end of the range. * Note that 'end' is exclusive while 'lend' is inclusive. */ start = (lstart + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; if (lend == -1) /* * lend == -1 indicates end-of-file so we have to set 'end' * to the highest possible pgoff_t and since the type is * unsigned we're using -1. */ end = -1; else end = (lend + 1) >> PAGE_CACHE_SHIFT; |
d7339071f [PATCH] reiser4: ... |
244 |
|
1da177e4c Linux-2.6.12-rc2 |
245 |
pagevec_init(&pvec, 0); |
b85e0effd mm: consistent tr... |
246 |
index = start; |
0cd6144aa mm + fs: prepare ... |
247 248 249 |
while (index < end && pagevec_lookup_entries(&pvec, mapping, index, min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) { |
1da177e4c Linux-2.6.12-rc2 |
250 251 |
for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; |
1da177e4c Linux-2.6.12-rc2 |
252 |
|
b85e0effd mm: consistent tr... |
253 |
/* We rely upon deletion not changing page->index */ |
0cd6144aa mm + fs: prepare ... |
254 |
index = indices[i]; |
5a7203947 mm: teach truncat... |
255 |
if (index >= end) |
d7339071f [PATCH] reiser4: ... |
256 |
break; |
d7339071f [PATCH] reiser4: ... |
257 |
|
0cd6144aa mm + fs: prepare ... |
258 259 260 261 |
if (radix_tree_exceptional_entry(page)) { clear_exceptional_entry(mapping, index, page); continue; } |
529ae9aaa mm: rename page t... |
262 |
if (!trylock_page(page)) |
1da177e4c Linux-2.6.12-rc2 |
263 |
continue; |
b85e0effd mm: consistent tr... |
264 |
WARN_ON(page->index != index); |
1da177e4c Linux-2.6.12-rc2 |
265 266 267 268 |
if (PageWriteback(page)) { unlock_page(page); continue; } |
750b4987b HWPOISON: Refacto... |
269 |
truncate_inode_page(mapping, page); |
1da177e4c Linux-2.6.12-rc2 |
270 271 |
unlock_page(page); } |
0cd6144aa mm + fs: prepare ... |
272 |
pagevec_remove_exceptionals(&pvec); |
1da177e4c Linux-2.6.12-rc2 |
273 274 |
pagevec_release(&pvec); cond_resched(); |
b85e0effd mm: consistent tr... |
275 |
index++; |
1da177e4c Linux-2.6.12-rc2 |
276 |
} |
5a7203947 mm: teach truncat... |
277 |
if (partial_start) { |
1da177e4c Linux-2.6.12-rc2 |
278 279 |
struct page *page = find_lock_page(mapping, start - 1); if (page) { |
5a7203947 mm: teach truncat... |
280 281 282 283 284 285 |
unsigned int top = PAGE_CACHE_SIZE; if (start > end) { /* Truncation within a single page */ top = partial_end; partial_end = 0; } |
1da177e4c Linux-2.6.12-rc2 |
286 |
wait_on_page_writeback(page); |
5a7203947 mm: teach truncat... |
287 288 289 290 291 |
zero_user_segment(page, partial_start, top); cleancache_invalidate_page(mapping, page); if (page_has_private(page)) do_invalidatepage(page, partial_start, top - partial_start); |
1da177e4c Linux-2.6.12-rc2 |
292 293 294 295 |
unlock_page(page); page_cache_release(page); } } |
5a7203947 mm: teach truncat... |
296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 |
if (partial_end) { struct page *page = find_lock_page(mapping, end); if (page) { wait_on_page_writeback(page); zero_user_segment(page, 0, partial_end); cleancache_invalidate_page(mapping, page); if (page_has_private(page)) do_invalidatepage(page, 0, partial_end); unlock_page(page); page_cache_release(page); } } /* * If the truncation happened within a single page no pages * will be released, just zeroed, so we can bail out now. */ if (start >= end) return; |
1da177e4c Linux-2.6.12-rc2 |
315 |
|
b85e0effd mm: consistent tr... |
316 |
index = start; |
1da177e4c Linux-2.6.12-rc2 |
317 318 |
for ( ; ; ) { cond_resched(); |
0cd6144aa mm + fs: prepare ... |
319 |
if (!pagevec_lookup_entries(&pvec, mapping, index, |
792ceaefe mm/fs: fix pessim... |
320 321 |
min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) { /* If all gone from start onwards, we're done */ |
b85e0effd mm: consistent tr... |
322 |
if (index == start) |
1da177e4c Linux-2.6.12-rc2 |
323 |
break; |
792ceaefe mm/fs: fix pessim... |
324 |
/* Otherwise restart to make sure all gone */ |
b85e0effd mm: consistent tr... |
325 |
index = start; |
1da177e4c Linux-2.6.12-rc2 |
326 327 |
continue; } |
0cd6144aa mm + fs: prepare ... |
328 |
if (index == start && indices[0] >= end) { |
792ceaefe mm/fs: fix pessim... |
329 |
/* All gone out of hole to be punched, we're done */ |
0cd6144aa mm + fs: prepare ... |
330 |
pagevec_remove_exceptionals(&pvec); |
d7339071f [PATCH] reiser4: ... |
331 332 333 |
pagevec_release(&pvec); break; } |
1da177e4c Linux-2.6.12-rc2 |
334 335 |
for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; |
b85e0effd mm: consistent tr... |
336 |
/* We rely upon deletion not changing page->index */ |
0cd6144aa mm + fs: prepare ... |
337 |
index = indices[i]; |
792ceaefe mm/fs: fix pessim... |
338 339 340 |
if (index >= end) { /* Restart punch to make sure all gone */ index = start - 1; |
d7339071f [PATCH] reiser4: ... |
341 |
break; |
792ceaefe mm/fs: fix pessim... |
342 |
} |
b85e0effd mm: consistent tr... |
343 |
|
0cd6144aa mm + fs: prepare ... |
344 345 346 347 |
if (radix_tree_exceptional_entry(page)) { clear_exceptional_entry(mapping, index, page); continue; } |
1da177e4c Linux-2.6.12-rc2 |
348 |
lock_page(page); |
b85e0effd mm: consistent tr... |
349 |
WARN_ON(page->index != index); |
1da177e4c Linux-2.6.12-rc2 |
350 |
wait_on_page_writeback(page); |
750b4987b HWPOISON: Refacto... |
351 |
truncate_inode_page(mapping, page); |
1da177e4c Linux-2.6.12-rc2 |
352 353 |
unlock_page(page); } |
0cd6144aa mm + fs: prepare ... |
354 |
pagevec_remove_exceptionals(&pvec); |
1da177e4c Linux-2.6.12-rc2 |
355 |
pagevec_release(&pvec); |
b85e0effd mm: consistent tr... |
356 |
index++; |
1da177e4c Linux-2.6.12-rc2 |
357 |
} |
3167760f8 mm: cleancache: s... |
358 |
cleancache_invalidate_inode(mapping); |
1da177e4c Linux-2.6.12-rc2 |
359 |
} |
d7339071f [PATCH] reiser4: ... |
360 |
EXPORT_SYMBOL(truncate_inode_pages_range); |
1da177e4c Linux-2.6.12-rc2 |
361 |
|
d7339071f [PATCH] reiser4: ... |
362 363 364 365 366 |
/** * truncate_inode_pages - truncate *all* the pages from an offset * @mapping: mapping to truncate * @lstart: offset from which to truncate * |
1b1dcc1b5 [PATCH] mutex sub... |
367 |
* Called under (and serialised by) inode->i_mutex. |
08142579b mm: fix assertion... |
368 369 370 371 372 |
* * Note: When this function returns, there can be a page in the process of * deletion (inside __delete_from_page_cache()) in the specified range. Thus * mapping->nrpages can be non-zero when this function returns even after * truncation of the whole mapping. |
d7339071f [PATCH] reiser4: ... |
373 374 375 376 377 |
*/ void truncate_inode_pages(struct address_space *mapping, loff_t lstart) { truncate_inode_pages_range(mapping, lstart, (loff_t)-1); } |
1da177e4c Linux-2.6.12-rc2 |
378 |
EXPORT_SYMBOL(truncate_inode_pages); |
286973552 mm: remove __inva... |
379 |
/** |
91b0abe36 mm + fs: store sh... |
380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 |
* truncate_inode_pages_final - truncate *all* pages before inode dies * @mapping: mapping to truncate * * Called under (and serialized by) inode->i_mutex. * * Filesystems have to use this in the .evict_inode path to inform the * VM that this is the final truncate and the inode is going away. */ void truncate_inode_pages_final(struct address_space *mapping) { unsigned long nrshadows; unsigned long nrpages; /* * Page reclaim can not participate in regular inode lifetime * management (can't call iput()) and thus can race with the * inode teardown. Tell it when the address space is exiting, * so that it does not install eviction information after the * final truncate has begun. */ mapping_set_exiting(mapping); /* * When reclaim installs eviction entries, it increases * nrshadows first, then decreases nrpages. Make sure we see * this in the right order or we might miss an entry. */ nrpages = mapping->nrpages; smp_rmb(); nrshadows = mapping->nrshadows; if (nrpages || nrshadows) { /* * As truncation uses a lockless tree lookup, cycle * the tree lock to make sure any ongoing tree * modification that does not see AS_EXITING is * completed before starting the final truncate. */ spin_lock_irq(&mapping->tree_lock); spin_unlock_irq(&mapping->tree_lock); truncate_inode_pages(mapping, 0); } } EXPORT_SYMBOL(truncate_inode_pages_final); /** |
286973552 mm: remove __inva... |
427 428 429 430 431 432 433 434 435 436 437 438 439 |
* invalidate_mapping_pages - Invalidate all the unlocked pages of one inode * @mapping: the address_space which holds the pages to invalidate * @start: the offset 'from' which to invalidate * @end: the offset 'to' which to invalidate (inclusive) * * This function only removes the unlocked pages, if you want to * remove all the pages of one inode, you must call truncate_inode_pages. * * invalidate_mapping_pages() will not block on IO activity. It will not * invalidate pages which are dirty, locked, under writeback or mapped into * pagetables. */ unsigned long invalidate_mapping_pages(struct address_space *mapping, |
315601809 mm: deactivate in... |
440 |
pgoff_t start, pgoff_t end) |
1da177e4c Linux-2.6.12-rc2 |
441 |
{ |
0cd6144aa mm + fs: prepare ... |
442 |
pgoff_t indices[PAGEVEC_SIZE]; |
1da177e4c Linux-2.6.12-rc2 |
443 |
struct pagevec pvec; |
b85e0effd mm: consistent tr... |
444 |
pgoff_t index = start; |
315601809 mm: deactivate in... |
445 446 |
unsigned long ret; unsigned long count = 0; |
1da177e4c Linux-2.6.12-rc2 |
447 448 449 |
int i; pagevec_init(&pvec, 0); |
0cd6144aa mm + fs: prepare ... |
450 451 452 |
while (index <= end && pagevec_lookup_entries(&pvec, mapping, index, min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1, indices)) { |
1da177e4c Linux-2.6.12-rc2 |
453 454 |
for (i = 0; i < pagevec_count(&pvec); i++) { struct page *page = pvec.pages[i]; |
e0f23603f [PATCH] Remove se... |
455 |
|
b85e0effd mm: consistent tr... |
456 |
/* We rely upon deletion not changing page->index */ |
0cd6144aa mm + fs: prepare ... |
457 |
index = indices[i]; |
b85e0effd mm: consistent tr... |
458 459 |
if (index > end) break; |
e0f23603f [PATCH] Remove se... |
460 |
|
0cd6144aa mm + fs: prepare ... |
461 462 463 464 |
if (radix_tree_exceptional_entry(page)) { clear_exceptional_entry(mapping, index, page); continue; } |
b85e0effd mm: consistent tr... |
465 466 467 |
if (!trylock_page(page)) continue; WARN_ON(page->index != index); |
315601809 mm: deactivate in... |
468 |
ret = invalidate_inode_page(page); |
1da177e4c Linux-2.6.12-rc2 |
469 |
unlock_page(page); |
315601809 mm: deactivate in... |
470 471 472 473 474 |
/* * Invalidation is a hint that the page is no longer * of interest and try to speed up its reclaim. */ if (!ret) |
cc5993bd7 mm: rename deacti... |
475 |
deactivate_file_page(page); |
315601809 mm: deactivate in... |
476 |
count += ret; |
1da177e4c Linux-2.6.12-rc2 |
477 |
} |
0cd6144aa mm + fs: prepare ... |
478 |
pagevec_remove_exceptionals(&pvec); |
1da177e4c Linux-2.6.12-rc2 |
479 |
pagevec_release(&pvec); |
286973552 mm: remove __inva... |
480 |
cond_resched(); |
b85e0effd mm: consistent tr... |
481 |
index++; |
1da177e4c Linux-2.6.12-rc2 |
482 |
} |
315601809 mm: deactivate in... |
483 |
return count; |
1da177e4c Linux-2.6.12-rc2 |
484 |
} |
54bc48552 [PATCH] Export in... |
485 |
EXPORT_SYMBOL(invalidate_mapping_pages); |
1da177e4c Linux-2.6.12-rc2 |
486 |
|
bd4c8ce41 [PATCH] invalidat... |
487 488 489 490 |
/* * This is like invalidate_complete_page(), except it ignores the page's * refcount. We do this because invalidate_inode_pages2() needs stronger * invalidation guarantees, and cannot afford to leave pages behind because |
2706a1b89 vmscan: fix comme... |
491 492 |
* shrink_page_list() has a temp ref on them, or because they're transiently * sitting in the lru_cache_add() pagevecs. |
bd4c8ce41 [PATCH] invalidat... |
493 494 495 496 |
*/ static int invalidate_complete_page2(struct address_space *mapping, struct page *page) { |
c4843a759 memcg: add per cg... |
497 498 |
struct mem_cgroup *memcg; unsigned long flags; |
bd4c8ce41 [PATCH] invalidat... |
499 500 |
if (page->mapping != mapping) return 0; |
266cf658e FS-Cache: Recruit... |
501 |
if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) |
bd4c8ce41 [PATCH] invalidat... |
502 |
return 0; |
c4843a759 memcg: add per cg... |
503 504 |
memcg = mem_cgroup_begin_page_stat(page); spin_lock_irqsave(&mapping->tree_lock, flags); |
bd4c8ce41 [PATCH] invalidat... |
505 506 |
if (PageDirty(page)) goto failed; |
266cf658e FS-Cache: Recruit... |
507 |
BUG_ON(page_has_private(page)); |
c4843a759 memcg: add per cg... |
508 509 510 |
__delete_from_page_cache(page, NULL, memcg); spin_unlock_irqrestore(&mapping->tree_lock, flags); mem_cgroup_end_page_stat(memcg); |
6072d13c4 Call the filesyst... |
511 512 513 |
if (mapping->a_ops->freepage) mapping->a_ops->freepage(page); |
bd4c8ce41 [PATCH] invalidat... |
514 515 516 |
page_cache_release(page); /* pagecache ref */ return 1; failed: |
c4843a759 memcg: add per cg... |
517 518 |
spin_unlock_irqrestore(&mapping->tree_lock, flags); mem_cgroup_end_page_stat(memcg); |
bd4c8ce41 [PATCH] invalidat... |
519 520 |
return 0; } |
e3db7691e [PATCH] NFS: Fix ... |
521 522 523 524 525 526 527 528 |
static int do_launder_page(struct address_space *mapping, struct page *page) { if (!PageDirty(page)) return 0; if (page->mapping != mapping || mapping->a_ops->launder_page == NULL) return 0; return mapping->a_ops->launder_page(page); } |
1da177e4c Linux-2.6.12-rc2 |
529 530 |
/** * invalidate_inode_pages2_range - remove range of pages from an address_space |
67be2dd1b [PATCH] DocBook: ... |
531 |
* @mapping: the address_space |
1da177e4c Linux-2.6.12-rc2 |
532 533 534 535 536 537 |
* @start: the page offset 'from' which to invalidate * @end: the page offset 'to' which to invalidate (inclusive) * * Any pages which are found to be mapped into pagetables are unmapped prior to * invalidation. * |
6ccfa806a VFS: fix dio writ... |
538 |
* Returns -EBUSY if any pages could not be invalidated. |
1da177e4c Linux-2.6.12-rc2 |
539 540 541 542 |
*/ int invalidate_inode_pages2_range(struct address_space *mapping, pgoff_t start, pgoff_t end) { |
0cd6144aa mm + fs: prepare ... |
543 |
pgoff_t indices[PAGEVEC_SIZE]; |
1da177e4c Linux-2.6.12-rc2 |
544 |
struct pagevec pvec; |
b85e0effd mm: consistent tr... |
545 |
pgoff_t index; |
1da177e4c Linux-2.6.12-rc2 |
546 547 |
int i; int ret = 0; |
0dd1334fa fix invalidate_in... |
548 |
int ret2 = 0; |
1da177e4c Linux-2.6.12-rc2 |
549 |
int did_range_unmap = 0; |
1da177e4c Linux-2.6.12-rc2 |
550 |
|
3167760f8 mm: cleancache: s... |
551 |
cleancache_invalidate_inode(mapping); |
1da177e4c Linux-2.6.12-rc2 |
552 |
pagevec_init(&pvec, 0); |
b85e0effd mm: consistent tr... |
553 |
index = start; |
0cd6144aa mm + fs: prepare ... |
554 555 556 |
while (index <= end && pagevec_lookup_entries(&pvec, mapping, index, min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1, indices)) { |
7b965e088 [PATCH] VM: inval... |
557 |
for (i = 0; i < pagevec_count(&pvec); i++) { |
1da177e4c Linux-2.6.12-rc2 |
558 |
struct page *page = pvec.pages[i]; |
b85e0effd mm: consistent tr... |
559 560 |
/* We rely upon deletion not changing page->index */ |
0cd6144aa mm + fs: prepare ... |
561 |
index = indices[i]; |
b85e0effd mm: consistent tr... |
562 563 |
if (index > end) break; |
1da177e4c Linux-2.6.12-rc2 |
564 |
|
0cd6144aa mm + fs: prepare ... |
565 566 567 568 |
if (radix_tree_exceptional_entry(page)) { clear_exceptional_entry(mapping, index, page); continue; } |
1da177e4c Linux-2.6.12-rc2 |
569 |
lock_page(page); |
b85e0effd mm: consistent tr... |
570 |
WARN_ON(page->index != index); |
1da177e4c Linux-2.6.12-rc2 |
571 572 573 574 |
if (page->mapping != mapping) { unlock_page(page); continue; } |
1da177e4c Linux-2.6.12-rc2 |
575 |
wait_on_page_writeback(page); |
d00806b18 mm: fix fault vs ... |
576 |
if (page_mapped(page)) { |
1da177e4c Linux-2.6.12-rc2 |
577 578 579 580 581 |
if (!did_range_unmap) { /* * Zap the rest of the file in one hit. */ unmap_mapping_range(mapping, |
b85e0effd mm: consistent tr... |
582 583 584 |
(loff_t)index << PAGE_CACHE_SHIFT, (loff_t)(1 + end - index) << PAGE_CACHE_SHIFT, |
1da177e4c Linux-2.6.12-rc2 |
585 586 587 588 589 590 591 |
0); did_range_unmap = 1; } else { /* * Just zap this page */ unmap_mapping_range(mapping, |
b85e0effd mm: consistent tr... |
592 593 |
(loff_t)index << PAGE_CACHE_SHIFT, PAGE_CACHE_SIZE, 0); |
1da177e4c Linux-2.6.12-rc2 |
594 595 |
} } |
d00806b18 mm: fix fault vs ... |
596 |
BUG_ON(page_mapped(page)); |
0dd1334fa fix invalidate_in... |
597 598 599 |
ret2 = do_launder_page(mapping, page); if (ret2 == 0) { if (!invalidate_complete_page2(mapping, page)) |
6ccfa806a VFS: fix dio writ... |
600 |
ret2 = -EBUSY; |
0dd1334fa fix invalidate_in... |
601 602 603 |
} if (ret2 < 0) ret = ret2; |
1da177e4c Linux-2.6.12-rc2 |
604 605 |
unlock_page(page); } |
0cd6144aa mm + fs: prepare ... |
606 |
pagevec_remove_exceptionals(&pvec); |
1da177e4c Linux-2.6.12-rc2 |
607 608 |
pagevec_release(&pvec); cond_resched(); |
b85e0effd mm: consistent tr... |
609 |
index++; |
1da177e4c Linux-2.6.12-rc2 |
610 |
} |
3167760f8 mm: cleancache: s... |
611 |
cleancache_invalidate_inode(mapping); |
1da177e4c Linux-2.6.12-rc2 |
612 613 614 615 616 617 |
return ret; } EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range); /** * invalidate_inode_pages2 - remove all pages from an address_space |
67be2dd1b [PATCH] DocBook: ... |
618 |
* @mapping: the address_space |
1da177e4c Linux-2.6.12-rc2 |
619 620 621 622 |
* * Any pages which are found to be mapped into pagetables are unmapped prior to * invalidation. * |
e9de25dda mm: fix comments ... |
623 |
* Returns -EBUSY if any pages could not be invalidated. |
1da177e4c Linux-2.6.12-rc2 |
624 625 626 627 628 629 |
*/ int invalidate_inode_pages2(struct address_space *mapping) { return invalidate_inode_pages2_range(mapping, 0, -1); } EXPORT_SYMBOL_GPL(invalidate_inode_pages2); |
25d9e2d15 truncate: new hel... |
630 631 632 633 |
/** * truncate_pagecache - unmap and remove pagecache that has been truncated * @inode: inode |
8a549bea5 mm: tidy vmtrunca... |
634 |
* @newsize: new file size |
25d9e2d15 truncate: new hel... |
635 636 637 638 639 640 641 642 643 644 645 |
* * inode's new i_size must already be written before truncate_pagecache * is called. * * This function should typically be called before the filesystem * releases resources associated with the freed range (eg. deallocates * blocks). This way, pagecache will always stay logically coherent * with on-disk format, and the filesystem would not have to deal with * situations such as writepage being called for a page that has already * had its underlying blocks deallocated. */ |
7caef2676 truncate: drop 'o... |
646 |
void truncate_pagecache(struct inode *inode, loff_t newsize) |
25d9e2d15 truncate: new hel... |
647 |
{ |
cedabed49 vfs: Fix vmtrunca... |
648 |
struct address_space *mapping = inode->i_mapping; |
8a549bea5 mm: tidy vmtrunca... |
649 |
loff_t holebegin = round_up(newsize, PAGE_SIZE); |
cedabed49 vfs: Fix vmtrunca... |
650 651 652 653 654 655 656 657 658 659 |
/* * unmap_mapping_range is called twice, first simply for * efficiency so that truncate_inode_pages does fewer * single-page unmaps. However after this first call, and * before truncate_inode_pages finishes, it is possible for * private pages to be COWed, which remain after * truncate_inode_pages finishes, hence the second * unmap_mapping_range call must be made for correctness. */ |
8a549bea5 mm: tidy vmtrunca... |
660 661 662 |
unmap_mapping_range(mapping, holebegin, 0, 1); truncate_inode_pages(mapping, newsize); unmap_mapping_range(mapping, holebegin, 0, 1); |
25d9e2d15 truncate: new hel... |
663 664 665 666 |
} EXPORT_SYMBOL(truncate_pagecache); /** |
2c27c65ed check ATTR_SIZE c... |
667 668 669 670 |
* truncate_setsize - update inode and pagecache for a new file size * @inode: inode * @newsize: new file size * |
382e27daa mm: fix truncate_... |
671 672 673 |
* truncate_setsize updates i_size and performs pagecache truncation (if * necessary) to @newsize. It will be typically be called from the filesystem's * setattr function when ATTR_SIZE is passed in. |
2c27c65ed check ATTR_SIZE c... |
674 |
* |
77783d064 mm: Fix comment b... |
675 676 677 |
* Must be called with a lock serializing truncates and writes (generally * i_mutex but e.g. xfs uses a different lock) and before all filesystem * specific block truncation has been performed. |
2c27c65ed check ATTR_SIZE c... |
678 679 680 |
*/ void truncate_setsize(struct inode *inode, loff_t newsize) { |
90a802027 vfs: fix data cor... |
681 |
loff_t oldsize = inode->i_size; |
2c27c65ed check ATTR_SIZE c... |
682 |
i_size_write(inode, newsize); |
90a802027 vfs: fix data cor... |
683 684 |
if (newsize > oldsize) pagecache_isize_extended(inode, oldsize, newsize); |
7caef2676 truncate: drop 'o... |
685 |
truncate_pagecache(inode, newsize); |
2c27c65ed check ATTR_SIZE c... |
686 687 688 689 |
} EXPORT_SYMBOL(truncate_setsize); /** |
90a802027 vfs: fix data cor... |
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 |
* pagecache_isize_extended - update pagecache after extension of i_size * @inode: inode for which i_size was extended * @from: original inode size * @to: new inode size * * Handle extension of inode size either caused by extending truncate or by * write starting after current i_size. We mark the page straddling current * i_size RO so that page_mkwrite() is called on the nearest write access to * the page. This way filesystem can be sure that page_mkwrite() is called on * the page before user writes to the page via mmap after the i_size has been * changed. * * The function must be called after i_size is updated so that page fault * coming after we unlock the page will already see the new i_size. * The function must be called while we still hold i_mutex - this not only * makes sure i_size is stable but also that userspace cannot observe new * i_size value before we are prepared to store mmap writes at new inode size. */ void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to) { int bsize = 1 << inode->i_blkbits; loff_t rounded_from; struct page *page; pgoff_t index; |
90a802027 vfs: fix data cor... |
714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 |
WARN_ON(to > inode->i_size); if (from >= to || bsize == PAGE_CACHE_SIZE) return; /* Page straddling @from will not have any hole block created? */ rounded_from = round_up(from, bsize); if (to <= rounded_from || !(rounded_from & (PAGE_CACHE_SIZE - 1))) return; index = from >> PAGE_CACHE_SHIFT; page = find_lock_page(inode->i_mapping, index); /* Page not cached? Nothing to do */ if (!page) return; /* * See clear_page_dirty_for_io() for details why set_page_dirty() * is needed. */ if (page_mkclean(page)) set_page_dirty(page); unlock_page(page); page_cache_release(page); } EXPORT_SYMBOL(pagecache_isize_extended); /** |
623e3db9f mm for fs: add tr... |
740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 |
* truncate_pagecache_range - unmap and remove pagecache that is hole-punched * @inode: inode * @lstart: offset of beginning of hole * @lend: offset of last byte of hole * * This function should typically be called before the filesystem * releases resources associated with the freed range (eg. deallocates * blocks). This way, pagecache will always stay logically coherent * with on-disk format, and the filesystem would not have to deal with * situations such as writepage being called for a page that has already * had its underlying blocks deallocated. */ void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend) { struct address_space *mapping = inode->i_mapping; loff_t unmap_start = round_up(lstart, PAGE_SIZE); loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1; /* * This rounding is currently just for example: unmap_mapping_range * expands its hole outwards, whereas we want it to contract the hole * inwards. However, existing callers of truncate_pagecache_range are |
5a7203947 mm: teach truncat... |
761 762 |
* doing their own page rounding first. Note that unmap_mapping_range * allows holelen 0 for all, and we allow lend -1 for end of file. |
623e3db9f mm for fs: add tr... |
763 764 765 766 767 768 769 770 771 772 773 774 775 |
*/ /* * Unlike in truncate_pagecache, unmap_mapping_range is called only * once (before truncating pagecache), and without "even_cows" flag: * hole-punching should not remove private COWed pages from the hole. */ if ((u64)unmap_end > (u64)unmap_start) unmap_mapping_range(mapping, unmap_start, 1 + unmap_end - unmap_start, 0); truncate_inode_pages_range(mapping, lstart, lend); } EXPORT_SYMBOL(truncate_pagecache_range); |