Blame view

mm/truncate.c 24 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
3
4
5
  /*
   * mm/truncate.c - code for taking down pages from address_spaces
   *
   * Copyright (C) 2002, Linus Torvalds
   *
e1f8e8744   Francois Cami   Remove Andrew Mor...
6
   * 10Sep2002	Andrew Morton
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
7
8
9
10
   *		Initial version.
   */
  
  #include <linux/kernel.h>
4af3c9cc4   Alexey Dobriyan   Drop some headers...
11
  #include <linux/backing-dev.h>
f9fe48bec   Ross Zwisler   dax: support dirt...
12
  #include <linux/dax.h>
5a0e3ad6a   Tejun Heo   include cleanup: ...
13
  #include <linux/gfp.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
14
  #include <linux/mm.h>
0fd0e6b05   Nick Piggin   [PATCH] page inva...
15
  #include <linux/swap.h>
b95f1b31b   Paul Gortmaker   mm: Map most file...
16
  #include <linux/export.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
17
  #include <linux/pagemap.h>
01f2705da   Nate Diller   fs: convert core ...
18
  #include <linux/highmem.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
19
  #include <linux/pagevec.h>
e08748ce0   Andrew Morton   [PATCH] io-accoun...
20
  #include <linux/task_io_accounting_ops.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
21
  #include <linux/buffer_head.h>	/* grr. try_to_release_page,
aaa4059bc   Jan Kara   [PATCH] ext3: Fix...
22
  				   do_invalidatepage */
c515e1fd3   Dan Magenheimer   mm/fs: add hooks ...
23
  #include <linux/cleancache.h>
90a802027   Jan Kara   vfs: fix data cor...
24
  #include <linux/rmap.h>
ba470de43   Rik van Riel   mmap: handle mloc...
25
  #include "internal.h"
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
26

0cd6144aa   Johannes Weiner   mm + fs: prepare ...
27
28
29
  static void clear_exceptional_entry(struct address_space *mapping,
  				    pgoff_t index, void *entry)
  {
449dd6984   Johannes Weiner   mm: keep page cac...
30
31
  	struct radix_tree_node *node;
  	void **slot;
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
32
33
34
  	/* Handled by shmem itself */
  	if (shmem_mapping(mapping))
  		return;
f9fe48bec   Ross Zwisler   dax: support dirt...
35
  	if (dax_mapping(mapping)) {
ac401cc78   Jan Kara   dax: New fault lo...
36
37
  		dax_delete_mapping_entry(mapping, index);
  		return;
f9fe48bec   Ross Zwisler   dax: support dirt...
38
  	}
ac401cc78   Jan Kara   dax: New fault lo...
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
  	spin_lock_irq(&mapping->tree_lock);
  	/*
  	 * Regular page slots are stabilized by the page lock even
  	 * without the tree itself locked.  These unlocked entries
  	 * need verification under the tree lock.
  	 */
  	if (!__radix_tree_lookup(&mapping->page_tree, index, &node,
  				&slot))
  		goto unlock;
  	if (*slot != entry)
  		goto unlock;
  	radix_tree_replace_slot(slot, NULL);
  	mapping->nrexceptional--;
  	if (!node)
  		goto unlock;
  	workingset_node_shadows_dec(node);
  	/*
  	 * Don't track node without shadow entries.
  	 *
  	 * Avoid acquiring the list_lru lock if already untracked.
  	 * The list_empty() test is safe as node->private_list is
  	 * protected by mapping->tree_lock.
  	 */
  	if (!workingset_node_shadows(node) &&
  	    !list_empty(&node->private_list))
  		list_lru_del(&workingset_shadow_nodes,
  				&node->private_list);
  	__radix_tree_delete_node(&mapping->page_tree, node);
449dd6984   Johannes Weiner   mm: keep page cac...
67
  unlock:
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
68
69
  	spin_unlock_irq(&mapping->tree_lock);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
70

cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
71
  /**
28bc44d7d   Fengguang Wu   do_invalidatepage...
72
   * do_invalidatepage - invalidate part or all of a page
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
73
   * @page: the page which is affected
d47992f86   Lukas Czerner   mm: change invali...
74
75
   * @offset: start of the range to invalidate
   * @length: length of the range to invalidate
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
76
77
78
79
80
81
82
83
84
85
   *
   * do_invalidatepage() is called when all or part of the page has become
   * invalidated by a truncate operation.
   *
   * do_invalidatepage() does not have to release all buffers, but it must
   * ensure that no dirty buffer is left outside @offset and that no I/O
   * is underway against any of the blocks which are outside the truncation
   * point.  Because the caller is about to free (and possibly reuse) those
   * blocks on-disk.
   */
d47992f86   Lukas Czerner   mm: change invali...
86
87
  void do_invalidatepage(struct page *page, unsigned int offset,
  		       unsigned int length)
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
88
  {
d47992f86   Lukas Czerner   mm: change invali...
89
  	void (*invalidatepage)(struct page *, unsigned int, unsigned int);
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
90
  	invalidatepage = page->mapping->a_ops->invalidatepage;
9361401eb   David Howells   [PATCH] BLOCK: Ma...
91
  #ifdef CONFIG_BLOCK
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
92
93
  	if (!invalidatepage)
  		invalidatepage = block_invalidatepage;
9361401eb   David Howells   [PATCH] BLOCK: Ma...
94
  #endif
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
95
  	if (invalidatepage)
d47992f86   Lukas Czerner   mm: change invali...
96
  		(*invalidatepage)(page, offset, length);
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
97
  }
ecdfc9787   Linus Torvalds   Resurrect 'try_to...
98
  /*
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
99
   * If truncate cannot remove the fs-private metadata from the page, the page
62e1c5530   Shaohua Li   page migraton: ha...
100
   * becomes orphaned.  It will be left on the LRU and may even be mapped into
54cb8821d   Nick Piggin   mm: merge populat...
101
   * user pagetables if we're racing with filemap_fault().
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
102
103
104
   *
   * We need to bale out if page->mapping is no longer equal to the original
   * mapping.  This happens a) when the VM reclaimed the page while we waited on
fc0ecff69   Andrew Morton   [PATCH] remove in...
105
   * its lock, b) when a concurrent invalidate_mapping_pages got there first and
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
106
107
   * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
   */
750b4987b   Nick Piggin   HWPOISON: Refacto...
108
  static int
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
109
110
111
  truncate_complete_page(struct address_space *mapping, struct page *page)
  {
  	if (page->mapping != mapping)
750b4987b   Nick Piggin   HWPOISON: Refacto...
112
  		return -EIO;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
113

266cf658e   David Howells   FS-Cache: Recruit...
114
  	if (page_has_private(page))
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
115
  		do_invalidatepage(page, 0, PAGE_SIZE);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
116

b9ea25152   Konstantin Khlebnikov   page_writeback: c...
117
118
119
120
121
  	/*
  	 * Some filesystems seem to re-dirty the page even after
  	 * the VM has canceled the dirty bit (eg ext3 journaling).
  	 * Hence dirty accounting check is placed after invalidation.
  	 */
11f81becc   Tejun Heo   page_writeback: r...
122
  	cancel_dirty_page(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
123
  	ClearPageMappedToDisk(page);
5adc7b518   Minchan Kim   mm: truncate: cha...
124
  	delete_from_page_cache(page);
750b4987b   Nick Piggin   HWPOISON: Refacto...
125
  	return 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
126
127
128
  }
  
  /*
fc0ecff69   Andrew Morton   [PATCH] remove in...
129
   * This is for invalidate_mapping_pages().  That function can be called at
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
130
   * any time, and is not supposed to throw away dirty pages.  But pages can
0fd0e6b05   Nick Piggin   [PATCH] page inva...
131
132
   * be marked dirty at any time too, so use remove_mapping which safely
   * discards clean, unused pages.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
133
134
135
136
137
138
   *
   * Returns non-zero if the page was successfully invalidated.
   */
  static int
  invalidate_complete_page(struct address_space *mapping, struct page *page)
  {
0fd0e6b05   Nick Piggin   [PATCH] page inva...
139
  	int ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
140
141
  	if (page->mapping != mapping)
  		return 0;
266cf658e   David Howells   FS-Cache: Recruit...
142
  	if (page_has_private(page) && !try_to_release_page(page, 0))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
143
  		return 0;
0fd0e6b05   Nick Piggin   [PATCH] page inva...
144
  	ret = remove_mapping(mapping, page);
0fd0e6b05   Nick Piggin   [PATCH] page inva...
145
146
  
  	return ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
147
  }
750b4987b   Nick Piggin   HWPOISON: Refacto...
148
149
  int truncate_inode_page(struct address_space *mapping, struct page *page)
  {
fc127da08   Kirill A. Shutemov   truncate: handle ...
150
151
152
153
  	loff_t holelen;
  	VM_BUG_ON_PAGE(PageTail(page), page);
  
  	holelen = PageTransHuge(page) ? HPAGE_PMD_SIZE : PAGE_SIZE;
750b4987b   Nick Piggin   HWPOISON: Refacto...
154
155
  	if (page_mapped(page)) {
  		unmap_mapping_range(mapping,
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
156
  				   (loff_t)page->index << PAGE_SHIFT,
fc127da08   Kirill A. Shutemov   truncate: handle ...
157
  				   holelen, 0);
750b4987b   Nick Piggin   HWPOISON: Refacto...
158
159
160
  	}
  	return truncate_complete_page(mapping, page);
  }
83f786680   Wu Fengguang   HWPOISON: Add inv...
161
  /*
257187362   Andi Kleen   HWPOISON: Define ...
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
   * Used to get rid of pages on hardware memory corruption.
   */
  int generic_error_remove_page(struct address_space *mapping, struct page *page)
  {
  	if (!mapping)
  		return -EINVAL;
  	/*
  	 * Only punch for normal data pages for now.
  	 * Handling other types like directories would need more auditing.
  	 */
  	if (!S_ISREG(mapping->host->i_mode))
  		return -EIO;
  	return truncate_inode_page(mapping, page);
  }
  EXPORT_SYMBOL(generic_error_remove_page);
  
  /*
83f786680   Wu Fengguang   HWPOISON: Add inv...
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
   * Safely invalidate one page from its pagecache mapping.
   * It only drops clean, unused pages. The page must be locked.
   *
   * Returns 1 if the page is successfully invalidated, otherwise 0.
   */
  int invalidate_inode_page(struct page *page)
  {
  	struct address_space *mapping = page_mapping(page);
  	if (!mapping)
  		return 0;
  	if (PageDirty(page) || PageWriteback(page))
  		return 0;
  	if (page_mapped(page))
  		return 0;
  	return invalidate_complete_page(mapping, page);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
195
  /**
73c1e2043   Liu Bo   mm: fix comment t...
196
   * truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
197
198
   * @mapping: mapping to truncate
   * @lstart: offset from which to truncate
5a7203947   Lukas Czerner   mm: teach truncat...
199
   * @lend: offset to which to truncate (inclusive)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
200
   *
d7339071f   Hans Reiser   [PATCH] reiser4: ...
201
   * Truncate the page cache, removing the pages that are between
5a7203947   Lukas Czerner   mm: teach truncat...
202
203
   * specified offsets (and zeroing out partial pages
   * if lstart or lend + 1 is not page aligned).
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
204
205
206
207
208
209
210
   *
   * Truncate takes two passes - the first pass is nonblocking.  It will not
   * block on page locks and it will not block on writeback.  The second pass
   * will wait.  This is to prevent as much IO as possible in the affected region.
   * The first pass will remove most pages, so the search cost of the second pass
   * is low.
   *
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
211
212
213
   * We pass down the cache-hot hint to the page freeing code.  Even if the
   * mapping is large, it is probably the case that the final pages are the most
   * recently touched, and freeing happens in ascending file offset order.
5a7203947   Lukas Czerner   mm: teach truncat...
214
215
216
217
   *
   * Note that since ->invalidatepage() accepts range to invalidate
   * truncate_inode_pages_range is able to handle cases where lend + 1 is not
   * page aligned properly.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
218
   */
d7339071f   Hans Reiser   [PATCH] reiser4: ...
219
220
  void truncate_inode_pages_range(struct address_space *mapping,
  				loff_t lstart, loff_t lend)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
221
  {
5a7203947   Lukas Czerner   mm: teach truncat...
222
223
224
225
226
  	pgoff_t		start;		/* inclusive */
  	pgoff_t		end;		/* exclusive */
  	unsigned int	partial_start;	/* inclusive */
  	unsigned int	partial_end;	/* exclusive */
  	struct pagevec	pvec;
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
227
  	pgoff_t		indices[PAGEVEC_SIZE];
5a7203947   Lukas Czerner   mm: teach truncat...
228
229
  	pgoff_t		index;
  	int		i;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
230

3167760f8   Dan Magenheimer   mm: cleancache: s...
231
  	cleancache_invalidate_inode(mapping);
f9fe48bec   Ross Zwisler   dax: support dirt...
232
  	if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
233
  		return;
5a7203947   Lukas Czerner   mm: teach truncat...
234
  	/* Offsets within partial pages */
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
235
236
  	partial_start = lstart & (PAGE_SIZE - 1);
  	partial_end = (lend + 1) & (PAGE_SIZE - 1);
5a7203947   Lukas Czerner   mm: teach truncat...
237
238
239
240
241
242
243
  
  	/*
  	 * 'start' and 'end' always covers the range of pages to be fully
  	 * truncated. Partial pages are covered with 'partial_start' at the
  	 * start of the range and 'partial_end' at the end of the range.
  	 * Note that 'end' is exclusive while 'lend' is inclusive.
  	 */
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
244
  	start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
5a7203947   Lukas Czerner   mm: teach truncat...
245
246
247
248
249
250
251
252
  	if (lend == -1)
  		/*
  		 * lend == -1 indicates end-of-file so we have to set 'end'
  		 * to the highest possible pgoff_t and since the type is
  		 * unsigned we're using -1.
  		 */
  		end = -1;
  	else
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
253
  		end = (lend + 1) >> PAGE_SHIFT;
d7339071f   Hans Reiser   [PATCH] reiser4: ...
254

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
255
  	pagevec_init(&pvec, 0);
b85e0effd   Hugh Dickins   mm: consistent tr...
256
  	index = start;
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
257
258
259
  	while (index < end && pagevec_lookup_entries(&pvec, mapping, index,
  			min(end - index, (pgoff_t)PAGEVEC_SIZE),
  			indices)) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
260
261
  		for (i = 0; i < pagevec_count(&pvec); i++) {
  			struct page *page = pvec.pages[i];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
262

b85e0effd   Hugh Dickins   mm: consistent tr...
263
  			/* We rely upon deletion not changing page->index */
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
264
  			index = indices[i];
5a7203947   Lukas Czerner   mm: teach truncat...
265
  			if (index >= end)
d7339071f   Hans Reiser   [PATCH] reiser4: ...
266
  				break;
d7339071f   Hans Reiser   [PATCH] reiser4: ...
267

0cd6144aa   Johannes Weiner   mm + fs: prepare ...
268
269
270
271
  			if (radix_tree_exceptional_entry(page)) {
  				clear_exceptional_entry(mapping, index, page);
  				continue;
  			}
529ae9aaa   Nick Piggin   mm: rename page t...
272
  			if (!trylock_page(page))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
273
  				continue;
5cbc198ae   Kirill A. Shutemov   mm: fix false-pos...
274
  			WARN_ON(page_to_index(page) != index);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
275
276
277
278
  			if (PageWriteback(page)) {
  				unlock_page(page);
  				continue;
  			}
750b4987b   Nick Piggin   HWPOISON: Refacto...
279
  			truncate_inode_page(mapping, page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
280
281
  			unlock_page(page);
  		}
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
282
  		pagevec_remove_exceptionals(&pvec);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
283
284
  		pagevec_release(&pvec);
  		cond_resched();
b85e0effd   Hugh Dickins   mm: consistent tr...
285
  		index++;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
286
  	}
5a7203947   Lukas Czerner   mm: teach truncat...
287
  	if (partial_start) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
288
289
  		struct page *page = find_lock_page(mapping, start - 1);
  		if (page) {
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
290
  			unsigned int top = PAGE_SIZE;
5a7203947   Lukas Czerner   mm: teach truncat...
291
292
293
294
295
  			if (start > end) {
  				/* Truncation within a single page */
  				top = partial_end;
  				partial_end = 0;
  			}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
296
  			wait_on_page_writeback(page);
5a7203947   Lukas Czerner   mm: teach truncat...
297
298
299
300
301
  			zero_user_segment(page, partial_start, top);
  			cleancache_invalidate_page(mapping, page);
  			if (page_has_private(page))
  				do_invalidatepage(page, partial_start,
  						  top - partial_start);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
302
  			unlock_page(page);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
303
  			put_page(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
304
305
  		}
  	}
5a7203947   Lukas Czerner   mm: teach truncat...
306
307
308
309
310
311
312
313
314
315
  	if (partial_end) {
  		struct page *page = find_lock_page(mapping, end);
  		if (page) {
  			wait_on_page_writeback(page);
  			zero_user_segment(page, 0, partial_end);
  			cleancache_invalidate_page(mapping, page);
  			if (page_has_private(page))
  				do_invalidatepage(page, 0,
  						  partial_end);
  			unlock_page(page);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
316
  			put_page(page);
5a7203947   Lukas Czerner   mm: teach truncat...
317
318
319
320
321
322
323
324
  		}
  	}
  	/*
  	 * If the truncation happened within a single page no pages
  	 * will be released, just zeroed, so we can bail out now.
  	 */
  	if (start >= end)
  		return;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
325

b85e0effd   Hugh Dickins   mm: consistent tr...
326
  	index = start;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
327
328
  	for ( ; ; ) {
  		cond_resched();
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
329
  		if (!pagevec_lookup_entries(&pvec, mapping, index,
792ceaefe   Hugh Dickins   mm/fs: fix pessim...
330
331
  			min(end - index, (pgoff_t)PAGEVEC_SIZE), indices)) {
  			/* If all gone from start onwards, we're done */
b85e0effd   Hugh Dickins   mm: consistent tr...
332
  			if (index == start)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
333
  				break;
792ceaefe   Hugh Dickins   mm/fs: fix pessim...
334
  			/* Otherwise restart to make sure all gone */
b85e0effd   Hugh Dickins   mm: consistent tr...
335
  			index = start;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
336
337
  			continue;
  		}
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
338
  		if (index == start && indices[0] >= end) {
792ceaefe   Hugh Dickins   mm/fs: fix pessim...
339
  			/* All gone out of hole to be punched, we're done */
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
340
  			pagevec_remove_exceptionals(&pvec);
d7339071f   Hans Reiser   [PATCH] reiser4: ...
341
342
343
  			pagevec_release(&pvec);
  			break;
  		}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
344
345
  		for (i = 0; i < pagevec_count(&pvec); i++) {
  			struct page *page = pvec.pages[i];
b85e0effd   Hugh Dickins   mm: consistent tr...
346
  			/* We rely upon deletion not changing page->index */
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
347
  			index = indices[i];
792ceaefe   Hugh Dickins   mm/fs: fix pessim...
348
349
350
  			if (index >= end) {
  				/* Restart punch to make sure all gone */
  				index = start - 1;
d7339071f   Hans Reiser   [PATCH] reiser4: ...
351
  				break;
792ceaefe   Hugh Dickins   mm/fs: fix pessim...
352
  			}
b85e0effd   Hugh Dickins   mm: consistent tr...
353

0cd6144aa   Johannes Weiner   mm + fs: prepare ...
354
355
356
357
  			if (radix_tree_exceptional_entry(page)) {
  				clear_exceptional_entry(mapping, index, page);
  				continue;
  			}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
358
  			lock_page(page);
5cbc198ae   Kirill A. Shutemov   mm: fix false-pos...
359
  			WARN_ON(page_to_index(page) != index);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
360
  			wait_on_page_writeback(page);
750b4987b   Nick Piggin   HWPOISON: Refacto...
361
  			truncate_inode_page(mapping, page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
362
363
  			unlock_page(page);
  		}
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
364
  		pagevec_remove_exceptionals(&pvec);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
365
  		pagevec_release(&pvec);
b85e0effd   Hugh Dickins   mm: consistent tr...
366
  		index++;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
367
  	}
3167760f8   Dan Magenheimer   mm: cleancache: s...
368
  	cleancache_invalidate_inode(mapping);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
369
  }
d7339071f   Hans Reiser   [PATCH] reiser4: ...
370
  EXPORT_SYMBOL(truncate_inode_pages_range);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
371

d7339071f   Hans Reiser   [PATCH] reiser4: ...
372
373
374
375
376
  /**
   * truncate_inode_pages - truncate *all* the pages from an offset
   * @mapping: mapping to truncate
   * @lstart: offset from which to truncate
   *
1b1dcc1b5   Jes Sorensen   [PATCH] mutex sub...
377
   * Called under (and serialised by) inode->i_mutex.
08142579b   Jan Kara   mm: fix assertion...
378
379
380
381
382
   *
   * Note: When this function returns, there can be a page in the process of
   * deletion (inside __delete_from_page_cache()) in the specified range.  Thus
   * mapping->nrpages can be non-zero when this function returns even after
   * truncation of the whole mapping.
d7339071f   Hans Reiser   [PATCH] reiser4: ...
383
384
385
386
387
   */
  void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
  {
  	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
388
  EXPORT_SYMBOL(truncate_inode_pages);
286973552   Mike Waychison   mm: remove __inva...
389
  /**
91b0abe36   Johannes Weiner   mm + fs: store sh...
390
391
392
393
394
395
396
397
398
399
   * truncate_inode_pages_final - truncate *all* pages before inode dies
   * @mapping: mapping to truncate
   *
   * Called under (and serialized by) inode->i_mutex.
   *
   * Filesystems have to use this in the .evict_inode path to inform the
   * VM that this is the final truncate and the inode is going away.
   */
  void truncate_inode_pages_final(struct address_space *mapping)
  {
f9fe48bec   Ross Zwisler   dax: support dirt...
400
  	unsigned long nrexceptional;
91b0abe36   Johannes Weiner   mm + fs: store sh...
401
402
403
404
405
406
407
408
409
410
411
412
413
  	unsigned long nrpages;
  
  	/*
  	 * Page reclaim can not participate in regular inode lifetime
  	 * management (can't call iput()) and thus can race with the
  	 * inode teardown.  Tell it when the address space is exiting,
  	 * so that it does not install eviction information after the
  	 * final truncate has begun.
  	 */
  	mapping_set_exiting(mapping);
  
  	/*
  	 * When reclaim installs eviction entries, it increases
f9fe48bec   Ross Zwisler   dax: support dirt...
414
  	 * nrexceptional first, then decreases nrpages.  Make sure we see
91b0abe36   Johannes Weiner   mm + fs: store sh...
415
416
417
418
  	 * this in the right order or we might miss an entry.
  	 */
  	nrpages = mapping->nrpages;
  	smp_rmb();
f9fe48bec   Ross Zwisler   dax: support dirt...
419
  	nrexceptional = mapping->nrexceptional;
91b0abe36   Johannes Weiner   mm + fs: store sh...
420

f9fe48bec   Ross Zwisler   dax: support dirt...
421
  	if (nrpages || nrexceptional) {
91b0abe36   Johannes Weiner   mm + fs: store sh...
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
  		/*
  		 * As truncation uses a lockless tree lookup, cycle
  		 * the tree lock to make sure any ongoing tree
  		 * modification that does not see AS_EXITING is
  		 * completed before starting the final truncate.
  		 */
  		spin_lock_irq(&mapping->tree_lock);
  		spin_unlock_irq(&mapping->tree_lock);
  
  		truncate_inode_pages(mapping, 0);
  	}
  }
  EXPORT_SYMBOL(truncate_inode_pages_final);
  
  /**
286973552   Mike Waychison   mm: remove __inva...
437
438
439
440
441
442
443
444
445
446
447
448
449
   * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
   * @mapping: the address_space which holds the pages to invalidate
   * @start: the offset 'from' which to invalidate
   * @end: the offset 'to' which to invalidate (inclusive)
   *
   * This function only removes the unlocked pages, if you want to
   * remove all the pages of one inode, you must call truncate_inode_pages.
   *
   * invalidate_mapping_pages() will not block on IO activity. It will not
   * invalidate pages which are dirty, locked, under writeback or mapped into
   * pagetables.
   */
  unsigned long invalidate_mapping_pages(struct address_space *mapping,
315601809   Minchan Kim   mm: deactivate in...
450
  		pgoff_t start, pgoff_t end)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
451
  {
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
452
  	pgoff_t indices[PAGEVEC_SIZE];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
453
  	struct pagevec pvec;
b85e0effd   Hugh Dickins   mm: consistent tr...
454
  	pgoff_t index = start;
315601809   Minchan Kim   mm: deactivate in...
455
456
  	unsigned long ret;
  	unsigned long count = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
457
458
459
  	int i;
  
  	pagevec_init(&pvec, 0);
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
460
461
462
  	while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
  			min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
  			indices)) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
463
464
  		for (i = 0; i < pagevec_count(&pvec); i++) {
  			struct page *page = pvec.pages[i];
e0f23603f   NeilBrown   [PATCH] Remove se...
465

b85e0effd   Hugh Dickins   mm: consistent tr...
466
  			/* We rely upon deletion not changing page->index */
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
467
  			index = indices[i];
b85e0effd   Hugh Dickins   mm: consistent tr...
468
469
  			if (index > end)
  				break;
e0f23603f   NeilBrown   [PATCH] Remove se...
470

0cd6144aa   Johannes Weiner   mm + fs: prepare ...
471
472
473
474
  			if (radix_tree_exceptional_entry(page)) {
  				clear_exceptional_entry(mapping, index, page);
  				continue;
  			}
b85e0effd   Hugh Dickins   mm: consistent tr...
475
476
  			if (!trylock_page(page))
  				continue;
fc127da08   Kirill A. Shutemov   truncate: handle ...
477

5cbc198ae   Kirill A. Shutemov   mm: fix false-pos...
478
  			WARN_ON(page_to_index(page) != index);
fc127da08   Kirill A. Shutemov   truncate: handle ...
479
480
481
482
483
484
485
486
487
488
489
490
  
  			/* Middle of THP: skip */
  			if (PageTransTail(page)) {
  				unlock_page(page);
  				continue;
  			} else if (PageTransHuge(page)) {
  				index += HPAGE_PMD_NR - 1;
  				i += HPAGE_PMD_NR - 1;
  				/* 'end' is in the middle of THP */
  				if (index ==  round_down(end, HPAGE_PMD_NR))
  					continue;
  			}
315601809   Minchan Kim   mm: deactivate in...
491
  			ret = invalidate_inode_page(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
492
  			unlock_page(page);
315601809   Minchan Kim   mm: deactivate in...
493
494
495
496
497
  			/*
  			 * Invalidation is a hint that the page is no longer
  			 * of interest and try to speed up its reclaim.
  			 */
  			if (!ret)
cc5993bd7   Minchan Kim   mm: rename deacti...
498
  				deactivate_file_page(page);
315601809   Minchan Kim   mm: deactivate in...
499
  			count += ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
500
  		}
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
501
  		pagevec_remove_exceptionals(&pvec);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
502
  		pagevec_release(&pvec);
286973552   Mike Waychison   mm: remove __inva...
503
  		cond_resched();
b85e0effd   Hugh Dickins   mm: consistent tr...
504
  		index++;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
505
  	}
315601809   Minchan Kim   mm: deactivate in...
506
  	return count;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
507
  }
54bc48552   Anton Altaparmakov   [PATCH] Export in...
508
  EXPORT_SYMBOL(invalidate_mapping_pages);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
509

bd4c8ce41   Andrew Morton   [PATCH] invalidat...
510
511
512
513
  /*
   * This is like invalidate_complete_page(), except it ignores the page's
   * refcount.  We do this because invalidate_inode_pages2() needs stronger
   * invalidation guarantees, and cannot afford to leave pages behind because
2706a1b89   Anderson Briglia   vmscan: fix comme...
514
515
   * shrink_page_list() has a temp ref on them, or because they're transiently
   * sitting in the lru_cache_add() pagevecs.
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
516
517
518
519
   */
  static int
  invalidate_complete_page2(struct address_space *mapping, struct page *page)
  {
c4843a759   Greg Thelen   memcg: add per cg...
520
  	unsigned long flags;
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
521
522
  	if (page->mapping != mapping)
  		return 0;
266cf658e   David Howells   FS-Cache: Recruit...
523
  	if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
524
  		return 0;
c4843a759   Greg Thelen   memcg: add per cg...
525
  	spin_lock_irqsave(&mapping->tree_lock, flags);
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
526
527
  	if (PageDirty(page))
  		goto failed;
266cf658e   David Howells   FS-Cache: Recruit...
528
  	BUG_ON(page_has_private(page));
62cccb8c8   Johannes Weiner   mm: simplify lock...
529
  	__delete_from_page_cache(page, NULL);
c4843a759   Greg Thelen   memcg: add per cg...
530
  	spin_unlock_irqrestore(&mapping->tree_lock, flags);
6072d13c4   Linus Torvalds   Call the filesyst...
531
532
533
  
  	if (mapping->a_ops->freepage)
  		mapping->a_ops->freepage(page);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
534
  	put_page(page);	/* pagecache ref */
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
535
536
  	return 1;
  failed:
c4843a759   Greg Thelen   memcg: add per cg...
537
  	spin_unlock_irqrestore(&mapping->tree_lock, flags);
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
538
539
  	return 0;
  }
e3db7691e   Trond Myklebust   [PATCH] NFS: Fix ...
540
541
542
543
544
545
546
547
  static int do_launder_page(struct address_space *mapping, struct page *page)
  {
  	if (!PageDirty(page))
  		return 0;
  	if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
  		return 0;
  	return mapping->a_ops->launder_page(page);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
548
549
  /**
   * invalidate_inode_pages2_range - remove range of pages from an address_space
67be2dd1b   Martin Waitz   [PATCH] DocBook: ...
550
   * @mapping: the address_space
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
551
552
553
554
555
556
   * @start: the page offset 'from' which to invalidate
   * @end: the page offset 'to' which to invalidate (inclusive)
   *
   * Any pages which are found to be mapped into pagetables are unmapped prior to
   * invalidation.
   *
6ccfa806a   Hisashi Hifumi   VFS: fix dio writ...
557
   * Returns -EBUSY if any pages could not be invalidated.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
558
559
560
561
   */
  int invalidate_inode_pages2_range(struct address_space *mapping,
  				  pgoff_t start, pgoff_t end)
  {
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
562
  	pgoff_t indices[PAGEVEC_SIZE];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
563
  	struct pagevec pvec;
b85e0effd   Hugh Dickins   mm: consistent tr...
564
  	pgoff_t index;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
565
566
  	int i;
  	int ret = 0;
0dd1334fa   Hisashi Hifumi   fix invalidate_in...
567
  	int ret2 = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
568
  	int did_range_unmap = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
569

3167760f8   Dan Magenheimer   mm: cleancache: s...
570
  	cleancache_invalidate_inode(mapping);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
571
  	pagevec_init(&pvec, 0);
b85e0effd   Hugh Dickins   mm: consistent tr...
572
  	index = start;
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
573
574
575
  	while (index <= end && pagevec_lookup_entries(&pvec, mapping, index,
  			min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1,
  			indices)) {
7b965e088   Trond Myklebust   [PATCH] VM: inval...
576
  		for (i = 0; i < pagevec_count(&pvec); i++) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
577
  			struct page *page = pvec.pages[i];
b85e0effd   Hugh Dickins   mm: consistent tr...
578
579
  
  			/* We rely upon deletion not changing page->index */
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
580
  			index = indices[i];
b85e0effd   Hugh Dickins   mm: consistent tr...
581
582
  			if (index > end)
  				break;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
583

0cd6144aa   Johannes Weiner   mm + fs: prepare ...
584
585
586
587
  			if (radix_tree_exceptional_entry(page)) {
  				clear_exceptional_entry(mapping, index, page);
  				continue;
  			}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
588
  			lock_page(page);
5cbc198ae   Kirill A. Shutemov   mm: fix false-pos...
589
  			WARN_ON(page_to_index(page) != index);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
590
591
592
593
  			if (page->mapping != mapping) {
  				unlock_page(page);
  				continue;
  			}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
594
  			wait_on_page_writeback(page);
d00806b18   Nick Piggin   mm: fix fault vs ...
595
  			if (page_mapped(page)) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
596
597
598
599
600
  				if (!did_range_unmap) {
  					/*
  					 * Zap the rest of the file in one hit.
  					 */
  					unmap_mapping_range(mapping,
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
601
  					   (loff_t)index << PAGE_SHIFT,
b85e0effd   Hugh Dickins   mm: consistent tr...
602
  					   (loff_t)(1 + end - index)
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
603
604
  							 << PAGE_SHIFT,
  							 0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
605
606
607
608
609
610
  					did_range_unmap = 1;
  				} else {
  					/*
  					 * Just zap this page
  					 */
  					unmap_mapping_range(mapping,
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
611
612
  					   (loff_t)index << PAGE_SHIFT,
  					   PAGE_SIZE, 0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
613
614
  				}
  			}
d00806b18   Nick Piggin   mm: fix fault vs ...
615
  			BUG_ON(page_mapped(page));
0dd1334fa   Hisashi Hifumi   fix invalidate_in...
616
617
618
  			ret2 = do_launder_page(mapping, page);
  			if (ret2 == 0) {
  				if (!invalidate_complete_page2(mapping, page))
6ccfa806a   Hisashi Hifumi   VFS: fix dio writ...
619
  					ret2 = -EBUSY;
0dd1334fa   Hisashi Hifumi   fix invalidate_in...
620
621
622
  			}
  			if (ret2 < 0)
  				ret = ret2;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
623
624
  			unlock_page(page);
  		}
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
625
  		pagevec_remove_exceptionals(&pvec);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
626
627
  		pagevec_release(&pvec);
  		cond_resched();
b85e0effd   Hugh Dickins   mm: consistent tr...
628
  		index++;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
629
  	}
3167760f8   Dan Magenheimer   mm: cleancache: s...
630
  	cleancache_invalidate_inode(mapping);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
631
632
633
634
635
636
  	return ret;
  }
  EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
  
  /**
   * invalidate_inode_pages2 - remove all pages from an address_space
67be2dd1b   Martin Waitz   [PATCH] DocBook: ...
637
   * @mapping: the address_space
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
638
639
640
641
   *
   * Any pages which are found to be mapped into pagetables are unmapped prior to
   * invalidation.
   *
e9de25dda   Peng Tao   mm: fix comments ...
642
   * Returns -EBUSY if any pages could not be invalidated.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
643
644
645
646
647
648
   */
  int invalidate_inode_pages2(struct address_space *mapping)
  {
  	return invalidate_inode_pages2_range(mapping, 0, -1);
  }
  EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
25d9e2d15   npiggin@suse.de   truncate: new hel...
649
650
651
652
  
  /**
   * truncate_pagecache - unmap and remove pagecache that has been truncated
   * @inode: inode
8a549bea5   Hugh Dickins   mm: tidy vmtrunca...
653
   * @newsize: new file size
25d9e2d15   npiggin@suse.de   truncate: new hel...
654
655
656
657
658
659
660
661
662
663
664
   *
   * inode's new i_size must already be written before truncate_pagecache
   * is called.
   *
   * This function should typically be called before the filesystem
   * releases resources associated with the freed range (eg. deallocates
   * blocks). This way, pagecache will always stay logically coherent
   * with on-disk format, and the filesystem would not have to deal with
   * situations such as writepage being called for a page that has already
   * had its underlying blocks deallocated.
   */
7caef2676   Kirill A. Shutemov   truncate: drop 'o...
665
  void truncate_pagecache(struct inode *inode, loff_t newsize)
25d9e2d15   npiggin@suse.de   truncate: new hel...
666
  {
cedabed49   OGAWA Hirofumi   vfs: Fix vmtrunca...
667
  	struct address_space *mapping = inode->i_mapping;
8a549bea5   Hugh Dickins   mm: tidy vmtrunca...
668
  	loff_t holebegin = round_up(newsize, PAGE_SIZE);
cedabed49   OGAWA Hirofumi   vfs: Fix vmtrunca...
669
670
671
672
673
674
675
676
677
678
  
  	/*
  	 * unmap_mapping_range is called twice, first simply for
  	 * efficiency so that truncate_inode_pages does fewer
  	 * single-page unmaps.  However after this first call, and
  	 * before truncate_inode_pages finishes, it is possible for
  	 * private pages to be COWed, which remain after
  	 * truncate_inode_pages finishes, hence the second
  	 * unmap_mapping_range call must be made for correctness.
  	 */
8a549bea5   Hugh Dickins   mm: tidy vmtrunca...
679
680
681
  	unmap_mapping_range(mapping, holebegin, 0, 1);
  	truncate_inode_pages(mapping, newsize);
  	unmap_mapping_range(mapping, holebegin, 0, 1);
25d9e2d15   npiggin@suse.de   truncate: new hel...
682
683
684
685
  }
  EXPORT_SYMBOL(truncate_pagecache);
  
  /**
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
686
687
688
689
   * truncate_setsize - update inode and pagecache for a new file size
   * @inode: inode
   * @newsize: new file size
   *
382e27daa   Jan Kara   mm: fix truncate_...
690
691
692
   * truncate_setsize updates i_size and performs pagecache truncation (if
   * necessary) to @newsize. It will be typically be called from the filesystem's
   * setattr function when ATTR_SIZE is passed in.
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
693
   *
77783d064   Jan Kara   mm: Fix comment b...
694
695
696
   * Must be called with a lock serializing truncates and writes (generally
   * i_mutex but e.g. xfs uses a different lock) and before all filesystem
   * specific block truncation has been performed.
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
697
698
699
   */
  void truncate_setsize(struct inode *inode, loff_t newsize)
  {
90a802027   Jan Kara   vfs: fix data cor...
700
  	loff_t oldsize = inode->i_size;
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
701
  	i_size_write(inode, newsize);
90a802027   Jan Kara   vfs: fix data cor...
702
703
  	if (newsize > oldsize)
  		pagecache_isize_extended(inode, oldsize, newsize);
7caef2676   Kirill A. Shutemov   truncate: drop 'o...
704
  	truncate_pagecache(inode, newsize);
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
705
706
707
708
  }
  EXPORT_SYMBOL(truncate_setsize);
  
  /**
90a802027   Jan Kara   vfs: fix data cor...
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
   * pagecache_isize_extended - update pagecache after extension of i_size
   * @inode:	inode for which i_size was extended
   * @from:	original inode size
   * @to:		new inode size
   *
   * Handle extension of inode size either caused by extending truncate or by
   * write starting after current i_size. We mark the page straddling current
   * i_size RO so that page_mkwrite() is called on the nearest write access to
   * the page.  This way filesystem can be sure that page_mkwrite() is called on
   * the page before user writes to the page via mmap after the i_size has been
   * changed.
   *
   * The function must be called after i_size is updated so that page fault
   * coming after we unlock the page will already see the new i_size.
   * The function must be called while we still hold i_mutex - this not only
   * makes sure i_size is stable but also that userspace cannot observe new
   * i_size value before we are prepared to store mmap writes at new inode size.
   */
  void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to)
  {
  	int bsize = 1 << inode->i_blkbits;
  	loff_t rounded_from;
  	struct page *page;
  	pgoff_t index;
90a802027   Jan Kara   vfs: fix data cor...
733
  	WARN_ON(to > inode->i_size);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
734
  	if (from >= to || bsize == PAGE_SIZE)
90a802027   Jan Kara   vfs: fix data cor...
735
736
737
  		return;
  	/* Page straddling @from will not have any hole block created? */
  	rounded_from = round_up(from, bsize);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
738
  	if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))
90a802027   Jan Kara   vfs: fix data cor...
739
  		return;
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
740
  	index = from >> PAGE_SHIFT;
90a802027   Jan Kara   vfs: fix data cor...
741
742
743
744
745
746
747
748
749
750
751
  	page = find_lock_page(inode->i_mapping, index);
  	/* Page not cached? Nothing to do */
  	if (!page)
  		return;
  	/*
  	 * See clear_page_dirty_for_io() for details why set_page_dirty()
  	 * is needed.
  	 */
  	if (page_mkclean(page))
  		set_page_dirty(page);
  	unlock_page(page);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
752
  	put_page(page);
90a802027   Jan Kara   vfs: fix data cor...
753
754
755
756
  }
  EXPORT_SYMBOL(pagecache_isize_extended);
  
  /**
623e3db9f   Hugh Dickins   mm for fs: add tr...
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
   * truncate_pagecache_range - unmap and remove pagecache that is hole-punched
   * @inode: inode
   * @lstart: offset of beginning of hole
   * @lend: offset of last byte of hole
   *
   * This function should typically be called before the filesystem
   * releases resources associated with the freed range (eg. deallocates
   * blocks). This way, pagecache will always stay logically coherent
   * with on-disk format, and the filesystem would not have to deal with
   * situations such as writepage being called for a page that has already
   * had its underlying blocks deallocated.
   */
  void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend)
  {
  	struct address_space *mapping = inode->i_mapping;
  	loff_t unmap_start = round_up(lstart, PAGE_SIZE);
  	loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;
  	/*
  	 * This rounding is currently just for example: unmap_mapping_range
  	 * expands its hole outwards, whereas we want it to contract the hole
  	 * inwards.  However, existing callers of truncate_pagecache_range are
5a7203947   Lukas Czerner   mm: teach truncat...
778
779
  	 * doing their own page rounding first.  Note that unmap_mapping_range
  	 * allows holelen 0 for all, and we allow lend -1 for end of file.
623e3db9f   Hugh Dickins   mm for fs: add tr...
780
781
782
783
784
785
786
787
788
789
790
791
792
  	 */
  
  	/*
  	 * Unlike in truncate_pagecache, unmap_mapping_range is called only
  	 * once (before truncating pagecache), and without "even_cows" flag:
  	 * hole-punching should not remove private COWed pages from the hole.
  	 */
  	if ((u64)unmap_end > (u64)unmap_start)
  		unmap_mapping_range(mapping, unmap_start,
  				    1 + unmap_end - unmap_start, 0);
  	truncate_inode_pages_range(mapping, lstart, lend);
  }
  EXPORT_SYMBOL(truncate_pagecache_range);