Blame view

mm/truncate.c 25 KB
457c89965   Thomas Gleixner   treewide: Add SPD...
1
  // SPDX-License-Identifier: GPL-2.0-only
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2
3
4
5
6
  /*
   * mm/truncate.c - code for taking down pages from address_spaces
   *
   * Copyright (C) 2002, Linus Torvalds
   *
e1f8e8744   Francois Cami   Remove Andrew Mor...
7
   * 10Sep2002	Andrew Morton
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
8
9
10
11
   *		Initial version.
   */
  
  #include <linux/kernel.h>
4af3c9cc4   Alexey Dobriyan   Drop some headers...
12
  #include <linux/backing-dev.h>
f9fe48bec   Ross Zwisler   dax: support dirt...
13
  #include <linux/dax.h>
5a0e3ad6a   Tejun Heo   include cleanup: ...
14
  #include <linux/gfp.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
15
  #include <linux/mm.h>
0fd0e6b05   Nick Piggin   [PATCH] page inva...
16
  #include <linux/swap.h>
b95f1b31b   Paul Gortmaker   mm: Map most file...
17
  #include <linux/export.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
18
  #include <linux/pagemap.h>
01f2705da   Nate Diller   fs: convert core ...
19
  #include <linux/highmem.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
20
  #include <linux/pagevec.h>
e08748ce0   Andrew Morton   [PATCH] io-accoun...
21
  #include <linux/task_io_accounting_ops.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
22
  #include <linux/buffer_head.h>	/* grr. try_to_release_page,
aaa4059bc   Jan Kara   [PATCH] ext3: Fix...
23
  				   do_invalidatepage */
3a4f8a0b3   Hugh Dickins   mm: remove shmem_...
24
  #include <linux/shmem_fs.h>
c515e1fd3   Dan Magenheimer   mm/fs: add hooks ...
25
  #include <linux/cleancache.h>
90a802027   Jan Kara   vfs: fix data cor...
26
  #include <linux/rmap.h>
ba470de43   Rik van Riel   mmap: handle mloc...
27
  #include "internal.h"
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
28

f21875991   Mel Gorman   mm, truncate: rem...
29
30
31
32
33
34
35
  /*
   * Regular page slots are stabilized by the page lock even without the tree
   * itself locked.  These unlocked entries need verification under the tree
   * lock.
   */
  static inline void __clear_shadow_entry(struct address_space *mapping,
  				pgoff_t index, void *entry)
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
36
  {
69b6c1319   Matthew Wilcox   mm: Convert trunc...
37
  	XA_STATE(xas, &mapping->i_pages, index);
449dd6984   Johannes Weiner   mm: keep page cac...
38

69b6c1319   Matthew Wilcox   mm: Convert trunc...
39
40
  	xas_set_update(&xas, workingset_update_node);
  	if (xas_load(&xas) != entry)
f21875991   Mel Gorman   mm, truncate: rem...
41
  		return;
69b6c1319   Matthew Wilcox   mm: Convert trunc...
42
  	xas_store(&xas, NULL);
f21875991   Mel Gorman   mm, truncate: rem...
43
44
45
46
47
  }
  
  static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
  			       void *entry)
  {
b93b01631   Matthew Wilcox   page cache: use x...
48
  	xa_lock_irq(&mapping->i_pages);
f21875991   Mel Gorman   mm, truncate: rem...
49
  	__clear_shadow_entry(mapping, index, entry);
b93b01631   Matthew Wilcox   page cache: use x...
50
  	xa_unlock_irq(&mapping->i_pages);
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
51
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
52

c6dcf52c2   Jan Kara   mm: Invalidate DA...
53
  /*
f21875991   Mel Gorman   mm, truncate: rem...
54
55
56
   * Unconditionally remove exceptional entries. Usually called from truncate
   * path. Note that the pagevec may be altered by this function by removing
   * exceptional entries similar to what pagevec_remove_exceptionals does.
c6dcf52c2   Jan Kara   mm: Invalidate DA...
57
   */
f21875991   Mel Gorman   mm, truncate: rem...
58
  static void truncate_exceptional_pvec_entries(struct address_space *mapping,
31d270fd9   Matthew Wilcox (Oracle)   mm: add an 'end' ...
59
  				struct pagevec *pvec, pgoff_t *indices)
c6dcf52c2   Jan Kara   mm: Invalidate DA...
60
  {
f21875991   Mel Gorman   mm, truncate: rem...
61
  	int i, j;
31d270fd9   Matthew Wilcox (Oracle)   mm: add an 'end' ...
62
  	bool dax;
f21875991   Mel Gorman   mm, truncate: rem...
63

c6dcf52c2   Jan Kara   mm: Invalidate DA...
64
65
66
  	/* Handled by shmem itself */
  	if (shmem_mapping(mapping))
  		return;
f21875991   Mel Gorman   mm, truncate: rem...
67
  	for (j = 0; j < pagevec_count(pvec); j++)
3159f943a   Matthew Wilcox   xarray: Replace e...
68
  		if (xa_is_value(pvec->pages[j]))
f21875991   Mel Gorman   mm, truncate: rem...
69
70
71
  			break;
  
  	if (j == pagevec_count(pvec))
c6dcf52c2   Jan Kara   mm: Invalidate DA...
72
  		return;
f21875991   Mel Gorman   mm, truncate: rem...
73
74
  
  	dax = dax_mapping(mapping);
31d270fd9   Matthew Wilcox (Oracle)   mm: add an 'end' ...
75
  	if (!dax)
b93b01631   Matthew Wilcox   page cache: use x...
76
  		xa_lock_irq(&mapping->i_pages);
f21875991   Mel Gorman   mm, truncate: rem...
77
78
79
80
  
  	for (i = j; i < pagevec_count(pvec); i++) {
  		struct page *page = pvec->pages[i];
  		pgoff_t index = indices[i];
3159f943a   Matthew Wilcox   xarray: Replace e...
81
  		if (!xa_is_value(page)) {
f21875991   Mel Gorman   mm, truncate: rem...
82
83
84
  			pvec->pages[j++] = page;
  			continue;
  		}
f21875991   Mel Gorman   mm, truncate: rem...
85
86
87
88
89
90
  		if (unlikely(dax)) {
  			dax_delete_mapping_entry(mapping, index);
  			continue;
  		}
  
  		__clear_shadow_entry(mapping, index, page);
c6dcf52c2   Jan Kara   mm: Invalidate DA...
91
  	}
f21875991   Mel Gorman   mm, truncate: rem...
92

31d270fd9   Matthew Wilcox (Oracle)   mm: add an 'end' ...
93
  	if (!dax)
b93b01631   Matthew Wilcox   page cache: use x...
94
  		xa_unlock_irq(&mapping->i_pages);
f21875991   Mel Gorman   mm, truncate: rem...
95
  	pvec->nr = j;
c6dcf52c2   Jan Kara   mm: Invalidate DA...
96
97
98
99
  }
  
  /*
   * Invalidate exceptional entry if easily possible. This handles exceptional
4636e70bb   Ross Zwisler   dax: prevent inva...
100
   * entries for invalidate_inode_pages().
c6dcf52c2   Jan Kara   mm: Invalidate DA...
101
102
103
104
   */
  static int invalidate_exceptional_entry(struct address_space *mapping,
  					pgoff_t index, void *entry)
  {
4636e70bb   Ross Zwisler   dax: prevent inva...
105
106
  	/* Handled by shmem itself, or for DAX we do nothing. */
  	if (shmem_mapping(mapping) || dax_mapping(mapping))
c6dcf52c2   Jan Kara   mm: Invalidate DA...
107
  		return 1;
c6dcf52c2   Jan Kara   mm: Invalidate DA...
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
  	clear_shadow_entry(mapping, index, entry);
  	return 1;
  }
  
  /*
   * Invalidate exceptional entry if clean. This handles exceptional entries for
   * invalidate_inode_pages2() so for DAX it evicts only clean entries.
   */
  static int invalidate_exceptional_entry2(struct address_space *mapping,
  					 pgoff_t index, void *entry)
  {
  	/* Handled by shmem itself */
  	if (shmem_mapping(mapping))
  		return 1;
  	if (dax_mapping(mapping))
  		return dax_invalidate_mapping_entry_sync(mapping, index);
  	clear_shadow_entry(mapping, index, entry);
  	return 1;
  }
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
127
  /**
28bc44d7d   Fengguang Wu   do_invalidatepage...
128
   * do_invalidatepage - invalidate part or all of a page
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
129
   * @page: the page which is affected
d47992f86   Lukas Czerner   mm: change invali...
130
131
   * @offset: start of the range to invalidate
   * @length: length of the range to invalidate
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
132
133
134
135
136
137
138
139
140
141
   *
   * do_invalidatepage() is called when all or part of the page has become
   * invalidated by a truncate operation.
   *
   * do_invalidatepage() does not have to release all buffers, but it must
   * ensure that no dirty buffer is left outside @offset and that no I/O
   * is underway against any of the blocks which are outside the truncation
   * point.  Because the caller is about to free (and possibly reuse) those
   * blocks on-disk.
   */
d47992f86   Lukas Czerner   mm: change invali...
142
143
  void do_invalidatepage(struct page *page, unsigned int offset,
  		       unsigned int length)
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
144
  {
d47992f86   Lukas Czerner   mm: change invali...
145
  	void (*invalidatepage)(struct page *, unsigned int, unsigned int);
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
146
  	invalidatepage = page->mapping->a_ops->invalidatepage;
9361401eb   David Howells   [PATCH] BLOCK: Ma...
147
  #ifdef CONFIG_BLOCK
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
148
149
  	if (!invalidatepage)
  		invalidatepage = block_invalidatepage;
9361401eb   David Howells   [PATCH] BLOCK: Ma...
150
  #endif
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
151
  	if (invalidatepage)
d47992f86   Lukas Czerner   mm: change invali...
152
  		(*invalidatepage)(page, offset, length);
cf9a2ae8d   David Howells   [PATCH] BLOCK: Mo...
153
  }
ecdfc9787   Linus Torvalds   Resurrect 'try_to...
154
  /*
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
155
   * If truncate cannot remove the fs-private metadata from the page, the page
62e1c5530   Shaohua Li   page migraton: ha...
156
   * becomes orphaned.  It will be left on the LRU and may even be mapped into
54cb8821d   Nick Piggin   mm: merge populat...
157
   * user pagetables if we're racing with filemap_fault().
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
158
   *
fc3a5ac52   Matthew Wilcox (Oracle)   mm/truncate: fix ...
159
   * We need to bail out if page->mapping is no longer equal to the original
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
160
   * mapping.  This happens a) when the VM reclaimed the page while we waited on
fc0ecff69   Andrew Morton   [PATCH] remove in...
161
   * its lock, b) when a concurrent invalidate_mapping_pages got there first and
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
162
163
   * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
   */
22061a1ff   Hugh Dickins   mm/thp: unmap_map...
164
  static void truncate_cleanup_page(struct page *page)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
165
  {
22061a1ff   Hugh Dickins   mm/thp: unmap_map...
166
167
  	if (page_mapped(page))
  		unmap_mapping_page(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
168

266cf658e   David Howells   FS-Cache: Recruit...
169
  	if (page_has_private(page))
fc3a5ac52   Matthew Wilcox (Oracle)   mm/truncate: fix ...
170
  		do_invalidatepage(page, 0, thp_size(page));
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
171

b9ea25152   Konstantin Khlebnikov   page_writeback: c...
172
173
174
175
176
  	/*
  	 * Some filesystems seem to re-dirty the page even after
  	 * the VM has canceled the dirty bit (eg ext3 journaling).
  	 * Hence dirty accounting check is placed after invalidation.
  	 */
11f81becc   Tejun Heo   page_writeback: r...
177
  	cancel_dirty_page(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
178
  	ClearPageMappedToDisk(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
179
180
181
  }
  
  /*
fc0ecff69   Andrew Morton   [PATCH] remove in...
182
   * This is for invalidate_mapping_pages().  That function can be called at
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
183
   * any time, and is not supposed to throw away dirty pages.  But pages can
0fd0e6b05   Nick Piggin   [PATCH] page inva...
184
185
   * be marked dirty at any time too, so use remove_mapping which safely
   * discards clean, unused pages.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
186
187
188
189
190
191
   *
   * Returns non-zero if the page was successfully invalidated.
   */
  static int
  invalidate_complete_page(struct address_space *mapping, struct page *page)
  {
0fd0e6b05   Nick Piggin   [PATCH] page inva...
192
  	int ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
193
194
  	if (page->mapping != mapping)
  		return 0;
266cf658e   David Howells   FS-Cache: Recruit...
195
  	if (page_has_private(page) && !try_to_release_page(page, 0))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
196
  		return 0;
0fd0e6b05   Nick Piggin   [PATCH] page inva...
197
  	ret = remove_mapping(mapping, page);
0fd0e6b05   Nick Piggin   [PATCH] page inva...
198
199
  
  	return ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
200
  }
750b4987b   Nick Piggin   HWPOISON: Refacto...
201
202
  int truncate_inode_page(struct address_space *mapping, struct page *page)
  {
fc127da08   Kirill A. Shutemov   truncate: handle ...
203
  	VM_BUG_ON_PAGE(PageTail(page), page);
9f4e41f47   Jan Kara   mm: refactor trun...
204
205
  	if (page->mapping != mapping)
  		return -EIO;
22061a1ff   Hugh Dickins   mm/thp: unmap_map...
206
  	truncate_cleanup_page(page);
9f4e41f47   Jan Kara   mm: refactor trun...
207
208
  	delete_from_page_cache(page);
  	return 0;
750b4987b   Nick Piggin   HWPOISON: Refacto...
209
  }
83f786680   Wu Fengguang   HWPOISON: Add inv...
210
  /*
257187362   Andi Kleen   HWPOISON: Define ...
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
   * Used to get rid of pages on hardware memory corruption.
   */
  int generic_error_remove_page(struct address_space *mapping, struct page *page)
  {
  	if (!mapping)
  		return -EINVAL;
  	/*
  	 * Only punch for normal data pages for now.
  	 * Handling other types like directories would need more auditing.
  	 */
  	if (!S_ISREG(mapping->host->i_mode))
  		return -EIO;
  	return truncate_inode_page(mapping, page);
  }
  EXPORT_SYMBOL(generic_error_remove_page);
  
  /*
83f786680   Wu Fengguang   HWPOISON: Add inv...
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
   * Safely invalidate one page from its pagecache mapping.
   * It only drops clean, unused pages. The page must be locked.
   *
   * Returns 1 if the page is successfully invalidated, otherwise 0.
   */
  int invalidate_inode_page(struct page *page)
  {
  	struct address_space *mapping = page_mapping(page);
  	if (!mapping)
  		return 0;
  	if (PageDirty(page) || PageWriteback(page))
  		return 0;
  	if (page_mapped(page))
  		return 0;
  	return invalidate_complete_page(mapping, page);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
244
  /**
73c1e2043   Liu Bo   mm: fix comment t...
245
   * truncate_inode_pages_range - truncate range of pages specified by start & end byte offsets
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
246
247
   * @mapping: mapping to truncate
   * @lstart: offset from which to truncate
5a7203947   Lukas Czerner   mm: teach truncat...
248
   * @lend: offset to which to truncate (inclusive)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
249
   *
d7339071f   Hans Reiser   [PATCH] reiser4: ...
250
   * Truncate the page cache, removing the pages that are between
5a7203947   Lukas Czerner   mm: teach truncat...
251
252
   * specified offsets (and zeroing out partial pages
   * if lstart or lend + 1 is not page aligned).
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
253
254
255
256
257
258
259
   *
   * Truncate takes two passes - the first pass is nonblocking.  It will not
   * block on page locks and it will not block on writeback.  The second pass
   * will wait.  This is to prevent as much IO as possible in the affected region.
   * The first pass will remove most pages, so the search cost of the second pass
   * is low.
   *
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
260
261
262
   * We pass down the cache-hot hint to the page freeing code.  Even if the
   * mapping is large, it is probably the case that the final pages are the most
   * recently touched, and freeing happens in ascending file offset order.
5a7203947   Lukas Czerner   mm: teach truncat...
263
264
265
266
   *
   * Note that since ->invalidatepage() accepts range to invalidate
   * truncate_inode_pages_range is able to handle cases where lend + 1 is not
   * page aligned properly.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
267
   */
d7339071f   Hans Reiser   [PATCH] reiser4: ...
268
269
  void truncate_inode_pages_range(struct address_space *mapping,
  				loff_t lstart, loff_t lend)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
270
  {
5a7203947   Lukas Czerner   mm: teach truncat...
271
272
273
274
275
  	pgoff_t		start;		/* inclusive */
  	pgoff_t		end;		/* exclusive */
  	unsigned int	partial_start;	/* inclusive */
  	unsigned int	partial_end;	/* exclusive */
  	struct pagevec	pvec;
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
276
  	pgoff_t		indices[PAGEVEC_SIZE];
5a7203947   Lukas Czerner   mm: teach truncat...
277
278
  	pgoff_t		index;
  	int		i;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
279

7716506ad   Matthew Wilcox (Oracle)   mm: introduce and...
280
  	if (mapping_empty(mapping))
34ccb69ea   Andrey Ryabinin   mm/truncate: avoi...
281
  		goto out;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
282

5a7203947   Lukas Czerner   mm: teach truncat...
283
  	/* Offsets within partial pages */
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
284
285
  	partial_start = lstart & (PAGE_SIZE - 1);
  	partial_end = (lend + 1) & (PAGE_SIZE - 1);
5a7203947   Lukas Czerner   mm: teach truncat...
286
287
288
289
290
291
292
  
  	/*
  	 * 'start' and 'end' always covers the range of pages to be fully
  	 * truncated. Partial pages are covered with 'partial_start' at the
  	 * start of the range and 'partial_end' at the end of the range.
  	 * Note that 'end' is exclusive while 'lend' is inclusive.
  	 */
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
293
  	start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
5a7203947   Lukas Czerner   mm: teach truncat...
294
295
296
297
298
299
300
301
  	if (lend == -1)
  		/*
  		 * lend == -1 indicates end-of-file so we have to set 'end'
  		 * to the highest possible pgoff_t and since the type is
  		 * unsigned we're using -1.
  		 */
  		end = -1;
  	else
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
302
  		end = (lend + 1) >> PAGE_SHIFT;
d7339071f   Hans Reiser   [PATCH] reiser4: ...
303

866798201   Mel Gorman   mm, pagevec: remo...
304
  	pagevec_init(&pvec);
b85e0effd   Hugh Dickins   mm: consistent tr...
305
  	index = start;
5c211ba29   Matthew Wilcox (Oracle)   mm: add and use f...
306
307
308
  	while (index < end && find_lock_entries(mapping, index, end - 1,
  			&pvec, indices)) {
  		index = indices[pagevec_count(&pvec) - 1] + 1;
31d270fd9   Matthew Wilcox (Oracle)   mm: add an 'end' ...
309
  		truncate_exceptional_pvec_entries(mapping, &pvec, indices);
5c211ba29   Matthew Wilcox (Oracle)   mm: add and use f...
310
  		for (i = 0; i < pagevec_count(&pvec); i++)
22061a1ff   Hugh Dickins   mm/thp: unmap_map...
311
  			truncate_cleanup_page(pvec.pages[i]);
5c211ba29   Matthew Wilcox (Oracle)   mm: add and use f...
312
313
314
  		delete_from_page_cache_batch(mapping, &pvec);
  		for (i = 0; i < pagevec_count(&pvec); i++)
  			unlock_page(pvec.pages[i]);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
315
316
317
  		pagevec_release(&pvec);
  		cond_resched();
  	}
5c211ba29   Matthew Wilcox (Oracle)   mm: add and use f...
318

5a7203947   Lukas Czerner   mm: teach truncat...
319
  	if (partial_start) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
320
321
  		struct page *page = find_lock_page(mapping, start - 1);
  		if (page) {
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
322
  			unsigned int top = PAGE_SIZE;
5a7203947   Lukas Czerner   mm: teach truncat...
323
324
325
326
327
  			if (start > end) {
  				/* Truncation within a single page */
  				top = partial_end;
  				partial_end = 0;
  			}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
328
  			wait_on_page_writeback(page);
5a7203947   Lukas Czerner   mm: teach truncat...
329
330
331
332
333
  			zero_user_segment(page, partial_start, top);
  			cleancache_invalidate_page(mapping, page);
  			if (page_has_private(page))
  				do_invalidatepage(page, partial_start,
  						  top - partial_start);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
334
  			unlock_page(page);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
335
  			put_page(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
336
337
  		}
  	}
5a7203947   Lukas Czerner   mm: teach truncat...
338
339
340
341
342
343
344
345
346
347
  	if (partial_end) {
  		struct page *page = find_lock_page(mapping, end);
  		if (page) {
  			wait_on_page_writeback(page);
  			zero_user_segment(page, 0, partial_end);
  			cleancache_invalidate_page(mapping, page);
  			if (page_has_private(page))
  				do_invalidatepage(page, 0,
  						  partial_end);
  			unlock_page(page);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
348
  			put_page(page);
5a7203947   Lukas Czerner   mm: teach truncat...
349
350
351
352
353
354
355
  		}
  	}
  	/*
  	 * If the truncation happened within a single page no pages
  	 * will be released, just zeroed, so we can bail out now.
  	 */
  	if (start >= end)
34ccb69ea   Andrey Ryabinin   mm/truncate: avoi...
356
  		goto out;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
357

b85e0effd   Hugh Dickins   mm: consistent tr...
358
  	index = start;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
359
360
  	for ( ; ; ) {
  		cond_resched();
a656a2024   Matthew Wilcox (Oracle)   mm: remove pageve...
361
  		if (!find_get_entries(mapping, index, end - 1, &pvec,
38cefeb33   Matthew Wilcox (Oracle)   mm: remove nr_ent...
362
  				indices)) {
792ceaefe   Hugh Dickins   mm/fs: fix pessim...
363
  			/* If all gone from start onwards, we're done */
b85e0effd   Hugh Dickins   mm: consistent tr...
364
  			if (index == start)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
365
  				break;
792ceaefe   Hugh Dickins   mm/fs: fix pessim...
366
  			/* Otherwise restart to make sure all gone */
b85e0effd   Hugh Dickins   mm: consistent tr...
367
  			index = start;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
368
369
  			continue;
  		}
f21875991   Mel Gorman   mm, truncate: rem...
370

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
371
372
  		for (i = 0; i < pagevec_count(&pvec); i++) {
  			struct page *page = pvec.pages[i];
b85e0effd   Hugh Dickins   mm: consistent tr...
373
  			/* We rely upon deletion not changing page->index */
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
374
  			index = indices[i];
b85e0effd   Hugh Dickins   mm: consistent tr...
375

3159f943a   Matthew Wilcox   xarray: Replace e...
376
  			if (xa_is_value(page))
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
377
  				continue;
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
378

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
379
  			lock_page(page);
5cbc198ae   Kirill A. Shutemov   mm: fix false-pos...
380
  			WARN_ON(page_to_index(page) != index);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
381
  			wait_on_page_writeback(page);
750b4987b   Nick Piggin   HWPOISON: Refacto...
382
  			truncate_inode_page(mapping, page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
383
384
  			unlock_page(page);
  		}
31d270fd9   Matthew Wilcox (Oracle)   mm: add an 'end' ...
385
  		truncate_exceptional_pvec_entries(mapping, &pvec, indices);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
386
  		pagevec_release(&pvec);
b85e0effd   Hugh Dickins   mm: consistent tr...
387
  		index++;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
388
  	}
34ccb69ea   Andrey Ryabinin   mm/truncate: avoi...
389
390
  
  out:
3167760f8   Dan Magenheimer   mm: cleancache: s...
391
  	cleancache_invalidate_inode(mapping);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
392
  }
d7339071f   Hans Reiser   [PATCH] reiser4: ...
393
  EXPORT_SYMBOL(truncate_inode_pages_range);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
394

d7339071f   Hans Reiser   [PATCH] reiser4: ...
395
396
397
398
399
  /**
   * truncate_inode_pages - truncate *all* the pages from an offset
   * @mapping: mapping to truncate
   * @lstart: offset from which to truncate
   *
730633f0b   Jan Kara   mm: Protect opera...
400
401
   * Called under (and serialised by) inode->i_rwsem and
   * mapping->invalidate_lock.
08142579b   Jan Kara   mm: fix assertion...
402
403
404
405
406
   *
   * Note: When this function returns, there can be a page in the process of
   * deletion (inside __delete_from_page_cache()) in the specified range.  Thus
   * mapping->nrpages can be non-zero when this function returns even after
   * truncation of the whole mapping.
d7339071f   Hans Reiser   [PATCH] reiser4: ...
407
408
409
410
411
   */
  void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
  {
  	truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
412
  EXPORT_SYMBOL(truncate_inode_pages);
286973552   Mike Waychison   mm: remove __inva...
413
  /**
91b0abe36   Johannes Weiner   mm + fs: store sh...
414
415
416
   * truncate_inode_pages_final - truncate *all* pages before inode dies
   * @mapping: mapping to truncate
   *
9608703e4   Jan Kara   mm: Fix comments ...
417
   * Called under (and serialized by) inode->i_rwsem.
91b0abe36   Johannes Weiner   mm + fs: store sh...
418
419
420
421
422
423
   *
   * Filesystems have to use this in the .evict_inode path to inform the
   * VM that this is the final truncate and the inode is going away.
   */
  void truncate_inode_pages_final(struct address_space *mapping)
  {
91b0abe36   Johannes Weiner   mm + fs: store sh...
424
425
426
427
428
429
430
431
  	/*
  	 * Page reclaim can not participate in regular inode lifetime
  	 * management (can't call iput()) and thus can race with the
  	 * inode teardown.  Tell it when the address space is exiting,
  	 * so that it does not install eviction information after the
  	 * final truncate has begun.
  	 */
  	mapping_set_exiting(mapping);
7716506ad   Matthew Wilcox (Oracle)   mm: introduce and...
432
  	if (!mapping_empty(mapping)) {
91b0abe36   Johannes Weiner   mm + fs: store sh...
433
434
435
436
437
438
  		/*
  		 * As truncation uses a lockless tree lookup, cycle
  		 * the tree lock to make sure any ongoing tree
  		 * modification that does not see AS_EXITING is
  		 * completed before starting the final truncate.
  		 */
b93b01631   Matthew Wilcox   page cache: use x...
439
440
  		xa_lock_irq(&mapping->i_pages);
  		xa_unlock_irq(&mapping->i_pages);
91b0abe36   Johannes Weiner   mm + fs: store sh...
441
  	}
6ff38bd40   Pavel Tikhomirov   mm: cleancache: f...
442
443
444
445
446
447
  
  	/*
  	 * Cleancache needs notification even if there are no pages or shadow
  	 * entries.
  	 */
  	truncate_inode_pages(mapping, 0);
91b0abe36   Johannes Weiner   mm + fs: store sh...
448
449
  }
  EXPORT_SYMBOL(truncate_inode_pages_final);
a77eedbc8   Jason Yan   mm/truncate.c: ma...
450
  static unsigned long __invalidate_mapping_pages(struct address_space *mapping,
eb1d7a65f   Yafang Shao   mm, fadvise: impr...
451
  		pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
452
  {
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
453
  	pgoff_t indices[PAGEVEC_SIZE];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
454
  	struct pagevec pvec;
b85e0effd   Hugh Dickins   mm: consistent tr...
455
  	pgoff_t index = start;
315601809   Minchan Kim   mm: deactivate in...
456
457
  	unsigned long ret;
  	unsigned long count = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
458
  	int i;
866798201   Mel Gorman   mm, pagevec: remo...
459
  	pagevec_init(&pvec);
5c211ba29   Matthew Wilcox (Oracle)   mm: add and use f...
460
  	while (find_lock_entries(mapping, index, end, &pvec, indices)) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
461
462
  		for (i = 0; i < pagevec_count(&pvec); i++) {
  			struct page *page = pvec.pages[i];
e0f23603f   NeilBrown   [PATCH] Remove se...
463

b85e0effd   Hugh Dickins   mm: consistent tr...
464
  			/* We rely upon deletion not changing page->index */
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
465
  			index = indices[i];
e0f23603f   NeilBrown   [PATCH] Remove se...
466

3159f943a   Matthew Wilcox   xarray: Replace e...
467
  			if (xa_is_value(page)) {
7ae12c809   Johannes Weiner   fs: inode: count ...
468
469
470
  				count += invalidate_exceptional_entry(mapping,
  								      index,
  								      page);
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
471
472
  				continue;
  			}
5c211ba29   Matthew Wilcox (Oracle)   mm: add and use f...
473
  			index += thp_nr_pages(page) - 1;
fc127da08   Kirill A. Shutemov   truncate: handle ...
474

315601809   Minchan Kim   mm: deactivate in...
475
  			ret = invalidate_inode_page(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
476
  			unlock_page(page);
315601809   Minchan Kim   mm: deactivate in...
477
478
479
480
  			/*
  			 * Invalidation is a hint that the page is no longer
  			 * of interest and try to speed up its reclaim.
  			 */
eb1d7a65f   Yafang Shao   mm, fadvise: impr...
481
  			if (!ret) {
cc5993bd7   Minchan Kim   mm: rename deacti...
482
  				deactivate_file_page(page);
eb1d7a65f   Yafang Shao   mm, fadvise: impr...
483
484
485
486
  				/* It is likely on the pagevec of a remote CPU */
  				if (nr_pagevec)
  					(*nr_pagevec)++;
  			}
315601809   Minchan Kim   mm: deactivate in...
487
  			count += ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
488
  		}
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
489
  		pagevec_remove_exceptionals(&pvec);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
490
  		pagevec_release(&pvec);
286973552   Mike Waychison   mm: remove __inva...
491
  		cond_resched();
b85e0effd   Hugh Dickins   mm: consistent tr...
492
  		index++;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
493
  	}
315601809   Minchan Kim   mm: deactivate in...
494
  	return count;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
495
  }
eb1d7a65f   Yafang Shao   mm, fadvise: impr...
496
497
  
  /**
7ae12c809   Johannes Weiner   fs: inode: count ...
498
499
   * invalidate_mapping_pages - Invalidate all clean, unlocked cache of one inode
   * @mapping: the address_space which holds the cache to invalidate
eb1d7a65f   Yafang Shao   mm, fadvise: impr...
500
501
502
   * @start: the offset 'from' which to invalidate
   * @end: the offset 'to' which to invalidate (inclusive)
   *
7ae12c809   Johannes Weiner   fs: inode: count ...
503
504
   * This function removes pages that are clean, unmapped and unlocked,
   * as well as shadow entries. It will not block on IO activity.
eb1d7a65f   Yafang Shao   mm, fadvise: impr...
505
   *
7ae12c809   Johannes Weiner   fs: inode: count ...
506
507
   * If you want to remove all the pages of one inode, regardless of
   * their use and writeback state, use truncate_inode_pages().
eb1d7a65f   Yafang Shao   mm, fadvise: impr...
508
   *
7ae12c809   Johannes Weiner   fs: inode: count ...
509
   * Return: the number of the cache entries that were invalidated
eb1d7a65f   Yafang Shao   mm, fadvise: impr...
510
511
512
513
514
515
   */
  unsigned long invalidate_mapping_pages(struct address_space *mapping,
  		pgoff_t start, pgoff_t end)
  {
  	return __invalidate_mapping_pages(mapping, start, end, NULL);
  }
54bc48552   Anton Altaparmakov   [PATCH] Export in...
516
  EXPORT_SYMBOL(invalidate_mapping_pages);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
517

eb1d7a65f   Yafang Shao   mm, fadvise: impr...
518
  /**
649c6dfed   Alex Shi   mm/truncate: add ...
519
520
521
522
523
524
   * invalidate_mapping_pagevec - Invalidate all the unlocked pages of one inode
   * @mapping: the address_space which holds the pages to invalidate
   * @start: the offset 'from' which to invalidate
   * @end: the offset 'to' which to invalidate (inclusive)
   * @nr_pagevec: invalidate failed page number for caller
   *
a00cda3f0   Mauro Carvalho Chehab   mm: fix kernel-do...
525
526
527
   * This helper is similar to invalidate_mapping_pages(), except that it accounts
   * for pages that are likely on a pagevec and counts them in @nr_pagevec, which
   * will be used by the caller.
eb1d7a65f   Yafang Shao   mm, fadvise: impr...
528
529
530
531
532
533
   */
  void invalidate_mapping_pagevec(struct address_space *mapping,
  		pgoff_t start, pgoff_t end, unsigned long *nr_pagevec)
  {
  	__invalidate_mapping_pages(mapping, start, end, nr_pagevec);
  }
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
534
535
536
537
  /*
   * This is like invalidate_complete_page(), except it ignores the page's
   * refcount.  We do this because invalidate_inode_pages2() needs stronger
   * invalidation guarantees, and cannot afford to leave pages behind because
2706a1b89   Anderson Briglia   vmscan: fix comme...
538
539
   * shrink_page_list() has a temp ref on them, or because they're transiently
   * sitting in the lru_cache_add() pagevecs.
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
540
541
542
543
544
545
   */
  static int
  invalidate_complete_page2(struct address_space *mapping, struct page *page)
  {
  	if (page->mapping != mapping)
  		return 0;
266cf658e   David Howells   FS-Cache: Recruit...
546
  	if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
547
  		return 0;
304725097   Johannes Weiner   mm: remove irqsav...
548
  	xa_lock_irq(&mapping->i_pages);
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
549
550
  	if (PageDirty(page))
  		goto failed;
266cf658e   David Howells   FS-Cache: Recruit...
551
  	BUG_ON(page_has_private(page));
62cccb8c8   Johannes Weiner   mm: simplify lock...
552
  	__delete_from_page_cache(page, NULL);
304725097   Johannes Weiner   mm: remove irqsav...
553
  	xa_unlock_irq(&mapping->i_pages);
6072d13c4   Linus Torvalds   Call the filesyst...
554
555
556
  
  	if (mapping->a_ops->freepage)
  		mapping->a_ops->freepage(page);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
557
  	put_page(page);	/* pagecache ref */
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
558
559
  	return 1;
  failed:
304725097   Johannes Weiner   mm: remove irqsav...
560
  	xa_unlock_irq(&mapping->i_pages);
bd4c8ce41   Andrew Morton   [PATCH] invalidat...
561
562
  	return 0;
  }
e3db7691e   Trond Myklebust   [PATCH] NFS: Fix ...
563
564
565
566
567
568
569
570
  static int do_launder_page(struct address_space *mapping, struct page *page)
  {
  	if (!PageDirty(page))
  		return 0;
  	if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
  		return 0;
  	return mapping->a_ops->launder_page(page);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
571
572
  /**
   * invalidate_inode_pages2_range - remove range of pages from an address_space
67be2dd1b   Martin Waitz   [PATCH] DocBook: ...
573
   * @mapping: the address_space
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
574
575
576
577
578
579
   * @start: the page offset 'from' which to invalidate
   * @end: the page offset 'to' which to invalidate (inclusive)
   *
   * Any pages which are found to be mapped into pagetables are unmapped prior to
   * invalidation.
   *
a862f68a8   Mike Rapoport   docs/core-api/mm:...
580
   * Return: -EBUSY if any pages could not be invalidated.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
581
582
583
584
   */
  int invalidate_inode_pages2_range(struct address_space *mapping,
  				  pgoff_t start, pgoff_t end)
  {
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
585
  	pgoff_t indices[PAGEVEC_SIZE];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
586
  	struct pagevec pvec;
b85e0effd   Hugh Dickins   mm: consistent tr...
587
  	pgoff_t index;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
588
589
  	int i;
  	int ret = 0;
0dd1334fa   Hisashi Hifumi   fix invalidate_in...
590
  	int ret2 = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
591
  	int did_range_unmap = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
592

7716506ad   Matthew Wilcox (Oracle)   mm: introduce and...
593
  	if (mapping_empty(mapping))
34ccb69ea   Andrey Ryabinin   mm/truncate: avoi...
594
  		goto out;
32691f0fb   Andrey Ryabinin   mm/truncate: bail...
595

866798201   Mel Gorman   mm, pagevec: remo...
596
  	pagevec_init(&pvec);
b85e0effd   Hugh Dickins   mm: consistent tr...
597
  	index = start;
a656a2024   Matthew Wilcox (Oracle)   mm: remove pageve...
598
  	while (find_get_entries(mapping, index, end, &pvec, indices)) {
7b965e088   Trond Myklebust   [PATCH] VM: inval...
599
  		for (i = 0; i < pagevec_count(&pvec); i++) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
600
  			struct page *page = pvec.pages[i];
b85e0effd   Hugh Dickins   mm: consistent tr...
601
602
  
  			/* We rely upon deletion not changing page->index */
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
603
  			index = indices[i];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
604

3159f943a   Matthew Wilcox   xarray: Replace e...
605
  			if (xa_is_value(page)) {
c6dcf52c2   Jan Kara   mm: Invalidate DA...
606
607
608
  				if (!invalidate_exceptional_entry2(mapping,
  								   index, page))
  					ret = -EBUSY;
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
609
610
  				continue;
  			}
22061a1ff   Hugh Dickins   mm/thp: unmap_map...
611
612
613
614
615
616
617
618
619
  			if (!did_range_unmap && page_mapped(page)) {
  				/*
  				 * If page is mapped, before taking its lock,
  				 * zap the rest of the file in one hit.
  				 */
  				unmap_mapping_pages(mapping, index,
  						(1 + end - index), false);
  				did_range_unmap = 1;
  			}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
620
  			lock_page(page);
5cbc198ae   Kirill A. Shutemov   mm: fix false-pos...
621
  			WARN_ON(page_to_index(page) != index);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
622
623
624
625
  			if (page->mapping != mapping) {
  				unlock_page(page);
  				continue;
  			}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
626
  			wait_on_page_writeback(page);
22061a1ff   Hugh Dickins   mm/thp: unmap_map...
627
628
629
  
  			if (page_mapped(page))
  				unmap_mapping_page(page);
d00806b18   Nick Piggin   mm: fix fault vs ...
630
  			BUG_ON(page_mapped(page));
22061a1ff   Hugh Dickins   mm/thp: unmap_map...
631

0dd1334fa   Hisashi Hifumi   fix invalidate_in...
632
633
634
  			ret2 = do_launder_page(mapping, page);
  			if (ret2 == 0) {
  				if (!invalidate_complete_page2(mapping, page))
6ccfa806a   Hisashi Hifumi   VFS: fix dio writ...
635
  					ret2 = -EBUSY;
0dd1334fa   Hisashi Hifumi   fix invalidate_in...
636
637
638
  			}
  			if (ret2 < 0)
  				ret = ret2;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
639
640
  			unlock_page(page);
  		}
0cd6144aa   Johannes Weiner   mm + fs: prepare ...
641
  		pagevec_remove_exceptionals(&pvec);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
642
643
  		pagevec_release(&pvec);
  		cond_resched();
b85e0effd   Hugh Dickins   mm: consistent tr...
644
  		index++;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
645
  	}
cd656375f   Jan Kara   mm: fix data corr...
646
  	/*
69b6c1319   Matthew Wilcox   mm: Convert trunc...
647
  	 * For DAX we invalidate page tables after invalidating page cache.  We
cd656375f   Jan Kara   mm: fix data corr...
648
649
  	 * could invalidate page tables while invalidating each entry however
  	 * that would be expensive. And doing range unmapping before doesn't
69b6c1319   Matthew Wilcox   mm: Convert trunc...
650
  	 * work as we have no cheap way to find whether page cache entry didn't
cd656375f   Jan Kara   mm: fix data corr...
651
652
653
  	 * get remapped later.
  	 */
  	if (dax_mapping(mapping)) {
977fbdcd5   Matthew Wilcox   mm: add unmap_map...
654
  		unmap_mapping_pages(mapping, start, end - start + 1, false);
cd656375f   Jan Kara   mm: fix data corr...
655
  	}
34ccb69ea   Andrey Ryabinin   mm/truncate: avoi...
656
  out:
3167760f8   Dan Magenheimer   mm: cleancache: s...
657
  	cleancache_invalidate_inode(mapping);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
658
659
660
661
662
663
  	return ret;
  }
  EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
  
  /**
   * invalidate_inode_pages2 - remove all pages from an address_space
67be2dd1b   Martin Waitz   [PATCH] DocBook: ...
664
   * @mapping: the address_space
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
665
666
667
668
   *
   * Any pages which are found to be mapped into pagetables are unmapped prior to
   * invalidation.
   *
a862f68a8   Mike Rapoport   docs/core-api/mm:...
669
   * Return: -EBUSY if any pages could not be invalidated.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
670
671
672
673
674
675
   */
  int invalidate_inode_pages2(struct address_space *mapping)
  {
  	return invalidate_inode_pages2_range(mapping, 0, -1);
  }
  EXPORT_SYMBOL_GPL(invalidate_inode_pages2);
25d9e2d15   npiggin@suse.de   truncate: new hel...
676
677
678
679
  
  /**
   * truncate_pagecache - unmap and remove pagecache that has been truncated
   * @inode: inode
8a549bea5   Hugh Dickins   mm: tidy vmtrunca...
680
   * @newsize: new file size
25d9e2d15   npiggin@suse.de   truncate: new hel...
681
682
683
684
685
686
687
688
689
690
691
   *
   * inode's new i_size must already be written before truncate_pagecache
   * is called.
   *
   * This function should typically be called before the filesystem
   * releases resources associated with the freed range (eg. deallocates
   * blocks). This way, pagecache will always stay logically coherent
   * with on-disk format, and the filesystem would not have to deal with
   * situations such as writepage being called for a page that has already
   * had its underlying blocks deallocated.
   */
7caef2676   Kirill A. Shutemov   truncate: drop 'o...
692
  void truncate_pagecache(struct inode *inode, loff_t newsize)
25d9e2d15   npiggin@suse.de   truncate: new hel...
693
  {
cedabed49   OGAWA Hirofumi   vfs: Fix vmtrunca...
694
  	struct address_space *mapping = inode->i_mapping;
8a549bea5   Hugh Dickins   mm: tidy vmtrunca...
695
  	loff_t holebegin = round_up(newsize, PAGE_SIZE);
cedabed49   OGAWA Hirofumi   vfs: Fix vmtrunca...
696
697
698
699
700
701
702
703
704
705
  
  	/*
  	 * unmap_mapping_range is called twice, first simply for
  	 * efficiency so that truncate_inode_pages does fewer
  	 * single-page unmaps.  However after this first call, and
  	 * before truncate_inode_pages finishes, it is possible for
  	 * private pages to be COWed, which remain after
  	 * truncate_inode_pages finishes, hence the second
  	 * unmap_mapping_range call must be made for correctness.
  	 */
8a549bea5   Hugh Dickins   mm: tidy vmtrunca...
706
707
708
  	unmap_mapping_range(mapping, holebegin, 0, 1);
  	truncate_inode_pages(mapping, newsize);
  	unmap_mapping_range(mapping, holebegin, 0, 1);
25d9e2d15   npiggin@suse.de   truncate: new hel...
709
710
711
712
  }
  EXPORT_SYMBOL(truncate_pagecache);
  
  /**
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
713
714
715
716
   * truncate_setsize - update inode and pagecache for a new file size
   * @inode: inode
   * @newsize: new file size
   *
382e27daa   Jan Kara   mm: fix truncate_...
717
718
719
   * truncate_setsize updates i_size and performs pagecache truncation (if
   * necessary) to @newsize. It will be typically be called from the filesystem's
   * setattr function when ATTR_SIZE is passed in.
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
720
   *
77783d064   Jan Kara   mm: Fix comment b...
721
   * Must be called with a lock serializing truncates and writes (generally
9608703e4   Jan Kara   mm: Fix comments ...
722
   * i_rwsem but e.g. xfs uses a different lock) and before all filesystem
77783d064   Jan Kara   mm: Fix comment b...
723
   * specific block truncation has been performed.
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
724
725
726
   */
  void truncate_setsize(struct inode *inode, loff_t newsize)
  {
90a802027   Jan Kara   vfs: fix data cor...
727
  	loff_t oldsize = inode->i_size;
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
728
  	i_size_write(inode, newsize);
90a802027   Jan Kara   vfs: fix data cor...
729
730
  	if (newsize > oldsize)
  		pagecache_isize_extended(inode, oldsize, newsize);
7caef2676   Kirill A. Shutemov   truncate: drop 'o...
731
  	truncate_pagecache(inode, newsize);
2c27c65ed   Christoph Hellwig   check ATTR_SIZE c...
732
733
734
735
  }
  EXPORT_SYMBOL(truncate_setsize);
  
  /**
90a802027   Jan Kara   vfs: fix data cor...
736
737
738
739
740
741
742
743
744
745
746
747
748
749
   * pagecache_isize_extended - update pagecache after extension of i_size
   * @inode:	inode for which i_size was extended
   * @from:	original inode size
   * @to:		new inode size
   *
   * Handle extension of inode size either caused by extending truncate or by
   * write starting after current i_size. We mark the page straddling current
   * i_size RO so that page_mkwrite() is called on the nearest write access to
   * the page.  This way filesystem can be sure that page_mkwrite() is called on
   * the page before user writes to the page via mmap after the i_size has been
   * changed.
   *
   * The function must be called after i_size is updated so that page fault
   * coming after we unlock the page will already see the new i_size.
9608703e4   Jan Kara   mm: Fix comments ...
750
   * The function must be called while we still hold i_rwsem - this not only
90a802027   Jan Kara   vfs: fix data cor...
751
752
753
754
755
   * makes sure i_size is stable but also that userspace cannot observe new
   * i_size value before we are prepared to store mmap writes at new inode size.
   */
  void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to)
  {
93407472a   Fabian Frederick   fs: add i_blocksi...
756
  	int bsize = i_blocksize(inode);
90a802027   Jan Kara   vfs: fix data cor...
757
758
759
  	loff_t rounded_from;
  	struct page *page;
  	pgoff_t index;
90a802027   Jan Kara   vfs: fix data cor...
760
  	WARN_ON(to > inode->i_size);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
761
  	if (from >= to || bsize == PAGE_SIZE)
90a802027   Jan Kara   vfs: fix data cor...
762
763
764
  		return;
  	/* Page straddling @from will not have any hole block created? */
  	rounded_from = round_up(from, bsize);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
765
  	if (to <= rounded_from || !(rounded_from & (PAGE_SIZE - 1)))
90a802027   Jan Kara   vfs: fix data cor...
766
  		return;
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
767
  	index = from >> PAGE_SHIFT;
90a802027   Jan Kara   vfs: fix data cor...
768
769
770
771
772
773
774
775
776
777
778
  	page = find_lock_page(inode->i_mapping, index);
  	/* Page not cached? Nothing to do */
  	if (!page)
  		return;
  	/*
  	 * See clear_page_dirty_for_io() for details why set_page_dirty()
  	 * is needed.
  	 */
  	if (page_mkclean(page))
  		set_page_dirty(page);
  	unlock_page(page);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
779
  	put_page(page);
90a802027   Jan Kara   vfs: fix data cor...
780
781
782
783
  }
  EXPORT_SYMBOL(pagecache_isize_extended);
  
  /**
623e3db9f   Hugh Dickins   mm for fs: add tr...
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
   * truncate_pagecache_range - unmap and remove pagecache that is hole-punched
   * @inode: inode
   * @lstart: offset of beginning of hole
   * @lend: offset of last byte of hole
   *
   * This function should typically be called before the filesystem
   * releases resources associated with the freed range (eg. deallocates
   * blocks). This way, pagecache will always stay logically coherent
   * with on-disk format, and the filesystem would not have to deal with
   * situations such as writepage being called for a page that has already
   * had its underlying blocks deallocated.
   */
  void truncate_pagecache_range(struct inode *inode, loff_t lstart, loff_t lend)
  {
  	struct address_space *mapping = inode->i_mapping;
  	loff_t unmap_start = round_up(lstart, PAGE_SIZE);
  	loff_t unmap_end = round_down(1 + lend, PAGE_SIZE) - 1;
  	/*
  	 * This rounding is currently just for example: unmap_mapping_range
  	 * expands its hole outwards, whereas we want it to contract the hole
  	 * inwards.  However, existing callers of truncate_pagecache_range are
5a7203947   Lukas Czerner   mm: teach truncat...
805
806
  	 * doing their own page rounding first.  Note that unmap_mapping_range
  	 * allows holelen 0 for all, and we allow lend -1 for end of file.
623e3db9f   Hugh Dickins   mm for fs: add tr...
807
808
809
810
811
812
813
814
815
816
817
818
819
  	 */
  
  	/*
  	 * Unlike in truncate_pagecache, unmap_mapping_range is called only
  	 * once (before truncating pagecache), and without "even_cows" flag:
  	 * hole-punching should not remove private COWed pages from the hole.
  	 */
  	if ((u64)unmap_end > (u64)unmap_start)
  		unmap_mapping_range(mapping, unmap_start,
  				    1 + unmap_end - unmap_start, 0);
  	truncate_inode_pages_range(mapping, lstart, lend);
  }
  EXPORT_SYMBOL(truncate_pagecache_range);