Blame view

fs/mpage.c 20.5 KB
b24413180   Greg Kroah-Hartman   License cleanup: ...
1
  // SPDX-License-Identifier: GPL-2.0
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
2
3
4
5
6
7
8
9
  /*
   * fs/mpage.c
   *
   * Copyright (C) 2002, Linus Torvalds.
   *
   * Contains functions related to preparing and submitting BIOs which contain
   * multiple pagecache pages.
   *
e1f8e8744   Francois Cami   Remove Andrew Mor...
10
   * 15May2002	Andrew Morton
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
11
12
13
14
15
16
   *		Initial version
   * 27Jun2002	axboe@suse.de
   *		use bio_add_page() to build bio's just the right size
   */
  
  #include <linux/kernel.h>
630d9c472   Paul Gortmaker   fs: reduce the us...
17
  #include <linux/export.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
18
19
  #include <linux/mm.h>
  #include <linux/kdev_t.h>
5a0e3ad6a   Tejun Heo   include cleanup: ...
20
  #include <linux/gfp.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
21
22
23
24
25
26
27
  #include <linux/bio.h>
  #include <linux/fs.h>
  #include <linux/buffer_head.h>
  #include <linux/blkdev.h>
  #include <linux/highmem.h>
  #include <linux/prefetch.h>
  #include <linux/mpage.h>
02c43638e   Andrew Morton   fs/mpage.c:mpage_...
28
  #include <linux/mm_inline.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
29
30
31
  #include <linux/writeback.h>
  #include <linux/backing-dev.h>
  #include <linux/pagevec.h>
c515e1fd3   Dan Magenheimer   mm/fs: add hooks ...
32
  #include <linux/cleancache.h>
4db96b71e   Akinobu Mita   vfs: guard end of...
33
  #include "internal.h"
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
34
35
36
37
38
39
40
41
42
43
44
45
46
  
  /*
   * I/O completion handler for multipage BIOs.
   *
   * The mpage code never puts partial pages into a BIO (except for end-of-file).
   * If a page does not map to a contiguous run of blocks then it simply falls
   * back to block_read_full_page().
   *
   * Why is this?  If a page's completion depends on a number of different BIOs
   * which can complete in any order (or at the same time) then determining the
   * status of that page is hard.  See end_buffer_async_read() for the details.
   * There is no point in duplicating all that complexity.
   */
4246a0b63   Christoph Hellwig   block: add a bi_e...
47
  static void mpage_end_io(struct bio *bio)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
48
  {
2c30c71bd   Kent Overstreet   block: Convert va...
49
  	struct bio_vec *bv;
6dc4f100c   Ming Lei   block: allow bio_...
50
  	struct bvec_iter_all iter_all;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
51

2b070cfe5   Christoph Hellwig   block: remove the...
52
  	bio_for_each_segment_all(bv, bio, iter_all) {
2c30c71bd   Kent Overstreet   block: Convert va...
53
  		struct page *page = bv->bv_page;
3f289dcb4   Tejun Heo   block: make bdev_...
54
55
  		page_endio(page, bio_op(bio),
  			   blk_status_to_errno(bio->bi_status));
2c30c71bd   Kent Overstreet   block: Convert va...
56
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
57
  	bio_put(bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
58
  }
eed25cd5b   Mike Christie   mpage: use bio op...
59
  static struct bio *mpage_bio_submit(int op, int op_flags, struct bio *bio)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
60
  {
c32b0d4b3   Hai Shan   fs/mpage.c: conso...
61
  	bio->bi_end_io = mpage_end_io;
eed25cd5b   Mike Christie   mpage: use bio op...
62
  	bio_set_op_attrs(bio, op, op_flags);
83c9c5471   Ming Lei   fs: move guard_bi...
63
  	guard_bio_eod(bio);
4e49ea4a3   Mike Christie   block/fs/drivers:...
64
  	submit_bio(bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
65
66
67
68
69
70
  	return NULL;
  }
  
  static struct bio *
  mpage_alloc(struct block_device *bdev,
  		sector_t first_sector, int nr_vecs,
dd0fc66fb   Al Viro   [PATCH] gfp flags...
71
  		gfp_t gfp_flags)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
72
73
  {
  	struct bio *bio;
8a5c743e3   Michal Hocko   mm, memcg: use co...
74
75
  	/* Restrict the given (page cache) mask for slab allocations */
  	gfp_flags &= GFP_KERNEL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
76
77
78
79
80
81
82
83
  	bio = bio_alloc(gfp_flags, nr_vecs);
  
  	if (bio == NULL && (current->flags & PF_MEMALLOC)) {
  		while (!bio && (nr_vecs /= 2))
  			bio = bio_alloc(gfp_flags, nr_vecs);
  	}
  
  	if (bio) {
74d46992e   Christoph Hellwig   block: replace bi...
84
  		bio_set_dev(bio, bdev);
4f024f379   Kent Overstreet   block: Abstract o...
85
  		bio->bi_iter.bi_sector = first_sector;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
86
87
88
89
90
  	}
  	return bio;
  }
  
  /*
d4388340a   Matthew Wilcox (Oracle)   fs: convert mpage...
91
   * support function for mpage_readahead.  The fs supplied get_block might
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
   * return an up to date buffer.  This is used to map that buffer into
   * the page, which allows readpage to avoid triggering a duplicate call
   * to get_block.
   *
   * The idea is to avoid adding buffers to pages that don't already have
   * them.  So when the buffer is up to date and the page size == block size,
   * this marks the page up to date instead of adding new buffers.
   */
  static void 
  map_buffer_to_page(struct page *page, struct buffer_head *bh, int page_block) 
  {
  	struct inode *inode = page->mapping->host;
  	struct buffer_head *page_bh, *head;
  	int block = 0;
  
  	if (!page_has_buffers(page)) {
  		/*
  		 * don't make any buffers if there is only one buffer on
  		 * the page and the page just needs to be set up to date
  		 */
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
112
  		if (inode->i_blkbits == PAGE_SHIFT &&
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
113
114
115
116
  		    buffer_uptodate(bh)) {
  			SetPageUptodate(page);    
  			return;
  		}
93407472a   Fabian Frederick   fs: add i_blocksi...
117
  		create_empty_buffers(page, i_blocksize(inode), 0);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
118
119
120
121
122
123
124
125
126
127
128
129
130
131
  	}
  	head = page_buffers(page);
  	page_bh = head;
  	do {
  		if (block == page_block) {
  			page_bh->b_state = bh->b_state;
  			page_bh->b_bdev = bh->b_bdev;
  			page_bh->b_blocknr = bh->b_blocknr;
  			break;
  		}
  		page_bh = page_bh->b_this_page;
  		block++;
  	} while (page_bh != head);
  }
357c12065   Jens Axboe   mpage: add argume...
132
133
134
135
  struct mpage_readpage_args {
  	struct bio *bio;
  	struct page *page;
  	unsigned int nr_pages;
74c8164e1   Jens Axboe   mpage: mpage_read...
136
  	bool is_readahead;
357c12065   Jens Axboe   mpage: add argume...
137
138
139
140
  	sector_t last_block_in_bio;
  	struct buffer_head map_bh;
  	unsigned long first_logical_block;
  	get_block_t *get_block;
357c12065   Jens Axboe   mpage: add argume...
141
  };
fa30bd058   Badari Pulavarty   [PATCH] map multi...
142
143
144
145
146
147
148
149
150
  /*
   * This is the worker routine which does all the work of mapping the disk
   * blocks and constructs largest possible bios, submits them for IO if the
   * blocks are not contiguous on the disk.
   *
   * We pass a buffer_head back and forth and use its buffer_mapped() flag to
   * represent the validity of its disk mapping and to decide when to do the next
   * get_block() call.
   */
357c12065   Jens Axboe   mpage: add argume...
151
  static struct bio *do_mpage_readpage(struct mpage_readpage_args *args)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
152
  {
357c12065   Jens Axboe   mpage: add argume...
153
  	struct page *page = args->page;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
154
155
  	struct inode *inode = page->mapping->host;
  	const unsigned blkbits = inode->i_blkbits;
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
156
  	const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
157
  	const unsigned blocksize = 1 << blkbits;
357c12065   Jens Axboe   mpage: add argume...
158
  	struct buffer_head *map_bh = &args->map_bh;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
159
160
  	sector_t block_in_file;
  	sector_t last_block;
fa30bd058   Badari Pulavarty   [PATCH] map multi...
161
  	sector_t last_block_in_file;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
162
163
164
165
  	sector_t blocks[MAX_BUF_PER_PAGE];
  	unsigned page_block;
  	unsigned first_hole = blocks_per_page;
  	struct block_device *bdev = NULL;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
166
167
  	int length;
  	int fully_mapped = 1;
74c8164e1   Jens Axboe   mpage: mpage_read...
168
  	int op_flags;
fa30bd058   Badari Pulavarty   [PATCH] map multi...
169
170
  	unsigned nblocks;
  	unsigned relative_block;
74c8164e1   Jens Axboe   mpage: mpage_read...
171
172
173
174
175
176
177
178
179
  	gfp_t gfp;
  
  	if (args->is_readahead) {
  		op_flags = REQ_RAHEAD;
  		gfp = readahead_gfp_mask(page->mapping);
  	} else {
  		op_flags = 0;
  		gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
180
181
182
  
  	if (page_has_buffers(page))
  		goto confused;
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
183
  	block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
357c12065   Jens Axboe   mpage: add argume...
184
  	last_block = block_in_file + args->nr_pages * blocks_per_page;
fa30bd058   Badari Pulavarty   [PATCH] map multi...
185
186
187
188
189
190
191
192
193
  	last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
  	if (last_block > last_block_in_file)
  		last_block = last_block_in_file;
  	page_block = 0;
  
  	/*
  	 * Map blocks using the result from the previous get_blocks call first.
  	 */
  	nblocks = map_bh->b_size >> blkbits;
357c12065   Jens Axboe   mpage: add argume...
194
195
196
197
  	if (buffer_mapped(map_bh) &&
  			block_in_file > args->first_logical_block &&
  			block_in_file < (args->first_logical_block + nblocks)) {
  		unsigned map_offset = block_in_file - args->first_logical_block;
fa30bd058   Badari Pulavarty   [PATCH] map multi...
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
  		unsigned last = nblocks - map_offset;
  
  		for (relative_block = 0; ; relative_block++) {
  			if (relative_block == last) {
  				clear_buffer_mapped(map_bh);
  				break;
  			}
  			if (page_block == blocks_per_page)
  				break;
  			blocks[page_block] = map_bh->b_blocknr + map_offset +
  						relative_block;
  			page_block++;
  			block_in_file++;
  		}
  		bdev = map_bh->b_bdev;
  	}
  
  	/*
  	 * Then do more get_blocks calls until we are done with this page.
  	 */
  	map_bh->b_page = page;
  	while (page_block < blocks_per_page) {
  		map_bh->b_state = 0;
  		map_bh->b_size = 0;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
222

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
223
  		if (block_in_file < last_block) {
fa30bd058   Badari Pulavarty   [PATCH] map multi...
224
  			map_bh->b_size = (last_block-block_in_file) << blkbits;
357c12065   Jens Axboe   mpage: add argume...
225
  			if (args->get_block(inode, block_in_file, map_bh, 0))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
226
  				goto confused;
357c12065   Jens Axboe   mpage: add argume...
227
  			args->first_logical_block = block_in_file;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
228
  		}
fa30bd058   Badari Pulavarty   [PATCH] map multi...
229
  		if (!buffer_mapped(map_bh)) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
230
231
232
  			fully_mapped = 0;
  			if (first_hole == blocks_per_page)
  				first_hole = page_block;
fa30bd058   Badari Pulavarty   [PATCH] map multi...
233
234
  			page_block++;
  			block_in_file++;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
235
236
237
238
239
240
241
242
243
  			continue;
  		}
  
  		/* some filesystems will copy data into the page during
  		 * the get_block call, in which case we don't want to
  		 * read it again.  map_buffer_to_page copies the data
  		 * we just collected from get_block into the page's buffers
  		 * so readpage doesn't have to repeat the get_block call
  		 */
fa30bd058   Badari Pulavarty   [PATCH] map multi...
244
245
  		if (buffer_uptodate(map_bh)) {
  			map_buffer_to_page(page, map_bh, page_block);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
246
247
248
249
250
251
252
  			goto confused;
  		}
  	
  		if (first_hole != blocks_per_page)
  			goto confused;		/* hole -> non-hole */
  
  		/* Contiguous blocks? */
fa30bd058   Badari Pulavarty   [PATCH] map multi...
253
  		if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
254
  			goto confused;
fa30bd058   Badari Pulavarty   [PATCH] map multi...
255
256
257
258
259
260
261
262
263
264
265
266
  		nblocks = map_bh->b_size >> blkbits;
  		for (relative_block = 0; ; relative_block++) {
  			if (relative_block == nblocks) {
  				clear_buffer_mapped(map_bh);
  				break;
  			} else if (page_block == blocks_per_page)
  				break;
  			blocks[page_block] = map_bh->b_blocknr+relative_block;
  			page_block++;
  			block_in_file++;
  		}
  		bdev = map_bh->b_bdev;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
267
268
269
  	}
  
  	if (first_hole != blocks_per_page) {
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
270
  		zero_user_segment(page, first_hole << blkbits, PAGE_SIZE);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
271
272
273
274
275
276
277
278
  		if (first_hole == 0) {
  			SetPageUptodate(page);
  			unlock_page(page);
  			goto out;
  		}
  	} else if (fully_mapped) {
  		SetPageMappedToDisk(page);
  	}
c515e1fd3   Dan Magenheimer   mm/fs: add hooks ...
279
280
281
282
283
  	if (fully_mapped && blocks_per_page == 1 && !PageUptodate(page) &&
  	    cleancache_get_page(page) == 0) {
  		SetPageUptodate(page);
  		goto confused;
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
284
285
286
  	/*
  	 * This page will go to BIO.  Do we need to send this BIO off first?
  	 */
357c12065   Jens Axboe   mpage: add argume...
287
  	if (args->bio && (args->last_block_in_bio != blocks[0] - 1))
74c8164e1   Jens Axboe   mpage: mpage_read...
288
  		args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
289
290
  
  alloc_new:
357c12065   Jens Axboe   mpage: add argume...
291
  	if (args->bio == NULL) {
47a191fd3   Matthew Wilcox   fs/block_dev.c: a...
292
293
294
295
296
  		if (first_hole == blocks_per_page) {
  			if (!bdev_read_page(bdev, blocks[0] << (blkbits - 9),
  								page))
  				goto out;
  		}
357c12065   Jens Axboe   mpage: add argume...
297
298
299
  		args->bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
  					min_t(int, args->nr_pages,
  					      BIO_MAX_PAGES),
74c8164e1   Jens Axboe   mpage: mpage_read...
300
  					gfp);
357c12065   Jens Axboe   mpage: add argume...
301
  		if (args->bio == NULL)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
302
303
304
305
  			goto confused;
  	}
  
  	length = first_hole << blkbits;
357c12065   Jens Axboe   mpage: add argume...
306
  	if (bio_add_page(args->bio, page, length, 0) < length) {
74c8164e1   Jens Axboe   mpage: mpage_read...
307
  		args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
308
309
  		goto alloc_new;
  	}
357c12065   Jens Axboe   mpage: add argume...
310
  	relative_block = block_in_file - args->first_logical_block;
38c8e6180   Miquel van Smoorenburg   do_mpage_readpage...
311
312
313
  	nblocks = map_bh->b_size >> blkbits;
  	if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
  	    (first_hole != blocks_per_page))
74c8164e1   Jens Axboe   mpage: mpage_read...
314
  		args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
315
  	else
357c12065   Jens Axboe   mpage: add argume...
316
  		args->last_block_in_bio = blocks[blocks_per_page - 1];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
317
  out:
357c12065   Jens Axboe   mpage: add argume...
318
  	return args->bio;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
319
320
  
  confused:
357c12065   Jens Axboe   mpage: add argume...
321
  	if (args->bio)
74c8164e1   Jens Axboe   mpage: mpage_read...
322
  		args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
323
  	if (!PageUptodate(page))
357c12065   Jens Axboe   mpage: add argume...
324
  		block_read_full_page(page, args->get_block);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
325
326
327
328
  	else
  		unlock_page(page);
  	goto out;
  }
67be2dd1b   Martin Waitz   [PATCH] DocBook: ...
329
  /**
d4388340a   Matthew Wilcox (Oracle)   fs: convert mpage...
330
331
   * mpage_readahead - start reads against pages
   * @rac: Describes which pages to read.
67be2dd1b   Martin Waitz   [PATCH] DocBook: ...
332
333
334
335
336
337
338
339
340
341
342
343
344
   * @get_block: The filesystem's block mapper function.
   *
   * This function walks the pages and the blocks within each page, building and
   * emitting large BIOs.
   *
   * If anything unusual happens, such as:
   *
   * - encountering a page which has buffers
   * - encountering a page which has a non-hole after a hole
   * - encountering a page with non-contiguous blocks
   *
   * then this code just gives up and calls the buffer_head-based read function.
   * It does handle a page which has holes at the end - that is a common case:
ea1754a08   Kirill A. Shutemov   mm, fs: remove re...
345
   * the end-of-file on blocksize < PAGE_SIZE setups.
67be2dd1b   Martin Waitz   [PATCH] DocBook: ...
346
347
348
349
350
351
352
353
354
   *
   * BH_Boundary explanation:
   *
   * There is a problem.  The mpage read code assembles several pages, gets all
   * their disk mappings, and then submits them all.  That's fine, but obtaining
   * the disk mappings may require I/O.  Reads of indirect blocks, for example.
   *
   * So an mpage read of the first 16 blocks of an ext2 file will cause I/O to be
   * submitted in the following order:
0117d4272   Mauro Carvalho Chehab   fs: add a blank l...
355
   *
67be2dd1b   Martin Waitz   [PATCH] DocBook: ...
356
   * 	12 0 1 2 3 4 5 6 7 8 9 10 11 13 14 15 16
78a4a50a8   Randy Dunlap   docbook: fix file...
357
   *
67be2dd1b   Martin Waitz   [PATCH] DocBook: ...
358
359
360
361
362
363
364
365
366
367
   * because the indirect block has to be read to get the mappings of blocks
   * 13,14,15,16.  Obviously, this impacts performance.
   *
   * So what we do it to allow the filesystem's get_block() function to set
   * BH_Boundary when it maps block 11.  BH_Boundary says: mapping of the block
   * after this one will require I/O against a block which is probably close to
   * this one.  So you should push what I/O you have currently accumulated.
   *
   * This all causes the disk requests to be issued in the correct order.
   */
d4388340a   Matthew Wilcox (Oracle)   fs: convert mpage...
368
  void mpage_readahead(struct readahead_control *rac, get_block_t get_block)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
369
  {
d4388340a   Matthew Wilcox (Oracle)   fs: convert mpage...
370
  	struct page *page;
357c12065   Jens Axboe   mpage: add argume...
371
372
  	struct mpage_readpage_args args = {
  		.get_block = get_block,
74c8164e1   Jens Axboe   mpage: mpage_read...
373
  		.is_readahead = true,
357c12065   Jens Axboe   mpage: add argume...
374
  	};
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
375

d4388340a   Matthew Wilcox (Oracle)   fs: convert mpage...
376
  	while ((page = readahead_page(rac))) {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
377
  		prefetchw(&page->flags);
d4388340a   Matthew Wilcox (Oracle)   fs: convert mpage...
378
379
380
  		args.page = page;
  		args.nr_pages = readahead_count(rac);
  		args.bio = do_mpage_readpage(&args);
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
381
  		put_page(page);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
382
  	}
357c12065   Jens Axboe   mpage: add argume...
383
  	if (args.bio)
74c8164e1   Jens Axboe   mpage: mpage_read...
384
  		mpage_bio_submit(REQ_OP_READ, REQ_RAHEAD, args.bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
385
  }
d4388340a   Matthew Wilcox (Oracle)   fs: convert mpage...
386
  EXPORT_SYMBOL(mpage_readahead);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
387
388
389
390
391
392
  
  /*
   * This isn't called much at all
   */
  int mpage_readpage(struct page *page, get_block_t get_block)
  {
357c12065   Jens Axboe   mpage: add argume...
393
394
395
396
  	struct mpage_readpage_args args = {
  		.page = page,
  		.nr_pages = 1,
  		.get_block = get_block,
357c12065   Jens Axboe   mpage: add argume...
397
398
399
400
401
  	};
  
  	args.bio = do_mpage_readpage(&args);
  	if (args.bio)
  		mpage_bio_submit(REQ_OP_READ, 0, args.bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
  	return 0;
  }
  EXPORT_SYMBOL(mpage_readpage);
  
  /*
   * Writing is not so simple.
   *
   * If the page has buffers then they will be used for obtaining the disk
   * mapping.  We only support pages which are fully mapped-and-dirty, with a
   * special case for pages which are unmapped at the end: end-of-file.
   *
   * If the page has no buffers (preferred) then the page is mapped here.
   *
   * If all blocks are found to be contiguous then the page can go into the
   * BIO.  Otherwise fall back to the mapping's writepage().
   * 
   * FIXME: This code wants an estimate of how many pages are still to be
   * written, so it can intelligently allocate a suitably-sized BIO.  For now,
   * just allocate full-size (16-page) BIOs.
   */
0ea971801   Miklos Szeredi   consolidate gener...
422

ced117c73   Dmitri Vorobiev   Remove two unneed...
423
424
425
426
427
428
  struct mpage_data {
  	struct bio *bio;
  	sector_t last_block_in_bio;
  	get_block_t *get_block;
  	unsigned use_writepage;
  };
90768eee4   Matthew Wilcox   fs/mpage.c: facto...
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
  /*
   * We have our BIO, so we can now mark the buffers clean.  Make
   * sure to only clean buffers which we know we'll be writing.
   */
  static void clean_buffers(struct page *page, unsigned first_unmapped)
  {
  	unsigned buffer_counter = 0;
  	struct buffer_head *bh, *head;
  	if (!page_has_buffers(page))
  		return;
  	head = page_buffers(page);
  	bh = head;
  
  	do {
  		if (buffer_counter++ == first_unmapped)
  			break;
  		clear_buffer_dirty(bh);
  		bh = bh->b_this_page;
  	} while (bh != head);
  
  	/*
  	 * we cannot drop the bh if the page is not uptodate or a concurrent
  	 * readpage would fail to serialize with the bh and it would read from
  	 * disk before we reach the platter.
  	 */
  	if (buffer_heads_over_limit && PageUptodate(page))
  		try_to_free_buffers(page);
  }
f892760aa   Matthew Wilcox   fs/mpage.c: fix m...
457
458
459
460
461
462
463
464
465
  /*
   * For situations where we want to clean all buffers attached to a page.
   * We don't need to calculate how many buffers are attached to the page,
   * we just need to specify a number larger than the maximum number of buffers.
   */
  void clean_page_buffers(struct page *page)
  {
  	clean_buffers(page, ~0U);
  }
ced117c73   Dmitri Vorobiev   Remove two unneed...
466
  static int __mpage_writepage(struct page *page, struct writeback_control *wbc,
29a814d2e   Alex Tomas   vfs: add hooks fo...
467
  		      void *data)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
468
  {
0ea971801   Miklos Szeredi   consolidate gener...
469
470
  	struct mpage_data *mpd = data;
  	struct bio *bio = mpd->bio;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
471
472
473
474
  	struct address_space *mapping = page->mapping;
  	struct inode *inode = page->mapping->host;
  	const unsigned blkbits = inode->i_blkbits;
  	unsigned long end_index;
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
475
  	const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
476
477
478
479
480
481
482
483
484
485
486
487
  	sector_t last_block;
  	sector_t block_in_file;
  	sector_t blocks[MAX_BUF_PER_PAGE];
  	unsigned page_block;
  	unsigned first_unmapped = blocks_per_page;
  	struct block_device *bdev = NULL;
  	int boundary = 0;
  	sector_t boundary_block = 0;
  	struct block_device *boundary_bdev = NULL;
  	int length;
  	struct buffer_head map_bh;
  	loff_t i_size = i_size_read(inode);
0ea971801   Miklos Szeredi   consolidate gener...
488
  	int ret = 0;
7637241e6   Jens Axboe   writeback: add wb...
489
  	int op_flags = wbc_to_write_flags(wbc);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
  
  	if (page_has_buffers(page)) {
  		struct buffer_head *head = page_buffers(page);
  		struct buffer_head *bh = head;
  
  		/* If they're all mapped and dirty, do it */
  		page_block = 0;
  		do {
  			BUG_ON(buffer_locked(bh));
  			if (!buffer_mapped(bh)) {
  				/*
  				 * unmapped dirty buffers are created by
  				 * __set_page_dirty_buffers -> mmapped data
  				 */
  				if (buffer_dirty(bh))
  					goto confused;
  				if (first_unmapped == blocks_per_page)
  					first_unmapped = page_block;
  				continue;
  			}
  
  			if (first_unmapped != blocks_per_page)
  				goto confused;	/* hole -> non-hole */
  
  			if (!buffer_dirty(bh) || !buffer_uptodate(bh))
  				goto confused;
  			if (page_block) {
  				if (bh->b_blocknr != blocks[page_block-1] + 1)
  					goto confused;
  			}
  			blocks[page_block++] = bh->b_blocknr;
  			boundary = buffer_boundary(bh);
  			if (boundary) {
  				boundary_block = bh->b_blocknr;
  				boundary_bdev = bh->b_bdev;
  			}
  			bdev = bh->b_bdev;
  		} while ((bh = bh->b_this_page) != head);
  
  		if (first_unmapped)
  			goto page_is_mapped;
  
  		/*
  		 * Page has buffers, but they are all unmapped. The page was
  		 * created by pagein or read over a hole which was handled by
  		 * block_read_full_page().  If this address_space is also
d4388340a   Matthew Wilcox (Oracle)   fs: convert mpage...
536
  		 * using mpage_readahead then this can rarely happen.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
537
538
539
540
541
542
543
544
  		 */
  		goto confused;
  	}
  
  	/*
  	 * The page has no buffers: map it to disk
  	 */
  	BUG_ON(!PageUptodate(page));
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
545
  	block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
546
547
548
549
550
  	last_block = (i_size - 1) >> blkbits;
  	map_bh.b_page = page;
  	for (page_block = 0; page_block < blocks_per_page; ) {
  
  		map_bh.b_state = 0;
b0cf2321c   Badari Pulavarty   [PATCH] pass b_si...
551
  		map_bh.b_size = 1 << blkbits;
0ea971801   Miklos Szeredi   consolidate gener...
552
  		if (mpd->get_block(inode, block_in_file, &map_bh, 1))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
553
554
  			goto confused;
  		if (buffer_new(&map_bh))
e64855c6c   Jan Kara   fs: Add helper to...
555
  			clean_bdev_bh_alias(&map_bh);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
  		if (buffer_boundary(&map_bh)) {
  			boundary_block = map_bh.b_blocknr;
  			boundary_bdev = map_bh.b_bdev;
  		}
  		if (page_block) {
  			if (map_bh.b_blocknr != blocks[page_block-1] + 1)
  				goto confused;
  		}
  		blocks[page_block++] = map_bh.b_blocknr;
  		boundary = buffer_boundary(&map_bh);
  		bdev = map_bh.b_bdev;
  		if (block_in_file == last_block)
  			break;
  		block_in_file++;
  	}
  	BUG_ON(page_block == 0);
  
  	first_unmapped = page_block;
  
  page_is_mapped:
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
576
  	end_index = i_size >> PAGE_SHIFT;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
577
578
579
  	if (page->index >= end_index) {
  		/*
  		 * The page straddles i_size.  It must be zeroed out on each
2a61aa401   Adam Buchbinder   Fix misspellings ...
580
  		 * and every writepage invocation because it may be mmapped.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
581
582
583
584
585
  		 * "A file is mapped in multiples of the page size.  For a file
  		 * that is not a multiple of the page size, the remaining memory
  		 * is zeroed when mapped, and writes to that region are not
  		 * written out to the file."
  		 */
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
586
  		unsigned offset = i_size & (PAGE_SIZE - 1);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
587
588
589
  
  		if (page->index > end_index || !offset)
  			goto confused;
09cbfeaf1   Kirill A. Shutemov   mm, fs: get rid o...
590
  		zero_user_segment(page, offset, PAGE_SIZE);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
591
592
593
594
595
  	}
  
  	/*
  	 * This page will go to BIO.  Do we need to send this BIO off first?
  	 */
0ea971801   Miklos Szeredi   consolidate gener...
596
  	if (bio && mpd->last_block_in_bio != blocks[0] - 1)
eed25cd5b   Mike Christie   mpage: use bio op...
597
  		bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
598
599
600
  
  alloc_new:
  	if (bio == NULL) {
47a191fd3   Matthew Wilcox   fs/block_dev.c: a...
601
602
  		if (first_unmapped == blocks_per_page) {
  			if (!bdev_write_page(bdev, blocks[0] << (blkbits - 9),
f892760aa   Matthew Wilcox   fs/mpage.c: fix m...
603
  								page, wbc))
47a191fd3   Matthew Wilcox   fs/block_dev.c: a...
604
  				goto out;
47a191fd3   Matthew Wilcox   fs/block_dev.c: a...
605
  		}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
606
  		bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
b54ffb73c   Kent Overstreet   block: remove bio...
607
  				BIO_MAX_PAGES, GFP_NOFS|__GFP_HIGH);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
608
609
  		if (bio == NULL)
  			goto confused;
429b3fb02   Tejun Heo   mpage: make __mpa...
610

b16b1deb5   Tejun Heo   writeback: make w...
611
  		wbc_init_bio(wbc, bio);
8e8f92988   Jens Axboe   fs: add support f...
612
  		bio->bi_write_hint = inode->i_write_hint;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
613
614
615
616
617
618
619
  	}
  
  	/*
  	 * Must try to add the page before marking the buffer clean or
  	 * the confused fail path above (OOM) will be very confused when
  	 * it finds all bh marked clean (i.e. it will not write anything)
  	 */
34e51a5e1   Tejun Heo   blkcg, writeback:...
620
  	wbc_account_cgroup_owner(wbc, page, PAGE_SIZE);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
621
622
  	length = first_unmapped << blkbits;
  	if (bio_add_page(bio, page, length, 0) < length) {
eed25cd5b   Mike Christie   mpage: use bio op...
623
  		bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
624
625
  		goto alloc_new;
  	}
90768eee4   Matthew Wilcox   fs/mpage.c: facto...
626
  	clean_buffers(page, first_unmapped);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
627
628
629
630
631
  
  	BUG_ON(PageWriteback(page));
  	set_page_writeback(page);
  	unlock_page(page);
  	if (boundary || (first_unmapped != blocks_per_page)) {
eed25cd5b   Mike Christie   mpage: use bio op...
632
  		bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
633
634
635
636
637
  		if (boundary_block) {
  			write_boundary_block(boundary_bdev,
  					boundary_block, 1 << blkbits);
  		}
  	} else {
0ea971801   Miklos Szeredi   consolidate gener...
638
  		mpd->last_block_in_bio = blocks[blocks_per_page - 1];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
639
640
641
642
643
  	}
  	goto out;
  
  confused:
  	if (bio)
eed25cd5b   Mike Christie   mpage: use bio op...
644
  		bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
645

0ea971801   Miklos Szeredi   consolidate gener...
646
647
  	if (mpd->use_writepage) {
  		ret = mapping->a_ops->writepage(page, wbc);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
648
  	} else {
0ea971801   Miklos Szeredi   consolidate gener...
649
  		ret = -EAGAIN;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
650
651
652
653
654
  		goto out;
  	}
  	/*
  	 * The caller has a ref on the inode, so *mapping is stable
  	 */
0ea971801   Miklos Szeredi   consolidate gener...
655
  	mapping_set_error(mapping, ret);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
656
  out:
0ea971801   Miklos Szeredi   consolidate gener...
657
658
  	mpd->bio = bio;
  	return ret;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
659
660
661
  }
  
  /**
78a4a50a8   Randy Dunlap   docbook: fix file...
662
   * mpage_writepages - walk the list of dirty pages of the given address space & writepage() all of them
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
   * @mapping: address space structure to write
   * @wbc: subtract the number of written pages from *@wbc->nr_to_write
   * @get_block: the filesystem's block mapper function.
   *             If this is NULL then use a_ops->writepage.  Otherwise, go
   *             direct-to-BIO.
   *
   * This is a library function, which implements the writepages()
   * address_space_operation.
   *
   * If a page is already under I/O, generic_writepages() skips it, even
   * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
   * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
   * and msync() need to guarantee that all the data which was dirty at the time
   * the call was made get new I/O started against them.  If wbc->sync_mode is
   * WB_SYNC_ALL then we were called for data integrity and we must wait for
   * existing IO to complete.
   */
  int
  mpage_writepages(struct address_space *mapping,
  		struct writeback_control *wbc, get_block_t get_block)
  {
2ed1a6bcf   Jens Axboe   fs: make mpage re...
684
  	struct blk_plug plug;
0ea971801   Miklos Szeredi   consolidate gener...
685
  	int ret;
2ed1a6bcf   Jens Axboe   fs: make mpage re...
686
  	blk_start_plug(&plug);
0ea971801   Miklos Szeredi   consolidate gener...
687
688
689
690
691
692
693
694
695
696
697
  	if (!get_block)
  		ret = generic_writepages(mapping, wbc);
  	else {
  		struct mpage_data mpd = {
  			.bio = NULL,
  			.last_block_in_bio = 0,
  			.get_block = get_block,
  			.use_writepage = 1,
  		};
  
  		ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
5948edbcb   Roman Pen   fs/mpage.c: forgo...
698
  		if (mpd.bio) {
eed25cd5b   Mike Christie   mpage: use bio op...
699
  			int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
70fd76140   Christoph Hellwig   block,fs: use REQ...
700
  				  REQ_SYNC : 0);
eed25cd5b   Mike Christie   mpage: use bio op...
701
  			mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
5948edbcb   Roman Pen   fs/mpage.c: forgo...
702
  		}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
703
  	}
2ed1a6bcf   Jens Axboe   fs: make mpage re...
704
  	blk_finish_plug(&plug);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
705
706
707
  	return ret;
  }
  EXPORT_SYMBOL(mpage_writepages);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
708
709
710
711
  
  int mpage_writepage(struct page *page, get_block_t get_block,
  	struct writeback_control *wbc)
  {
0ea971801   Miklos Szeredi   consolidate gener...
712
713
714
715
716
717
718
  	struct mpage_data mpd = {
  		.bio = NULL,
  		.last_block_in_bio = 0,
  		.get_block = get_block,
  		.use_writepage = 0,
  	};
  	int ret = __mpage_writepage(page, wbc, &mpd);
5948edbcb   Roman Pen   fs/mpage.c: forgo...
719
  	if (mpd.bio) {
eed25cd5b   Mike Christie   mpage: use bio op...
720
  		int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
70fd76140   Christoph Hellwig   block,fs: use REQ...
721
  			  REQ_SYNC : 0);
eed25cd5b   Mike Christie   mpage: use bio op...
722
  		mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
5948edbcb   Roman Pen   fs/mpage.c: forgo...
723
  	}
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
724
725
726
  	return ret;
  }
  EXPORT_SYMBOL(mpage_writepage);