Blame view
block/blk-lib.c
10.3 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
f31e7e402 blkdev: move blkd... |
2 3 4 5 6 7 8 9 10 11 |
/* * Functions related to generic helpers functions */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/scatterlist.h> #include "blk.h" |
4e49ea4a3 block/fs/drivers:... |
12 |
static struct bio *next_bio(struct bio *bio, unsigned int nr_pages, |
9082e87bf block: remove str... |
13 |
gfp_t gfp) |
f31e7e402 blkdev: move blkd... |
14 |
{ |
9082e87bf block: remove str... |
15 16 17 18 |
struct bio *new = bio_alloc(gfp, nr_pages); if (bio) { bio_chain(bio, new); |
4e49ea4a3 block/fs/drivers:... |
19 |
submit_bio(bio); |
9082e87bf block: remove str... |
20 |
} |
5dba3089e blkdev: Submit di... |
21 |
|
9082e87bf block: remove str... |
22 |
return new; |
f31e7e402 blkdev: move blkd... |
23 |
} |
38f252553 block: add __blkd... |
24 |
int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, |
288dab8a3 block: add a sepa... |
25 |
sector_t nr_sects, gfp_t gfp_mask, int flags, |
469e3216e block discard: us... |
26 |
struct bio **biop) |
f31e7e402 blkdev: move blkd... |
27 |
{ |
f31e7e402 blkdev: move blkd... |
28 |
struct request_queue *q = bdev_get_queue(bdev); |
38f252553 block: add __blkd... |
29 |
struct bio *bio = *biop; |
ef295ecf0 block: better op ... |
30 |
unsigned int op; |
28b2be203 block: require wr... |
31 |
sector_t bs_mask; |
f31e7e402 blkdev: move blkd... |
32 33 34 |
if (!q) return -ENXIO; |
288dab8a3 block: add a sepa... |
35 |
|
a13553c77 block: add bdev_r... |
36 37 |
if (bdev_read_only(bdev)) return -EPERM; |
288dab8a3 block: add a sepa... |
38 39 40 41 42 43 44 45 46 |
if (flags & BLKDEV_DISCARD_SECURE) { if (!blk_queue_secure_erase(q)) return -EOPNOTSUPP; op = REQ_OP_SECURE_ERASE; } else { if (!blk_queue_discard(q)) return -EOPNOTSUPP; op = REQ_OP_DISCARD; } |
f31e7e402 blkdev: move blkd... |
47 |
|
28b2be203 block: require wr... |
48 49 50 |
bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; if ((sector | nr_sects) & bs_mask) return -EINVAL; |
5dba3089e blkdev: Submit di... |
51 |
while (nr_sects) { |
744889b7c block: don't deal... |
52 53 |
unsigned int req_sects = nr_sects; sector_t end_sect; |
c6e666345 block: split disc... |
54 |
|
b88aef36b block: fix infini... |
55 56 |
if (!req_sects) goto fail; |
14657efd3 block: make sure ... |
57 |
req_sects = min(req_sects, bio_allowed_max_sectors(q)); |
a22c4d7e3 block: re-add dis... |
58 |
|
c6e666345 block: split disc... |
59 |
end_sect = sector + req_sects; |
c6e666345 block: split disc... |
60 |
|
f9d03f96b block: improve ha... |
61 |
bio = next_bio(bio, 0, gfp_mask); |
4f024f379 block: Abstract o... |
62 |
bio->bi_iter.bi_sector = sector; |
74d46992e block: replace bi... |
63 |
bio_set_dev(bio, bdev); |
288dab8a3 block: add a sepa... |
64 |
bio_set_op_attrs(bio, op, 0); |
f31e7e402 blkdev: move blkd... |
65 |
|
4f024f379 block: Abstract o... |
66 |
bio->bi_iter.bi_size = req_sects << 9; |
c6e666345 block: split disc... |
67 68 |
nr_sects -= req_sects; sector = end_sect; |
f31e7e402 blkdev: move blkd... |
69 |
|
c8123f8c9 block: add cond_r... |
70 71 72 73 74 75 76 |
/* * We can loop for a long time in here, if someone does * full device discards (like mkfs). Be nice and allow * us to schedule out to avoid softlocking if preempt * is disabled. */ cond_resched(); |
5dba3089e blkdev: Submit di... |
77 |
} |
38f252553 block: add __blkd... |
78 79 80 |
*biop = bio; return 0; |
b88aef36b block: fix infini... |
81 82 83 84 85 86 87 88 |
fail: if (bio) { submit_bio_wait(bio); bio_put(bio); } *biop = NULL; return -EOPNOTSUPP; |
38f252553 block: add __blkd... |
89 90 91 92 93 94 95 96 97 |
} EXPORT_SYMBOL(__blkdev_issue_discard); /** * blkdev_issue_discard - queue a discard * @bdev: blockdev to issue discard for * @sector: start sector * @nr_sects: number of sectors to discard * @gfp_mask: memory allocation flags (for bio_alloc) |
e554911c2 block: correct do... |
98 |
* @flags: BLKDEV_DISCARD_* flags to control behaviour |
38f252553 block: add __blkd... |
99 100 101 102 103 104 105 |
* * Description: * Issue a discard request for the sectors in question. */ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, unsigned long flags) { |
38f252553 block: add __blkd... |
106 107 108 |
struct bio *bio = NULL; struct blk_plug plug; int ret; |
38f252553 block: add __blkd... |
109 |
blk_start_plug(&plug); |
288dab8a3 block: add a sepa... |
110 |
ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags, |
38f252553 block: add __blkd... |
111 |
&bio); |
bbd848e0f block: reinstate ... |
112 |
if (!ret && bio) { |
4e49ea4a3 block/fs/drivers:... |
113 |
ret = submit_bio_wait(bio); |
48920ff2a block: remove the... |
114 |
if (ret == -EOPNOTSUPP) |
bbd848e0f block: reinstate ... |
115 |
ret = 0; |
05bd92ddd block: missing bi... |
116 |
bio_put(bio); |
bbd848e0f block: reinstate ... |
117 |
} |
0cfbcafca block: add plug f... |
118 |
blk_finish_plug(&plug); |
f31e7e402 blkdev: move blkd... |
119 |
|
bbd848e0f block: reinstate ... |
120 |
return ret; |
f31e7e402 blkdev: move blkd... |
121 122 |
} EXPORT_SYMBOL(blkdev_issue_discard); |
3f14d792f blkdev: add blkde... |
123 |
|
3f14d792f blkdev: add blkde... |
124 |
/** |
e73c23ff7 block: add async ... |
125 |
* __blkdev_issue_write_same - generate number of bios with same page |
4363ac7c1 block: Implement ... |
126 127 128 129 130 |
* @bdev: target blockdev * @sector: start sector * @nr_sects: number of sectors to write * @gfp_mask: memory allocation flags (for bio_alloc) * @page: page containing data to write |
e73c23ff7 block: add async ... |
131 |
* @biop: pointer to anchor bio |
4363ac7c1 block: Implement ... |
132 133 |
* * Description: |
e73c23ff7 block: add async ... |
134 |
* Generate and issue number of bios(REQ_OP_WRITE_SAME) with same page. |
4363ac7c1 block: Implement ... |
135 |
*/ |
e73c23ff7 block: add async ... |
136 137 138 |
static int __blkdev_issue_write_same(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct page *page, struct bio **biop) |
4363ac7c1 block: Implement ... |
139 |
{ |
4363ac7c1 block: Implement ... |
140 141 |
struct request_queue *q = bdev_get_queue(bdev); unsigned int max_write_same_sectors; |
e73c23ff7 block: add async ... |
142 |
struct bio *bio = *biop; |
28b2be203 block: require wr... |
143 |
sector_t bs_mask; |
4363ac7c1 block: Implement ... |
144 145 146 |
if (!q) return -ENXIO; |
a13553c77 block: add bdev_r... |
147 148 |
if (bdev_read_only(bdev)) return -EPERM; |
28b2be203 block: require wr... |
149 150 151 |
bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; if ((sector | nr_sects) & bs_mask) return -EINVAL; |
e73c23ff7 block: add async ... |
152 153 |
if (!bdev_write_same(bdev)) return -EOPNOTSUPP; |
b49a0871b block: remove spl... |
154 |
/* Ensure that max_write_same_sectors doesn't overflow bi_size */ |
1ea5c403d block: make sure ... |
155 |
max_write_same_sectors = bio_allowed_max_sectors(q); |
4363ac7c1 block: Implement ... |
156 |
|
4363ac7c1 block: Implement ... |
157 |
while (nr_sects) { |
4e49ea4a3 block/fs/drivers:... |
158 |
bio = next_bio(bio, 1, gfp_mask); |
4f024f379 block: Abstract o... |
159 |
bio->bi_iter.bi_sector = sector; |
74d46992e block: replace bi... |
160 |
bio_set_dev(bio, bdev); |
4363ac7c1 block: Implement ... |
161 162 163 164 |
bio->bi_vcnt = 1; bio->bi_io_vec->bv_page = page; bio->bi_io_vec->bv_offset = 0; bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev); |
95fe6c1a2 block, fs, mm, dr... |
165 |
bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0); |
4363ac7c1 block: Implement ... |
166 167 |
if (nr_sects > max_write_same_sectors) { |
4f024f379 block: Abstract o... |
168 |
bio->bi_iter.bi_size = max_write_same_sectors << 9; |
4363ac7c1 block: Implement ... |
169 170 171 |
nr_sects -= max_write_same_sectors; sector += max_write_same_sectors; } else { |
4f024f379 block: Abstract o... |
172 |
bio->bi_iter.bi_size = nr_sects << 9; |
4363ac7c1 block: Implement ... |
173 174 |
nr_sects = 0; } |
e73c23ff7 block: add async ... |
175 |
cond_resched(); |
4363ac7c1 block: Implement ... |
176 |
} |
e73c23ff7 block: add async ... |
177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 |
*biop = bio; return 0; } /** * blkdev_issue_write_same - queue a write same operation * @bdev: target blockdev * @sector: start sector * @nr_sects: number of sectors to write * @gfp_mask: memory allocation flags (for bio_alloc) * @page: page containing data * * Description: * Issue a write same request for the sectors in question. */ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct page *page) { struct bio *bio = NULL; struct blk_plug plug; int ret; blk_start_plug(&plug); ret = __blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, page, &bio); if (ret == 0 && bio) { |
4e49ea4a3 block/fs/drivers:... |
204 |
ret = submit_bio_wait(bio); |
05bd92ddd block: missing bi... |
205 206 |
bio_put(bio); } |
e73c23ff7 block: add async ... |
207 |
blk_finish_plug(&plug); |
3f40bf2c8 block: don't igno... |
208 |
return ret; |
4363ac7c1 block: Implement ... |
209 210 |
} EXPORT_SYMBOL(blkdev_issue_write_same); |
a6f0788ec block: add suppor... |
211 212 |
static int __blkdev_issue_write_zeroes(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, |
d928be9f8 block: add a REQ_... |
213 |
struct bio **biop, unsigned flags) |
a6f0788ec block: add suppor... |
214 215 216 217 218 219 220 |
{ struct bio *bio = *biop; unsigned int max_write_zeroes_sectors; struct request_queue *q = bdev_get_queue(bdev); if (!q) return -ENXIO; |
a13553c77 block: add bdev_r... |
221 222 |
if (bdev_read_only(bdev)) return -EPERM; |
a6f0788ec block: add suppor... |
223 224 225 226 227 228 229 230 231 |
/* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */ max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev); if (max_write_zeroes_sectors == 0) return -EOPNOTSUPP; while (nr_sects) { bio = next_bio(bio, 0, gfp_mask); bio->bi_iter.bi_sector = sector; |
74d46992e block: replace bi... |
232 |
bio_set_dev(bio, bdev); |
d928be9f8 block: add a REQ_... |
233 234 235 |
bio->bi_opf = REQ_OP_WRITE_ZEROES; if (flags & BLKDEV_ZERO_NOUNMAP) bio->bi_opf |= REQ_NOUNMAP; |
a6f0788ec block: add suppor... |
236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 |
if (nr_sects > max_write_zeroes_sectors) { bio->bi_iter.bi_size = max_write_zeroes_sectors << 9; nr_sects -= max_write_zeroes_sectors; sector += max_write_zeroes_sectors; } else { bio->bi_iter.bi_size = nr_sects << 9; nr_sects = 0; } cond_resched(); } *biop = bio; return 0; } |
615d22a51 block: Fix __blkd... |
251 252 253 254 255 256 257 258 |
/* * Convert a number of 512B sectors to a number of pages. * The result is limited to a number of pages that can fit into a BIO. * Also make sure that the result is always at least 1 (page) for the cases * where nr_sects is lower than the number of sectors in a page. */ static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects) { |
09c2c359b block: fix intege... |
259 |
sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512); |
615d22a51 block: Fix __blkd... |
260 |
|
09c2c359b block: fix intege... |
261 |
return min(pages, (sector_t)BIO_MAX_PAGES); |
615d22a51 block: Fix __blkd... |
262 |
} |
425a4dba7 block: factor out... |
263 264 265 266 267 268 269 270 271 272 273 |
static int __blkdev_issue_zero_pages(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct bio **biop) { struct request_queue *q = bdev_get_queue(bdev); struct bio *bio = *biop; int bi_size = 0; unsigned int sz; if (!q) return -ENXIO; |
a13553c77 block: add bdev_r... |
274 275 |
if (bdev_read_only(bdev)) return -EPERM; |
425a4dba7 block: factor out... |
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 |
while (nr_sects != 0) { bio = next_bio(bio, __blkdev_sectors_to_bio_pages(nr_sects), gfp_mask); bio->bi_iter.bi_sector = sector; bio_set_dev(bio, bdev); bio_set_op_attrs(bio, REQ_OP_WRITE, 0); while (nr_sects != 0) { sz = min((sector_t) PAGE_SIZE, nr_sects << 9); bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0); nr_sects -= bi_size >> 9; sector += bi_size >> 9; if (bi_size < sz) break; } cond_resched(); } *biop = bio; return 0; } |
a6f0788ec block: add suppor... |
297 |
/** |
e73c23ff7 block: add async ... |
298 |
* __blkdev_issue_zeroout - generate number of zero filed write bios |
3f14d792f blkdev: add blkde... |
299 300 301 302 |
* @bdev: blockdev to issue * @sector: start sector * @nr_sects: number of sectors to write * @gfp_mask: memory allocation flags (for bio_alloc) |
e73c23ff7 block: add async ... |
303 |
* @biop: pointer to anchor bio |
ee472d835 block: add a flag... |
304 |
* @flags: controls detailed behavior |
3f14d792f blkdev: add blkde... |
305 306 |
* * Description: |
ee472d835 block: add a flag... |
307 308 309 310 311 |
* Zero-fill a block range, either using hardware offload or by explicitly * writing zeroes to the device. * * If a device is using logical block provisioning, the underlying space will * not be released if %flags contains BLKDEV_ZERO_NOUNMAP. |
cb365b967 block: add a new ... |
312 313 314 |
* * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided. |
3f14d792f blkdev: add blkde... |
315 |
*/ |
e73c23ff7 block: add async ... |
316 317 |
int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, |
ee472d835 block: add a flag... |
318 |
unsigned flags) |
3f14d792f blkdev: add blkde... |
319 |
{ |
18edc8eaa blkdev: fix blkde... |
320 |
int ret; |
28b2be203 block: require wr... |
321 322 323 324 325 |
sector_t bs_mask; bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; if ((sector | nr_sects) & bs_mask) return -EINVAL; |
3f14d792f blkdev: add blkde... |
326 |
|
a6f0788ec block: add suppor... |
327 |
ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask, |
d928be9f8 block: add a REQ_... |
328 |
biop, flags); |
cb365b967 block: add a new ... |
329 |
if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK)) |
425a4dba7 block: factor out... |
330 |
return ret; |
3f14d792f blkdev: add blkde... |
331 |
|
425a4dba7 block: factor out... |
332 333 |
return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, biop); |
3f14d792f blkdev: add blkde... |
334 |
} |
e73c23ff7 block: add async ... |
335 |
EXPORT_SYMBOL(__blkdev_issue_zeroout); |
579e8f3c7 block: Make blkde... |
336 337 338 339 340 341 342 |
/** * blkdev_issue_zeroout - zero-fill a block range * @bdev: blockdev to write * @sector: start sector * @nr_sects: number of sectors to write * @gfp_mask: memory allocation flags (for bio_alloc) |
ee472d835 block: add a flag... |
343 |
* @flags: controls detailed behavior |
579e8f3c7 block: Make blkde... |
344 345 |
* * Description: |
ee472d835 block: add a flag... |
346 347 348 |
* Zero-fill a block range, either using hardware offload or by explicitly * writing zeroes to the device. See __blkdev_issue_zeroout() for the * valid values for %flags. |
579e8f3c7 block: Make blkde... |
349 |
*/ |
579e8f3c7 block: Make blkde... |
350 |
int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, |
ee472d835 block: add a flag... |
351 |
sector_t nr_sects, gfp_t gfp_mask, unsigned flags) |
579e8f3c7 block: Make blkde... |
352 |
{ |
d5ce4c31d block: cope with ... |
353 354 355 |
int ret = 0; sector_t bs_mask; struct bio *bio; |
e73c23ff7 block: add async ... |
356 |
struct blk_plug plug; |
d5ce4c31d block: cope with ... |
357 |
bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev); |
d93ba7a5a block: Add discar... |
358 |
|
d5ce4c31d block: cope with ... |
359 360 361 362 363 364 |
bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; if ((sector | nr_sects) & bs_mask) return -EINVAL; retry: bio = NULL; |
e73c23ff7 block: add async ... |
365 |
blk_start_plug(&plug); |
d5ce4c31d block: cope with ... |
366 367 368 369 370 371 372 373 374 375 |
if (try_write_zeroes) { ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask, &bio, flags); } else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) { ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, &bio); } else { /* No zeroing offload support */ ret = -EOPNOTSUPP; } |
e73c23ff7 block: add async ... |
376 377 378 379 380 |
if (ret == 0 && bio) { ret = submit_bio_wait(bio); bio_put(bio); } blk_finish_plug(&plug); |
d5ce4c31d block: cope with ... |
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 |
if (ret && try_write_zeroes) { if (!(flags & BLKDEV_ZERO_NOFALLBACK)) { try_write_zeroes = false; goto retry; } if (!bdev_write_zeroes_sectors(bdev)) { /* * Zeroing offload support was indicated, but the * device reported ILLEGAL REQUEST (for some devices * there is no non-destructive way to verify whether * WRITE ZEROES is actually supported). */ ret = -EOPNOTSUPP; } } |
579e8f3c7 block: Make blkde... |
396 |
|
e73c23ff7 block: add async ... |
397 |
return ret; |
579e8f3c7 block: Make blkde... |
398 |
} |
3f14d792f blkdev: add blkde... |
399 |
EXPORT_SYMBOL(blkdev_issue_zeroout); |