Blame view
block/blk-lib.c
9.92 KB
f31e7e402
|
1 2 3 4 5 6 7 8 9 10 |
/* * Functions related to generic helpers functions */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/scatterlist.h> #include "blk.h" |
4e49ea4a3
|
11 |
static struct bio *next_bio(struct bio *bio, unsigned int nr_pages, |
9082e87bf
|
12 |
gfp_t gfp) |
f31e7e402
|
13 |
{ |
9082e87bf
|
14 15 16 17 |
struct bio *new = bio_alloc(gfp, nr_pages); if (bio) { bio_chain(bio, new); |
4e49ea4a3
|
18 |
submit_bio(bio); |
9082e87bf
|
19 |
} |
5dba3089e
|
20 |
|
9082e87bf
|
21 |
return new; |
f31e7e402
|
22 |
} |
38f252553
|
23 |
int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, |
288dab8a3
|
24 |
sector_t nr_sects, gfp_t gfp_mask, int flags, |
469e3216e
|
25 |
struct bio **biop) |
f31e7e402
|
26 |
{ |
f31e7e402
|
27 |
struct request_queue *q = bdev_get_queue(bdev); |
38f252553
|
28 |
struct bio *bio = *biop; |
a22c4d7e3
|
29 |
unsigned int granularity; |
ef295ecf0
|
30 |
unsigned int op; |
a22c4d7e3
|
31 |
int alignment; |
28b2be203
|
32 |
sector_t bs_mask; |
f31e7e402
|
33 34 35 |
if (!q) return -ENXIO; |
288dab8a3
|
36 37 |
if (flags & BLKDEV_DISCARD_SECURE) { |
e950fdf71
|
38 39 |
if (flags & BLKDEV_DISCARD_ZERO) return -EOPNOTSUPP; |
288dab8a3
|
40 41 42 43 44 45 |
if (!blk_queue_secure_erase(q)) return -EOPNOTSUPP; op = REQ_OP_SECURE_ERASE; } else { if (!blk_queue_discard(q)) return -EOPNOTSUPP; |
e950fdf71
|
46 47 48 |
if ((flags & BLKDEV_DISCARD_ZERO) && !q->limits.discard_zeroes_data) return -EOPNOTSUPP; |
288dab8a3
|
49 50 |
op = REQ_OP_DISCARD; } |
f31e7e402
|
51 |
|
28b2be203
|
52 53 54 |
bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; if ((sector | nr_sects) & bs_mask) return -EINVAL; |
a22c4d7e3
|
55 56 57 |
/* Zero-sector (unknown) and one-sector granularities are the same. */ granularity = max(q->limits.discard_granularity >> 9, 1U); alignment = (bdev_discard_alignment(bdev) >> 9) % granularity; |
5dba3089e
|
58 |
while (nr_sects) { |
c6e666345
|
59 |
unsigned int req_sects; |
a22c4d7e3
|
60 |
sector_t end_sect, tmp; |
c6e666345
|
61 |
|
a22c4d7e3
|
62 63 |
/* Make sure bi_size doesn't overflow */ req_sects = min_t(sector_t, nr_sects, UINT_MAX >> 9); |
9082e87bf
|
64 |
/** |
a22c4d7e3
|
65 66 67 |
* If splitting a request, and the next starting sector would be * misaligned, stop the discard at the previous aligned sector. */ |
c6e666345
|
68 |
end_sect = sector + req_sects; |
a22c4d7e3
|
69 70 71 72 73 74 75 76 |
tmp = end_sect; if (req_sects < nr_sects && sector_div(tmp, granularity) != alignment) { end_sect = end_sect - alignment; sector_div(end_sect, granularity); end_sect = end_sect * granularity + alignment; req_sects = end_sect - sector; } |
c6e666345
|
77 |
|
f9d03f96b
|
78 |
bio = next_bio(bio, 0, gfp_mask); |
4f024f379
|
79 |
bio->bi_iter.bi_sector = sector; |
f31e7e402
|
80 |
bio->bi_bdev = bdev; |
288dab8a3
|
81 |
bio_set_op_attrs(bio, op, 0); |
f31e7e402
|
82 |
|
4f024f379
|
83 |
bio->bi_iter.bi_size = req_sects << 9; |
c6e666345
|
84 85 |
nr_sects -= req_sects; sector = end_sect; |
f31e7e402
|
86 |
|
c8123f8c9
|
87 88 89 90 91 92 93 |
/* * We can loop for a long time in here, if someone does * full device discards (like mkfs). Be nice and allow * us to schedule out to avoid softlocking if preempt * is disabled. */ cond_resched(); |
5dba3089e
|
94 |
} |
38f252553
|
95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
*biop = bio; return 0; } EXPORT_SYMBOL(__blkdev_issue_discard); /** * blkdev_issue_discard - queue a discard * @bdev: blockdev to issue discard for * @sector: start sector * @nr_sects: number of sectors to discard * @gfp_mask: memory allocation flags (for bio_alloc) * @flags: BLKDEV_IFL_* flags to control behaviour * * Description: * Issue a discard request for the sectors in question. */ int blkdev_issue_discard(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, unsigned long flags) { |
38f252553
|
115 116 117 |
struct bio *bio = NULL; struct blk_plug plug; int ret; |
38f252553
|
118 |
blk_start_plug(&plug); |
288dab8a3
|
119 |
ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags, |
38f252553
|
120 |
&bio); |
bbd848e0f
|
121 |
if (!ret && bio) { |
4e49ea4a3
|
122 |
ret = submit_bio_wait(bio); |
e950fdf71
|
123 |
if (ret == -EOPNOTSUPP && !(flags & BLKDEV_DISCARD_ZERO)) |
bbd848e0f
|
124 |
ret = 0; |
05bd92ddd
|
125 |
bio_put(bio); |
bbd848e0f
|
126 |
} |
0cfbcafca
|
127 |
blk_finish_plug(&plug); |
f31e7e402
|
128 |
|
bbd848e0f
|
129 |
return ret; |
f31e7e402
|
130 131 |
} EXPORT_SYMBOL(blkdev_issue_discard); |
3f14d792f
|
132 |
|
3f14d792f
|
133 |
/** |
e73c23ff7
|
134 |
* __blkdev_issue_write_same - generate number of bios with same page |
4363ac7c1
|
135 136 137 138 139 |
* @bdev: target blockdev * @sector: start sector * @nr_sects: number of sectors to write * @gfp_mask: memory allocation flags (for bio_alloc) * @page: page containing data to write |
e73c23ff7
|
140 |
* @biop: pointer to anchor bio |
4363ac7c1
|
141 142 |
* * Description: |
e73c23ff7
|
143 |
* Generate and issue number of bios(REQ_OP_WRITE_SAME) with same page. |
4363ac7c1
|
144 |
*/ |
e73c23ff7
|
145 146 147 |
static int __blkdev_issue_write_same(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct page *page, struct bio **biop) |
4363ac7c1
|
148 |
{ |
4363ac7c1
|
149 150 |
struct request_queue *q = bdev_get_queue(bdev); unsigned int max_write_same_sectors; |
e73c23ff7
|
151 |
struct bio *bio = *biop; |
28b2be203
|
152 |
sector_t bs_mask; |
4363ac7c1
|
153 154 155 |
if (!q) return -ENXIO; |
28b2be203
|
156 157 158 |
bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; if ((sector | nr_sects) & bs_mask) return -EINVAL; |
e73c23ff7
|
159 160 |
if (!bdev_write_same(bdev)) return -EOPNOTSUPP; |
b49a0871b
|
161 162 |
/* Ensure that max_write_same_sectors doesn't overflow bi_size */ max_write_same_sectors = UINT_MAX >> 9; |
4363ac7c1
|
163 |
|
4363ac7c1
|
164 |
while (nr_sects) { |
4e49ea4a3
|
165 |
bio = next_bio(bio, 1, gfp_mask); |
4f024f379
|
166 |
bio->bi_iter.bi_sector = sector; |
4363ac7c1
|
167 |
bio->bi_bdev = bdev; |
4363ac7c1
|
168 169 170 171 |
bio->bi_vcnt = 1; bio->bi_io_vec->bv_page = page; bio->bi_io_vec->bv_offset = 0; bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev); |
95fe6c1a2
|
172 |
bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0); |
4363ac7c1
|
173 174 |
if (nr_sects > max_write_same_sectors) { |
4f024f379
|
175 |
bio->bi_iter.bi_size = max_write_same_sectors << 9; |
4363ac7c1
|
176 177 178 |
nr_sects -= max_write_same_sectors; sector += max_write_same_sectors; } else { |
4f024f379
|
179 |
bio->bi_iter.bi_size = nr_sects << 9; |
4363ac7c1
|
180 181 |
nr_sects = 0; } |
e73c23ff7
|
182 |
cond_resched(); |
4363ac7c1
|
183 |
} |
e73c23ff7
|
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 |
*biop = bio; return 0; } /** * blkdev_issue_write_same - queue a write same operation * @bdev: target blockdev * @sector: start sector * @nr_sects: number of sectors to write * @gfp_mask: memory allocation flags (for bio_alloc) * @page: page containing data * * Description: * Issue a write same request for the sectors in question. */ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct page *page) { struct bio *bio = NULL; struct blk_plug plug; int ret; blk_start_plug(&plug); ret = __blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, page, &bio); if (ret == 0 && bio) { |
4e49ea4a3
|
211 |
ret = submit_bio_wait(bio); |
05bd92ddd
|
212 213 |
bio_put(bio); } |
e73c23ff7
|
214 |
blk_finish_plug(&plug); |
3f40bf2c8
|
215 |
return ret; |
4363ac7c1
|
216 217 218 219 |
} EXPORT_SYMBOL(blkdev_issue_write_same); /** |
a6f0788ec
|
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 |
* __blkdev_issue_write_zeroes - generate number of bios with WRITE ZEROES * @bdev: blockdev to issue * @sector: start sector * @nr_sects: number of sectors to write * @gfp_mask: memory allocation flags (for bio_alloc) * @biop: pointer to anchor bio * * Description: * Generate and issue number of bios(REQ_OP_WRITE_ZEROES) with zerofiled pages. */ static int __blkdev_issue_write_zeroes(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct bio **biop) { struct bio *bio = *biop; unsigned int max_write_zeroes_sectors; struct request_queue *q = bdev_get_queue(bdev); if (!q) return -ENXIO; /* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */ max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev); if (max_write_zeroes_sectors == 0) return -EOPNOTSUPP; while (nr_sects) { bio = next_bio(bio, 0, gfp_mask); bio->bi_iter.bi_sector = sector; bio->bi_bdev = bdev; bio_set_op_attrs(bio, REQ_OP_WRITE_ZEROES, 0); if (nr_sects > max_write_zeroes_sectors) { bio->bi_iter.bi_size = max_write_zeroes_sectors << 9; nr_sects -= max_write_zeroes_sectors; sector += max_write_zeroes_sectors; } else { bio->bi_iter.bi_size = nr_sects << 9; nr_sects = 0; } cond_resched(); } *biop = bio; return 0; } /** |
e73c23ff7
|
269 |
* __blkdev_issue_zeroout - generate number of zero filed write bios |
3f14d792f
|
270 271 272 273 |
* @bdev: blockdev to issue * @sector: start sector * @nr_sects: number of sectors to write * @gfp_mask: memory allocation flags (for bio_alloc) |
e73c23ff7
|
274 275 |
* @biop: pointer to anchor bio * @discard: discard flag |
3f14d792f
|
276 277 278 |
* * Description: * Generate and issue number of bios with zerofiled pages. |
3f14d792f
|
279 |
*/ |
e73c23ff7
|
280 281 282 |
int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, bool discard) |
3f14d792f
|
283 |
{ |
18edc8eaa
|
284 |
int ret; |
e73c23ff7
|
285 286 |
int bi_size = 0; struct bio *bio = *biop; |
0aeea1896
|
287 |
unsigned int sz; |
28b2be203
|
288 289 290 291 292 |
sector_t bs_mask; bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; if ((sector | nr_sects) & bs_mask) return -EINVAL; |
3f14d792f
|
293 |
|
e73c23ff7
|
294 295 296 297 298 299 |
if (discard) { ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, BLKDEV_DISCARD_ZERO, biop); if (ret == 0 || (ret && ret != -EOPNOTSUPP)) goto out; } |
a6f0788ec
|
300 301 302 303 |
ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask, biop); if (ret == 0 || (ret && ret != -EOPNOTSUPP)) goto out; |
e73c23ff7
|
304 305 306 307 308 309 |
ret = __blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, ZERO_PAGE(0), biop); if (ret == 0 || (ret && ret != -EOPNOTSUPP)) goto out; ret = 0; |
3f14d792f
|
310 |
while (nr_sects != 0) { |
4e49ea4a3
|
311 |
bio = next_bio(bio, min(nr_sects, (sector_t)BIO_MAX_PAGES), |
9082e87bf
|
312 |
gfp_mask); |
4f024f379
|
313 |
bio->bi_iter.bi_sector = sector; |
3f14d792f
|
314 |
bio->bi_bdev = bdev; |
95fe6c1a2
|
315 |
bio_set_op_attrs(bio, REQ_OP_WRITE, 0); |
3f14d792f
|
316 |
|
0341aafb7
|
317 318 |
while (nr_sects != 0) { sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects); |
e73c23ff7
|
319 320 321 322 |
bi_size = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0); nr_sects -= bi_size >> 9; sector += bi_size >> 9; if (bi_size < (sz << 9)) |
3f14d792f
|
323 324 |
break; } |
e73c23ff7
|
325 |
cond_resched(); |
3f14d792f
|
326 |
} |
3f14d792f
|
327 |
|
e73c23ff7
|
328 329 330 |
*biop = bio; out: return ret; |
3f14d792f
|
331 |
} |
e73c23ff7
|
332 |
EXPORT_SYMBOL(__blkdev_issue_zeroout); |
579e8f3c7
|
333 334 335 336 337 338 339 |
/** * blkdev_issue_zeroout - zero-fill a block range * @bdev: blockdev to write * @sector: start sector * @nr_sects: number of sectors to write * @gfp_mask: memory allocation flags (for bio_alloc) |
d93ba7a5a
|
340 |
* @discard: whether to discard the block range |
579e8f3c7
|
341 342 |
* * Description: |
d93ba7a5a
|
343 344 345 346 347 348 |
* Zero-fill a block range. If the discard flag is set and the block * device guarantees that subsequent READ operations to the block range * in question will return zeroes, the blocks will be discarded. Should * the discard request fail, if the discard flag is not set, or if * discard_zeroes_data is not supported, this function will resort to * zeroing the blocks manually, thus provisioning (allocating, |
a6f0788ec
|
349 350 |
* anchoring) them. If the block device supports WRITE ZEROES or WRITE SAME * command(s), blkdev_issue_zeroout() will use it to optimize the process of |
d93ba7a5a
|
351 352 |
* clearing the block range. Otherwise the zeroing will be performed * using regular WRITE calls. |
579e8f3c7
|
353 |
*/ |
579e8f3c7
|
354 |
int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, |
d93ba7a5a
|
355 |
sector_t nr_sects, gfp_t gfp_mask, bool discard) |
579e8f3c7
|
356 |
{ |
e73c23ff7
|
357 358 359 |
int ret; struct bio *bio = NULL; struct blk_plug plug; |
d93ba7a5a
|
360 |
|
e73c23ff7
|
361 362 363 364 365 366 367 368 |
blk_start_plug(&plug); ret = __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask, &bio, discard); if (ret == 0 && bio) { ret = submit_bio_wait(bio); bio_put(bio); } blk_finish_plug(&plug); |
579e8f3c7
|
369 |
|
e73c23ff7
|
370 |
return ret; |
579e8f3c7
|
371 |
} |
3f14d792f
|
372 |
EXPORT_SYMBOL(blkdev_issue_zeroout); |