Blame view
block/blk-map.c
15.7 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
86db1e297 block: continue l... |
2 3 4 5 |
/* * Functions related to mapping data to requests */ #include <linux/kernel.h> |
68db0cf10 sched/headers: Pr... |
6 |
#include <linux/sched/task_stack.h> |
86db1e297 block: continue l... |
7 8 9 |
#include <linux/module.h> #include <linux/bio.h> #include <linux/blkdev.h> |
26e49cfc7 block: pass iov_i... |
10 |
#include <linux/uio.h> |
86db1e297 block: continue l... |
11 12 |
#include "blk.h" |
130879f1e block: move bio_m... |
13 |
struct bio_map_data { |
f3256075b block: remove the... |
14 15 |
bool is_our_pages : 1; bool is_null_mapped : 1; |
130879f1e block: move bio_m... |
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 |
struct iov_iter iter; struct iovec iov[]; }; static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, gfp_t gfp_mask) { struct bio_map_data *bmd; if (data->nr_segs > UIO_MAXIOV) return NULL; bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); if (!bmd) return NULL; memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs); bmd->iter = *data; bmd->iter.iov = bmd->iov; return bmd; } /** * bio_copy_from_iter - copy all pages from iov_iter to bio * @bio: The &struct bio which describes the I/O as destination * @iter: iov_iter as source * * Copy all pages from iov_iter to bio. * Returns 0 on success, or error on failure. */ static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) { struct bio_vec *bvec; struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) { ssize_t ret; ret = copy_page_from_iter(bvec->bv_page, bvec->bv_offset, bvec->bv_len, iter); if (!iov_iter_count(iter)) break; if (ret < bvec->bv_len) return -EFAULT; } return 0; } /** * bio_copy_to_iter - copy all pages from bio to iov_iter * @bio: The &struct bio which describes the I/O as source * @iter: iov_iter as destination * * Copy all pages from bio to iov_iter. * Returns 0 on success, or error on failure. */ static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) { struct bio_vec *bvec; struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) { ssize_t ret; ret = copy_page_to_iter(bvec->bv_page, bvec->bv_offset, bvec->bv_len, &iter); if (!iov_iter_count(&iter)) break; if (ret < bvec->bv_len) return -EFAULT; } return 0; } /** * bio_uncopy_user - finish previously mapped bio * @bio: bio being terminated * * Free pages allocated from bio_copy_user_iov() and write back data * to user space in case of a read. */ static int bio_uncopy_user(struct bio *bio) { struct bio_map_data *bmd = bio->bi_private; int ret = 0; |
3310eebaf block: remove the... |
110 |
if (!bmd->is_null_mapped) { |
130879f1e block: move bio_m... |
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 |
/* * if we're in a workqueue, the request is orphaned, so * don't copy into a random user address space, just free * and return -EINTR so user space doesn't expect any data. */ if (!current->mm) ret = -EINTR; else if (bio_data_dir(bio) == READ) ret = bio_copy_to_iter(bio, bmd->iter); if (bmd->is_our_pages) bio_free_pages(bio); } kfree(bmd); bio_put(bio); return ret; } |
7589ad672 block: remove __b... |
127 128 |
static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, struct iov_iter *iter, gfp_t gfp_mask) |
130879f1e block: move bio_m... |
129 130 131 |
{ struct bio_map_data *bmd; struct page *page; |
7589ad672 block: remove __b... |
132 |
struct bio *bio, *bounce_bio; |
130879f1e block: move bio_m... |
133 134 135 136 137 138 139 |
int i = 0, ret; int nr_pages; unsigned int len = iter->count; unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; bmd = bio_alloc_map_data(iter, gfp_mask); if (!bmd) |
7589ad672 block: remove __b... |
140 |
return -ENOMEM; |
130879f1e block: move bio_m... |
141 142 143 144 145 146 |
/* * We need to do a deep copy of the iov_iter including the iovecs. * The caller provided iov might point to an on-stack or otherwise * shortlived one. */ |
f3256075b block: remove the... |
147 |
bmd->is_our_pages = !map_data; |
038597175 block: fix bmd->i... |
148 |
bmd->is_null_mapped = (map_data && map_data->null_mapped); |
130879f1e block: move bio_m... |
149 150 151 152 153 154 155 156 157 |
nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); if (nr_pages > BIO_MAX_PAGES) nr_pages = BIO_MAX_PAGES; ret = -ENOMEM; bio = bio_kmalloc(gfp_mask, nr_pages); if (!bio) goto out_bmd; |
7589ad672 block: remove __b... |
158 |
bio->bi_opf |= req_op(rq); |
130879f1e block: move bio_m... |
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 |
if (map_data) { nr_pages = 1 << map_data->page_order; i = map_data->offset / PAGE_SIZE; } while (len) { unsigned int bytes = PAGE_SIZE; bytes -= offset; if (bytes > len) bytes = len; if (map_data) { if (i == map_data->nr_entries * nr_pages) { ret = -ENOMEM; |
7589ad672 block: remove __b... |
175 |
goto cleanup; |
130879f1e block: move bio_m... |
176 177 178 179 180 181 182 |
} page = map_data->pages[i / nr_pages]; page += (i % nr_pages); i++; } else { |
7589ad672 block: remove __b... |
183 |
page = alloc_page(rq->q->bounce_gfp | gfp_mask); |
130879f1e block: move bio_m... |
184 185 |
if (!page) { ret = -ENOMEM; |
7589ad672 block: remove __b... |
186 |
goto cleanup; |
130879f1e block: move bio_m... |
187 188 |
} } |
7589ad672 block: remove __b... |
189 |
if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) { |
130879f1e block: move bio_m... |
190 191 192 193 194 195 196 197 |
if (!map_data) __free_page(page); break; } len -= bytes; offset = 0; } |
130879f1e block: move bio_m... |
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 |
if (map_data) map_data->offset += bio->bi_iter.bi_size; /* * success */ if ((iov_iter_rw(iter) == WRITE && (!map_data || !map_data->null_mapped)) || (map_data && map_data->from_user)) { ret = bio_copy_from_iter(bio, iter); if (ret) goto cleanup; } else { if (bmd->is_our_pages) zero_fill_bio(bio); iov_iter_advance(iter, bio->bi_iter.bi_size); } bio->bi_private = bmd; |
7589ad672 block: remove __b... |
217 218 219 220 221 222 223 224 225 226 227 228 |
bounce_bio = bio; ret = blk_rq_append_bio(rq, &bounce_bio); if (ret) goto cleanup; /* * We link the bounce buffer in and could have to traverse it later, so * we have to get a ref to prevent it from being freed */ bio_get(bounce_bio); return 0; |
130879f1e block: move bio_m... |
229 230 231 232 233 234 |
cleanup: if (!map_data) bio_free_pages(bio); bio_put(bio); out_bmd: kfree(bmd); |
7589ad672 block: remove __b... |
235 |
return ret; |
130879f1e block: move bio_m... |
236 |
} |
7589ad672 block: remove __b... |
237 238 |
static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) |
130879f1e block: move bio_m... |
239 |
{ |
7589ad672 block: remove __b... |
240 241 |
unsigned int max_sectors = queue_max_hw_sectors(rq->q); struct bio *bio, *bounce_bio; |
130879f1e block: move bio_m... |
242 |
int ret; |
7589ad672 block: remove __b... |
243 |
int j; |
130879f1e block: move bio_m... |
244 245 |
if (!iov_iter_count(iter)) |
7589ad672 block: remove __b... |
246 |
return -EINVAL; |
130879f1e block: move bio_m... |
247 248 249 |
bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES)); if (!bio) |
7589ad672 block: remove __b... |
250 251 |
return -ENOMEM; bio->bi_opf |= req_op(rq); |
130879f1e block: move bio_m... |
252 253 254 255 256 257 258 259 260 261 262 263 264 265 |
while (iov_iter_count(iter)) { struct page **pages; ssize_t bytes; size_t offs, added = 0; int npages; bytes = iov_iter_get_pages_alloc(iter, &pages, LONG_MAX, &offs); if (unlikely(bytes <= 0)) { ret = bytes ? bytes : -EFAULT; goto out_unmap; } npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); |
7589ad672 block: remove __b... |
266 |
if (unlikely(offs & queue_dma_alignment(rq->q))) { |
130879f1e block: move bio_m... |
267 268 269 270 271 272 273 274 275 276 |
ret = -EINVAL; j = 0; } else { for (j = 0; j < npages; j++) { struct page *page = pages[j]; unsigned int n = PAGE_SIZE - offs; bool same_page = false; if (n > bytes) n = bytes; |
7589ad672 block: remove __b... |
277 |
if (!bio_add_hw_page(rq->q, bio, page, n, offs, |
e45811057 block: rename __b... |
278 |
max_sectors, &same_page)) { |
130879f1e block: move bio_m... |
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 |
if (same_page) put_page(page); break; } added += n; bytes -= n; offs = 0; } iov_iter_advance(iter, added); } /* * release the pages we didn't map into the bio, if any */ while (j < npages) put_page(pages[j++]); kvfree(pages); /* couldn't stuff something into bio? */ if (bytes) break; } |
130879f1e block: move bio_m... |
300 |
/* |
7589ad672 block: remove __b... |
301 302 303 |
* Subtle: if we end up needing to bounce a bio, it would normally * disappear when its bi_end_io is run. However, we need the original * bio for the unmap, so grab an extra reference to it |
130879f1e block: move bio_m... |
304 305 |
*/ bio_get(bio); |
130879f1e block: move bio_m... |
306 |
|
7589ad672 block: remove __b... |
307 308 309 310 311 312 313 314 315 316 317 318 319 320 |
bounce_bio = bio; ret = blk_rq_append_bio(rq, &bounce_bio); if (ret) goto out_put_orig; /* * We link the bounce buffer in and could have to traverse it * later, so we have to get a ref to prevent it from being freed */ bio_get(bounce_bio); return 0; out_put_orig: bio_put(bio); |
130879f1e block: move bio_m... |
321 322 323 |
out_unmap: bio_release_pages(bio, false); bio_put(bio); |
7589ad672 block: remove __b... |
324 |
return ret; |
130879f1e block: move bio_m... |
325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 |
} /** * bio_unmap_user - unmap a bio * @bio: the bio being unmapped * * Unmap a bio previously mapped by bio_map_user_iov(). Must be called from * process context. * * bio_unmap_user() may sleep. */ static void bio_unmap_user(struct bio *bio) { bio_release_pages(bio, bio_data_dir(bio) == READ); bio_put(bio); bio_put(bio); } static void bio_invalidate_vmalloc_pages(struct bio *bio) { #ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE if (bio->bi_private && !op_is_write(bio_op(bio))) { unsigned long i, len = 0; for (i = 0; i < bio->bi_vcnt; i++) len += bio->bi_io_vec[i].bv_len; invalidate_kernel_vmap_range(bio->bi_private, len); } #endif } static void bio_map_kern_endio(struct bio *bio) { bio_invalidate_vmalloc_pages(bio); bio_put(bio); } /** * bio_map_kern - map kernel address into bio * @q: the struct request_queue for the bio * @data: pointer to buffer to map * @len: length in bytes * @gfp_mask: allocation flags for bio allocation * * Map the kernel address into a bio suitable for io to a block * device. Returns an error pointer in case of error. */ static struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len, gfp_t gfp_mask) { unsigned long kaddr = (unsigned long)data; unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; unsigned long start = kaddr >> PAGE_SHIFT; const int nr_pages = end - start; bool is_vmalloc = is_vmalloc_addr(data); struct page *page; int offset, i; struct bio *bio; bio = bio_kmalloc(gfp_mask, nr_pages); if (!bio) return ERR_PTR(-ENOMEM); if (is_vmalloc) { flush_kernel_vmap_range(data, len); bio->bi_private = data; } offset = offset_in_page(kaddr); for (i = 0; i < nr_pages; i++) { unsigned int bytes = PAGE_SIZE - offset; if (len <= 0) break; if (bytes > len) bytes = len; if (!is_vmalloc) page = virt_to_page(data); else page = vmalloc_to_page(data); if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes) { /* we don't support partial mappings */ bio_put(bio); return ERR_PTR(-EINVAL); } data += bytes; len -= bytes; offset = 0; } bio->bi_end_io = bio_map_kern_endio; return bio; } static void bio_copy_kern_endio(struct bio *bio) { bio_free_pages(bio); bio_put(bio); } static void bio_copy_kern_endio_read(struct bio *bio) { char *p = bio->bi_private; struct bio_vec *bvec; struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) { memcpy(p, page_address(bvec->bv_page), bvec->bv_len); p += bvec->bv_len; } bio_copy_kern_endio(bio); } /** * bio_copy_kern - copy kernel address into bio * @q: the struct request_queue for the bio * @data: pointer to buffer to copy * @len: length in bytes * @gfp_mask: allocation flags for bio and page allocation * @reading: data direction is READ * * copy the kernel address into a bio suitable for io to a block * device. Returns an error pointer in case of error. */ static struct bio *bio_copy_kern(struct request_queue *q, void *data, unsigned int len, gfp_t gfp_mask, int reading) { unsigned long kaddr = (unsigned long)data; unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; unsigned long start = kaddr >> PAGE_SHIFT; struct bio *bio; void *p = data; int nr_pages = 0; /* * Overflow, abort */ if (end < start) return ERR_PTR(-EINVAL); nr_pages = end - start; bio = bio_kmalloc(gfp_mask, nr_pages); if (!bio) return ERR_PTR(-ENOMEM); while (len) { struct page *page; unsigned int bytes = PAGE_SIZE; if (bytes > len) bytes = len; page = alloc_page(q->bounce_gfp | gfp_mask); if (!page) goto cleanup; if (!reading) memcpy(page_address(page), p, bytes); if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) break; len -= bytes; p += bytes; } if (reading) { bio->bi_end_io = bio_copy_kern_endio_read; bio->bi_private = data; } else { bio->bi_end_io = bio_copy_kern_endio; } return bio; cleanup: bio_free_pages(bio); bio_put(bio); return ERR_PTR(-ENOMEM); } |
98d61d5b1 block: simplify a... |
510 |
/* |
0abc2a103 block: fix blk_rq... |
511 512 |
* Append a bio to a passthrough request. Only works if the bio can be merged * into the request based on the driver constraints. |
98d61d5b1 block: simplify a... |
513 |
*/ |
0abc2a103 block: fix blk_rq... |
514 |
int blk_rq_append_bio(struct request *rq, struct bio **bio) |
86db1e297 block: continue l... |
515 |
{ |
0abc2a103 block: fix blk_rq... |
516 |
struct bio *orig_bio = *bio; |
14ccb66b3 block: remove the... |
517 518 519 |
struct bvec_iter iter; struct bio_vec bv; unsigned int nr_segs = 0; |
0abc2a103 block: fix blk_rq... |
520 521 |
blk_queue_bounce(rq->q, bio); |
caa4b0247 blk-map: call blk... |
522 |
|
14ccb66b3 block: remove the... |
523 524 |
bio_for_each_bvec(bv, *bio, iter) nr_segs++; |
98d61d5b1 block: simplify a... |
525 |
if (!rq->bio) { |
14ccb66b3 block: remove the... |
526 |
blk_rq_bio_prep(rq, *bio, nr_segs); |
98d61d5b1 block: simplify a... |
527 |
} else { |
14ccb66b3 block: remove the... |
528 |
if (!ll_back_merge_fn(rq, *bio, nr_segs)) { |
0abc2a103 block: fix blk_rq... |
529 530 531 532 |
if (orig_bio != *bio) { bio_put(*bio); *bio = orig_bio; } |
98d61d5b1 block: simplify a... |
533 |
return -EINVAL; |
0abc2a103 block: fix blk_rq... |
534 |
} |
98d61d5b1 block: simplify a... |
535 |
|
0abc2a103 block: fix blk_rq... |
536 537 538 |
rq->biotail->bi_next = *bio; rq->biotail = *bio; rq->__data_len += (*bio)->bi_iter.bi_size; |
a892c8d52 block: Inline enc... |
539 |
bio_crypt_free_ctx(*bio); |
86db1e297 block: continue l... |
540 |
} |
98d61d5b1 block: simplify a... |
541 |
|
86db1e297 block: continue l... |
542 543 |
return 0; } |
98d61d5b1 block: simplify a... |
544 |
EXPORT_SYMBOL(blk_rq_append_bio); |
86db1e297 block: continue l... |
545 |
|
86db1e297 block: continue l... |
546 |
/** |
aebf526b5 block: fold cmd_t... |
547 |
* blk_rq_map_user_iov - map user data to a request, for passthrough requests |
86db1e297 block: continue l... |
548 549 |
* @q: request queue where request should be inserted * @rq: request to map data to |
152e283fd block: introduce ... |
550 |
* @map_data: pointer to the rq_map_data holding pages (if necessary) |
26e49cfc7 block: pass iov_i... |
551 |
* @iter: iovec iterator |
a3bce90ed block: add gfp_ma... |
552 |
* @gfp_mask: memory allocation flags |
86db1e297 block: continue l... |
553 554 |
* * Description: |
710027a48 Add some block/ s... |
555 |
* Data will be mapped directly for zero copy I/O, if possible. Otherwise |
86db1e297 block: continue l... |
556 557 |
* a kernel bounce buffer is used. * |
710027a48 Add some block/ s... |
558 |
* A matching blk_rq_unmap_user() must be issued at the end of I/O, while |
86db1e297 block: continue l... |
559 560 561 562 563 564 565 566 567 |
* still in process context. * * Note: The mapped bio may need to be bounced through blk_queue_bounce() * before being submitted to the device, as pages mapped may be out of * reach. It's the callers responsibility to make sure this happens. The * original bio must be passed back in to blk_rq_unmap_user() for proper * unmapping. */ int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, |
26e49cfc7 block: pass iov_i... |
568 569 |
struct rq_map_data *map_data, const struct iov_iter *iter, gfp_t gfp_mask) |
86db1e297 block: continue l... |
570 |
{ |
357f435d8 fix the copy vs. ... |
571 572 |
bool copy = false; unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); |
4d6af73d9 block: support la... |
573 574 |
struct bio *bio = NULL; struct iov_iter i; |
69e0927b3 blk_rq_map_user_i... |
575 |
int ret = -EINVAL; |
86db1e297 block: continue l... |
576 |
|
a0ac402cf Don't feed anythi... |
577 578 |
if (!iter_is_iovec(iter)) goto fail; |
357f435d8 fix the copy vs. ... |
579 580 581 582 583 584 |
if (map_data) copy = true; else if (iov_iter_alignment(iter) & align) copy = true; else if (queue_virt_boundary(q)) copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); |
afdc1a780 block: add bio_co... |
585 |
|
4d6af73d9 block: support la... |
586 587 |
i = *iter; do { |
7589ad672 block: remove __b... |
588 589 590 591 |
if (copy) ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask); else ret = bio_map_user_iov(rq, &i, gfp_mask); |
4d6af73d9 block: support la... |
592 593 594 595 596 |
if (ret) goto unmap_rq; if (!bio) bio = rq->bio; } while (iov_iter_count(&i)); |
86db1e297 block: continue l... |
597 |
|
86db1e297 block: continue l... |
598 |
return 0; |
4d6af73d9 block: support la... |
599 600 |
unmap_rq: |
3b7995a98 block: fix memlea... |
601 |
blk_rq_unmap_user(bio); |
a0ac402cf Don't feed anythi... |
602 |
fail: |
4d6af73d9 block: support la... |
603 |
rq->bio = NULL; |
69e0927b3 blk_rq_map_user_i... |
604 |
return ret; |
86db1e297 block: continue l... |
605 |
} |
152e283fd block: introduce ... |
606 |
EXPORT_SYMBOL(blk_rq_map_user_iov); |
86db1e297 block: continue l... |
607 |
|
ddad8dd0a block: use blk_rq... |
608 609 610 611 |
int blk_rq_map_user(struct request_queue *q, struct request *rq, struct rq_map_data *map_data, void __user *ubuf, unsigned long len, gfp_t gfp_mask) { |
26e49cfc7 block: pass iov_i... |
612 613 |
struct iovec iov; struct iov_iter i; |
8f7e885a4 blk_rq_map_user()... |
614 |
int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i); |
ddad8dd0a block: use blk_rq... |
615 |
|
8f7e885a4 blk_rq_map_user()... |
616 617 |
if (unlikely(ret < 0)) return ret; |
ddad8dd0a block: use blk_rq... |
618 |
|
26e49cfc7 block: pass iov_i... |
619 |
return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask); |
ddad8dd0a block: use blk_rq... |
620 621 |
} EXPORT_SYMBOL(blk_rq_map_user); |
86db1e297 block: continue l... |
622 623 624 625 626 627 628 |
/** * blk_rq_unmap_user - unmap a request with user data * @bio: start of bio list * * Description: * Unmap a rq previously mapped by blk_rq_map_user(). The caller must * supply the original rq->bio from the blk_rq_map_user() return, since |
710027a48 Add some block/ s... |
629 |
* the I/O completion may have changed rq->bio. |
86db1e297 block: continue l... |
630 631 632 633 634 635 636 637 638 639 |
*/ int blk_rq_unmap_user(struct bio *bio) { struct bio *mapped_bio; int ret = 0, ret2; while (bio) { mapped_bio = bio; if (unlikely(bio_flagged(bio, BIO_BOUNCED))) mapped_bio = bio->bi_private; |
3310eebaf block: remove the... |
640 |
if (bio->bi_private) { |
7b63c052a block: remove __b... |
641 642 643 |
ret2 = bio_uncopy_user(mapped_bio); if (ret2 && !ret) ret = ret2; |
3310eebaf block: remove the... |
644 645 |
} else { bio_unmap_user(mapped_bio); |
7b63c052a block: remove __b... |
646 |
} |
86db1e297 block: continue l... |
647 648 649 650 651 652 653 654 |
mapped_bio = bio; bio = bio->bi_next; bio_put(mapped_bio); } return ret; } |
86db1e297 block: continue l... |
655 656 657 |
EXPORT_SYMBOL(blk_rq_unmap_user); /** |
aebf526b5 block: fold cmd_t... |
658 |
* blk_rq_map_kern - map kernel data to a request, for passthrough requests |
86db1e297 block: continue l... |
659 660 661 662 663 |
* @q: request queue where request should be inserted * @rq: request to fill * @kbuf: the kernel buffer * @len: length of user data * @gfp_mask: memory allocation flags |
68154e90c block: add dma al... |
664 665 666 |
* * Description: * Data will be mapped directly if possible. Otherwise a bounce |
e227867f1 treewide: Fix typ... |
667 |
* buffer is used. Can be called multiple times to append multiple |
3a5a39276 block: allow blk_... |
668 |
* buffers. |
86db1e297 block: continue l... |
669 670 671 672 |
*/ int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, unsigned int len, gfp_t gfp_mask) { |
68154e90c block: add dma al... |
673 |
int reading = rq_data_dir(rq) == READ; |
144177991 block: fix an add... |
674 |
unsigned long addr = (unsigned long) kbuf; |
0abc2a103 block: fix blk_rq... |
675 |
struct bio *bio, *orig_bio; |
3a5a39276 block: allow blk_... |
676 |
int ret; |
86db1e297 block: continue l... |
677 |
|
ae03bf639 block: Use access... |
678 |
if (len > (queue_max_hw_sectors(q) << 9)) |
86db1e297 block: continue l... |
679 680 681 |
return -EINVAL; if (!len || !kbuf) return -EINVAL; |
e64a0e169 block: remove RQF... |
682 |
if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf)) |
68154e90c block: add dma al... |
683 684 685 |
bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); else bio = bio_map_kern(q, kbuf, len, gfp_mask); |
86db1e297 block: continue l... |
686 687 |
if (IS_ERR(bio)) return PTR_ERR(bio); |
aebf526b5 block: fold cmd_t... |
688 689 |
bio->bi_opf &= ~REQ_OP_MASK; bio->bi_opf |= req_op(rq); |
86db1e297 block: continue l... |
690 |
|
0abc2a103 block: fix blk_rq... |
691 692 |
orig_bio = bio; ret = blk_rq_append_bio(rq, &bio); |
3a5a39276 block: allow blk_... |
693 694 |
if (unlikely(ret)) { /* request is too big */ |
0abc2a103 block: fix blk_rq... |
695 |
bio_put(orig_bio); |
3a5a39276 block: allow blk_... |
696 697 |
return ret; } |
86db1e297 block: continue l... |
698 699 |
return 0; } |
86db1e297 block: continue l... |
700 |
EXPORT_SYMBOL(blk_rq_map_kern); |