Blame view
drivers/md/dm-rq.c
21.4 KB
4cc96131a dm: move request-... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 |
/* * Copyright (C) 2016 Red Hat, Inc. All rights reserved. * * This file is released under the GPL. */ #include "dm-core.h" #include "dm-rq.h" #include <linux/elevator.h> /* for rq_end_sector() */ #include <linux/blk-mq.h> #define DM_MSG_PREFIX "core-rq" #define DM_MQ_NR_HW_QUEUES 1 #define DM_MQ_QUEUE_DEPTH 2048 static unsigned dm_mq_nr_hw_queues = DM_MQ_NR_HW_QUEUES; static unsigned dm_mq_queue_depth = DM_MQ_QUEUE_DEPTH; /* * Request-based DM's mempools' reserved IOs set by the user. */ #define RESERVED_REQUEST_BASED_IOS 256 static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS; |
b23df0d04 dm rq: simplify u... |
25 |
static bool use_blk_mq = IS_ENABLED(CONFIG_DM_MQ_DEFAULT); |
4cc96131a dm: move request-... |
26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
bool dm_use_blk_mq_default(void) { return use_blk_mq; } bool dm_use_blk_mq(struct mapped_device *md) { return md->use_blk_mq; } EXPORT_SYMBOL_GPL(dm_use_blk_mq); unsigned dm_get_reserved_rq_based_ios(void) { return __dm_get_module_param(&reserved_rq_based_ios, RESERVED_REQUEST_BASED_IOS, DM_RESERVED_MAX_IOS); } EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios); static unsigned dm_get_blk_mq_nr_hw_queues(void) { return __dm_get_module_param(&dm_mq_nr_hw_queues, 1, 32); } static unsigned dm_get_blk_mq_queue_depth(void) { return __dm_get_module_param(&dm_mq_queue_depth, DM_MQ_QUEUE_DEPTH, BLK_MQ_MAX_DEPTH); } int dm_request_based(struct mapped_device *md) { return blk_queue_stackable(md->queue); } static void dm_old_start_queue(struct request_queue *q) { unsigned long flags; spin_lock_irqsave(q->queue_lock, flags); if (blk_queue_stopped(q)) blk_start_queue(q); spin_unlock_irqrestore(q->queue_lock, flags); } |
9dbeaeaba dm rq: take reque... |
70 71 |
static void dm_mq_start_queue(struct request_queue *q) { |
f660174e8 blk-mq: use the i... |
72 |
blk_mq_unquiesce_queue(q); |
9dbeaeaba dm rq: take reque... |
73 74 |
blk_mq_kick_requeue_list(q); } |
4cc96131a dm: move request-... |
75 76 77 78 |
void dm_start_queue(struct request_queue *q) { if (!q->mq_ops) dm_old_start_queue(q); |
9dbeaeaba dm rq: take reque... |
79 80 |
else dm_mq_start_queue(q); |
4cc96131a dm: move request-... |
81 82 83 84 85 86 87 |
} static void dm_old_stop_queue(struct request_queue *q) { unsigned long flags; spin_lock_irqsave(q->queue_lock, flags); |
c533f249a dm rq: simplify d... |
88 89 |
if (!blk_queue_stopped(q)) blk_stop_queue(q); |
4cc96131a dm: move request-... |
90 91 |
spin_unlock_irqrestore(q->queue_lock, flags); } |
2397a15af dm rq: factor out... |
92 93 |
static void dm_mq_stop_queue(struct request_queue *q) { |
f0d33ab76 dm: Use BLK_MQ_S_... |
94 |
if (blk_mq_queue_stopped(q)) |
4cc96131a dm: move request-... |
95 |
return; |
2397a15af dm rq: factor out... |
96 |
|
7b17c2f72 dm: Fix a race co... |
97 |
blk_mq_quiesce_queue(q); |
4cc96131a dm: move request-... |
98 99 100 101 102 103 |
} void dm_stop_queue(struct request_queue *q) { if (!q->mq_ops) dm_old_stop_queue(q); |
2397a15af dm rq: factor out... |
104 105 |
else dm_mq_stop_queue(q); |
4cc96131a dm: move request-... |
106 |
} |
4cc96131a dm: move request-... |
107 108 109 110 111 112 113 114 |
/* * Partial completion handling for request-based dm */ static void end_clone_bio(struct bio *clone) { struct dm_rq_clone_bio_info *info = container_of(clone, struct dm_rq_clone_bio_info, clone); struct dm_rq_target_io *tio = info->tio; |
4cc96131a dm: move request-... |
115 |
unsigned int nr_bytes = info->orig->bi_iter.bi_size; |
4e4cbee93 block: switch bio... |
116 |
blk_status_t error = clone->bi_status; |
dc6364b51 dm rq: do not upd... |
117 |
bool is_last = !clone->bi_next; |
4cc96131a dm: move request-... |
118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 |
bio_put(clone); if (tio->error) /* * An error has already been detected on the request. * Once error occurred, just let clone->end_io() handle * the remainder. */ return; else if (error) { /* * Don't notice the error to the upper layer yet. * The error handling decision is made by the target driver, * when the request is completed. */ tio->error = error; |
dc6364b51 dm rq: do not upd... |
135 |
goto exit; |
4cc96131a dm: move request-... |
136 137 138 139 140 141 |
} /* * I/O for the bio successfully completed. * Notice the data completion to the upper layer. */ |
dc6364b51 dm rq: do not upd... |
142 |
tio->completed += nr_bytes; |
4cc96131a dm: move request-... |
143 144 145 146 147 148 |
/* * Update the original request. * Do not use blk_end_request() here, because it may complete * the original request before the clone, and break the ordering. */ |
dc6364b51 dm rq: do not upd... |
149 150 151 |
if (is_last) exit: blk_update_request(tio->orig, BLK_STS_OK, tio->completed); |
4cc96131a dm: move request-... |
152 153 154 155 |
} static struct dm_rq_target_io *tio_from_request(struct request *rq) { |
eb8db831b dm: always defer ... |
156 |
return blk_mq_rq_to_pdu(rq); |
4cc96131a dm: move request-... |
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 |
} static void rq_end_stats(struct mapped_device *md, struct request *orig) { if (unlikely(dm_stats_used(&md->stats))) { struct dm_rq_target_io *tio = tio_from_request(orig); tio->duration_jiffies = jiffies - tio->duration_jiffies; dm_stats_account_io(&md->stats, rq_data_dir(orig), blk_rq_pos(orig), tio->n_sectors, true, tio->duration_jiffies, &tio->stats_aux); } } /* * Don't touch any member of the md after calling this function because * the md may be freed in dm_put() at the end of this function. * Or do dm_get() before calling this function and dm_put() later. */ static void rq_completed(struct mapped_device *md, int rw, bool run_queue) { |
d15bb3a64 dm rq: fix a race... |
177 178 |
struct request_queue *q = md->queue; unsigned long flags; |
4cc96131a dm: move request-... |
179 180 181 182 183 184 185 186 187 188 189 190 |
atomic_dec(&md->pending[rw]); /* nudge anyone waiting on suspend queue */ if (!md_in_flight(md)) wake_up(&md->wait); /* * Run this off this callpath, as drivers could invoke end_io while * inside their request_fn (and holding the queue lock). Calling * back into ->request_fn() could deadlock attempting to grab the * queue lock again. */ |
d15bb3a64 dm rq: fix a race... |
191 192 193 194 195 |
if (!q->mq_ops && run_queue) { spin_lock_irqsave(q->queue_lock, flags); blk_run_queue_async(q); spin_unlock_irqrestore(q->queue_lock, flags); } |
4cc96131a dm: move request-... |
196 197 198 199 200 201 |
/* * dm_put() must be at the end of this function. See the comment above */ dm_put(md); } |
4cc96131a dm: move request-... |
202 203 204 205 206 |
/* * Complete the clone and the original request. * Must be called without clone's queue lock held, * see end_clone_request() for more details. */ |
2a842acab block: introduce ... |
207 |
static void dm_end_request(struct request *clone, blk_status_t error) |
4cc96131a dm: move request-... |
208 209 210 211 212 |
{ int rw = rq_data_dir(clone); struct dm_rq_target_io *tio = clone->end_io_data; struct mapped_device *md = tio->md; struct request *rq = tio->orig; |
eb8db831b dm: always defer ... |
213 214 |
blk_rq_unprep_clone(clone); tio->ti->type->release_clone_rq(clone); |
4cc96131a dm: move request-... |
215 |
|
4cc96131a dm: move request-... |
216 217 218 219 220 221 222 |
rq_end_stats(md, rq); if (!rq->q->mq_ops) blk_end_request_all(rq, error); else blk_mq_end_request(rq, error); rq_completed(md, rw, true); } |
4cc96131a dm: move request-... |
223 224 225 |
/* * Requeue the original request of a clone. */ |
d5c27f3ff dm rq: make dm-sq... |
226 |
static void dm_old_requeue_request(struct request *rq, unsigned long delay_ms) |
4cc96131a dm: move request-... |
227 228 229 230 231 232 |
{ struct request_queue *q = rq->q; unsigned long flags; spin_lock_irqsave(q->queue_lock, flags); blk_requeue_request(q, rq); |
d5c27f3ff dm rq: make dm-sq... |
233 |
blk_delay_queue(q, delay_ms); |
4cc96131a dm: move request-... |
234 235 |
spin_unlock_irqrestore(q->queue_lock, flags); } |
e0c107526 dm rq: introduce ... |
236 |
static void __dm_mq_kick_requeue_list(struct request_queue *q, unsigned long msecs) |
4cc96131a dm: move request-... |
237 |
{ |
52d7f1b5c blk-mq: Avoid tha... |
238 |
blk_mq_delay_kick_requeue_list(q, msecs); |
4cc96131a dm: move request-... |
239 |
} |
e0c107526 dm rq: introduce ... |
240 241 242 243 244 245 246 247 |
void dm_mq_kick_requeue_list(struct mapped_device *md) { __dm_mq_kick_requeue_list(dm_get_md_queue(md), 0); } EXPORT_SYMBOL(dm_mq_kick_requeue_list); static void dm_mq_delay_requeue_request(struct request *rq, unsigned long msecs) { |
2b053aca7 blk-mq: Add a kic... |
248 |
blk_mq_requeue_request(rq, false); |
e0c107526 dm rq: introduce ... |
249 250 |
__dm_mq_kick_requeue_list(rq->q, msecs); } |
fbc39b4ca dm rq: reduce arg... |
251 |
static void dm_requeue_original_request(struct dm_rq_target_io *tio, bool delay_requeue) |
4cc96131a dm: move request-... |
252 |
{ |
fbc39b4ca dm rq: reduce arg... |
253 254 |
struct mapped_device *md = tio->md; struct request *rq = tio->orig; |
4cc96131a dm: move request-... |
255 |
int rw = rq_data_dir(rq); |
d5c27f3ff dm rq: make dm-sq... |
256 |
unsigned long delay_ms = delay_requeue ? 100 : 0; |
4cc96131a dm: move request-... |
257 258 |
rq_end_stats(md, rq); |
eb8db831b dm: always defer ... |
259 260 261 262 |
if (tio->clone) { blk_rq_unprep_clone(tio->clone); tio->ti->type->release_clone_rq(tio->clone); } |
4cc96131a dm: move request-... |
263 264 |
if (!rq->q->mq_ops) |
d5c27f3ff dm rq: make dm-sq... |
265 |
dm_old_requeue_request(rq, delay_ms); |
4cc96131a dm: move request-... |
266 |
else |
d5c27f3ff dm rq: make dm-sq... |
267 |
dm_mq_delay_requeue_request(rq, delay_ms); |
4cc96131a dm: move request-... |
268 269 270 |
rq_completed(md, rw, false); } |
2a842acab block: introduce ... |
271 |
static void dm_done(struct request *clone, blk_status_t error, bool mapped) |
4cc96131a dm: move request-... |
272 |
{ |
7ed8578a9 dm rq: change ->r... |
273 |
int r = DM_ENDIO_DONE; |
4cc96131a dm: move request-... |
274 275 276 277 278 279 280 281 282 |
struct dm_rq_target_io *tio = clone->end_io_data; dm_request_endio_fn rq_end_io = NULL; if (tio->ti) { rq_end_io = tio->ti->type->rq_end_io; if (mapped && rq_end_io) r = rq_end_io(tio->ti, clone, error, &tio->info); } |
2a842acab block: introduce ... |
283 |
if (unlikely(error == BLK_STS_TARGET)) { |
ac62d6208 dm: support REQ_O... |
284 285 286 287 288 289 290 |
if (req_op(clone) == REQ_OP_WRITE_SAME && !clone->q->limits.max_write_same_sectors) disable_write_same(tio->md); if (req_op(clone) == REQ_OP_WRITE_ZEROES && !clone->q->limits.max_write_zeroes_sectors) disable_write_zeroes(tio->md); } |
4cc96131a dm: move request-... |
291 |
|
7ed8578a9 dm rq: change ->r... |
292 293 |
switch (r) { case DM_ENDIO_DONE: |
4cc96131a dm: move request-... |
294 |
/* The target wants to complete the I/O */ |
7ed8578a9 dm rq: change ->r... |
295 296 297 |
dm_end_request(clone, error); break; case DM_ENDIO_INCOMPLETE: |
4cc96131a dm: move request-... |
298 299 |
/* The target will handle the I/O */ return; |
7ed8578a9 dm rq: change ->r... |
300 |
case DM_ENDIO_REQUEUE: |
4cc96131a dm: move request-... |
301 |
/* The target wants to requeue the I/O */ |
fbc39b4ca dm rq: reduce arg... |
302 |
dm_requeue_original_request(tio, false); |
7ed8578a9 dm rq: change ->r... |
303 304 |
break; default: |
4cc96131a dm: move request-... |
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 |
DMWARN("unimplemented target endio return value: %d", r); BUG(); } } /* * Request completion handler for request-based dm */ static void dm_softirq_done(struct request *rq) { bool mapped = true; struct dm_rq_target_io *tio = tio_from_request(rq); struct request *clone = tio->clone; int rw; if (!clone) { |
61febef40 dm-rq: don't dere... |
321 322 323 |
struct mapped_device *md = tio->md; rq_end_stats(md, rq); |
4cc96131a dm: move request-... |
324 |
rw = rq_data_dir(rq); |
eb8db831b dm: always defer ... |
325 |
if (!rq->q->mq_ops) |
4cc96131a dm: move request-... |
326 |
blk_end_request_all(rq, tio->error); |
eb8db831b dm: always defer ... |
327 |
else |
4cc96131a dm: move request-... |
328 |
blk_mq_end_request(rq, tio->error); |
61febef40 dm-rq: don't dere... |
329 |
rq_completed(md, rw, false); |
4cc96131a dm: move request-... |
330 331 |
return; } |
e80640213 block: split out ... |
332 |
if (rq->rq_flags & RQF_FAILED) |
4cc96131a dm: move request-... |
333 334 335 336 337 338 339 340 341 |
mapped = false; dm_done(clone, tio->error, mapped); } /* * Complete the clone and the original request with the error status * through softirq context. */ |
2a842acab block: introduce ... |
342 |
static void dm_complete_request(struct request *rq, blk_status_t error) |
4cc96131a dm: move request-... |
343 344 345 346 347 348 349 |
{ struct dm_rq_target_io *tio = tio_from_request(rq); tio->error = error; if (!rq->q->mq_ops) blk_complete_request(rq); else |
08e0029aa blk-mq: remove th... |
350 |
blk_mq_complete_request(rq); |
4cc96131a dm: move request-... |
351 352 353 354 355 356 357 358 |
} /* * Complete the not-mapped clone and the original request with the error status * through softirq context. * Target's rq_end_io() function isn't called. * This may be used when the target's map_rq() or clone_and_map_rq() functions fail. */ |
2a842acab block: introduce ... |
359 |
static void dm_kill_unmapped_request(struct request *rq, blk_status_t error) |
4cc96131a dm: move request-... |
360 |
{ |
e80640213 block: split out ... |
361 |
rq->rq_flags |= RQF_FAILED; |
4cc96131a dm: move request-... |
362 363 364 365 366 367 |
dm_complete_request(rq, error); } /* * Called with the clone's queue lock held (in the case of .request_fn) */ |
2a842acab block: introduce ... |
368 |
static void end_clone_request(struct request *clone, blk_status_t error) |
4cc96131a dm: move request-... |
369 370 |
{ struct dm_rq_target_io *tio = clone->end_io_data; |
4cc96131a dm: move request-... |
371 372 373 374 375 376 377 378 379 380 381 382 383 |
/* * Actual request completion is done in a softirq context which doesn't * hold the clone's queue lock. Otherwise, deadlock could occur because: * - another request may be submitted by the upper level driver * of the stacking during the completion * - the submission which requires queue lock may be done * against this clone's queue */ dm_complete_request(tio->orig, error); } static void dm_dispatch_clone_request(struct request *clone, struct request *rq) { |
2a842acab block: introduce ... |
384 |
blk_status_t r; |
4cc96131a dm: move request-... |
385 386 |
if (blk_queue_io_stat(clone->q)) |
e80640213 block: split out ... |
387 |
clone->rq_flags |= RQF_IO_STAT; |
4cc96131a dm: move request-... |
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 |
clone->start_time = jiffies; r = blk_insert_cloned_request(clone->q, clone); if (r) /* must complete clone in terms of original request */ dm_complete_request(rq, r); } static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig, void *data) { struct dm_rq_target_io *tio = data; struct dm_rq_clone_bio_info *info = container_of(bio, struct dm_rq_clone_bio_info, clone); info->orig = bio_orig; info->tio = tio; bio->bi_end_io = end_clone_bio; return 0; } static int setup_clone(struct request *clone, struct request *rq, struct dm_rq_target_io *tio, gfp_t gfp_mask) { int r; r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask, dm_rq_bio_constructor, tio); if (r) return r; |
4cc96131a dm: move request-... |
419 420 421 422 423 424 425 |
clone->end_io = end_clone_request; clone->end_io_data = tio; tio->clone = clone; return 0; } |
4cc96131a dm: move request-... |
426 427 428 429 430 431 432 433 434 435 |
static void map_tio_request(struct kthread_work *work); static void init_tio(struct dm_rq_target_io *tio, struct request *rq, struct mapped_device *md) { tio->md = md; tio->ti = NULL; tio->clone = NULL; tio->orig = rq; tio->error = 0; |
dc6364b51 dm rq: do not upd... |
436 |
tio->completed = 0; |
4cc96131a dm: move request-... |
437 438 439 440 441 442 443 444 |
/* * Avoid initializing info for blk-mq; it passes * target-specific data through info.ptr * (see: dm_mq_init_request) */ if (!md->init_tio_pdu) memset(&tio->info, 0, sizeof(tio->info)); if (md->kworker_task) |
3989144f8 kthread: kthread ... |
445 |
kthread_init_work(&tio->work, map_tio_request); |
4cc96131a dm: move request-... |
446 |
} |
4cc96131a dm: move request-... |
447 448 |
/* * Returns: |
a8ac51e4a dm rq: add DM_MAP... |
449 450 |
* DM_MAPIO_* : the request has been processed as indicated * DM_MAPIO_REQUEUE : the original request needs to be immediately requeued |
4cc96131a dm: move request-... |
451 452 |
* < 0 : the request was completed due to failure */ |
fbc39b4ca dm rq: reduce arg... |
453 |
static int map_request(struct dm_rq_target_io *tio) |
4cc96131a dm: move request-... |
454 455 456 |
{ int r; struct dm_target *ti = tio->ti; |
fbc39b4ca dm rq: reduce arg... |
457 458 |
struct mapped_device *md = tio->md; struct request *rq = tio->orig; |
4cc96131a dm: move request-... |
459 |
struct request *clone = NULL; |
eb8db831b dm: always defer ... |
460 |
r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone); |
4cc96131a dm: move request-... |
461 462 463 464 465 |
switch (r) { case DM_MAPIO_SUBMITTED: /* The target has taken the I/O to submit by itself later */ break; case DM_MAPIO_REMAPPED: |
eb8db831b dm: always defer ... |
466 467 468 469 470 |
if (setup_clone(clone, rq, tio, GFP_ATOMIC)) { /* -ENOMEM */ ti->type->release_clone_rq(clone); return DM_MAPIO_REQUEUE; } |
4cc96131a dm: move request-... |
471 472 473 474 475 476 477 |
/* The target has remapped the I/O so dispatch it */ trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)), blk_rq_pos(rq)); dm_dispatch_clone_request(clone, rq); break; case DM_MAPIO_REQUEUE: /* The target wants to requeue the I/O */ |
a8ac51e4a dm rq: add DM_MAP... |
478 479 480 |
break; case DM_MAPIO_DELAY_REQUEUE: /* The target wants to requeue the I/O after a delay */ |
fbc39b4ca dm rq: reduce arg... |
481 |
dm_requeue_original_request(tio, true); |
4cc96131a dm: move request-... |
482 |
break; |
412445acb dm: introduce a n... |
483 |
case DM_MAPIO_KILL: |
4cc96131a dm: move request-... |
484 |
/* The target wants to complete the I/O */ |
2a842acab block: introduce ... |
485 |
dm_kill_unmapped_request(rq, BLK_STS_IOERR); |
ece072803 dm rq: add a miss... |
486 |
break; |
412445acb dm: introduce a n... |
487 488 489 |
default: DMWARN("unimplemented target map return value: %d", r); BUG(); |
4cc96131a dm: move request-... |
490 |
} |
a8ac51e4a dm rq: add DM_MAP... |
491 |
return r; |
4cc96131a dm: move request-... |
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 |
} static void dm_start_request(struct mapped_device *md, struct request *orig) { if (!orig->q->mq_ops) blk_start_request(orig); else blk_mq_start_request(orig); atomic_inc(&md->pending[rq_data_dir(orig)]); if (md->seq_rq_merge_deadline_usecs) { md->last_rq_pos = rq_end_sector(orig); md->last_rq_rw = rq_data_dir(orig); md->last_rq_start_time = ktime_get(); } if (unlikely(dm_stats_used(&md->stats))) { struct dm_rq_target_io *tio = tio_from_request(orig); tio->duration_jiffies = jiffies; tio->n_sectors = blk_rq_sectors(orig); dm_stats_account_io(&md->stats, rq_data_dir(orig), blk_rq_pos(orig), tio->n_sectors, false, 0, &tio->stats_aux); } /* * Hold the md reference here for the in-flight I/O. * We can't rely on the reference count by device opener, * because the device may be closed during the request completion * when all bios are completed. * See the comment in rq_completed() too. */ dm_get(md); } |
eb8db831b dm: always defer ... |
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 |
static int __dm_rq_init_rq(struct mapped_device *md, struct request *rq) { struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); /* * Must initialize md member of tio, otherwise it won't * be available in dm_mq_queue_rq. */ tio->md = md; if (md->init_tio_pdu) { /* target-specific per-io data is immediately after the tio */ tio->info.ptr = tio + 1; } return 0; } static int dm_rq_init_rq(struct request_queue *q, struct request *rq, gfp_t gfp) { return __dm_rq_init_rq(q->rq_alloc_data, rq); } |
4cc96131a dm: move request-... |
548 549 550 |
static void map_tio_request(struct kthread_work *work) { struct dm_rq_target_io *tio = container_of(work, struct dm_rq_target_io, work); |
4cc96131a dm: move request-... |
551 |
|
fbc39b4ca dm rq: reduce arg... |
552 553 |
if (map_request(tio) == DM_MAPIO_REQUEUE) dm_requeue_original_request(tio, false); |
4cc96131a dm: move request-... |
554 555 556 557 558 559 560 561 562 563 564 565 566 567 |
} ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf) { return sprintf(buf, "%u ", md->seq_rq_merge_deadline_usecs); } #define MAX_SEQ_RQ_MERGE_DEADLINE_USECS 100000 ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md, const char *buf, size_t count) { unsigned deadline; |
e83068a5f dm mpath: add opt... |
568 |
if (dm_get_md_type(md) != DM_TYPE_REQUEST_BASED) |
4cc96131a dm: move request-... |
569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 |
return count; if (kstrtouint(buf, 10, &deadline)) return -EINVAL; if (deadline > MAX_SEQ_RQ_MERGE_DEADLINE_USECS) deadline = MAX_SEQ_RQ_MERGE_DEADLINE_USECS; md->seq_rq_merge_deadline_usecs = deadline; return count; } static bool dm_old_request_peeked_before_merge_deadline(struct mapped_device *md) { ktime_t kt_deadline; if (!md->seq_rq_merge_deadline_usecs) return false; kt_deadline = ns_to_ktime((u64)md->seq_rq_merge_deadline_usecs * NSEC_PER_USEC); kt_deadline = ktime_add_safe(md->last_rq_start_time, kt_deadline); return !ktime_after(ktime_get(), kt_deadline); } /* * q->request_fn for old request-based dm. * Called with the queue lock held. */ static void dm_old_request_fn(struct request_queue *q) { struct mapped_device *md = q->queuedata; struct dm_target *ti = md->immutable_target; struct request *rq; struct dm_rq_target_io *tio; sector_t pos = 0; if (unlikely(!ti)) { int srcu_idx; struct dm_table *map = dm_get_live_table(md, &srcu_idx); |
4087a1fff dm rq: cope with ... |
610 611 612 613 |
if (unlikely(!map)) { dm_put_live_table(md, srcu_idx); return; } |
4cc96131a dm: move request-... |
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 |
ti = dm_table_find_target(map, pos); dm_put_live_table(md, srcu_idx); } /* * For suspend, check blk_queue_stopped() and increment * ->pending within a single queue_lock not to increment the * number of in-flight I/Os after the queue is stopped in * dm_suspend(). */ while (!blk_queue_stopped(q)) { rq = blk_peek_request(q); if (!rq) return; /* always use block 0 to find the target for flushes for now */ pos = 0; if (req_op(rq) != REQ_OP_FLUSH) pos = blk_rq_pos(rq); if ((dm_old_request_peeked_before_merge_deadline(md) && |
4f9c74c60 dm rq: replace 'b... |
635 |
md_in_flight(md) && rq->bio && !bio_multiple_segments(rq->bio) && |
4cc96131a dm: move request-... |
636 637 |
md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq)) || (ti->type->busy && ti->type->busy(ti))) { |
bd9f55ea1 dm: fix second bl... |
638 |
blk_delay_queue(q, 10); |
4cc96131a dm: move request-... |
639 640 641 642 643 644 |
return; } dm_start_request(md, rq); tio = tio_from_request(rq); |
eb8db831b dm: always defer ... |
645 |
init_tio(tio, rq, md); |
4cc96131a dm: move request-... |
646 647 |
/* Establish tio->ti before queuing work (map_tio_request) */ tio->ti = ti; |
3989144f8 kthread: kthread ... |
648 |
kthread_queue_work(&md->kworker, &tio->work); |
4cc96131a dm: move request-... |
649 650 651 652 653 654 655 |
BUG_ON(!irqs_disabled()); } } /* * Fully initialize a .request_fn request-based queue. */ |
eb8db831b dm: always defer ... |
656 |
int dm_old_init_request_queue(struct mapped_device *md, struct dm_table *t) |
4cc96131a dm: move request-... |
657 |
{ |
eb8db831b dm: always defer ... |
658 |
struct dm_target *immutable_tgt; |
4cc96131a dm: move request-... |
659 |
/* Fully initialize the queue */ |
eb8db831b dm: always defer ... |
660 661 |
md->queue->cmd_size = sizeof(struct dm_rq_target_io); md->queue->rq_alloc_data = md; |
5ea708d15 block: simplify b... |
662 |
md->queue->request_fn = dm_old_request_fn; |
eb8db831b dm: always defer ... |
663 664 665 666 667 668 669 670 |
md->queue->init_rq_fn = dm_rq_init_rq; immutable_tgt = dm_table_get_immutable_target(t); if (immutable_tgt && immutable_tgt->per_io_data_size) { /* any target-specific per-io data is immediately after the tio */ md->queue->cmd_size += immutable_tgt->per_io_data_size; md->init_tio_pdu = true; } |
5ea708d15 block: simplify b... |
671 |
if (blk_init_allocated_queue(md->queue) < 0) |
4cc96131a dm: move request-... |
672 673 674 675 676 677 678 |
return -EINVAL; /* disable dm_old_request_fn's merge heuristic by default */ md->seq_rq_merge_deadline_usecs = 0; dm_init_normal_md_queue(md); blk_queue_softirq_done(md->queue, dm_softirq_done); |
4cc96131a dm: move request-... |
679 680 |
/* Initialize the request-based DM worker thread */ |
3989144f8 kthread: kthread ... |
681 |
kthread_init_worker(&md->kworker); |
4cc96131a dm: move request-... |
682 683 |
md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker, "kdmwork-%s", dm_device_name(md)); |
937fa62e8 dm rq: clear kwor... |
684 685 686 687 688 |
if (IS_ERR(md->kworker_task)) { int error = PTR_ERR(md->kworker_task); md->kworker_task = NULL; return error; } |
4cc96131a dm: move request-... |
689 690 691 692 693 |
elv_register_queue(md->queue); return 0; } |
d6296d39e blk-mq: update ->... |
694 695 |
static int dm_mq_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node) |
4cc96131a dm: move request-... |
696 |
{ |
d6296d39e blk-mq: update ->... |
697 |
return __dm_rq_init_rq(set->driver_data, rq); |
4cc96131a dm: move request-... |
698 |
} |
fc17b6534 blk-mq: switch ->... |
699 |
static blk_status_t dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, |
4cc96131a dm: move request-... |
700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 |
const struct blk_mq_queue_data *bd) { struct request *rq = bd->rq; struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq); struct mapped_device *md = tio->md; struct dm_target *ti = md->immutable_target; if (unlikely(!ti)) { int srcu_idx; struct dm_table *map = dm_get_live_table(md, &srcu_idx); ti = dm_table_find_target(map, 0); dm_put_live_table(md, srcu_idx); } if (ti->type->busy && ti->type->busy(ti)) |
fc17b6534 blk-mq: switch ->... |
716 |
return BLK_STS_RESOURCE; |
4cc96131a dm: move request-... |
717 718 719 720 721 722 723 724 725 726 727 728 |
dm_start_request(md, rq); /* Init tio using md established in .init_request */ init_tio(tio, rq, md); /* * Establish tio->ti before calling map_request(). */ tio->ti = ti; /* Direct call is fine since .queue_rq allows allocations */ |
fbc39b4ca dm rq: reduce arg... |
729 |
if (map_request(tio) == DM_MAPIO_REQUEUE) { |
4cc96131a dm: move request-... |
730 731 732 |
/* Undo dm_start_request() before requeuing */ rq_end_stats(md, rq); rq_completed(md, rq_data_dir(rq), false); |
6077c2d70 dm rq: Avoid that... |
733 |
blk_mq_delay_run_hw_queue(hctx, 100/*ms*/); |
fc17b6534 blk-mq: switch ->... |
734 |
return BLK_STS_RESOURCE; |
4cc96131a dm: move request-... |
735 |
} |
fc17b6534 blk-mq: switch ->... |
736 |
return BLK_STS_OK; |
4cc96131a dm: move request-... |
737 |
} |
f363b089b blk-mq: constify ... |
738 |
static const struct blk_mq_ops dm_mq_ops = { |
4cc96131a dm: move request-... |
739 |
.queue_rq = dm_mq_queue_rq, |
4cc96131a dm: move request-... |
740 741 742 |
.complete = dm_softirq_done, .init_request = dm_mq_init_request, }; |
e83068a5f dm mpath: add opt... |
743 |
int dm_mq_init_request_queue(struct mapped_device *md, struct dm_table *t) |
4cc96131a dm: move request-... |
744 745 |
{ struct request_queue *q; |
e83068a5f dm mpath: add opt... |
746 |
struct dm_target *immutable_tgt; |
4cc96131a dm: move request-... |
747 |
int err; |
e83068a5f dm mpath: add opt... |
748 |
if (!dm_table_all_blk_mq_devices(t)) { |
4cc96131a dm: move request-... |
749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 |
DMERR("request-based dm-mq may only be stacked on blk-mq device(s)"); return -EINVAL; } md->tag_set = kzalloc_node(sizeof(struct blk_mq_tag_set), GFP_KERNEL, md->numa_node_id); if (!md->tag_set) return -ENOMEM; md->tag_set->ops = &dm_mq_ops; md->tag_set->queue_depth = dm_get_blk_mq_queue_depth(); md->tag_set->numa_node = md->numa_node_id; md->tag_set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; md->tag_set->nr_hw_queues = dm_get_blk_mq_nr_hw_queues(); md->tag_set->driver_data = md; md->tag_set->cmd_size = sizeof(struct dm_rq_target_io); |
e83068a5f dm mpath: add opt... |
765 |
immutable_tgt = dm_table_get_immutable_target(t); |
4cc96131a dm: move request-... |
766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 |
if (immutable_tgt && immutable_tgt->per_io_data_size) { /* any target-specific per-io data is immediately after the tio */ md->tag_set->cmd_size += immutable_tgt->per_io_data_size; md->init_tio_pdu = true; } err = blk_mq_alloc_tag_set(md->tag_set); if (err) goto out_kfree_tag_set; q = blk_mq_init_allocated_queue(md->tag_set, md->queue); if (IS_ERR(q)) { err = PTR_ERR(q); goto out_tag_set; } dm_init_md_queue(md); /* backfill 'mq' sysfs registration normally done in blk_register_queue */ |
23a601248 dm rq: check blk_... |
784 785 786 |
err = blk_mq_register_dev(disk_to_dev(md->disk), q); if (err) goto out_cleanup_queue; |
4cc96131a dm: move request-... |
787 788 |
return 0; |
23a601248 dm rq: check blk_... |
789 790 |
out_cleanup_queue: blk_cleanup_queue(q); |
4cc96131a dm: move request-... |
791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 |
out_tag_set: blk_mq_free_tag_set(md->tag_set); out_kfree_tag_set: kfree(md->tag_set); return err; } void dm_mq_cleanup_mapped_device(struct mapped_device *md) { if (md->tag_set) { blk_mq_free_tag_set(md->tag_set); kfree(md->tag_set); } } module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools"); module_param(use_blk_mq, bool, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(use_blk_mq, "Use block multiqueue for request-based DM devices"); module_param(dm_mq_nr_hw_queues, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(dm_mq_nr_hw_queues, "Number of hardware queues for request-based dm-mq devices"); module_param(dm_mq_queue_depth, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(dm_mq_queue_depth, "Queue depth for request-based dm-mq devices"); |