Blame view
block/blk-mq.c
72.3 KB
75bb4625b blk-mq: add file ... |
1 2 3 4 5 6 |
/* * Block multiqueue core code * * Copyright (C) 2013-2014 Jens Axboe * Copyright (C) 2013-2014 Christoph Hellwig */ |
320ae51fe blk-mq: new multi... |
7 8 9 10 11 |
#include <linux/kernel.h> #include <linux/module.h> #include <linux/backing-dev.h> #include <linux/bio.h> #include <linux/blkdev.h> |
f75782e4e block: kmemleak: ... |
12 |
#include <linux/kmemleak.h> |
320ae51fe blk-mq: new multi... |
13 14 15 16 17 18 19 20 21 22 |
#include <linux/mm.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/workqueue.h> #include <linux/smp.h> #include <linux/llist.h> #include <linux/list_sort.h> #include <linux/cpu.h> #include <linux/cache.h> #include <linux/sched/sysctl.h> |
105ab3d8c sched/headers: Pr... |
23 |
#include <linux/sched/topology.h> |
174cd4b1e sched/headers: Pr... |
24 |
#include <linux/sched/signal.h> |
320ae51fe blk-mq: new multi... |
25 |
#include <linux/delay.h> |
aedcd72f6 blk-mq: limit mem... |
26 |
#include <linux/crash_dump.h> |
88c7b2b75 blk-mq: prefetch ... |
27 |
#include <linux/prefetch.h> |
320ae51fe blk-mq: new multi... |
28 29 30 31 32 33 |
#include <trace/events/block.h> #include <linux/blk-mq.h> #include "blk.h" #include "blk-mq.h" |
9c1051aac blk-mq: untangle ... |
34 |
#include "blk-mq-debugfs.h" |
320ae51fe blk-mq: new multi... |
35 |
#include "blk-mq-tag.h" |
cf43e6be8 block: add scalab... |
36 |
#include "blk-stat.h" |
87760e5ee block: hook up wr... |
37 |
#include "blk-wbt.h" |
bd166ef18 blk-mq-sched: add... |
38 |
#include "blk-mq-sched.h" |
320ae51fe blk-mq: new multi... |
39 |
|
45cde2f81 block: add a poll... |
40 |
static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie); |
34dbad5d2 blk-stat: convert... |
41 42 |
static void blk_mq_poll_stats_start(struct request_queue *q); static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); |
720b8ccc4 blk-mq: Add a pol... |
43 44 45 |
static int blk_mq_poll_stats_bkt(const struct request *rq) { int ddir, bytes, bucket; |
99c749a4c blk-stat: kill bl... |
46 |
ddir = rq_data_dir(rq); |
720b8ccc4 blk-mq: Add a pol... |
47 48 49 50 51 52 53 54 55 56 57 |
bytes = blk_rq_bytes(rq); bucket = ddir + 2*(ilog2(bytes) - 9); if (bucket < 0) return -1; else if (bucket >= BLK_MQ_POLL_STATS_BKTS) return ddir + BLK_MQ_POLL_STATS_BKTS - 2; return bucket; } |
320ae51fe blk-mq: new multi... |
58 59 60 |
/* * Check if any of the ctx's have pending work in this hardware queue */ |
50e1dab86 blk-mq-sched: fix... |
61 |
bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) |
320ae51fe blk-mq: new multi... |
62 |
{ |
bd166ef18 blk-mq-sched: add... |
63 64 65 |
return sbitmap_any_bit_set(&hctx->ctx_map) || !list_empty_careful(&hctx->dispatch) || blk_mq_sched_has_work(hctx); |
1429d7c94 blk-mq: switch ct... |
66 |
} |
320ae51fe blk-mq: new multi... |
67 68 69 70 71 72 |
/* * Mark this ctx as having pending work in this hardware queue */ static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx) { |
88459642c blk-mq: abstract ... |
73 74 |
if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw)) sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw); |
1429d7c94 blk-mq: switch ct... |
75 76 77 78 79 |
} static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx) { |
88459642c blk-mq: abstract ... |
80 |
sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw); |
320ae51fe blk-mq: new multi... |
81 |
} |
f299b7c7a blk-mq: provide i... |
82 83 84 85 86 87 88 89 90 91 92 93 94 95 |
struct mq_inflight { struct hd_struct *part; unsigned int *inflight; }; static void blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq, void *priv, bool reserved) { struct mq_inflight *mi = priv; if (test_bit(REQ_ATOM_STARTED, &rq->atomic_flags) && !test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) { /* |
b8d62b3a9 blk-mq: enable ch... |
96 97 98 99 |
* index[0] counts the specific partition that was asked * for. index[1] counts the ones that are active on the * whole device, so increment that if mi->part is indeed * a partition, and not a whole device. |
f299b7c7a blk-mq: provide i... |
100 |
*/ |
b8d62b3a9 blk-mq: enable ch... |
101 |
if (rq->part == mi->part) |
f299b7c7a blk-mq: provide i... |
102 |
mi->inflight[0]++; |
b8d62b3a9 blk-mq: enable ch... |
103 104 |
if (mi->part->partno) mi->inflight[1]++; |
f299b7c7a blk-mq: provide i... |
105 106 107 108 109 110 111 |
} } void blk_mq_in_flight(struct request_queue *q, struct hd_struct *part, unsigned int inflight[2]) { struct mq_inflight mi = { .part = part, .inflight = inflight, }; |
b8d62b3a9 blk-mq: enable ch... |
112 |
inflight[0] = inflight[1] = 0; |
f299b7c7a blk-mq: provide i... |
113 114 |
blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi); } |
06beec871 blk-mq: fix sysfs... |
115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 |
static void blk_mq_check_inflight_rw(struct blk_mq_hw_ctx *hctx, struct request *rq, void *priv, bool reserved) { struct mq_inflight *mi = priv; if (rq->part == mi->part) mi->inflight[rq_data_dir(rq)]++; } void blk_mq_in_flight_rw(struct request_queue *q, struct hd_struct *part, unsigned int inflight[2]) { struct mq_inflight mi = { .part = part, .inflight = inflight, }; inflight[0] = inflight[1] = 0; blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight_rw, &mi); } |
1671d522c block: rename blk... |
133 |
void blk_freeze_queue_start(struct request_queue *q) |
43a5e4e21 block: blk-mq: su... |
134 |
{ |
4ecd4fef3 block: use an ato... |
135 |
int freeze_depth; |
cddd5d176 blk-mq: blk_mq_fr... |
136 |
|
4ecd4fef3 block: use an ato... |
137 138 |
freeze_depth = atomic_inc_return(&q->mq_freeze_depth); if (freeze_depth == 1) { |
3ef28e83a block: generic re... |
139 |
percpu_ref_kill(&q->q_usage_counter); |
b94ec2964 blk-mq: export bl... |
140 |
blk_mq_run_hw_queues(q, false); |
cddd5d176 blk-mq: blk_mq_fr... |
141 |
} |
f3af020b9 blk-mq: make mq_q... |
142 |
} |
1671d522c block: rename blk... |
143 |
EXPORT_SYMBOL_GPL(blk_freeze_queue_start); |
f3af020b9 blk-mq: make mq_q... |
144 |
|
6bae363ee blk-mq: Export bl... |
145 |
void blk_mq_freeze_queue_wait(struct request_queue *q) |
f3af020b9 blk-mq: make mq_q... |
146 |
{ |
3ef28e83a block: generic re... |
147 |
wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter)); |
43a5e4e21 block: blk-mq: su... |
148 |
} |
6bae363ee blk-mq: Export bl... |
149 |
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait); |
43a5e4e21 block: blk-mq: su... |
150 |
|
f91328c40 blk-mq: Provide f... |
151 152 153 154 155 156 157 158 |
int blk_mq_freeze_queue_wait_timeout(struct request_queue *q, unsigned long timeout) { return wait_event_timeout(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter), timeout); } EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout); |
43a5e4e21 block: blk-mq: su... |
159 |
|
f3af020b9 blk-mq: make mq_q... |
160 161 162 163 |
/* * Guarantee no request is in use, so we can change any data structure of * the queue afterward. */ |
3ef28e83a block: generic re... |
164 |
void blk_freeze_queue(struct request_queue *q) |
f3af020b9 blk-mq: make mq_q... |
165 |
{ |
3ef28e83a block: generic re... |
166 167 168 169 170 171 172 |
/* * In the !blk_mq case we are only calling this to kill the * q_usage_counter, otherwise this increases the freeze depth * and waits for it to return to zero. For this reason there is * no blk_unfreeze_queue(), and blk_freeze_queue() is not * exported to drivers as the only user for unfreeze is blk_mq. */ |
1671d522c block: rename blk... |
173 |
blk_freeze_queue_start(q); |
7e3acce11 block: drain queu... |
174 175 |
if (!q->mq_ops) blk_drain_queue(q); |
f3af020b9 blk-mq: make mq_q... |
176 177 |
blk_mq_freeze_queue_wait(q); } |
3ef28e83a block: generic re... |
178 179 180 181 182 183 184 185 186 |
void blk_mq_freeze_queue(struct request_queue *q) { /* * ...just an alias to keep freeze and unfreeze actions balanced * in the blk_mq_* namespace */ blk_freeze_queue(q); } |
c761d96b0 blk-mq: export bl... |
187 |
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue); |
f3af020b9 blk-mq: make mq_q... |
188 |
|
b4c6a0287 blk-mq: Export fr... |
189 |
void blk_mq_unfreeze_queue(struct request_queue *q) |
320ae51fe blk-mq: new multi... |
190 |
{ |
4ecd4fef3 block: use an ato... |
191 |
int freeze_depth; |
320ae51fe blk-mq: new multi... |
192 |
|
4ecd4fef3 block: use an ato... |
193 194 195 |
freeze_depth = atomic_dec_return(&q->mq_freeze_depth); WARN_ON_ONCE(freeze_depth < 0); if (!freeze_depth) { |
3ef28e83a block: generic re... |
196 |
percpu_ref_reinit(&q->q_usage_counter); |
320ae51fe blk-mq: new multi... |
197 |
wake_up_all(&q->mq_freeze_wq); |
add703fda blk-mq: use percp... |
198 |
} |
320ae51fe blk-mq: new multi... |
199 |
} |
b4c6a0287 blk-mq: Export fr... |
200 |
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); |
320ae51fe blk-mq: new multi... |
201 |
|
852ec8098 blk-mq: Make it s... |
202 203 204 205 206 207 208 209 210 211 212 213 214 |
/* * FIXME: replace the scsi_internal_device_*block_nowait() calls in the * mpt3sas driver such that this function can be removed. */ void blk_mq_quiesce_queue_nowait(struct request_queue *q) { unsigned long flags; spin_lock_irqsave(q->queue_lock, flags); queue_flag_set(QUEUE_FLAG_QUIESCED, q); spin_unlock_irqrestore(q->queue_lock, flags); } EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait); |
6a83e74d2 blk-mq: Introduce... |
215 |
/** |
69e07c4ad blk-mq: update co... |
216 |
* blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished |
6a83e74d2 blk-mq: Introduce... |
217 218 219 |
* @q: request queue. * * Note: this function does not prevent that the struct request end_io() |
69e07c4ad blk-mq: update co... |
220 221 222 |
* callback function is invoked. Once this function is returned, we make * sure no dispatch can happen until the queue is unquiesced via * blk_mq_unquiesce_queue(). |
6a83e74d2 blk-mq: Introduce... |
223 224 225 226 227 228 |
*/ void blk_mq_quiesce_queue(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; unsigned int i; bool rcu = false; |
1d9e9bc6b blk-mq: don't sto... |
229 |
blk_mq_quiesce_queue_nowait(q); |
f4560ffe8 blk-mq: use QUEUE... |
230 |
|
6a83e74d2 blk-mq: Introduce... |
231 232 |
queue_for_each_hw_ctx(q, hctx, i) { if (hctx->flags & BLK_MQ_F_BLOCKING) |
073196787 blk-mq: Reduce bl... |
233 |
synchronize_srcu(hctx->queue_rq_srcu); |
6a83e74d2 blk-mq: Introduce... |
234 235 236 237 238 239 240 |
else rcu = true; } if (rcu) synchronize_rcu(); } EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue); |
e4e739131 blk-mq: introduce... |
241 242 243 244 245 246 247 248 249 |
/* * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue() * @q: request queue. * * This function recovers queue into the state before quiescing * which is done by blk_mq_quiesce_queue. */ void blk_mq_unquiesce_queue(struct request_queue *q) { |
852ec8098 blk-mq: Make it s... |
250 251 252 |
unsigned long flags; spin_lock_irqsave(q->queue_lock, flags); |
f4560ffe8 blk-mq: use QUEUE... |
253 |
queue_flag_clear(QUEUE_FLAG_QUIESCED, q); |
852ec8098 blk-mq: Make it s... |
254 |
spin_unlock_irqrestore(q->queue_lock, flags); |
f4560ffe8 blk-mq: use QUEUE... |
255 |
|
1d9e9bc6b blk-mq: don't sto... |
256 257 |
/* dispatch requests which are inserted during quiescing */ blk_mq_run_hw_queues(q, true); |
e4e739131 blk-mq: introduce... |
258 259 |
} EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue); |
aed3ea94b block: wake up wa... |
260 261 262 263 264 265 266 267 |
void blk_mq_wake_waiters(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; unsigned int i; queue_for_each_hw_ctx(q, hctx, i) if (blk_mq_hw_queue_mapped(hctx)) blk_mq_tag_wakeup_all(hctx->tags, true); |
3fd5940cb blk-mq: Wake task... |
268 269 270 271 272 273 274 |
/* * If we are called because the queue has now been marked as * dying, we need to ensure that processes currently waiting on * the queue are notified as well. */ wake_up_all(&q->mq_freeze_wq); |
aed3ea94b block: wake up wa... |
275 |
} |
320ae51fe blk-mq: new multi... |
276 277 278 279 280 |
bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) { return blk_mq_has_free_tags(hctx->tags); } EXPORT_SYMBOL(blk_mq_can_queue); |
e4cdf1a1c blk-mq: remove __... |
281 282 |
static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data, unsigned int tag, unsigned int op) |
320ae51fe blk-mq: new multi... |
283 |
{ |
e4cdf1a1c blk-mq: remove __... |
284 285 |
struct blk_mq_tags *tags = blk_mq_tags_from_data(data); struct request *rq = tags->static_rqs[tag]; |
c3a148d20 blk-mq: Initializ... |
286 |
rq->rq_flags = 0; |
e4cdf1a1c blk-mq: remove __... |
287 288 289 290 291 292 293 294 295 296 297 298 |
if (data->flags & BLK_MQ_REQ_INTERNAL) { rq->tag = -1; rq->internal_tag = tag; } else { if (blk_mq_tag_busy(data->hctx)) { rq->rq_flags = RQF_MQ_INFLIGHT; atomic_inc(&data->hctx->nr_active); } rq->tag = tag; rq->internal_tag = -1; data->hctx->tags->rqs[rq->tag] = rq; } |
af76e555e blk-mq: initializ... |
299 300 |
INIT_LIST_HEAD(&rq->queuelist); /* csd/requeue_work/fifo_time is initialized before use */ |
e4cdf1a1c blk-mq: remove __... |
301 302 |
rq->q = data->q; rq->mq_ctx = data->ctx; |
ef295ecf0 block: better op ... |
303 |
rq->cmd_flags = op; |
e4cdf1a1c blk-mq: remove __... |
304 |
if (blk_queue_io_stat(data->q)) |
e80640213 block: split out ... |
305 |
rq->rq_flags |= RQF_IO_STAT; |
af76e555e blk-mq: initializ... |
306 307 |
/* do not touch atomic flags, it needs atomic ops against the timer */ rq->cpu = -1; |
af76e555e blk-mq: initializ... |
308 309 |
INIT_HLIST_NODE(&rq->hash); RB_CLEAR_NODE(&rq->rb_node); |
af76e555e blk-mq: initializ... |
310 311 |
rq->rq_disk = NULL; rq->part = NULL; |
3ee323723 blk-mq: always in... |
312 |
rq->start_time = jiffies; |
af76e555e blk-mq: initializ... |
313 314 |
#ifdef CONFIG_BLK_CGROUP rq->rl = NULL; |
0fec08b4e blk-mq: fix initi... |
315 |
set_start_time_ns(rq); |
af76e555e blk-mq: initializ... |
316 317 318 319 320 321 |
rq->io_start_time_ns = 0; #endif rq->nr_phys_segments = 0; #if defined(CONFIG_BLK_DEV_INTEGRITY) rq->nr_integrity_segments = 0; #endif |
af76e555e blk-mq: initializ... |
322 323 |
rq->special = NULL; /* tag was already set */ |
af76e555e blk-mq: initializ... |
324 |
rq->extra_len = 0; |
af76e555e blk-mq: initializ... |
325 |
|
af76e555e blk-mq: initializ... |
326 |
INIT_LIST_HEAD(&rq->timeout_list); |
f6be4fb4b blk-mq: ->timeout... |
327 |
rq->timeout = 0; |
af76e555e blk-mq: initializ... |
328 329 330 |
rq->end_io = NULL; rq->end_io_data = NULL; rq->next_rq = NULL; |
e4cdf1a1c blk-mq: remove __... |
331 332 |
data->ctx->rq_dispatched[op_is_sync(op)]++; return rq; |
5dee85772 blk-mq: initializ... |
333 |
} |
d2c0d3832 blk-mq: move blk_... |
334 335 336 337 338 339 |
static struct request *blk_mq_get_request(struct request_queue *q, struct bio *bio, unsigned int op, struct blk_mq_alloc_data *data) { struct elevator_queue *e = q->elevator; struct request *rq; |
e4cdf1a1c blk-mq: remove __... |
340 |
unsigned int tag; |
1ad43c007 blk-mq: don't lea... |
341 |
struct blk_mq_ctx *local_ctx = NULL; |
d2c0d3832 blk-mq: move blk_... |
342 343 344 345 |
blk_queue_enter_live(q); data->q = q; if (likely(!data->ctx)) |
1ad43c007 blk-mq: don't lea... |
346 |
data->ctx = local_ctx = blk_mq_get_ctx(q); |
d2c0d3832 blk-mq: move blk_... |
347 348 |
if (likely(!data->hctx)) data->hctx = blk_mq_map_queue(q, data->ctx->cpu); |
03a07c92a block: return on ... |
349 350 |
if (op & REQ_NOWAIT) data->flags |= BLK_MQ_REQ_NOWAIT; |
d2c0d3832 blk-mq: move blk_... |
351 352 353 354 355 356 357 358 |
if (e) { data->flags |= BLK_MQ_REQ_INTERNAL; /* * Flush requests are special and go directly to the * dispatch list. */ |
5bbf4e5a8 blk-mq-sched: uni... |
359 360 |
if (!op_is_flush(op) && e->type->ops.mq.limit_depth) e->type->ops.mq.limit_depth(op, data); |
d2c0d3832 blk-mq: move blk_... |
361 |
} |
e4cdf1a1c blk-mq: remove __... |
362 363 |
tag = blk_mq_get_tag(data); if (tag == BLK_MQ_TAG_FAIL) { |
1ad43c007 blk-mq: don't lea... |
364 365 366 367 |
if (local_ctx) { blk_mq_put_ctx(local_ctx); data->ctx = NULL; } |
037cebb85 blk-mq: streamlin... |
368 369 |
blk_queue_exit(q); return NULL; |
d2c0d3832 blk-mq: move blk_... |
370 |
} |
e4cdf1a1c blk-mq: remove __... |
371 |
rq = blk_mq_rq_ctx_init(data, tag, op); |
037cebb85 blk-mq: streamlin... |
372 373 |
if (!op_is_flush(op)) { rq->elv.icq = NULL; |
5bbf4e5a8 blk-mq-sched: uni... |
374 |
if (e && e->type->ops.mq.prepare_request) { |
44e8c2bff blk-mq: refactor ... |
375 376 |
if (e->type->icq_cache && rq_ioc(bio)) blk_mq_sched_assign_ioc(rq, bio); |
5bbf4e5a8 blk-mq-sched: uni... |
377 378 |
e->type->ops.mq.prepare_request(rq, bio); rq->rq_flags |= RQF_ELVPRIV; |
44e8c2bff blk-mq: refactor ... |
379 |
} |
037cebb85 blk-mq: streamlin... |
380 381 382 |
} data->hctx->queued++; return rq; |
d2c0d3832 blk-mq: move blk_... |
383 |
} |
cd6ce1482 block: Make reque... |
384 |
struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op, |
6f3b0e8bc blk-mq: add a fla... |
385 |
unsigned int flags) |
320ae51fe blk-mq: new multi... |
386 |
{ |
5a797e00d blk-mq: don't los... |
387 |
struct blk_mq_alloc_data alloc_data = { .flags = flags }; |
bd166ef18 blk-mq-sched: add... |
388 |
struct request *rq; |
a492f0754 block,scsi: fixup... |
389 |
int ret; |
320ae51fe blk-mq: new multi... |
390 |
|
6f3b0e8bc blk-mq: add a fla... |
391 |
ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT); |
a492f0754 block,scsi: fixup... |
392 393 |
if (ret) return ERR_PTR(ret); |
320ae51fe blk-mq: new multi... |
394 |
|
cd6ce1482 block: Make reque... |
395 |
rq = blk_mq_get_request(q, NULL, op, &alloc_data); |
3280d66a6 blk-mq: Fix queue... |
396 |
blk_queue_exit(q); |
841bac2c8 blk-mq: get rid o... |
397 |
|
bd166ef18 blk-mq-sched: add... |
398 |
if (!rq) |
a492f0754 block,scsi: fixup... |
399 |
return ERR_PTR(-EWOULDBLOCK); |
0c4de0f33 block: ensure bio... |
400 |
|
1ad43c007 blk-mq: don't lea... |
401 |
blk_mq_put_ctx(alloc_data.ctx); |
1ad43c007 blk-mq: don't lea... |
402 |
|
0c4de0f33 block: ensure bio... |
403 404 405 |
rq->__data_len = 0; rq->__sector = (sector_t) -1; rq->bio = rq->biotail = NULL; |
320ae51fe blk-mq: new multi... |
406 407 |
return rq; } |
4bb659b15 blk-mq: implement... |
408 |
EXPORT_SYMBOL(blk_mq_alloc_request); |
320ae51fe blk-mq: new multi... |
409 |
|
cd6ce1482 block: Make reque... |
410 411 |
struct request *blk_mq_alloc_request_hctx(struct request_queue *q, unsigned int op, unsigned int flags, unsigned int hctx_idx) |
1f5bd336b blk-mq: add blk_m... |
412 |
{ |
6d2809d51 blk-mq: make blk_... |
413 |
struct blk_mq_alloc_data alloc_data = { .flags = flags }; |
1f5bd336b blk-mq: add blk_m... |
414 |
struct request *rq; |
6d2809d51 blk-mq: make blk_... |
415 |
unsigned int cpu; |
1f5bd336b blk-mq: add blk_m... |
416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 |
int ret; /* * If the tag allocator sleeps we could get an allocation for a * different hardware context. No need to complicate the low level * allocator for this for the rare use case of a command tied to * a specific queue. */ if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT))) return ERR_PTR(-EINVAL); if (hctx_idx >= q->nr_hw_queues) return ERR_PTR(-EIO); ret = blk_queue_enter(q, true); if (ret) return ERR_PTR(ret); |
c8712c6a6 blk-mq: skip unma... |
433 434 435 436 |
/* * Check if the hardware context is actually mapped to anything. * If not tell the caller that it should skip this queue. */ |
6d2809d51 blk-mq: make blk_... |
437 438 439 440 |
alloc_data.hctx = q->queue_hw_ctx[hctx_idx]; if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) { blk_queue_exit(q); return ERR_PTR(-EXDEV); |
c8712c6a6 blk-mq: skip unma... |
441 |
} |
6d2809d51 blk-mq: make blk_... |
442 443 |
cpu = cpumask_first(alloc_data.hctx->cpumask); alloc_data.ctx = __blk_mq_get_ctx(q, cpu); |
1f5bd336b blk-mq: add blk_m... |
444 |
|
cd6ce1482 block: Make reque... |
445 |
rq = blk_mq_get_request(q, NULL, op, &alloc_data); |
3280d66a6 blk-mq: Fix queue... |
446 |
blk_queue_exit(q); |
c8712c6a6 blk-mq: skip unma... |
447 |
|
6d2809d51 blk-mq: make blk_... |
448 449 450 451 |
if (!rq) return ERR_PTR(-EWOULDBLOCK); return rq; |
1f5bd336b blk-mq: add blk_m... |
452 453 |
} EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx); |
6af54051a blk-mq: simplify ... |
454 |
void blk_mq_free_request(struct request *rq) |
320ae51fe blk-mq: new multi... |
455 |
{ |
320ae51fe blk-mq: new multi... |
456 |
struct request_queue *q = rq->q; |
6af54051a blk-mq: simplify ... |
457 458 459 460 |
struct elevator_queue *e = q->elevator; struct blk_mq_ctx *ctx = rq->mq_ctx; struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu); const int sched_tag = rq->internal_tag; |
5bbf4e5a8 blk-mq-sched: uni... |
461 |
if (rq->rq_flags & RQF_ELVPRIV) { |
6af54051a blk-mq: simplify ... |
462 463 464 465 466 467 468 |
if (e && e->type->ops.mq.finish_request) e->type->ops.mq.finish_request(rq); if (rq->elv.icq) { put_io_context(rq->elv.icq->ioc); rq->elv.icq = NULL; } } |
320ae51fe blk-mq: new multi... |
469 |
|
6af54051a blk-mq: simplify ... |
470 |
ctx->rq_completed[rq_is_sync(rq)]++; |
e80640213 block: split out ... |
471 |
if (rq->rq_flags & RQF_MQ_INFLIGHT) |
0d2602ca3 blk-mq: improve s... |
472 |
atomic_dec(&hctx->nr_active); |
87760e5ee block: hook up wr... |
473 474 |
wbt_done(q->rq_wb, &rq->issue_stat); |
0d2602ca3 blk-mq: improve s... |
475 |
|
af76e555e blk-mq: initializ... |
476 |
clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags); |
06426adf0 blk-mq: implement... |
477 |
clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags); |
bd166ef18 blk-mq-sched: add... |
478 479 480 |
if (rq->tag != -1) blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag); if (sched_tag != -1) |
c05f8525f blk-mq-sched: mak... |
481 |
blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag); |
6d8c6c0f9 blk-mq: Restart a... |
482 |
blk_mq_sched_restart(hctx); |
3ef28e83a block: generic re... |
483 |
blk_queue_exit(q); |
320ae51fe blk-mq: new multi... |
484 |
} |
1a3b595a2 blk-mq: export bl... |
485 |
EXPORT_SYMBOL_GPL(blk_mq_free_request); |
320ae51fe blk-mq: new multi... |
486 |
|
2a842acab block: introduce ... |
487 |
inline void __blk_mq_end_request(struct request *rq, blk_status_t error) |
320ae51fe blk-mq: new multi... |
488 |
{ |
0d11e6aca blk-mq: fix use-a... |
489 |
blk_account_io_done(rq); |
91b63639c blk-mq: bidi support |
490 |
if (rq->end_io) { |
87760e5ee block: hook up wr... |
491 |
wbt_done(rq->q->rq_wb, &rq->issue_stat); |
320ae51fe blk-mq: new multi... |
492 |
rq->end_io(rq, error); |
91b63639c blk-mq: bidi support |
493 494 495 |
} else { if (unlikely(blk_bidi_rq(rq))) blk_mq_free_request(rq->next_rq); |
320ae51fe blk-mq: new multi... |
496 |
blk_mq_free_request(rq); |
91b63639c blk-mq: bidi support |
497 |
} |
320ae51fe blk-mq: new multi... |
498 |
} |
c8a446ad6 blk-mq: rename bl... |
499 |
EXPORT_SYMBOL(__blk_mq_end_request); |
63151a449 blk-mq: allow dri... |
500 |
|
2a842acab block: introduce ... |
501 |
void blk_mq_end_request(struct request *rq, blk_status_t error) |
63151a449 blk-mq: allow dri... |
502 503 504 |
{ if (blk_update_request(rq, error, blk_rq_bytes(rq))) BUG(); |
c8a446ad6 blk-mq: rename bl... |
505 |
__blk_mq_end_request(rq, error); |
63151a449 blk-mq: allow dri... |
506 |
} |
c8a446ad6 blk-mq: rename bl... |
507 |
EXPORT_SYMBOL(blk_mq_end_request); |
320ae51fe blk-mq: new multi... |
508 |
|
30a91cb4e blk-mq: rework I/... |
509 |
static void __blk_mq_complete_request_remote(void *data) |
320ae51fe blk-mq: new multi... |
510 |
{ |
3d6efbf62 blk-mq: use __smp... |
511 |
struct request *rq = data; |
320ae51fe blk-mq: new multi... |
512 |
|
30a91cb4e blk-mq: rework I/... |
513 |
rq->q->softirq_done_fn(rq); |
320ae51fe blk-mq: new multi... |
514 |
} |
320ae51fe blk-mq: new multi... |
515 |
|
453f83418 blk-mq: simplify ... |
516 |
static void __blk_mq_complete_request(struct request *rq) |
320ae51fe blk-mq: new multi... |
517 518 |
{ struct blk_mq_ctx *ctx = rq->mq_ctx; |
385352016 blk-mq: respect r... |
519 |
bool shared = false; |
320ae51fe blk-mq: new multi... |
520 |
int cpu; |
453f83418 blk-mq: simplify ... |
521 522 523 524 525 526 |
if (rq->internal_tag != -1) blk_mq_sched_completed_request(rq); if (rq->rq_flags & RQF_STATS) { blk_mq_poll_stats_start(rq->q); blk_stat_add(rq); } |
385352016 blk-mq: respect r... |
527 |
if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) { |
30a91cb4e blk-mq: rework I/... |
528 529 530 |
rq->q->softirq_done_fn(rq); return; } |
320ae51fe blk-mq: new multi... |
531 532 |
cpu = get_cpu(); |
385352016 blk-mq: respect r... |
533 534 535 536 |
if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags)) shared = cpus_share_cache(cpu, ctx->cpu); if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) { |
30a91cb4e blk-mq: rework I/... |
537 |
rq->csd.func = __blk_mq_complete_request_remote; |
3d6efbf62 blk-mq: use __smp... |
538 539 |
rq->csd.info = rq; rq->csd.flags = 0; |
c46fff2a3 smp: Rename __smp... |
540 |
smp_call_function_single_async(ctx->cpu, &rq->csd); |
3d6efbf62 blk-mq: use __smp... |
541 |
} else { |
30a91cb4e blk-mq: rework I/... |
542 |
rq->q->softirq_done_fn(rq); |
3d6efbf62 blk-mq: use __smp... |
543 |
} |
320ae51fe blk-mq: new multi... |
544 545 |
put_cpu(); } |
30a91cb4e blk-mq: rework I/... |
546 547 548 549 550 551 552 553 554 |
/** * blk_mq_complete_request - end I/O on a request * @rq: the request being processed * * Description: * Ends all I/O on a request. It does not handle partial completions. * The actual completion happens out-of-order, through a IPI handler. **/ |
08e0029aa blk-mq: remove th... |
555 |
void blk_mq_complete_request(struct request *rq) |
30a91cb4e blk-mq: rework I/... |
556 |
{ |
95f096849 blk-mq: allow non... |
557 558 559 |
struct request_queue *q = rq->q; if (unlikely(blk_should_fake_timeout(q))) |
30a91cb4e blk-mq: rework I/... |
560 |
return; |
08e0029aa blk-mq: remove th... |
561 |
if (!blk_mark_rq_complete(rq)) |
ed851860b blk-mq: push IPI ... |
562 |
__blk_mq_complete_request(rq); |
30a91cb4e blk-mq: rework I/... |
563 564 |
} EXPORT_SYMBOL(blk_mq_complete_request); |
320ae51fe blk-mq: new multi... |
565 |
|
973c01919 blk-mq: Export if... |
566 567 568 569 570 |
int blk_mq_request_started(struct request *rq) { return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags); } EXPORT_SYMBOL_GPL(blk_mq_request_started); |
e2490073c blk-mq: call blk_... |
571 |
void blk_mq_start_request(struct request *rq) |
320ae51fe blk-mq: new multi... |
572 573 |
{ struct request_queue *q = rq->q; |
bd166ef18 blk-mq-sched: add... |
574 |
blk_mq_sched_started_request(rq); |
320ae51fe blk-mq: new multi... |
575 |
trace_block_rq_issue(q, rq); |
cf43e6be8 block: add scalab... |
576 |
if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { |
88eeca495 block: track requ... |
577 |
blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq)); |
cf43e6be8 block: add scalab... |
578 |
rq->rq_flags |= RQF_STATS; |
87760e5ee block: hook up wr... |
579 |
wbt_issue(q->rq_wb, &rq->issue_stat); |
cf43e6be8 block: add scalab... |
580 |
} |
2b8393b43 blk-mq: add timer... |
581 |
blk_add_timer(rq); |
87ee7b112 blk-mq: fix race ... |
582 583 |
/* |
538b75341 blk-mq: request d... |
584 585 586 587 588 589 |
* Ensure that ->deadline is visible before set the started * flag and clear the completed flag. */ smp_mb__before_atomic(); /* |
87ee7b112 blk-mq: fix race ... |
590 591 592 593 594 |
* Mark us as started and clear complete. Complete might have been * set if requeue raced with timeout, which then marked it as * complete. So be sure to clear complete again when we start * the request, otherwise we'll ignore the completion event. */ |
4b570521b blk-mq: request i... |
595 596 597 598 |
if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) set_bit(REQ_ATOM_STARTED, &rq->atomic_flags); if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); |
49f5baa51 blk-mq: pair blk_... |
599 600 601 602 603 604 605 606 607 |
if (q->dma_drain_size && blk_rq_bytes(rq)) { /* * Make sure space for the drain appears. We know we can do * this because max_hw_segments has been adjusted to be one * fewer than the device can handle. */ rq->nr_phys_segments++; } |
320ae51fe blk-mq: new multi... |
608 |
} |
e2490073c blk-mq: call blk_... |
609 |
EXPORT_SYMBOL(blk_mq_start_request); |
320ae51fe blk-mq: new multi... |
610 |
|
d9d149a39 blk-mq: comment o... |
611 612 |
/* * When we reach here because queue is busy, REQ_ATOM_COMPLETE |
48b99c9d6 blk-mq: fix a typ... |
613 |
* flag isn't set yet, so there may be race with timeout handler, |
d9d149a39 blk-mq: comment o... |
614 615 616 617 618 619 |
* but given rq->deadline is just set in .queue_rq() under * this situation, the race won't be possible in reality because * rq->timeout should be set as big enough to cover the window * between blk_mq_start_request() called from .queue_rq() and * clearing REQ_ATOM_STARTED here. */ |
ed0791b2f blk-mq: add blk_m... |
620 |
static void __blk_mq_requeue_request(struct request *rq) |
320ae51fe blk-mq: new multi... |
621 622 623 624 |
{ struct request_queue *q = rq->q; trace_block_rq_requeue(q, rq); |
87760e5ee block: hook up wr... |
625 |
wbt_requeue(q->rq_wb, &rq->issue_stat); |
49f5baa51 blk-mq: pair blk_... |
626 |
|
e2490073c blk-mq: call blk_... |
627 628 629 630 |
if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) { if (q->dma_drain_size && blk_rq_bytes(rq)) rq->nr_phys_segments--; } |
320ae51fe blk-mq: new multi... |
631 |
} |
2b053aca7 blk-mq: Add a kic... |
632 |
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) |
ed0791b2f blk-mq: add blk_m... |
633 |
{ |
ed0791b2f blk-mq: add blk_m... |
634 |
__blk_mq_requeue_request(rq); |
ed0791b2f blk-mq: add blk_m... |
635 |
|
ccddee811 blk-mq: don't cal... |
636 637 |
/* this request will be re-inserted to io scheduler queue */ blk_mq_sched_requeue_request(rq); |
ed0791b2f blk-mq: add blk_m... |
638 |
BUG_ON(blk_queued_rq(rq)); |
2b053aca7 blk-mq: Add a kic... |
639 |
blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); |
ed0791b2f blk-mq: add blk_m... |
640 641 |
} EXPORT_SYMBOL(blk_mq_requeue_request); |
6fca6a611 blk-mq: add helpe... |
642 643 644 |
static void blk_mq_requeue_work(struct work_struct *work) { struct request_queue *q = |
2849450ad blk-mq: introduce... |
645 |
container_of(work, struct request_queue, requeue_work.work); |
6fca6a611 blk-mq: add helpe... |
646 647 |
LIST_HEAD(rq_list); struct request *rq, *next; |
6fca6a611 blk-mq: add helpe... |
648 |
|
18e9781d4 blk-mq: blk_mq_re... |
649 |
spin_lock_irq(&q->requeue_lock); |
6fca6a611 blk-mq: add helpe... |
650 |
list_splice_init(&q->requeue_list, &rq_list); |
18e9781d4 blk-mq: blk_mq_re... |
651 |
spin_unlock_irq(&q->requeue_lock); |
6fca6a611 blk-mq: add helpe... |
652 653 |
list_for_each_entry_safe(rq, next, &rq_list, queuelist) { |
e80640213 block: split out ... |
654 |
if (!(rq->rq_flags & RQF_SOFTBARRIER)) |
6fca6a611 blk-mq: add helpe... |
655 |
continue; |
e80640213 block: split out ... |
656 |
rq->rq_flags &= ~RQF_SOFTBARRIER; |
6fca6a611 blk-mq: add helpe... |
657 |
list_del_init(&rq->queuelist); |
bd6737f1a blk-mq-sched: add... |
658 |
blk_mq_sched_insert_request(rq, true, false, false, true); |
6fca6a611 blk-mq: add helpe... |
659 660 661 662 663 |
} while (!list_empty(&rq_list)) { rq = list_entry(rq_list.next, struct request, queuelist); list_del_init(&rq->queuelist); |
bd6737f1a blk-mq-sched: add... |
664 |
blk_mq_sched_insert_request(rq, false, false, false, true); |
6fca6a611 blk-mq: add helpe... |
665 |
} |
52d7f1b5c blk-mq: Avoid tha... |
666 |
blk_mq_run_hw_queues(q, false); |
6fca6a611 blk-mq: add helpe... |
667 |
} |
2b053aca7 blk-mq: Add a kic... |
668 669 |
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, bool kick_requeue_list) |
6fca6a611 blk-mq: add helpe... |
670 671 672 673 674 675 676 677 |
{ struct request_queue *q = rq->q; unsigned long flags; /* * We abuse this flag that is otherwise used by the I/O scheduler to * request head insertation from the workqueue. */ |
e80640213 block: split out ... |
678 |
BUG_ON(rq->rq_flags & RQF_SOFTBARRIER); |
6fca6a611 blk-mq: add helpe... |
679 680 681 |
spin_lock_irqsave(&q->requeue_lock, flags); if (at_head) { |
e80640213 block: split out ... |
682 |
rq->rq_flags |= RQF_SOFTBARRIER; |
6fca6a611 blk-mq: add helpe... |
683 684 685 686 687 |
list_add(&rq->queuelist, &q->requeue_list); } else { list_add_tail(&rq->queuelist, &q->requeue_list); } spin_unlock_irqrestore(&q->requeue_lock, flags); |
2b053aca7 blk-mq: Add a kic... |
688 689 690 |
if (kick_requeue_list) blk_mq_kick_requeue_list(q); |
6fca6a611 blk-mq: add helpe... |
691 692 693 694 695 |
} EXPORT_SYMBOL(blk_mq_add_to_requeue_list); void blk_mq_kick_requeue_list(struct request_queue *q) { |
2849450ad blk-mq: introduce... |
696 |
kblockd_schedule_delayed_work(&q->requeue_work, 0); |
6fca6a611 blk-mq: add helpe... |
697 698 |
} EXPORT_SYMBOL(blk_mq_kick_requeue_list); |
2849450ad blk-mq: introduce... |
699 700 701 |
void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs) { |
d4acf3650 block: Make blk_m... |
702 703 |
kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, msecs_to_jiffies(msecs)); |
2849450ad blk-mq: introduce... |
704 705 |
} EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list); |
0e62f51f8 blk-mq: let blk_m... |
706 707 |
struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag) { |
88c7b2b75 blk-mq: prefetch ... |
708 709 |
if (tag < tags->nr_tags) { prefetch(tags->rqs[tag]); |
4ee86babe blk-mq: add bound... |
710 |
return tags->rqs[tag]; |
88c7b2b75 blk-mq: prefetch ... |
711 |
} |
4ee86babe blk-mq: add bound... |
712 713 |
return NULL; |
24d2f9030 blk-mq: split out... |
714 715 |
} EXPORT_SYMBOL(blk_mq_tag_to_rq); |
320ae51fe blk-mq: new multi... |
716 |
struct blk_mq_timeout_data { |
46f92d42e blk-mq: unshared ... |
717 718 |
unsigned long next; unsigned int next_set; |
320ae51fe blk-mq: new multi... |
719 |
}; |
904158376 block: fix blk_ab... |
720 |
void blk_mq_rq_timed_out(struct request *req, bool reserved) |
320ae51fe blk-mq: new multi... |
721 |
{ |
f8a5b1224 blk-mq: make mq_o... |
722 |
const struct blk_mq_ops *ops = req->q->mq_ops; |
46f92d42e blk-mq: unshared ... |
723 |
enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER; |
87ee7b112 blk-mq: fix race ... |
724 725 726 727 728 729 730 |
/* * We know that complete is set at this point. If STARTED isn't set * anymore, then the request isn't active and the "timeout" should * just be ignored. This can happen due to the bitflag ordering. * Timeout first checks if STARTED is set, and if it is, assumes * the request is active. But if we race with completion, then |
48b99c9d6 blk-mq: fix a typ... |
731 |
* both flags will get cleared. So check here again, and ignore |
87ee7b112 blk-mq: fix race ... |
732 733 |
* a timeout event with a request that isn't active. */ |
46f92d42e blk-mq: unshared ... |
734 735 |
if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags)) return; |
87ee7b112 blk-mq: fix race ... |
736 |
|
46f92d42e blk-mq: unshared ... |
737 |
if (ops->timeout) |
0152fb6b5 blk-mq: pass a re... |
738 |
ret = ops->timeout(req, reserved); |
46f92d42e blk-mq: unshared ... |
739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 |
switch (ret) { case BLK_EH_HANDLED: __blk_mq_complete_request(req); break; case BLK_EH_RESET_TIMER: blk_add_timer(req); blk_clear_rq_complete(req); break; case BLK_EH_NOT_HANDLED: break; default: printk(KERN_ERR "block: bad eh return: %d ", ret); break; } |
87ee7b112 blk-mq: fix race ... |
755 |
} |
5b3f25fc3 blk-mq: Allow req... |
756 |
|
81481eb42 blk-mq: fix and s... |
757 758 759 760 |
static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, struct request *rq, void *priv, bool reserved) { struct blk_mq_timeout_data *data = priv; |
87ee7b112 blk-mq: fix race ... |
761 |
|
95a496037 blk-mq: don't com... |
762 |
if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) |
46f92d42e blk-mq: unshared ... |
763 |
return; |
87ee7b112 blk-mq: fix race ... |
764 |
|
d9d149a39 blk-mq: comment o... |
765 766 767 768 769 770 771 772 773 774 775 776 777 |
/* * The rq being checked may have been freed and reallocated * out already here, we avoid this race by checking rq->deadline * and REQ_ATOM_COMPLETE flag together: * * - if rq->deadline is observed as new value because of * reusing, the rq won't be timed out because of timing. * - if rq->deadline is observed as previous value, * REQ_ATOM_COMPLETE flag won't be cleared in reuse path * because we put a barrier between setting rq->deadline * and clearing the flag in blk_mq_start_request(), so * this rq won't be timed out too. */ |
46f92d42e blk-mq: unshared ... |
778 779 |
if (time_after_eq(jiffies, rq->deadline)) { if (!blk_mark_rq_complete(rq)) |
0152fb6b5 blk-mq: pass a re... |
780 |
blk_mq_rq_timed_out(rq, reserved); |
46f92d42e blk-mq: unshared ... |
781 782 783 784 |
} else if (!data->next_set || time_after(data->next, rq->deadline)) { data->next = rq->deadline; data->next_set = 1; } |
87ee7b112 blk-mq: fix race ... |
785 |
} |
287922eb0 block: defer time... |
786 |
static void blk_mq_timeout_work(struct work_struct *work) |
320ae51fe blk-mq: new multi... |
787 |
{ |
287922eb0 block: defer time... |
788 789 |
struct request_queue *q = container_of(work, struct request_queue, timeout_work); |
81481eb42 blk-mq: fix and s... |
790 791 792 793 |
struct blk_mq_timeout_data data = { .next = 0, .next_set = 0, }; |
81481eb42 blk-mq: fix and s... |
794 |
int i; |
320ae51fe blk-mq: new multi... |
795 |
|
71f79fb31 blk-mq: Allow tim... |
796 797 798 799 800 801 802 803 804 |
/* A deadlock might occur if a request is stuck requiring a * timeout at the same time a queue freeze is waiting * completion, since the timeout code would not be able to * acquire the queue reference here. * * That's why we don't use blk_queue_enter here; instead, we use * percpu_ref_tryget directly, because we need to be able to * obtain a reference even in the short window between the queue * starting to freeze, by dropping the first reference in |
1671d522c block: rename blk... |
805 |
* blk_freeze_queue_start, and the moment the last request is |
71f79fb31 blk-mq: Allow tim... |
806 807 808 809 |
* consumed, marked by the instant q_usage_counter reaches * zero. */ if (!percpu_ref_tryget(&q->q_usage_counter)) |
287922eb0 block: defer time... |
810 |
return; |
0bf6cd5b9 blk-mq: factor ou... |
811 |
blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data); |
320ae51fe blk-mq: new multi... |
812 |
|
81481eb42 blk-mq: fix and s... |
813 814 815 |
if (data.next_set) { data.next = blk_rq_timeout(round_jiffies_up(data.next)); mod_timer(&q->timeout, data.next); |
0d2602ca3 blk-mq: improve s... |
816 |
} else { |
0bf6cd5b9 blk-mq: factor ou... |
817 |
struct blk_mq_hw_ctx *hctx; |
f054b56c9 blk-mq: fix race ... |
818 819 820 821 822 |
queue_for_each_hw_ctx(q, hctx, i) { /* the hctx may be unmapped, so check it here */ if (blk_mq_hw_queue_mapped(hctx)) blk_mq_tag_idle(hctx); } |
0d2602ca3 blk-mq: improve s... |
823 |
} |
287922eb0 block: defer time... |
824 |
blk_queue_exit(q); |
320ae51fe blk-mq: new multi... |
825 |
} |
88459642c blk-mq: abstract ... |
826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 |
struct flush_busy_ctx_data { struct blk_mq_hw_ctx *hctx; struct list_head *list; }; static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data) { struct flush_busy_ctx_data *flush_data = data; struct blk_mq_hw_ctx *hctx = flush_data->hctx; struct blk_mq_ctx *ctx = hctx->ctxs[bitnr]; sbitmap_clear_bit(sb, bitnr); spin_lock(&ctx->lock); list_splice_tail_init(&ctx->rq_list, flush_data->list); spin_unlock(&ctx->lock); return true; } |
320ae51fe blk-mq: new multi... |
843 |
/* |
1429d7c94 blk-mq: switch ct... |
844 845 846 |
* Process software queues that have been marked busy, splicing them * to the for-dispatch */ |
2c3ad6679 blk-mq: export so... |
847 |
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list) |
1429d7c94 blk-mq: switch ct... |
848 |
{ |
88459642c blk-mq: abstract ... |
849 850 851 852 |
struct flush_busy_ctx_data data = { .hctx = hctx, .list = list, }; |
1429d7c94 blk-mq: switch ct... |
853 |
|
88459642c blk-mq: abstract ... |
854 |
sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data); |
1429d7c94 blk-mq: switch ct... |
855 |
} |
2c3ad6679 blk-mq: export so... |
856 |
EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs); |
1429d7c94 blk-mq: switch ct... |
857 |
|
703fd1c0f blk-mq: account h... |
858 859 860 861 |
static inline unsigned int queued_to_index(unsigned int queued) { if (!queued) return 0; |
1429d7c94 blk-mq: switch ct... |
862 |
|
703fd1c0f blk-mq: account h... |
863 |
return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); |
1429d7c94 blk-mq: switch ct... |
864 |
} |
bd6737f1a blk-mq-sched: add... |
865 866 |
bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx, bool wait) |
bd166ef18 blk-mq-sched: add... |
867 868 869 |
{ struct blk_mq_alloc_data data = { .q = rq->q, |
bd166ef18 blk-mq-sched: add... |
870 871 872 |
.hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu), .flags = wait ? 0 : BLK_MQ_REQ_NOWAIT, }; |
5feeacdd4 blk-mq: add might... |
873 |
might_sleep_if(wait); |
81380ca10 blk-mq: use the r... |
874 875 |
if (rq->tag != -1) goto done; |
bd166ef18 blk-mq-sched: add... |
876 |
|
415b806de blk-mq-sched: All... |
877 878 |
if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag)) data.flags |= BLK_MQ_REQ_RESERVED; |
bd166ef18 blk-mq-sched: add... |
879 880 |
rq->tag = blk_mq_get_tag(&data); if (rq->tag >= 0) { |
200e86b33 blk-mq: only appl... |
881 882 883 884 |
if (blk_mq_tag_busy(data.hctx)) { rq->rq_flags |= RQF_MQ_INFLIGHT; atomic_inc(&data.hctx->nr_active); } |
bd166ef18 blk-mq-sched: add... |
885 |
data.hctx->tags->rqs[rq->tag] = rq; |
bd166ef18 blk-mq-sched: add... |
886 |
} |
81380ca10 blk-mq: use the r... |
887 888 889 890 |
done: if (hctx) *hctx = data.hctx; return rq->tag != -1; |
bd166ef18 blk-mq-sched: add... |
891 |
} |
113285b47 blk-mq: ensure th... |
892 893 |
static void __blk_mq_put_driver_tag(struct blk_mq_hw_ctx *hctx, struct request *rq) |
99cf1dc58 blk-mq: release d... |
894 |
{ |
99cf1dc58 blk-mq: release d... |
895 896 897 898 899 900 901 902 |
blk_mq_put_tag(hctx, hctx->tags, rq->mq_ctx, rq->tag); rq->tag = -1; if (rq->rq_flags & RQF_MQ_INFLIGHT) { rq->rq_flags &= ~RQF_MQ_INFLIGHT; atomic_dec(&hctx->nr_active); } } |
113285b47 blk-mq: ensure th... |
903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 |
static void blk_mq_put_driver_tag_hctx(struct blk_mq_hw_ctx *hctx, struct request *rq) { if (rq->tag == -1 || rq->internal_tag == -1) return; __blk_mq_put_driver_tag(hctx, rq); } static void blk_mq_put_driver_tag(struct request *rq) { struct blk_mq_hw_ctx *hctx; if (rq->tag == -1 || rq->internal_tag == -1) return; hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu); __blk_mq_put_driver_tag(hctx, rq); } |
bd166ef18 blk-mq-sched: add... |
922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 |
/* * If we fail getting a driver tag because all the driver tags are already * assigned and on the dispatch list, BUT the first entry does not have a * tag, then we could deadlock. For that case, move entries with assigned * driver tags to the front, leaving the set of tagged requests in the * same order, and the untagged set in the same order. */ static bool reorder_tags_to_front(struct list_head *list) { struct request *rq, *tmp, *first = NULL; list_for_each_entry_safe_reverse(rq, tmp, list, queuelist) { if (rq == first) break; if (rq->tag != -1) { list_move(&rq->queuelist, list); if (!first) first = rq; } } return first != NULL; } |
ac6424b98 sched/wait: Renam... |
945 |
static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, int flags, |
da55f2cc7 blk-mq: use sbq w... |
946 947 948 949 950 |
void *key) { struct blk_mq_hw_ctx *hctx; hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait); |
2055da973 sched/wait: Disam... |
951 |
list_del(&wait->entry); |
da55f2cc7 blk-mq: use sbq w... |
952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 |
clear_bit_unlock(BLK_MQ_S_TAG_WAITING, &hctx->state); blk_mq_run_hw_queue(hctx, true); return 1; } static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx *hctx) { struct sbq_wait_state *ws; /* * The TAG_WAITING bit serves as a lock protecting hctx->dispatch_wait. * The thread which wins the race to grab this bit adds the hardware * queue to the wait queue. */ if (test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state) || test_and_set_bit_lock(BLK_MQ_S_TAG_WAITING, &hctx->state)) return false; init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake); ws = bt_wait_ptr(&hctx->tags->bitmap_tags, hctx); /* * As soon as this returns, it's no longer safe to fiddle with * hctx->dispatch_wait, since a completion can wake up the wait queue * and unlock the bit. */ add_wait_queue(&ws->wait, &hctx->dispatch_wait); return true; } |
81380ca10 blk-mq: use the r... |
981 |
bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list) |
320ae51fe blk-mq: new multi... |
982 |
{ |
81380ca10 blk-mq: use the r... |
983 |
struct blk_mq_hw_ctx *hctx; |
320ae51fe blk-mq: new multi... |
984 |
struct request *rq; |
fc17b6534 blk-mq: switch ->... |
985 |
int errors, queued; |
320ae51fe blk-mq: new multi... |
986 |
|
81380ca10 blk-mq: use the r... |
987 988 |
if (list_empty(list)) return false; |
320ae51fe blk-mq: new multi... |
989 |
/* |
320ae51fe blk-mq: new multi... |
990 991 |
* Now process all the entries, sending them to the driver. */ |
93efe9817 blk-mq: include e... |
992 |
errors = queued = 0; |
81380ca10 blk-mq: use the r... |
993 |
do { |
74c450521 blk-mq: add a 'li... |
994 |
struct blk_mq_queue_data bd; |
fc17b6534 blk-mq: switch ->... |
995 |
blk_status_t ret; |
320ae51fe blk-mq: new multi... |
996 |
|
f04c3df3e blk-mq: abstract ... |
997 |
rq = list_first_entry(list, struct request, queuelist); |
bd166ef18 blk-mq-sched: add... |
998 999 1000 |
if (!blk_mq_get_driver_tag(rq, &hctx, false)) { if (!queued && reorder_tags_to_front(list)) continue; |
3c782d67c blk-mq: fix poten... |
1001 1002 |
/* |
da55f2cc7 blk-mq: use sbq w... |
1003 1004 |
* The initial allocation attempt failed, so we need to * rerun the hardware queue when a tag is freed. |
3c782d67c blk-mq: fix poten... |
1005 |
*/ |
807b10417 blk-mq: make driv... |
1006 1007 1008 1009 1010 1011 1012 1013 1014 |
if (!blk_mq_dispatch_wait_add(hctx)) break; /* * It's possible that a tag was freed in the window * between the allocation failure and adding the * hardware queue to the wait queue. */ if (!blk_mq_get_driver_tag(rq, &hctx, false)) |
3c782d67c blk-mq: fix poten... |
1015 |
break; |
bd166ef18 blk-mq-sched: add... |
1016 |
} |
da55f2cc7 blk-mq: use sbq w... |
1017 |
|
320ae51fe blk-mq: new multi... |
1018 |
list_del_init(&rq->queuelist); |
320ae51fe blk-mq: new multi... |
1019 |
|
74c450521 blk-mq: add a 'li... |
1020 |
bd.rq = rq; |
113285b47 blk-mq: ensure th... |
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 |
/* * Flag last if we have no more requests, or if we have more * but can't assign a driver tag to it. */ if (list_empty(list)) bd.last = true; else { struct request *nxt; nxt = list_first_entry(list, struct request, queuelist); bd.last = !blk_mq_get_driver_tag(nxt, NULL, false); } |
74c450521 blk-mq: add a 'li... |
1034 1035 |
ret = q->mq_ops->queue_rq(hctx, &bd); |
fc17b6534 blk-mq: switch ->... |
1036 |
if (ret == BLK_STS_RESOURCE) { |
113285b47 blk-mq: ensure th... |
1037 |
blk_mq_put_driver_tag_hctx(hctx, rq); |
f04c3df3e blk-mq: abstract ... |
1038 |
list_add(&rq->queuelist, list); |
ed0791b2f blk-mq: add blk_m... |
1039 |
__blk_mq_requeue_request(rq); |
320ae51fe blk-mq: new multi... |
1040 |
break; |
fc17b6534 blk-mq: switch ->... |
1041 1042 1043 |
} if (unlikely(ret != BLK_STS_OK)) { |
93efe9817 blk-mq: include e... |
1044 |
errors++; |
2a842acab block: introduce ... |
1045 |
blk_mq_end_request(rq, BLK_STS_IOERR); |
fc17b6534 blk-mq: switch ->... |
1046 |
continue; |
320ae51fe blk-mq: new multi... |
1047 |
} |
fc17b6534 blk-mq: switch ->... |
1048 |
queued++; |
81380ca10 blk-mq: use the r... |
1049 |
} while (!list_empty(list)); |
320ae51fe blk-mq: new multi... |
1050 |
|
703fd1c0f blk-mq: account h... |
1051 |
hctx->dispatched[queued_to_index(queued)]++; |
320ae51fe blk-mq: new multi... |
1052 1053 1054 1055 1056 |
/* * Any items that need requeuing? Stuff them into hctx->dispatch, * that is where we will continue on next queue run. */ |
f04c3df3e blk-mq: abstract ... |
1057 |
if (!list_empty(list)) { |
113285b47 blk-mq: ensure th... |
1058 |
/* |
710c785f8 blk-mq: Clarify c... |
1059 1060 |
* If an I/O scheduler has been configured and we got a driver * tag for the next request already, free it again. |
113285b47 blk-mq: ensure th... |
1061 1062 1063 |
*/ rq = list_first_entry(list, struct request, queuelist); blk_mq_put_driver_tag(rq); |
320ae51fe blk-mq: new multi... |
1064 |
spin_lock(&hctx->lock); |
c13660a08 blk-mq-sched: cha... |
1065 |
list_splice_init(list, &hctx->dispatch); |
320ae51fe blk-mq: new multi... |
1066 |
spin_unlock(&hctx->lock); |
f04c3df3e blk-mq: abstract ... |
1067 |
|
9ba52e581 blk-mq: don't los... |
1068 |
/* |
710c785f8 blk-mq: Clarify c... |
1069 1070 1071 |
* If SCHED_RESTART was set by the caller of this function and * it is no longer set that means that it was cleared by another * thread and hence that a queue rerun is needed. |
9ba52e581 blk-mq: don't los... |
1072 |
* |
710c785f8 blk-mq: Clarify c... |
1073 1074 1075 1076 |
* If TAG_WAITING is set that means that an I/O scheduler has * been configured and another thread is waiting for a driver * tag. To guarantee fairness, do not rerun this hardware queue * but let the other thread grab the driver tag. |
bd166ef18 blk-mq-sched: add... |
1077 |
* |
710c785f8 blk-mq: Clarify c... |
1078 1079 1080 1081 1082 1083 1084 |
* If no I/O scheduler has been configured it is possible that * the hardware queue got stopped and restarted before requests * were pushed back onto the dispatch list. Rerun the queue to * avoid starvation. Notes: * - blk_mq_run_hw_queue() checks whether or not a queue has * been stopped before rerunning a queue. * - Some but not all block drivers stop a queue before |
fc17b6534 blk-mq: switch ->... |
1085 |
* returning BLK_STS_RESOURCE. Two exceptions are scsi-mq |
710c785f8 blk-mq: Clarify c... |
1086 |
* and dm-rq. |
bd166ef18 blk-mq-sched: add... |
1087 |
*/ |
da55f2cc7 blk-mq: use sbq w... |
1088 1089 |
if (!blk_mq_sched_needs_restart(hctx) && !test_bit(BLK_MQ_S_TAG_WAITING, &hctx->state)) |
bd166ef18 blk-mq-sched: add... |
1090 |
blk_mq_run_hw_queue(hctx, true); |
320ae51fe blk-mq: new multi... |
1091 |
} |
f04c3df3e blk-mq: abstract ... |
1092 |
|
93efe9817 blk-mq: include e... |
1093 |
return (queued + errors) != 0; |
f04c3df3e blk-mq: abstract ... |
1094 |
} |
6a83e74d2 blk-mq: Introduce... |
1095 1096 1097 |
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx) { int srcu_idx; |
b7a71e66d blk-mq: add warni... |
1098 1099 1100 |
/* * We should be running this queue from one of the CPUs that * are mapped to it. |
a1dfcb01e blk-mq: turn WARN... |
1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 |
* * There are at least two related races now between setting * hctx->next_cpu from blk_mq_hctx_next_cpu() and running * __blk_mq_run_hw_queue(): * * - hctx->next_cpu is found offline in blk_mq_hctx_next_cpu(), * but later it becomes online, then this warning is harmless * at all * * - hctx->next_cpu is found online in blk_mq_hctx_next_cpu(), * but later it becomes offline, then the warning can't be * triggered, and we depend on blk-mq timeout handler to * handle dispatched requests to this hctx |
b7a71e66d blk-mq: add warni... |
1114 |
*/ |
a1dfcb01e blk-mq: turn WARN... |
1115 1116 1117 1118 1119 1120 1121 1122 |
if (!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) && cpu_online(hctx->next_cpu)) { printk(KERN_WARNING "run queue from wrong CPU %d, hctx %s ", raw_smp_processor_id(), cpumask_empty(hctx->cpumask) ? "inactive": "active"); dump_stack(); } |
6a83e74d2 blk-mq: Introduce... |
1123 |
|
b7a71e66d blk-mq: add warni... |
1124 1125 1126 1127 1128 |
/* * We can't run the queue inline with ints disabled. Ensure that * we catch bad users of this early. */ WARN_ON_ONCE(in_interrupt()); |
6a83e74d2 blk-mq: Introduce... |
1129 1130 |
if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { rcu_read_lock(); |
bd166ef18 blk-mq-sched: add... |
1131 |
blk_mq_sched_dispatch_requests(hctx); |
6a83e74d2 blk-mq: Introduce... |
1132 1133 |
rcu_read_unlock(); } else { |
bf4907c05 blk-mq: fix sched... |
1134 |
might_sleep(); |
073196787 blk-mq: Reduce bl... |
1135 |
srcu_idx = srcu_read_lock(hctx->queue_rq_srcu); |
bd166ef18 blk-mq-sched: add... |
1136 |
blk_mq_sched_dispatch_requests(hctx); |
073196787 blk-mq: Reduce bl... |
1137 |
srcu_read_unlock(hctx->queue_rq_srcu, srcu_idx); |
6a83e74d2 blk-mq: Introduce... |
1138 1139 |
} } |
506e931f9 blk-mq: add basic... |
1140 1141 1142 1143 1144 1145 1146 1147 |
/* * It'd be great if the workqueue API had a way to pass * in a mask and had some smarts for more clever placement. * For now we just round-robin here, switching for every * BLK_MQ_CPU_WORK_BATCH queued items. */ static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx) { |
b657d7e63 blk-mq: handle th... |
1148 1149 |
if (hctx->queue->nr_hw_queues == 1) return WORK_CPU_UNBOUND; |
506e931f9 blk-mq: add basic... |
1150 1151 |
if (--hctx->next_cpu_batch <= 0) { |
c02ebfddd blk-mq: Always sc... |
1152 |
int next_cpu; |
506e931f9 blk-mq: add basic... |
1153 1154 1155 1156 1157 1158 1159 1160 |
next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask); if (next_cpu >= nr_cpu_ids) next_cpu = cpumask_first(hctx->cpumask); hctx->next_cpu = next_cpu; hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; } |
b657d7e63 blk-mq: handle th... |
1161 |
return hctx->next_cpu; |
506e931f9 blk-mq: add basic... |
1162 |
} |
7587a5ae7 blk-mq: Introduce... |
1163 1164 |
static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async, unsigned long msecs) |
320ae51fe blk-mq: new multi... |
1165 |
{ |
5435c023b blk-mq: Warn when... |
1166 1167 1168 1169 |
if (WARN_ON_ONCE(!blk_mq_hw_queue_mapped(hctx))) return; if (unlikely(blk_mq_hctx_stopped(hctx))) |
320ae51fe blk-mq: new multi... |
1170 |
return; |
1b792f2f9 blk-mq: add flag ... |
1171 |
if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) { |
2a90d4aae blk-mq: use get_c... |
1172 1173 |
int cpu = get_cpu(); if (cpumask_test_cpu(cpu, hctx->cpumask)) { |
398205b83 blk_mq: call pree... |
1174 |
__blk_mq_run_hw_queue(hctx); |
2a90d4aae blk-mq: use get_c... |
1175 |
put_cpu(); |
398205b83 blk_mq: call pree... |
1176 1177 |
return; } |
e4043dcf3 blk-mq: ensure th... |
1178 |
|
2a90d4aae blk-mq: use get_c... |
1179 |
put_cpu(); |
e4043dcf3 blk-mq: ensure th... |
1180 |
} |
398205b83 blk_mq: call pree... |
1181 |
|
9f9937379 blk-mq: unify hct... |
1182 1183 1184 |
kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, msecs_to_jiffies(msecs)); |
7587a5ae7 blk-mq: Introduce... |
1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 |
} void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) { __blk_mq_delay_run_hw_queue(hctx, true, msecs); } EXPORT_SYMBOL(blk_mq_delay_run_hw_queue); void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) { __blk_mq_delay_run_hw_queue(hctx, async, 0); |
320ae51fe blk-mq: new multi... |
1196 |
} |
5b7272729 blk-mq: export he... |
1197 |
EXPORT_SYMBOL(blk_mq_run_hw_queue); |
320ae51fe blk-mq: new multi... |
1198 |
|
b94ec2964 blk-mq: export bl... |
1199 |
void blk_mq_run_hw_queues(struct request_queue *q, bool async) |
320ae51fe blk-mq: new multi... |
1200 1201 1202 1203 1204 |
{ struct blk_mq_hw_ctx *hctx; int i; queue_for_each_hw_ctx(q, hctx, i) { |
bd166ef18 blk-mq-sched: add... |
1205 |
if (!blk_mq_hctx_has_pending(hctx) || |
5d1b25c1e blk-mq: Introduce... |
1206 |
blk_mq_hctx_stopped(hctx)) |
320ae51fe blk-mq: new multi... |
1207 |
continue; |
b94ec2964 blk-mq: export bl... |
1208 |
blk_mq_run_hw_queue(hctx, async); |
320ae51fe blk-mq: new multi... |
1209 1210 |
} } |
b94ec2964 blk-mq: export bl... |
1211 |
EXPORT_SYMBOL(blk_mq_run_hw_queues); |
320ae51fe blk-mq: new multi... |
1212 |
|
fd0014430 blk-mq: Introduce... |
1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 |
/** * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped * @q: request queue. * * The caller is responsible for serializing this function against * blk_mq_{start,stop}_hw_queue(). */ bool blk_mq_queue_stopped(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; int i; queue_for_each_hw_ctx(q, hctx, i) if (blk_mq_hctx_stopped(hctx)) return true; return false; } EXPORT_SYMBOL(blk_mq_queue_stopped); |
39a70c76b blk-mq: clarify d... |
1232 1233 1234 |
/* * This function is often used for pausing .queue_rq() by driver when * there isn't enough resource or some conditions aren't satisfied, and |
4d6062193 block: Fix two co... |
1235 |
* BLK_STS_RESOURCE is usually returned. |
39a70c76b blk-mq: clarify d... |
1236 1237 1238 1239 1240 |
* * We do not guarantee that dispatch can be drained or blocked * after blk_mq_stop_hw_queue() returns. Please use * blk_mq_quiesce_queue() for that requirement. */ |
2719aa217 blk-mq: don't use... |
1241 1242 |
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx) { |
641a9ed60 Revert "blk-mq: d... |
1243 |
cancel_delayed_work(&hctx->run_work); |
280d45f6c blk-mq: add blk_m... |
1244 |
|
641a9ed60 Revert "blk-mq: d... |
1245 |
set_bit(BLK_MQ_S_STOPPED, &hctx->state); |
2719aa217 blk-mq: don't use... |
1246 |
} |
641a9ed60 Revert "blk-mq: d... |
1247 |
EXPORT_SYMBOL(blk_mq_stop_hw_queue); |
2719aa217 blk-mq: don't use... |
1248 |
|
39a70c76b blk-mq: clarify d... |
1249 1250 1251 |
/* * This function is often used for pausing .queue_rq() by driver when * there isn't enough resource or some conditions aren't satisfied, and |
4d6062193 block: Fix two co... |
1252 |
* BLK_STS_RESOURCE is usually returned. |
39a70c76b blk-mq: clarify d... |
1253 1254 1255 1256 1257 |
* * We do not guarantee that dispatch can be drained or blocked * after blk_mq_stop_hw_queues() returns. Please use * blk_mq_quiesce_queue() for that requirement. */ |
2719aa217 blk-mq: don't use... |
1258 1259 |
void blk_mq_stop_hw_queues(struct request_queue *q) { |
641a9ed60 Revert "blk-mq: d... |
1260 1261 1262 1263 1264 |
struct blk_mq_hw_ctx *hctx; int i; queue_for_each_hw_ctx(q, hctx, i) blk_mq_stop_hw_queue(hctx); |
280d45f6c blk-mq: add blk_m... |
1265 1266 |
} EXPORT_SYMBOL(blk_mq_stop_hw_queues); |
320ae51fe blk-mq: new multi... |
1267 1268 1269 |
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx) { clear_bit(BLK_MQ_S_STOPPED, &hctx->state); |
e4043dcf3 blk-mq: ensure th... |
1270 |
|
0ffbce80c blk-mq: blk_mq_st... |
1271 |
blk_mq_run_hw_queue(hctx, false); |
320ae51fe blk-mq: new multi... |
1272 1273 |
} EXPORT_SYMBOL(blk_mq_start_hw_queue); |
2f2685565 blk-mq: add blk_m... |
1274 1275 1276 1277 1278 1279 1280 1281 1282 |
void blk_mq_start_hw_queues(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; int i; queue_for_each_hw_ctx(q, hctx, i) blk_mq_start_hw_queue(hctx); } EXPORT_SYMBOL(blk_mq_start_hw_queues); |
ae911c5e7 blk-mq: add blk_m... |
1283 1284 1285 1286 1287 1288 1289 1290 1291 |
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) { if (!blk_mq_hctx_stopped(hctx)) return; clear_bit(BLK_MQ_S_STOPPED, &hctx->state); blk_mq_run_hw_queue(hctx, async); } EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue); |
1b4a32585 blk-mq: add async... |
1292 |
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async) |
320ae51fe blk-mq: new multi... |
1293 1294 1295 |
{ struct blk_mq_hw_ctx *hctx; int i; |
ae911c5e7 blk-mq: add blk_m... |
1296 1297 |
queue_for_each_hw_ctx(q, hctx, i) blk_mq_start_stopped_hw_queue(hctx, async); |
320ae51fe blk-mq: new multi... |
1298 1299 |
} EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues); |
70f4db639 blk-mq: add blk_m... |
1300 |
static void blk_mq_run_work_fn(struct work_struct *work) |
320ae51fe blk-mq: new multi... |
1301 1302 |
{ struct blk_mq_hw_ctx *hctx; |
9f9937379 blk-mq: unify hct... |
1303 |
hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work); |
320ae51fe blk-mq: new multi... |
1304 |
|
21c6e939a blk-mq: unify hct... |
1305 1306 1307 1308 1309 1310 1311 1312 |
/* * If we are stopped, don't run the queue. The exception is if * BLK_MQ_S_START_ON_RUN is set. For that case, we auto-clear * the STOPPED bit and run it. */ if (test_bit(BLK_MQ_S_STOPPED, &hctx->state)) { if (!test_bit(BLK_MQ_S_START_ON_RUN, &hctx->state)) return; |
7587a5ae7 blk-mq: Introduce... |
1313 |
|
21c6e939a blk-mq: unify hct... |
1314 1315 1316 |
clear_bit(BLK_MQ_S_START_ON_RUN, &hctx->state); clear_bit(BLK_MQ_S_STOPPED, &hctx->state); } |
7587a5ae7 blk-mq: Introduce... |
1317 1318 1319 |
__blk_mq_run_hw_queue(hctx); } |
70f4db639 blk-mq: add blk_m... |
1320 1321 1322 |
void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs) { |
5435c023b blk-mq: Warn when... |
1323 |
if (WARN_ON_ONCE(!blk_mq_hw_queue_mapped(hctx))) |
19c66e59c blk-mq: prevent u... |
1324 |
return; |
70f4db639 blk-mq: add blk_m... |
1325 |
|
21c6e939a blk-mq: unify hct... |
1326 1327 1328 1329 1330 |
/* * Stop the hw queue, then modify currently delayed work. * This should prevent us from running the queue prematurely. * Mark the queue as auto-clearing STOPPED when it runs. */ |
7e79dadce blk-mq: stop hard... |
1331 |
blk_mq_stop_hw_queue(hctx); |
21c6e939a blk-mq: unify hct... |
1332 1333 1334 1335 |
set_bit(BLK_MQ_S_START_ON_RUN, &hctx->state); kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work, msecs_to_jiffies(msecs)); |
70f4db639 blk-mq: add blk_m... |
1336 1337 |
} EXPORT_SYMBOL(blk_mq_delay_queue); |
cfd0c552a blk-mq: mark ctx ... |
1338 |
static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx, |
cfd0c552a blk-mq: mark ctx ... |
1339 1340 |
struct request *rq, bool at_head) |
320ae51fe blk-mq: new multi... |
1341 |
{ |
e57690fe0 blk-mq: don't ove... |
1342 |
struct blk_mq_ctx *ctx = rq->mq_ctx; |
7b6078146 blk-mq: Document ... |
1343 |
lockdep_assert_held(&ctx->lock); |
01b983c9f blk-mq: add blktr... |
1344 |
trace_block_rq_insert(hctx->queue, rq); |
72a0a36e2 blk-mq: support a... |
1345 1346 1347 1348 |
if (at_head) list_add(&rq->queuelist, &ctx->rq_list); else list_add_tail(&rq->queuelist, &ctx->rq_list); |
cfd0c552a blk-mq: mark ctx ... |
1349 |
} |
4bb659b15 blk-mq: implement... |
1350 |
|
2c3ad6679 blk-mq: export so... |
1351 1352 |
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, bool at_head) |
cfd0c552a blk-mq: mark ctx ... |
1353 1354 |
{ struct blk_mq_ctx *ctx = rq->mq_ctx; |
7b6078146 blk-mq: Document ... |
1355 |
lockdep_assert_held(&ctx->lock); |
e57690fe0 blk-mq: don't ove... |
1356 |
__blk_mq_insert_req_list(hctx, rq, at_head); |
320ae51fe blk-mq: new multi... |
1357 |
blk_mq_hctx_mark_pending(hctx, ctx); |
320ae51fe blk-mq: new multi... |
1358 |
} |
157f377be block: directly i... |
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 |
/* * Should only be used carefully, when the caller knows we want to * bypass a potential IO scheduler on the target device. */ void blk_mq_request_bypass_insert(struct request *rq) { struct blk_mq_ctx *ctx = rq->mq_ctx; struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(rq->q, ctx->cpu); spin_lock(&hctx->lock); list_add_tail(&rq->queuelist, &hctx->dispatch); spin_unlock(&hctx->lock); blk_mq_run_hw_queue(hctx, false); } |
bd166ef18 blk-mq-sched: add... |
1374 1375 |
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, struct list_head *list) |
320ae51fe blk-mq: new multi... |
1376 1377 |
{ |
320ae51fe blk-mq: new multi... |
1378 1379 1380 1381 1382 1383 1384 1385 1386 |
/* * preemption doesn't flush plug list, so it's possible ctx->cpu is * offline now */ spin_lock(&ctx->lock); while (!list_empty(list)) { struct request *rq; rq = list_first_entry(list, struct request, queuelist); |
e57690fe0 blk-mq: don't ove... |
1387 |
BUG_ON(rq->mq_ctx != ctx); |
320ae51fe blk-mq: new multi... |
1388 |
list_del_init(&rq->queuelist); |
e57690fe0 blk-mq: don't ove... |
1389 |
__blk_mq_insert_req_list(hctx, rq, false); |
320ae51fe blk-mq: new multi... |
1390 |
} |
cfd0c552a blk-mq: mark ctx ... |
1391 |
blk_mq_hctx_mark_pending(hctx, ctx); |
320ae51fe blk-mq: new multi... |
1392 |
spin_unlock(&ctx->lock); |
320ae51fe blk-mq: new multi... |
1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 |
} static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b) { struct request *rqa = container_of(a, struct request, queuelist); struct request *rqb = container_of(b, struct request, queuelist); return !(rqa->mq_ctx < rqb->mq_ctx || (rqa->mq_ctx == rqb->mq_ctx && blk_rq_pos(rqa) < blk_rq_pos(rqb))); } void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule) { struct blk_mq_ctx *this_ctx; struct request_queue *this_q; struct request *rq; LIST_HEAD(list); LIST_HEAD(ctx_list); unsigned int depth; list_splice_init(&plug->mq_list, &list); list_sort(NULL, &list, plug_ctx_cmp); this_q = NULL; this_ctx = NULL; depth = 0; while (!list_empty(&list)) { rq = list_entry_rq(list.next); list_del_init(&rq->queuelist); BUG_ON(!rq->q); if (rq->mq_ctx != this_ctx) { if (this_ctx) { |
8e2e2192e blk-mq: I/O and t... |
1428 |
trace_block_unplug(this_q, depth, !from_schedule); |
bd166ef18 blk-mq-sched: add... |
1429 1430 1431 |
blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list, from_schedule); |
320ae51fe blk-mq: new multi... |
1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 |
} this_ctx = rq->mq_ctx; this_q = rq->q; depth = 0; } depth++; list_add_tail(&rq->queuelist, &ctx_list); } /* * If 'this_ctx' is set, we know we have entries to complete * on 'ctx_list'. Do those. */ if (this_ctx) { |
8e2e2192e blk-mq: I/O and t... |
1448 |
trace_block_unplug(this_q, depth, !from_schedule); |
bd166ef18 blk-mq-sched: add... |
1449 1450 |
blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list, from_schedule); |
320ae51fe blk-mq: new multi... |
1451 1452 1453 1454 1455 |
} } static void blk_mq_bio_to_request(struct request *rq, struct bio *bio) { |
da8d7f079 block: Export blk... |
1456 |
blk_init_request_from_bio(rq, bio); |
4b570521b blk-mq: request i... |
1457 |
|
6e85eaf30 blk-mq: blk_accou... |
1458 |
blk_account_io_start(rq, true); |
320ae51fe blk-mq: new multi... |
1459 |
} |
274a5843f blk-mq: don't all... |
1460 1461 1462 1463 1464 |
static inline bool hctx_allow_merges(struct blk_mq_hw_ctx *hctx) { return (hctx->flags & BLK_MQ_F_SHOULD_MERGE) && !blk_queue_nomerges(hctx->queue); } |
ab42f35d9 blk-mq: merge bio... |
1465 1466 1467 1468 1469 1470 1471 |
static inline void blk_mq_queue_io(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, struct request *rq) { spin_lock(&ctx->lock); __blk_mq_insert_request(hctx, rq, false); spin_unlock(&ctx->lock); |
07068d5b8 blk-mq: split mak... |
1472 |
} |
14ec77f35 blk-mq: Add bio_i... |
1473 |
|
fd2d33267 blk-mq: add suppo... |
1474 1475 |
static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq) { |
bd166ef18 blk-mq-sched: add... |
1476 1477 1478 1479 |
if (rq->tag != -1) return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false); return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true); |
fd2d33267 blk-mq: add suppo... |
1480 |
} |
d964f04a8 blk-mq: fix direc... |
1481 1482 1483 |
static void __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, struct request *rq, blk_qc_t *cookie, bool may_sleep) |
f984df1f0 blk-mq: do limite... |
1484 |
{ |
f984df1f0 blk-mq: do limite... |
1485 |
struct request_queue *q = rq->q; |
f984df1f0 blk-mq: do limite... |
1486 1487 |
struct blk_mq_queue_data bd = { .rq = rq, |
d945a365a blk-mq: use true ... |
1488 |
.last = true, |
f984df1f0 blk-mq: do limite... |
1489 |
}; |
bd166ef18 blk-mq-sched: add... |
1490 |
blk_qc_t new_cookie; |
f06345add blk-mq: fixup typ... |
1491 |
blk_status_t ret; |
d964f04a8 blk-mq: fix direc... |
1492 |
bool run_queue = true; |
f4560ffe8 blk-mq: use QUEUE... |
1493 1494 |
/* RCU or SRCU read lock is needed before checking quiesced flag */ if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) { |
d964f04a8 blk-mq: fix direc... |
1495 1496 1497 |
run_queue = false; goto insert; } |
f984df1f0 blk-mq: do limite... |
1498 |
|
bd166ef18 blk-mq-sched: add... |
1499 |
if (q->elevator) |
2253efc85 blk-mq: Move more... |
1500 |
goto insert; |
d964f04a8 blk-mq: fix direc... |
1501 |
if (!blk_mq_get_driver_tag(rq, NULL, false)) |
bd166ef18 blk-mq-sched: add... |
1502 1503 1504 |
goto insert; new_cookie = request_to_qc_t(hctx, rq); |
f984df1f0 blk-mq: do limite... |
1505 1506 1507 1508 1509 1510 |
/* * For OK queue, we are done. For error, kill it. Any other * error (busy), just add it to our list as we previously * would have done */ ret = q->mq_ops->queue_rq(hctx, &bd); |
fc17b6534 blk-mq: switch ->... |
1511 1512 |
switch (ret) { case BLK_STS_OK: |
7b371636f blk-mq: return ta... |
1513 |
*cookie = new_cookie; |
2253efc85 blk-mq: Move more... |
1514 |
return; |
fc17b6534 blk-mq: switch ->... |
1515 1516 1517 1518 |
case BLK_STS_RESOURCE: __blk_mq_requeue_request(rq); goto insert; default: |
7b371636f blk-mq: return ta... |
1519 |
*cookie = BLK_QC_T_NONE; |
fc17b6534 blk-mq: switch ->... |
1520 |
blk_mq_end_request(rq, ret); |
2253efc85 blk-mq: Move more... |
1521 |
return; |
f984df1f0 blk-mq: do limite... |
1522 |
} |
7b371636f blk-mq: return ta... |
1523 |
|
2253efc85 blk-mq: Move more... |
1524 |
insert: |
d964f04a8 blk-mq: fix direc... |
1525 |
blk_mq_sched_insert_request(rq, false, run_queue, false, may_sleep); |
f984df1f0 blk-mq: do limite... |
1526 |
} |
5eb6126e1 blk-mq: improve b... |
1527 1528 1529 1530 1531 |
static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx, struct request *rq, blk_qc_t *cookie) { if (!(hctx->flags & BLK_MQ_F_BLOCKING)) { rcu_read_lock(); |
d964f04a8 blk-mq: fix direc... |
1532 |
__blk_mq_try_issue_directly(hctx, rq, cookie, false); |
5eb6126e1 blk-mq: improve b... |
1533 1534 |
rcu_read_unlock(); } else { |
bf4907c05 blk-mq: fix sched... |
1535 1536 1537 |
unsigned int srcu_idx; might_sleep(); |
073196787 blk-mq: Reduce bl... |
1538 |
srcu_idx = srcu_read_lock(hctx->queue_rq_srcu); |
d964f04a8 blk-mq: fix direc... |
1539 |
__blk_mq_try_issue_directly(hctx, rq, cookie, true); |
073196787 blk-mq: Reduce bl... |
1540 |
srcu_read_unlock(hctx->queue_rq_srcu, srcu_idx); |
5eb6126e1 blk-mq: improve b... |
1541 1542 |
} } |
dece16353 block: change ->m... |
1543 |
static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio) |
07068d5b8 blk-mq: split mak... |
1544 |
{ |
ef295ecf0 block: better op ... |
1545 |
const int is_sync = op_is_sync(bio->bi_opf); |
f73f44eb0 block: add a op_i... |
1546 |
const int is_flush_fua = op_is_flush(bio->bi_opf); |
5a797e00d blk-mq: don't los... |
1547 |
struct blk_mq_alloc_data data = { .flags = 0 }; |
07068d5b8 blk-mq: split mak... |
1548 |
struct request *rq; |
5eb6126e1 blk-mq: improve b... |
1549 |
unsigned int request_count = 0; |
f984df1f0 blk-mq: do limite... |
1550 |
struct blk_plug *plug; |
5b3f341f0 blk-mq: make plug... |
1551 |
struct request *same_queue_rq = NULL; |
7b371636f blk-mq: return ta... |
1552 |
blk_qc_t cookie; |
87760e5ee block: hook up wr... |
1553 |
unsigned int wb_acct; |
07068d5b8 blk-mq: split mak... |
1554 1555 |
blk_queue_bounce(q, &bio); |
af67c31fb blk: remove bio_s... |
1556 |
blk_queue_split(q, &bio); |
f36ea50ca blk-mq: NVMe 512B... |
1557 |
|
e23947bd7 bio-integrity: fo... |
1558 |
if (!bio_integrity_prep(bio)) |
dece16353 block: change ->m... |
1559 |
return BLK_QC_T_NONE; |
07068d5b8 blk-mq: split mak... |
1560 |
|
87c279e61 blk-mq: really fi... |
1561 1562 1563 |
if (!is_flush_fua && !blk_queue_nomerges(q) && blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq)) return BLK_QC_T_NONE; |
f984df1f0 blk-mq: do limite... |
1564 |
|
bd166ef18 blk-mq-sched: add... |
1565 1566 |
if (blk_mq_sched_bio_merge(q, bio)) return BLK_QC_T_NONE; |
87760e5ee block: hook up wr... |
1567 |
wb_acct = wbt_wait(q->rq_wb, bio, NULL); |
bd166ef18 blk-mq-sched: add... |
1568 |
trace_block_getrq(q, bio, bio->bi_opf); |
d2c0d3832 blk-mq: move blk_... |
1569 |
rq = blk_mq_get_request(q, bio, bio->bi_opf, &data); |
87760e5ee block: hook up wr... |
1570 1571 |
if (unlikely(!rq)) { __wbt_done(q->rq_wb, wb_acct); |
03a07c92a block: return on ... |
1572 1573 |
if (bio->bi_opf & REQ_NOWAIT) bio_wouldblock_error(bio); |
dece16353 block: change ->m... |
1574 |
return BLK_QC_T_NONE; |
87760e5ee block: hook up wr... |
1575 1576 1577 |
} wbt_track(&rq->issue_stat, wb_acct); |
07068d5b8 blk-mq: split mak... |
1578 |
|
fd2d33267 blk-mq: add suppo... |
1579 |
cookie = request_to_qc_t(data.hctx, rq); |
07068d5b8 blk-mq: split mak... |
1580 |
|
f984df1f0 blk-mq: do limite... |
1581 |
plug = current->plug; |
07068d5b8 blk-mq: split mak... |
1582 |
if (unlikely(is_flush_fua)) { |
f984df1f0 blk-mq: do limite... |
1583 |
blk_mq_put_ctx(data.ctx); |
07068d5b8 blk-mq: split mak... |
1584 |
blk_mq_bio_to_request(rq, bio); |
a4d907b6a blk-mq: streamlin... |
1585 1586 1587 |
if (q->elevator) { blk_mq_sched_insert_request(rq, false, true, true, true); |
6a83e74d2 blk-mq: Introduce... |
1588 |
} else { |
a4d907b6a blk-mq: streamlin... |
1589 1590 |
blk_insert_flush(rq); blk_mq_run_hw_queue(data.hctx, true); |
6a83e74d2 blk-mq: Introduce... |
1591 |
} |
a4d907b6a blk-mq: streamlin... |
1592 |
} else if (plug && q->nr_hw_queues == 1) { |
600271d90 blk-mq: immediate... |
1593 |
struct request *last = NULL; |
b00c53e8f blk-mq: fix sched... |
1594 |
blk_mq_put_ctx(data.ctx); |
e6c4438ba blk-mq: fix plugg... |
1595 |
blk_mq_bio_to_request(rq, bio); |
0a6219a95 block: deal with ... |
1596 1597 1598 1599 1600 1601 1602 |
/* * @request_count may become stale because of schedule * out, so check the list again. */ if (list_empty(&plug->mq_list)) request_count = 0; |
254d259da blk-mq: merge mq ... |
1603 1604 |
else if (blk_queue_nomerges(q)) request_count = blk_plug_queued_count(q); |
676d06077 blk-mq: fix for t... |
1605 |
if (!request_count) |
e6c4438ba blk-mq: fix plugg... |
1606 |
trace_block_plug(q); |
600271d90 blk-mq: immediate... |
1607 1608 |
else last = list_entry_rq(plug->mq_list.prev); |
b094f89ca blk-mq: fix calli... |
1609 |
|
600271d90 blk-mq: immediate... |
1610 1611 |
if (request_count >= BLK_MAX_REQUEST_COUNT || (last && blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) { |
e6c4438ba blk-mq: fix plugg... |
1612 1613 |
blk_flush_plug_list(plug, false); trace_block_plug(q); |
320ae51fe blk-mq: new multi... |
1614 |
} |
b094f89ca blk-mq: fix calli... |
1615 |
|
e6c4438ba blk-mq: fix plugg... |
1616 |
list_add_tail(&rq->queuelist, &plug->mq_list); |
2299722c4 blk-mq: split the... |
1617 |
} else if (plug && !blk_queue_nomerges(q)) { |
bd166ef18 blk-mq-sched: add... |
1618 |
blk_mq_bio_to_request(rq, bio); |
07068d5b8 blk-mq: split mak... |
1619 |
|
07068d5b8 blk-mq: split mak... |
1620 |
/* |
6a83e74d2 blk-mq: Introduce... |
1621 |
* We do limited plugging. If the bio can be merged, do that. |
f984df1f0 blk-mq: do limite... |
1622 1623 |
* Otherwise the existing request in the plug list will be * issued. So the plug list will have one request at most |
2299722c4 blk-mq: split the... |
1624 1625 |
* The plug list might get flushed before this. If that happens, * the plug list is empty, and same_queue_rq is invalid. |
07068d5b8 blk-mq: split mak... |
1626 |
*/ |
2299722c4 blk-mq: split the... |
1627 1628 1629 1630 1631 |
if (list_empty(&plug->mq_list)) same_queue_rq = NULL; if (same_queue_rq) list_del_init(&same_queue_rq->queuelist); list_add_tail(&rq->queuelist, &plug->mq_list); |
bf4907c05 blk-mq: fix sched... |
1632 |
blk_mq_put_ctx(data.ctx); |
dad7a3be4 blk-mq: pass corr... |
1633 1634 1635 |
if (same_queue_rq) { data.hctx = blk_mq_map_queue(q, same_queue_rq->mq_ctx->cpu); |
2299722c4 blk-mq: split the... |
1636 1637 |
blk_mq_try_issue_directly(data.hctx, same_queue_rq, &cookie); |
dad7a3be4 blk-mq: pass corr... |
1638 |
} |
a4d907b6a blk-mq: streamlin... |
1639 |
} else if (q->nr_hw_queues > 1 && is_sync) { |
bf4907c05 blk-mq: fix sched... |
1640 |
blk_mq_put_ctx(data.ctx); |
2299722c4 blk-mq: split the... |
1641 |
blk_mq_bio_to_request(rq, bio); |
2299722c4 blk-mq: split the... |
1642 |
blk_mq_try_issue_directly(data.hctx, rq, &cookie); |
a4d907b6a blk-mq: streamlin... |
1643 |
} else if (q->elevator) { |
b00c53e8f blk-mq: fix sched... |
1644 |
blk_mq_put_ctx(data.ctx); |
bd166ef18 blk-mq-sched: add... |
1645 |
blk_mq_bio_to_request(rq, bio); |
a4d907b6a blk-mq: streamlin... |
1646 |
blk_mq_sched_insert_request(rq, false, true, true, true); |
ab42f35d9 blk-mq: merge bio... |
1647 |
} else { |
b00c53e8f blk-mq: fix sched... |
1648 |
blk_mq_put_ctx(data.ctx); |
ab42f35d9 blk-mq: merge bio... |
1649 1650 |
blk_mq_bio_to_request(rq, bio); blk_mq_queue_io(data.hctx, data.ctx, rq); |
a4d907b6a blk-mq: streamlin... |
1651 |
blk_mq_run_hw_queue(data.hctx, true); |
ab42f35d9 blk-mq: merge bio... |
1652 |
} |
320ae51fe blk-mq: new multi... |
1653 |
|
7b371636f blk-mq: return ta... |
1654 |
return cookie; |
320ae51fe blk-mq: new multi... |
1655 |
} |
cc71a6f43 blk-mq: abstract ... |
1656 1657 |
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx) |
95363efde blk-mq: allow blk... |
1658 |
{ |
e9b267d91 blk-mq: add ->ini... |
1659 |
struct page *page; |
320ae51fe blk-mq: new multi... |
1660 |
|
24d2f9030 blk-mq: split out... |
1661 |
if (tags->rqs && set->ops->exit_request) { |
e9b267d91 blk-mq: add ->ini... |
1662 |
int i; |
320ae51fe blk-mq: new multi... |
1663 |
|
24d2f9030 blk-mq: split out... |
1664 |
for (i = 0; i < tags->nr_tags; i++) { |
2af8cbe30 blk-mq: split tag... |
1665 1666 1667 |
struct request *rq = tags->static_rqs[i]; if (!rq) |
e9b267d91 blk-mq: add ->ini... |
1668 |
continue; |
d6296d39e blk-mq: update ->... |
1669 |
set->ops->exit_request(set, rq, hctx_idx); |
2af8cbe30 blk-mq: split tag... |
1670 |
tags->static_rqs[i] = NULL; |
e9b267d91 blk-mq: add ->ini... |
1671 |
} |
320ae51fe blk-mq: new multi... |
1672 |
} |
320ae51fe blk-mq: new multi... |
1673 |
|
24d2f9030 blk-mq: split out... |
1674 1675 |
while (!list_empty(&tags->page_list)) { page = list_first_entry(&tags->page_list, struct page, lru); |
6753471c0 blk-mq: uses page... |
1676 |
list_del_init(&page->lru); |
f75782e4e block: kmemleak: ... |
1677 1678 1679 1680 1681 |
/* * Remove kmemleak object previously allocated in * blk_mq_init_rq_map(). */ kmemleak_free(page_address(page)); |
320ae51fe blk-mq: new multi... |
1682 1683 |
__free_pages(page, page->private); } |
cc71a6f43 blk-mq: abstract ... |
1684 |
} |
320ae51fe blk-mq: new multi... |
1685 |
|
cc71a6f43 blk-mq: abstract ... |
1686 1687 |
void blk_mq_free_rq_map(struct blk_mq_tags *tags) { |
24d2f9030 blk-mq: split out... |
1688 |
kfree(tags->rqs); |
cc71a6f43 blk-mq: abstract ... |
1689 |
tags->rqs = NULL; |
2af8cbe30 blk-mq: split tag... |
1690 1691 |
kfree(tags->static_rqs); tags->static_rqs = NULL; |
320ae51fe blk-mq: new multi... |
1692 |
|
24d2f9030 blk-mq: split out... |
1693 |
blk_mq_free_tags(tags); |
320ae51fe blk-mq: new multi... |
1694 |
} |
cc71a6f43 blk-mq: abstract ... |
1695 1696 1697 1698 |
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, unsigned int hctx_idx, unsigned int nr_tags, unsigned int reserved_tags) |
320ae51fe blk-mq: new multi... |
1699 |
{ |
24d2f9030 blk-mq: split out... |
1700 |
struct blk_mq_tags *tags; |
59f082e46 blk-mq: allocate ... |
1701 |
int node; |
320ae51fe blk-mq: new multi... |
1702 |
|
59f082e46 blk-mq: allocate ... |
1703 1704 1705 1706 1707 |
node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); if (node == NUMA_NO_NODE) node = set->numa_node; tags = blk_mq_init_tags(nr_tags, reserved_tags, node, |
24391c0dc blk-mq: add tag a... |
1708 |
BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags)); |
24d2f9030 blk-mq: split out... |
1709 1710 |
if (!tags) return NULL; |
320ae51fe blk-mq: new multi... |
1711 |
|
cc71a6f43 blk-mq: abstract ... |
1712 |
tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *), |
36e1f3d10 blk-mq: Avoid mem... |
1713 |
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, |
59f082e46 blk-mq: allocate ... |
1714 |
node); |
24d2f9030 blk-mq: split out... |
1715 1716 1717 1718 |
if (!tags->rqs) { blk_mq_free_tags(tags); return NULL; } |
320ae51fe blk-mq: new multi... |
1719 |
|
2af8cbe30 blk-mq: split tag... |
1720 1721 |
tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY, |
59f082e46 blk-mq: allocate ... |
1722 |
node); |
2af8cbe30 blk-mq: split tag... |
1723 1724 1725 1726 1727 |
if (!tags->static_rqs) { kfree(tags->rqs); blk_mq_free_tags(tags); return NULL; } |
cc71a6f43 blk-mq: abstract ... |
1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 |
return tags; } static size_t order_to_size(unsigned int order) { return (size_t)PAGE_SIZE << order; } int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, unsigned int hctx_idx, unsigned int depth) { unsigned int i, j, entries_per_page, max_order = 4; size_t rq_size, left; |
59f082e46 blk-mq: allocate ... |
1741 1742 1743 1744 1745 |
int node; node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx); if (node == NUMA_NO_NODE) node = set->numa_node; |
cc71a6f43 blk-mq: abstract ... |
1746 1747 |
INIT_LIST_HEAD(&tags->page_list); |
320ae51fe blk-mq: new multi... |
1748 1749 1750 1751 |
/* * rq_size is the size of the request plus driver payload, rounded * to the cacheline size */ |
24d2f9030 blk-mq: split out... |
1752 |
rq_size = round_up(sizeof(struct request) + set->cmd_size, |
320ae51fe blk-mq: new multi... |
1753 |
cache_line_size()); |
cc71a6f43 blk-mq: abstract ... |
1754 |
left = rq_size * depth; |
320ae51fe blk-mq: new multi... |
1755 |
|
cc71a6f43 blk-mq: abstract ... |
1756 |
for (i = 0; i < depth; ) { |
320ae51fe blk-mq: new multi... |
1757 1758 1759 1760 |
int this_order = max_order; struct page *page; int to_do; void *p; |
b3a834b15 blk-mq: fix undef... |
1761 |
while (this_order && left < order_to_size(this_order - 1)) |
320ae51fe blk-mq: new multi... |
1762 1763 1764 |
this_order--; do { |
59f082e46 blk-mq: allocate ... |
1765 |
page = alloc_pages_node(node, |
36e1f3d10 blk-mq: Avoid mem... |
1766 |
GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO, |
a51644054 blk-mq: scale dep... |
1767 |
this_order); |
320ae51fe blk-mq: new multi... |
1768 1769 1770 1771 1772 1773 1774 1775 1776 |
if (page) break; if (!this_order--) break; if (order_to_size(this_order) < rq_size) break; } while (1); if (!page) |
24d2f9030 blk-mq: split out... |
1777 |
goto fail; |
320ae51fe blk-mq: new multi... |
1778 1779 |
page->private = this_order; |
24d2f9030 blk-mq: split out... |
1780 |
list_add_tail(&page->lru, &tags->page_list); |
320ae51fe blk-mq: new multi... |
1781 1782 |
p = page_address(page); |
f75782e4e block: kmemleak: ... |
1783 1784 1785 1786 |
/* * Allow kmemleak to scan these pages as they contain pointers * to additional allocations like via ops->init_request(). */ |
36e1f3d10 blk-mq: Avoid mem... |
1787 |
kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO); |
320ae51fe blk-mq: new multi... |
1788 |
entries_per_page = order_to_size(this_order) / rq_size; |
cc71a6f43 blk-mq: abstract ... |
1789 |
to_do = min(entries_per_page, depth - i); |
320ae51fe blk-mq: new multi... |
1790 1791 |
left -= to_do * rq_size; for (j = 0; j < to_do; j++) { |
2af8cbe30 blk-mq: split tag... |
1792 1793 1794 |
struct request *rq = p; tags->static_rqs[i] = rq; |
24d2f9030 blk-mq: split out... |
1795 |
if (set->ops->init_request) { |
d6296d39e blk-mq: update ->... |
1796 |
if (set->ops->init_request(set, rq, hctx_idx, |
59f082e46 blk-mq: allocate ... |
1797 |
node)) { |
2af8cbe30 blk-mq: split tag... |
1798 |
tags->static_rqs[i] = NULL; |
24d2f9030 blk-mq: split out... |
1799 |
goto fail; |
a51644054 blk-mq: scale dep... |
1800 |
} |
e9b267d91 blk-mq: add ->ini... |
1801 |
} |
320ae51fe blk-mq: new multi... |
1802 1803 1804 1805 |
p += rq_size; i++; } } |
cc71a6f43 blk-mq: abstract ... |
1806 |
return 0; |
320ae51fe blk-mq: new multi... |
1807 |
|
24d2f9030 blk-mq: split out... |
1808 |
fail: |
cc71a6f43 blk-mq: abstract ... |
1809 1810 |
blk_mq_free_rqs(set, tags, hctx_idx); return -ENOMEM; |
320ae51fe blk-mq: new multi... |
1811 |
} |
e57690fe0 blk-mq: don't ove... |
1812 1813 1814 1815 1816 |
/* * 'cpu' is going away. splice any existing rq_list entries from this * software queue to the hw queue dispatch list, and ensure that it * gets run. */ |
9467f8596 blk-mq/cpu-notif:... |
1817 |
static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) |
484b4061e blk-mq: save memo... |
1818 |
{ |
9467f8596 blk-mq/cpu-notif:... |
1819 |
struct blk_mq_hw_ctx *hctx; |
484b4061e blk-mq: save memo... |
1820 1821 |
struct blk_mq_ctx *ctx; LIST_HEAD(tmp); |
9467f8596 blk-mq/cpu-notif:... |
1822 |
hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead); |
e57690fe0 blk-mq: don't ove... |
1823 |
ctx = __blk_mq_get_ctx(hctx->queue, cpu); |
484b4061e blk-mq: save memo... |
1824 1825 1826 1827 1828 1829 1830 1831 1832 |
spin_lock(&ctx->lock); if (!list_empty(&ctx->rq_list)) { list_splice_init(&ctx->rq_list, &tmp); blk_mq_hctx_clear_pending(hctx, ctx); } spin_unlock(&ctx->lock); if (list_empty(&tmp)) |
9467f8596 blk-mq/cpu-notif:... |
1833 |
return 0; |
484b4061e blk-mq: save memo... |
1834 |
|
e57690fe0 blk-mq: don't ove... |
1835 1836 1837 |
spin_lock(&hctx->lock); list_splice_tail_init(&tmp, &hctx->dispatch); spin_unlock(&hctx->lock); |
484b4061e blk-mq: save memo... |
1838 1839 |
blk_mq_run_hw_queue(hctx, true); |
9467f8596 blk-mq/cpu-notif:... |
1840 |
return 0; |
484b4061e blk-mq: save memo... |
1841 |
} |
9467f8596 blk-mq/cpu-notif:... |
1842 |
static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx) |
484b4061e blk-mq: save memo... |
1843 |
{ |
9467f8596 blk-mq/cpu-notif:... |
1844 1845 |
cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); |
484b4061e blk-mq: save memo... |
1846 |
} |
c3b4afca7 blk-mq: free hctx... |
1847 |
/* hctx->ctxs will be freed in queue's release handler */ |
08e98fc60 blk-mq: handle fa... |
1848 1849 1850 1851 |
static void blk_mq_exit_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) { |
9c1051aac blk-mq: untangle ... |
1852 |
blk_mq_debugfs_unregister_hctx(hctx); |
8976d64b2 blk-mq: fix kerne... |
1853 1854 |
if (blk_mq_hw_queue_mapped(hctx)) blk_mq_tag_idle(hctx); |
08e98fc60 blk-mq: handle fa... |
1855 |
|
f70ced091 blk-mq: support p... |
1856 |
if (set->ops->exit_request) |
d6296d39e blk-mq: update ->... |
1857 |
set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx); |
f70ced091 blk-mq: support p... |
1858 |
|
93252632e blk-mq-sched: set... |
1859 |
blk_mq_sched_exit_hctx(q, hctx, hctx_idx); |
08e98fc60 blk-mq: handle fa... |
1860 1861 |
if (set->ops->exit_hctx) set->ops->exit_hctx(hctx, hctx_idx); |
6a83e74d2 blk-mq: Introduce... |
1862 |
if (hctx->flags & BLK_MQ_F_BLOCKING) |
073196787 blk-mq: Reduce bl... |
1863 |
cleanup_srcu_struct(hctx->queue_rq_srcu); |
6a83e74d2 blk-mq: Introduce... |
1864 |
|
9467f8596 blk-mq/cpu-notif:... |
1865 |
blk_mq_remove_cpuhp(hctx); |
f70ced091 blk-mq: support p... |
1866 |
blk_free_flush_queue(hctx->fq); |
88459642c blk-mq: abstract ... |
1867 |
sbitmap_free(&hctx->ctx_map); |
08e98fc60 blk-mq: handle fa... |
1868 |
} |
624dbe475 blk-mq: avoid cod... |
1869 1870 1871 1872 1873 1874 1875 1876 1877 |
static void blk_mq_exit_hw_queues(struct request_queue *q, struct blk_mq_tag_set *set, int nr_queue) { struct blk_mq_hw_ctx *hctx; unsigned int i; queue_for_each_hw_ctx(q, hctx, i) { if (i == nr_queue) break; |
08e98fc60 blk-mq: handle fa... |
1878 |
blk_mq_exit_hctx(q, set, hctx, i); |
624dbe475 blk-mq: avoid cod... |
1879 |
} |
624dbe475 blk-mq: avoid cod... |
1880 |
} |
08e98fc60 blk-mq: handle fa... |
1881 1882 1883 |
static int blk_mq_init_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) |
320ae51fe blk-mq: new multi... |
1884 |
{ |
08e98fc60 blk-mq: handle fa... |
1885 1886 1887 1888 1889 |
int node; node = hctx->numa_node; if (node == NUMA_NO_NODE) node = hctx->numa_node = set->numa_node; |
9f9937379 blk-mq: unify hct... |
1890 |
INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); |
08e98fc60 blk-mq: handle fa... |
1891 1892 1893 |
spin_lock_init(&hctx->lock); INIT_LIST_HEAD(&hctx->dispatch); hctx->queue = q; |
2404e607a blk-mq: avoid exc... |
1894 |
hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED; |
08e98fc60 blk-mq: handle fa... |
1895 |
|
9467f8596 blk-mq/cpu-notif:... |
1896 |
cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead); |
08e98fc60 blk-mq: handle fa... |
1897 1898 |
hctx->tags = set->tags[hctx_idx]; |
320ae51fe blk-mq: new multi... |
1899 1900 |
/* |
08e98fc60 blk-mq: handle fa... |
1901 1902 |
* Allocate space for all possible cpus to avoid allocation at * runtime |
320ae51fe blk-mq: new multi... |
1903 |
*/ |
08e98fc60 blk-mq: handle fa... |
1904 1905 1906 1907 |
hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *), GFP_KERNEL, node); if (!hctx->ctxs) goto unregister_cpu_notifier; |
320ae51fe blk-mq: new multi... |
1908 |
|
88459642c blk-mq: abstract ... |
1909 1910 |
if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL, node)) |
08e98fc60 blk-mq: handle fa... |
1911 |
goto free_ctxs; |
320ae51fe blk-mq: new multi... |
1912 |
|
08e98fc60 blk-mq: handle fa... |
1913 |
hctx->nr_ctx = 0; |
320ae51fe blk-mq: new multi... |
1914 |
|
08e98fc60 blk-mq: handle fa... |
1915 1916 1917 |
if (set->ops->init_hctx && set->ops->init_hctx(hctx, set->driver_data, hctx_idx)) goto free_bitmap; |
320ae51fe blk-mq: new multi... |
1918 |
|
93252632e blk-mq-sched: set... |
1919 1920 |
if (blk_mq_sched_init_hctx(q, hctx, hctx_idx)) goto exit_hctx; |
f70ced091 blk-mq: support p... |
1921 1922 |
hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size); if (!hctx->fq) |
93252632e blk-mq-sched: set... |
1923 |
goto sched_exit_hctx; |
320ae51fe blk-mq: new multi... |
1924 |
|
f70ced091 blk-mq: support p... |
1925 |
if (set->ops->init_request && |
d6296d39e blk-mq: update ->... |
1926 1927 |
set->ops->init_request(set, hctx->fq->flush_rq, hctx_idx, node)) |
f70ced091 blk-mq: support p... |
1928 |
goto free_fq; |
320ae51fe blk-mq: new multi... |
1929 |
|
6a83e74d2 blk-mq: Introduce... |
1930 |
if (hctx->flags & BLK_MQ_F_BLOCKING) |
073196787 blk-mq: Reduce bl... |
1931 |
init_srcu_struct(hctx->queue_rq_srcu); |
6a83e74d2 blk-mq: Introduce... |
1932 |
|
9c1051aac blk-mq: untangle ... |
1933 |
blk_mq_debugfs_register_hctx(q, hctx); |
08e98fc60 blk-mq: handle fa... |
1934 |
return 0; |
320ae51fe blk-mq: new multi... |
1935 |
|
f70ced091 blk-mq: support p... |
1936 1937 |
free_fq: kfree(hctx->fq); |
93252632e blk-mq-sched: set... |
1938 1939 |
sched_exit_hctx: blk_mq_sched_exit_hctx(q, hctx, hctx_idx); |
f70ced091 blk-mq: support p... |
1940 1941 1942 |
exit_hctx: if (set->ops->exit_hctx) set->ops->exit_hctx(hctx, hctx_idx); |
08e98fc60 blk-mq: handle fa... |
1943 |
free_bitmap: |
88459642c blk-mq: abstract ... |
1944 |
sbitmap_free(&hctx->ctx_map); |
08e98fc60 blk-mq: handle fa... |
1945 1946 1947 |
free_ctxs: kfree(hctx->ctxs); unregister_cpu_notifier: |
9467f8596 blk-mq/cpu-notif:... |
1948 |
blk_mq_remove_cpuhp(hctx); |
08e98fc60 blk-mq: handle fa... |
1949 1950 |
return -1; } |
320ae51fe blk-mq: new multi... |
1951 |
|
320ae51fe blk-mq: new multi... |
1952 1953 1954 1955 1956 1957 1958 1959 |
static void blk_mq_init_cpu_queues(struct request_queue *q, unsigned int nr_hw_queues) { unsigned int i; for_each_possible_cpu(i) { struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i); struct blk_mq_hw_ctx *hctx; |
320ae51fe blk-mq: new multi... |
1960 1961 1962 1963 |
__ctx->cpu = i; spin_lock_init(&__ctx->lock); INIT_LIST_HEAD(&__ctx->rq_list); __ctx->queue = q; |
4b855ad37 blk-mq: Create hc... |
1964 1965 |
/* If the cpu isn't present, the cpu is mapped to first hctx */ if (!cpu_present(i)) |
320ae51fe blk-mq: new multi... |
1966 |
continue; |
7d7e0f90b blk-mq: remove ->... |
1967 |
hctx = blk_mq_map_queue(q, i); |
e4043dcf3 blk-mq: ensure th... |
1968 |
|
320ae51fe blk-mq: new multi... |
1969 1970 1971 1972 1973 |
/* * Set local node, IFF we have more than one hw queue. If * not, we remain on the home node of the device */ if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) |
bffed4571 blk-mq: Avoid mem... |
1974 |
hctx->numa_node = local_memory_node(cpu_to_node(i)); |
320ae51fe blk-mq: new multi... |
1975 1976 |
} } |
cc71a6f43 blk-mq: abstract ... |
1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 |
static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx) { int ret = 0; set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx, set->queue_depth, set->reserved_tags); if (!set->tags[hctx_idx]) return false; ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx, set->queue_depth); if (!ret) return true; blk_mq_free_rq_map(set->tags[hctx_idx]); set->tags[hctx_idx] = NULL; return false; } static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set, unsigned int hctx_idx) { |
bd166ef18 blk-mq-sched: add... |
1999 2000 2001 2002 2003 |
if (set->tags[hctx_idx]) { blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx); blk_mq_free_rq_map(set->tags[hctx_idx]); set->tags[hctx_idx] = NULL; } |
cc71a6f43 blk-mq: abstract ... |
2004 |
} |
4b855ad37 blk-mq: Create hc... |
2005 |
static void blk_mq_map_swqueue(struct request_queue *q) |
320ae51fe blk-mq: new multi... |
2006 |
{ |
d1b1cea1e blk-mq: Fix faile... |
2007 |
unsigned int i, hctx_idx; |
320ae51fe blk-mq: new multi... |
2008 2009 |
struct blk_mq_hw_ctx *hctx; struct blk_mq_ctx *ctx; |
2a34c0872 blk-mq: fix CPU h... |
2010 |
struct blk_mq_tag_set *set = q->tag_set; |
320ae51fe blk-mq: new multi... |
2011 |
|
60de074ba blk-mq: fix deadl... |
2012 2013 2014 2015 |
/* * Avoid others reading imcomplete hctx->cpumask through sysfs */ mutex_lock(&q->sysfs_lock); |
320ae51fe blk-mq: new multi... |
2016 |
queue_for_each_hw_ctx(q, hctx, i) { |
e4043dcf3 blk-mq: ensure th... |
2017 |
cpumask_clear(hctx->cpumask); |
320ae51fe blk-mq: new multi... |
2018 2019 2020 2021 |
hctx->nr_ctx = 0; } /* |
4b855ad37 blk-mq: Create hc... |
2022 2023 2024 |
* Map software to hardware queues. * * If the cpu isn't present, the cpu is mapped to first hctx. |
320ae51fe blk-mq: new multi... |
2025 |
*/ |
4b855ad37 blk-mq: Create hc... |
2026 |
for_each_present_cpu(i) { |
d1b1cea1e blk-mq: Fix faile... |
2027 2028 |
hctx_idx = q->mq_map[i]; /* unmapped hw queue can be remapped after CPU topo changed */ |
cc71a6f43 blk-mq: abstract ... |
2029 2030 |
if (!set->tags[hctx_idx] && !__blk_mq_alloc_rq_map(set, hctx_idx)) { |
d1b1cea1e blk-mq: Fix faile... |
2031 2032 2033 2034 2035 2036 |
/* * If tags initialization fail for some hctx, * that hctx won't be brought online. In this * case, remap the current ctx to hctx[0] which * is guaranteed to always have tags allocated */ |
cc71a6f43 blk-mq: abstract ... |
2037 |
q->mq_map[i] = 0; |
d1b1cea1e blk-mq: Fix faile... |
2038 |
} |
897bb0c7f blk-mq: Use prope... |
2039 |
ctx = per_cpu_ptr(q->queue_ctx, i); |
7d7e0f90b blk-mq: remove ->... |
2040 |
hctx = blk_mq_map_queue(q, i); |
868f2f0b7 blk-mq: dynamic h... |
2041 |
|
e4043dcf3 blk-mq: ensure th... |
2042 |
cpumask_set_cpu(i, hctx->cpumask); |
320ae51fe blk-mq: new multi... |
2043 2044 2045 |
ctx->index_hw = hctx->nr_ctx; hctx->ctxs[hctx->nr_ctx++] = ctx; } |
506e931f9 blk-mq: add basic... |
2046 |
|
60de074ba blk-mq: fix deadl... |
2047 |
mutex_unlock(&q->sysfs_lock); |
506e931f9 blk-mq: add basic... |
2048 |
queue_for_each_hw_ctx(q, hctx, i) { |
484b4061e blk-mq: save memo... |
2049 |
/* |
a68aafa5b blk-mq: correct a... |
2050 2051 |
* If no software queues are mapped to this hardware queue, * disable it and free the request entries. |
484b4061e blk-mq: save memo... |
2052 2053 |
*/ if (!hctx->nr_ctx) { |
d1b1cea1e blk-mq: Fix faile... |
2054 2055 2056 2057 |
/* Never unmap queue 0. We need it as a * fallback in case of a new remap fails * allocation */ |
cc71a6f43 blk-mq: abstract ... |
2058 2059 |
if (i && set->tags[i]) blk_mq_free_map_and_requests(set, i); |
2a34c0872 blk-mq: fix CPU h... |
2060 |
hctx->tags = NULL; |
484b4061e blk-mq: save memo... |
2061 2062 |
continue; } |
2a34c0872 blk-mq: fix CPU h... |
2063 2064 |
hctx->tags = set->tags[i]; WARN_ON(!hctx->tags); |
484b4061e blk-mq: save memo... |
2065 |
/* |
889fa31f0 blk-mq: reduce un... |
2066 2067 2068 2069 |
* Set the map size to the number of mapped software queues. * This is more accurate and more efficient than looping * over all possibly mapped software queues. */ |
88459642c blk-mq: abstract ... |
2070 |
sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx); |
889fa31f0 blk-mq: reduce un... |
2071 2072 |
/* |
484b4061e blk-mq: save memo... |
2073 2074 |
* Initialize batch roundrobin counts */ |
506e931f9 blk-mq: add basic... |
2075 2076 2077 |
hctx->next_cpu = cpumask_first(hctx->cpumask); hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; } |
320ae51fe blk-mq: new multi... |
2078 |
} |
8e8320c93 blk-mq: fix perfo... |
2079 2080 2081 2082 |
/* * Caller needs to ensure that we're either frozen/quiesced, or that * the queue isn't live yet. */ |
2404e607a blk-mq: avoid exc... |
2083 |
static void queue_set_hctx_shared(struct request_queue *q, bool shared) |
0d2602ca3 blk-mq: improve s... |
2084 2085 |
{ struct blk_mq_hw_ctx *hctx; |
0d2602ca3 blk-mq: improve s... |
2086 |
int i; |
2404e607a blk-mq: avoid exc... |
2087 |
queue_for_each_hw_ctx(q, hctx, i) { |
8e8320c93 blk-mq: fix perfo... |
2088 2089 2090 |
if (shared) { if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) atomic_inc(&q->shared_hctx_restart); |
2404e607a blk-mq: avoid exc... |
2091 |
hctx->flags |= BLK_MQ_F_TAG_SHARED; |
8e8320c93 blk-mq: fix perfo... |
2092 2093 2094 |
} else { if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) atomic_dec(&q->shared_hctx_restart); |
2404e607a blk-mq: avoid exc... |
2095 |
hctx->flags &= ~BLK_MQ_F_TAG_SHARED; |
8e8320c93 blk-mq: fix perfo... |
2096 |
} |
2404e607a blk-mq: avoid exc... |
2097 2098 |
} } |
8e8320c93 blk-mq: fix perfo... |
2099 2100 |
static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set, bool shared) |
2404e607a blk-mq: avoid exc... |
2101 2102 |
{ struct request_queue *q; |
0d2602ca3 blk-mq: improve s... |
2103 |
|
705cda97e blk-mq: Make it s... |
2104 |
lockdep_assert_held(&set->tag_list_lock); |
0d2602ca3 blk-mq: improve s... |
2105 2106 |
list_for_each_entry(q, &set->tag_list, tag_set_list) { blk_mq_freeze_queue(q); |
2404e607a blk-mq: avoid exc... |
2107 |
queue_set_hctx_shared(q, shared); |
0d2602ca3 blk-mq: improve s... |
2108 2109 2110 2111 2112 2113 2114 |
blk_mq_unfreeze_queue(q); } } static void blk_mq_del_queue_tag_set(struct request_queue *q) { struct blk_mq_tag_set *set = q->tag_set; |
0d2602ca3 blk-mq: improve s... |
2115 |
mutex_lock(&set->tag_list_lock); |
705cda97e blk-mq: Make it s... |
2116 |
list_del_rcu(&q->tag_set_list); |
2404e607a blk-mq: avoid exc... |
2117 2118 2119 2120 2121 2122 |
if (list_is_singular(&set->tag_list)) { /* just transitioned to unshared */ set->flags &= ~BLK_MQ_F_TAG_SHARED; /* update existing queue */ blk_mq_update_tag_set_depth(set, false); } |
0d2602ca3 blk-mq: improve s... |
2123 |
mutex_unlock(&set->tag_list_lock); |
705cda97e blk-mq: Make it s... |
2124 |
synchronize_rcu(); |
ba502bf2b blk-mq: reinit q-... |
2125 |
INIT_LIST_HEAD(&q->tag_set_list); |
0d2602ca3 blk-mq: improve s... |
2126 2127 2128 2129 2130 2131 2132 2133 |
} static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set, struct request_queue *q) { q->tag_set = set; mutex_lock(&set->tag_list_lock); |
2404e607a blk-mq: avoid exc... |
2134 2135 2136 2137 2138 2139 2140 2141 2142 |
/* Check to see if we're transitioning to shared (from 1 to 2 queues). */ if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) { set->flags |= BLK_MQ_F_TAG_SHARED; /* update existing queue */ blk_mq_update_tag_set_depth(set, true); } if (set->flags & BLK_MQ_F_TAG_SHARED) queue_set_hctx_shared(q, true); |
705cda97e blk-mq: Make it s... |
2143 |
list_add_tail_rcu(&q->tag_set_list, &set->tag_list); |
2404e607a blk-mq: avoid exc... |
2144 |
|
0d2602ca3 blk-mq: improve s... |
2145 2146 |
mutex_unlock(&set->tag_list_lock); } |
e09aae7ed blk-mq: release m... |
2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 |
/* * It is the actual release handler for mq, but we do it from * request queue's release handler for avoiding use-after-free * and headache because q->mq_kobj shouldn't have been introduced, * but we can't group ctx/kctx kobj without it. */ void blk_mq_release(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; unsigned int i; /* hctx kobj stays in hctx */ |
c3b4afca7 blk-mq: free hctx... |
2159 2160 2161 |
queue_for_each_hw_ctx(q, hctx, i) { if (!hctx) continue; |
6c8b232ef blk-mq: make life... |
2162 |
kobject_put(&hctx->kobj); |
c3b4afca7 blk-mq: free hctx... |
2163 |
} |
e09aae7ed blk-mq: release m... |
2164 |
|
a723bab3d blk-mq: Fix use a... |
2165 |
q->mq_map = NULL; |
e09aae7ed blk-mq: release m... |
2166 |
kfree(q->queue_hw_ctx); |
7ea5fe31c blk-mq: make life... |
2167 2168 2169 2170 2171 |
/* * release .mq_kobj and sw queue's kobject now because * both share lifetime with request queue. */ blk_mq_sysfs_deinit(q); |
e09aae7ed blk-mq: release m... |
2172 2173 |
free_percpu(q->queue_ctx); } |
24d2f9030 blk-mq: split out... |
2174 |
struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set) |
320ae51fe blk-mq: new multi... |
2175 |
{ |
b62c21b71 blk-mq: add blk_m... |
2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 |
struct request_queue *uninit_q, *q; uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node); if (!uninit_q) return ERR_PTR(-ENOMEM); q = blk_mq_init_allocated_queue(set, uninit_q); if (IS_ERR(q)) blk_cleanup_queue(uninit_q); return q; } EXPORT_SYMBOL(blk_mq_init_queue); |
073196787 blk-mq: Reduce bl... |
2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 |
static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set) { int hw_ctx_size = sizeof(struct blk_mq_hw_ctx); BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, queue_rq_srcu), __alignof__(struct blk_mq_hw_ctx)) != sizeof(struct blk_mq_hw_ctx)); if (tag_set->flags & BLK_MQ_F_BLOCKING) hw_ctx_size += sizeof(struct srcu_struct); return hw_ctx_size; } |
868f2f0b7 blk-mq: dynamic h... |
2202 2203 |
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set, struct request_queue *q) |
b62c21b71 blk-mq: add blk_m... |
2204 |
{ |
868f2f0b7 blk-mq: dynamic h... |
2205 2206 |
int i, j; struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx; |
f14bbe77a blk-mq: pass in s... |
2207 |
|
868f2f0b7 blk-mq: dynamic h... |
2208 |
blk_mq_sysfs_unregister(q); |
fb1ef85d5 blk-mq: fix race ... |
2209 2210 2211 |
/* protect against switching io scheduler */ mutex_lock(&q->sysfs_lock); |
24d2f9030 blk-mq: split out... |
2212 |
for (i = 0; i < set->nr_hw_queues; i++) { |
868f2f0b7 blk-mq: dynamic h... |
2213 |
int node; |
f14bbe77a blk-mq: pass in s... |
2214 |
|
868f2f0b7 blk-mq: dynamic h... |
2215 2216 2217 2218 |
if (hctxs[i]) continue; node = blk_mq_hw_queue_to_node(q->mq_map, i); |
073196787 blk-mq: Reduce bl... |
2219 |
hctxs[i] = kzalloc_node(blk_mq_hw_ctx_size(set), |
cdef54dd8 blk-mq: remove al... |
2220 |
GFP_KERNEL, node); |
320ae51fe blk-mq: new multi... |
2221 |
if (!hctxs[i]) |
868f2f0b7 blk-mq: dynamic h... |
2222 |
break; |
320ae51fe blk-mq: new multi... |
2223 |
|
a86073e48 blk-mq: allocate ... |
2224 |
if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL, |
868f2f0b7 blk-mq: dynamic h... |
2225 2226 2227 2228 2229 |
node)) { kfree(hctxs[i]); hctxs[i] = NULL; break; } |
e4043dcf3 blk-mq: ensure th... |
2230 |
|
0d2602ca3 blk-mq: improve s... |
2231 |
atomic_set(&hctxs[i]->nr_active, 0); |
f14bbe77a blk-mq: pass in s... |
2232 |
hctxs[i]->numa_node = node; |
320ae51fe blk-mq: new multi... |
2233 |
hctxs[i]->queue_num = i; |
868f2f0b7 blk-mq: dynamic h... |
2234 2235 2236 2237 2238 2239 2240 2241 |
if (blk_mq_init_hctx(q, set, hctxs[i], i)) { free_cpumask_var(hctxs[i]->cpumask); kfree(hctxs[i]); hctxs[i] = NULL; break; } blk_mq_hctx_kobj_init(hctxs[i]); |
320ae51fe blk-mq: new multi... |
2242 |
} |
868f2f0b7 blk-mq: dynamic h... |
2243 2244 2245 2246 |
for (j = i; j < q->nr_hw_queues; j++) { struct blk_mq_hw_ctx *hctx = hctxs[j]; if (hctx) { |
cc71a6f43 blk-mq: abstract ... |
2247 2248 |
if (hctx->tags) blk_mq_free_map_and_requests(set, j); |
868f2f0b7 blk-mq: dynamic h... |
2249 |
blk_mq_exit_hctx(q, set, hctx, j); |
868f2f0b7 blk-mq: dynamic h... |
2250 |
kobject_put(&hctx->kobj); |
868f2f0b7 blk-mq: dynamic h... |
2251 2252 2253 2254 2255 |
hctxs[j] = NULL; } } q->nr_hw_queues = i; |
fb1ef85d5 blk-mq: fix race ... |
2256 |
mutex_unlock(&q->sysfs_lock); |
868f2f0b7 blk-mq: dynamic h... |
2257 2258 2259 2260 2261 2262 |
blk_mq_sysfs_register(q); } struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set, struct request_queue *q) { |
668416721 blk-mq: mark requ... |
2263 2264 |
/* mark the queue as mq asap */ q->mq_ops = set->ops; |
34dbad5d2 blk-stat: convert... |
2265 |
q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn, |
720b8ccc4 blk-mq: Add a pol... |
2266 2267 |
blk_mq_poll_stats_bkt, BLK_MQ_POLL_STATS_BKTS, q); |
34dbad5d2 blk-stat: convert... |
2268 2269 |
if (!q->poll_cb) goto err_exit; |
868f2f0b7 blk-mq: dynamic h... |
2270 2271 |
q->queue_ctx = alloc_percpu(struct blk_mq_ctx); if (!q->queue_ctx) |
c7de57263 blk-mq: clear q->... |
2272 |
goto err_exit; |
868f2f0b7 blk-mq: dynamic h... |
2273 |
|
737f98cfe blk-mq: initializ... |
2274 2275 |
/* init q->mq_kobj and sw queues' kobjects */ blk_mq_sysfs_init(q); |
868f2f0b7 blk-mq: dynamic h... |
2276 2277 2278 2279 |
q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)), GFP_KERNEL, set->numa_node); if (!q->queue_hw_ctx) goto err_percpu; |
bdd17e75c blk-mq: only allo... |
2280 |
q->mq_map = set->mq_map; |
868f2f0b7 blk-mq: dynamic h... |
2281 2282 2283 2284 |
blk_mq_realloc_hw_ctxs(set, q); if (!q->nr_hw_queues) goto err_hctxs; |
320ae51fe blk-mq: new multi... |
2285 |
|
287922eb0 block: defer time... |
2286 |
INIT_WORK(&q->timeout_work, blk_mq_timeout_work); |
e56f698bd blk-mq: set defau... |
2287 |
blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ); |
320ae51fe blk-mq: new multi... |
2288 2289 |
q->nr_queues = nr_cpu_ids; |
320ae51fe blk-mq: new multi... |
2290 |
|
94eddfbea blk-mq: ensure th... |
2291 |
q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT; |
320ae51fe blk-mq: new multi... |
2292 |
|
05f1dd531 block: add queue ... |
2293 2294 |
if (!(set->flags & BLK_MQ_F_SG_MERGE)) q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE; |
1be036e94 blk-mq: initializ... |
2295 |
q->sg_reserved_size = INT_MAX; |
2849450ad blk-mq: introduce... |
2296 |
INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work); |
6fca6a611 blk-mq: add helpe... |
2297 2298 |
INIT_LIST_HEAD(&q->requeue_list); spin_lock_init(&q->requeue_lock); |
254d259da blk-mq: merge mq ... |
2299 |
blk_queue_make_request(q, blk_mq_make_request); |
45cde2f81 block: add a poll... |
2300 2301 |
if (q->mq_ops->poll) q->poll_fn = blk_mq_poll; |
07068d5b8 blk-mq: split mak... |
2302 |
|
eba717682 blk-mq: initializ... |
2303 2304 2305 2306 |
/* * Do this after blk_queue_make_request() overrides it... */ q->nr_requests = set->queue_depth; |
64f1c21e8 blk-mq: make the ... |
2307 2308 2309 2310 |
/* * Default to classic polling */ q->poll_nsec = -1; |
24d2f9030 blk-mq: split out... |
2311 2312 |
if (set->ops->complete) blk_queue_softirq_done(q, set->ops->complete); |
30a91cb4e blk-mq: rework I/... |
2313 |
|
24d2f9030 blk-mq: split out... |
2314 |
blk_mq_init_cpu_queues(q, set->nr_hw_queues); |
0d2602ca3 blk-mq: improve s... |
2315 |
blk_mq_add_queue_tag_set(set, q); |
4b855ad37 blk-mq: Create hc... |
2316 |
blk_mq_map_swqueue(q); |
4593fdbe7 blk-mq: fix sysfs... |
2317 |
|
d34849913 blk-mq-sched: all... |
2318 2319 2320 2321 2322 2323 2324 |
if (!(set->flags & BLK_MQ_F_NO_SCHED)) { int ret; ret = blk_mq_sched_init(q); if (ret) return ERR_PTR(ret); } |
320ae51fe blk-mq: new multi... |
2325 |
return q; |
18741986a blk-mq: rework fl... |
2326 |
|
320ae51fe blk-mq: new multi... |
2327 |
err_hctxs: |
868f2f0b7 blk-mq: dynamic h... |
2328 |
kfree(q->queue_hw_ctx); |
320ae51fe blk-mq: new multi... |
2329 |
err_percpu: |
868f2f0b7 blk-mq: dynamic h... |
2330 |
free_percpu(q->queue_ctx); |
c7de57263 blk-mq: clear q->... |
2331 2332 |
err_exit: q->mq_ops = NULL; |
320ae51fe blk-mq: new multi... |
2333 2334 |
return ERR_PTR(-ENOMEM); } |
b62c21b71 blk-mq: add blk_m... |
2335 |
EXPORT_SYMBOL(blk_mq_init_allocated_queue); |
320ae51fe blk-mq: new multi... |
2336 2337 2338 |
void blk_mq_free_queue(struct request_queue *q) { |
624dbe475 blk-mq: avoid cod... |
2339 |
struct blk_mq_tag_set *set = q->tag_set; |
320ae51fe blk-mq: new multi... |
2340 |
|
0d2602ca3 blk-mq: improve s... |
2341 |
blk_mq_del_queue_tag_set(q); |
624dbe475 blk-mq: avoid cod... |
2342 |
blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); |
320ae51fe blk-mq: new multi... |
2343 |
} |
320ae51fe blk-mq: new multi... |
2344 2345 |
/* Basically redo blk_mq_init_queue with queue frozen */ |
4b855ad37 blk-mq: Create hc... |
2346 |
static void blk_mq_queue_reinit(struct request_queue *q) |
320ae51fe blk-mq: new multi... |
2347 |
{ |
4ecd4fef3 block: use an ato... |
2348 |
WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); |
320ae51fe blk-mq: new multi... |
2349 |
|
9c1051aac blk-mq: untangle ... |
2350 |
blk_mq_debugfs_unregister_hctxs(q); |
67aec14ce blk-mq: make the ... |
2351 |
blk_mq_sysfs_unregister(q); |
320ae51fe blk-mq: new multi... |
2352 2353 2354 2355 2356 |
/* * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe * we should change hctx numa_node according to new topology (this * involves free and re-allocate memory, worthy doing?) */ |
4b855ad37 blk-mq: Create hc... |
2357 |
blk_mq_map_swqueue(q); |
320ae51fe blk-mq: new multi... |
2358 |
|
67aec14ce blk-mq: make the ... |
2359 |
blk_mq_sysfs_register(q); |
9c1051aac blk-mq: untangle ... |
2360 |
blk_mq_debugfs_register_hctxs(q); |
320ae51fe blk-mq: new multi... |
2361 |
} |
a51644054 blk-mq: scale dep... |
2362 2363 2364 |
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) { int i; |
cc71a6f43 blk-mq: abstract ... |
2365 2366 |
for (i = 0; i < set->nr_hw_queues; i++) if (!__blk_mq_alloc_rq_map(set, i)) |
a51644054 blk-mq: scale dep... |
2367 |
goto out_unwind; |
a51644054 blk-mq: scale dep... |
2368 2369 2370 2371 2372 |
return 0; out_unwind: while (--i >= 0) |
cc71a6f43 blk-mq: abstract ... |
2373 |
blk_mq_free_rq_map(set->tags[i]); |
a51644054 blk-mq: scale dep... |
2374 |
|
a51644054 blk-mq: scale dep... |
2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 |
return -ENOMEM; } /* * Allocate the request maps associated with this tag_set. Note that this * may reduce the depth asked for, if memory is tight. set->queue_depth * will be updated to reflect the allocated depth. */ static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) { unsigned int depth; int err; depth = set->queue_depth; do { err = __blk_mq_alloc_rq_maps(set); if (!err) break; set->queue_depth >>= 1; if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) { err = -ENOMEM; break; } } while (set->queue_depth); if (!set->queue_depth || err) { pr_err("blk-mq: failed to allocate request map "); return -ENOMEM; } if (depth != set->queue_depth) pr_info("blk-mq: reduced tag depth (%u -> %u) ", depth, set->queue_depth); return 0; } |
ebe8bddb6 blk-mq: remap que... |
2414 2415 |
static int blk_mq_update_queue_map(struct blk_mq_tag_set *set) { |
eaa077800 blk-mq: avoid to ... |
2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 |
if (set->ops->map_queues) { int cpu; /* * transport .map_queues is usually done in the following * way: * * for (queue = 0; queue < set->nr_hw_queues; queue++) { * mask = get_cpu_mask(queue) * for_each_cpu(cpu, mask) * set->mq_map[cpu] = queue; * } * * When we need to remap, the table has to be cleared for * killing stale mapping since one CPU may not be mapped * to any hw queue. */ for_each_possible_cpu(cpu) set->mq_map[cpu] = 0; |
ebe8bddb6 blk-mq: remap que... |
2434 |
return set->ops->map_queues(set); |
eaa077800 blk-mq: avoid to ... |
2435 |
} else |
ebe8bddb6 blk-mq: remap que... |
2436 2437 |
return blk_mq_map_queues(set); } |
a4391c646 blk-mq: bump max ... |
2438 2439 2440 2441 2442 2443 |
/* * Alloc a tag set to be associated with one or more request queues. * May fail with EINVAL for various error conditions. May adjust the * requested depth down, if if it too large. In that case, the set * value will be stored in set->queue_depth. */ |
24d2f9030 blk-mq: split out... |
2444 2445 |
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set) { |
da695ba23 blk-mq: allow the... |
2446 |
int ret; |
205fb5f5b blk-mq: add blk_m... |
2447 |
BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS); |
24d2f9030 blk-mq: split out... |
2448 2449 |
if (!set->nr_hw_queues) return -EINVAL; |
a4391c646 blk-mq: bump max ... |
2450 |
if (!set->queue_depth) |
24d2f9030 blk-mq: split out... |
2451 2452 2453 |
return -EINVAL; if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) return -EINVAL; |
7d7e0f90b blk-mq: remove ->... |
2454 |
if (!set->ops->queue_rq) |
24d2f9030 blk-mq: split out... |
2455 |
return -EINVAL; |
a4391c646 blk-mq: bump max ... |
2456 2457 2458 2459 2460 2461 |
if (set->queue_depth > BLK_MQ_MAX_DEPTH) { pr_info("blk-mq: reduced tag depth to %u ", BLK_MQ_MAX_DEPTH); set->queue_depth = BLK_MQ_MAX_DEPTH; } |
24d2f9030 blk-mq: split out... |
2462 |
|
6637fadf2 blk-mq: move the ... |
2463 2464 2465 2466 2467 2468 2469 2470 2471 |
/* * If a crashdump is active, then we are potentially in a very * memory constrained environment. Limit us to 1 queue and * 64 tags to prevent using too much memory. */ if (is_kdump_kernel()) { set->nr_hw_queues = 1; set->queue_depth = min(64U, set->queue_depth); } |
868f2f0b7 blk-mq: dynamic h... |
2472 2473 2474 2475 2476 |
/* * There is no use for more h/w queues than cpus. */ if (set->nr_hw_queues > nr_cpu_ids) set->nr_hw_queues = nr_cpu_ids; |
6637fadf2 blk-mq: move the ... |
2477 |
|
868f2f0b7 blk-mq: dynamic h... |
2478 |
set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *), |
24d2f9030 blk-mq: split out... |
2479 2480 |
GFP_KERNEL, set->numa_node); if (!set->tags) |
a51644054 blk-mq: scale dep... |
2481 |
return -ENOMEM; |
24d2f9030 blk-mq: split out... |
2482 |
|
da695ba23 blk-mq: allow the... |
2483 2484 2485 |
ret = -ENOMEM; set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids, GFP_KERNEL, set->numa_node); |
bdd17e75c blk-mq: only allo... |
2486 2487 |
if (!set->mq_map) goto out_free_tags; |
ebe8bddb6 blk-mq: remap que... |
2488 |
ret = blk_mq_update_queue_map(set); |
da695ba23 blk-mq: allow the... |
2489 2490 2491 2492 2493 |
if (ret) goto out_free_mq_map; ret = blk_mq_alloc_rq_maps(set); if (ret) |
bdd17e75c blk-mq: only allo... |
2494 |
goto out_free_mq_map; |
24d2f9030 blk-mq: split out... |
2495 |
|
0d2602ca3 blk-mq: improve s... |
2496 2497 |
mutex_init(&set->tag_list_lock); INIT_LIST_HEAD(&set->tag_list); |
24d2f9030 blk-mq: split out... |
2498 |
return 0; |
bdd17e75c blk-mq: only allo... |
2499 2500 2501 2502 2503 |
out_free_mq_map: kfree(set->mq_map); set->mq_map = NULL; out_free_tags: |
5676e7b6d blk-mq: cleanup a... |
2504 2505 |
kfree(set->tags); set->tags = NULL; |
da695ba23 blk-mq: allow the... |
2506 |
return ret; |
24d2f9030 blk-mq: split out... |
2507 2508 2509 2510 2511 2512 |
} EXPORT_SYMBOL(blk_mq_alloc_tag_set); void blk_mq_free_tag_set(struct blk_mq_tag_set *set) { int i; |
cc71a6f43 blk-mq: abstract ... |
2513 2514 |
for (i = 0; i < nr_cpu_ids; i++) blk_mq_free_map_and_requests(set, i); |
484b4061e blk-mq: save memo... |
2515 |
|
bdd17e75c blk-mq: only allo... |
2516 2517 |
kfree(set->mq_map); set->mq_map = NULL; |
981bd189f blk-mq: fix leak ... |
2518 |
kfree(set->tags); |
5676e7b6d blk-mq: cleanup a... |
2519 |
set->tags = NULL; |
24d2f9030 blk-mq: split out... |
2520 2521 |
} EXPORT_SYMBOL(blk_mq_free_tag_set); |
e3a2b3f93 blk-mq: allow cha... |
2522 2523 2524 2525 2526 |
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) { struct blk_mq_tag_set *set = q->tag_set; struct blk_mq_hw_ctx *hctx; int i, ret; |
bd166ef18 blk-mq-sched: add... |
2527 |
if (!set) |
e3a2b3f93 blk-mq: allow cha... |
2528 |
return -EINVAL; |
70f36b600 blk-mq: allow res... |
2529 |
blk_mq_freeze_queue(q); |
70f36b600 blk-mq: allow res... |
2530 |
|
e3a2b3f93 blk-mq: allow cha... |
2531 2532 |
ret = 0; queue_for_each_hw_ctx(q, hctx, i) { |
e9137d4b9 blk-mq: Fix NULL ... |
2533 2534 |
if (!hctx->tags) continue; |
bd166ef18 blk-mq-sched: add... |
2535 2536 2537 2538 |
/* * If we're using an MQ scheduler, just update the scheduler * queue depth. This is similar to what the old code would do. */ |
70f36b600 blk-mq: allow res... |
2539 2540 2541 2542 2543 2544 2545 2546 |
if (!hctx->sched_tags) { ret = blk_mq_tag_update_depth(hctx, &hctx->tags, min(nr, set->queue_depth), false); } else { ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags, nr, true); } |
e3a2b3f93 blk-mq: allow cha... |
2547 2548 2549 2550 2551 2552 |
if (ret) break; } if (!ret) q->nr_requests = nr; |
70f36b600 blk-mq: allow res... |
2553 |
blk_mq_unfreeze_queue(q); |
70f36b600 blk-mq: allow res... |
2554 |
|
e3a2b3f93 blk-mq: allow cha... |
2555 2556 |
return ret; } |
e4dc2b32d blk-mq: Take tags... |
2557 2558 |
static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) |
868f2f0b7 blk-mq: dynamic h... |
2559 2560 |
{ struct request_queue *q; |
705cda97e blk-mq: Make it s... |
2561 |
lockdep_assert_held(&set->tag_list_lock); |
868f2f0b7 blk-mq: dynamic h... |
2562 2563 2564 2565 2566 2567 2568 2569 2570 |
if (nr_hw_queues > nr_cpu_ids) nr_hw_queues = nr_cpu_ids; if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues) return; list_for_each_entry(q, &set->tag_list, tag_set_list) blk_mq_freeze_queue(q); set->nr_hw_queues = nr_hw_queues; |
ebe8bddb6 blk-mq: remap que... |
2571 |
blk_mq_update_queue_map(set); |
868f2f0b7 blk-mq: dynamic h... |
2572 2573 |
list_for_each_entry(q, &set->tag_list, tag_set_list) { blk_mq_realloc_hw_ctxs(set, q); |
4b855ad37 blk-mq: Create hc... |
2574 |
blk_mq_queue_reinit(q); |
868f2f0b7 blk-mq: dynamic h... |
2575 2576 2577 2578 2579 |
} list_for_each_entry(q, &set->tag_list, tag_set_list) blk_mq_unfreeze_queue(q); } |
e4dc2b32d blk-mq: Take tags... |
2580 2581 2582 2583 2584 2585 2586 |
void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues) { mutex_lock(&set->tag_list_lock); __blk_mq_update_nr_hw_queues(set, nr_hw_queues); mutex_unlock(&set->tag_list_lock); } |
868f2f0b7 blk-mq: dynamic h... |
2587 |
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues); |
34dbad5d2 blk-stat: convert... |
2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 |
/* Enable polling stats and return whether they were already enabled. */ static bool blk_poll_stats_enable(struct request_queue *q) { if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || test_and_set_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags)) return true; blk_stat_add_callback(q, q->poll_cb); return false; } static void blk_mq_poll_stats_start(struct request_queue *q) { /* * We don't arm the callback if polling stats are not enabled or the * callback is already active. */ if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) || blk_stat_is_active(q->poll_cb)) return; blk_stat_activate_msecs(q->poll_cb, 100); } static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb) { struct request_queue *q = cb->data; |
720b8ccc4 blk-mq: Add a pol... |
2614 |
int bucket; |
34dbad5d2 blk-stat: convert... |
2615 |
|
720b8ccc4 blk-mq: Add a pol... |
2616 2617 2618 2619 |
for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) { if (cb->stat[bucket].nr_samples) q->poll_stat[bucket] = cb->stat[bucket]; } |
34dbad5d2 blk-stat: convert... |
2620 |
} |
64f1c21e8 blk-mq: make the ... |
2621 2622 2623 2624 |
static unsigned long blk_mq_poll_nsecs(struct request_queue *q, struct blk_mq_hw_ctx *hctx, struct request *rq) { |
64f1c21e8 blk-mq: make the ... |
2625 |
unsigned long ret = 0; |
720b8ccc4 blk-mq: Add a pol... |
2626 |
int bucket; |
64f1c21e8 blk-mq: make the ... |
2627 2628 2629 2630 2631 |
/* * If stats collection isn't on, don't sleep but turn it on for * future users */ |
34dbad5d2 blk-stat: convert... |
2632 |
if (!blk_poll_stats_enable(q)) |
64f1c21e8 blk-mq: make the ... |
2633 2634 2635 |
return 0; /* |
64f1c21e8 blk-mq: make the ... |
2636 2637 2638 2639 2640 |
* As an optimistic guess, use half of the mean service time * for this type of request. We can (and should) make this smarter. * For instance, if the completion latencies are tight, we can * get closer than just half the mean. This is especially * important on devices where the completion latencies are longer |
720b8ccc4 blk-mq: Add a pol... |
2641 2642 |
* than ~10 usec. We do use the stats for the relevant IO size * if available which does lead to better estimates. |
64f1c21e8 blk-mq: make the ... |
2643 |
*/ |
720b8ccc4 blk-mq: Add a pol... |
2644 2645 2646 2647 2648 2649 |
bucket = blk_mq_poll_stats_bkt(rq); if (bucket < 0) return ret; if (q->poll_stat[bucket].nr_samples) ret = (q->poll_stat[bucket].mean + 1) / 2; |
64f1c21e8 blk-mq: make the ... |
2650 2651 2652 |
return ret; } |
06426adf0 blk-mq: implement... |
2653 |
static bool blk_mq_poll_hybrid_sleep(struct request_queue *q, |
64f1c21e8 blk-mq: make the ... |
2654 |
struct blk_mq_hw_ctx *hctx, |
06426adf0 blk-mq: implement... |
2655 2656 2657 2658 |
struct request *rq) { struct hrtimer_sleeper hs; enum hrtimer_mode mode; |
64f1c21e8 blk-mq: make the ... |
2659 |
unsigned int nsecs; |
06426adf0 blk-mq: implement... |
2660 |
ktime_t kt; |
64f1c21e8 blk-mq: make the ... |
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 |
if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags)) return false; /* * poll_nsec can be: * * -1: don't ever hybrid sleep * 0: use half of prev avg * >0: use this specific value */ if (q->poll_nsec == -1) return false; else if (q->poll_nsec > 0) nsecs = q->poll_nsec; else nsecs = blk_mq_poll_nsecs(q, hctx, rq); if (!nsecs) |
06426adf0 blk-mq: implement... |
2679 2680 2681 2682 2683 2684 2685 2686 |
return false; set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags); /* * This will be replaced with the stats tracking code, using * 'avg_completion_time / 2' as the pre-sleep target. */ |
8b0e19531 ktime: Cleanup kt... |
2687 |
kt = nsecs; |
06426adf0 blk-mq: implement... |
2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 |
mode = HRTIMER_MODE_REL; hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode); hrtimer_set_expires(&hs.timer, kt); hrtimer_init_sleeper(&hs, current); do { if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) break; set_current_state(TASK_UNINTERRUPTIBLE); hrtimer_start_expires(&hs.timer, mode); if (hs.task) io_schedule(); hrtimer_cancel(&hs.timer); mode = HRTIMER_MODE_ABS; } while (hs.task && !signal_pending(current)); __set_current_state(TASK_RUNNING); destroy_hrtimer_on_stack(&hs.timer); return true; } |
bbd7bb701 block: move poll ... |
2709 2710 2711 2712 |
static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq) { struct request_queue *q = hctx->queue; long state; |
06426adf0 blk-mq: implement... |
2713 2714 2715 2716 2717 2718 2719 |
/* * If we sleep, have the caller restart the poll loop to reset * the state. Like for the other success return cases, the * caller is responsible for checking if the IO completed. If * the IO isn't complete, we'll get called again and will go * straight to the busy poll loop. */ |
64f1c21e8 blk-mq: make the ... |
2720 |
if (blk_mq_poll_hybrid_sleep(q, hctx, rq)) |
06426adf0 blk-mq: implement... |
2721 |
return true; |
bbd7bb701 block: move poll ... |
2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 |
hctx->poll_considered++; state = current->state; while (!need_resched()) { int ret; hctx->poll_invoked++; ret = q->mq_ops->poll(hctx, rq->tag); if (ret > 0) { hctx->poll_success++; set_current_state(TASK_RUNNING); return true; } if (signal_pending_state(state, current)) set_current_state(TASK_RUNNING); if (current->state == TASK_RUNNING) return true; if (ret < 0) break; cpu_relax(); } return false; } |
45cde2f81 block: add a poll... |
2749 |
static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) |
bbd7bb701 block: move poll ... |
2750 2751 |
{ struct blk_mq_hw_ctx *hctx; |
bbd7bb701 block: move poll ... |
2752 |
struct request *rq; |
45cde2f81 block: add a poll... |
2753 |
if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags)) |
bbd7bb701 block: move poll ... |
2754 |
return false; |
bbd7bb701 block: move poll ... |
2755 |
hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)]; |
bd166ef18 blk-mq-sched: add... |
2756 2757 |
if (!blk_qc_t_is_internal(cookie)) rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie)); |
3a07bb1d7 blk-mq: fix poten... |
2758 |
else { |
bd166ef18 blk-mq-sched: add... |
2759 |
rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie)); |
3a07bb1d7 blk-mq: fix poten... |
2760 2761 2762 2763 2764 2765 2766 2767 2768 |
/* * With scheduling, if the request has completed, we'll * get a NULL return here, as we clear the sched tag when * that happens. The request still remains valid, like always, * so we should be safe with just the NULL check. */ if (!rq) return false; } |
bbd7bb701 block: move poll ... |
2769 2770 2771 |
return __blk_mq_poll(hctx, rq); } |
bbd7bb701 block: move poll ... |
2772 |
|
320ae51fe blk-mq: new multi... |
2773 2774 |
static int __init blk_mq_init(void) { |
9467f8596 blk-mq/cpu-notif:... |
2775 2776 |
cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, blk_mq_hctx_notify_dead); |
320ae51fe blk-mq: new multi... |
2777 2778 2779 |
return 0; } subsys_initcall(blk_mq_init); |