Blame view
block/blk-rq-qos.h
4.96 KB
3dcf60bcb
|
1 |
/* SPDX-License-Identifier: GPL-2.0 */ |
a79050434
|
2 3 4 5 6 7 8 9 |
#ifndef RQ_QOS_H #define RQ_QOS_H #include <linux/kernel.h> #include <linux/blkdev.h> #include <linux/blk_types.h> #include <linux/atomic.h> #include <linux/wait.h> |
cc56694f1
|
10 11 12 |
#include "blk-mq-debugfs.h" struct blk_mq_debugfs_attr; |
a79050434
|
13 14 |
enum rq_qos_id { RQ_QOS_WBT, |
beab17fc2
|
15 |
RQ_QOS_LATENCY, |
7caa47151
|
16 |
RQ_QOS_COST, |
a79050434
|
17 18 19 20 21 22 23 24 25 26 27 28 |
}; struct rq_wait { wait_queue_head_t wait; atomic_t inflight; }; struct rq_qos { struct rq_qos_ops *ops; struct request_queue *q; enum rq_qos_id id; struct rq_qos *next; |
cc56694f1
|
29 30 31 |
#ifdef CONFIG_BLK_DEBUG_FS struct dentry *debugfs_dir; #endif |
a79050434
|
32 33 34 |
}; struct rq_qos_ops { |
d53375608
|
35 |
void (*throttle)(struct rq_qos *, struct bio *); |
c1c80384c
|
36 |
void (*track)(struct rq_qos *, struct request *, struct bio *); |
d3e65ffff
|
37 |
void (*merge)(struct rq_qos *, struct request *, struct bio *); |
a79050434
|
38 39 40 |
void (*issue)(struct rq_qos *, struct request *); void (*requeue)(struct rq_qos *, struct request *); void (*done)(struct rq_qos *, struct request *); |
67b42d0bf
|
41 |
void (*done_bio)(struct rq_qos *, struct bio *); |
c1c80384c
|
42 |
void (*cleanup)(struct rq_qos *, struct bio *); |
9677a3e01
|
43 |
void (*queue_depth_changed)(struct rq_qos *); |
a79050434
|
44 |
void (*exit)(struct rq_qos *); |
cc56694f1
|
45 |
const struct blk_mq_debugfs_attr *debugfs_attrs; |
a79050434
|
46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
}; struct rq_depth { unsigned int max_depth; int scale_step; bool scaled_max; unsigned int queue_depth; unsigned int default_depth; }; static inline struct rq_qos *rq_qos_id(struct request_queue *q, enum rq_qos_id id) { struct rq_qos *rqos; for (rqos = q->rq_qos; rqos; rqos = rqos->next) { if (rqos->id == id) break; } return rqos; } static inline struct rq_qos *wbt_rq_qos(struct request_queue *q) { return rq_qos_id(q, RQ_QOS_WBT); } static inline struct rq_qos *blkcg_rq_qos(struct request_queue *q) { |
beab17fc2
|
76 |
return rq_qos_id(q, RQ_QOS_LATENCY); |
a79050434
|
77 |
} |
cc56694f1
|
78 79 80 81 82 |
static inline const char *rq_qos_id_to_name(enum rq_qos_id id) { switch (id) { case RQ_QOS_WBT: return "wbt"; |
beab17fc2
|
83 84 |
case RQ_QOS_LATENCY: return "latency"; |
7caa47151
|
85 86 |
case RQ_QOS_COST: return "cost"; |
cc56694f1
|
87 88 89 |
} return "unknown"; } |
a79050434
|
90 91 92 93 94 95 96 97 98 99 |
static inline void rq_wait_init(struct rq_wait *rq_wait) { atomic_set(&rq_wait->inflight, 0); init_waitqueue_head(&rq_wait->wait); } static inline void rq_qos_add(struct request_queue *q, struct rq_qos *rqos) { rqos->next = q->rq_qos; q->rq_qos = rqos; |
cc56694f1
|
100 101 102 |
if (rqos->ops->debugfs_attrs) blk_mq_debugfs_register_rqos(rqos); |
a79050434
|
103 104 105 106 |
} static inline void rq_qos_del(struct request_queue *q, struct rq_qos *rqos) { |
307f4065b
|
107 108 109 110 111 |
struct rq_qos **cur; for (cur = &q->rq_qos; *cur; cur = &(*cur)->next) { if (*cur == rqos) { *cur = rqos->next; |
a79050434
|
112 113 |
break; } |
a79050434
|
114 |
} |
cc56694f1
|
115 116 |
blk_mq_debugfs_unregister_rqos(rqos); |
a79050434
|
117 |
} |
84f603246
|
118 119 120 121 122 123 |
typedef bool (acquire_inflight_cb_t)(struct rq_wait *rqw, void *private_data); typedef void (cleanup_cb_t)(struct rq_wait *rqw, void *private_data); void rq_qos_wait(struct rq_wait *rqw, void *private_data, acquire_inflight_cb_t *acquire_inflight_cb, cleanup_cb_t *cleanup_cb); |
22f17952c
|
124 |
bool rq_wait_inc_below(struct rq_wait *rq_wait, unsigned int limit); |
b84477d3e
|
125 126 |
bool rq_depth_scale_up(struct rq_depth *rqd); bool rq_depth_scale_down(struct rq_depth *rqd, bool hard_throttle); |
a79050434
|
127 |
bool rq_depth_calc_max_depth(struct rq_depth *rqd); |
e50454544
|
128 129 130 131 132 133 |
void __rq_qos_cleanup(struct rq_qos *rqos, struct bio *bio); void __rq_qos_done(struct rq_qos *rqos, struct request *rq); void __rq_qos_issue(struct rq_qos *rqos, struct request *rq); void __rq_qos_requeue(struct rq_qos *rqos, struct request *rq); void __rq_qos_throttle(struct rq_qos *rqos, struct bio *bio); void __rq_qos_track(struct rq_qos *rqos, struct request *rq, struct bio *bio); |
d3e65ffff
|
134 |
void __rq_qos_merge(struct rq_qos *rqos, struct request *rq, struct bio *bio); |
e50454544
|
135 |
void __rq_qos_done_bio(struct rq_qos *rqos, struct bio *bio); |
9677a3e01
|
136 |
void __rq_qos_queue_depth_changed(struct rq_qos *rqos); |
e50454544
|
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
static inline void rq_qos_cleanup(struct request_queue *q, struct bio *bio) { if (q->rq_qos) __rq_qos_cleanup(q->rq_qos, bio); } static inline void rq_qos_done(struct request_queue *q, struct request *rq) { if (q->rq_qos) __rq_qos_done(q->rq_qos, rq); } static inline void rq_qos_issue(struct request_queue *q, struct request *rq) { if (q->rq_qos) __rq_qos_issue(q->rq_qos, rq); } static inline void rq_qos_requeue(struct request_queue *q, struct request *rq) { if (q->rq_qos) __rq_qos_requeue(q->rq_qos, rq); } static inline void rq_qos_done_bio(struct request_queue *q, struct bio *bio) { if (q->rq_qos) __rq_qos_done_bio(q->rq_qos, bio); } static inline void rq_qos_throttle(struct request_queue *q, struct bio *bio) { |
13369816c
|
170 171 172 173 174 |
/* * BIO_TRACKED lets controllers know that a bio went through the * normal rq_qos path. */ bio_set_flag(bio, BIO_TRACKED); |
e50454544
|
175 176 177 178 179 180 181 182 183 184 |
if (q->rq_qos) __rq_qos_throttle(q->rq_qos, bio); } static inline void rq_qos_track(struct request_queue *q, struct request *rq, struct bio *bio) { if (q->rq_qos) __rq_qos_track(q->rq_qos, rq, bio); } |
d3e65ffff
|
185 186 187 188 189 190 |
static inline void rq_qos_merge(struct request_queue *q, struct request *rq, struct bio *bio) { if (q->rq_qos) __rq_qos_merge(q->rq_qos, rq, bio); } |
9677a3e01
|
191 192 193 194 195 |
static inline void rq_qos_queue_depth_changed(struct request_queue *q) { if (q->rq_qos) __rq_qos_queue_depth_changed(q->rq_qos); } |
a79050434
|
196 |
void rq_qos_exit(struct request_queue *); |
e50454544
|
197 |
|
a79050434
|
198 |
#endif |