Blame view
block/blk.h
13.3 KB
b24413180 License cleanup: ... |
1 |
/* SPDX-License-Identifier: GPL-2.0 */ |
8324aa91d block: split tag ... |
2 3 |
#ifndef BLK_INTERNAL_H #define BLK_INTERNAL_H |
a73f730d0 block, cfq: move ... |
4 |
#include <linux/idr.h> |
f70ced091 blk-mq: support p... |
5 |
#include <linux/blk-mq.h> |
c6a564ffa block: move the p... |
6 |
#include <linux/part_stat.h> |
a892c8d52 block: Inline enc... |
7 |
#include <linux/blk-crypto.h> |
c39ae60df block: remove ARC... |
8 |
#include <xen/xen.h> |
a892c8d52 block: Inline enc... |
9 |
#include "blk-crypto-internal.h" |
f70ced091 blk-mq: support p... |
10 |
#include "blk-mq.h" |
c3e221921 block: free sched... |
11 |
#include "blk-mq-sched.h" |
a73f730d0 block, cfq: move ... |
12 |
|
0d2602ca3 blk-mq: improve s... |
13 14 |
/* Max future timer expiry for timeouts */ #define BLK_MAX_TIMEOUT (5 * HZ) |
18fbda91c block: use same b... |
15 |
extern struct dentry *blk_debugfs_root; |
18fbda91c block: use same b... |
16 |
|
7c94e1c15 block: introduce ... |
17 |
struct blk_flush_queue { |
7c94e1c15 block: introduce ... |
18 19 |
unsigned int flush_pending_idx:1; unsigned int flush_running_idx:1; |
8d6996630 block: fix null p... |
20 |
blk_status_t rq_status; |
7c94e1c15 block: introduce ... |
21 22 23 24 |
unsigned long flush_pending_since; struct list_head flush_queue[2]; struct list_head flush_data_in_flight; struct request *flush_rq; |
0048b4837 blk-mq: fix race ... |
25 |
|
b3c6a5997 block: Fix a lock... |
26 |
struct lock_class_key key; |
7c94e1c15 block: introduce ... |
27 28 |
spinlock_t mq_flush_lock; }; |
8324aa91d block: split tag ... |
29 30 |
extern struct kmem_cache *blk_requestq_cachep; extern struct kobj_type blk_queue_ktype; |
a73f730d0 block, cfq: move ... |
31 |
extern struct ida blk_queue_ida; |
8324aa91d block: split tag ... |
32 |
|
f9afca4d3 blk-mq: pass in r... |
33 34 |
static inline struct blk_flush_queue * blk_get_flush_queue(struct request_queue *q, struct blk_mq_ctx *ctx) |
7c94e1c15 block: introduce ... |
35 |
{ |
8ccdf4a37 blk-mq: save queu... |
36 |
return blk_mq_map_queue(q, REQ_OP_FLUSH, ctx)->fq; |
7c94e1c15 block: introduce ... |
37 |
} |
09ac46c42 block: misc updat... |
38 39 40 41 |
static inline void __blk_get_queue(struct request_queue *q) { kobject_get(&q->kobj); } |
8d6996630 block: fix null p... |
42 43 44 45 46 |
static inline bool is_flush_rq(struct request *req, struct blk_mq_hw_ctx *hctx) { return hctx->fq->flush_rq == req; } |
754a15726 block: remove unn... |
47 48 |
struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size, gfp_t flags); |
f70ced091 blk-mq: support p... |
49 |
void blk_free_flush_queue(struct blk_flush_queue *q); |
f35526557 block: introduce ... |
50 |
|
3ef28e83a block: generic re... |
51 |
void blk_freeze_queue(struct request_queue *q); |
3dccdae54 block: merge BIOV... |
52 53 |
static inline bool biovec_phys_mergeable(struct request_queue *q, struct bio_vec *vec1, struct bio_vec *vec2) |
6a9f5f240 block: simplify B... |
54 |
{ |
3dccdae54 block: merge BIOV... |
55 |
unsigned long mask = queue_segment_boundary(q); |
6e768461c block: remove bve... |
56 57 |
phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset; phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset; |
3dccdae54 block: merge BIOV... |
58 59 |
if (addr1 + vec1->bv_len != addr2) |
6a9f5f240 block: simplify B... |
60 |
return false; |
0383ad437 block: pass page ... |
61 |
if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) |
6a9f5f240 block: simplify B... |
62 |
return false; |
3dccdae54 block: merge BIOV... |
63 64 |
if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) return false; |
6a9f5f240 block: simplify B... |
65 66 |
return true; } |
27ca1d4ed block: move req_g... |
67 68 69 |
static inline bool __bvec_gap_to_prev(struct request_queue *q, struct bio_vec *bprv, unsigned int offset) { |
df376b2ed block: respect vi... |
70 |
return (offset & queue_virt_boundary(q)) || |
27ca1d4ed block: move req_g... |
71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); } /* * Check if adding a bio_vec after bprv with offset would create a gap in * the SG list. Most drivers don't care about this, but some do. */ static inline bool bvec_gap_to_prev(struct request_queue *q, struct bio_vec *bprv, unsigned int offset) { if (!queue_virt_boundary(q)) return false; return __bvec_gap_to_prev(q, bprv, offset); } |
1aa0a133f block: mark blk_r... |
85 86 87 88 89 90 91 92 93 94 95 |
static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio, unsigned int nr_segs) { rq->nr_phys_segments = nr_segs; rq->__data_len = bio->bi_iter.bi_size; rq->bio = rq->biotail = bio; rq->ioprio = bio_prio(bio); if (bio->bi_disk) rq->rq_disk = bio->bi_disk; } |
5a48fc147 block: blk_flush_... |
96 97 |
#ifdef CONFIG_BLK_DEV_INTEGRITY void blk_flush_integrity(void); |
7c20f1168 bio-integrity: st... |
98 |
bool __bio_integrity_endio(struct bio *); |
ece841abb block: fix memlea... |
99 |
void bio_integrity_free(struct bio *bio); |
7c20f1168 bio-integrity: st... |
100 101 102 103 104 105 |
static inline bool bio_integrity_endio(struct bio *bio) { if (bio_integrity(bio)) return __bio_integrity_endio(bio); return true; } |
43b729bfe block: move integ... |
106 |
|
92cf2fd15 block: remove the... |
107 108 |
bool blk_integrity_merge_rq(struct request_queue *, struct request *, struct request *); |
d59da4199 block: remove the... |
109 110 |
bool blk_integrity_merge_bio(struct request_queue *, struct request *, struct bio *); |
92cf2fd15 block: remove the... |
111 |
|
43b729bfe block: move integ... |
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 |
static inline bool integrity_req_gap_back_merge(struct request *req, struct bio *next) { struct bio_integrity_payload *bip = bio_integrity(req->bio); struct bio_integrity_payload *bip_next = bio_integrity(next); return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], bip_next->bip_vec[0].bv_offset); } static inline bool integrity_req_gap_front_merge(struct request *req, struct bio *bio) { struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_integrity_payload *bip_next = bio_integrity(req->bio); return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], bip_next->bip_vec[0].bv_offset); } |
581e26004 block: move block... |
131 132 133 |
void blk_integrity_add(struct gendisk *); void blk_integrity_del(struct gendisk *); |
43b729bfe block: move integ... |
134 |
#else /* CONFIG_BLK_DEV_INTEGRITY */ |
92cf2fd15 block: remove the... |
135 136 137 138 139 |
static inline bool blk_integrity_merge_rq(struct request_queue *rq, struct request *r1, struct request *r2) { return true; } |
d59da4199 block: remove the... |
140 141 142 143 144 |
static inline bool blk_integrity_merge_bio(struct request_queue *rq, struct request *r, struct bio *b) { return true; } |
43b729bfe block: move integ... |
145 146 147 148 149 150 151 152 153 154 |
static inline bool integrity_req_gap_back_merge(struct request *req, struct bio *next) { return false; } static inline bool integrity_req_gap_front_merge(struct request *req, struct bio *bio) { return false; } |
5a48fc147 block: blk_flush_... |
155 156 157 |
static inline void blk_flush_integrity(void) { } |
7c20f1168 bio-integrity: st... |
158 159 160 161 |
static inline bool bio_integrity_endio(struct bio *bio) { return true; } |
ece841abb block: fix memlea... |
162 163 164 |
static inline void bio_integrity_free(struct bio *bio) { } |
581e26004 block: move block... |
165 166 167 168 169 170 |
static inline void blk_integrity_add(struct gendisk *disk) { } static inline void blk_integrity_del(struct gendisk *disk) { } |
43b729bfe block: move integ... |
171 |
#endif /* CONFIG_BLK_DEV_INTEGRITY */ |
8324aa91d block: split tag ... |
172 |
|
0d2602ca3 blk-mq: improve s... |
173 |
unsigned long blk_rq_timeout(unsigned long timeout); |
87ee7b112 blk-mq: fix race ... |
174 |
void blk_add_timer(struct request *req); |
320ae51fe blk-mq: new multi... |
175 |
|
320ae51fe blk-mq: new multi... |
176 |
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, |
14ccb66b3 block: remove the... |
177 |
unsigned int nr_segs, struct request **same_queue_rq); |
bdc6a287b block: Move blk_m... |
178 179 |
bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, struct bio *bio, unsigned int nr_segs); |
320ae51fe blk-mq: new multi... |
180 |
|
b5af37ab3 block: add a blk_... |
181 |
void blk_account_io_start(struct request *req); |
522a77756 block: consolidat... |
182 |
void blk_account_io_done(struct request *req, u64 now); |
320ae51fe blk-mq: new multi... |
183 |
|
242f9dcb8 block: unify requ... |
184 |
/* |
158dbda00 block: reorganize... |
185 186 |
* Internal elevator interface */ |
e80640213 block: split out ... |
187 |
#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED) |
158dbda00 block: reorganize... |
188 |
|
ae1b15396 block: reimplemen... |
189 |
void blk_insert_flush(struct request *rq); |
dd831006d block: misc clean... |
190 |
|
954b4a5ce block: Change ele... |
191 |
void elevator_init_mq(struct request_queue *q); |
d48ece209 blk-mq: init hctx... |
192 193 |
int elevator_switch_mq(struct request_queue *q, struct elevator_type *new_e); |
c3e221921 block: free sched... |
194 |
void __elevator_exit(struct request_queue *, struct elevator_queue *); |
cecf5d87f block: split .sys... |
195 |
int elv_register_queue(struct request_queue *q, bool uevent); |
83d016ac8 block: Unexport e... |
196 |
void elv_unregister_queue(struct request_queue *q); |
c3e221921 block: free sched... |
197 198 199 |
static inline void elevator_exit(struct request_queue *q, struct elevator_queue *e) { |
284b94be1 blk-mq: move lock... |
200 |
lockdep_assert_held(&q->sysfs_lock); |
c3e221921 block: free sched... |
201 202 203 |
blk_mq_sched_free_requests(q); __elevator_exit(q, e); } |
807d4af2f block: add a __di... |
204 |
struct hd_struct *__disk_get_part(struct gendisk *disk, int partno); |
3ad5cee5c block: move sysfs... |
205 206 207 208 209 210 211 212 213 214 |
ssize_t part_size_show(struct device *dev, struct device_attribute *attr, char *buf); ssize_t part_stat_show(struct device *dev, struct device_attribute *attr, char *buf); ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr, char *buf); ssize_t part_fail_show(struct device *dev, struct device_attribute *attr, char *buf); ssize_t part_fail_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count); |
581d4e28d block: add fault ... |
215 216 217 |
ssize_t part_timeout_show(struct device *, struct device_attribute *, char *); ssize_t part_timeout_store(struct device *, struct device_attribute *, const char *, size_t); |
581d4e28d block: add fault ... |
218 |
|
f695ca388 block: remove the... |
219 |
void __blk_queue_split(struct bio **bio, unsigned int *nr_segs); |
14ccb66b3 block: remove the... |
220 221 |
int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs); |
5e84ea3a9 block: attempt to... |
222 223 |
int blk_attempt_req_merge(struct request_queue *q, struct request *rq, struct request *next); |
e9cd19c0c block: simplify b... |
224 |
unsigned int blk_recalc_rq_segments(struct request *rq); |
80a761fd3 block: implement ... |
225 |
void blk_rq_set_mixed_merge(struct request *rq); |
050c8ea80 block: separate o... |
226 |
bool blk_rq_merge_ok(struct request *rq, struct bio *bio); |
34fe7c054 block: enumify EL... |
227 |
enum elv_merge blk_try_merge(struct request *rq, struct bio *bio); |
d6d481969 block: ll_rw_blk.... |
228 |
|
ff88972c8 proper prototype ... |
229 |
int blk_dev_init(void); |
c2553b584 block: make blk_d... |
230 231 232 233 |
/* * Contribute to IO statistics IFF: * * a) it's attached to a gendisk, and |
48d9b0d43 block: account st... |
234 |
* b) the queue had IO stats enabled when this request was started |
c2553b584 block: make blk_d... |
235 |
*/ |
599d067dd block: change ret... |
236 |
static inline bool blk_do_io_stat(struct request *rq) |
fb8ec18c3 block: fix oops i... |
237 |
{ |
48d9b0d43 block: account st... |
238 |
return rq->rq_disk && (rq->rq_flags & RQF_IO_STAT); |
fb8ec18c3 block: fix oops i... |
239 |
} |
6cf7677f1 block: move req_s... |
240 241 242 243 244 245 |
static inline void req_set_nomerge(struct request_queue *q, struct request *req) { req->cmd_flags |= REQ_NOMERGE; if (req == q->last_merge) q->last_merge = NULL; } |
f2dbd76a0 block, cfq: repla... |
246 |
/* |
1adfc5e41 block: make sure ... |
247 248 249 250 251 252 253 254 255 256 |
* The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size * is defined as 'unsigned int', meantime it has to aligned to with logical * block size which is the minimum accepted unit by hardware. */ static inline unsigned int bio_allowed_max_sectors(struct request_queue *q) { return round_down(UINT_MAX, queue_logical_block_size(q)) >> 9; } /* |
9b15d109a block: improve di... |
257 258 259 260 261 262 263 264 265 266 267 268 269 270 |
* The max bio size which is aligned to q->limits.discard_granularity. This * is a hint to split large discard bio in generic block layer, then if device * driver needs to split the discard bio into smaller ones, their bi_size can * be very probably and easily aligned to discard_granularity of the device's * queue. */ static inline unsigned int bio_aligned_discard_max_sectors( struct request_queue *q) { return round_down(UINT_MAX, q->limits.discard_granularity) >> SECTOR_SHIFT; } /* |
f2dbd76a0 block, cfq: repla... |
271 272 273 |
* Internal io_context interface */ void get_io_context(struct io_context *ioc); |
47fdd4ca9 block, cfq: move ... |
274 |
struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q); |
24acfc34f block: interface ... |
275 276 |
struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q, gfp_t gfp_mask); |
7e5a87944 block, cfq: move ... |
277 |
void ioc_clear_queue(struct request_queue *q); |
f2dbd76a0 block, cfq: repla... |
278 |
|
24acfc34f block: interface ... |
279 |
int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node); |
f2dbd76a0 block, cfq: repla... |
280 |
|
f2dbd76a0 block, cfq: repla... |
281 282 283 |
/* * Internal throttling interface */ |
bc9fcbf9c block: move blk_t... |
284 |
#ifdef CONFIG_BLK_DEV_THROTTLING |
bc9fcbf9c block: move blk_t... |
285 286 |
extern int blk_throtl_init(struct request_queue *q); extern void blk_throtl_exit(struct request_queue *q); |
d61fcfa4b blk-throttle: cho... |
287 |
extern void blk_throtl_register_queue(struct request_queue *q); |
db18a53e5 blk-cgroup: remov... |
288 |
bool blk_throtl_bio(struct bio *bio); |
bc9fcbf9c block: move blk_t... |
289 |
#else /* CONFIG_BLK_DEV_THROTTLING */ |
bc9fcbf9c block: move blk_t... |
290 291 |
static inline int blk_throtl_init(struct request_queue *q) { return 0; } static inline void blk_throtl_exit(struct request_queue *q) { } |
d61fcfa4b blk-throttle: cho... |
292 |
static inline void blk_throtl_register_queue(struct request_queue *q) { } |
db18a53e5 blk-cgroup: remov... |
293 |
static inline bool blk_throtl_bio(struct bio *bio) { return false; } |
bc9fcbf9c block: move blk_t... |
294 |
#endif /* CONFIG_BLK_DEV_THROTTLING */ |
297e3d854 blk-throttle: mak... |
295 296 297 298 |
#ifdef CONFIG_BLK_DEV_THROTTLING_LOW extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page); extern ssize_t blk_throtl_sample_time_store(struct request_queue *q, const char *page, size_t count); |
9e234eeaf blk-throttle: add... |
299 |
extern void blk_throtl_bio_endio(struct bio *bio); |
b9147dd1b blk-throttle: add... |
300 |
extern void blk_throtl_stat_add(struct request *rq, u64 time); |
9e234eeaf blk-throttle: add... |
301 302 |
#else static inline void blk_throtl_bio_endio(struct bio *bio) { } |
b9147dd1b blk-throttle: add... |
303 |
static inline void blk_throtl_stat_add(struct request *rq, u64 time) { } |
297e3d854 blk-throttle: mak... |
304 |
#endif |
bc9fcbf9c block: move blk_t... |
305 |
|
3bce016a4 block: move bounc... |
306 307 308 309 310 311 312 313 314 315 316 317 |
#ifdef CONFIG_BOUNCE extern int init_emergency_isa_pool(void); extern void blk_queue_bounce(struct request_queue *q, struct bio **bio); #else static inline int init_emergency_isa_pool(void) { return 0; } static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio) { } #endif /* CONFIG_BOUNCE */ |
d70675121 block: introduce ... |
318 319 320 321 322 |
#ifdef CONFIG_BLK_CGROUP_IOLATENCY extern int blk_iolatency_init(struct request_queue *q); #else static inline int blk_iolatency_init(struct request_queue *q) { return 0; } #endif |
a2d6b3a2d block: Improve zo... |
323 |
struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp); |
bf5054569 block: Introduce ... |
324 325 326 327 328 |
#ifdef CONFIG_BLK_DEV_ZONED void blk_queue_free_zone_bitmaps(struct request_queue *q); #else static inline void blk_queue_free_zone_bitmaps(struct request_queue *q) {} #endif |
581e26004 block: move block... |
329 330 331 332 333 334 335 336 337 |
struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector); int blk_alloc_devt(struct hd_struct *part, dev_t *devt); void blk_free_devt(dev_t devt); void blk_invalidate_devt(dev_t devt); char *disk_name(struct gendisk *hd, int partno, char *buf); #define ADDPART_FLAG_NONE 0 #define ADDPART_FLAG_RAID 1 #define ADDPART_FLAG_WHOLEDISK 2 |
8328eb283 block: remove the... |
338 |
void delete_partition(struct hd_struct *part); |
fa9156ae5 block: refactor b... |
339 340 341 342 343 |
int bdev_add_partition(struct block_device *bdev, int partno, sector_t start, sector_t length); int bdev_del_partition(struct block_device *bdev, int partno); int bdev_resize_partition(struct block_device *bdev, int partno, sector_t start, sector_t length); |
581e26004 block: move block... |
344 |
int disk_expand_part_tbl(struct gendisk *disk, int target); |
8da2892e2 block: cleanup hd... |
345 |
int hd_ref_init(struct hd_struct *part); |
581e26004 block: move block... |
346 |
|
27eb3af9a block: don't hold... |
347 |
/* no need to get/put refcount of part0 */ |
581e26004 block: move block... |
348 349 |
static inline int hd_struct_try_get(struct hd_struct *part) { |
27eb3af9a block: don't hold... |
350 351 352 |
if (part->partno) return percpu_ref_tryget_live(&part->ref); return 1; |
581e26004 block: move block... |
353 354 355 356 |
} static inline void hd_struct_put(struct hd_struct *part) { |
27eb3af9a block: don't hold... |
357 358 |
if (part->partno) percpu_ref_put(&part->ref); |
581e26004 block: move block... |
359 |
} |
581e26004 block: move block... |
360 361 |
static inline void hd_free_part(struct hd_struct *part) { |
58d4f14fc block: always use... |
362 |
free_percpu(part->dkstats); |
581e26004 block: move block... |
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 |
kfree(part->info); percpu_ref_exit(&part->ref); } /* * Any access of part->nr_sects which is not protected by partition * bd_mutex or gendisk bdev bd_mutex, should be done using this * accessor function. * * Code written along the lines of i_size_read() and i_size_write(). * CONFIG_PREEMPTION case optimizes the case of UP kernel with preemption * on. */ static inline sector_t part_nr_sects_read(struct hd_struct *part) { #if BITS_PER_LONG==32 && defined(CONFIG_SMP) sector_t nr_sects; unsigned seq; do { seq = read_seqcount_begin(&part->nr_sects_seq); nr_sects = part->nr_sects; } while (read_seqcount_retry(&part->nr_sects_seq, seq)); return nr_sects; #elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) sector_t nr_sects; preempt_disable(); nr_sects = part->nr_sects; preempt_enable(); return nr_sects; #else return part->nr_sects; #endif } /* * Should be called with mutex lock held (typically bd_mutex) of partition * to provide mutual exlusion among writers otherwise seqcount might be * left in wrong state leaving the readers spinning infinitely. */ static inline void part_nr_sects_write(struct hd_struct *part, sector_t size) { #if BITS_PER_LONG==32 && defined(CONFIG_SMP) |
15b81ce5a block: nr_sects_w... |
406 |
preempt_disable(); |
581e26004 block: move block... |
407 408 409 |
write_seqcount_begin(&part->nr_sects_seq); part->nr_sects = size; write_seqcount_end(&part->nr_sects_seq); |
15b81ce5a block: nr_sects_w... |
410 |
preempt_enable(); |
581e26004 block: move block... |
411 412 413 414 415 416 417 418 |
#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) preempt_disable(); part->nr_sects = size; preempt_enable(); #else part->nr_sects = size; #endif } |
e45811057 block: rename __b... |
419 |
int bio_add_hw_page(struct request_queue *q, struct bio *bio, |
130879f1e block: move bio_m... |
420 |
struct page *page, unsigned int len, unsigned int offset, |
e45811057 block: rename __b... |
421 |
unsigned int max_sectors, bool *same_page); |
130879f1e block: move bio_m... |
422 |
|
bc9fcbf9c block: move blk_t... |
423 |
#endif /* BLK_INTERNAL_H */ |