Commit 55c022bbddb2c056b5dff1bd1b1758d31b6d64c9
Committed by
Jens Axboe
1 parent
719c0c5906
Exists in
master
and in
4 other branches
block: avoid building too big plug list
When I test fio script with big I/O depth, I found the total throughput drops compared to some relative small I/O depth. The reason is the thread accumulates big requests in its plug list and causes some delays (surely this depends on CPU speed). I thought we'd better have a threshold for requests. When a threshold reaches, this means there is no request merge and queue lock contention isn't severe when pushing per-task requests to queue, so the main advantages of blk plug don't exist. We can force a plug list flush in this case. With this, my test throughput actually increases and almost equals to small I/O depth. Another side effect is irq off time decreases in blk_flush_plug_list() for big I/O depth. The BLK_MAX_REQUEST_COUNT is choosen arbitarily, but 16 is efficiently to reduce lock contention to me. But I'm open here, 32 is ok in my test too. Signed-off-by: Shaohua Li <shaohua.li@intel.com> Signed-off-by: Jens Axboe <jaxboe@fusionio.com>
Showing 2 changed files with 8 additions and 0 deletions Inline Diff
block/blk-core.c
1 | /* | 1 | /* |
2 | * Copyright (C) 1991, 1992 Linus Torvalds | 2 | * Copyright (C) 1991, 1992 Linus Torvalds |
3 | * Copyright (C) 1994, Karl Keyte: Added support for disk statistics | 3 | * Copyright (C) 1994, Karl Keyte: Added support for disk statistics |
4 | * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE | 4 | * Elevator latency, (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE |
5 | * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de> | 5 | * Queue request tables / lock, selectable elevator, Jens Axboe <axboe@suse.de> |
6 | * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au> | 6 | * kernel-doc documentation started by NeilBrown <neilb@cse.unsw.edu.au> |
7 | * - July2000 | 7 | * - July2000 |
8 | * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001 | 8 | * bio rewrite, highmem i/o, etc, Jens Axboe <axboe@suse.de> - may 2001 |
9 | */ | 9 | */ |
10 | 10 | ||
11 | /* | 11 | /* |
12 | * This handles all read/write requests to block devices | 12 | * This handles all read/write requests to block devices |
13 | */ | 13 | */ |
14 | #include <linux/kernel.h> | 14 | #include <linux/kernel.h> |
15 | #include <linux/module.h> | 15 | #include <linux/module.h> |
16 | #include <linux/backing-dev.h> | 16 | #include <linux/backing-dev.h> |
17 | #include <linux/bio.h> | 17 | #include <linux/bio.h> |
18 | #include <linux/blkdev.h> | 18 | #include <linux/blkdev.h> |
19 | #include <linux/highmem.h> | 19 | #include <linux/highmem.h> |
20 | #include <linux/mm.h> | 20 | #include <linux/mm.h> |
21 | #include <linux/kernel_stat.h> | 21 | #include <linux/kernel_stat.h> |
22 | #include <linux/string.h> | 22 | #include <linux/string.h> |
23 | #include <linux/init.h> | 23 | #include <linux/init.h> |
24 | #include <linux/completion.h> | 24 | #include <linux/completion.h> |
25 | #include <linux/slab.h> | 25 | #include <linux/slab.h> |
26 | #include <linux/swap.h> | 26 | #include <linux/swap.h> |
27 | #include <linux/writeback.h> | 27 | #include <linux/writeback.h> |
28 | #include <linux/task_io_accounting_ops.h> | 28 | #include <linux/task_io_accounting_ops.h> |
29 | #include <linux/fault-inject.h> | 29 | #include <linux/fault-inject.h> |
30 | #include <linux/list_sort.h> | 30 | #include <linux/list_sort.h> |
31 | 31 | ||
32 | #define CREATE_TRACE_POINTS | 32 | #define CREATE_TRACE_POINTS |
33 | #include <trace/events/block.h> | 33 | #include <trace/events/block.h> |
34 | 34 | ||
35 | #include "blk.h" | 35 | #include "blk.h" |
36 | 36 | ||
37 | EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap); | 37 | EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap); |
38 | EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); | 38 | EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap); |
39 | EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); | 39 | EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete); |
40 | 40 | ||
41 | static int __make_request(struct request_queue *q, struct bio *bio); | 41 | static int __make_request(struct request_queue *q, struct bio *bio); |
42 | 42 | ||
43 | /* | 43 | /* |
44 | * For the allocated request tables | 44 | * For the allocated request tables |
45 | */ | 45 | */ |
46 | static struct kmem_cache *request_cachep; | 46 | static struct kmem_cache *request_cachep; |
47 | 47 | ||
48 | /* | 48 | /* |
49 | * For queue allocation | 49 | * For queue allocation |
50 | */ | 50 | */ |
51 | struct kmem_cache *blk_requestq_cachep; | 51 | struct kmem_cache *blk_requestq_cachep; |
52 | 52 | ||
53 | /* | 53 | /* |
54 | * Controlling structure to kblockd | 54 | * Controlling structure to kblockd |
55 | */ | 55 | */ |
56 | static struct workqueue_struct *kblockd_workqueue; | 56 | static struct workqueue_struct *kblockd_workqueue; |
57 | 57 | ||
58 | static void drive_stat_acct(struct request *rq, int new_io) | 58 | static void drive_stat_acct(struct request *rq, int new_io) |
59 | { | 59 | { |
60 | struct hd_struct *part; | 60 | struct hd_struct *part; |
61 | int rw = rq_data_dir(rq); | 61 | int rw = rq_data_dir(rq); |
62 | int cpu; | 62 | int cpu; |
63 | 63 | ||
64 | if (!blk_do_io_stat(rq)) | 64 | if (!blk_do_io_stat(rq)) |
65 | return; | 65 | return; |
66 | 66 | ||
67 | cpu = part_stat_lock(); | 67 | cpu = part_stat_lock(); |
68 | 68 | ||
69 | if (!new_io) { | 69 | if (!new_io) { |
70 | part = rq->part; | 70 | part = rq->part; |
71 | part_stat_inc(cpu, part, merges[rw]); | 71 | part_stat_inc(cpu, part, merges[rw]); |
72 | } else { | 72 | } else { |
73 | part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); | 73 | part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq)); |
74 | if (!hd_struct_try_get(part)) { | 74 | if (!hd_struct_try_get(part)) { |
75 | /* | 75 | /* |
76 | * The partition is already being removed, | 76 | * The partition is already being removed, |
77 | * the request will be accounted on the disk only | 77 | * the request will be accounted on the disk only |
78 | * | 78 | * |
79 | * We take a reference on disk->part0 although that | 79 | * We take a reference on disk->part0 although that |
80 | * partition will never be deleted, so we can treat | 80 | * partition will never be deleted, so we can treat |
81 | * it as any other partition. | 81 | * it as any other partition. |
82 | */ | 82 | */ |
83 | part = &rq->rq_disk->part0; | 83 | part = &rq->rq_disk->part0; |
84 | hd_struct_get(part); | 84 | hd_struct_get(part); |
85 | } | 85 | } |
86 | part_round_stats(cpu, part); | 86 | part_round_stats(cpu, part); |
87 | part_inc_in_flight(part, rw); | 87 | part_inc_in_flight(part, rw); |
88 | rq->part = part; | 88 | rq->part = part; |
89 | } | 89 | } |
90 | 90 | ||
91 | part_stat_unlock(); | 91 | part_stat_unlock(); |
92 | } | 92 | } |
93 | 93 | ||
94 | void blk_queue_congestion_threshold(struct request_queue *q) | 94 | void blk_queue_congestion_threshold(struct request_queue *q) |
95 | { | 95 | { |
96 | int nr; | 96 | int nr; |
97 | 97 | ||
98 | nr = q->nr_requests - (q->nr_requests / 8) + 1; | 98 | nr = q->nr_requests - (q->nr_requests / 8) + 1; |
99 | if (nr > q->nr_requests) | 99 | if (nr > q->nr_requests) |
100 | nr = q->nr_requests; | 100 | nr = q->nr_requests; |
101 | q->nr_congestion_on = nr; | 101 | q->nr_congestion_on = nr; |
102 | 102 | ||
103 | nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1; | 103 | nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1; |
104 | if (nr < 1) | 104 | if (nr < 1) |
105 | nr = 1; | 105 | nr = 1; |
106 | q->nr_congestion_off = nr; | 106 | q->nr_congestion_off = nr; |
107 | } | 107 | } |
108 | 108 | ||
109 | /** | 109 | /** |
110 | * blk_get_backing_dev_info - get the address of a queue's backing_dev_info | 110 | * blk_get_backing_dev_info - get the address of a queue's backing_dev_info |
111 | * @bdev: device | 111 | * @bdev: device |
112 | * | 112 | * |
113 | * Locates the passed device's request queue and returns the address of its | 113 | * Locates the passed device's request queue and returns the address of its |
114 | * backing_dev_info | 114 | * backing_dev_info |
115 | * | 115 | * |
116 | * Will return NULL if the request queue cannot be located. | 116 | * Will return NULL if the request queue cannot be located. |
117 | */ | 117 | */ |
118 | struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev) | 118 | struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev) |
119 | { | 119 | { |
120 | struct backing_dev_info *ret = NULL; | 120 | struct backing_dev_info *ret = NULL; |
121 | struct request_queue *q = bdev_get_queue(bdev); | 121 | struct request_queue *q = bdev_get_queue(bdev); |
122 | 122 | ||
123 | if (q) | 123 | if (q) |
124 | ret = &q->backing_dev_info; | 124 | ret = &q->backing_dev_info; |
125 | return ret; | 125 | return ret; |
126 | } | 126 | } |
127 | EXPORT_SYMBOL(blk_get_backing_dev_info); | 127 | EXPORT_SYMBOL(blk_get_backing_dev_info); |
128 | 128 | ||
129 | void blk_rq_init(struct request_queue *q, struct request *rq) | 129 | void blk_rq_init(struct request_queue *q, struct request *rq) |
130 | { | 130 | { |
131 | memset(rq, 0, sizeof(*rq)); | 131 | memset(rq, 0, sizeof(*rq)); |
132 | 132 | ||
133 | INIT_LIST_HEAD(&rq->queuelist); | 133 | INIT_LIST_HEAD(&rq->queuelist); |
134 | INIT_LIST_HEAD(&rq->timeout_list); | 134 | INIT_LIST_HEAD(&rq->timeout_list); |
135 | rq->cpu = -1; | 135 | rq->cpu = -1; |
136 | rq->q = q; | 136 | rq->q = q; |
137 | rq->__sector = (sector_t) -1; | 137 | rq->__sector = (sector_t) -1; |
138 | INIT_HLIST_NODE(&rq->hash); | 138 | INIT_HLIST_NODE(&rq->hash); |
139 | RB_CLEAR_NODE(&rq->rb_node); | 139 | RB_CLEAR_NODE(&rq->rb_node); |
140 | rq->cmd = rq->__cmd; | 140 | rq->cmd = rq->__cmd; |
141 | rq->cmd_len = BLK_MAX_CDB; | 141 | rq->cmd_len = BLK_MAX_CDB; |
142 | rq->tag = -1; | 142 | rq->tag = -1; |
143 | rq->ref_count = 1; | 143 | rq->ref_count = 1; |
144 | rq->start_time = jiffies; | 144 | rq->start_time = jiffies; |
145 | set_start_time_ns(rq); | 145 | set_start_time_ns(rq); |
146 | rq->part = NULL; | 146 | rq->part = NULL; |
147 | } | 147 | } |
148 | EXPORT_SYMBOL(blk_rq_init); | 148 | EXPORT_SYMBOL(blk_rq_init); |
149 | 149 | ||
150 | static void req_bio_endio(struct request *rq, struct bio *bio, | 150 | static void req_bio_endio(struct request *rq, struct bio *bio, |
151 | unsigned int nbytes, int error) | 151 | unsigned int nbytes, int error) |
152 | { | 152 | { |
153 | if (error) | 153 | if (error) |
154 | clear_bit(BIO_UPTODATE, &bio->bi_flags); | 154 | clear_bit(BIO_UPTODATE, &bio->bi_flags); |
155 | else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) | 155 | else if (!test_bit(BIO_UPTODATE, &bio->bi_flags)) |
156 | error = -EIO; | 156 | error = -EIO; |
157 | 157 | ||
158 | if (unlikely(nbytes > bio->bi_size)) { | 158 | if (unlikely(nbytes > bio->bi_size)) { |
159 | printk(KERN_ERR "%s: want %u bytes done, %u left\n", | 159 | printk(KERN_ERR "%s: want %u bytes done, %u left\n", |
160 | __func__, nbytes, bio->bi_size); | 160 | __func__, nbytes, bio->bi_size); |
161 | nbytes = bio->bi_size; | 161 | nbytes = bio->bi_size; |
162 | } | 162 | } |
163 | 163 | ||
164 | if (unlikely(rq->cmd_flags & REQ_QUIET)) | 164 | if (unlikely(rq->cmd_flags & REQ_QUIET)) |
165 | set_bit(BIO_QUIET, &bio->bi_flags); | 165 | set_bit(BIO_QUIET, &bio->bi_flags); |
166 | 166 | ||
167 | bio->bi_size -= nbytes; | 167 | bio->bi_size -= nbytes; |
168 | bio->bi_sector += (nbytes >> 9); | 168 | bio->bi_sector += (nbytes >> 9); |
169 | 169 | ||
170 | if (bio_integrity(bio)) | 170 | if (bio_integrity(bio)) |
171 | bio_integrity_advance(bio, nbytes); | 171 | bio_integrity_advance(bio, nbytes); |
172 | 172 | ||
173 | /* don't actually finish bio if it's part of flush sequence */ | 173 | /* don't actually finish bio if it's part of flush sequence */ |
174 | if (bio->bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) | 174 | if (bio->bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ)) |
175 | bio_endio(bio, error); | 175 | bio_endio(bio, error); |
176 | } | 176 | } |
177 | 177 | ||
178 | void blk_dump_rq_flags(struct request *rq, char *msg) | 178 | void blk_dump_rq_flags(struct request *rq, char *msg) |
179 | { | 179 | { |
180 | int bit; | 180 | int bit; |
181 | 181 | ||
182 | printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg, | 182 | printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg, |
183 | rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type, | 183 | rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type, |
184 | rq->cmd_flags); | 184 | rq->cmd_flags); |
185 | 185 | ||
186 | printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", | 186 | printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n", |
187 | (unsigned long long)blk_rq_pos(rq), | 187 | (unsigned long long)blk_rq_pos(rq), |
188 | blk_rq_sectors(rq), blk_rq_cur_sectors(rq)); | 188 | blk_rq_sectors(rq), blk_rq_cur_sectors(rq)); |
189 | printk(KERN_INFO " bio %p, biotail %p, buffer %p, len %u\n", | 189 | printk(KERN_INFO " bio %p, biotail %p, buffer %p, len %u\n", |
190 | rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq)); | 190 | rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq)); |
191 | 191 | ||
192 | if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { | 192 | if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { |
193 | printk(KERN_INFO " cdb: "); | 193 | printk(KERN_INFO " cdb: "); |
194 | for (bit = 0; bit < BLK_MAX_CDB; bit++) | 194 | for (bit = 0; bit < BLK_MAX_CDB; bit++) |
195 | printk("%02x ", rq->cmd[bit]); | 195 | printk("%02x ", rq->cmd[bit]); |
196 | printk("\n"); | 196 | printk("\n"); |
197 | } | 197 | } |
198 | } | 198 | } |
199 | EXPORT_SYMBOL(blk_dump_rq_flags); | 199 | EXPORT_SYMBOL(blk_dump_rq_flags); |
200 | 200 | ||
201 | static void blk_delay_work(struct work_struct *work) | 201 | static void blk_delay_work(struct work_struct *work) |
202 | { | 202 | { |
203 | struct request_queue *q; | 203 | struct request_queue *q; |
204 | 204 | ||
205 | q = container_of(work, struct request_queue, delay_work.work); | 205 | q = container_of(work, struct request_queue, delay_work.work); |
206 | spin_lock_irq(q->queue_lock); | 206 | spin_lock_irq(q->queue_lock); |
207 | __blk_run_queue(q); | 207 | __blk_run_queue(q); |
208 | spin_unlock_irq(q->queue_lock); | 208 | spin_unlock_irq(q->queue_lock); |
209 | } | 209 | } |
210 | 210 | ||
211 | /** | 211 | /** |
212 | * blk_delay_queue - restart queueing after defined interval | 212 | * blk_delay_queue - restart queueing after defined interval |
213 | * @q: The &struct request_queue in question | 213 | * @q: The &struct request_queue in question |
214 | * @msecs: Delay in msecs | 214 | * @msecs: Delay in msecs |
215 | * | 215 | * |
216 | * Description: | 216 | * Description: |
217 | * Sometimes queueing needs to be postponed for a little while, to allow | 217 | * Sometimes queueing needs to be postponed for a little while, to allow |
218 | * resources to come back. This function will make sure that queueing is | 218 | * resources to come back. This function will make sure that queueing is |
219 | * restarted around the specified time. | 219 | * restarted around the specified time. |
220 | */ | 220 | */ |
221 | void blk_delay_queue(struct request_queue *q, unsigned long msecs) | 221 | void blk_delay_queue(struct request_queue *q, unsigned long msecs) |
222 | { | 222 | { |
223 | queue_delayed_work(kblockd_workqueue, &q->delay_work, | 223 | queue_delayed_work(kblockd_workqueue, &q->delay_work, |
224 | msecs_to_jiffies(msecs)); | 224 | msecs_to_jiffies(msecs)); |
225 | } | 225 | } |
226 | EXPORT_SYMBOL(blk_delay_queue); | 226 | EXPORT_SYMBOL(blk_delay_queue); |
227 | 227 | ||
228 | /** | 228 | /** |
229 | * blk_start_queue - restart a previously stopped queue | 229 | * blk_start_queue - restart a previously stopped queue |
230 | * @q: The &struct request_queue in question | 230 | * @q: The &struct request_queue in question |
231 | * | 231 | * |
232 | * Description: | 232 | * Description: |
233 | * blk_start_queue() will clear the stop flag on the queue, and call | 233 | * blk_start_queue() will clear the stop flag on the queue, and call |
234 | * the request_fn for the queue if it was in a stopped state when | 234 | * the request_fn for the queue if it was in a stopped state when |
235 | * entered. Also see blk_stop_queue(). Queue lock must be held. | 235 | * entered. Also see blk_stop_queue(). Queue lock must be held. |
236 | **/ | 236 | **/ |
237 | void blk_start_queue(struct request_queue *q) | 237 | void blk_start_queue(struct request_queue *q) |
238 | { | 238 | { |
239 | WARN_ON(!irqs_disabled()); | 239 | WARN_ON(!irqs_disabled()); |
240 | 240 | ||
241 | queue_flag_clear(QUEUE_FLAG_STOPPED, q); | 241 | queue_flag_clear(QUEUE_FLAG_STOPPED, q); |
242 | __blk_run_queue(q); | 242 | __blk_run_queue(q); |
243 | } | 243 | } |
244 | EXPORT_SYMBOL(blk_start_queue); | 244 | EXPORT_SYMBOL(blk_start_queue); |
245 | 245 | ||
246 | /** | 246 | /** |
247 | * blk_stop_queue - stop a queue | 247 | * blk_stop_queue - stop a queue |
248 | * @q: The &struct request_queue in question | 248 | * @q: The &struct request_queue in question |
249 | * | 249 | * |
250 | * Description: | 250 | * Description: |
251 | * The Linux block layer assumes that a block driver will consume all | 251 | * The Linux block layer assumes that a block driver will consume all |
252 | * entries on the request queue when the request_fn strategy is called. | 252 | * entries on the request queue when the request_fn strategy is called. |
253 | * Often this will not happen, because of hardware limitations (queue | 253 | * Often this will not happen, because of hardware limitations (queue |
254 | * depth settings). If a device driver gets a 'queue full' response, | 254 | * depth settings). If a device driver gets a 'queue full' response, |
255 | * or if it simply chooses not to queue more I/O at one point, it can | 255 | * or if it simply chooses not to queue more I/O at one point, it can |
256 | * call this function to prevent the request_fn from being called until | 256 | * call this function to prevent the request_fn from being called until |
257 | * the driver has signalled it's ready to go again. This happens by calling | 257 | * the driver has signalled it's ready to go again. This happens by calling |
258 | * blk_start_queue() to restart queue operations. Queue lock must be held. | 258 | * blk_start_queue() to restart queue operations. Queue lock must be held. |
259 | **/ | 259 | **/ |
260 | void blk_stop_queue(struct request_queue *q) | 260 | void blk_stop_queue(struct request_queue *q) |
261 | { | 261 | { |
262 | __cancel_delayed_work(&q->delay_work); | 262 | __cancel_delayed_work(&q->delay_work); |
263 | queue_flag_set(QUEUE_FLAG_STOPPED, q); | 263 | queue_flag_set(QUEUE_FLAG_STOPPED, q); |
264 | } | 264 | } |
265 | EXPORT_SYMBOL(blk_stop_queue); | 265 | EXPORT_SYMBOL(blk_stop_queue); |
266 | 266 | ||
267 | /** | 267 | /** |
268 | * blk_sync_queue - cancel any pending callbacks on a queue | 268 | * blk_sync_queue - cancel any pending callbacks on a queue |
269 | * @q: the queue | 269 | * @q: the queue |
270 | * | 270 | * |
271 | * Description: | 271 | * Description: |
272 | * The block layer may perform asynchronous callback activity | 272 | * The block layer may perform asynchronous callback activity |
273 | * on a queue, such as calling the unplug function after a timeout. | 273 | * on a queue, such as calling the unplug function after a timeout. |
274 | * A block device may call blk_sync_queue to ensure that any | 274 | * A block device may call blk_sync_queue to ensure that any |
275 | * such activity is cancelled, thus allowing it to release resources | 275 | * such activity is cancelled, thus allowing it to release resources |
276 | * that the callbacks might use. The caller must already have made sure | 276 | * that the callbacks might use. The caller must already have made sure |
277 | * that its ->make_request_fn will not re-add plugging prior to calling | 277 | * that its ->make_request_fn will not re-add plugging prior to calling |
278 | * this function. | 278 | * this function. |
279 | * | 279 | * |
280 | * This function does not cancel any asynchronous activity arising | 280 | * This function does not cancel any asynchronous activity arising |
281 | * out of elevator or throttling code. That would require elevaotor_exit() | 281 | * out of elevator or throttling code. That would require elevaotor_exit() |
282 | * and blk_throtl_exit() to be called with queue lock initialized. | 282 | * and blk_throtl_exit() to be called with queue lock initialized. |
283 | * | 283 | * |
284 | */ | 284 | */ |
285 | void blk_sync_queue(struct request_queue *q) | 285 | void blk_sync_queue(struct request_queue *q) |
286 | { | 286 | { |
287 | del_timer_sync(&q->timeout); | 287 | del_timer_sync(&q->timeout); |
288 | cancel_delayed_work_sync(&q->delay_work); | 288 | cancel_delayed_work_sync(&q->delay_work); |
289 | } | 289 | } |
290 | EXPORT_SYMBOL(blk_sync_queue); | 290 | EXPORT_SYMBOL(blk_sync_queue); |
291 | 291 | ||
292 | /** | 292 | /** |
293 | * __blk_run_queue - run a single device queue | 293 | * __blk_run_queue - run a single device queue |
294 | * @q: The queue to run | 294 | * @q: The queue to run |
295 | * | 295 | * |
296 | * Description: | 296 | * Description: |
297 | * See @blk_run_queue. This variant must be called with the queue lock | 297 | * See @blk_run_queue. This variant must be called with the queue lock |
298 | * held and interrupts disabled. | 298 | * held and interrupts disabled. |
299 | */ | 299 | */ |
300 | void __blk_run_queue(struct request_queue *q) | 300 | void __blk_run_queue(struct request_queue *q) |
301 | { | 301 | { |
302 | if (unlikely(blk_queue_stopped(q))) | 302 | if (unlikely(blk_queue_stopped(q))) |
303 | return; | 303 | return; |
304 | 304 | ||
305 | q->request_fn(q); | 305 | q->request_fn(q); |
306 | } | 306 | } |
307 | EXPORT_SYMBOL(__blk_run_queue); | 307 | EXPORT_SYMBOL(__blk_run_queue); |
308 | 308 | ||
309 | /** | 309 | /** |
310 | * blk_run_queue_async - run a single device queue in workqueue context | 310 | * blk_run_queue_async - run a single device queue in workqueue context |
311 | * @q: The queue to run | 311 | * @q: The queue to run |
312 | * | 312 | * |
313 | * Description: | 313 | * Description: |
314 | * Tells kblockd to perform the equivalent of @blk_run_queue on behalf | 314 | * Tells kblockd to perform the equivalent of @blk_run_queue on behalf |
315 | * of us. | 315 | * of us. |
316 | */ | 316 | */ |
317 | void blk_run_queue_async(struct request_queue *q) | 317 | void blk_run_queue_async(struct request_queue *q) |
318 | { | 318 | { |
319 | if (likely(!blk_queue_stopped(q))) { | 319 | if (likely(!blk_queue_stopped(q))) { |
320 | __cancel_delayed_work(&q->delay_work); | 320 | __cancel_delayed_work(&q->delay_work); |
321 | queue_delayed_work(kblockd_workqueue, &q->delay_work, 0); | 321 | queue_delayed_work(kblockd_workqueue, &q->delay_work, 0); |
322 | } | 322 | } |
323 | } | 323 | } |
324 | EXPORT_SYMBOL(blk_run_queue_async); | 324 | EXPORT_SYMBOL(blk_run_queue_async); |
325 | 325 | ||
326 | /** | 326 | /** |
327 | * blk_run_queue - run a single device queue | 327 | * blk_run_queue - run a single device queue |
328 | * @q: The queue to run | 328 | * @q: The queue to run |
329 | * | 329 | * |
330 | * Description: | 330 | * Description: |
331 | * Invoke request handling on this queue, if it has pending work to do. | 331 | * Invoke request handling on this queue, if it has pending work to do. |
332 | * May be used to restart queueing when a request has completed. | 332 | * May be used to restart queueing when a request has completed. |
333 | */ | 333 | */ |
334 | void blk_run_queue(struct request_queue *q) | 334 | void blk_run_queue(struct request_queue *q) |
335 | { | 335 | { |
336 | unsigned long flags; | 336 | unsigned long flags; |
337 | 337 | ||
338 | spin_lock_irqsave(q->queue_lock, flags); | 338 | spin_lock_irqsave(q->queue_lock, flags); |
339 | __blk_run_queue(q); | 339 | __blk_run_queue(q); |
340 | spin_unlock_irqrestore(q->queue_lock, flags); | 340 | spin_unlock_irqrestore(q->queue_lock, flags); |
341 | } | 341 | } |
342 | EXPORT_SYMBOL(blk_run_queue); | 342 | EXPORT_SYMBOL(blk_run_queue); |
343 | 343 | ||
344 | void blk_put_queue(struct request_queue *q) | 344 | void blk_put_queue(struct request_queue *q) |
345 | { | 345 | { |
346 | kobject_put(&q->kobj); | 346 | kobject_put(&q->kobj); |
347 | } | 347 | } |
348 | EXPORT_SYMBOL(blk_put_queue); | 348 | EXPORT_SYMBOL(blk_put_queue); |
349 | 349 | ||
350 | /* | 350 | /* |
351 | * Note: If a driver supplied the queue lock, it should not zap that lock | 351 | * Note: If a driver supplied the queue lock, it should not zap that lock |
352 | * unexpectedly as some queue cleanup components like elevator_exit() and | 352 | * unexpectedly as some queue cleanup components like elevator_exit() and |
353 | * blk_throtl_exit() need queue lock. | 353 | * blk_throtl_exit() need queue lock. |
354 | */ | 354 | */ |
355 | void blk_cleanup_queue(struct request_queue *q) | 355 | void blk_cleanup_queue(struct request_queue *q) |
356 | { | 356 | { |
357 | /* | 357 | /* |
358 | * We know we have process context here, so we can be a little | 358 | * We know we have process context here, so we can be a little |
359 | * cautious and ensure that pending block actions on this device | 359 | * cautious and ensure that pending block actions on this device |
360 | * are done before moving on. Going into this function, we should | 360 | * are done before moving on. Going into this function, we should |
361 | * not have processes doing IO to this device. | 361 | * not have processes doing IO to this device. |
362 | */ | 362 | */ |
363 | blk_sync_queue(q); | 363 | blk_sync_queue(q); |
364 | 364 | ||
365 | del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer); | 365 | del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer); |
366 | mutex_lock(&q->sysfs_lock); | 366 | mutex_lock(&q->sysfs_lock); |
367 | queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q); | 367 | queue_flag_set_unlocked(QUEUE_FLAG_DEAD, q); |
368 | mutex_unlock(&q->sysfs_lock); | 368 | mutex_unlock(&q->sysfs_lock); |
369 | 369 | ||
370 | if (q->elevator) | 370 | if (q->elevator) |
371 | elevator_exit(q->elevator); | 371 | elevator_exit(q->elevator); |
372 | 372 | ||
373 | blk_throtl_exit(q); | 373 | blk_throtl_exit(q); |
374 | 374 | ||
375 | blk_put_queue(q); | 375 | blk_put_queue(q); |
376 | } | 376 | } |
377 | EXPORT_SYMBOL(blk_cleanup_queue); | 377 | EXPORT_SYMBOL(blk_cleanup_queue); |
378 | 378 | ||
379 | static int blk_init_free_list(struct request_queue *q) | 379 | static int blk_init_free_list(struct request_queue *q) |
380 | { | 380 | { |
381 | struct request_list *rl = &q->rq; | 381 | struct request_list *rl = &q->rq; |
382 | 382 | ||
383 | if (unlikely(rl->rq_pool)) | 383 | if (unlikely(rl->rq_pool)) |
384 | return 0; | 384 | return 0; |
385 | 385 | ||
386 | rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0; | 386 | rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0; |
387 | rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0; | 387 | rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0; |
388 | rl->elvpriv = 0; | 388 | rl->elvpriv = 0; |
389 | init_waitqueue_head(&rl->wait[BLK_RW_SYNC]); | 389 | init_waitqueue_head(&rl->wait[BLK_RW_SYNC]); |
390 | init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]); | 390 | init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]); |
391 | 391 | ||
392 | rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab, | 392 | rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab, |
393 | mempool_free_slab, request_cachep, q->node); | 393 | mempool_free_slab, request_cachep, q->node); |
394 | 394 | ||
395 | if (!rl->rq_pool) | 395 | if (!rl->rq_pool) |
396 | return -ENOMEM; | 396 | return -ENOMEM; |
397 | 397 | ||
398 | return 0; | 398 | return 0; |
399 | } | 399 | } |
400 | 400 | ||
401 | struct request_queue *blk_alloc_queue(gfp_t gfp_mask) | 401 | struct request_queue *blk_alloc_queue(gfp_t gfp_mask) |
402 | { | 402 | { |
403 | return blk_alloc_queue_node(gfp_mask, -1); | 403 | return blk_alloc_queue_node(gfp_mask, -1); |
404 | } | 404 | } |
405 | EXPORT_SYMBOL(blk_alloc_queue); | 405 | EXPORT_SYMBOL(blk_alloc_queue); |
406 | 406 | ||
407 | struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) | 407 | struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id) |
408 | { | 408 | { |
409 | struct request_queue *q; | 409 | struct request_queue *q; |
410 | int err; | 410 | int err; |
411 | 411 | ||
412 | q = kmem_cache_alloc_node(blk_requestq_cachep, | 412 | q = kmem_cache_alloc_node(blk_requestq_cachep, |
413 | gfp_mask | __GFP_ZERO, node_id); | 413 | gfp_mask | __GFP_ZERO, node_id); |
414 | if (!q) | 414 | if (!q) |
415 | return NULL; | 415 | return NULL; |
416 | 416 | ||
417 | q->backing_dev_info.ra_pages = | 417 | q->backing_dev_info.ra_pages = |
418 | (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; | 418 | (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE; |
419 | q->backing_dev_info.state = 0; | 419 | q->backing_dev_info.state = 0; |
420 | q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY; | 420 | q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY; |
421 | q->backing_dev_info.name = "block"; | 421 | q->backing_dev_info.name = "block"; |
422 | 422 | ||
423 | err = bdi_init(&q->backing_dev_info); | 423 | err = bdi_init(&q->backing_dev_info); |
424 | if (err) { | 424 | if (err) { |
425 | kmem_cache_free(blk_requestq_cachep, q); | 425 | kmem_cache_free(blk_requestq_cachep, q); |
426 | return NULL; | 426 | return NULL; |
427 | } | 427 | } |
428 | 428 | ||
429 | if (blk_throtl_init(q)) { | 429 | if (blk_throtl_init(q)) { |
430 | kmem_cache_free(blk_requestq_cachep, q); | 430 | kmem_cache_free(blk_requestq_cachep, q); |
431 | return NULL; | 431 | return NULL; |
432 | } | 432 | } |
433 | 433 | ||
434 | setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, | 434 | setup_timer(&q->backing_dev_info.laptop_mode_wb_timer, |
435 | laptop_mode_timer_fn, (unsigned long) q); | 435 | laptop_mode_timer_fn, (unsigned long) q); |
436 | setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); | 436 | setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q); |
437 | INIT_LIST_HEAD(&q->timeout_list); | 437 | INIT_LIST_HEAD(&q->timeout_list); |
438 | INIT_LIST_HEAD(&q->flush_queue[0]); | 438 | INIT_LIST_HEAD(&q->flush_queue[0]); |
439 | INIT_LIST_HEAD(&q->flush_queue[1]); | 439 | INIT_LIST_HEAD(&q->flush_queue[1]); |
440 | INIT_LIST_HEAD(&q->flush_data_in_flight); | 440 | INIT_LIST_HEAD(&q->flush_data_in_flight); |
441 | INIT_DELAYED_WORK(&q->delay_work, blk_delay_work); | 441 | INIT_DELAYED_WORK(&q->delay_work, blk_delay_work); |
442 | 442 | ||
443 | kobject_init(&q->kobj, &blk_queue_ktype); | 443 | kobject_init(&q->kobj, &blk_queue_ktype); |
444 | 444 | ||
445 | mutex_init(&q->sysfs_lock); | 445 | mutex_init(&q->sysfs_lock); |
446 | spin_lock_init(&q->__queue_lock); | 446 | spin_lock_init(&q->__queue_lock); |
447 | 447 | ||
448 | /* | 448 | /* |
449 | * By default initialize queue_lock to internal lock and driver can | 449 | * By default initialize queue_lock to internal lock and driver can |
450 | * override it later if need be. | 450 | * override it later if need be. |
451 | */ | 451 | */ |
452 | q->queue_lock = &q->__queue_lock; | 452 | q->queue_lock = &q->__queue_lock; |
453 | 453 | ||
454 | return q; | 454 | return q; |
455 | } | 455 | } |
456 | EXPORT_SYMBOL(blk_alloc_queue_node); | 456 | EXPORT_SYMBOL(blk_alloc_queue_node); |
457 | 457 | ||
458 | /** | 458 | /** |
459 | * blk_init_queue - prepare a request queue for use with a block device | 459 | * blk_init_queue - prepare a request queue for use with a block device |
460 | * @rfn: The function to be called to process requests that have been | 460 | * @rfn: The function to be called to process requests that have been |
461 | * placed on the queue. | 461 | * placed on the queue. |
462 | * @lock: Request queue spin lock | 462 | * @lock: Request queue spin lock |
463 | * | 463 | * |
464 | * Description: | 464 | * Description: |
465 | * If a block device wishes to use the standard request handling procedures, | 465 | * If a block device wishes to use the standard request handling procedures, |
466 | * which sorts requests and coalesces adjacent requests, then it must | 466 | * which sorts requests and coalesces adjacent requests, then it must |
467 | * call blk_init_queue(). The function @rfn will be called when there | 467 | * call blk_init_queue(). The function @rfn will be called when there |
468 | * are requests on the queue that need to be processed. If the device | 468 | * are requests on the queue that need to be processed. If the device |
469 | * supports plugging, then @rfn may not be called immediately when requests | 469 | * supports plugging, then @rfn may not be called immediately when requests |
470 | * are available on the queue, but may be called at some time later instead. | 470 | * are available on the queue, but may be called at some time later instead. |
471 | * Plugged queues are generally unplugged when a buffer belonging to one | 471 | * Plugged queues are generally unplugged when a buffer belonging to one |
472 | * of the requests on the queue is needed, or due to memory pressure. | 472 | * of the requests on the queue is needed, or due to memory pressure. |
473 | * | 473 | * |
474 | * @rfn is not required, or even expected, to remove all requests off the | 474 | * @rfn is not required, or even expected, to remove all requests off the |
475 | * queue, but only as many as it can handle at a time. If it does leave | 475 | * queue, but only as many as it can handle at a time. If it does leave |
476 | * requests on the queue, it is responsible for arranging that the requests | 476 | * requests on the queue, it is responsible for arranging that the requests |
477 | * get dealt with eventually. | 477 | * get dealt with eventually. |
478 | * | 478 | * |
479 | * The queue spin lock must be held while manipulating the requests on the | 479 | * The queue spin lock must be held while manipulating the requests on the |
480 | * request queue; this lock will be taken also from interrupt context, so irq | 480 | * request queue; this lock will be taken also from interrupt context, so irq |
481 | * disabling is needed for it. | 481 | * disabling is needed for it. |
482 | * | 482 | * |
483 | * Function returns a pointer to the initialized request queue, or %NULL if | 483 | * Function returns a pointer to the initialized request queue, or %NULL if |
484 | * it didn't succeed. | 484 | * it didn't succeed. |
485 | * | 485 | * |
486 | * Note: | 486 | * Note: |
487 | * blk_init_queue() must be paired with a blk_cleanup_queue() call | 487 | * blk_init_queue() must be paired with a blk_cleanup_queue() call |
488 | * when the block device is deactivated (such as at module unload). | 488 | * when the block device is deactivated (such as at module unload). |
489 | **/ | 489 | **/ |
490 | 490 | ||
491 | struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock) | 491 | struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock) |
492 | { | 492 | { |
493 | return blk_init_queue_node(rfn, lock, -1); | 493 | return blk_init_queue_node(rfn, lock, -1); |
494 | } | 494 | } |
495 | EXPORT_SYMBOL(blk_init_queue); | 495 | EXPORT_SYMBOL(blk_init_queue); |
496 | 496 | ||
497 | struct request_queue * | 497 | struct request_queue * |
498 | blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id) | 498 | blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id) |
499 | { | 499 | { |
500 | struct request_queue *uninit_q, *q; | 500 | struct request_queue *uninit_q, *q; |
501 | 501 | ||
502 | uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id); | 502 | uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id); |
503 | if (!uninit_q) | 503 | if (!uninit_q) |
504 | return NULL; | 504 | return NULL; |
505 | 505 | ||
506 | q = blk_init_allocated_queue_node(uninit_q, rfn, lock, node_id); | 506 | q = blk_init_allocated_queue_node(uninit_q, rfn, lock, node_id); |
507 | if (!q) | 507 | if (!q) |
508 | blk_cleanup_queue(uninit_q); | 508 | blk_cleanup_queue(uninit_q); |
509 | 509 | ||
510 | return q; | 510 | return q; |
511 | } | 511 | } |
512 | EXPORT_SYMBOL(blk_init_queue_node); | 512 | EXPORT_SYMBOL(blk_init_queue_node); |
513 | 513 | ||
514 | struct request_queue * | 514 | struct request_queue * |
515 | blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, | 515 | blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn, |
516 | spinlock_t *lock) | 516 | spinlock_t *lock) |
517 | { | 517 | { |
518 | return blk_init_allocated_queue_node(q, rfn, lock, -1); | 518 | return blk_init_allocated_queue_node(q, rfn, lock, -1); |
519 | } | 519 | } |
520 | EXPORT_SYMBOL(blk_init_allocated_queue); | 520 | EXPORT_SYMBOL(blk_init_allocated_queue); |
521 | 521 | ||
522 | struct request_queue * | 522 | struct request_queue * |
523 | blk_init_allocated_queue_node(struct request_queue *q, request_fn_proc *rfn, | 523 | blk_init_allocated_queue_node(struct request_queue *q, request_fn_proc *rfn, |
524 | spinlock_t *lock, int node_id) | 524 | spinlock_t *lock, int node_id) |
525 | { | 525 | { |
526 | if (!q) | 526 | if (!q) |
527 | return NULL; | 527 | return NULL; |
528 | 528 | ||
529 | q->node = node_id; | 529 | q->node = node_id; |
530 | if (blk_init_free_list(q)) | 530 | if (blk_init_free_list(q)) |
531 | return NULL; | 531 | return NULL; |
532 | 532 | ||
533 | q->request_fn = rfn; | 533 | q->request_fn = rfn; |
534 | q->prep_rq_fn = NULL; | 534 | q->prep_rq_fn = NULL; |
535 | q->unprep_rq_fn = NULL; | 535 | q->unprep_rq_fn = NULL; |
536 | q->queue_flags = QUEUE_FLAG_DEFAULT; | 536 | q->queue_flags = QUEUE_FLAG_DEFAULT; |
537 | 537 | ||
538 | /* Override internal queue lock with supplied lock pointer */ | 538 | /* Override internal queue lock with supplied lock pointer */ |
539 | if (lock) | 539 | if (lock) |
540 | q->queue_lock = lock; | 540 | q->queue_lock = lock; |
541 | 541 | ||
542 | /* | 542 | /* |
543 | * This also sets hw/phys segments, boundary and size | 543 | * This also sets hw/phys segments, boundary and size |
544 | */ | 544 | */ |
545 | blk_queue_make_request(q, __make_request); | 545 | blk_queue_make_request(q, __make_request); |
546 | 546 | ||
547 | q->sg_reserved_size = INT_MAX; | 547 | q->sg_reserved_size = INT_MAX; |
548 | 548 | ||
549 | /* | 549 | /* |
550 | * all done | 550 | * all done |
551 | */ | 551 | */ |
552 | if (!elevator_init(q, NULL)) { | 552 | if (!elevator_init(q, NULL)) { |
553 | blk_queue_congestion_threshold(q); | 553 | blk_queue_congestion_threshold(q); |
554 | return q; | 554 | return q; |
555 | } | 555 | } |
556 | 556 | ||
557 | return NULL; | 557 | return NULL; |
558 | } | 558 | } |
559 | EXPORT_SYMBOL(blk_init_allocated_queue_node); | 559 | EXPORT_SYMBOL(blk_init_allocated_queue_node); |
560 | 560 | ||
561 | int blk_get_queue(struct request_queue *q) | 561 | int blk_get_queue(struct request_queue *q) |
562 | { | 562 | { |
563 | if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) { | 563 | if (likely(!test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) { |
564 | kobject_get(&q->kobj); | 564 | kobject_get(&q->kobj); |
565 | return 0; | 565 | return 0; |
566 | } | 566 | } |
567 | 567 | ||
568 | return 1; | 568 | return 1; |
569 | } | 569 | } |
570 | EXPORT_SYMBOL(blk_get_queue); | 570 | EXPORT_SYMBOL(blk_get_queue); |
571 | 571 | ||
572 | static inline void blk_free_request(struct request_queue *q, struct request *rq) | 572 | static inline void blk_free_request(struct request_queue *q, struct request *rq) |
573 | { | 573 | { |
574 | if (rq->cmd_flags & REQ_ELVPRIV) | 574 | if (rq->cmd_flags & REQ_ELVPRIV) |
575 | elv_put_request(q, rq); | 575 | elv_put_request(q, rq); |
576 | mempool_free(rq, q->rq.rq_pool); | 576 | mempool_free(rq, q->rq.rq_pool); |
577 | } | 577 | } |
578 | 578 | ||
579 | static struct request * | 579 | static struct request * |
580 | blk_alloc_request(struct request_queue *q, int flags, int priv, gfp_t gfp_mask) | 580 | blk_alloc_request(struct request_queue *q, int flags, int priv, gfp_t gfp_mask) |
581 | { | 581 | { |
582 | struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask); | 582 | struct request *rq = mempool_alloc(q->rq.rq_pool, gfp_mask); |
583 | 583 | ||
584 | if (!rq) | 584 | if (!rq) |
585 | return NULL; | 585 | return NULL; |
586 | 586 | ||
587 | blk_rq_init(q, rq); | 587 | blk_rq_init(q, rq); |
588 | 588 | ||
589 | rq->cmd_flags = flags | REQ_ALLOCED; | 589 | rq->cmd_flags = flags | REQ_ALLOCED; |
590 | 590 | ||
591 | if (priv) { | 591 | if (priv) { |
592 | if (unlikely(elv_set_request(q, rq, gfp_mask))) { | 592 | if (unlikely(elv_set_request(q, rq, gfp_mask))) { |
593 | mempool_free(rq, q->rq.rq_pool); | 593 | mempool_free(rq, q->rq.rq_pool); |
594 | return NULL; | 594 | return NULL; |
595 | } | 595 | } |
596 | rq->cmd_flags |= REQ_ELVPRIV; | 596 | rq->cmd_flags |= REQ_ELVPRIV; |
597 | } | 597 | } |
598 | 598 | ||
599 | return rq; | 599 | return rq; |
600 | } | 600 | } |
601 | 601 | ||
602 | /* | 602 | /* |
603 | * ioc_batching returns true if the ioc is a valid batching request and | 603 | * ioc_batching returns true if the ioc is a valid batching request and |
604 | * should be given priority access to a request. | 604 | * should be given priority access to a request. |
605 | */ | 605 | */ |
606 | static inline int ioc_batching(struct request_queue *q, struct io_context *ioc) | 606 | static inline int ioc_batching(struct request_queue *q, struct io_context *ioc) |
607 | { | 607 | { |
608 | if (!ioc) | 608 | if (!ioc) |
609 | return 0; | 609 | return 0; |
610 | 610 | ||
611 | /* | 611 | /* |
612 | * Make sure the process is able to allocate at least 1 request | 612 | * Make sure the process is able to allocate at least 1 request |
613 | * even if the batch times out, otherwise we could theoretically | 613 | * even if the batch times out, otherwise we could theoretically |
614 | * lose wakeups. | 614 | * lose wakeups. |
615 | */ | 615 | */ |
616 | return ioc->nr_batch_requests == q->nr_batching || | 616 | return ioc->nr_batch_requests == q->nr_batching || |
617 | (ioc->nr_batch_requests > 0 | 617 | (ioc->nr_batch_requests > 0 |
618 | && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME)); | 618 | && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME)); |
619 | } | 619 | } |
620 | 620 | ||
621 | /* | 621 | /* |
622 | * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This | 622 | * ioc_set_batching sets ioc to be a new "batcher" if it is not one. This |
623 | * will cause the process to be a "batcher" on all queues in the system. This | 623 | * will cause the process to be a "batcher" on all queues in the system. This |
624 | * is the behaviour we want though - once it gets a wakeup it should be given | 624 | * is the behaviour we want though - once it gets a wakeup it should be given |
625 | * a nice run. | 625 | * a nice run. |
626 | */ | 626 | */ |
627 | static void ioc_set_batching(struct request_queue *q, struct io_context *ioc) | 627 | static void ioc_set_batching(struct request_queue *q, struct io_context *ioc) |
628 | { | 628 | { |
629 | if (!ioc || ioc_batching(q, ioc)) | 629 | if (!ioc || ioc_batching(q, ioc)) |
630 | return; | 630 | return; |
631 | 631 | ||
632 | ioc->nr_batch_requests = q->nr_batching; | 632 | ioc->nr_batch_requests = q->nr_batching; |
633 | ioc->last_waited = jiffies; | 633 | ioc->last_waited = jiffies; |
634 | } | 634 | } |
635 | 635 | ||
636 | static void __freed_request(struct request_queue *q, int sync) | 636 | static void __freed_request(struct request_queue *q, int sync) |
637 | { | 637 | { |
638 | struct request_list *rl = &q->rq; | 638 | struct request_list *rl = &q->rq; |
639 | 639 | ||
640 | if (rl->count[sync] < queue_congestion_off_threshold(q)) | 640 | if (rl->count[sync] < queue_congestion_off_threshold(q)) |
641 | blk_clear_queue_congested(q, sync); | 641 | blk_clear_queue_congested(q, sync); |
642 | 642 | ||
643 | if (rl->count[sync] + 1 <= q->nr_requests) { | 643 | if (rl->count[sync] + 1 <= q->nr_requests) { |
644 | if (waitqueue_active(&rl->wait[sync])) | 644 | if (waitqueue_active(&rl->wait[sync])) |
645 | wake_up(&rl->wait[sync]); | 645 | wake_up(&rl->wait[sync]); |
646 | 646 | ||
647 | blk_clear_queue_full(q, sync); | 647 | blk_clear_queue_full(q, sync); |
648 | } | 648 | } |
649 | } | 649 | } |
650 | 650 | ||
651 | /* | 651 | /* |
652 | * A request has just been released. Account for it, update the full and | 652 | * A request has just been released. Account for it, update the full and |
653 | * congestion status, wake up any waiters. Called under q->queue_lock. | 653 | * congestion status, wake up any waiters. Called under q->queue_lock. |
654 | */ | 654 | */ |
655 | static void freed_request(struct request_queue *q, int sync, int priv) | 655 | static void freed_request(struct request_queue *q, int sync, int priv) |
656 | { | 656 | { |
657 | struct request_list *rl = &q->rq; | 657 | struct request_list *rl = &q->rq; |
658 | 658 | ||
659 | rl->count[sync]--; | 659 | rl->count[sync]--; |
660 | if (priv) | 660 | if (priv) |
661 | rl->elvpriv--; | 661 | rl->elvpriv--; |
662 | 662 | ||
663 | __freed_request(q, sync); | 663 | __freed_request(q, sync); |
664 | 664 | ||
665 | if (unlikely(rl->starved[sync ^ 1])) | 665 | if (unlikely(rl->starved[sync ^ 1])) |
666 | __freed_request(q, sync ^ 1); | 666 | __freed_request(q, sync ^ 1); |
667 | } | 667 | } |
668 | 668 | ||
669 | /* | 669 | /* |
670 | * Determine if elevator data should be initialized when allocating the | 670 | * Determine if elevator data should be initialized when allocating the |
671 | * request associated with @bio. | 671 | * request associated with @bio. |
672 | */ | 672 | */ |
673 | static bool blk_rq_should_init_elevator(struct bio *bio) | 673 | static bool blk_rq_should_init_elevator(struct bio *bio) |
674 | { | 674 | { |
675 | if (!bio) | 675 | if (!bio) |
676 | return true; | 676 | return true; |
677 | 677 | ||
678 | /* | 678 | /* |
679 | * Flush requests do not use the elevator so skip initialization. | 679 | * Flush requests do not use the elevator so skip initialization. |
680 | * This allows a request to share the flush and elevator data. | 680 | * This allows a request to share the flush and elevator data. |
681 | */ | 681 | */ |
682 | if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) | 682 | if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) |
683 | return false; | 683 | return false; |
684 | 684 | ||
685 | return true; | 685 | return true; |
686 | } | 686 | } |
687 | 687 | ||
688 | /* | 688 | /* |
689 | * Get a free request, queue_lock must be held. | 689 | * Get a free request, queue_lock must be held. |
690 | * Returns NULL on failure, with queue_lock held. | 690 | * Returns NULL on failure, with queue_lock held. |
691 | * Returns !NULL on success, with queue_lock *not held*. | 691 | * Returns !NULL on success, with queue_lock *not held*. |
692 | */ | 692 | */ |
693 | static struct request *get_request(struct request_queue *q, int rw_flags, | 693 | static struct request *get_request(struct request_queue *q, int rw_flags, |
694 | struct bio *bio, gfp_t gfp_mask) | 694 | struct bio *bio, gfp_t gfp_mask) |
695 | { | 695 | { |
696 | struct request *rq = NULL; | 696 | struct request *rq = NULL; |
697 | struct request_list *rl = &q->rq; | 697 | struct request_list *rl = &q->rq; |
698 | struct io_context *ioc = NULL; | 698 | struct io_context *ioc = NULL; |
699 | const bool is_sync = rw_is_sync(rw_flags) != 0; | 699 | const bool is_sync = rw_is_sync(rw_flags) != 0; |
700 | int may_queue, priv = 0; | 700 | int may_queue, priv = 0; |
701 | 701 | ||
702 | may_queue = elv_may_queue(q, rw_flags); | 702 | may_queue = elv_may_queue(q, rw_flags); |
703 | if (may_queue == ELV_MQUEUE_NO) | 703 | if (may_queue == ELV_MQUEUE_NO) |
704 | goto rq_starved; | 704 | goto rq_starved; |
705 | 705 | ||
706 | if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) { | 706 | if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) { |
707 | if (rl->count[is_sync]+1 >= q->nr_requests) { | 707 | if (rl->count[is_sync]+1 >= q->nr_requests) { |
708 | ioc = current_io_context(GFP_ATOMIC, q->node); | 708 | ioc = current_io_context(GFP_ATOMIC, q->node); |
709 | /* | 709 | /* |
710 | * The queue will fill after this allocation, so set | 710 | * The queue will fill after this allocation, so set |
711 | * it as full, and mark this process as "batching". | 711 | * it as full, and mark this process as "batching". |
712 | * This process will be allowed to complete a batch of | 712 | * This process will be allowed to complete a batch of |
713 | * requests, others will be blocked. | 713 | * requests, others will be blocked. |
714 | */ | 714 | */ |
715 | if (!blk_queue_full(q, is_sync)) { | 715 | if (!blk_queue_full(q, is_sync)) { |
716 | ioc_set_batching(q, ioc); | 716 | ioc_set_batching(q, ioc); |
717 | blk_set_queue_full(q, is_sync); | 717 | blk_set_queue_full(q, is_sync); |
718 | } else { | 718 | } else { |
719 | if (may_queue != ELV_MQUEUE_MUST | 719 | if (may_queue != ELV_MQUEUE_MUST |
720 | && !ioc_batching(q, ioc)) { | 720 | && !ioc_batching(q, ioc)) { |
721 | /* | 721 | /* |
722 | * The queue is full and the allocating | 722 | * The queue is full and the allocating |
723 | * process is not a "batcher", and not | 723 | * process is not a "batcher", and not |
724 | * exempted by the IO scheduler | 724 | * exempted by the IO scheduler |
725 | */ | 725 | */ |
726 | goto out; | 726 | goto out; |
727 | } | 727 | } |
728 | } | 728 | } |
729 | } | 729 | } |
730 | blk_set_queue_congested(q, is_sync); | 730 | blk_set_queue_congested(q, is_sync); |
731 | } | 731 | } |
732 | 732 | ||
733 | /* | 733 | /* |
734 | * Only allow batching queuers to allocate up to 50% over the defined | 734 | * Only allow batching queuers to allocate up to 50% over the defined |
735 | * limit of requests, otherwise we could have thousands of requests | 735 | * limit of requests, otherwise we could have thousands of requests |
736 | * allocated with any setting of ->nr_requests | 736 | * allocated with any setting of ->nr_requests |
737 | */ | 737 | */ |
738 | if (rl->count[is_sync] >= (3 * q->nr_requests / 2)) | 738 | if (rl->count[is_sync] >= (3 * q->nr_requests / 2)) |
739 | goto out; | 739 | goto out; |
740 | 740 | ||
741 | rl->count[is_sync]++; | 741 | rl->count[is_sync]++; |
742 | rl->starved[is_sync] = 0; | 742 | rl->starved[is_sync] = 0; |
743 | 743 | ||
744 | if (blk_rq_should_init_elevator(bio)) { | 744 | if (blk_rq_should_init_elevator(bio)) { |
745 | priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); | 745 | priv = !test_bit(QUEUE_FLAG_ELVSWITCH, &q->queue_flags); |
746 | if (priv) | 746 | if (priv) |
747 | rl->elvpriv++; | 747 | rl->elvpriv++; |
748 | } | 748 | } |
749 | 749 | ||
750 | if (blk_queue_io_stat(q)) | 750 | if (blk_queue_io_stat(q)) |
751 | rw_flags |= REQ_IO_STAT; | 751 | rw_flags |= REQ_IO_STAT; |
752 | spin_unlock_irq(q->queue_lock); | 752 | spin_unlock_irq(q->queue_lock); |
753 | 753 | ||
754 | rq = blk_alloc_request(q, rw_flags, priv, gfp_mask); | 754 | rq = blk_alloc_request(q, rw_flags, priv, gfp_mask); |
755 | if (unlikely(!rq)) { | 755 | if (unlikely(!rq)) { |
756 | /* | 756 | /* |
757 | * Allocation failed presumably due to memory. Undo anything | 757 | * Allocation failed presumably due to memory. Undo anything |
758 | * we might have messed up. | 758 | * we might have messed up. |
759 | * | 759 | * |
760 | * Allocating task should really be put onto the front of the | 760 | * Allocating task should really be put onto the front of the |
761 | * wait queue, but this is pretty rare. | 761 | * wait queue, but this is pretty rare. |
762 | */ | 762 | */ |
763 | spin_lock_irq(q->queue_lock); | 763 | spin_lock_irq(q->queue_lock); |
764 | freed_request(q, is_sync, priv); | 764 | freed_request(q, is_sync, priv); |
765 | 765 | ||
766 | /* | 766 | /* |
767 | * in the very unlikely event that allocation failed and no | 767 | * in the very unlikely event that allocation failed and no |
768 | * requests for this direction was pending, mark us starved | 768 | * requests for this direction was pending, mark us starved |
769 | * so that freeing of a request in the other direction will | 769 | * so that freeing of a request in the other direction will |
770 | * notice us. another possible fix would be to split the | 770 | * notice us. another possible fix would be to split the |
771 | * rq mempool into READ and WRITE | 771 | * rq mempool into READ and WRITE |
772 | */ | 772 | */ |
773 | rq_starved: | 773 | rq_starved: |
774 | if (unlikely(rl->count[is_sync] == 0)) | 774 | if (unlikely(rl->count[is_sync] == 0)) |
775 | rl->starved[is_sync] = 1; | 775 | rl->starved[is_sync] = 1; |
776 | 776 | ||
777 | goto out; | 777 | goto out; |
778 | } | 778 | } |
779 | 779 | ||
780 | /* | 780 | /* |
781 | * ioc may be NULL here, and ioc_batching will be false. That's | 781 | * ioc may be NULL here, and ioc_batching will be false. That's |
782 | * OK, if the queue is under the request limit then requests need | 782 | * OK, if the queue is under the request limit then requests need |
783 | * not count toward the nr_batch_requests limit. There will always | 783 | * not count toward the nr_batch_requests limit. There will always |
784 | * be some limit enforced by BLK_BATCH_TIME. | 784 | * be some limit enforced by BLK_BATCH_TIME. |
785 | */ | 785 | */ |
786 | if (ioc_batching(q, ioc)) | 786 | if (ioc_batching(q, ioc)) |
787 | ioc->nr_batch_requests--; | 787 | ioc->nr_batch_requests--; |
788 | 788 | ||
789 | trace_block_getrq(q, bio, rw_flags & 1); | 789 | trace_block_getrq(q, bio, rw_flags & 1); |
790 | out: | 790 | out: |
791 | return rq; | 791 | return rq; |
792 | } | 792 | } |
793 | 793 | ||
794 | /* | 794 | /* |
795 | * No available requests for this queue, wait for some requests to become | 795 | * No available requests for this queue, wait for some requests to become |
796 | * available. | 796 | * available. |
797 | * | 797 | * |
798 | * Called with q->queue_lock held, and returns with it unlocked. | 798 | * Called with q->queue_lock held, and returns with it unlocked. |
799 | */ | 799 | */ |
800 | static struct request *get_request_wait(struct request_queue *q, int rw_flags, | 800 | static struct request *get_request_wait(struct request_queue *q, int rw_flags, |
801 | struct bio *bio) | 801 | struct bio *bio) |
802 | { | 802 | { |
803 | const bool is_sync = rw_is_sync(rw_flags) != 0; | 803 | const bool is_sync = rw_is_sync(rw_flags) != 0; |
804 | struct request *rq; | 804 | struct request *rq; |
805 | 805 | ||
806 | rq = get_request(q, rw_flags, bio, GFP_NOIO); | 806 | rq = get_request(q, rw_flags, bio, GFP_NOIO); |
807 | while (!rq) { | 807 | while (!rq) { |
808 | DEFINE_WAIT(wait); | 808 | DEFINE_WAIT(wait); |
809 | struct io_context *ioc; | 809 | struct io_context *ioc; |
810 | struct request_list *rl = &q->rq; | 810 | struct request_list *rl = &q->rq; |
811 | 811 | ||
812 | prepare_to_wait_exclusive(&rl->wait[is_sync], &wait, | 812 | prepare_to_wait_exclusive(&rl->wait[is_sync], &wait, |
813 | TASK_UNINTERRUPTIBLE); | 813 | TASK_UNINTERRUPTIBLE); |
814 | 814 | ||
815 | trace_block_sleeprq(q, bio, rw_flags & 1); | 815 | trace_block_sleeprq(q, bio, rw_flags & 1); |
816 | 816 | ||
817 | spin_unlock_irq(q->queue_lock); | 817 | spin_unlock_irq(q->queue_lock); |
818 | io_schedule(); | 818 | io_schedule(); |
819 | 819 | ||
820 | /* | 820 | /* |
821 | * After sleeping, we become a "batching" process and | 821 | * After sleeping, we become a "batching" process and |
822 | * will be able to allocate at least one request, and | 822 | * will be able to allocate at least one request, and |
823 | * up to a big batch of them for a small period time. | 823 | * up to a big batch of them for a small period time. |
824 | * See ioc_batching, ioc_set_batching | 824 | * See ioc_batching, ioc_set_batching |
825 | */ | 825 | */ |
826 | ioc = current_io_context(GFP_NOIO, q->node); | 826 | ioc = current_io_context(GFP_NOIO, q->node); |
827 | ioc_set_batching(q, ioc); | 827 | ioc_set_batching(q, ioc); |
828 | 828 | ||
829 | spin_lock_irq(q->queue_lock); | 829 | spin_lock_irq(q->queue_lock); |
830 | finish_wait(&rl->wait[is_sync], &wait); | 830 | finish_wait(&rl->wait[is_sync], &wait); |
831 | 831 | ||
832 | rq = get_request(q, rw_flags, bio, GFP_NOIO); | 832 | rq = get_request(q, rw_flags, bio, GFP_NOIO); |
833 | }; | 833 | }; |
834 | 834 | ||
835 | return rq; | 835 | return rq; |
836 | } | 836 | } |
837 | 837 | ||
838 | struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask) | 838 | struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask) |
839 | { | 839 | { |
840 | struct request *rq; | 840 | struct request *rq; |
841 | 841 | ||
842 | BUG_ON(rw != READ && rw != WRITE); | 842 | BUG_ON(rw != READ && rw != WRITE); |
843 | 843 | ||
844 | spin_lock_irq(q->queue_lock); | 844 | spin_lock_irq(q->queue_lock); |
845 | if (gfp_mask & __GFP_WAIT) { | 845 | if (gfp_mask & __GFP_WAIT) { |
846 | rq = get_request_wait(q, rw, NULL); | 846 | rq = get_request_wait(q, rw, NULL); |
847 | } else { | 847 | } else { |
848 | rq = get_request(q, rw, NULL, gfp_mask); | 848 | rq = get_request(q, rw, NULL, gfp_mask); |
849 | if (!rq) | 849 | if (!rq) |
850 | spin_unlock_irq(q->queue_lock); | 850 | spin_unlock_irq(q->queue_lock); |
851 | } | 851 | } |
852 | /* q->queue_lock is unlocked at this point */ | 852 | /* q->queue_lock is unlocked at this point */ |
853 | 853 | ||
854 | return rq; | 854 | return rq; |
855 | } | 855 | } |
856 | EXPORT_SYMBOL(blk_get_request); | 856 | EXPORT_SYMBOL(blk_get_request); |
857 | 857 | ||
858 | /** | 858 | /** |
859 | * blk_make_request - given a bio, allocate a corresponding struct request. | 859 | * blk_make_request - given a bio, allocate a corresponding struct request. |
860 | * @q: target request queue | 860 | * @q: target request queue |
861 | * @bio: The bio describing the memory mappings that will be submitted for IO. | 861 | * @bio: The bio describing the memory mappings that will be submitted for IO. |
862 | * It may be a chained-bio properly constructed by block/bio layer. | 862 | * It may be a chained-bio properly constructed by block/bio layer. |
863 | * @gfp_mask: gfp flags to be used for memory allocation | 863 | * @gfp_mask: gfp flags to be used for memory allocation |
864 | * | 864 | * |
865 | * blk_make_request is the parallel of generic_make_request for BLOCK_PC | 865 | * blk_make_request is the parallel of generic_make_request for BLOCK_PC |
866 | * type commands. Where the struct request needs to be farther initialized by | 866 | * type commands. Where the struct request needs to be farther initialized by |
867 | * the caller. It is passed a &struct bio, which describes the memory info of | 867 | * the caller. It is passed a &struct bio, which describes the memory info of |
868 | * the I/O transfer. | 868 | * the I/O transfer. |
869 | * | 869 | * |
870 | * The caller of blk_make_request must make sure that bi_io_vec | 870 | * The caller of blk_make_request must make sure that bi_io_vec |
871 | * are set to describe the memory buffers. That bio_data_dir() will return | 871 | * are set to describe the memory buffers. That bio_data_dir() will return |
872 | * the needed direction of the request. (And all bio's in the passed bio-chain | 872 | * the needed direction of the request. (And all bio's in the passed bio-chain |
873 | * are properly set accordingly) | 873 | * are properly set accordingly) |
874 | * | 874 | * |
875 | * If called under none-sleepable conditions, mapped bio buffers must not | 875 | * If called under none-sleepable conditions, mapped bio buffers must not |
876 | * need bouncing, by calling the appropriate masked or flagged allocator, | 876 | * need bouncing, by calling the appropriate masked or flagged allocator, |
877 | * suitable for the target device. Otherwise the call to blk_queue_bounce will | 877 | * suitable for the target device. Otherwise the call to blk_queue_bounce will |
878 | * BUG. | 878 | * BUG. |
879 | * | 879 | * |
880 | * WARNING: When allocating/cloning a bio-chain, careful consideration should be | 880 | * WARNING: When allocating/cloning a bio-chain, careful consideration should be |
881 | * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for | 881 | * given to how you allocate bios. In particular, you cannot use __GFP_WAIT for |
882 | * anything but the first bio in the chain. Otherwise you risk waiting for IO | 882 | * anything but the first bio in the chain. Otherwise you risk waiting for IO |
883 | * completion of a bio that hasn't been submitted yet, thus resulting in a | 883 | * completion of a bio that hasn't been submitted yet, thus resulting in a |
884 | * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead | 884 | * deadlock. Alternatively bios should be allocated using bio_kmalloc() instead |
885 | * of bio_alloc(), as that avoids the mempool deadlock. | 885 | * of bio_alloc(), as that avoids the mempool deadlock. |
886 | * If possible a big IO should be split into smaller parts when allocation | 886 | * If possible a big IO should be split into smaller parts when allocation |
887 | * fails. Partial allocation should not be an error, or you risk a live-lock. | 887 | * fails. Partial allocation should not be an error, or you risk a live-lock. |
888 | */ | 888 | */ |
889 | struct request *blk_make_request(struct request_queue *q, struct bio *bio, | 889 | struct request *blk_make_request(struct request_queue *q, struct bio *bio, |
890 | gfp_t gfp_mask) | 890 | gfp_t gfp_mask) |
891 | { | 891 | { |
892 | struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask); | 892 | struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask); |
893 | 893 | ||
894 | if (unlikely(!rq)) | 894 | if (unlikely(!rq)) |
895 | return ERR_PTR(-ENOMEM); | 895 | return ERR_PTR(-ENOMEM); |
896 | 896 | ||
897 | for_each_bio(bio) { | 897 | for_each_bio(bio) { |
898 | struct bio *bounce_bio = bio; | 898 | struct bio *bounce_bio = bio; |
899 | int ret; | 899 | int ret; |
900 | 900 | ||
901 | blk_queue_bounce(q, &bounce_bio); | 901 | blk_queue_bounce(q, &bounce_bio); |
902 | ret = blk_rq_append_bio(q, rq, bounce_bio); | 902 | ret = blk_rq_append_bio(q, rq, bounce_bio); |
903 | if (unlikely(ret)) { | 903 | if (unlikely(ret)) { |
904 | blk_put_request(rq); | 904 | blk_put_request(rq); |
905 | return ERR_PTR(ret); | 905 | return ERR_PTR(ret); |
906 | } | 906 | } |
907 | } | 907 | } |
908 | 908 | ||
909 | return rq; | 909 | return rq; |
910 | } | 910 | } |
911 | EXPORT_SYMBOL(blk_make_request); | 911 | EXPORT_SYMBOL(blk_make_request); |
912 | 912 | ||
913 | /** | 913 | /** |
914 | * blk_requeue_request - put a request back on queue | 914 | * blk_requeue_request - put a request back on queue |
915 | * @q: request queue where request should be inserted | 915 | * @q: request queue where request should be inserted |
916 | * @rq: request to be inserted | 916 | * @rq: request to be inserted |
917 | * | 917 | * |
918 | * Description: | 918 | * Description: |
919 | * Drivers often keep queueing requests until the hardware cannot accept | 919 | * Drivers often keep queueing requests until the hardware cannot accept |
920 | * more, when that condition happens we need to put the request back | 920 | * more, when that condition happens we need to put the request back |
921 | * on the queue. Must be called with queue lock held. | 921 | * on the queue. Must be called with queue lock held. |
922 | */ | 922 | */ |
923 | void blk_requeue_request(struct request_queue *q, struct request *rq) | 923 | void blk_requeue_request(struct request_queue *q, struct request *rq) |
924 | { | 924 | { |
925 | blk_delete_timer(rq); | 925 | blk_delete_timer(rq); |
926 | blk_clear_rq_complete(rq); | 926 | blk_clear_rq_complete(rq); |
927 | trace_block_rq_requeue(q, rq); | 927 | trace_block_rq_requeue(q, rq); |
928 | 928 | ||
929 | if (blk_rq_tagged(rq)) | 929 | if (blk_rq_tagged(rq)) |
930 | blk_queue_end_tag(q, rq); | 930 | blk_queue_end_tag(q, rq); |
931 | 931 | ||
932 | BUG_ON(blk_queued_rq(rq)); | 932 | BUG_ON(blk_queued_rq(rq)); |
933 | 933 | ||
934 | elv_requeue_request(q, rq); | 934 | elv_requeue_request(q, rq); |
935 | } | 935 | } |
936 | EXPORT_SYMBOL(blk_requeue_request); | 936 | EXPORT_SYMBOL(blk_requeue_request); |
937 | 937 | ||
938 | static void add_acct_request(struct request_queue *q, struct request *rq, | 938 | static void add_acct_request(struct request_queue *q, struct request *rq, |
939 | int where) | 939 | int where) |
940 | { | 940 | { |
941 | drive_stat_acct(rq, 1); | 941 | drive_stat_acct(rq, 1); |
942 | __elv_add_request(q, rq, where); | 942 | __elv_add_request(q, rq, where); |
943 | } | 943 | } |
944 | 944 | ||
945 | /** | 945 | /** |
946 | * blk_insert_request - insert a special request into a request queue | 946 | * blk_insert_request - insert a special request into a request queue |
947 | * @q: request queue where request should be inserted | 947 | * @q: request queue where request should be inserted |
948 | * @rq: request to be inserted | 948 | * @rq: request to be inserted |
949 | * @at_head: insert request at head or tail of queue | 949 | * @at_head: insert request at head or tail of queue |
950 | * @data: private data | 950 | * @data: private data |
951 | * | 951 | * |
952 | * Description: | 952 | * Description: |
953 | * Many block devices need to execute commands asynchronously, so they don't | 953 | * Many block devices need to execute commands asynchronously, so they don't |
954 | * block the whole kernel from preemption during request execution. This is | 954 | * block the whole kernel from preemption during request execution. This is |
955 | * accomplished normally by inserting aritficial requests tagged as | 955 | * accomplished normally by inserting aritficial requests tagged as |
956 | * REQ_TYPE_SPECIAL in to the corresponding request queue, and letting them | 956 | * REQ_TYPE_SPECIAL in to the corresponding request queue, and letting them |
957 | * be scheduled for actual execution by the request queue. | 957 | * be scheduled for actual execution by the request queue. |
958 | * | 958 | * |
959 | * We have the option of inserting the head or the tail of the queue. | 959 | * We have the option of inserting the head or the tail of the queue. |
960 | * Typically we use the tail for new ioctls and so forth. We use the head | 960 | * Typically we use the tail for new ioctls and so forth. We use the head |
961 | * of the queue for things like a QUEUE_FULL message from a device, or a | 961 | * of the queue for things like a QUEUE_FULL message from a device, or a |
962 | * host that is unable to accept a particular command. | 962 | * host that is unable to accept a particular command. |
963 | */ | 963 | */ |
964 | void blk_insert_request(struct request_queue *q, struct request *rq, | 964 | void blk_insert_request(struct request_queue *q, struct request *rq, |
965 | int at_head, void *data) | 965 | int at_head, void *data) |
966 | { | 966 | { |
967 | int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK; | 967 | int where = at_head ? ELEVATOR_INSERT_FRONT : ELEVATOR_INSERT_BACK; |
968 | unsigned long flags; | 968 | unsigned long flags; |
969 | 969 | ||
970 | /* | 970 | /* |
971 | * tell I/O scheduler that this isn't a regular read/write (ie it | 971 | * tell I/O scheduler that this isn't a regular read/write (ie it |
972 | * must not attempt merges on this) and that it acts as a soft | 972 | * must not attempt merges on this) and that it acts as a soft |
973 | * barrier | 973 | * barrier |
974 | */ | 974 | */ |
975 | rq->cmd_type = REQ_TYPE_SPECIAL; | 975 | rq->cmd_type = REQ_TYPE_SPECIAL; |
976 | 976 | ||
977 | rq->special = data; | 977 | rq->special = data; |
978 | 978 | ||
979 | spin_lock_irqsave(q->queue_lock, flags); | 979 | spin_lock_irqsave(q->queue_lock, flags); |
980 | 980 | ||
981 | /* | 981 | /* |
982 | * If command is tagged, release the tag | 982 | * If command is tagged, release the tag |
983 | */ | 983 | */ |
984 | if (blk_rq_tagged(rq)) | 984 | if (blk_rq_tagged(rq)) |
985 | blk_queue_end_tag(q, rq); | 985 | blk_queue_end_tag(q, rq); |
986 | 986 | ||
987 | add_acct_request(q, rq, where); | 987 | add_acct_request(q, rq, where); |
988 | __blk_run_queue(q); | 988 | __blk_run_queue(q); |
989 | spin_unlock_irqrestore(q->queue_lock, flags); | 989 | spin_unlock_irqrestore(q->queue_lock, flags); |
990 | } | 990 | } |
991 | EXPORT_SYMBOL(blk_insert_request); | 991 | EXPORT_SYMBOL(blk_insert_request); |
992 | 992 | ||
993 | static void part_round_stats_single(int cpu, struct hd_struct *part, | 993 | static void part_round_stats_single(int cpu, struct hd_struct *part, |
994 | unsigned long now) | 994 | unsigned long now) |
995 | { | 995 | { |
996 | if (now == part->stamp) | 996 | if (now == part->stamp) |
997 | return; | 997 | return; |
998 | 998 | ||
999 | if (part_in_flight(part)) { | 999 | if (part_in_flight(part)) { |
1000 | __part_stat_add(cpu, part, time_in_queue, | 1000 | __part_stat_add(cpu, part, time_in_queue, |
1001 | part_in_flight(part) * (now - part->stamp)); | 1001 | part_in_flight(part) * (now - part->stamp)); |
1002 | __part_stat_add(cpu, part, io_ticks, (now - part->stamp)); | 1002 | __part_stat_add(cpu, part, io_ticks, (now - part->stamp)); |
1003 | } | 1003 | } |
1004 | part->stamp = now; | 1004 | part->stamp = now; |
1005 | } | 1005 | } |
1006 | 1006 | ||
1007 | /** | 1007 | /** |
1008 | * part_round_stats() - Round off the performance stats on a struct disk_stats. | 1008 | * part_round_stats() - Round off the performance stats on a struct disk_stats. |
1009 | * @cpu: cpu number for stats access | 1009 | * @cpu: cpu number for stats access |
1010 | * @part: target partition | 1010 | * @part: target partition |
1011 | * | 1011 | * |
1012 | * The average IO queue length and utilisation statistics are maintained | 1012 | * The average IO queue length and utilisation statistics are maintained |
1013 | * by observing the current state of the queue length and the amount of | 1013 | * by observing the current state of the queue length and the amount of |
1014 | * time it has been in this state for. | 1014 | * time it has been in this state for. |
1015 | * | 1015 | * |
1016 | * Normally, that accounting is done on IO completion, but that can result | 1016 | * Normally, that accounting is done on IO completion, but that can result |
1017 | * in more than a second's worth of IO being accounted for within any one | 1017 | * in more than a second's worth of IO being accounted for within any one |
1018 | * second, leading to >100% utilisation. To deal with that, we call this | 1018 | * second, leading to >100% utilisation. To deal with that, we call this |
1019 | * function to do a round-off before returning the results when reading | 1019 | * function to do a round-off before returning the results when reading |
1020 | * /proc/diskstats. This accounts immediately for all queue usage up to | 1020 | * /proc/diskstats. This accounts immediately for all queue usage up to |
1021 | * the current jiffies and restarts the counters again. | 1021 | * the current jiffies and restarts the counters again. |
1022 | */ | 1022 | */ |
1023 | void part_round_stats(int cpu, struct hd_struct *part) | 1023 | void part_round_stats(int cpu, struct hd_struct *part) |
1024 | { | 1024 | { |
1025 | unsigned long now = jiffies; | 1025 | unsigned long now = jiffies; |
1026 | 1026 | ||
1027 | if (part->partno) | 1027 | if (part->partno) |
1028 | part_round_stats_single(cpu, &part_to_disk(part)->part0, now); | 1028 | part_round_stats_single(cpu, &part_to_disk(part)->part0, now); |
1029 | part_round_stats_single(cpu, part, now); | 1029 | part_round_stats_single(cpu, part, now); |
1030 | } | 1030 | } |
1031 | EXPORT_SYMBOL_GPL(part_round_stats); | 1031 | EXPORT_SYMBOL_GPL(part_round_stats); |
1032 | 1032 | ||
1033 | /* | 1033 | /* |
1034 | * queue lock must be held | 1034 | * queue lock must be held |
1035 | */ | 1035 | */ |
1036 | void __blk_put_request(struct request_queue *q, struct request *req) | 1036 | void __blk_put_request(struct request_queue *q, struct request *req) |
1037 | { | 1037 | { |
1038 | if (unlikely(!q)) | 1038 | if (unlikely(!q)) |
1039 | return; | 1039 | return; |
1040 | if (unlikely(--req->ref_count)) | 1040 | if (unlikely(--req->ref_count)) |
1041 | return; | 1041 | return; |
1042 | 1042 | ||
1043 | elv_completed_request(q, req); | 1043 | elv_completed_request(q, req); |
1044 | 1044 | ||
1045 | /* this is a bio leak */ | 1045 | /* this is a bio leak */ |
1046 | WARN_ON(req->bio != NULL); | 1046 | WARN_ON(req->bio != NULL); |
1047 | 1047 | ||
1048 | /* | 1048 | /* |
1049 | * Request may not have originated from ll_rw_blk. if not, | 1049 | * Request may not have originated from ll_rw_blk. if not, |
1050 | * it didn't come out of our reserved rq pools | 1050 | * it didn't come out of our reserved rq pools |
1051 | */ | 1051 | */ |
1052 | if (req->cmd_flags & REQ_ALLOCED) { | 1052 | if (req->cmd_flags & REQ_ALLOCED) { |
1053 | int is_sync = rq_is_sync(req) != 0; | 1053 | int is_sync = rq_is_sync(req) != 0; |
1054 | int priv = req->cmd_flags & REQ_ELVPRIV; | 1054 | int priv = req->cmd_flags & REQ_ELVPRIV; |
1055 | 1055 | ||
1056 | BUG_ON(!list_empty(&req->queuelist)); | 1056 | BUG_ON(!list_empty(&req->queuelist)); |
1057 | BUG_ON(!hlist_unhashed(&req->hash)); | 1057 | BUG_ON(!hlist_unhashed(&req->hash)); |
1058 | 1058 | ||
1059 | blk_free_request(q, req); | 1059 | blk_free_request(q, req); |
1060 | freed_request(q, is_sync, priv); | 1060 | freed_request(q, is_sync, priv); |
1061 | } | 1061 | } |
1062 | } | 1062 | } |
1063 | EXPORT_SYMBOL_GPL(__blk_put_request); | 1063 | EXPORT_SYMBOL_GPL(__blk_put_request); |
1064 | 1064 | ||
1065 | void blk_put_request(struct request *req) | 1065 | void blk_put_request(struct request *req) |
1066 | { | 1066 | { |
1067 | unsigned long flags; | 1067 | unsigned long flags; |
1068 | struct request_queue *q = req->q; | 1068 | struct request_queue *q = req->q; |
1069 | 1069 | ||
1070 | spin_lock_irqsave(q->queue_lock, flags); | 1070 | spin_lock_irqsave(q->queue_lock, flags); |
1071 | __blk_put_request(q, req); | 1071 | __blk_put_request(q, req); |
1072 | spin_unlock_irqrestore(q->queue_lock, flags); | 1072 | spin_unlock_irqrestore(q->queue_lock, flags); |
1073 | } | 1073 | } |
1074 | EXPORT_SYMBOL(blk_put_request); | 1074 | EXPORT_SYMBOL(blk_put_request); |
1075 | 1075 | ||
1076 | /** | 1076 | /** |
1077 | * blk_add_request_payload - add a payload to a request | 1077 | * blk_add_request_payload - add a payload to a request |
1078 | * @rq: request to update | 1078 | * @rq: request to update |
1079 | * @page: page backing the payload | 1079 | * @page: page backing the payload |
1080 | * @len: length of the payload. | 1080 | * @len: length of the payload. |
1081 | * | 1081 | * |
1082 | * This allows to later add a payload to an already submitted request by | 1082 | * This allows to later add a payload to an already submitted request by |
1083 | * a block driver. The driver needs to take care of freeing the payload | 1083 | * a block driver. The driver needs to take care of freeing the payload |
1084 | * itself. | 1084 | * itself. |
1085 | * | 1085 | * |
1086 | * Note that this is a quite horrible hack and nothing but handling of | 1086 | * Note that this is a quite horrible hack and nothing but handling of |
1087 | * discard requests should ever use it. | 1087 | * discard requests should ever use it. |
1088 | */ | 1088 | */ |
1089 | void blk_add_request_payload(struct request *rq, struct page *page, | 1089 | void blk_add_request_payload(struct request *rq, struct page *page, |
1090 | unsigned int len) | 1090 | unsigned int len) |
1091 | { | 1091 | { |
1092 | struct bio *bio = rq->bio; | 1092 | struct bio *bio = rq->bio; |
1093 | 1093 | ||
1094 | bio->bi_io_vec->bv_page = page; | 1094 | bio->bi_io_vec->bv_page = page; |
1095 | bio->bi_io_vec->bv_offset = 0; | 1095 | bio->bi_io_vec->bv_offset = 0; |
1096 | bio->bi_io_vec->bv_len = len; | 1096 | bio->bi_io_vec->bv_len = len; |
1097 | 1097 | ||
1098 | bio->bi_size = len; | 1098 | bio->bi_size = len; |
1099 | bio->bi_vcnt = 1; | 1099 | bio->bi_vcnt = 1; |
1100 | bio->bi_phys_segments = 1; | 1100 | bio->bi_phys_segments = 1; |
1101 | 1101 | ||
1102 | rq->__data_len = rq->resid_len = len; | 1102 | rq->__data_len = rq->resid_len = len; |
1103 | rq->nr_phys_segments = 1; | 1103 | rq->nr_phys_segments = 1; |
1104 | rq->buffer = bio_data(bio); | 1104 | rq->buffer = bio_data(bio); |
1105 | } | 1105 | } |
1106 | EXPORT_SYMBOL_GPL(blk_add_request_payload); | 1106 | EXPORT_SYMBOL_GPL(blk_add_request_payload); |
1107 | 1107 | ||
1108 | static bool bio_attempt_back_merge(struct request_queue *q, struct request *req, | 1108 | static bool bio_attempt_back_merge(struct request_queue *q, struct request *req, |
1109 | struct bio *bio) | 1109 | struct bio *bio) |
1110 | { | 1110 | { |
1111 | const int ff = bio->bi_rw & REQ_FAILFAST_MASK; | 1111 | const int ff = bio->bi_rw & REQ_FAILFAST_MASK; |
1112 | 1112 | ||
1113 | if (!ll_back_merge_fn(q, req, bio)) | 1113 | if (!ll_back_merge_fn(q, req, bio)) |
1114 | return false; | 1114 | return false; |
1115 | 1115 | ||
1116 | trace_block_bio_backmerge(q, bio); | 1116 | trace_block_bio_backmerge(q, bio); |
1117 | 1117 | ||
1118 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) | 1118 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) |
1119 | blk_rq_set_mixed_merge(req); | 1119 | blk_rq_set_mixed_merge(req); |
1120 | 1120 | ||
1121 | req->biotail->bi_next = bio; | 1121 | req->biotail->bi_next = bio; |
1122 | req->biotail = bio; | 1122 | req->biotail = bio; |
1123 | req->__data_len += bio->bi_size; | 1123 | req->__data_len += bio->bi_size; |
1124 | req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); | 1124 | req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); |
1125 | 1125 | ||
1126 | drive_stat_acct(req, 0); | 1126 | drive_stat_acct(req, 0); |
1127 | elv_bio_merged(q, req, bio); | 1127 | elv_bio_merged(q, req, bio); |
1128 | return true; | 1128 | return true; |
1129 | } | 1129 | } |
1130 | 1130 | ||
1131 | static bool bio_attempt_front_merge(struct request_queue *q, | 1131 | static bool bio_attempt_front_merge(struct request_queue *q, |
1132 | struct request *req, struct bio *bio) | 1132 | struct request *req, struct bio *bio) |
1133 | { | 1133 | { |
1134 | const int ff = bio->bi_rw & REQ_FAILFAST_MASK; | 1134 | const int ff = bio->bi_rw & REQ_FAILFAST_MASK; |
1135 | 1135 | ||
1136 | if (!ll_front_merge_fn(q, req, bio)) | 1136 | if (!ll_front_merge_fn(q, req, bio)) |
1137 | return false; | 1137 | return false; |
1138 | 1138 | ||
1139 | trace_block_bio_frontmerge(q, bio); | 1139 | trace_block_bio_frontmerge(q, bio); |
1140 | 1140 | ||
1141 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) | 1141 | if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff) |
1142 | blk_rq_set_mixed_merge(req); | 1142 | blk_rq_set_mixed_merge(req); |
1143 | 1143 | ||
1144 | bio->bi_next = req->bio; | 1144 | bio->bi_next = req->bio; |
1145 | req->bio = bio; | 1145 | req->bio = bio; |
1146 | 1146 | ||
1147 | /* | 1147 | /* |
1148 | * may not be valid. if the low level driver said | 1148 | * may not be valid. if the low level driver said |
1149 | * it didn't need a bounce buffer then it better | 1149 | * it didn't need a bounce buffer then it better |
1150 | * not touch req->buffer either... | 1150 | * not touch req->buffer either... |
1151 | */ | 1151 | */ |
1152 | req->buffer = bio_data(bio); | 1152 | req->buffer = bio_data(bio); |
1153 | req->__sector = bio->bi_sector; | 1153 | req->__sector = bio->bi_sector; |
1154 | req->__data_len += bio->bi_size; | 1154 | req->__data_len += bio->bi_size; |
1155 | req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); | 1155 | req->ioprio = ioprio_best(req->ioprio, bio_prio(bio)); |
1156 | 1156 | ||
1157 | drive_stat_acct(req, 0); | 1157 | drive_stat_acct(req, 0); |
1158 | elv_bio_merged(q, req, bio); | 1158 | elv_bio_merged(q, req, bio); |
1159 | return true; | 1159 | return true; |
1160 | } | 1160 | } |
1161 | 1161 | ||
1162 | /* | 1162 | /* |
1163 | * Attempts to merge with the plugged list in the current process. Returns | 1163 | * Attempts to merge with the plugged list in the current process. Returns |
1164 | * true if merge was successful, otherwise false. | 1164 | * true if merge was successful, otherwise false. |
1165 | */ | 1165 | */ |
1166 | static bool attempt_plug_merge(struct task_struct *tsk, struct request_queue *q, | 1166 | static bool attempt_plug_merge(struct task_struct *tsk, struct request_queue *q, |
1167 | struct bio *bio) | 1167 | struct bio *bio) |
1168 | { | 1168 | { |
1169 | struct blk_plug *plug; | 1169 | struct blk_plug *plug; |
1170 | struct request *rq; | 1170 | struct request *rq; |
1171 | bool ret = false; | 1171 | bool ret = false; |
1172 | 1172 | ||
1173 | plug = tsk->plug; | 1173 | plug = tsk->plug; |
1174 | if (!plug) | 1174 | if (!plug) |
1175 | goto out; | 1175 | goto out; |
1176 | 1176 | ||
1177 | list_for_each_entry_reverse(rq, &plug->list, queuelist) { | 1177 | list_for_each_entry_reverse(rq, &plug->list, queuelist) { |
1178 | int el_ret; | 1178 | int el_ret; |
1179 | 1179 | ||
1180 | if (rq->q != q) | 1180 | if (rq->q != q) |
1181 | continue; | 1181 | continue; |
1182 | 1182 | ||
1183 | el_ret = elv_try_merge(rq, bio); | 1183 | el_ret = elv_try_merge(rq, bio); |
1184 | if (el_ret == ELEVATOR_BACK_MERGE) { | 1184 | if (el_ret == ELEVATOR_BACK_MERGE) { |
1185 | ret = bio_attempt_back_merge(q, rq, bio); | 1185 | ret = bio_attempt_back_merge(q, rq, bio); |
1186 | if (ret) | 1186 | if (ret) |
1187 | break; | 1187 | break; |
1188 | } else if (el_ret == ELEVATOR_FRONT_MERGE) { | 1188 | } else if (el_ret == ELEVATOR_FRONT_MERGE) { |
1189 | ret = bio_attempt_front_merge(q, rq, bio); | 1189 | ret = bio_attempt_front_merge(q, rq, bio); |
1190 | if (ret) | 1190 | if (ret) |
1191 | break; | 1191 | break; |
1192 | } | 1192 | } |
1193 | } | 1193 | } |
1194 | out: | 1194 | out: |
1195 | return ret; | 1195 | return ret; |
1196 | } | 1196 | } |
1197 | 1197 | ||
1198 | void init_request_from_bio(struct request *req, struct bio *bio) | 1198 | void init_request_from_bio(struct request *req, struct bio *bio) |
1199 | { | 1199 | { |
1200 | req->cpu = bio->bi_comp_cpu; | 1200 | req->cpu = bio->bi_comp_cpu; |
1201 | req->cmd_type = REQ_TYPE_FS; | 1201 | req->cmd_type = REQ_TYPE_FS; |
1202 | 1202 | ||
1203 | req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK; | 1203 | req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK; |
1204 | if (bio->bi_rw & REQ_RAHEAD) | 1204 | if (bio->bi_rw & REQ_RAHEAD) |
1205 | req->cmd_flags |= REQ_FAILFAST_MASK; | 1205 | req->cmd_flags |= REQ_FAILFAST_MASK; |
1206 | 1206 | ||
1207 | req->errors = 0; | 1207 | req->errors = 0; |
1208 | req->__sector = bio->bi_sector; | 1208 | req->__sector = bio->bi_sector; |
1209 | req->ioprio = bio_prio(bio); | 1209 | req->ioprio = bio_prio(bio); |
1210 | blk_rq_bio_prep(req->q, req, bio); | 1210 | blk_rq_bio_prep(req->q, req, bio); |
1211 | } | 1211 | } |
1212 | 1212 | ||
1213 | static int __make_request(struct request_queue *q, struct bio *bio) | 1213 | static int __make_request(struct request_queue *q, struct bio *bio) |
1214 | { | 1214 | { |
1215 | const bool sync = !!(bio->bi_rw & REQ_SYNC); | 1215 | const bool sync = !!(bio->bi_rw & REQ_SYNC); |
1216 | struct blk_plug *plug; | 1216 | struct blk_plug *plug; |
1217 | int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT; | 1217 | int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT; |
1218 | struct request *req; | 1218 | struct request *req; |
1219 | 1219 | ||
1220 | /* | 1220 | /* |
1221 | * low level driver can indicate that it wants pages above a | 1221 | * low level driver can indicate that it wants pages above a |
1222 | * certain limit bounced to low memory (ie for highmem, or even | 1222 | * certain limit bounced to low memory (ie for highmem, or even |
1223 | * ISA dma in theory) | 1223 | * ISA dma in theory) |
1224 | */ | 1224 | */ |
1225 | blk_queue_bounce(q, &bio); | 1225 | blk_queue_bounce(q, &bio); |
1226 | 1226 | ||
1227 | if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { | 1227 | if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) { |
1228 | spin_lock_irq(q->queue_lock); | 1228 | spin_lock_irq(q->queue_lock); |
1229 | where = ELEVATOR_INSERT_FLUSH; | 1229 | where = ELEVATOR_INSERT_FLUSH; |
1230 | goto get_rq; | 1230 | goto get_rq; |
1231 | } | 1231 | } |
1232 | 1232 | ||
1233 | /* | 1233 | /* |
1234 | * Check if we can merge with the plugged list before grabbing | 1234 | * Check if we can merge with the plugged list before grabbing |
1235 | * any locks. | 1235 | * any locks. |
1236 | */ | 1236 | */ |
1237 | if (attempt_plug_merge(current, q, bio)) | 1237 | if (attempt_plug_merge(current, q, bio)) |
1238 | goto out; | 1238 | goto out; |
1239 | 1239 | ||
1240 | spin_lock_irq(q->queue_lock); | 1240 | spin_lock_irq(q->queue_lock); |
1241 | 1241 | ||
1242 | el_ret = elv_merge(q, &req, bio); | 1242 | el_ret = elv_merge(q, &req, bio); |
1243 | if (el_ret == ELEVATOR_BACK_MERGE) { | 1243 | if (el_ret == ELEVATOR_BACK_MERGE) { |
1244 | if (bio_attempt_back_merge(q, req, bio)) { | 1244 | if (bio_attempt_back_merge(q, req, bio)) { |
1245 | if (!attempt_back_merge(q, req)) | 1245 | if (!attempt_back_merge(q, req)) |
1246 | elv_merged_request(q, req, el_ret); | 1246 | elv_merged_request(q, req, el_ret); |
1247 | goto out_unlock; | 1247 | goto out_unlock; |
1248 | } | 1248 | } |
1249 | } else if (el_ret == ELEVATOR_FRONT_MERGE) { | 1249 | } else if (el_ret == ELEVATOR_FRONT_MERGE) { |
1250 | if (bio_attempt_front_merge(q, req, bio)) { | 1250 | if (bio_attempt_front_merge(q, req, bio)) { |
1251 | if (!attempt_front_merge(q, req)) | 1251 | if (!attempt_front_merge(q, req)) |
1252 | elv_merged_request(q, req, el_ret); | 1252 | elv_merged_request(q, req, el_ret); |
1253 | goto out_unlock; | 1253 | goto out_unlock; |
1254 | } | 1254 | } |
1255 | } | 1255 | } |
1256 | 1256 | ||
1257 | get_rq: | 1257 | get_rq: |
1258 | /* | 1258 | /* |
1259 | * This sync check and mask will be re-done in init_request_from_bio(), | 1259 | * This sync check and mask will be re-done in init_request_from_bio(), |
1260 | * but we need to set it earlier to expose the sync flag to the | 1260 | * but we need to set it earlier to expose the sync flag to the |
1261 | * rq allocator and io schedulers. | 1261 | * rq allocator and io schedulers. |
1262 | */ | 1262 | */ |
1263 | rw_flags = bio_data_dir(bio); | 1263 | rw_flags = bio_data_dir(bio); |
1264 | if (sync) | 1264 | if (sync) |
1265 | rw_flags |= REQ_SYNC; | 1265 | rw_flags |= REQ_SYNC; |
1266 | 1266 | ||
1267 | /* | 1267 | /* |
1268 | * Grab a free request. This is might sleep but can not fail. | 1268 | * Grab a free request. This is might sleep but can not fail. |
1269 | * Returns with the queue unlocked. | 1269 | * Returns with the queue unlocked. |
1270 | */ | 1270 | */ |
1271 | req = get_request_wait(q, rw_flags, bio); | 1271 | req = get_request_wait(q, rw_flags, bio); |
1272 | 1272 | ||
1273 | /* | 1273 | /* |
1274 | * After dropping the lock and possibly sleeping here, our request | 1274 | * After dropping the lock and possibly sleeping here, our request |
1275 | * may now be mergeable after it had proven unmergeable (above). | 1275 | * may now be mergeable after it had proven unmergeable (above). |
1276 | * We don't worry about that case for efficiency. It won't happen | 1276 | * We don't worry about that case for efficiency. It won't happen |
1277 | * often, and the elevators are able to handle it. | 1277 | * often, and the elevators are able to handle it. |
1278 | */ | 1278 | */ |
1279 | init_request_from_bio(req, bio); | 1279 | init_request_from_bio(req, bio); |
1280 | 1280 | ||
1281 | if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) || | 1281 | if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags) || |
1282 | bio_flagged(bio, BIO_CPU_AFFINE)) { | 1282 | bio_flagged(bio, BIO_CPU_AFFINE)) { |
1283 | req->cpu = blk_cpu_to_group(get_cpu()); | 1283 | req->cpu = blk_cpu_to_group(get_cpu()); |
1284 | put_cpu(); | 1284 | put_cpu(); |
1285 | } | 1285 | } |
1286 | 1286 | ||
1287 | plug = current->plug; | 1287 | plug = current->plug; |
1288 | if (plug) { | 1288 | if (plug) { |
1289 | /* | 1289 | /* |
1290 | * If this is the first request added after a plug, fire | 1290 | * If this is the first request added after a plug, fire |
1291 | * of a plug trace. If others have been added before, check | 1291 | * of a plug trace. If others have been added before, check |
1292 | * if we have multiple devices in this plug. If so, make a | 1292 | * if we have multiple devices in this plug. If so, make a |
1293 | * note to sort the list before dispatch. | 1293 | * note to sort the list before dispatch. |
1294 | */ | 1294 | */ |
1295 | if (list_empty(&plug->list)) | 1295 | if (list_empty(&plug->list)) |
1296 | trace_block_plug(q); | 1296 | trace_block_plug(q); |
1297 | else if (!plug->should_sort) { | 1297 | else if (!plug->should_sort) { |
1298 | struct request *__rq; | 1298 | struct request *__rq; |
1299 | 1299 | ||
1300 | __rq = list_entry_rq(plug->list.prev); | 1300 | __rq = list_entry_rq(plug->list.prev); |
1301 | if (__rq->q != q) | 1301 | if (__rq->q != q) |
1302 | plug->should_sort = 1; | 1302 | plug->should_sort = 1; |
1303 | } | 1303 | } |
1304 | list_add_tail(&req->queuelist, &plug->list); | 1304 | list_add_tail(&req->queuelist, &plug->list); |
1305 | plug->count++; | ||
1305 | drive_stat_acct(req, 1); | 1306 | drive_stat_acct(req, 1); |
1307 | if (plug->count >= BLK_MAX_REQUEST_COUNT) | ||
1308 | blk_flush_plug_list(plug, false); | ||
1306 | } else { | 1309 | } else { |
1307 | spin_lock_irq(q->queue_lock); | 1310 | spin_lock_irq(q->queue_lock); |
1308 | add_acct_request(q, req, where); | 1311 | add_acct_request(q, req, where); |
1309 | __blk_run_queue(q); | 1312 | __blk_run_queue(q); |
1310 | out_unlock: | 1313 | out_unlock: |
1311 | spin_unlock_irq(q->queue_lock); | 1314 | spin_unlock_irq(q->queue_lock); |
1312 | } | 1315 | } |
1313 | out: | 1316 | out: |
1314 | return 0; | 1317 | return 0; |
1315 | } | 1318 | } |
1316 | 1319 | ||
1317 | /* | 1320 | /* |
1318 | * If bio->bi_dev is a partition, remap the location | 1321 | * If bio->bi_dev is a partition, remap the location |
1319 | */ | 1322 | */ |
1320 | static inline void blk_partition_remap(struct bio *bio) | 1323 | static inline void blk_partition_remap(struct bio *bio) |
1321 | { | 1324 | { |
1322 | struct block_device *bdev = bio->bi_bdev; | 1325 | struct block_device *bdev = bio->bi_bdev; |
1323 | 1326 | ||
1324 | if (bio_sectors(bio) && bdev != bdev->bd_contains) { | 1327 | if (bio_sectors(bio) && bdev != bdev->bd_contains) { |
1325 | struct hd_struct *p = bdev->bd_part; | 1328 | struct hd_struct *p = bdev->bd_part; |
1326 | 1329 | ||
1327 | bio->bi_sector += p->start_sect; | 1330 | bio->bi_sector += p->start_sect; |
1328 | bio->bi_bdev = bdev->bd_contains; | 1331 | bio->bi_bdev = bdev->bd_contains; |
1329 | 1332 | ||
1330 | trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio, | 1333 | trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio, |
1331 | bdev->bd_dev, | 1334 | bdev->bd_dev, |
1332 | bio->bi_sector - p->start_sect); | 1335 | bio->bi_sector - p->start_sect); |
1333 | } | 1336 | } |
1334 | } | 1337 | } |
1335 | 1338 | ||
1336 | static void handle_bad_sector(struct bio *bio) | 1339 | static void handle_bad_sector(struct bio *bio) |
1337 | { | 1340 | { |
1338 | char b[BDEVNAME_SIZE]; | 1341 | char b[BDEVNAME_SIZE]; |
1339 | 1342 | ||
1340 | printk(KERN_INFO "attempt to access beyond end of device\n"); | 1343 | printk(KERN_INFO "attempt to access beyond end of device\n"); |
1341 | printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n", | 1344 | printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n", |
1342 | bdevname(bio->bi_bdev, b), | 1345 | bdevname(bio->bi_bdev, b), |
1343 | bio->bi_rw, | 1346 | bio->bi_rw, |
1344 | (unsigned long long)bio->bi_sector + bio_sectors(bio), | 1347 | (unsigned long long)bio->bi_sector + bio_sectors(bio), |
1345 | (long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9)); | 1348 | (long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9)); |
1346 | 1349 | ||
1347 | set_bit(BIO_EOF, &bio->bi_flags); | 1350 | set_bit(BIO_EOF, &bio->bi_flags); |
1348 | } | 1351 | } |
1349 | 1352 | ||
1350 | #ifdef CONFIG_FAIL_MAKE_REQUEST | 1353 | #ifdef CONFIG_FAIL_MAKE_REQUEST |
1351 | 1354 | ||
1352 | static DECLARE_FAULT_ATTR(fail_make_request); | 1355 | static DECLARE_FAULT_ATTR(fail_make_request); |
1353 | 1356 | ||
1354 | static int __init setup_fail_make_request(char *str) | 1357 | static int __init setup_fail_make_request(char *str) |
1355 | { | 1358 | { |
1356 | return setup_fault_attr(&fail_make_request, str); | 1359 | return setup_fault_attr(&fail_make_request, str); |
1357 | } | 1360 | } |
1358 | __setup("fail_make_request=", setup_fail_make_request); | 1361 | __setup("fail_make_request=", setup_fail_make_request); |
1359 | 1362 | ||
1360 | static int should_fail_request(struct bio *bio) | 1363 | static int should_fail_request(struct bio *bio) |
1361 | { | 1364 | { |
1362 | struct hd_struct *part = bio->bi_bdev->bd_part; | 1365 | struct hd_struct *part = bio->bi_bdev->bd_part; |
1363 | 1366 | ||
1364 | if (part_to_disk(part)->part0.make_it_fail || part->make_it_fail) | 1367 | if (part_to_disk(part)->part0.make_it_fail || part->make_it_fail) |
1365 | return should_fail(&fail_make_request, bio->bi_size); | 1368 | return should_fail(&fail_make_request, bio->bi_size); |
1366 | 1369 | ||
1367 | return 0; | 1370 | return 0; |
1368 | } | 1371 | } |
1369 | 1372 | ||
1370 | static int __init fail_make_request_debugfs(void) | 1373 | static int __init fail_make_request_debugfs(void) |
1371 | { | 1374 | { |
1372 | return init_fault_attr_dentries(&fail_make_request, | 1375 | return init_fault_attr_dentries(&fail_make_request, |
1373 | "fail_make_request"); | 1376 | "fail_make_request"); |
1374 | } | 1377 | } |
1375 | 1378 | ||
1376 | late_initcall(fail_make_request_debugfs); | 1379 | late_initcall(fail_make_request_debugfs); |
1377 | 1380 | ||
1378 | #else /* CONFIG_FAIL_MAKE_REQUEST */ | 1381 | #else /* CONFIG_FAIL_MAKE_REQUEST */ |
1379 | 1382 | ||
1380 | static inline int should_fail_request(struct bio *bio) | 1383 | static inline int should_fail_request(struct bio *bio) |
1381 | { | 1384 | { |
1382 | return 0; | 1385 | return 0; |
1383 | } | 1386 | } |
1384 | 1387 | ||
1385 | #endif /* CONFIG_FAIL_MAKE_REQUEST */ | 1388 | #endif /* CONFIG_FAIL_MAKE_REQUEST */ |
1386 | 1389 | ||
1387 | /* | 1390 | /* |
1388 | * Check whether this bio extends beyond the end of the device. | 1391 | * Check whether this bio extends beyond the end of the device. |
1389 | */ | 1392 | */ |
1390 | static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors) | 1393 | static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors) |
1391 | { | 1394 | { |
1392 | sector_t maxsector; | 1395 | sector_t maxsector; |
1393 | 1396 | ||
1394 | if (!nr_sectors) | 1397 | if (!nr_sectors) |
1395 | return 0; | 1398 | return 0; |
1396 | 1399 | ||
1397 | /* Test device or partition size, when known. */ | 1400 | /* Test device or partition size, when known. */ |
1398 | maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9; | 1401 | maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9; |
1399 | if (maxsector) { | 1402 | if (maxsector) { |
1400 | sector_t sector = bio->bi_sector; | 1403 | sector_t sector = bio->bi_sector; |
1401 | 1404 | ||
1402 | if (maxsector < nr_sectors || maxsector - nr_sectors < sector) { | 1405 | if (maxsector < nr_sectors || maxsector - nr_sectors < sector) { |
1403 | /* | 1406 | /* |
1404 | * This may well happen - the kernel calls bread() | 1407 | * This may well happen - the kernel calls bread() |
1405 | * without checking the size of the device, e.g., when | 1408 | * without checking the size of the device, e.g., when |
1406 | * mounting a device. | 1409 | * mounting a device. |
1407 | */ | 1410 | */ |
1408 | handle_bad_sector(bio); | 1411 | handle_bad_sector(bio); |
1409 | return 1; | 1412 | return 1; |
1410 | } | 1413 | } |
1411 | } | 1414 | } |
1412 | 1415 | ||
1413 | return 0; | 1416 | return 0; |
1414 | } | 1417 | } |
1415 | 1418 | ||
1416 | /** | 1419 | /** |
1417 | * generic_make_request - hand a buffer to its device driver for I/O | 1420 | * generic_make_request - hand a buffer to its device driver for I/O |
1418 | * @bio: The bio describing the location in memory and on the device. | 1421 | * @bio: The bio describing the location in memory and on the device. |
1419 | * | 1422 | * |
1420 | * generic_make_request() is used to make I/O requests of block | 1423 | * generic_make_request() is used to make I/O requests of block |
1421 | * devices. It is passed a &struct bio, which describes the I/O that needs | 1424 | * devices. It is passed a &struct bio, which describes the I/O that needs |
1422 | * to be done. | 1425 | * to be done. |
1423 | * | 1426 | * |
1424 | * generic_make_request() does not return any status. The | 1427 | * generic_make_request() does not return any status. The |
1425 | * success/failure status of the request, along with notification of | 1428 | * success/failure status of the request, along with notification of |
1426 | * completion, is delivered asynchronously through the bio->bi_end_io | 1429 | * completion, is delivered asynchronously through the bio->bi_end_io |
1427 | * function described (one day) else where. | 1430 | * function described (one day) else where. |
1428 | * | 1431 | * |
1429 | * The caller of generic_make_request must make sure that bi_io_vec | 1432 | * The caller of generic_make_request must make sure that bi_io_vec |
1430 | * are set to describe the memory buffer, and that bi_dev and bi_sector are | 1433 | * are set to describe the memory buffer, and that bi_dev and bi_sector are |
1431 | * set to describe the device address, and the | 1434 | * set to describe the device address, and the |
1432 | * bi_end_io and optionally bi_private are set to describe how | 1435 | * bi_end_io and optionally bi_private are set to describe how |
1433 | * completion notification should be signaled. | 1436 | * completion notification should be signaled. |
1434 | * | 1437 | * |
1435 | * generic_make_request and the drivers it calls may use bi_next if this | 1438 | * generic_make_request and the drivers it calls may use bi_next if this |
1436 | * bio happens to be merged with someone else, and may change bi_dev and | 1439 | * bio happens to be merged with someone else, and may change bi_dev and |
1437 | * bi_sector for remaps as it sees fit. So the values of these fields | 1440 | * bi_sector for remaps as it sees fit. So the values of these fields |
1438 | * should NOT be depended on after the call to generic_make_request. | 1441 | * should NOT be depended on after the call to generic_make_request. |
1439 | */ | 1442 | */ |
1440 | static inline void __generic_make_request(struct bio *bio) | 1443 | static inline void __generic_make_request(struct bio *bio) |
1441 | { | 1444 | { |
1442 | struct request_queue *q; | 1445 | struct request_queue *q; |
1443 | sector_t old_sector; | 1446 | sector_t old_sector; |
1444 | int ret, nr_sectors = bio_sectors(bio); | 1447 | int ret, nr_sectors = bio_sectors(bio); |
1445 | dev_t old_dev; | 1448 | dev_t old_dev; |
1446 | int err = -EIO; | 1449 | int err = -EIO; |
1447 | 1450 | ||
1448 | might_sleep(); | 1451 | might_sleep(); |
1449 | 1452 | ||
1450 | if (bio_check_eod(bio, nr_sectors)) | 1453 | if (bio_check_eod(bio, nr_sectors)) |
1451 | goto end_io; | 1454 | goto end_io; |
1452 | 1455 | ||
1453 | /* | 1456 | /* |
1454 | * Resolve the mapping until finished. (drivers are | 1457 | * Resolve the mapping until finished. (drivers are |
1455 | * still free to implement/resolve their own stacking | 1458 | * still free to implement/resolve their own stacking |
1456 | * by explicitly returning 0) | 1459 | * by explicitly returning 0) |
1457 | * | 1460 | * |
1458 | * NOTE: we don't repeat the blk_size check for each new device. | 1461 | * NOTE: we don't repeat the blk_size check for each new device. |
1459 | * Stacking drivers are expected to know what they are doing. | 1462 | * Stacking drivers are expected to know what they are doing. |
1460 | */ | 1463 | */ |
1461 | old_sector = -1; | 1464 | old_sector = -1; |
1462 | old_dev = 0; | 1465 | old_dev = 0; |
1463 | do { | 1466 | do { |
1464 | char b[BDEVNAME_SIZE]; | 1467 | char b[BDEVNAME_SIZE]; |
1465 | 1468 | ||
1466 | q = bdev_get_queue(bio->bi_bdev); | 1469 | q = bdev_get_queue(bio->bi_bdev); |
1467 | if (unlikely(!q)) { | 1470 | if (unlikely(!q)) { |
1468 | printk(KERN_ERR | 1471 | printk(KERN_ERR |
1469 | "generic_make_request: Trying to access " | 1472 | "generic_make_request: Trying to access " |
1470 | "nonexistent block-device %s (%Lu)\n", | 1473 | "nonexistent block-device %s (%Lu)\n", |
1471 | bdevname(bio->bi_bdev, b), | 1474 | bdevname(bio->bi_bdev, b), |
1472 | (long long) bio->bi_sector); | 1475 | (long long) bio->bi_sector); |
1473 | goto end_io; | 1476 | goto end_io; |
1474 | } | 1477 | } |
1475 | 1478 | ||
1476 | if (unlikely(!(bio->bi_rw & REQ_DISCARD) && | 1479 | if (unlikely(!(bio->bi_rw & REQ_DISCARD) && |
1477 | nr_sectors > queue_max_hw_sectors(q))) { | 1480 | nr_sectors > queue_max_hw_sectors(q))) { |
1478 | printk(KERN_ERR "bio too big device %s (%u > %u)\n", | 1481 | printk(KERN_ERR "bio too big device %s (%u > %u)\n", |
1479 | bdevname(bio->bi_bdev, b), | 1482 | bdevname(bio->bi_bdev, b), |
1480 | bio_sectors(bio), | 1483 | bio_sectors(bio), |
1481 | queue_max_hw_sectors(q)); | 1484 | queue_max_hw_sectors(q)); |
1482 | goto end_io; | 1485 | goto end_io; |
1483 | } | 1486 | } |
1484 | 1487 | ||
1485 | if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) | 1488 | if (unlikely(test_bit(QUEUE_FLAG_DEAD, &q->queue_flags))) |
1486 | goto end_io; | 1489 | goto end_io; |
1487 | 1490 | ||
1488 | if (should_fail_request(bio)) | 1491 | if (should_fail_request(bio)) |
1489 | goto end_io; | 1492 | goto end_io; |
1490 | 1493 | ||
1491 | /* | 1494 | /* |
1492 | * If this device has partitions, remap block n | 1495 | * If this device has partitions, remap block n |
1493 | * of partition p to block n+start(p) of the disk. | 1496 | * of partition p to block n+start(p) of the disk. |
1494 | */ | 1497 | */ |
1495 | blk_partition_remap(bio); | 1498 | blk_partition_remap(bio); |
1496 | 1499 | ||
1497 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) | 1500 | if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) |
1498 | goto end_io; | 1501 | goto end_io; |
1499 | 1502 | ||
1500 | if (old_sector != -1) | 1503 | if (old_sector != -1) |
1501 | trace_block_bio_remap(q, bio, old_dev, old_sector); | 1504 | trace_block_bio_remap(q, bio, old_dev, old_sector); |
1502 | 1505 | ||
1503 | old_sector = bio->bi_sector; | 1506 | old_sector = bio->bi_sector; |
1504 | old_dev = bio->bi_bdev->bd_dev; | 1507 | old_dev = bio->bi_bdev->bd_dev; |
1505 | 1508 | ||
1506 | if (bio_check_eod(bio, nr_sectors)) | 1509 | if (bio_check_eod(bio, nr_sectors)) |
1507 | goto end_io; | 1510 | goto end_io; |
1508 | 1511 | ||
1509 | /* | 1512 | /* |
1510 | * Filter flush bio's early so that make_request based | 1513 | * Filter flush bio's early so that make_request based |
1511 | * drivers without flush support don't have to worry | 1514 | * drivers without flush support don't have to worry |
1512 | * about them. | 1515 | * about them. |
1513 | */ | 1516 | */ |
1514 | if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) { | 1517 | if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) { |
1515 | bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA); | 1518 | bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA); |
1516 | if (!nr_sectors) { | 1519 | if (!nr_sectors) { |
1517 | err = 0; | 1520 | err = 0; |
1518 | goto end_io; | 1521 | goto end_io; |
1519 | } | 1522 | } |
1520 | } | 1523 | } |
1521 | 1524 | ||
1522 | if ((bio->bi_rw & REQ_DISCARD) && | 1525 | if ((bio->bi_rw & REQ_DISCARD) && |
1523 | (!blk_queue_discard(q) || | 1526 | (!blk_queue_discard(q) || |
1524 | ((bio->bi_rw & REQ_SECURE) && | 1527 | ((bio->bi_rw & REQ_SECURE) && |
1525 | !blk_queue_secdiscard(q)))) { | 1528 | !blk_queue_secdiscard(q)))) { |
1526 | err = -EOPNOTSUPP; | 1529 | err = -EOPNOTSUPP; |
1527 | goto end_io; | 1530 | goto end_io; |
1528 | } | 1531 | } |
1529 | 1532 | ||
1530 | if (blk_throtl_bio(q, &bio)) | 1533 | if (blk_throtl_bio(q, &bio)) |
1531 | goto end_io; | 1534 | goto end_io; |
1532 | 1535 | ||
1533 | /* | 1536 | /* |
1534 | * If bio = NULL, bio has been throttled and will be submitted | 1537 | * If bio = NULL, bio has been throttled and will be submitted |
1535 | * later. | 1538 | * later. |
1536 | */ | 1539 | */ |
1537 | if (!bio) | 1540 | if (!bio) |
1538 | break; | 1541 | break; |
1539 | 1542 | ||
1540 | trace_block_bio_queue(q, bio); | 1543 | trace_block_bio_queue(q, bio); |
1541 | 1544 | ||
1542 | ret = q->make_request_fn(q, bio); | 1545 | ret = q->make_request_fn(q, bio); |
1543 | } while (ret); | 1546 | } while (ret); |
1544 | 1547 | ||
1545 | return; | 1548 | return; |
1546 | 1549 | ||
1547 | end_io: | 1550 | end_io: |
1548 | bio_endio(bio, err); | 1551 | bio_endio(bio, err); |
1549 | } | 1552 | } |
1550 | 1553 | ||
1551 | /* | 1554 | /* |
1552 | * We only want one ->make_request_fn to be active at a time, | 1555 | * We only want one ->make_request_fn to be active at a time, |
1553 | * else stack usage with stacked devices could be a problem. | 1556 | * else stack usage with stacked devices could be a problem. |
1554 | * So use current->bio_list to keep a list of requests | 1557 | * So use current->bio_list to keep a list of requests |
1555 | * submited by a make_request_fn function. | 1558 | * submited by a make_request_fn function. |
1556 | * current->bio_list is also used as a flag to say if | 1559 | * current->bio_list is also used as a flag to say if |
1557 | * generic_make_request is currently active in this task or not. | 1560 | * generic_make_request is currently active in this task or not. |
1558 | * If it is NULL, then no make_request is active. If it is non-NULL, | 1561 | * If it is NULL, then no make_request is active. If it is non-NULL, |
1559 | * then a make_request is active, and new requests should be added | 1562 | * then a make_request is active, and new requests should be added |
1560 | * at the tail | 1563 | * at the tail |
1561 | */ | 1564 | */ |
1562 | void generic_make_request(struct bio *bio) | 1565 | void generic_make_request(struct bio *bio) |
1563 | { | 1566 | { |
1564 | struct bio_list bio_list_on_stack; | 1567 | struct bio_list bio_list_on_stack; |
1565 | 1568 | ||
1566 | if (current->bio_list) { | 1569 | if (current->bio_list) { |
1567 | /* make_request is active */ | 1570 | /* make_request is active */ |
1568 | bio_list_add(current->bio_list, bio); | 1571 | bio_list_add(current->bio_list, bio); |
1569 | return; | 1572 | return; |
1570 | } | 1573 | } |
1571 | /* following loop may be a bit non-obvious, and so deserves some | 1574 | /* following loop may be a bit non-obvious, and so deserves some |
1572 | * explanation. | 1575 | * explanation. |
1573 | * Before entering the loop, bio->bi_next is NULL (as all callers | 1576 | * Before entering the loop, bio->bi_next is NULL (as all callers |
1574 | * ensure that) so we have a list with a single bio. | 1577 | * ensure that) so we have a list with a single bio. |
1575 | * We pretend that we have just taken it off a longer list, so | 1578 | * We pretend that we have just taken it off a longer list, so |
1576 | * we assign bio_list to a pointer to the bio_list_on_stack, | 1579 | * we assign bio_list to a pointer to the bio_list_on_stack, |
1577 | * thus initialising the bio_list of new bios to be | 1580 | * thus initialising the bio_list of new bios to be |
1578 | * added. __generic_make_request may indeed add some more bios | 1581 | * added. __generic_make_request may indeed add some more bios |
1579 | * through a recursive call to generic_make_request. If it | 1582 | * through a recursive call to generic_make_request. If it |
1580 | * did, we find a non-NULL value in bio_list and re-enter the loop | 1583 | * did, we find a non-NULL value in bio_list and re-enter the loop |
1581 | * from the top. In this case we really did just take the bio | 1584 | * from the top. In this case we really did just take the bio |
1582 | * of the top of the list (no pretending) and so remove it from | 1585 | * of the top of the list (no pretending) and so remove it from |
1583 | * bio_list, and call into __generic_make_request again. | 1586 | * bio_list, and call into __generic_make_request again. |
1584 | * | 1587 | * |
1585 | * The loop was structured like this to make only one call to | 1588 | * The loop was structured like this to make only one call to |
1586 | * __generic_make_request (which is important as it is large and | 1589 | * __generic_make_request (which is important as it is large and |
1587 | * inlined) and to keep the structure simple. | 1590 | * inlined) and to keep the structure simple. |
1588 | */ | 1591 | */ |
1589 | BUG_ON(bio->bi_next); | 1592 | BUG_ON(bio->bi_next); |
1590 | bio_list_init(&bio_list_on_stack); | 1593 | bio_list_init(&bio_list_on_stack); |
1591 | current->bio_list = &bio_list_on_stack; | 1594 | current->bio_list = &bio_list_on_stack; |
1592 | do { | 1595 | do { |
1593 | __generic_make_request(bio); | 1596 | __generic_make_request(bio); |
1594 | bio = bio_list_pop(current->bio_list); | 1597 | bio = bio_list_pop(current->bio_list); |
1595 | } while (bio); | 1598 | } while (bio); |
1596 | current->bio_list = NULL; /* deactivate */ | 1599 | current->bio_list = NULL; /* deactivate */ |
1597 | } | 1600 | } |
1598 | EXPORT_SYMBOL(generic_make_request); | 1601 | EXPORT_SYMBOL(generic_make_request); |
1599 | 1602 | ||
1600 | /** | 1603 | /** |
1601 | * submit_bio - submit a bio to the block device layer for I/O | 1604 | * submit_bio - submit a bio to the block device layer for I/O |
1602 | * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead) | 1605 | * @rw: whether to %READ or %WRITE, or maybe to %READA (read ahead) |
1603 | * @bio: The &struct bio which describes the I/O | 1606 | * @bio: The &struct bio which describes the I/O |
1604 | * | 1607 | * |
1605 | * submit_bio() is very similar in purpose to generic_make_request(), and | 1608 | * submit_bio() is very similar in purpose to generic_make_request(), and |
1606 | * uses that function to do most of the work. Both are fairly rough | 1609 | * uses that function to do most of the work. Both are fairly rough |
1607 | * interfaces; @bio must be presetup and ready for I/O. | 1610 | * interfaces; @bio must be presetup and ready for I/O. |
1608 | * | 1611 | * |
1609 | */ | 1612 | */ |
1610 | void submit_bio(int rw, struct bio *bio) | 1613 | void submit_bio(int rw, struct bio *bio) |
1611 | { | 1614 | { |
1612 | int count = bio_sectors(bio); | 1615 | int count = bio_sectors(bio); |
1613 | 1616 | ||
1614 | bio->bi_rw |= rw; | 1617 | bio->bi_rw |= rw; |
1615 | 1618 | ||
1616 | /* | 1619 | /* |
1617 | * If it's a regular read/write or a barrier with data attached, | 1620 | * If it's a regular read/write or a barrier with data attached, |
1618 | * go through the normal accounting stuff before submission. | 1621 | * go through the normal accounting stuff before submission. |
1619 | */ | 1622 | */ |
1620 | if (bio_has_data(bio) && !(rw & REQ_DISCARD)) { | 1623 | if (bio_has_data(bio) && !(rw & REQ_DISCARD)) { |
1621 | if (rw & WRITE) { | 1624 | if (rw & WRITE) { |
1622 | count_vm_events(PGPGOUT, count); | 1625 | count_vm_events(PGPGOUT, count); |
1623 | } else { | 1626 | } else { |
1624 | task_io_account_read(bio->bi_size); | 1627 | task_io_account_read(bio->bi_size); |
1625 | count_vm_events(PGPGIN, count); | 1628 | count_vm_events(PGPGIN, count); |
1626 | } | 1629 | } |
1627 | 1630 | ||
1628 | if (unlikely(block_dump)) { | 1631 | if (unlikely(block_dump)) { |
1629 | char b[BDEVNAME_SIZE]; | 1632 | char b[BDEVNAME_SIZE]; |
1630 | printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", | 1633 | printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n", |
1631 | current->comm, task_pid_nr(current), | 1634 | current->comm, task_pid_nr(current), |
1632 | (rw & WRITE) ? "WRITE" : "READ", | 1635 | (rw & WRITE) ? "WRITE" : "READ", |
1633 | (unsigned long long)bio->bi_sector, | 1636 | (unsigned long long)bio->bi_sector, |
1634 | bdevname(bio->bi_bdev, b), | 1637 | bdevname(bio->bi_bdev, b), |
1635 | count); | 1638 | count); |
1636 | } | 1639 | } |
1637 | } | 1640 | } |
1638 | 1641 | ||
1639 | generic_make_request(bio); | 1642 | generic_make_request(bio); |
1640 | } | 1643 | } |
1641 | EXPORT_SYMBOL(submit_bio); | 1644 | EXPORT_SYMBOL(submit_bio); |
1642 | 1645 | ||
1643 | /** | 1646 | /** |
1644 | * blk_rq_check_limits - Helper function to check a request for the queue limit | 1647 | * blk_rq_check_limits - Helper function to check a request for the queue limit |
1645 | * @q: the queue | 1648 | * @q: the queue |
1646 | * @rq: the request being checked | 1649 | * @rq: the request being checked |
1647 | * | 1650 | * |
1648 | * Description: | 1651 | * Description: |
1649 | * @rq may have been made based on weaker limitations of upper-level queues | 1652 | * @rq may have been made based on weaker limitations of upper-level queues |
1650 | * in request stacking drivers, and it may violate the limitation of @q. | 1653 | * in request stacking drivers, and it may violate the limitation of @q. |
1651 | * Since the block layer and the underlying device driver trust @rq | 1654 | * Since the block layer and the underlying device driver trust @rq |
1652 | * after it is inserted to @q, it should be checked against @q before | 1655 | * after it is inserted to @q, it should be checked against @q before |
1653 | * the insertion using this generic function. | 1656 | * the insertion using this generic function. |
1654 | * | 1657 | * |
1655 | * This function should also be useful for request stacking drivers | 1658 | * This function should also be useful for request stacking drivers |
1656 | * in some cases below, so export this function. | 1659 | * in some cases below, so export this function. |
1657 | * Request stacking drivers like request-based dm may change the queue | 1660 | * Request stacking drivers like request-based dm may change the queue |
1658 | * limits while requests are in the queue (e.g. dm's table swapping). | 1661 | * limits while requests are in the queue (e.g. dm's table swapping). |
1659 | * Such request stacking drivers should check those requests agaist | 1662 | * Such request stacking drivers should check those requests agaist |
1660 | * the new queue limits again when they dispatch those requests, | 1663 | * the new queue limits again when they dispatch those requests, |
1661 | * although such checkings are also done against the old queue limits | 1664 | * although such checkings are also done against the old queue limits |
1662 | * when submitting requests. | 1665 | * when submitting requests. |
1663 | */ | 1666 | */ |
1664 | int blk_rq_check_limits(struct request_queue *q, struct request *rq) | 1667 | int blk_rq_check_limits(struct request_queue *q, struct request *rq) |
1665 | { | 1668 | { |
1666 | if (rq->cmd_flags & REQ_DISCARD) | 1669 | if (rq->cmd_flags & REQ_DISCARD) |
1667 | return 0; | 1670 | return 0; |
1668 | 1671 | ||
1669 | if (blk_rq_sectors(rq) > queue_max_sectors(q) || | 1672 | if (blk_rq_sectors(rq) > queue_max_sectors(q) || |
1670 | blk_rq_bytes(rq) > queue_max_hw_sectors(q) << 9) { | 1673 | blk_rq_bytes(rq) > queue_max_hw_sectors(q) << 9) { |
1671 | printk(KERN_ERR "%s: over max size limit.\n", __func__); | 1674 | printk(KERN_ERR "%s: over max size limit.\n", __func__); |
1672 | return -EIO; | 1675 | return -EIO; |
1673 | } | 1676 | } |
1674 | 1677 | ||
1675 | /* | 1678 | /* |
1676 | * queue's settings related to segment counting like q->bounce_pfn | 1679 | * queue's settings related to segment counting like q->bounce_pfn |
1677 | * may differ from that of other stacking queues. | 1680 | * may differ from that of other stacking queues. |
1678 | * Recalculate it to check the request correctly on this queue's | 1681 | * Recalculate it to check the request correctly on this queue's |
1679 | * limitation. | 1682 | * limitation. |
1680 | */ | 1683 | */ |
1681 | blk_recalc_rq_segments(rq); | 1684 | blk_recalc_rq_segments(rq); |
1682 | if (rq->nr_phys_segments > queue_max_segments(q)) { | 1685 | if (rq->nr_phys_segments > queue_max_segments(q)) { |
1683 | printk(KERN_ERR "%s: over max segments limit.\n", __func__); | 1686 | printk(KERN_ERR "%s: over max segments limit.\n", __func__); |
1684 | return -EIO; | 1687 | return -EIO; |
1685 | } | 1688 | } |
1686 | 1689 | ||
1687 | return 0; | 1690 | return 0; |
1688 | } | 1691 | } |
1689 | EXPORT_SYMBOL_GPL(blk_rq_check_limits); | 1692 | EXPORT_SYMBOL_GPL(blk_rq_check_limits); |
1690 | 1693 | ||
1691 | /** | 1694 | /** |
1692 | * blk_insert_cloned_request - Helper for stacking drivers to submit a request | 1695 | * blk_insert_cloned_request - Helper for stacking drivers to submit a request |
1693 | * @q: the queue to submit the request | 1696 | * @q: the queue to submit the request |
1694 | * @rq: the request being queued | 1697 | * @rq: the request being queued |
1695 | */ | 1698 | */ |
1696 | int blk_insert_cloned_request(struct request_queue *q, struct request *rq) | 1699 | int blk_insert_cloned_request(struct request_queue *q, struct request *rq) |
1697 | { | 1700 | { |
1698 | unsigned long flags; | 1701 | unsigned long flags; |
1699 | 1702 | ||
1700 | if (blk_rq_check_limits(q, rq)) | 1703 | if (blk_rq_check_limits(q, rq)) |
1701 | return -EIO; | 1704 | return -EIO; |
1702 | 1705 | ||
1703 | #ifdef CONFIG_FAIL_MAKE_REQUEST | 1706 | #ifdef CONFIG_FAIL_MAKE_REQUEST |
1704 | if (rq->rq_disk && rq->rq_disk->part0.make_it_fail && | 1707 | if (rq->rq_disk && rq->rq_disk->part0.make_it_fail && |
1705 | should_fail(&fail_make_request, blk_rq_bytes(rq))) | 1708 | should_fail(&fail_make_request, blk_rq_bytes(rq))) |
1706 | return -EIO; | 1709 | return -EIO; |
1707 | #endif | 1710 | #endif |
1708 | 1711 | ||
1709 | spin_lock_irqsave(q->queue_lock, flags); | 1712 | spin_lock_irqsave(q->queue_lock, flags); |
1710 | 1713 | ||
1711 | /* | 1714 | /* |
1712 | * Submitting request must be dequeued before calling this function | 1715 | * Submitting request must be dequeued before calling this function |
1713 | * because it will be linked to another request_queue | 1716 | * because it will be linked to another request_queue |
1714 | */ | 1717 | */ |
1715 | BUG_ON(blk_queued_rq(rq)); | 1718 | BUG_ON(blk_queued_rq(rq)); |
1716 | 1719 | ||
1717 | add_acct_request(q, rq, ELEVATOR_INSERT_BACK); | 1720 | add_acct_request(q, rq, ELEVATOR_INSERT_BACK); |
1718 | spin_unlock_irqrestore(q->queue_lock, flags); | 1721 | spin_unlock_irqrestore(q->queue_lock, flags); |
1719 | 1722 | ||
1720 | return 0; | 1723 | return 0; |
1721 | } | 1724 | } |
1722 | EXPORT_SYMBOL_GPL(blk_insert_cloned_request); | 1725 | EXPORT_SYMBOL_GPL(blk_insert_cloned_request); |
1723 | 1726 | ||
1724 | /** | 1727 | /** |
1725 | * blk_rq_err_bytes - determine number of bytes till the next failure boundary | 1728 | * blk_rq_err_bytes - determine number of bytes till the next failure boundary |
1726 | * @rq: request to examine | 1729 | * @rq: request to examine |
1727 | * | 1730 | * |
1728 | * Description: | 1731 | * Description: |
1729 | * A request could be merge of IOs which require different failure | 1732 | * A request could be merge of IOs which require different failure |
1730 | * handling. This function determines the number of bytes which | 1733 | * handling. This function determines the number of bytes which |
1731 | * can be failed from the beginning of the request without | 1734 | * can be failed from the beginning of the request without |
1732 | * crossing into area which need to be retried further. | 1735 | * crossing into area which need to be retried further. |
1733 | * | 1736 | * |
1734 | * Return: | 1737 | * Return: |
1735 | * The number of bytes to fail. | 1738 | * The number of bytes to fail. |
1736 | * | 1739 | * |
1737 | * Context: | 1740 | * Context: |
1738 | * queue_lock must be held. | 1741 | * queue_lock must be held. |
1739 | */ | 1742 | */ |
1740 | unsigned int blk_rq_err_bytes(const struct request *rq) | 1743 | unsigned int blk_rq_err_bytes(const struct request *rq) |
1741 | { | 1744 | { |
1742 | unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; | 1745 | unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK; |
1743 | unsigned int bytes = 0; | 1746 | unsigned int bytes = 0; |
1744 | struct bio *bio; | 1747 | struct bio *bio; |
1745 | 1748 | ||
1746 | if (!(rq->cmd_flags & REQ_MIXED_MERGE)) | 1749 | if (!(rq->cmd_flags & REQ_MIXED_MERGE)) |
1747 | return blk_rq_bytes(rq); | 1750 | return blk_rq_bytes(rq); |
1748 | 1751 | ||
1749 | /* | 1752 | /* |
1750 | * Currently the only 'mixing' which can happen is between | 1753 | * Currently the only 'mixing' which can happen is between |
1751 | * different fastfail types. We can safely fail portions | 1754 | * different fastfail types. We can safely fail portions |
1752 | * which have all the failfast bits that the first one has - | 1755 | * which have all the failfast bits that the first one has - |
1753 | * the ones which are at least as eager to fail as the first | 1756 | * the ones which are at least as eager to fail as the first |
1754 | * one. | 1757 | * one. |
1755 | */ | 1758 | */ |
1756 | for (bio = rq->bio; bio; bio = bio->bi_next) { | 1759 | for (bio = rq->bio; bio; bio = bio->bi_next) { |
1757 | if ((bio->bi_rw & ff) != ff) | 1760 | if ((bio->bi_rw & ff) != ff) |
1758 | break; | 1761 | break; |
1759 | bytes += bio->bi_size; | 1762 | bytes += bio->bi_size; |
1760 | } | 1763 | } |
1761 | 1764 | ||
1762 | /* this could lead to infinite loop */ | 1765 | /* this could lead to infinite loop */ |
1763 | BUG_ON(blk_rq_bytes(rq) && !bytes); | 1766 | BUG_ON(blk_rq_bytes(rq) && !bytes); |
1764 | return bytes; | 1767 | return bytes; |
1765 | } | 1768 | } |
1766 | EXPORT_SYMBOL_GPL(blk_rq_err_bytes); | 1769 | EXPORT_SYMBOL_GPL(blk_rq_err_bytes); |
1767 | 1770 | ||
1768 | static void blk_account_io_completion(struct request *req, unsigned int bytes) | 1771 | static void blk_account_io_completion(struct request *req, unsigned int bytes) |
1769 | { | 1772 | { |
1770 | if (blk_do_io_stat(req)) { | 1773 | if (blk_do_io_stat(req)) { |
1771 | const int rw = rq_data_dir(req); | 1774 | const int rw = rq_data_dir(req); |
1772 | struct hd_struct *part; | 1775 | struct hd_struct *part; |
1773 | int cpu; | 1776 | int cpu; |
1774 | 1777 | ||
1775 | cpu = part_stat_lock(); | 1778 | cpu = part_stat_lock(); |
1776 | part = req->part; | 1779 | part = req->part; |
1777 | part_stat_add(cpu, part, sectors[rw], bytes >> 9); | 1780 | part_stat_add(cpu, part, sectors[rw], bytes >> 9); |
1778 | part_stat_unlock(); | 1781 | part_stat_unlock(); |
1779 | } | 1782 | } |
1780 | } | 1783 | } |
1781 | 1784 | ||
1782 | static void blk_account_io_done(struct request *req) | 1785 | static void blk_account_io_done(struct request *req) |
1783 | { | 1786 | { |
1784 | /* | 1787 | /* |
1785 | * Account IO completion. flush_rq isn't accounted as a | 1788 | * Account IO completion. flush_rq isn't accounted as a |
1786 | * normal IO on queueing nor completion. Accounting the | 1789 | * normal IO on queueing nor completion. Accounting the |
1787 | * containing request is enough. | 1790 | * containing request is enough. |
1788 | */ | 1791 | */ |
1789 | if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) { | 1792 | if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) { |
1790 | unsigned long duration = jiffies - req->start_time; | 1793 | unsigned long duration = jiffies - req->start_time; |
1791 | const int rw = rq_data_dir(req); | 1794 | const int rw = rq_data_dir(req); |
1792 | struct hd_struct *part; | 1795 | struct hd_struct *part; |
1793 | int cpu; | 1796 | int cpu; |
1794 | 1797 | ||
1795 | cpu = part_stat_lock(); | 1798 | cpu = part_stat_lock(); |
1796 | part = req->part; | 1799 | part = req->part; |
1797 | 1800 | ||
1798 | part_stat_inc(cpu, part, ios[rw]); | 1801 | part_stat_inc(cpu, part, ios[rw]); |
1799 | part_stat_add(cpu, part, ticks[rw], duration); | 1802 | part_stat_add(cpu, part, ticks[rw], duration); |
1800 | part_round_stats(cpu, part); | 1803 | part_round_stats(cpu, part); |
1801 | part_dec_in_flight(part, rw); | 1804 | part_dec_in_flight(part, rw); |
1802 | 1805 | ||
1803 | hd_struct_put(part); | 1806 | hd_struct_put(part); |
1804 | part_stat_unlock(); | 1807 | part_stat_unlock(); |
1805 | } | 1808 | } |
1806 | } | 1809 | } |
1807 | 1810 | ||
1808 | /** | 1811 | /** |
1809 | * blk_peek_request - peek at the top of a request queue | 1812 | * blk_peek_request - peek at the top of a request queue |
1810 | * @q: request queue to peek at | 1813 | * @q: request queue to peek at |
1811 | * | 1814 | * |
1812 | * Description: | 1815 | * Description: |
1813 | * Return the request at the top of @q. The returned request | 1816 | * Return the request at the top of @q. The returned request |
1814 | * should be started using blk_start_request() before LLD starts | 1817 | * should be started using blk_start_request() before LLD starts |
1815 | * processing it. | 1818 | * processing it. |
1816 | * | 1819 | * |
1817 | * Return: | 1820 | * Return: |
1818 | * Pointer to the request at the top of @q if available. Null | 1821 | * Pointer to the request at the top of @q if available. Null |
1819 | * otherwise. | 1822 | * otherwise. |
1820 | * | 1823 | * |
1821 | * Context: | 1824 | * Context: |
1822 | * queue_lock must be held. | 1825 | * queue_lock must be held. |
1823 | */ | 1826 | */ |
1824 | struct request *blk_peek_request(struct request_queue *q) | 1827 | struct request *blk_peek_request(struct request_queue *q) |
1825 | { | 1828 | { |
1826 | struct request *rq; | 1829 | struct request *rq; |
1827 | int ret; | 1830 | int ret; |
1828 | 1831 | ||
1829 | while ((rq = __elv_next_request(q)) != NULL) { | 1832 | while ((rq = __elv_next_request(q)) != NULL) { |
1830 | if (!(rq->cmd_flags & REQ_STARTED)) { | 1833 | if (!(rq->cmd_flags & REQ_STARTED)) { |
1831 | /* | 1834 | /* |
1832 | * This is the first time the device driver | 1835 | * This is the first time the device driver |
1833 | * sees this request (possibly after | 1836 | * sees this request (possibly after |
1834 | * requeueing). Notify IO scheduler. | 1837 | * requeueing). Notify IO scheduler. |
1835 | */ | 1838 | */ |
1836 | if (rq->cmd_flags & REQ_SORTED) | 1839 | if (rq->cmd_flags & REQ_SORTED) |
1837 | elv_activate_rq(q, rq); | 1840 | elv_activate_rq(q, rq); |
1838 | 1841 | ||
1839 | /* | 1842 | /* |
1840 | * just mark as started even if we don't start | 1843 | * just mark as started even if we don't start |
1841 | * it, a request that has been delayed should | 1844 | * it, a request that has been delayed should |
1842 | * not be passed by new incoming requests | 1845 | * not be passed by new incoming requests |
1843 | */ | 1846 | */ |
1844 | rq->cmd_flags |= REQ_STARTED; | 1847 | rq->cmd_flags |= REQ_STARTED; |
1845 | trace_block_rq_issue(q, rq); | 1848 | trace_block_rq_issue(q, rq); |
1846 | } | 1849 | } |
1847 | 1850 | ||
1848 | if (!q->boundary_rq || q->boundary_rq == rq) { | 1851 | if (!q->boundary_rq || q->boundary_rq == rq) { |
1849 | q->end_sector = rq_end_sector(rq); | 1852 | q->end_sector = rq_end_sector(rq); |
1850 | q->boundary_rq = NULL; | 1853 | q->boundary_rq = NULL; |
1851 | } | 1854 | } |
1852 | 1855 | ||
1853 | if (rq->cmd_flags & REQ_DONTPREP) | 1856 | if (rq->cmd_flags & REQ_DONTPREP) |
1854 | break; | 1857 | break; |
1855 | 1858 | ||
1856 | if (q->dma_drain_size && blk_rq_bytes(rq)) { | 1859 | if (q->dma_drain_size && blk_rq_bytes(rq)) { |
1857 | /* | 1860 | /* |
1858 | * make sure space for the drain appears we | 1861 | * make sure space for the drain appears we |
1859 | * know we can do this because max_hw_segments | 1862 | * know we can do this because max_hw_segments |
1860 | * has been adjusted to be one fewer than the | 1863 | * has been adjusted to be one fewer than the |
1861 | * device can handle | 1864 | * device can handle |
1862 | */ | 1865 | */ |
1863 | rq->nr_phys_segments++; | 1866 | rq->nr_phys_segments++; |
1864 | } | 1867 | } |
1865 | 1868 | ||
1866 | if (!q->prep_rq_fn) | 1869 | if (!q->prep_rq_fn) |
1867 | break; | 1870 | break; |
1868 | 1871 | ||
1869 | ret = q->prep_rq_fn(q, rq); | 1872 | ret = q->prep_rq_fn(q, rq); |
1870 | if (ret == BLKPREP_OK) { | 1873 | if (ret == BLKPREP_OK) { |
1871 | break; | 1874 | break; |
1872 | } else if (ret == BLKPREP_DEFER) { | 1875 | } else if (ret == BLKPREP_DEFER) { |
1873 | /* | 1876 | /* |
1874 | * the request may have been (partially) prepped. | 1877 | * the request may have been (partially) prepped. |
1875 | * we need to keep this request in the front to | 1878 | * we need to keep this request in the front to |
1876 | * avoid resource deadlock. REQ_STARTED will | 1879 | * avoid resource deadlock. REQ_STARTED will |
1877 | * prevent other fs requests from passing this one. | 1880 | * prevent other fs requests from passing this one. |
1878 | */ | 1881 | */ |
1879 | if (q->dma_drain_size && blk_rq_bytes(rq) && | 1882 | if (q->dma_drain_size && blk_rq_bytes(rq) && |
1880 | !(rq->cmd_flags & REQ_DONTPREP)) { | 1883 | !(rq->cmd_flags & REQ_DONTPREP)) { |
1881 | /* | 1884 | /* |
1882 | * remove the space for the drain we added | 1885 | * remove the space for the drain we added |
1883 | * so that we don't add it again | 1886 | * so that we don't add it again |
1884 | */ | 1887 | */ |
1885 | --rq->nr_phys_segments; | 1888 | --rq->nr_phys_segments; |
1886 | } | 1889 | } |
1887 | 1890 | ||
1888 | rq = NULL; | 1891 | rq = NULL; |
1889 | break; | 1892 | break; |
1890 | } else if (ret == BLKPREP_KILL) { | 1893 | } else if (ret == BLKPREP_KILL) { |
1891 | rq->cmd_flags |= REQ_QUIET; | 1894 | rq->cmd_flags |= REQ_QUIET; |
1892 | /* | 1895 | /* |
1893 | * Mark this request as started so we don't trigger | 1896 | * Mark this request as started so we don't trigger |
1894 | * any debug logic in the end I/O path. | 1897 | * any debug logic in the end I/O path. |
1895 | */ | 1898 | */ |
1896 | blk_start_request(rq); | 1899 | blk_start_request(rq); |
1897 | __blk_end_request_all(rq, -EIO); | 1900 | __blk_end_request_all(rq, -EIO); |
1898 | } else { | 1901 | } else { |
1899 | printk(KERN_ERR "%s: bad return=%d\n", __func__, ret); | 1902 | printk(KERN_ERR "%s: bad return=%d\n", __func__, ret); |
1900 | break; | 1903 | break; |
1901 | } | 1904 | } |
1902 | } | 1905 | } |
1903 | 1906 | ||
1904 | return rq; | 1907 | return rq; |
1905 | } | 1908 | } |
1906 | EXPORT_SYMBOL(blk_peek_request); | 1909 | EXPORT_SYMBOL(blk_peek_request); |
1907 | 1910 | ||
1908 | void blk_dequeue_request(struct request *rq) | 1911 | void blk_dequeue_request(struct request *rq) |
1909 | { | 1912 | { |
1910 | struct request_queue *q = rq->q; | 1913 | struct request_queue *q = rq->q; |
1911 | 1914 | ||
1912 | BUG_ON(list_empty(&rq->queuelist)); | 1915 | BUG_ON(list_empty(&rq->queuelist)); |
1913 | BUG_ON(ELV_ON_HASH(rq)); | 1916 | BUG_ON(ELV_ON_HASH(rq)); |
1914 | 1917 | ||
1915 | list_del_init(&rq->queuelist); | 1918 | list_del_init(&rq->queuelist); |
1916 | 1919 | ||
1917 | /* | 1920 | /* |
1918 | * the time frame between a request being removed from the lists | 1921 | * the time frame between a request being removed from the lists |
1919 | * and to it is freed is accounted as io that is in progress at | 1922 | * and to it is freed is accounted as io that is in progress at |
1920 | * the driver side. | 1923 | * the driver side. |
1921 | */ | 1924 | */ |
1922 | if (blk_account_rq(rq)) { | 1925 | if (blk_account_rq(rq)) { |
1923 | q->in_flight[rq_is_sync(rq)]++; | 1926 | q->in_flight[rq_is_sync(rq)]++; |
1924 | set_io_start_time_ns(rq); | 1927 | set_io_start_time_ns(rq); |
1925 | } | 1928 | } |
1926 | } | 1929 | } |
1927 | 1930 | ||
1928 | /** | 1931 | /** |
1929 | * blk_start_request - start request processing on the driver | 1932 | * blk_start_request - start request processing on the driver |
1930 | * @req: request to dequeue | 1933 | * @req: request to dequeue |
1931 | * | 1934 | * |
1932 | * Description: | 1935 | * Description: |
1933 | * Dequeue @req and start timeout timer on it. This hands off the | 1936 | * Dequeue @req and start timeout timer on it. This hands off the |
1934 | * request to the driver. | 1937 | * request to the driver. |
1935 | * | 1938 | * |
1936 | * Block internal functions which don't want to start timer should | 1939 | * Block internal functions which don't want to start timer should |
1937 | * call blk_dequeue_request(). | 1940 | * call blk_dequeue_request(). |
1938 | * | 1941 | * |
1939 | * Context: | 1942 | * Context: |
1940 | * queue_lock must be held. | 1943 | * queue_lock must be held. |
1941 | */ | 1944 | */ |
1942 | void blk_start_request(struct request *req) | 1945 | void blk_start_request(struct request *req) |
1943 | { | 1946 | { |
1944 | blk_dequeue_request(req); | 1947 | blk_dequeue_request(req); |
1945 | 1948 | ||
1946 | /* | 1949 | /* |
1947 | * We are now handing the request to the hardware, initialize | 1950 | * We are now handing the request to the hardware, initialize |
1948 | * resid_len to full count and add the timeout handler. | 1951 | * resid_len to full count and add the timeout handler. |
1949 | */ | 1952 | */ |
1950 | req->resid_len = blk_rq_bytes(req); | 1953 | req->resid_len = blk_rq_bytes(req); |
1951 | if (unlikely(blk_bidi_rq(req))) | 1954 | if (unlikely(blk_bidi_rq(req))) |
1952 | req->next_rq->resid_len = blk_rq_bytes(req->next_rq); | 1955 | req->next_rq->resid_len = blk_rq_bytes(req->next_rq); |
1953 | 1956 | ||
1954 | blk_add_timer(req); | 1957 | blk_add_timer(req); |
1955 | } | 1958 | } |
1956 | EXPORT_SYMBOL(blk_start_request); | 1959 | EXPORT_SYMBOL(blk_start_request); |
1957 | 1960 | ||
1958 | /** | 1961 | /** |
1959 | * blk_fetch_request - fetch a request from a request queue | 1962 | * blk_fetch_request - fetch a request from a request queue |
1960 | * @q: request queue to fetch a request from | 1963 | * @q: request queue to fetch a request from |
1961 | * | 1964 | * |
1962 | * Description: | 1965 | * Description: |
1963 | * Return the request at the top of @q. The request is started on | 1966 | * Return the request at the top of @q. The request is started on |
1964 | * return and LLD can start processing it immediately. | 1967 | * return and LLD can start processing it immediately. |
1965 | * | 1968 | * |
1966 | * Return: | 1969 | * Return: |
1967 | * Pointer to the request at the top of @q if available. Null | 1970 | * Pointer to the request at the top of @q if available. Null |
1968 | * otherwise. | 1971 | * otherwise. |
1969 | * | 1972 | * |
1970 | * Context: | 1973 | * Context: |
1971 | * queue_lock must be held. | 1974 | * queue_lock must be held. |
1972 | */ | 1975 | */ |
1973 | struct request *blk_fetch_request(struct request_queue *q) | 1976 | struct request *blk_fetch_request(struct request_queue *q) |
1974 | { | 1977 | { |
1975 | struct request *rq; | 1978 | struct request *rq; |
1976 | 1979 | ||
1977 | rq = blk_peek_request(q); | 1980 | rq = blk_peek_request(q); |
1978 | if (rq) | 1981 | if (rq) |
1979 | blk_start_request(rq); | 1982 | blk_start_request(rq); |
1980 | return rq; | 1983 | return rq; |
1981 | } | 1984 | } |
1982 | EXPORT_SYMBOL(blk_fetch_request); | 1985 | EXPORT_SYMBOL(blk_fetch_request); |
1983 | 1986 | ||
1984 | /** | 1987 | /** |
1985 | * blk_update_request - Special helper function for request stacking drivers | 1988 | * blk_update_request - Special helper function for request stacking drivers |
1986 | * @req: the request being processed | 1989 | * @req: the request being processed |
1987 | * @error: %0 for success, < %0 for error | 1990 | * @error: %0 for success, < %0 for error |
1988 | * @nr_bytes: number of bytes to complete @req | 1991 | * @nr_bytes: number of bytes to complete @req |
1989 | * | 1992 | * |
1990 | * Description: | 1993 | * Description: |
1991 | * Ends I/O on a number of bytes attached to @req, but doesn't complete | 1994 | * Ends I/O on a number of bytes attached to @req, but doesn't complete |
1992 | * the request structure even if @req doesn't have leftover. | 1995 | * the request structure even if @req doesn't have leftover. |
1993 | * If @req has leftover, sets it up for the next range of segments. | 1996 | * If @req has leftover, sets it up for the next range of segments. |
1994 | * | 1997 | * |
1995 | * This special helper function is only for request stacking drivers | 1998 | * This special helper function is only for request stacking drivers |
1996 | * (e.g. request-based dm) so that they can handle partial completion. | 1999 | * (e.g. request-based dm) so that they can handle partial completion. |
1997 | * Actual device drivers should use blk_end_request instead. | 2000 | * Actual device drivers should use blk_end_request instead. |
1998 | * | 2001 | * |
1999 | * Passing the result of blk_rq_bytes() as @nr_bytes guarantees | 2002 | * Passing the result of blk_rq_bytes() as @nr_bytes guarantees |
2000 | * %false return from this function. | 2003 | * %false return from this function. |
2001 | * | 2004 | * |
2002 | * Return: | 2005 | * Return: |
2003 | * %false - this request doesn't have any more data | 2006 | * %false - this request doesn't have any more data |
2004 | * %true - this request has more data | 2007 | * %true - this request has more data |
2005 | **/ | 2008 | **/ |
2006 | bool blk_update_request(struct request *req, int error, unsigned int nr_bytes) | 2009 | bool blk_update_request(struct request *req, int error, unsigned int nr_bytes) |
2007 | { | 2010 | { |
2008 | int total_bytes, bio_nbytes, next_idx = 0; | 2011 | int total_bytes, bio_nbytes, next_idx = 0; |
2009 | struct bio *bio; | 2012 | struct bio *bio; |
2010 | 2013 | ||
2011 | if (!req->bio) | 2014 | if (!req->bio) |
2012 | return false; | 2015 | return false; |
2013 | 2016 | ||
2014 | trace_block_rq_complete(req->q, req); | 2017 | trace_block_rq_complete(req->q, req); |
2015 | 2018 | ||
2016 | /* | 2019 | /* |
2017 | * For fs requests, rq is just carrier of independent bio's | 2020 | * For fs requests, rq is just carrier of independent bio's |
2018 | * and each partial completion should be handled separately. | 2021 | * and each partial completion should be handled separately. |
2019 | * Reset per-request error on each partial completion. | 2022 | * Reset per-request error on each partial completion. |
2020 | * | 2023 | * |
2021 | * TODO: tj: This is too subtle. It would be better to let | 2024 | * TODO: tj: This is too subtle. It would be better to let |
2022 | * low level drivers do what they see fit. | 2025 | * low level drivers do what they see fit. |
2023 | */ | 2026 | */ |
2024 | if (req->cmd_type == REQ_TYPE_FS) | 2027 | if (req->cmd_type == REQ_TYPE_FS) |
2025 | req->errors = 0; | 2028 | req->errors = 0; |
2026 | 2029 | ||
2027 | if (error && req->cmd_type == REQ_TYPE_FS && | 2030 | if (error && req->cmd_type == REQ_TYPE_FS && |
2028 | !(req->cmd_flags & REQ_QUIET)) { | 2031 | !(req->cmd_flags & REQ_QUIET)) { |
2029 | char *error_type; | 2032 | char *error_type; |
2030 | 2033 | ||
2031 | switch (error) { | 2034 | switch (error) { |
2032 | case -ENOLINK: | 2035 | case -ENOLINK: |
2033 | error_type = "recoverable transport"; | 2036 | error_type = "recoverable transport"; |
2034 | break; | 2037 | break; |
2035 | case -EREMOTEIO: | 2038 | case -EREMOTEIO: |
2036 | error_type = "critical target"; | 2039 | error_type = "critical target"; |
2037 | break; | 2040 | break; |
2038 | case -EBADE: | 2041 | case -EBADE: |
2039 | error_type = "critical nexus"; | 2042 | error_type = "critical nexus"; |
2040 | break; | 2043 | break; |
2041 | case -EIO: | 2044 | case -EIO: |
2042 | default: | 2045 | default: |
2043 | error_type = "I/O"; | 2046 | error_type = "I/O"; |
2044 | break; | 2047 | break; |
2045 | } | 2048 | } |
2046 | printk(KERN_ERR "end_request: %s error, dev %s, sector %llu\n", | 2049 | printk(KERN_ERR "end_request: %s error, dev %s, sector %llu\n", |
2047 | error_type, req->rq_disk ? req->rq_disk->disk_name : "?", | 2050 | error_type, req->rq_disk ? req->rq_disk->disk_name : "?", |
2048 | (unsigned long long)blk_rq_pos(req)); | 2051 | (unsigned long long)blk_rq_pos(req)); |
2049 | } | 2052 | } |
2050 | 2053 | ||
2051 | blk_account_io_completion(req, nr_bytes); | 2054 | blk_account_io_completion(req, nr_bytes); |
2052 | 2055 | ||
2053 | total_bytes = bio_nbytes = 0; | 2056 | total_bytes = bio_nbytes = 0; |
2054 | while ((bio = req->bio) != NULL) { | 2057 | while ((bio = req->bio) != NULL) { |
2055 | int nbytes; | 2058 | int nbytes; |
2056 | 2059 | ||
2057 | if (nr_bytes >= bio->bi_size) { | 2060 | if (nr_bytes >= bio->bi_size) { |
2058 | req->bio = bio->bi_next; | 2061 | req->bio = bio->bi_next; |
2059 | nbytes = bio->bi_size; | 2062 | nbytes = bio->bi_size; |
2060 | req_bio_endio(req, bio, nbytes, error); | 2063 | req_bio_endio(req, bio, nbytes, error); |
2061 | next_idx = 0; | 2064 | next_idx = 0; |
2062 | bio_nbytes = 0; | 2065 | bio_nbytes = 0; |
2063 | } else { | 2066 | } else { |
2064 | int idx = bio->bi_idx + next_idx; | 2067 | int idx = bio->bi_idx + next_idx; |
2065 | 2068 | ||
2066 | if (unlikely(idx >= bio->bi_vcnt)) { | 2069 | if (unlikely(idx >= bio->bi_vcnt)) { |
2067 | blk_dump_rq_flags(req, "__end_that"); | 2070 | blk_dump_rq_flags(req, "__end_that"); |
2068 | printk(KERN_ERR "%s: bio idx %d >= vcnt %d\n", | 2071 | printk(KERN_ERR "%s: bio idx %d >= vcnt %d\n", |
2069 | __func__, idx, bio->bi_vcnt); | 2072 | __func__, idx, bio->bi_vcnt); |
2070 | break; | 2073 | break; |
2071 | } | 2074 | } |
2072 | 2075 | ||
2073 | nbytes = bio_iovec_idx(bio, idx)->bv_len; | 2076 | nbytes = bio_iovec_idx(bio, idx)->bv_len; |
2074 | BIO_BUG_ON(nbytes > bio->bi_size); | 2077 | BIO_BUG_ON(nbytes > bio->bi_size); |
2075 | 2078 | ||
2076 | /* | 2079 | /* |
2077 | * not a complete bvec done | 2080 | * not a complete bvec done |
2078 | */ | 2081 | */ |
2079 | if (unlikely(nbytes > nr_bytes)) { | 2082 | if (unlikely(nbytes > nr_bytes)) { |
2080 | bio_nbytes += nr_bytes; | 2083 | bio_nbytes += nr_bytes; |
2081 | total_bytes += nr_bytes; | 2084 | total_bytes += nr_bytes; |
2082 | break; | 2085 | break; |
2083 | } | 2086 | } |
2084 | 2087 | ||
2085 | /* | 2088 | /* |
2086 | * advance to the next vector | 2089 | * advance to the next vector |
2087 | */ | 2090 | */ |
2088 | next_idx++; | 2091 | next_idx++; |
2089 | bio_nbytes += nbytes; | 2092 | bio_nbytes += nbytes; |
2090 | } | 2093 | } |
2091 | 2094 | ||
2092 | total_bytes += nbytes; | 2095 | total_bytes += nbytes; |
2093 | nr_bytes -= nbytes; | 2096 | nr_bytes -= nbytes; |
2094 | 2097 | ||
2095 | bio = req->bio; | 2098 | bio = req->bio; |
2096 | if (bio) { | 2099 | if (bio) { |
2097 | /* | 2100 | /* |
2098 | * end more in this run, or just return 'not-done' | 2101 | * end more in this run, or just return 'not-done' |
2099 | */ | 2102 | */ |
2100 | if (unlikely(nr_bytes <= 0)) | 2103 | if (unlikely(nr_bytes <= 0)) |
2101 | break; | 2104 | break; |
2102 | } | 2105 | } |
2103 | } | 2106 | } |
2104 | 2107 | ||
2105 | /* | 2108 | /* |
2106 | * completely done | 2109 | * completely done |
2107 | */ | 2110 | */ |
2108 | if (!req->bio) { | 2111 | if (!req->bio) { |
2109 | /* | 2112 | /* |
2110 | * Reset counters so that the request stacking driver | 2113 | * Reset counters so that the request stacking driver |
2111 | * can find how many bytes remain in the request | 2114 | * can find how many bytes remain in the request |
2112 | * later. | 2115 | * later. |
2113 | */ | 2116 | */ |
2114 | req->__data_len = 0; | 2117 | req->__data_len = 0; |
2115 | return false; | 2118 | return false; |
2116 | } | 2119 | } |
2117 | 2120 | ||
2118 | /* | 2121 | /* |
2119 | * if the request wasn't completed, update state | 2122 | * if the request wasn't completed, update state |
2120 | */ | 2123 | */ |
2121 | if (bio_nbytes) { | 2124 | if (bio_nbytes) { |
2122 | req_bio_endio(req, bio, bio_nbytes, error); | 2125 | req_bio_endio(req, bio, bio_nbytes, error); |
2123 | bio->bi_idx += next_idx; | 2126 | bio->bi_idx += next_idx; |
2124 | bio_iovec(bio)->bv_offset += nr_bytes; | 2127 | bio_iovec(bio)->bv_offset += nr_bytes; |
2125 | bio_iovec(bio)->bv_len -= nr_bytes; | 2128 | bio_iovec(bio)->bv_len -= nr_bytes; |
2126 | } | 2129 | } |
2127 | 2130 | ||
2128 | req->__data_len -= total_bytes; | 2131 | req->__data_len -= total_bytes; |
2129 | req->buffer = bio_data(req->bio); | 2132 | req->buffer = bio_data(req->bio); |
2130 | 2133 | ||
2131 | /* update sector only for requests with clear definition of sector */ | 2134 | /* update sector only for requests with clear definition of sector */ |
2132 | if (req->cmd_type == REQ_TYPE_FS || (req->cmd_flags & REQ_DISCARD)) | 2135 | if (req->cmd_type == REQ_TYPE_FS || (req->cmd_flags & REQ_DISCARD)) |
2133 | req->__sector += total_bytes >> 9; | 2136 | req->__sector += total_bytes >> 9; |
2134 | 2137 | ||
2135 | /* mixed attributes always follow the first bio */ | 2138 | /* mixed attributes always follow the first bio */ |
2136 | if (req->cmd_flags & REQ_MIXED_MERGE) { | 2139 | if (req->cmd_flags & REQ_MIXED_MERGE) { |
2137 | req->cmd_flags &= ~REQ_FAILFAST_MASK; | 2140 | req->cmd_flags &= ~REQ_FAILFAST_MASK; |
2138 | req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK; | 2141 | req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK; |
2139 | } | 2142 | } |
2140 | 2143 | ||
2141 | /* | 2144 | /* |
2142 | * If total number of sectors is less than the first segment | 2145 | * If total number of sectors is less than the first segment |
2143 | * size, something has gone terribly wrong. | 2146 | * size, something has gone terribly wrong. |
2144 | */ | 2147 | */ |
2145 | if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) { | 2148 | if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) { |
2146 | blk_dump_rq_flags(req, "request botched"); | 2149 | blk_dump_rq_flags(req, "request botched"); |
2147 | req->__data_len = blk_rq_cur_bytes(req); | 2150 | req->__data_len = blk_rq_cur_bytes(req); |
2148 | } | 2151 | } |
2149 | 2152 | ||
2150 | /* recalculate the number of segments */ | 2153 | /* recalculate the number of segments */ |
2151 | blk_recalc_rq_segments(req); | 2154 | blk_recalc_rq_segments(req); |
2152 | 2155 | ||
2153 | return true; | 2156 | return true; |
2154 | } | 2157 | } |
2155 | EXPORT_SYMBOL_GPL(blk_update_request); | 2158 | EXPORT_SYMBOL_GPL(blk_update_request); |
2156 | 2159 | ||
2157 | static bool blk_update_bidi_request(struct request *rq, int error, | 2160 | static bool blk_update_bidi_request(struct request *rq, int error, |
2158 | unsigned int nr_bytes, | 2161 | unsigned int nr_bytes, |
2159 | unsigned int bidi_bytes) | 2162 | unsigned int bidi_bytes) |
2160 | { | 2163 | { |
2161 | if (blk_update_request(rq, error, nr_bytes)) | 2164 | if (blk_update_request(rq, error, nr_bytes)) |
2162 | return true; | 2165 | return true; |
2163 | 2166 | ||
2164 | /* Bidi request must be completed as a whole */ | 2167 | /* Bidi request must be completed as a whole */ |
2165 | if (unlikely(blk_bidi_rq(rq)) && | 2168 | if (unlikely(blk_bidi_rq(rq)) && |
2166 | blk_update_request(rq->next_rq, error, bidi_bytes)) | 2169 | blk_update_request(rq->next_rq, error, bidi_bytes)) |
2167 | return true; | 2170 | return true; |
2168 | 2171 | ||
2169 | if (blk_queue_add_random(rq->q)) | 2172 | if (blk_queue_add_random(rq->q)) |
2170 | add_disk_randomness(rq->rq_disk); | 2173 | add_disk_randomness(rq->rq_disk); |
2171 | 2174 | ||
2172 | return false; | 2175 | return false; |
2173 | } | 2176 | } |
2174 | 2177 | ||
2175 | /** | 2178 | /** |
2176 | * blk_unprep_request - unprepare a request | 2179 | * blk_unprep_request - unprepare a request |
2177 | * @req: the request | 2180 | * @req: the request |
2178 | * | 2181 | * |
2179 | * This function makes a request ready for complete resubmission (or | 2182 | * This function makes a request ready for complete resubmission (or |
2180 | * completion). It happens only after all error handling is complete, | 2183 | * completion). It happens only after all error handling is complete, |
2181 | * so represents the appropriate moment to deallocate any resources | 2184 | * so represents the appropriate moment to deallocate any resources |
2182 | * that were allocated to the request in the prep_rq_fn. The queue | 2185 | * that were allocated to the request in the prep_rq_fn. The queue |
2183 | * lock is held when calling this. | 2186 | * lock is held when calling this. |
2184 | */ | 2187 | */ |
2185 | void blk_unprep_request(struct request *req) | 2188 | void blk_unprep_request(struct request *req) |
2186 | { | 2189 | { |
2187 | struct request_queue *q = req->q; | 2190 | struct request_queue *q = req->q; |
2188 | 2191 | ||
2189 | req->cmd_flags &= ~REQ_DONTPREP; | 2192 | req->cmd_flags &= ~REQ_DONTPREP; |
2190 | if (q->unprep_rq_fn) | 2193 | if (q->unprep_rq_fn) |
2191 | q->unprep_rq_fn(q, req); | 2194 | q->unprep_rq_fn(q, req); |
2192 | } | 2195 | } |
2193 | EXPORT_SYMBOL_GPL(blk_unprep_request); | 2196 | EXPORT_SYMBOL_GPL(blk_unprep_request); |
2194 | 2197 | ||
2195 | /* | 2198 | /* |
2196 | * queue lock must be held | 2199 | * queue lock must be held |
2197 | */ | 2200 | */ |
2198 | static void blk_finish_request(struct request *req, int error) | 2201 | static void blk_finish_request(struct request *req, int error) |
2199 | { | 2202 | { |
2200 | if (blk_rq_tagged(req)) | 2203 | if (blk_rq_tagged(req)) |
2201 | blk_queue_end_tag(req->q, req); | 2204 | blk_queue_end_tag(req->q, req); |
2202 | 2205 | ||
2203 | BUG_ON(blk_queued_rq(req)); | 2206 | BUG_ON(blk_queued_rq(req)); |
2204 | 2207 | ||
2205 | if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS) | 2208 | if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS) |
2206 | laptop_io_completion(&req->q->backing_dev_info); | 2209 | laptop_io_completion(&req->q->backing_dev_info); |
2207 | 2210 | ||
2208 | blk_delete_timer(req); | 2211 | blk_delete_timer(req); |
2209 | 2212 | ||
2210 | if (req->cmd_flags & REQ_DONTPREP) | 2213 | if (req->cmd_flags & REQ_DONTPREP) |
2211 | blk_unprep_request(req); | 2214 | blk_unprep_request(req); |
2212 | 2215 | ||
2213 | 2216 | ||
2214 | blk_account_io_done(req); | 2217 | blk_account_io_done(req); |
2215 | 2218 | ||
2216 | if (req->end_io) | 2219 | if (req->end_io) |
2217 | req->end_io(req, error); | 2220 | req->end_io(req, error); |
2218 | else { | 2221 | else { |
2219 | if (blk_bidi_rq(req)) | 2222 | if (blk_bidi_rq(req)) |
2220 | __blk_put_request(req->next_rq->q, req->next_rq); | 2223 | __blk_put_request(req->next_rq->q, req->next_rq); |
2221 | 2224 | ||
2222 | __blk_put_request(req->q, req); | 2225 | __blk_put_request(req->q, req); |
2223 | } | 2226 | } |
2224 | } | 2227 | } |
2225 | 2228 | ||
2226 | /** | 2229 | /** |
2227 | * blk_end_bidi_request - Complete a bidi request | 2230 | * blk_end_bidi_request - Complete a bidi request |
2228 | * @rq: the request to complete | 2231 | * @rq: the request to complete |
2229 | * @error: %0 for success, < %0 for error | 2232 | * @error: %0 for success, < %0 for error |
2230 | * @nr_bytes: number of bytes to complete @rq | 2233 | * @nr_bytes: number of bytes to complete @rq |
2231 | * @bidi_bytes: number of bytes to complete @rq->next_rq | 2234 | * @bidi_bytes: number of bytes to complete @rq->next_rq |
2232 | * | 2235 | * |
2233 | * Description: | 2236 | * Description: |
2234 | * Ends I/O on a number of bytes attached to @rq and @rq->next_rq. | 2237 | * Ends I/O on a number of bytes attached to @rq and @rq->next_rq. |
2235 | * Drivers that supports bidi can safely call this member for any | 2238 | * Drivers that supports bidi can safely call this member for any |
2236 | * type of request, bidi or uni. In the later case @bidi_bytes is | 2239 | * type of request, bidi or uni. In the later case @bidi_bytes is |
2237 | * just ignored. | 2240 | * just ignored. |
2238 | * | 2241 | * |
2239 | * Return: | 2242 | * Return: |
2240 | * %false - we are done with this request | 2243 | * %false - we are done with this request |
2241 | * %true - still buffers pending for this request | 2244 | * %true - still buffers pending for this request |
2242 | **/ | 2245 | **/ |
2243 | static bool blk_end_bidi_request(struct request *rq, int error, | 2246 | static bool blk_end_bidi_request(struct request *rq, int error, |
2244 | unsigned int nr_bytes, unsigned int bidi_bytes) | 2247 | unsigned int nr_bytes, unsigned int bidi_bytes) |
2245 | { | 2248 | { |
2246 | struct request_queue *q = rq->q; | 2249 | struct request_queue *q = rq->q; |
2247 | unsigned long flags; | 2250 | unsigned long flags; |
2248 | 2251 | ||
2249 | if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes)) | 2252 | if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes)) |
2250 | return true; | 2253 | return true; |
2251 | 2254 | ||
2252 | spin_lock_irqsave(q->queue_lock, flags); | 2255 | spin_lock_irqsave(q->queue_lock, flags); |
2253 | blk_finish_request(rq, error); | 2256 | blk_finish_request(rq, error); |
2254 | spin_unlock_irqrestore(q->queue_lock, flags); | 2257 | spin_unlock_irqrestore(q->queue_lock, flags); |
2255 | 2258 | ||
2256 | return false; | 2259 | return false; |
2257 | } | 2260 | } |
2258 | 2261 | ||
2259 | /** | 2262 | /** |
2260 | * __blk_end_bidi_request - Complete a bidi request with queue lock held | 2263 | * __blk_end_bidi_request - Complete a bidi request with queue lock held |
2261 | * @rq: the request to complete | 2264 | * @rq: the request to complete |
2262 | * @error: %0 for success, < %0 for error | 2265 | * @error: %0 for success, < %0 for error |
2263 | * @nr_bytes: number of bytes to complete @rq | 2266 | * @nr_bytes: number of bytes to complete @rq |
2264 | * @bidi_bytes: number of bytes to complete @rq->next_rq | 2267 | * @bidi_bytes: number of bytes to complete @rq->next_rq |
2265 | * | 2268 | * |
2266 | * Description: | 2269 | * Description: |
2267 | * Identical to blk_end_bidi_request() except that queue lock is | 2270 | * Identical to blk_end_bidi_request() except that queue lock is |
2268 | * assumed to be locked on entry and remains so on return. | 2271 | * assumed to be locked on entry and remains so on return. |
2269 | * | 2272 | * |
2270 | * Return: | 2273 | * Return: |
2271 | * %false - we are done with this request | 2274 | * %false - we are done with this request |
2272 | * %true - still buffers pending for this request | 2275 | * %true - still buffers pending for this request |
2273 | **/ | 2276 | **/ |
2274 | static bool __blk_end_bidi_request(struct request *rq, int error, | 2277 | static bool __blk_end_bidi_request(struct request *rq, int error, |
2275 | unsigned int nr_bytes, unsigned int bidi_bytes) | 2278 | unsigned int nr_bytes, unsigned int bidi_bytes) |
2276 | { | 2279 | { |
2277 | if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes)) | 2280 | if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes)) |
2278 | return true; | 2281 | return true; |
2279 | 2282 | ||
2280 | blk_finish_request(rq, error); | 2283 | blk_finish_request(rq, error); |
2281 | 2284 | ||
2282 | return false; | 2285 | return false; |
2283 | } | 2286 | } |
2284 | 2287 | ||
2285 | /** | 2288 | /** |
2286 | * blk_end_request - Helper function for drivers to complete the request. | 2289 | * blk_end_request - Helper function for drivers to complete the request. |
2287 | * @rq: the request being processed | 2290 | * @rq: the request being processed |
2288 | * @error: %0 for success, < %0 for error | 2291 | * @error: %0 for success, < %0 for error |
2289 | * @nr_bytes: number of bytes to complete | 2292 | * @nr_bytes: number of bytes to complete |
2290 | * | 2293 | * |
2291 | * Description: | 2294 | * Description: |
2292 | * Ends I/O on a number of bytes attached to @rq. | 2295 | * Ends I/O on a number of bytes attached to @rq. |
2293 | * If @rq has leftover, sets it up for the next range of segments. | 2296 | * If @rq has leftover, sets it up for the next range of segments. |
2294 | * | 2297 | * |
2295 | * Return: | 2298 | * Return: |
2296 | * %false - we are done with this request | 2299 | * %false - we are done with this request |
2297 | * %true - still buffers pending for this request | 2300 | * %true - still buffers pending for this request |
2298 | **/ | 2301 | **/ |
2299 | bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes) | 2302 | bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes) |
2300 | { | 2303 | { |
2301 | return blk_end_bidi_request(rq, error, nr_bytes, 0); | 2304 | return blk_end_bidi_request(rq, error, nr_bytes, 0); |
2302 | } | 2305 | } |
2303 | EXPORT_SYMBOL(blk_end_request); | 2306 | EXPORT_SYMBOL(blk_end_request); |
2304 | 2307 | ||
2305 | /** | 2308 | /** |
2306 | * blk_end_request_all - Helper function for drives to finish the request. | 2309 | * blk_end_request_all - Helper function for drives to finish the request. |
2307 | * @rq: the request to finish | 2310 | * @rq: the request to finish |
2308 | * @error: %0 for success, < %0 for error | 2311 | * @error: %0 for success, < %0 for error |
2309 | * | 2312 | * |
2310 | * Description: | 2313 | * Description: |
2311 | * Completely finish @rq. | 2314 | * Completely finish @rq. |
2312 | */ | 2315 | */ |
2313 | void blk_end_request_all(struct request *rq, int error) | 2316 | void blk_end_request_all(struct request *rq, int error) |
2314 | { | 2317 | { |
2315 | bool pending; | 2318 | bool pending; |
2316 | unsigned int bidi_bytes = 0; | 2319 | unsigned int bidi_bytes = 0; |
2317 | 2320 | ||
2318 | if (unlikely(blk_bidi_rq(rq))) | 2321 | if (unlikely(blk_bidi_rq(rq))) |
2319 | bidi_bytes = blk_rq_bytes(rq->next_rq); | 2322 | bidi_bytes = blk_rq_bytes(rq->next_rq); |
2320 | 2323 | ||
2321 | pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes); | 2324 | pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes); |
2322 | BUG_ON(pending); | 2325 | BUG_ON(pending); |
2323 | } | 2326 | } |
2324 | EXPORT_SYMBOL(blk_end_request_all); | 2327 | EXPORT_SYMBOL(blk_end_request_all); |
2325 | 2328 | ||
2326 | /** | 2329 | /** |
2327 | * blk_end_request_cur - Helper function to finish the current request chunk. | 2330 | * blk_end_request_cur - Helper function to finish the current request chunk. |
2328 | * @rq: the request to finish the current chunk for | 2331 | * @rq: the request to finish the current chunk for |
2329 | * @error: %0 for success, < %0 for error | 2332 | * @error: %0 for success, < %0 for error |
2330 | * | 2333 | * |
2331 | * Description: | 2334 | * Description: |
2332 | * Complete the current consecutively mapped chunk from @rq. | 2335 | * Complete the current consecutively mapped chunk from @rq. |
2333 | * | 2336 | * |
2334 | * Return: | 2337 | * Return: |
2335 | * %false - we are done with this request | 2338 | * %false - we are done with this request |
2336 | * %true - still buffers pending for this request | 2339 | * %true - still buffers pending for this request |
2337 | */ | 2340 | */ |
2338 | bool blk_end_request_cur(struct request *rq, int error) | 2341 | bool blk_end_request_cur(struct request *rq, int error) |
2339 | { | 2342 | { |
2340 | return blk_end_request(rq, error, blk_rq_cur_bytes(rq)); | 2343 | return blk_end_request(rq, error, blk_rq_cur_bytes(rq)); |
2341 | } | 2344 | } |
2342 | EXPORT_SYMBOL(blk_end_request_cur); | 2345 | EXPORT_SYMBOL(blk_end_request_cur); |
2343 | 2346 | ||
2344 | /** | 2347 | /** |
2345 | * blk_end_request_err - Finish a request till the next failure boundary. | 2348 | * blk_end_request_err - Finish a request till the next failure boundary. |
2346 | * @rq: the request to finish till the next failure boundary for | 2349 | * @rq: the request to finish till the next failure boundary for |
2347 | * @error: must be negative errno | 2350 | * @error: must be negative errno |
2348 | * | 2351 | * |
2349 | * Description: | 2352 | * Description: |
2350 | * Complete @rq till the next failure boundary. | 2353 | * Complete @rq till the next failure boundary. |
2351 | * | 2354 | * |
2352 | * Return: | 2355 | * Return: |
2353 | * %false - we are done with this request | 2356 | * %false - we are done with this request |
2354 | * %true - still buffers pending for this request | 2357 | * %true - still buffers pending for this request |
2355 | */ | 2358 | */ |
2356 | bool blk_end_request_err(struct request *rq, int error) | 2359 | bool blk_end_request_err(struct request *rq, int error) |
2357 | { | 2360 | { |
2358 | WARN_ON(error >= 0); | 2361 | WARN_ON(error >= 0); |
2359 | return blk_end_request(rq, error, blk_rq_err_bytes(rq)); | 2362 | return blk_end_request(rq, error, blk_rq_err_bytes(rq)); |
2360 | } | 2363 | } |
2361 | EXPORT_SYMBOL_GPL(blk_end_request_err); | 2364 | EXPORT_SYMBOL_GPL(blk_end_request_err); |
2362 | 2365 | ||
2363 | /** | 2366 | /** |
2364 | * __blk_end_request - Helper function for drivers to complete the request. | 2367 | * __blk_end_request - Helper function for drivers to complete the request. |
2365 | * @rq: the request being processed | 2368 | * @rq: the request being processed |
2366 | * @error: %0 for success, < %0 for error | 2369 | * @error: %0 for success, < %0 for error |
2367 | * @nr_bytes: number of bytes to complete | 2370 | * @nr_bytes: number of bytes to complete |
2368 | * | 2371 | * |
2369 | * Description: | 2372 | * Description: |
2370 | * Must be called with queue lock held unlike blk_end_request(). | 2373 | * Must be called with queue lock held unlike blk_end_request(). |
2371 | * | 2374 | * |
2372 | * Return: | 2375 | * Return: |
2373 | * %false - we are done with this request | 2376 | * %false - we are done with this request |
2374 | * %true - still buffers pending for this request | 2377 | * %true - still buffers pending for this request |
2375 | **/ | 2378 | **/ |
2376 | bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes) | 2379 | bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes) |
2377 | { | 2380 | { |
2378 | return __blk_end_bidi_request(rq, error, nr_bytes, 0); | 2381 | return __blk_end_bidi_request(rq, error, nr_bytes, 0); |
2379 | } | 2382 | } |
2380 | EXPORT_SYMBOL(__blk_end_request); | 2383 | EXPORT_SYMBOL(__blk_end_request); |
2381 | 2384 | ||
2382 | /** | 2385 | /** |
2383 | * __blk_end_request_all - Helper function for drives to finish the request. | 2386 | * __blk_end_request_all - Helper function for drives to finish the request. |
2384 | * @rq: the request to finish | 2387 | * @rq: the request to finish |
2385 | * @error: %0 for success, < %0 for error | 2388 | * @error: %0 for success, < %0 for error |
2386 | * | 2389 | * |
2387 | * Description: | 2390 | * Description: |
2388 | * Completely finish @rq. Must be called with queue lock held. | 2391 | * Completely finish @rq. Must be called with queue lock held. |
2389 | */ | 2392 | */ |
2390 | void __blk_end_request_all(struct request *rq, int error) | 2393 | void __blk_end_request_all(struct request *rq, int error) |
2391 | { | 2394 | { |
2392 | bool pending; | 2395 | bool pending; |
2393 | unsigned int bidi_bytes = 0; | 2396 | unsigned int bidi_bytes = 0; |
2394 | 2397 | ||
2395 | if (unlikely(blk_bidi_rq(rq))) | 2398 | if (unlikely(blk_bidi_rq(rq))) |
2396 | bidi_bytes = blk_rq_bytes(rq->next_rq); | 2399 | bidi_bytes = blk_rq_bytes(rq->next_rq); |
2397 | 2400 | ||
2398 | pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes); | 2401 | pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes); |
2399 | BUG_ON(pending); | 2402 | BUG_ON(pending); |
2400 | } | 2403 | } |
2401 | EXPORT_SYMBOL(__blk_end_request_all); | 2404 | EXPORT_SYMBOL(__blk_end_request_all); |
2402 | 2405 | ||
2403 | /** | 2406 | /** |
2404 | * __blk_end_request_cur - Helper function to finish the current request chunk. | 2407 | * __blk_end_request_cur - Helper function to finish the current request chunk. |
2405 | * @rq: the request to finish the current chunk for | 2408 | * @rq: the request to finish the current chunk for |
2406 | * @error: %0 for success, < %0 for error | 2409 | * @error: %0 for success, < %0 for error |
2407 | * | 2410 | * |
2408 | * Description: | 2411 | * Description: |
2409 | * Complete the current consecutively mapped chunk from @rq. Must | 2412 | * Complete the current consecutively mapped chunk from @rq. Must |
2410 | * be called with queue lock held. | 2413 | * be called with queue lock held. |
2411 | * | 2414 | * |
2412 | * Return: | 2415 | * Return: |
2413 | * %false - we are done with this request | 2416 | * %false - we are done with this request |
2414 | * %true - still buffers pending for this request | 2417 | * %true - still buffers pending for this request |
2415 | */ | 2418 | */ |
2416 | bool __blk_end_request_cur(struct request *rq, int error) | 2419 | bool __blk_end_request_cur(struct request *rq, int error) |
2417 | { | 2420 | { |
2418 | return __blk_end_request(rq, error, blk_rq_cur_bytes(rq)); | 2421 | return __blk_end_request(rq, error, blk_rq_cur_bytes(rq)); |
2419 | } | 2422 | } |
2420 | EXPORT_SYMBOL(__blk_end_request_cur); | 2423 | EXPORT_SYMBOL(__blk_end_request_cur); |
2421 | 2424 | ||
2422 | /** | 2425 | /** |
2423 | * __blk_end_request_err - Finish a request till the next failure boundary. | 2426 | * __blk_end_request_err - Finish a request till the next failure boundary. |
2424 | * @rq: the request to finish till the next failure boundary for | 2427 | * @rq: the request to finish till the next failure boundary for |
2425 | * @error: must be negative errno | 2428 | * @error: must be negative errno |
2426 | * | 2429 | * |
2427 | * Description: | 2430 | * Description: |
2428 | * Complete @rq till the next failure boundary. Must be called | 2431 | * Complete @rq till the next failure boundary. Must be called |
2429 | * with queue lock held. | 2432 | * with queue lock held. |
2430 | * | 2433 | * |
2431 | * Return: | 2434 | * Return: |
2432 | * %false - we are done with this request | 2435 | * %false - we are done with this request |
2433 | * %true - still buffers pending for this request | 2436 | * %true - still buffers pending for this request |
2434 | */ | 2437 | */ |
2435 | bool __blk_end_request_err(struct request *rq, int error) | 2438 | bool __blk_end_request_err(struct request *rq, int error) |
2436 | { | 2439 | { |
2437 | WARN_ON(error >= 0); | 2440 | WARN_ON(error >= 0); |
2438 | return __blk_end_request(rq, error, blk_rq_err_bytes(rq)); | 2441 | return __blk_end_request(rq, error, blk_rq_err_bytes(rq)); |
2439 | } | 2442 | } |
2440 | EXPORT_SYMBOL_GPL(__blk_end_request_err); | 2443 | EXPORT_SYMBOL_GPL(__blk_end_request_err); |
2441 | 2444 | ||
2442 | void blk_rq_bio_prep(struct request_queue *q, struct request *rq, | 2445 | void blk_rq_bio_prep(struct request_queue *q, struct request *rq, |
2443 | struct bio *bio) | 2446 | struct bio *bio) |
2444 | { | 2447 | { |
2445 | /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */ | 2448 | /* Bit 0 (R/W) is identical in rq->cmd_flags and bio->bi_rw */ |
2446 | rq->cmd_flags |= bio->bi_rw & REQ_WRITE; | 2449 | rq->cmd_flags |= bio->bi_rw & REQ_WRITE; |
2447 | 2450 | ||
2448 | if (bio_has_data(bio)) { | 2451 | if (bio_has_data(bio)) { |
2449 | rq->nr_phys_segments = bio_phys_segments(q, bio); | 2452 | rq->nr_phys_segments = bio_phys_segments(q, bio); |
2450 | rq->buffer = bio_data(bio); | 2453 | rq->buffer = bio_data(bio); |
2451 | } | 2454 | } |
2452 | rq->__data_len = bio->bi_size; | 2455 | rq->__data_len = bio->bi_size; |
2453 | rq->bio = rq->biotail = bio; | 2456 | rq->bio = rq->biotail = bio; |
2454 | 2457 | ||
2455 | if (bio->bi_bdev) | 2458 | if (bio->bi_bdev) |
2456 | rq->rq_disk = bio->bi_bdev->bd_disk; | 2459 | rq->rq_disk = bio->bi_bdev->bd_disk; |
2457 | } | 2460 | } |
2458 | 2461 | ||
2459 | #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE | 2462 | #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE |
2460 | /** | 2463 | /** |
2461 | * rq_flush_dcache_pages - Helper function to flush all pages in a request | 2464 | * rq_flush_dcache_pages - Helper function to flush all pages in a request |
2462 | * @rq: the request to be flushed | 2465 | * @rq: the request to be flushed |
2463 | * | 2466 | * |
2464 | * Description: | 2467 | * Description: |
2465 | * Flush all pages in @rq. | 2468 | * Flush all pages in @rq. |
2466 | */ | 2469 | */ |
2467 | void rq_flush_dcache_pages(struct request *rq) | 2470 | void rq_flush_dcache_pages(struct request *rq) |
2468 | { | 2471 | { |
2469 | struct req_iterator iter; | 2472 | struct req_iterator iter; |
2470 | struct bio_vec *bvec; | 2473 | struct bio_vec *bvec; |
2471 | 2474 | ||
2472 | rq_for_each_segment(bvec, rq, iter) | 2475 | rq_for_each_segment(bvec, rq, iter) |
2473 | flush_dcache_page(bvec->bv_page); | 2476 | flush_dcache_page(bvec->bv_page); |
2474 | } | 2477 | } |
2475 | EXPORT_SYMBOL_GPL(rq_flush_dcache_pages); | 2478 | EXPORT_SYMBOL_GPL(rq_flush_dcache_pages); |
2476 | #endif | 2479 | #endif |
2477 | 2480 | ||
2478 | /** | 2481 | /** |
2479 | * blk_lld_busy - Check if underlying low-level drivers of a device are busy | 2482 | * blk_lld_busy - Check if underlying low-level drivers of a device are busy |
2480 | * @q : the queue of the device being checked | 2483 | * @q : the queue of the device being checked |
2481 | * | 2484 | * |
2482 | * Description: | 2485 | * Description: |
2483 | * Check if underlying low-level drivers of a device are busy. | 2486 | * Check if underlying low-level drivers of a device are busy. |
2484 | * If the drivers want to export their busy state, they must set own | 2487 | * If the drivers want to export their busy state, they must set own |
2485 | * exporting function using blk_queue_lld_busy() first. | 2488 | * exporting function using blk_queue_lld_busy() first. |
2486 | * | 2489 | * |
2487 | * Basically, this function is used only by request stacking drivers | 2490 | * Basically, this function is used only by request stacking drivers |
2488 | * to stop dispatching requests to underlying devices when underlying | 2491 | * to stop dispatching requests to underlying devices when underlying |
2489 | * devices are busy. This behavior helps more I/O merging on the queue | 2492 | * devices are busy. This behavior helps more I/O merging on the queue |
2490 | * of the request stacking driver and prevents I/O throughput regression | 2493 | * of the request stacking driver and prevents I/O throughput regression |
2491 | * on burst I/O load. | 2494 | * on burst I/O load. |
2492 | * | 2495 | * |
2493 | * Return: | 2496 | * Return: |
2494 | * 0 - Not busy (The request stacking driver should dispatch request) | 2497 | * 0 - Not busy (The request stacking driver should dispatch request) |
2495 | * 1 - Busy (The request stacking driver should stop dispatching request) | 2498 | * 1 - Busy (The request stacking driver should stop dispatching request) |
2496 | */ | 2499 | */ |
2497 | int blk_lld_busy(struct request_queue *q) | 2500 | int blk_lld_busy(struct request_queue *q) |
2498 | { | 2501 | { |
2499 | if (q->lld_busy_fn) | 2502 | if (q->lld_busy_fn) |
2500 | return q->lld_busy_fn(q); | 2503 | return q->lld_busy_fn(q); |
2501 | 2504 | ||
2502 | return 0; | 2505 | return 0; |
2503 | } | 2506 | } |
2504 | EXPORT_SYMBOL_GPL(blk_lld_busy); | 2507 | EXPORT_SYMBOL_GPL(blk_lld_busy); |
2505 | 2508 | ||
2506 | /** | 2509 | /** |
2507 | * blk_rq_unprep_clone - Helper function to free all bios in a cloned request | 2510 | * blk_rq_unprep_clone - Helper function to free all bios in a cloned request |
2508 | * @rq: the clone request to be cleaned up | 2511 | * @rq: the clone request to be cleaned up |
2509 | * | 2512 | * |
2510 | * Description: | 2513 | * Description: |
2511 | * Free all bios in @rq for a cloned request. | 2514 | * Free all bios in @rq for a cloned request. |
2512 | */ | 2515 | */ |
2513 | void blk_rq_unprep_clone(struct request *rq) | 2516 | void blk_rq_unprep_clone(struct request *rq) |
2514 | { | 2517 | { |
2515 | struct bio *bio; | 2518 | struct bio *bio; |
2516 | 2519 | ||
2517 | while ((bio = rq->bio) != NULL) { | 2520 | while ((bio = rq->bio) != NULL) { |
2518 | rq->bio = bio->bi_next; | 2521 | rq->bio = bio->bi_next; |
2519 | 2522 | ||
2520 | bio_put(bio); | 2523 | bio_put(bio); |
2521 | } | 2524 | } |
2522 | } | 2525 | } |
2523 | EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); | 2526 | EXPORT_SYMBOL_GPL(blk_rq_unprep_clone); |
2524 | 2527 | ||
2525 | /* | 2528 | /* |
2526 | * Copy attributes of the original request to the clone request. | 2529 | * Copy attributes of the original request to the clone request. |
2527 | * The actual data parts (e.g. ->cmd, ->buffer, ->sense) are not copied. | 2530 | * The actual data parts (e.g. ->cmd, ->buffer, ->sense) are not copied. |
2528 | */ | 2531 | */ |
2529 | static void __blk_rq_prep_clone(struct request *dst, struct request *src) | 2532 | static void __blk_rq_prep_clone(struct request *dst, struct request *src) |
2530 | { | 2533 | { |
2531 | dst->cpu = src->cpu; | 2534 | dst->cpu = src->cpu; |
2532 | dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE; | 2535 | dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE; |
2533 | dst->cmd_type = src->cmd_type; | 2536 | dst->cmd_type = src->cmd_type; |
2534 | dst->__sector = blk_rq_pos(src); | 2537 | dst->__sector = blk_rq_pos(src); |
2535 | dst->__data_len = blk_rq_bytes(src); | 2538 | dst->__data_len = blk_rq_bytes(src); |
2536 | dst->nr_phys_segments = src->nr_phys_segments; | 2539 | dst->nr_phys_segments = src->nr_phys_segments; |
2537 | dst->ioprio = src->ioprio; | 2540 | dst->ioprio = src->ioprio; |
2538 | dst->extra_len = src->extra_len; | 2541 | dst->extra_len = src->extra_len; |
2539 | } | 2542 | } |
2540 | 2543 | ||
2541 | /** | 2544 | /** |
2542 | * blk_rq_prep_clone - Helper function to setup clone request | 2545 | * blk_rq_prep_clone - Helper function to setup clone request |
2543 | * @rq: the request to be setup | 2546 | * @rq: the request to be setup |
2544 | * @rq_src: original request to be cloned | 2547 | * @rq_src: original request to be cloned |
2545 | * @bs: bio_set that bios for clone are allocated from | 2548 | * @bs: bio_set that bios for clone are allocated from |
2546 | * @gfp_mask: memory allocation mask for bio | 2549 | * @gfp_mask: memory allocation mask for bio |
2547 | * @bio_ctr: setup function to be called for each clone bio. | 2550 | * @bio_ctr: setup function to be called for each clone bio. |
2548 | * Returns %0 for success, non %0 for failure. | 2551 | * Returns %0 for success, non %0 for failure. |
2549 | * @data: private data to be passed to @bio_ctr | 2552 | * @data: private data to be passed to @bio_ctr |
2550 | * | 2553 | * |
2551 | * Description: | 2554 | * Description: |
2552 | * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq. | 2555 | * Clones bios in @rq_src to @rq, and copies attributes of @rq_src to @rq. |
2553 | * The actual data parts of @rq_src (e.g. ->cmd, ->buffer, ->sense) | 2556 | * The actual data parts of @rq_src (e.g. ->cmd, ->buffer, ->sense) |
2554 | * are not copied, and copying such parts is the caller's responsibility. | 2557 | * are not copied, and copying such parts is the caller's responsibility. |
2555 | * Also, pages which the original bios are pointing to are not copied | 2558 | * Also, pages which the original bios are pointing to are not copied |
2556 | * and the cloned bios just point same pages. | 2559 | * and the cloned bios just point same pages. |
2557 | * So cloned bios must be completed before original bios, which means | 2560 | * So cloned bios must be completed before original bios, which means |
2558 | * the caller must complete @rq before @rq_src. | 2561 | * the caller must complete @rq before @rq_src. |
2559 | */ | 2562 | */ |
2560 | int blk_rq_prep_clone(struct request *rq, struct request *rq_src, | 2563 | int blk_rq_prep_clone(struct request *rq, struct request *rq_src, |
2561 | struct bio_set *bs, gfp_t gfp_mask, | 2564 | struct bio_set *bs, gfp_t gfp_mask, |
2562 | int (*bio_ctr)(struct bio *, struct bio *, void *), | 2565 | int (*bio_ctr)(struct bio *, struct bio *, void *), |
2563 | void *data) | 2566 | void *data) |
2564 | { | 2567 | { |
2565 | struct bio *bio, *bio_src; | 2568 | struct bio *bio, *bio_src; |
2566 | 2569 | ||
2567 | if (!bs) | 2570 | if (!bs) |
2568 | bs = fs_bio_set; | 2571 | bs = fs_bio_set; |
2569 | 2572 | ||
2570 | blk_rq_init(NULL, rq); | 2573 | blk_rq_init(NULL, rq); |
2571 | 2574 | ||
2572 | __rq_for_each_bio(bio_src, rq_src) { | 2575 | __rq_for_each_bio(bio_src, rq_src) { |
2573 | bio = bio_alloc_bioset(gfp_mask, bio_src->bi_max_vecs, bs); | 2576 | bio = bio_alloc_bioset(gfp_mask, bio_src->bi_max_vecs, bs); |
2574 | if (!bio) | 2577 | if (!bio) |
2575 | goto free_and_out; | 2578 | goto free_and_out; |
2576 | 2579 | ||
2577 | __bio_clone(bio, bio_src); | 2580 | __bio_clone(bio, bio_src); |
2578 | 2581 | ||
2579 | if (bio_integrity(bio_src) && | 2582 | if (bio_integrity(bio_src) && |
2580 | bio_integrity_clone(bio, bio_src, gfp_mask, bs)) | 2583 | bio_integrity_clone(bio, bio_src, gfp_mask, bs)) |
2581 | goto free_and_out; | 2584 | goto free_and_out; |
2582 | 2585 | ||
2583 | if (bio_ctr && bio_ctr(bio, bio_src, data)) | 2586 | if (bio_ctr && bio_ctr(bio, bio_src, data)) |
2584 | goto free_and_out; | 2587 | goto free_and_out; |
2585 | 2588 | ||
2586 | if (rq->bio) { | 2589 | if (rq->bio) { |
2587 | rq->biotail->bi_next = bio; | 2590 | rq->biotail->bi_next = bio; |
2588 | rq->biotail = bio; | 2591 | rq->biotail = bio; |
2589 | } else | 2592 | } else |
2590 | rq->bio = rq->biotail = bio; | 2593 | rq->bio = rq->biotail = bio; |
2591 | } | 2594 | } |
2592 | 2595 | ||
2593 | __blk_rq_prep_clone(rq, rq_src); | 2596 | __blk_rq_prep_clone(rq, rq_src); |
2594 | 2597 | ||
2595 | return 0; | 2598 | return 0; |
2596 | 2599 | ||
2597 | free_and_out: | 2600 | free_and_out: |
2598 | if (bio) | 2601 | if (bio) |
2599 | bio_free(bio, bs); | 2602 | bio_free(bio, bs); |
2600 | blk_rq_unprep_clone(rq); | 2603 | blk_rq_unprep_clone(rq); |
2601 | 2604 | ||
2602 | return -ENOMEM; | 2605 | return -ENOMEM; |
2603 | } | 2606 | } |
2604 | EXPORT_SYMBOL_GPL(blk_rq_prep_clone); | 2607 | EXPORT_SYMBOL_GPL(blk_rq_prep_clone); |
2605 | 2608 | ||
2606 | int kblockd_schedule_work(struct request_queue *q, struct work_struct *work) | 2609 | int kblockd_schedule_work(struct request_queue *q, struct work_struct *work) |
2607 | { | 2610 | { |
2608 | return queue_work(kblockd_workqueue, work); | 2611 | return queue_work(kblockd_workqueue, work); |
2609 | } | 2612 | } |
2610 | EXPORT_SYMBOL(kblockd_schedule_work); | 2613 | EXPORT_SYMBOL(kblockd_schedule_work); |
2611 | 2614 | ||
2612 | int kblockd_schedule_delayed_work(struct request_queue *q, | 2615 | int kblockd_schedule_delayed_work(struct request_queue *q, |
2613 | struct delayed_work *dwork, unsigned long delay) | 2616 | struct delayed_work *dwork, unsigned long delay) |
2614 | { | 2617 | { |
2615 | return queue_delayed_work(kblockd_workqueue, dwork, delay); | 2618 | return queue_delayed_work(kblockd_workqueue, dwork, delay); |
2616 | } | 2619 | } |
2617 | EXPORT_SYMBOL(kblockd_schedule_delayed_work); | 2620 | EXPORT_SYMBOL(kblockd_schedule_delayed_work); |
2618 | 2621 | ||
2619 | #define PLUG_MAGIC 0x91827364 | 2622 | #define PLUG_MAGIC 0x91827364 |
2620 | 2623 | ||
2621 | void blk_start_plug(struct blk_plug *plug) | 2624 | void blk_start_plug(struct blk_plug *plug) |
2622 | { | 2625 | { |
2623 | struct task_struct *tsk = current; | 2626 | struct task_struct *tsk = current; |
2624 | 2627 | ||
2625 | plug->magic = PLUG_MAGIC; | 2628 | plug->magic = PLUG_MAGIC; |
2626 | INIT_LIST_HEAD(&plug->list); | 2629 | INIT_LIST_HEAD(&plug->list); |
2627 | INIT_LIST_HEAD(&plug->cb_list); | 2630 | INIT_LIST_HEAD(&plug->cb_list); |
2628 | plug->should_sort = 0; | 2631 | plug->should_sort = 0; |
2632 | plug->count = 0; | ||
2629 | 2633 | ||
2630 | /* | 2634 | /* |
2631 | * If this is a nested plug, don't actually assign it. It will be | 2635 | * If this is a nested plug, don't actually assign it. It will be |
2632 | * flushed on its own. | 2636 | * flushed on its own. |
2633 | */ | 2637 | */ |
2634 | if (!tsk->plug) { | 2638 | if (!tsk->plug) { |
2635 | /* | 2639 | /* |
2636 | * Store ordering should not be needed here, since a potential | 2640 | * Store ordering should not be needed here, since a potential |
2637 | * preempt will imply a full memory barrier | 2641 | * preempt will imply a full memory barrier |
2638 | */ | 2642 | */ |
2639 | tsk->plug = plug; | 2643 | tsk->plug = plug; |
2640 | } | 2644 | } |
2641 | } | 2645 | } |
2642 | EXPORT_SYMBOL(blk_start_plug); | 2646 | EXPORT_SYMBOL(blk_start_plug); |
2643 | 2647 | ||
2644 | static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) | 2648 | static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b) |
2645 | { | 2649 | { |
2646 | struct request *rqa = container_of(a, struct request, queuelist); | 2650 | struct request *rqa = container_of(a, struct request, queuelist); |
2647 | struct request *rqb = container_of(b, struct request, queuelist); | 2651 | struct request *rqb = container_of(b, struct request, queuelist); |
2648 | 2652 | ||
2649 | return !(rqa->q <= rqb->q); | 2653 | return !(rqa->q <= rqb->q); |
2650 | } | 2654 | } |
2651 | 2655 | ||
2652 | /* | 2656 | /* |
2653 | * If 'from_schedule' is true, then postpone the dispatch of requests | 2657 | * If 'from_schedule' is true, then postpone the dispatch of requests |
2654 | * until a safe kblockd context. We due this to avoid accidental big | 2658 | * until a safe kblockd context. We due this to avoid accidental big |
2655 | * additional stack usage in driver dispatch, in places where the originally | 2659 | * additional stack usage in driver dispatch, in places where the originally |
2656 | * plugger did not intend it. | 2660 | * plugger did not intend it. |
2657 | */ | 2661 | */ |
2658 | static void queue_unplugged(struct request_queue *q, unsigned int depth, | 2662 | static void queue_unplugged(struct request_queue *q, unsigned int depth, |
2659 | bool from_schedule) | 2663 | bool from_schedule) |
2660 | __releases(q->queue_lock) | 2664 | __releases(q->queue_lock) |
2661 | { | 2665 | { |
2662 | trace_block_unplug(q, depth, !from_schedule); | 2666 | trace_block_unplug(q, depth, !from_schedule); |
2663 | 2667 | ||
2664 | /* | 2668 | /* |
2665 | * If we are punting this to kblockd, then we can safely drop | 2669 | * If we are punting this to kblockd, then we can safely drop |
2666 | * the queue_lock before waking kblockd (which needs to take | 2670 | * the queue_lock before waking kblockd (which needs to take |
2667 | * this lock). | 2671 | * this lock). |
2668 | */ | 2672 | */ |
2669 | if (from_schedule) { | 2673 | if (from_schedule) { |
2670 | spin_unlock(q->queue_lock); | 2674 | spin_unlock(q->queue_lock); |
2671 | blk_run_queue_async(q); | 2675 | blk_run_queue_async(q); |
2672 | } else { | 2676 | } else { |
2673 | __blk_run_queue(q); | 2677 | __blk_run_queue(q); |
2674 | spin_unlock(q->queue_lock); | 2678 | spin_unlock(q->queue_lock); |
2675 | } | 2679 | } |
2676 | 2680 | ||
2677 | } | 2681 | } |
2678 | 2682 | ||
2679 | static void flush_plug_callbacks(struct blk_plug *plug) | 2683 | static void flush_plug_callbacks(struct blk_plug *plug) |
2680 | { | 2684 | { |
2681 | LIST_HEAD(callbacks); | 2685 | LIST_HEAD(callbacks); |
2682 | 2686 | ||
2683 | if (list_empty(&plug->cb_list)) | 2687 | if (list_empty(&plug->cb_list)) |
2684 | return; | 2688 | return; |
2685 | 2689 | ||
2686 | list_splice_init(&plug->cb_list, &callbacks); | 2690 | list_splice_init(&plug->cb_list, &callbacks); |
2687 | 2691 | ||
2688 | while (!list_empty(&callbacks)) { | 2692 | while (!list_empty(&callbacks)) { |
2689 | struct blk_plug_cb *cb = list_first_entry(&callbacks, | 2693 | struct blk_plug_cb *cb = list_first_entry(&callbacks, |
2690 | struct blk_plug_cb, | 2694 | struct blk_plug_cb, |
2691 | list); | 2695 | list); |
2692 | list_del(&cb->list); | 2696 | list_del(&cb->list); |
2693 | cb->callback(cb); | 2697 | cb->callback(cb); |
2694 | } | 2698 | } |
2695 | } | 2699 | } |
2696 | 2700 | ||
2697 | void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) | 2701 | void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) |
2698 | { | 2702 | { |
2699 | struct request_queue *q; | 2703 | struct request_queue *q; |
2700 | unsigned long flags; | 2704 | unsigned long flags; |
2701 | struct request *rq; | 2705 | struct request *rq; |
2702 | LIST_HEAD(list); | 2706 | LIST_HEAD(list); |
2703 | unsigned int depth; | 2707 | unsigned int depth; |
2704 | 2708 | ||
2705 | BUG_ON(plug->magic != PLUG_MAGIC); | 2709 | BUG_ON(plug->magic != PLUG_MAGIC); |
2706 | 2710 | ||
2707 | flush_plug_callbacks(plug); | 2711 | flush_plug_callbacks(plug); |
2708 | if (list_empty(&plug->list)) | 2712 | if (list_empty(&plug->list)) |
2709 | return; | 2713 | return; |
2710 | 2714 | ||
2711 | list_splice_init(&plug->list, &list); | 2715 | list_splice_init(&plug->list, &list); |
2716 | plug->count = 0; | ||
2712 | 2717 | ||
2713 | if (plug->should_sort) { | 2718 | if (plug->should_sort) { |
2714 | list_sort(NULL, &list, plug_rq_cmp); | 2719 | list_sort(NULL, &list, plug_rq_cmp); |
2715 | plug->should_sort = 0; | 2720 | plug->should_sort = 0; |
2716 | } | 2721 | } |
2717 | 2722 | ||
2718 | q = NULL; | 2723 | q = NULL; |
2719 | depth = 0; | 2724 | depth = 0; |
2720 | 2725 | ||
2721 | /* | 2726 | /* |
2722 | * Save and disable interrupts here, to avoid doing it for every | 2727 | * Save and disable interrupts here, to avoid doing it for every |
2723 | * queue lock we have to take. | 2728 | * queue lock we have to take. |
2724 | */ | 2729 | */ |
2725 | local_irq_save(flags); | 2730 | local_irq_save(flags); |
2726 | while (!list_empty(&list)) { | 2731 | while (!list_empty(&list)) { |
2727 | rq = list_entry_rq(list.next); | 2732 | rq = list_entry_rq(list.next); |
2728 | list_del_init(&rq->queuelist); | 2733 | list_del_init(&rq->queuelist); |
2729 | BUG_ON(!rq->q); | 2734 | BUG_ON(!rq->q); |
2730 | if (rq->q != q) { | 2735 | if (rq->q != q) { |
2731 | /* | 2736 | /* |
2732 | * This drops the queue lock | 2737 | * This drops the queue lock |
2733 | */ | 2738 | */ |
2734 | if (q) | 2739 | if (q) |
2735 | queue_unplugged(q, depth, from_schedule); | 2740 | queue_unplugged(q, depth, from_schedule); |
2736 | q = rq->q; | 2741 | q = rq->q; |
2737 | depth = 0; | 2742 | depth = 0; |
2738 | spin_lock(q->queue_lock); | 2743 | spin_lock(q->queue_lock); |
2739 | } | 2744 | } |
2740 | /* | 2745 | /* |
2741 | * rq is already accounted, so use raw insert | 2746 | * rq is already accounted, so use raw insert |
2742 | */ | 2747 | */ |
2743 | if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA)) | 2748 | if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA)) |
2744 | __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH); | 2749 | __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH); |
2745 | else | 2750 | else |
2746 | __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE); | 2751 | __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE); |
2747 | 2752 | ||
2748 | depth++; | 2753 | depth++; |
2749 | } | 2754 | } |
2750 | 2755 | ||
2751 | /* | 2756 | /* |
2752 | * This drops the queue lock | 2757 | * This drops the queue lock |
2753 | */ | 2758 | */ |
2754 | if (q) | 2759 | if (q) |
2755 | queue_unplugged(q, depth, from_schedule); | 2760 | queue_unplugged(q, depth, from_schedule); |
2756 | 2761 | ||
2757 | local_irq_restore(flags); | 2762 | local_irq_restore(flags); |
2758 | } | 2763 | } |
2759 | 2764 | ||
2760 | void blk_finish_plug(struct blk_plug *plug) | 2765 | void blk_finish_plug(struct blk_plug *plug) |
2761 | { | 2766 | { |
2762 | blk_flush_plug_list(plug, false); | 2767 | blk_flush_plug_list(plug, false); |
2763 | 2768 | ||
2764 | if (plug == current->plug) | 2769 | if (plug == current->plug) |
2765 | current->plug = NULL; | 2770 | current->plug = NULL; |
2766 | } | 2771 | } |
2767 | EXPORT_SYMBOL(blk_finish_plug); | 2772 | EXPORT_SYMBOL(blk_finish_plug); |
2768 | 2773 | ||
2769 | int __init blk_dev_init(void) | 2774 | int __init blk_dev_init(void) |
2770 | { | 2775 | { |
2771 | BUILD_BUG_ON(__REQ_NR_BITS > 8 * | 2776 | BUILD_BUG_ON(__REQ_NR_BITS > 8 * |
2772 | sizeof(((struct request *)0)->cmd_flags)); | 2777 | sizeof(((struct request *)0)->cmd_flags)); |
2773 | 2778 | ||
2774 | /* used for unplugging and affects IO latency/throughput - HIGHPRI */ | 2779 | /* used for unplugging and affects IO latency/throughput - HIGHPRI */ |
2775 | kblockd_workqueue = alloc_workqueue("kblockd", | 2780 | kblockd_workqueue = alloc_workqueue("kblockd", |
2776 | WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); | 2781 | WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); |
2777 | if (!kblockd_workqueue) | 2782 | if (!kblockd_workqueue) |
2778 | panic("Failed to create kblockd\n"); | 2783 | panic("Failed to create kblockd\n"); |
2779 | 2784 | ||
2780 | request_cachep = kmem_cache_create("blkdev_requests", | 2785 | request_cachep = kmem_cache_create("blkdev_requests", |
2781 | sizeof(struct request), 0, SLAB_PANIC, NULL); | 2786 | sizeof(struct request), 0, SLAB_PANIC, NULL); |
2782 | 2787 | ||
2783 | blk_requestq_cachep = kmem_cache_create("blkdev_queue", | 2788 | blk_requestq_cachep = kmem_cache_create("blkdev_queue", |
2784 | sizeof(struct request_queue), 0, SLAB_PANIC, NULL); | 2789 | sizeof(struct request_queue), 0, SLAB_PANIC, NULL); |
2785 | 2790 | ||
2786 | return 0; | 2791 | return 0; |
2787 | } | 2792 | } |
2788 | 2793 |
include/linux/blkdev.h
1 | #ifndef _LINUX_BLKDEV_H | 1 | #ifndef _LINUX_BLKDEV_H |
2 | #define _LINUX_BLKDEV_H | 2 | #define _LINUX_BLKDEV_H |
3 | 3 | ||
4 | #ifdef CONFIG_BLOCK | 4 | #ifdef CONFIG_BLOCK |
5 | 5 | ||
6 | #include <linux/sched.h> | 6 | #include <linux/sched.h> |
7 | #include <linux/major.h> | 7 | #include <linux/major.h> |
8 | #include <linux/genhd.h> | 8 | #include <linux/genhd.h> |
9 | #include <linux/list.h> | 9 | #include <linux/list.h> |
10 | #include <linux/timer.h> | 10 | #include <linux/timer.h> |
11 | #include <linux/workqueue.h> | 11 | #include <linux/workqueue.h> |
12 | #include <linux/pagemap.h> | 12 | #include <linux/pagemap.h> |
13 | #include <linux/backing-dev.h> | 13 | #include <linux/backing-dev.h> |
14 | #include <linux/wait.h> | 14 | #include <linux/wait.h> |
15 | #include <linux/mempool.h> | 15 | #include <linux/mempool.h> |
16 | #include <linux/bio.h> | 16 | #include <linux/bio.h> |
17 | #include <linux/module.h> | 17 | #include <linux/module.h> |
18 | #include <linux/stringify.h> | 18 | #include <linux/stringify.h> |
19 | #include <linux/gfp.h> | 19 | #include <linux/gfp.h> |
20 | #include <linux/bsg.h> | 20 | #include <linux/bsg.h> |
21 | #include <linux/smp.h> | 21 | #include <linux/smp.h> |
22 | 22 | ||
23 | #include <asm/scatterlist.h> | 23 | #include <asm/scatterlist.h> |
24 | 24 | ||
25 | struct scsi_ioctl_command; | 25 | struct scsi_ioctl_command; |
26 | 26 | ||
27 | struct request_queue; | 27 | struct request_queue; |
28 | struct elevator_queue; | 28 | struct elevator_queue; |
29 | struct request_pm_state; | 29 | struct request_pm_state; |
30 | struct blk_trace; | 30 | struct blk_trace; |
31 | struct request; | 31 | struct request; |
32 | struct sg_io_hdr; | 32 | struct sg_io_hdr; |
33 | 33 | ||
34 | #define BLKDEV_MIN_RQ 4 | 34 | #define BLKDEV_MIN_RQ 4 |
35 | #define BLKDEV_MAX_RQ 128 /* Default maximum */ | 35 | #define BLKDEV_MAX_RQ 128 /* Default maximum */ |
36 | 36 | ||
37 | struct request; | 37 | struct request; |
38 | typedef void (rq_end_io_fn)(struct request *, int); | 38 | typedef void (rq_end_io_fn)(struct request *, int); |
39 | 39 | ||
40 | struct request_list { | 40 | struct request_list { |
41 | /* | 41 | /* |
42 | * count[], starved[], and wait[] are indexed by | 42 | * count[], starved[], and wait[] are indexed by |
43 | * BLK_RW_SYNC/BLK_RW_ASYNC | 43 | * BLK_RW_SYNC/BLK_RW_ASYNC |
44 | */ | 44 | */ |
45 | int count[2]; | 45 | int count[2]; |
46 | int starved[2]; | 46 | int starved[2]; |
47 | int elvpriv; | 47 | int elvpriv; |
48 | mempool_t *rq_pool; | 48 | mempool_t *rq_pool; |
49 | wait_queue_head_t wait[2]; | 49 | wait_queue_head_t wait[2]; |
50 | }; | 50 | }; |
51 | 51 | ||
52 | /* | 52 | /* |
53 | * request command types | 53 | * request command types |
54 | */ | 54 | */ |
55 | enum rq_cmd_type_bits { | 55 | enum rq_cmd_type_bits { |
56 | REQ_TYPE_FS = 1, /* fs request */ | 56 | REQ_TYPE_FS = 1, /* fs request */ |
57 | REQ_TYPE_BLOCK_PC, /* scsi command */ | 57 | REQ_TYPE_BLOCK_PC, /* scsi command */ |
58 | REQ_TYPE_SENSE, /* sense request */ | 58 | REQ_TYPE_SENSE, /* sense request */ |
59 | REQ_TYPE_PM_SUSPEND, /* suspend request */ | 59 | REQ_TYPE_PM_SUSPEND, /* suspend request */ |
60 | REQ_TYPE_PM_RESUME, /* resume request */ | 60 | REQ_TYPE_PM_RESUME, /* resume request */ |
61 | REQ_TYPE_PM_SHUTDOWN, /* shutdown request */ | 61 | REQ_TYPE_PM_SHUTDOWN, /* shutdown request */ |
62 | REQ_TYPE_SPECIAL, /* driver defined type */ | 62 | REQ_TYPE_SPECIAL, /* driver defined type */ |
63 | /* | 63 | /* |
64 | * for ATA/ATAPI devices. this really doesn't belong here, ide should | 64 | * for ATA/ATAPI devices. this really doesn't belong here, ide should |
65 | * use REQ_TYPE_SPECIAL and use rq->cmd[0] with the range of driver | 65 | * use REQ_TYPE_SPECIAL and use rq->cmd[0] with the range of driver |
66 | * private REQ_LB opcodes to differentiate what type of request this is | 66 | * private REQ_LB opcodes to differentiate what type of request this is |
67 | */ | 67 | */ |
68 | REQ_TYPE_ATA_TASKFILE, | 68 | REQ_TYPE_ATA_TASKFILE, |
69 | REQ_TYPE_ATA_PC, | 69 | REQ_TYPE_ATA_PC, |
70 | }; | 70 | }; |
71 | 71 | ||
72 | #define BLK_MAX_CDB 16 | 72 | #define BLK_MAX_CDB 16 |
73 | 73 | ||
74 | /* | 74 | /* |
75 | * try to put the fields that are referenced together in the same cacheline. | 75 | * try to put the fields that are referenced together in the same cacheline. |
76 | * if you modify this structure, be sure to check block/blk-core.c:blk_rq_init() | 76 | * if you modify this structure, be sure to check block/blk-core.c:blk_rq_init() |
77 | * as well! | 77 | * as well! |
78 | */ | 78 | */ |
79 | struct request { | 79 | struct request { |
80 | struct list_head queuelist; | 80 | struct list_head queuelist; |
81 | struct call_single_data csd; | 81 | struct call_single_data csd; |
82 | 82 | ||
83 | struct request_queue *q; | 83 | struct request_queue *q; |
84 | 84 | ||
85 | unsigned int cmd_flags; | 85 | unsigned int cmd_flags; |
86 | enum rq_cmd_type_bits cmd_type; | 86 | enum rq_cmd_type_bits cmd_type; |
87 | unsigned long atomic_flags; | 87 | unsigned long atomic_flags; |
88 | 88 | ||
89 | int cpu; | 89 | int cpu; |
90 | 90 | ||
91 | /* the following two fields are internal, NEVER access directly */ | 91 | /* the following two fields are internal, NEVER access directly */ |
92 | unsigned int __data_len; /* total data len */ | 92 | unsigned int __data_len; /* total data len */ |
93 | sector_t __sector; /* sector cursor */ | 93 | sector_t __sector; /* sector cursor */ |
94 | 94 | ||
95 | struct bio *bio; | 95 | struct bio *bio; |
96 | struct bio *biotail; | 96 | struct bio *biotail; |
97 | 97 | ||
98 | struct hlist_node hash; /* merge hash */ | 98 | struct hlist_node hash; /* merge hash */ |
99 | /* | 99 | /* |
100 | * The rb_node is only used inside the io scheduler, requests | 100 | * The rb_node is only used inside the io scheduler, requests |
101 | * are pruned when moved to the dispatch queue. So let the | 101 | * are pruned when moved to the dispatch queue. So let the |
102 | * completion_data share space with the rb_node. | 102 | * completion_data share space with the rb_node. |
103 | */ | 103 | */ |
104 | union { | 104 | union { |
105 | struct rb_node rb_node; /* sort/lookup */ | 105 | struct rb_node rb_node; /* sort/lookup */ |
106 | void *completion_data; | 106 | void *completion_data; |
107 | }; | 107 | }; |
108 | 108 | ||
109 | /* | 109 | /* |
110 | * Three pointers are available for the IO schedulers, if they need | 110 | * Three pointers are available for the IO schedulers, if they need |
111 | * more they have to dynamically allocate it. Flush requests are | 111 | * more they have to dynamically allocate it. Flush requests are |
112 | * never put on the IO scheduler. So let the flush fields share | 112 | * never put on the IO scheduler. So let the flush fields share |
113 | * space with the three elevator_private pointers. | 113 | * space with the three elevator_private pointers. |
114 | */ | 114 | */ |
115 | union { | 115 | union { |
116 | void *elevator_private[3]; | 116 | void *elevator_private[3]; |
117 | struct { | 117 | struct { |
118 | unsigned int seq; | 118 | unsigned int seq; |
119 | struct list_head list; | 119 | struct list_head list; |
120 | } flush; | 120 | } flush; |
121 | }; | 121 | }; |
122 | 122 | ||
123 | struct gendisk *rq_disk; | 123 | struct gendisk *rq_disk; |
124 | struct hd_struct *part; | 124 | struct hd_struct *part; |
125 | unsigned long start_time; | 125 | unsigned long start_time; |
126 | #ifdef CONFIG_BLK_CGROUP | 126 | #ifdef CONFIG_BLK_CGROUP |
127 | unsigned long long start_time_ns; | 127 | unsigned long long start_time_ns; |
128 | unsigned long long io_start_time_ns; /* when passed to hardware */ | 128 | unsigned long long io_start_time_ns; /* when passed to hardware */ |
129 | #endif | 129 | #endif |
130 | /* Number of scatter-gather DMA addr+len pairs after | 130 | /* Number of scatter-gather DMA addr+len pairs after |
131 | * physical address coalescing is performed. | 131 | * physical address coalescing is performed. |
132 | */ | 132 | */ |
133 | unsigned short nr_phys_segments; | 133 | unsigned short nr_phys_segments; |
134 | #if defined(CONFIG_BLK_DEV_INTEGRITY) | 134 | #if defined(CONFIG_BLK_DEV_INTEGRITY) |
135 | unsigned short nr_integrity_segments; | 135 | unsigned short nr_integrity_segments; |
136 | #endif | 136 | #endif |
137 | 137 | ||
138 | unsigned short ioprio; | 138 | unsigned short ioprio; |
139 | 139 | ||
140 | int ref_count; | 140 | int ref_count; |
141 | 141 | ||
142 | void *special; /* opaque pointer available for LLD use */ | 142 | void *special; /* opaque pointer available for LLD use */ |
143 | char *buffer; /* kaddr of the current segment if available */ | 143 | char *buffer; /* kaddr of the current segment if available */ |
144 | 144 | ||
145 | int tag; | 145 | int tag; |
146 | int errors; | 146 | int errors; |
147 | 147 | ||
148 | /* | 148 | /* |
149 | * when request is used as a packet command carrier | 149 | * when request is used as a packet command carrier |
150 | */ | 150 | */ |
151 | unsigned char __cmd[BLK_MAX_CDB]; | 151 | unsigned char __cmd[BLK_MAX_CDB]; |
152 | unsigned char *cmd; | 152 | unsigned char *cmd; |
153 | unsigned short cmd_len; | 153 | unsigned short cmd_len; |
154 | 154 | ||
155 | unsigned int extra_len; /* length of alignment and padding */ | 155 | unsigned int extra_len; /* length of alignment and padding */ |
156 | unsigned int sense_len; | 156 | unsigned int sense_len; |
157 | unsigned int resid_len; /* residual count */ | 157 | unsigned int resid_len; /* residual count */ |
158 | void *sense; | 158 | void *sense; |
159 | 159 | ||
160 | unsigned long deadline; | 160 | unsigned long deadline; |
161 | struct list_head timeout_list; | 161 | struct list_head timeout_list; |
162 | unsigned int timeout; | 162 | unsigned int timeout; |
163 | int retries; | 163 | int retries; |
164 | 164 | ||
165 | /* | 165 | /* |
166 | * completion callback. | 166 | * completion callback. |
167 | */ | 167 | */ |
168 | rq_end_io_fn *end_io; | 168 | rq_end_io_fn *end_io; |
169 | void *end_io_data; | 169 | void *end_io_data; |
170 | 170 | ||
171 | /* for bidi */ | 171 | /* for bidi */ |
172 | struct request *next_rq; | 172 | struct request *next_rq; |
173 | }; | 173 | }; |
174 | 174 | ||
175 | static inline unsigned short req_get_ioprio(struct request *req) | 175 | static inline unsigned short req_get_ioprio(struct request *req) |
176 | { | 176 | { |
177 | return req->ioprio; | 177 | return req->ioprio; |
178 | } | 178 | } |
179 | 179 | ||
180 | /* | 180 | /* |
181 | * State information carried for REQ_TYPE_PM_SUSPEND and REQ_TYPE_PM_RESUME | 181 | * State information carried for REQ_TYPE_PM_SUSPEND and REQ_TYPE_PM_RESUME |
182 | * requests. Some step values could eventually be made generic. | 182 | * requests. Some step values could eventually be made generic. |
183 | */ | 183 | */ |
184 | struct request_pm_state | 184 | struct request_pm_state |
185 | { | 185 | { |
186 | /* PM state machine step value, currently driver specific */ | 186 | /* PM state machine step value, currently driver specific */ |
187 | int pm_step; | 187 | int pm_step; |
188 | /* requested PM state value (S1, S2, S3, S4, ...) */ | 188 | /* requested PM state value (S1, S2, S3, S4, ...) */ |
189 | u32 pm_state; | 189 | u32 pm_state; |
190 | void* data; /* for driver use */ | 190 | void* data; /* for driver use */ |
191 | }; | 191 | }; |
192 | 192 | ||
193 | #include <linux/elevator.h> | 193 | #include <linux/elevator.h> |
194 | 194 | ||
195 | typedef void (request_fn_proc) (struct request_queue *q); | 195 | typedef void (request_fn_proc) (struct request_queue *q); |
196 | typedef int (make_request_fn) (struct request_queue *q, struct bio *bio); | 196 | typedef int (make_request_fn) (struct request_queue *q, struct bio *bio); |
197 | typedef int (prep_rq_fn) (struct request_queue *, struct request *); | 197 | typedef int (prep_rq_fn) (struct request_queue *, struct request *); |
198 | typedef void (unprep_rq_fn) (struct request_queue *, struct request *); | 198 | typedef void (unprep_rq_fn) (struct request_queue *, struct request *); |
199 | 199 | ||
200 | struct bio_vec; | 200 | struct bio_vec; |
201 | struct bvec_merge_data { | 201 | struct bvec_merge_data { |
202 | struct block_device *bi_bdev; | 202 | struct block_device *bi_bdev; |
203 | sector_t bi_sector; | 203 | sector_t bi_sector; |
204 | unsigned bi_size; | 204 | unsigned bi_size; |
205 | unsigned long bi_rw; | 205 | unsigned long bi_rw; |
206 | }; | 206 | }; |
207 | typedef int (merge_bvec_fn) (struct request_queue *, struct bvec_merge_data *, | 207 | typedef int (merge_bvec_fn) (struct request_queue *, struct bvec_merge_data *, |
208 | struct bio_vec *); | 208 | struct bio_vec *); |
209 | typedef void (softirq_done_fn)(struct request *); | 209 | typedef void (softirq_done_fn)(struct request *); |
210 | typedef int (dma_drain_needed_fn)(struct request *); | 210 | typedef int (dma_drain_needed_fn)(struct request *); |
211 | typedef int (lld_busy_fn) (struct request_queue *q); | 211 | typedef int (lld_busy_fn) (struct request_queue *q); |
212 | 212 | ||
213 | enum blk_eh_timer_return { | 213 | enum blk_eh_timer_return { |
214 | BLK_EH_NOT_HANDLED, | 214 | BLK_EH_NOT_HANDLED, |
215 | BLK_EH_HANDLED, | 215 | BLK_EH_HANDLED, |
216 | BLK_EH_RESET_TIMER, | 216 | BLK_EH_RESET_TIMER, |
217 | }; | 217 | }; |
218 | 218 | ||
219 | typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *); | 219 | typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *); |
220 | 220 | ||
221 | enum blk_queue_state { | 221 | enum blk_queue_state { |
222 | Queue_down, | 222 | Queue_down, |
223 | Queue_up, | 223 | Queue_up, |
224 | }; | 224 | }; |
225 | 225 | ||
226 | struct blk_queue_tag { | 226 | struct blk_queue_tag { |
227 | struct request **tag_index; /* map of busy tags */ | 227 | struct request **tag_index; /* map of busy tags */ |
228 | unsigned long *tag_map; /* bit map of free/busy tags */ | 228 | unsigned long *tag_map; /* bit map of free/busy tags */ |
229 | int busy; /* current depth */ | 229 | int busy; /* current depth */ |
230 | int max_depth; /* what we will send to device */ | 230 | int max_depth; /* what we will send to device */ |
231 | int real_max_depth; /* what the array can hold */ | 231 | int real_max_depth; /* what the array can hold */ |
232 | atomic_t refcnt; /* map can be shared */ | 232 | atomic_t refcnt; /* map can be shared */ |
233 | }; | 233 | }; |
234 | 234 | ||
235 | #define BLK_SCSI_MAX_CMDS (256) | 235 | #define BLK_SCSI_MAX_CMDS (256) |
236 | #define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8)) | 236 | #define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8)) |
237 | 237 | ||
238 | struct queue_limits { | 238 | struct queue_limits { |
239 | unsigned long bounce_pfn; | 239 | unsigned long bounce_pfn; |
240 | unsigned long seg_boundary_mask; | 240 | unsigned long seg_boundary_mask; |
241 | 241 | ||
242 | unsigned int max_hw_sectors; | 242 | unsigned int max_hw_sectors; |
243 | unsigned int max_sectors; | 243 | unsigned int max_sectors; |
244 | unsigned int max_segment_size; | 244 | unsigned int max_segment_size; |
245 | unsigned int physical_block_size; | 245 | unsigned int physical_block_size; |
246 | unsigned int alignment_offset; | 246 | unsigned int alignment_offset; |
247 | unsigned int io_min; | 247 | unsigned int io_min; |
248 | unsigned int io_opt; | 248 | unsigned int io_opt; |
249 | unsigned int max_discard_sectors; | 249 | unsigned int max_discard_sectors; |
250 | unsigned int discard_granularity; | 250 | unsigned int discard_granularity; |
251 | unsigned int discard_alignment; | 251 | unsigned int discard_alignment; |
252 | 252 | ||
253 | unsigned short logical_block_size; | 253 | unsigned short logical_block_size; |
254 | unsigned short max_segments; | 254 | unsigned short max_segments; |
255 | unsigned short max_integrity_segments; | 255 | unsigned short max_integrity_segments; |
256 | 256 | ||
257 | unsigned char misaligned; | 257 | unsigned char misaligned; |
258 | unsigned char discard_misaligned; | 258 | unsigned char discard_misaligned; |
259 | unsigned char cluster; | 259 | unsigned char cluster; |
260 | unsigned char discard_zeroes_data; | 260 | unsigned char discard_zeroes_data; |
261 | }; | 261 | }; |
262 | 262 | ||
263 | struct request_queue | 263 | struct request_queue |
264 | { | 264 | { |
265 | /* | 265 | /* |
266 | * Together with queue_head for cacheline sharing | 266 | * Together with queue_head for cacheline sharing |
267 | */ | 267 | */ |
268 | struct list_head queue_head; | 268 | struct list_head queue_head; |
269 | struct request *last_merge; | 269 | struct request *last_merge; |
270 | struct elevator_queue *elevator; | 270 | struct elevator_queue *elevator; |
271 | 271 | ||
272 | /* | 272 | /* |
273 | * the queue request freelist, one for reads and one for writes | 273 | * the queue request freelist, one for reads and one for writes |
274 | */ | 274 | */ |
275 | struct request_list rq; | 275 | struct request_list rq; |
276 | 276 | ||
277 | request_fn_proc *request_fn; | 277 | request_fn_proc *request_fn; |
278 | make_request_fn *make_request_fn; | 278 | make_request_fn *make_request_fn; |
279 | prep_rq_fn *prep_rq_fn; | 279 | prep_rq_fn *prep_rq_fn; |
280 | unprep_rq_fn *unprep_rq_fn; | 280 | unprep_rq_fn *unprep_rq_fn; |
281 | merge_bvec_fn *merge_bvec_fn; | 281 | merge_bvec_fn *merge_bvec_fn; |
282 | softirq_done_fn *softirq_done_fn; | 282 | softirq_done_fn *softirq_done_fn; |
283 | rq_timed_out_fn *rq_timed_out_fn; | 283 | rq_timed_out_fn *rq_timed_out_fn; |
284 | dma_drain_needed_fn *dma_drain_needed; | 284 | dma_drain_needed_fn *dma_drain_needed; |
285 | lld_busy_fn *lld_busy_fn; | 285 | lld_busy_fn *lld_busy_fn; |
286 | 286 | ||
287 | /* | 287 | /* |
288 | * Dispatch queue sorting | 288 | * Dispatch queue sorting |
289 | */ | 289 | */ |
290 | sector_t end_sector; | 290 | sector_t end_sector; |
291 | struct request *boundary_rq; | 291 | struct request *boundary_rq; |
292 | 292 | ||
293 | /* | 293 | /* |
294 | * Delayed queue handling | 294 | * Delayed queue handling |
295 | */ | 295 | */ |
296 | struct delayed_work delay_work; | 296 | struct delayed_work delay_work; |
297 | 297 | ||
298 | struct backing_dev_info backing_dev_info; | 298 | struct backing_dev_info backing_dev_info; |
299 | 299 | ||
300 | /* | 300 | /* |
301 | * The queue owner gets to use this for whatever they like. | 301 | * The queue owner gets to use this for whatever they like. |
302 | * ll_rw_blk doesn't touch it. | 302 | * ll_rw_blk doesn't touch it. |
303 | */ | 303 | */ |
304 | void *queuedata; | 304 | void *queuedata; |
305 | 305 | ||
306 | /* | 306 | /* |
307 | * queue needs bounce pages for pages above this limit | 307 | * queue needs bounce pages for pages above this limit |
308 | */ | 308 | */ |
309 | gfp_t bounce_gfp; | 309 | gfp_t bounce_gfp; |
310 | 310 | ||
311 | /* | 311 | /* |
312 | * various queue flags, see QUEUE_* below | 312 | * various queue flags, see QUEUE_* below |
313 | */ | 313 | */ |
314 | unsigned long queue_flags; | 314 | unsigned long queue_flags; |
315 | 315 | ||
316 | /* | 316 | /* |
317 | * protects queue structures from reentrancy. ->__queue_lock should | 317 | * protects queue structures from reentrancy. ->__queue_lock should |
318 | * _never_ be used directly, it is queue private. always use | 318 | * _never_ be used directly, it is queue private. always use |
319 | * ->queue_lock. | 319 | * ->queue_lock. |
320 | */ | 320 | */ |
321 | spinlock_t __queue_lock; | 321 | spinlock_t __queue_lock; |
322 | spinlock_t *queue_lock; | 322 | spinlock_t *queue_lock; |
323 | 323 | ||
324 | /* | 324 | /* |
325 | * queue kobject | 325 | * queue kobject |
326 | */ | 326 | */ |
327 | struct kobject kobj; | 327 | struct kobject kobj; |
328 | 328 | ||
329 | /* | 329 | /* |
330 | * queue settings | 330 | * queue settings |
331 | */ | 331 | */ |
332 | unsigned long nr_requests; /* Max # of requests */ | 332 | unsigned long nr_requests; /* Max # of requests */ |
333 | unsigned int nr_congestion_on; | 333 | unsigned int nr_congestion_on; |
334 | unsigned int nr_congestion_off; | 334 | unsigned int nr_congestion_off; |
335 | unsigned int nr_batching; | 335 | unsigned int nr_batching; |
336 | 336 | ||
337 | void *dma_drain_buffer; | 337 | void *dma_drain_buffer; |
338 | unsigned int dma_drain_size; | 338 | unsigned int dma_drain_size; |
339 | unsigned int dma_pad_mask; | 339 | unsigned int dma_pad_mask; |
340 | unsigned int dma_alignment; | 340 | unsigned int dma_alignment; |
341 | 341 | ||
342 | struct blk_queue_tag *queue_tags; | 342 | struct blk_queue_tag *queue_tags; |
343 | struct list_head tag_busy_list; | 343 | struct list_head tag_busy_list; |
344 | 344 | ||
345 | unsigned int nr_sorted; | 345 | unsigned int nr_sorted; |
346 | unsigned int in_flight[2]; | 346 | unsigned int in_flight[2]; |
347 | 347 | ||
348 | unsigned int rq_timeout; | 348 | unsigned int rq_timeout; |
349 | struct timer_list timeout; | 349 | struct timer_list timeout; |
350 | struct list_head timeout_list; | 350 | struct list_head timeout_list; |
351 | 351 | ||
352 | struct queue_limits limits; | 352 | struct queue_limits limits; |
353 | 353 | ||
354 | /* | 354 | /* |
355 | * sg stuff | 355 | * sg stuff |
356 | */ | 356 | */ |
357 | unsigned int sg_timeout; | 357 | unsigned int sg_timeout; |
358 | unsigned int sg_reserved_size; | 358 | unsigned int sg_reserved_size; |
359 | int node; | 359 | int node; |
360 | #ifdef CONFIG_BLK_DEV_IO_TRACE | 360 | #ifdef CONFIG_BLK_DEV_IO_TRACE |
361 | struct blk_trace *blk_trace; | 361 | struct blk_trace *blk_trace; |
362 | #endif | 362 | #endif |
363 | /* | 363 | /* |
364 | * for flush operations | 364 | * for flush operations |
365 | */ | 365 | */ |
366 | unsigned int flush_flags; | 366 | unsigned int flush_flags; |
367 | unsigned int flush_not_queueable:1; | 367 | unsigned int flush_not_queueable:1; |
368 | unsigned int flush_queue_delayed:1; | 368 | unsigned int flush_queue_delayed:1; |
369 | unsigned int flush_pending_idx:1; | 369 | unsigned int flush_pending_idx:1; |
370 | unsigned int flush_running_idx:1; | 370 | unsigned int flush_running_idx:1; |
371 | unsigned long flush_pending_since; | 371 | unsigned long flush_pending_since; |
372 | struct list_head flush_queue[2]; | 372 | struct list_head flush_queue[2]; |
373 | struct list_head flush_data_in_flight; | 373 | struct list_head flush_data_in_flight; |
374 | struct request flush_rq; | 374 | struct request flush_rq; |
375 | 375 | ||
376 | struct mutex sysfs_lock; | 376 | struct mutex sysfs_lock; |
377 | 377 | ||
378 | #if defined(CONFIG_BLK_DEV_BSG) | 378 | #if defined(CONFIG_BLK_DEV_BSG) |
379 | struct bsg_class_device bsg_dev; | 379 | struct bsg_class_device bsg_dev; |
380 | #endif | 380 | #endif |
381 | 381 | ||
382 | #ifdef CONFIG_BLK_DEV_THROTTLING | 382 | #ifdef CONFIG_BLK_DEV_THROTTLING |
383 | /* Throttle data */ | 383 | /* Throttle data */ |
384 | struct throtl_data *td; | 384 | struct throtl_data *td; |
385 | #endif | 385 | #endif |
386 | }; | 386 | }; |
387 | 387 | ||
388 | #define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */ | 388 | #define QUEUE_FLAG_QUEUED 1 /* uses generic tag queueing */ |
389 | #define QUEUE_FLAG_STOPPED 2 /* queue is stopped */ | 389 | #define QUEUE_FLAG_STOPPED 2 /* queue is stopped */ |
390 | #define QUEUE_FLAG_SYNCFULL 3 /* read queue has been filled */ | 390 | #define QUEUE_FLAG_SYNCFULL 3 /* read queue has been filled */ |
391 | #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */ | 391 | #define QUEUE_FLAG_ASYNCFULL 4 /* write queue has been filled */ |
392 | #define QUEUE_FLAG_DEAD 5 /* queue being torn down */ | 392 | #define QUEUE_FLAG_DEAD 5 /* queue being torn down */ |
393 | #define QUEUE_FLAG_ELVSWITCH 6 /* don't use elevator, just do FIFO */ | 393 | #define QUEUE_FLAG_ELVSWITCH 6 /* don't use elevator, just do FIFO */ |
394 | #define QUEUE_FLAG_BIDI 7 /* queue supports bidi requests */ | 394 | #define QUEUE_FLAG_BIDI 7 /* queue supports bidi requests */ |
395 | #define QUEUE_FLAG_NOMERGES 8 /* disable merge attempts */ | 395 | #define QUEUE_FLAG_NOMERGES 8 /* disable merge attempts */ |
396 | #define QUEUE_FLAG_SAME_COMP 9 /* force complete on same CPU */ | 396 | #define QUEUE_FLAG_SAME_COMP 9 /* force complete on same CPU */ |
397 | #define QUEUE_FLAG_FAIL_IO 10 /* fake timeout */ | 397 | #define QUEUE_FLAG_FAIL_IO 10 /* fake timeout */ |
398 | #define QUEUE_FLAG_STACKABLE 11 /* supports request stacking */ | 398 | #define QUEUE_FLAG_STACKABLE 11 /* supports request stacking */ |
399 | #define QUEUE_FLAG_NONROT 12 /* non-rotational device (SSD) */ | 399 | #define QUEUE_FLAG_NONROT 12 /* non-rotational device (SSD) */ |
400 | #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */ | 400 | #define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT /* paravirt device */ |
401 | #define QUEUE_FLAG_IO_STAT 13 /* do IO stats */ | 401 | #define QUEUE_FLAG_IO_STAT 13 /* do IO stats */ |
402 | #define QUEUE_FLAG_DISCARD 14 /* supports DISCARD */ | 402 | #define QUEUE_FLAG_DISCARD 14 /* supports DISCARD */ |
403 | #define QUEUE_FLAG_NOXMERGES 15 /* No extended merges */ | 403 | #define QUEUE_FLAG_NOXMERGES 15 /* No extended merges */ |
404 | #define QUEUE_FLAG_ADD_RANDOM 16 /* Contributes to random pool */ | 404 | #define QUEUE_FLAG_ADD_RANDOM 16 /* Contributes to random pool */ |
405 | #define QUEUE_FLAG_SECDISCARD 17 /* supports SECDISCARD */ | 405 | #define QUEUE_FLAG_SECDISCARD 17 /* supports SECDISCARD */ |
406 | 406 | ||
407 | #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ | 407 | #define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \ |
408 | (1 << QUEUE_FLAG_STACKABLE) | \ | 408 | (1 << QUEUE_FLAG_STACKABLE) | \ |
409 | (1 << QUEUE_FLAG_SAME_COMP) | \ | 409 | (1 << QUEUE_FLAG_SAME_COMP) | \ |
410 | (1 << QUEUE_FLAG_ADD_RANDOM)) | 410 | (1 << QUEUE_FLAG_ADD_RANDOM)) |
411 | 411 | ||
412 | static inline int queue_is_locked(struct request_queue *q) | 412 | static inline int queue_is_locked(struct request_queue *q) |
413 | { | 413 | { |
414 | #ifdef CONFIG_SMP | 414 | #ifdef CONFIG_SMP |
415 | spinlock_t *lock = q->queue_lock; | 415 | spinlock_t *lock = q->queue_lock; |
416 | return lock && spin_is_locked(lock); | 416 | return lock && spin_is_locked(lock); |
417 | #else | 417 | #else |
418 | return 1; | 418 | return 1; |
419 | #endif | 419 | #endif |
420 | } | 420 | } |
421 | 421 | ||
422 | static inline void queue_flag_set_unlocked(unsigned int flag, | 422 | static inline void queue_flag_set_unlocked(unsigned int flag, |
423 | struct request_queue *q) | 423 | struct request_queue *q) |
424 | { | 424 | { |
425 | __set_bit(flag, &q->queue_flags); | 425 | __set_bit(flag, &q->queue_flags); |
426 | } | 426 | } |
427 | 427 | ||
428 | static inline int queue_flag_test_and_clear(unsigned int flag, | 428 | static inline int queue_flag_test_and_clear(unsigned int flag, |
429 | struct request_queue *q) | 429 | struct request_queue *q) |
430 | { | 430 | { |
431 | WARN_ON_ONCE(!queue_is_locked(q)); | 431 | WARN_ON_ONCE(!queue_is_locked(q)); |
432 | 432 | ||
433 | if (test_bit(flag, &q->queue_flags)) { | 433 | if (test_bit(flag, &q->queue_flags)) { |
434 | __clear_bit(flag, &q->queue_flags); | 434 | __clear_bit(flag, &q->queue_flags); |
435 | return 1; | 435 | return 1; |
436 | } | 436 | } |
437 | 437 | ||
438 | return 0; | 438 | return 0; |
439 | } | 439 | } |
440 | 440 | ||
441 | static inline int queue_flag_test_and_set(unsigned int flag, | 441 | static inline int queue_flag_test_and_set(unsigned int flag, |
442 | struct request_queue *q) | 442 | struct request_queue *q) |
443 | { | 443 | { |
444 | WARN_ON_ONCE(!queue_is_locked(q)); | 444 | WARN_ON_ONCE(!queue_is_locked(q)); |
445 | 445 | ||
446 | if (!test_bit(flag, &q->queue_flags)) { | 446 | if (!test_bit(flag, &q->queue_flags)) { |
447 | __set_bit(flag, &q->queue_flags); | 447 | __set_bit(flag, &q->queue_flags); |
448 | return 0; | 448 | return 0; |
449 | } | 449 | } |
450 | 450 | ||
451 | return 1; | 451 | return 1; |
452 | } | 452 | } |
453 | 453 | ||
454 | static inline void queue_flag_set(unsigned int flag, struct request_queue *q) | 454 | static inline void queue_flag_set(unsigned int flag, struct request_queue *q) |
455 | { | 455 | { |
456 | WARN_ON_ONCE(!queue_is_locked(q)); | 456 | WARN_ON_ONCE(!queue_is_locked(q)); |
457 | __set_bit(flag, &q->queue_flags); | 457 | __set_bit(flag, &q->queue_flags); |
458 | } | 458 | } |
459 | 459 | ||
460 | static inline void queue_flag_clear_unlocked(unsigned int flag, | 460 | static inline void queue_flag_clear_unlocked(unsigned int flag, |
461 | struct request_queue *q) | 461 | struct request_queue *q) |
462 | { | 462 | { |
463 | __clear_bit(flag, &q->queue_flags); | 463 | __clear_bit(flag, &q->queue_flags); |
464 | } | 464 | } |
465 | 465 | ||
466 | static inline int queue_in_flight(struct request_queue *q) | 466 | static inline int queue_in_flight(struct request_queue *q) |
467 | { | 467 | { |
468 | return q->in_flight[0] + q->in_flight[1]; | 468 | return q->in_flight[0] + q->in_flight[1]; |
469 | } | 469 | } |
470 | 470 | ||
471 | static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) | 471 | static inline void queue_flag_clear(unsigned int flag, struct request_queue *q) |
472 | { | 472 | { |
473 | WARN_ON_ONCE(!queue_is_locked(q)); | 473 | WARN_ON_ONCE(!queue_is_locked(q)); |
474 | __clear_bit(flag, &q->queue_flags); | 474 | __clear_bit(flag, &q->queue_flags); |
475 | } | 475 | } |
476 | 476 | ||
477 | #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) | 477 | #define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags) |
478 | #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) | 478 | #define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags) |
479 | #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) | 479 | #define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags) |
480 | #define blk_queue_noxmerges(q) \ | 480 | #define blk_queue_noxmerges(q) \ |
481 | test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) | 481 | test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags) |
482 | #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags) | 482 | #define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags) |
483 | #define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags) | 483 | #define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags) |
484 | #define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags) | 484 | #define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags) |
485 | #define blk_queue_stackable(q) \ | 485 | #define blk_queue_stackable(q) \ |
486 | test_bit(QUEUE_FLAG_STACKABLE, &(q)->queue_flags) | 486 | test_bit(QUEUE_FLAG_STACKABLE, &(q)->queue_flags) |
487 | #define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags) | 487 | #define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags) |
488 | #define blk_queue_secdiscard(q) (blk_queue_discard(q) && \ | 488 | #define blk_queue_secdiscard(q) (blk_queue_discard(q) && \ |
489 | test_bit(QUEUE_FLAG_SECDISCARD, &(q)->queue_flags)) | 489 | test_bit(QUEUE_FLAG_SECDISCARD, &(q)->queue_flags)) |
490 | 490 | ||
491 | #define blk_noretry_request(rq) \ | 491 | #define blk_noretry_request(rq) \ |
492 | ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ | 492 | ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \ |
493 | REQ_FAILFAST_DRIVER)) | 493 | REQ_FAILFAST_DRIVER)) |
494 | 494 | ||
495 | #define blk_account_rq(rq) \ | 495 | #define blk_account_rq(rq) \ |
496 | (((rq)->cmd_flags & REQ_STARTED) && \ | 496 | (((rq)->cmd_flags & REQ_STARTED) && \ |
497 | ((rq)->cmd_type == REQ_TYPE_FS || \ | 497 | ((rq)->cmd_type == REQ_TYPE_FS || \ |
498 | ((rq)->cmd_flags & REQ_DISCARD))) | 498 | ((rq)->cmd_flags & REQ_DISCARD))) |
499 | 499 | ||
500 | #define blk_pm_request(rq) \ | 500 | #define blk_pm_request(rq) \ |
501 | ((rq)->cmd_type == REQ_TYPE_PM_SUSPEND || \ | 501 | ((rq)->cmd_type == REQ_TYPE_PM_SUSPEND || \ |
502 | (rq)->cmd_type == REQ_TYPE_PM_RESUME) | 502 | (rq)->cmd_type == REQ_TYPE_PM_RESUME) |
503 | 503 | ||
504 | #define blk_rq_cpu_valid(rq) ((rq)->cpu != -1) | 504 | #define blk_rq_cpu_valid(rq) ((rq)->cpu != -1) |
505 | #define blk_bidi_rq(rq) ((rq)->next_rq != NULL) | 505 | #define blk_bidi_rq(rq) ((rq)->next_rq != NULL) |
506 | /* rq->queuelist of dequeued request must be list_empty() */ | 506 | /* rq->queuelist of dequeued request must be list_empty() */ |
507 | #define blk_queued_rq(rq) (!list_empty(&(rq)->queuelist)) | 507 | #define blk_queued_rq(rq) (!list_empty(&(rq)->queuelist)) |
508 | 508 | ||
509 | #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist) | 509 | #define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist) |
510 | 510 | ||
511 | #define rq_data_dir(rq) ((rq)->cmd_flags & 1) | 511 | #define rq_data_dir(rq) ((rq)->cmd_flags & 1) |
512 | 512 | ||
513 | static inline unsigned int blk_queue_cluster(struct request_queue *q) | 513 | static inline unsigned int blk_queue_cluster(struct request_queue *q) |
514 | { | 514 | { |
515 | return q->limits.cluster; | 515 | return q->limits.cluster; |
516 | } | 516 | } |
517 | 517 | ||
518 | /* | 518 | /* |
519 | * We regard a request as sync, if either a read or a sync write | 519 | * We regard a request as sync, if either a read or a sync write |
520 | */ | 520 | */ |
521 | static inline bool rw_is_sync(unsigned int rw_flags) | 521 | static inline bool rw_is_sync(unsigned int rw_flags) |
522 | { | 522 | { |
523 | return !(rw_flags & REQ_WRITE) || (rw_flags & REQ_SYNC); | 523 | return !(rw_flags & REQ_WRITE) || (rw_flags & REQ_SYNC); |
524 | } | 524 | } |
525 | 525 | ||
526 | static inline bool rq_is_sync(struct request *rq) | 526 | static inline bool rq_is_sync(struct request *rq) |
527 | { | 527 | { |
528 | return rw_is_sync(rq->cmd_flags); | 528 | return rw_is_sync(rq->cmd_flags); |
529 | } | 529 | } |
530 | 530 | ||
531 | static inline int blk_queue_full(struct request_queue *q, int sync) | 531 | static inline int blk_queue_full(struct request_queue *q, int sync) |
532 | { | 532 | { |
533 | if (sync) | 533 | if (sync) |
534 | return test_bit(QUEUE_FLAG_SYNCFULL, &q->queue_flags); | 534 | return test_bit(QUEUE_FLAG_SYNCFULL, &q->queue_flags); |
535 | return test_bit(QUEUE_FLAG_ASYNCFULL, &q->queue_flags); | 535 | return test_bit(QUEUE_FLAG_ASYNCFULL, &q->queue_flags); |
536 | } | 536 | } |
537 | 537 | ||
538 | static inline void blk_set_queue_full(struct request_queue *q, int sync) | 538 | static inline void blk_set_queue_full(struct request_queue *q, int sync) |
539 | { | 539 | { |
540 | if (sync) | 540 | if (sync) |
541 | queue_flag_set(QUEUE_FLAG_SYNCFULL, q); | 541 | queue_flag_set(QUEUE_FLAG_SYNCFULL, q); |
542 | else | 542 | else |
543 | queue_flag_set(QUEUE_FLAG_ASYNCFULL, q); | 543 | queue_flag_set(QUEUE_FLAG_ASYNCFULL, q); |
544 | } | 544 | } |
545 | 545 | ||
546 | static inline void blk_clear_queue_full(struct request_queue *q, int sync) | 546 | static inline void blk_clear_queue_full(struct request_queue *q, int sync) |
547 | { | 547 | { |
548 | if (sync) | 548 | if (sync) |
549 | queue_flag_clear(QUEUE_FLAG_SYNCFULL, q); | 549 | queue_flag_clear(QUEUE_FLAG_SYNCFULL, q); |
550 | else | 550 | else |
551 | queue_flag_clear(QUEUE_FLAG_ASYNCFULL, q); | 551 | queue_flag_clear(QUEUE_FLAG_ASYNCFULL, q); |
552 | } | 552 | } |
553 | 553 | ||
554 | 554 | ||
555 | /* | 555 | /* |
556 | * mergeable request must not have _NOMERGE or _BARRIER bit set, nor may | 556 | * mergeable request must not have _NOMERGE or _BARRIER bit set, nor may |
557 | * it already be started by driver. | 557 | * it already be started by driver. |
558 | */ | 558 | */ |
559 | #define RQ_NOMERGE_FLAGS \ | 559 | #define RQ_NOMERGE_FLAGS \ |
560 | (REQ_NOMERGE | REQ_STARTED | REQ_SOFTBARRIER | REQ_FLUSH | REQ_FUA) | 560 | (REQ_NOMERGE | REQ_STARTED | REQ_SOFTBARRIER | REQ_FLUSH | REQ_FUA) |
561 | #define rq_mergeable(rq) \ | 561 | #define rq_mergeable(rq) \ |
562 | (!((rq)->cmd_flags & RQ_NOMERGE_FLAGS) && \ | 562 | (!((rq)->cmd_flags & RQ_NOMERGE_FLAGS) && \ |
563 | (((rq)->cmd_flags & REQ_DISCARD) || \ | 563 | (((rq)->cmd_flags & REQ_DISCARD) || \ |
564 | (rq)->cmd_type == REQ_TYPE_FS)) | 564 | (rq)->cmd_type == REQ_TYPE_FS)) |
565 | 565 | ||
566 | /* | 566 | /* |
567 | * q->prep_rq_fn return values | 567 | * q->prep_rq_fn return values |
568 | */ | 568 | */ |
569 | #define BLKPREP_OK 0 /* serve it */ | 569 | #define BLKPREP_OK 0 /* serve it */ |
570 | #define BLKPREP_KILL 1 /* fatal error, kill */ | 570 | #define BLKPREP_KILL 1 /* fatal error, kill */ |
571 | #define BLKPREP_DEFER 2 /* leave on queue */ | 571 | #define BLKPREP_DEFER 2 /* leave on queue */ |
572 | 572 | ||
573 | extern unsigned long blk_max_low_pfn, blk_max_pfn; | 573 | extern unsigned long blk_max_low_pfn, blk_max_pfn; |
574 | 574 | ||
575 | /* | 575 | /* |
576 | * standard bounce addresses: | 576 | * standard bounce addresses: |
577 | * | 577 | * |
578 | * BLK_BOUNCE_HIGH : bounce all highmem pages | 578 | * BLK_BOUNCE_HIGH : bounce all highmem pages |
579 | * BLK_BOUNCE_ANY : don't bounce anything | 579 | * BLK_BOUNCE_ANY : don't bounce anything |
580 | * BLK_BOUNCE_ISA : bounce pages above ISA DMA boundary | 580 | * BLK_BOUNCE_ISA : bounce pages above ISA DMA boundary |
581 | */ | 581 | */ |
582 | 582 | ||
583 | #if BITS_PER_LONG == 32 | 583 | #if BITS_PER_LONG == 32 |
584 | #define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT) | 584 | #define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT) |
585 | #else | 585 | #else |
586 | #define BLK_BOUNCE_HIGH -1ULL | 586 | #define BLK_BOUNCE_HIGH -1ULL |
587 | #endif | 587 | #endif |
588 | #define BLK_BOUNCE_ANY (-1ULL) | 588 | #define BLK_BOUNCE_ANY (-1ULL) |
589 | #define BLK_BOUNCE_ISA (DMA_BIT_MASK(24)) | 589 | #define BLK_BOUNCE_ISA (DMA_BIT_MASK(24)) |
590 | 590 | ||
591 | /* | 591 | /* |
592 | * default timeout for SG_IO if none specified | 592 | * default timeout for SG_IO if none specified |
593 | */ | 593 | */ |
594 | #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ) | 594 | #define BLK_DEFAULT_SG_TIMEOUT (60 * HZ) |
595 | #define BLK_MIN_SG_TIMEOUT (7 * HZ) | 595 | #define BLK_MIN_SG_TIMEOUT (7 * HZ) |
596 | 596 | ||
597 | #ifdef CONFIG_BOUNCE | 597 | #ifdef CONFIG_BOUNCE |
598 | extern int init_emergency_isa_pool(void); | 598 | extern int init_emergency_isa_pool(void); |
599 | extern void blk_queue_bounce(struct request_queue *q, struct bio **bio); | 599 | extern void blk_queue_bounce(struct request_queue *q, struct bio **bio); |
600 | #else | 600 | #else |
601 | static inline int init_emergency_isa_pool(void) | 601 | static inline int init_emergency_isa_pool(void) |
602 | { | 602 | { |
603 | return 0; | 603 | return 0; |
604 | } | 604 | } |
605 | static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio) | 605 | static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio) |
606 | { | 606 | { |
607 | } | 607 | } |
608 | #endif /* CONFIG_MMU */ | 608 | #endif /* CONFIG_MMU */ |
609 | 609 | ||
610 | struct rq_map_data { | 610 | struct rq_map_data { |
611 | struct page **pages; | 611 | struct page **pages; |
612 | int page_order; | 612 | int page_order; |
613 | int nr_entries; | 613 | int nr_entries; |
614 | unsigned long offset; | 614 | unsigned long offset; |
615 | int null_mapped; | 615 | int null_mapped; |
616 | int from_user; | 616 | int from_user; |
617 | }; | 617 | }; |
618 | 618 | ||
619 | struct req_iterator { | 619 | struct req_iterator { |
620 | int i; | 620 | int i; |
621 | struct bio *bio; | 621 | struct bio *bio; |
622 | }; | 622 | }; |
623 | 623 | ||
624 | /* This should not be used directly - use rq_for_each_segment */ | 624 | /* This should not be used directly - use rq_for_each_segment */ |
625 | #define for_each_bio(_bio) \ | 625 | #define for_each_bio(_bio) \ |
626 | for (; _bio; _bio = _bio->bi_next) | 626 | for (; _bio; _bio = _bio->bi_next) |
627 | #define __rq_for_each_bio(_bio, rq) \ | 627 | #define __rq_for_each_bio(_bio, rq) \ |
628 | if ((rq->bio)) \ | 628 | if ((rq->bio)) \ |
629 | for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next) | 629 | for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next) |
630 | 630 | ||
631 | #define rq_for_each_segment(bvl, _rq, _iter) \ | 631 | #define rq_for_each_segment(bvl, _rq, _iter) \ |
632 | __rq_for_each_bio(_iter.bio, _rq) \ | 632 | __rq_for_each_bio(_iter.bio, _rq) \ |
633 | bio_for_each_segment(bvl, _iter.bio, _iter.i) | 633 | bio_for_each_segment(bvl, _iter.bio, _iter.i) |
634 | 634 | ||
635 | #define rq_iter_last(rq, _iter) \ | 635 | #define rq_iter_last(rq, _iter) \ |
636 | (_iter.bio->bi_next == NULL && _iter.i == _iter.bio->bi_vcnt-1) | 636 | (_iter.bio->bi_next == NULL && _iter.i == _iter.bio->bi_vcnt-1) |
637 | 637 | ||
638 | #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE | 638 | #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE |
639 | # error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform" | 639 | # error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform" |
640 | #endif | 640 | #endif |
641 | #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE | 641 | #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE |
642 | extern void rq_flush_dcache_pages(struct request *rq); | 642 | extern void rq_flush_dcache_pages(struct request *rq); |
643 | #else | 643 | #else |
644 | static inline void rq_flush_dcache_pages(struct request *rq) | 644 | static inline void rq_flush_dcache_pages(struct request *rq) |
645 | { | 645 | { |
646 | } | 646 | } |
647 | #endif | 647 | #endif |
648 | 648 | ||
649 | extern int blk_register_queue(struct gendisk *disk); | 649 | extern int blk_register_queue(struct gendisk *disk); |
650 | extern void blk_unregister_queue(struct gendisk *disk); | 650 | extern void blk_unregister_queue(struct gendisk *disk); |
651 | extern void generic_make_request(struct bio *bio); | 651 | extern void generic_make_request(struct bio *bio); |
652 | extern void blk_rq_init(struct request_queue *q, struct request *rq); | 652 | extern void blk_rq_init(struct request_queue *q, struct request *rq); |
653 | extern void blk_put_request(struct request *); | 653 | extern void blk_put_request(struct request *); |
654 | extern void __blk_put_request(struct request_queue *, struct request *); | 654 | extern void __blk_put_request(struct request_queue *, struct request *); |
655 | extern struct request *blk_get_request(struct request_queue *, int, gfp_t); | 655 | extern struct request *blk_get_request(struct request_queue *, int, gfp_t); |
656 | extern struct request *blk_make_request(struct request_queue *, struct bio *, | 656 | extern struct request *blk_make_request(struct request_queue *, struct bio *, |
657 | gfp_t); | 657 | gfp_t); |
658 | extern void blk_insert_request(struct request_queue *, struct request *, int, void *); | 658 | extern void blk_insert_request(struct request_queue *, struct request *, int, void *); |
659 | extern void blk_requeue_request(struct request_queue *, struct request *); | 659 | extern void blk_requeue_request(struct request_queue *, struct request *); |
660 | extern void blk_add_request_payload(struct request *rq, struct page *page, | 660 | extern void blk_add_request_payload(struct request *rq, struct page *page, |
661 | unsigned int len); | 661 | unsigned int len); |
662 | extern int blk_rq_check_limits(struct request_queue *q, struct request *rq); | 662 | extern int blk_rq_check_limits(struct request_queue *q, struct request *rq); |
663 | extern int blk_lld_busy(struct request_queue *q); | 663 | extern int blk_lld_busy(struct request_queue *q); |
664 | extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src, | 664 | extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src, |
665 | struct bio_set *bs, gfp_t gfp_mask, | 665 | struct bio_set *bs, gfp_t gfp_mask, |
666 | int (*bio_ctr)(struct bio *, struct bio *, void *), | 666 | int (*bio_ctr)(struct bio *, struct bio *, void *), |
667 | void *data); | 667 | void *data); |
668 | extern void blk_rq_unprep_clone(struct request *rq); | 668 | extern void blk_rq_unprep_clone(struct request *rq); |
669 | extern int blk_insert_cloned_request(struct request_queue *q, | 669 | extern int blk_insert_cloned_request(struct request_queue *q, |
670 | struct request *rq); | 670 | struct request *rq); |
671 | extern void blk_delay_queue(struct request_queue *, unsigned long); | 671 | extern void blk_delay_queue(struct request_queue *, unsigned long); |
672 | extern void blk_recount_segments(struct request_queue *, struct bio *); | 672 | extern void blk_recount_segments(struct request_queue *, struct bio *); |
673 | extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t, | 673 | extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t, |
674 | unsigned int, void __user *); | 674 | unsigned int, void __user *); |
675 | extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t, | 675 | extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t, |
676 | struct scsi_ioctl_command __user *); | 676 | struct scsi_ioctl_command __user *); |
677 | 677 | ||
678 | /* | 678 | /* |
679 | * A queue has just exitted congestion. Note this in the global counter of | 679 | * A queue has just exitted congestion. Note this in the global counter of |
680 | * congested queues, and wake up anyone who was waiting for requests to be | 680 | * congested queues, and wake up anyone who was waiting for requests to be |
681 | * put back. | 681 | * put back. |
682 | */ | 682 | */ |
683 | static inline void blk_clear_queue_congested(struct request_queue *q, int sync) | 683 | static inline void blk_clear_queue_congested(struct request_queue *q, int sync) |
684 | { | 684 | { |
685 | clear_bdi_congested(&q->backing_dev_info, sync); | 685 | clear_bdi_congested(&q->backing_dev_info, sync); |
686 | } | 686 | } |
687 | 687 | ||
688 | /* | 688 | /* |
689 | * A queue has just entered congestion. Flag that in the queue's VM-visible | 689 | * A queue has just entered congestion. Flag that in the queue's VM-visible |
690 | * state flags and increment the global gounter of congested queues. | 690 | * state flags and increment the global gounter of congested queues. |
691 | */ | 691 | */ |
692 | static inline void blk_set_queue_congested(struct request_queue *q, int sync) | 692 | static inline void blk_set_queue_congested(struct request_queue *q, int sync) |
693 | { | 693 | { |
694 | set_bdi_congested(&q->backing_dev_info, sync); | 694 | set_bdi_congested(&q->backing_dev_info, sync); |
695 | } | 695 | } |
696 | 696 | ||
697 | extern void blk_start_queue(struct request_queue *q); | 697 | extern void blk_start_queue(struct request_queue *q); |
698 | extern void blk_stop_queue(struct request_queue *q); | 698 | extern void blk_stop_queue(struct request_queue *q); |
699 | extern void blk_sync_queue(struct request_queue *q); | 699 | extern void blk_sync_queue(struct request_queue *q); |
700 | extern void __blk_stop_queue(struct request_queue *q); | 700 | extern void __blk_stop_queue(struct request_queue *q); |
701 | extern void __blk_run_queue(struct request_queue *q); | 701 | extern void __blk_run_queue(struct request_queue *q); |
702 | extern void blk_run_queue(struct request_queue *); | 702 | extern void blk_run_queue(struct request_queue *); |
703 | extern void blk_run_queue_async(struct request_queue *q); | 703 | extern void blk_run_queue_async(struct request_queue *q); |
704 | extern int blk_rq_map_user(struct request_queue *, struct request *, | 704 | extern int blk_rq_map_user(struct request_queue *, struct request *, |
705 | struct rq_map_data *, void __user *, unsigned long, | 705 | struct rq_map_data *, void __user *, unsigned long, |
706 | gfp_t); | 706 | gfp_t); |
707 | extern int blk_rq_unmap_user(struct bio *); | 707 | extern int blk_rq_unmap_user(struct bio *); |
708 | extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t); | 708 | extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t); |
709 | extern int blk_rq_map_user_iov(struct request_queue *, struct request *, | 709 | extern int blk_rq_map_user_iov(struct request_queue *, struct request *, |
710 | struct rq_map_data *, struct sg_iovec *, int, | 710 | struct rq_map_data *, struct sg_iovec *, int, |
711 | unsigned int, gfp_t); | 711 | unsigned int, gfp_t); |
712 | extern int blk_execute_rq(struct request_queue *, struct gendisk *, | 712 | extern int blk_execute_rq(struct request_queue *, struct gendisk *, |
713 | struct request *, int); | 713 | struct request *, int); |
714 | extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *, | 714 | extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *, |
715 | struct request *, int, rq_end_io_fn *); | 715 | struct request *, int, rq_end_io_fn *); |
716 | 716 | ||
717 | static inline struct request_queue *bdev_get_queue(struct block_device *bdev) | 717 | static inline struct request_queue *bdev_get_queue(struct block_device *bdev) |
718 | { | 718 | { |
719 | return bdev->bd_disk->queue; | 719 | return bdev->bd_disk->queue; |
720 | } | 720 | } |
721 | 721 | ||
722 | /* | 722 | /* |
723 | * blk_rq_pos() : the current sector | 723 | * blk_rq_pos() : the current sector |
724 | * blk_rq_bytes() : bytes left in the entire request | 724 | * blk_rq_bytes() : bytes left in the entire request |
725 | * blk_rq_cur_bytes() : bytes left in the current segment | 725 | * blk_rq_cur_bytes() : bytes left in the current segment |
726 | * blk_rq_err_bytes() : bytes left till the next error boundary | 726 | * blk_rq_err_bytes() : bytes left till the next error boundary |
727 | * blk_rq_sectors() : sectors left in the entire request | 727 | * blk_rq_sectors() : sectors left in the entire request |
728 | * blk_rq_cur_sectors() : sectors left in the current segment | 728 | * blk_rq_cur_sectors() : sectors left in the current segment |
729 | */ | 729 | */ |
730 | static inline sector_t blk_rq_pos(const struct request *rq) | 730 | static inline sector_t blk_rq_pos(const struct request *rq) |
731 | { | 731 | { |
732 | return rq->__sector; | 732 | return rq->__sector; |
733 | } | 733 | } |
734 | 734 | ||
735 | static inline unsigned int blk_rq_bytes(const struct request *rq) | 735 | static inline unsigned int blk_rq_bytes(const struct request *rq) |
736 | { | 736 | { |
737 | return rq->__data_len; | 737 | return rq->__data_len; |
738 | } | 738 | } |
739 | 739 | ||
740 | static inline int blk_rq_cur_bytes(const struct request *rq) | 740 | static inline int blk_rq_cur_bytes(const struct request *rq) |
741 | { | 741 | { |
742 | return rq->bio ? bio_cur_bytes(rq->bio) : 0; | 742 | return rq->bio ? bio_cur_bytes(rq->bio) : 0; |
743 | } | 743 | } |
744 | 744 | ||
745 | extern unsigned int blk_rq_err_bytes(const struct request *rq); | 745 | extern unsigned int blk_rq_err_bytes(const struct request *rq); |
746 | 746 | ||
747 | static inline unsigned int blk_rq_sectors(const struct request *rq) | 747 | static inline unsigned int blk_rq_sectors(const struct request *rq) |
748 | { | 748 | { |
749 | return blk_rq_bytes(rq) >> 9; | 749 | return blk_rq_bytes(rq) >> 9; |
750 | } | 750 | } |
751 | 751 | ||
752 | static inline unsigned int blk_rq_cur_sectors(const struct request *rq) | 752 | static inline unsigned int blk_rq_cur_sectors(const struct request *rq) |
753 | { | 753 | { |
754 | return blk_rq_cur_bytes(rq) >> 9; | 754 | return blk_rq_cur_bytes(rq) >> 9; |
755 | } | 755 | } |
756 | 756 | ||
757 | /* | 757 | /* |
758 | * Request issue related functions. | 758 | * Request issue related functions. |
759 | */ | 759 | */ |
760 | extern struct request *blk_peek_request(struct request_queue *q); | 760 | extern struct request *blk_peek_request(struct request_queue *q); |
761 | extern void blk_start_request(struct request *rq); | 761 | extern void blk_start_request(struct request *rq); |
762 | extern struct request *blk_fetch_request(struct request_queue *q); | 762 | extern struct request *blk_fetch_request(struct request_queue *q); |
763 | 763 | ||
764 | /* | 764 | /* |
765 | * Request completion related functions. | 765 | * Request completion related functions. |
766 | * | 766 | * |
767 | * blk_update_request() completes given number of bytes and updates | 767 | * blk_update_request() completes given number of bytes and updates |
768 | * the request without completing it. | 768 | * the request without completing it. |
769 | * | 769 | * |
770 | * blk_end_request() and friends. __blk_end_request() must be called | 770 | * blk_end_request() and friends. __blk_end_request() must be called |
771 | * with the request queue spinlock acquired. | 771 | * with the request queue spinlock acquired. |
772 | * | 772 | * |
773 | * Several drivers define their own end_request and call | 773 | * Several drivers define their own end_request and call |
774 | * blk_end_request() for parts of the original function. | 774 | * blk_end_request() for parts of the original function. |
775 | * This prevents code duplication in drivers. | 775 | * This prevents code duplication in drivers. |
776 | */ | 776 | */ |
777 | extern bool blk_update_request(struct request *rq, int error, | 777 | extern bool blk_update_request(struct request *rq, int error, |
778 | unsigned int nr_bytes); | 778 | unsigned int nr_bytes); |
779 | extern bool blk_end_request(struct request *rq, int error, | 779 | extern bool blk_end_request(struct request *rq, int error, |
780 | unsigned int nr_bytes); | 780 | unsigned int nr_bytes); |
781 | extern void blk_end_request_all(struct request *rq, int error); | 781 | extern void blk_end_request_all(struct request *rq, int error); |
782 | extern bool blk_end_request_cur(struct request *rq, int error); | 782 | extern bool blk_end_request_cur(struct request *rq, int error); |
783 | extern bool blk_end_request_err(struct request *rq, int error); | 783 | extern bool blk_end_request_err(struct request *rq, int error); |
784 | extern bool __blk_end_request(struct request *rq, int error, | 784 | extern bool __blk_end_request(struct request *rq, int error, |
785 | unsigned int nr_bytes); | 785 | unsigned int nr_bytes); |
786 | extern void __blk_end_request_all(struct request *rq, int error); | 786 | extern void __blk_end_request_all(struct request *rq, int error); |
787 | extern bool __blk_end_request_cur(struct request *rq, int error); | 787 | extern bool __blk_end_request_cur(struct request *rq, int error); |
788 | extern bool __blk_end_request_err(struct request *rq, int error); | 788 | extern bool __blk_end_request_err(struct request *rq, int error); |
789 | 789 | ||
790 | extern void blk_complete_request(struct request *); | 790 | extern void blk_complete_request(struct request *); |
791 | extern void __blk_complete_request(struct request *); | 791 | extern void __blk_complete_request(struct request *); |
792 | extern void blk_abort_request(struct request *); | 792 | extern void blk_abort_request(struct request *); |
793 | extern void blk_abort_queue(struct request_queue *); | 793 | extern void blk_abort_queue(struct request_queue *); |
794 | extern void blk_unprep_request(struct request *); | 794 | extern void blk_unprep_request(struct request *); |
795 | 795 | ||
796 | /* | 796 | /* |
797 | * Access functions for manipulating queue properties | 797 | * Access functions for manipulating queue properties |
798 | */ | 798 | */ |
799 | extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn, | 799 | extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn, |
800 | spinlock_t *lock, int node_id); | 800 | spinlock_t *lock, int node_id); |
801 | extern struct request_queue *blk_init_allocated_queue_node(struct request_queue *, | 801 | extern struct request_queue *blk_init_allocated_queue_node(struct request_queue *, |
802 | request_fn_proc *, | 802 | request_fn_proc *, |
803 | spinlock_t *, int node_id); | 803 | spinlock_t *, int node_id); |
804 | extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *); | 804 | extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *); |
805 | extern struct request_queue *blk_init_allocated_queue(struct request_queue *, | 805 | extern struct request_queue *blk_init_allocated_queue(struct request_queue *, |
806 | request_fn_proc *, spinlock_t *); | 806 | request_fn_proc *, spinlock_t *); |
807 | extern void blk_cleanup_queue(struct request_queue *); | 807 | extern void blk_cleanup_queue(struct request_queue *); |
808 | extern void blk_queue_make_request(struct request_queue *, make_request_fn *); | 808 | extern void blk_queue_make_request(struct request_queue *, make_request_fn *); |
809 | extern void blk_queue_bounce_limit(struct request_queue *, u64); | 809 | extern void blk_queue_bounce_limit(struct request_queue *, u64); |
810 | extern void blk_limits_max_hw_sectors(struct queue_limits *, unsigned int); | 810 | extern void blk_limits_max_hw_sectors(struct queue_limits *, unsigned int); |
811 | extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int); | 811 | extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int); |
812 | extern void blk_queue_max_segments(struct request_queue *, unsigned short); | 812 | extern void blk_queue_max_segments(struct request_queue *, unsigned short); |
813 | extern void blk_queue_max_segment_size(struct request_queue *, unsigned int); | 813 | extern void blk_queue_max_segment_size(struct request_queue *, unsigned int); |
814 | extern void blk_queue_max_discard_sectors(struct request_queue *q, | 814 | extern void blk_queue_max_discard_sectors(struct request_queue *q, |
815 | unsigned int max_discard_sectors); | 815 | unsigned int max_discard_sectors); |
816 | extern void blk_queue_logical_block_size(struct request_queue *, unsigned short); | 816 | extern void blk_queue_logical_block_size(struct request_queue *, unsigned short); |
817 | extern void blk_queue_physical_block_size(struct request_queue *, unsigned int); | 817 | extern void blk_queue_physical_block_size(struct request_queue *, unsigned int); |
818 | extern void blk_queue_alignment_offset(struct request_queue *q, | 818 | extern void blk_queue_alignment_offset(struct request_queue *q, |
819 | unsigned int alignment); | 819 | unsigned int alignment); |
820 | extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min); | 820 | extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min); |
821 | extern void blk_queue_io_min(struct request_queue *q, unsigned int min); | 821 | extern void blk_queue_io_min(struct request_queue *q, unsigned int min); |
822 | extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt); | 822 | extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt); |
823 | extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt); | 823 | extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt); |
824 | extern void blk_set_default_limits(struct queue_limits *lim); | 824 | extern void blk_set_default_limits(struct queue_limits *lim); |
825 | extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, | 825 | extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, |
826 | sector_t offset); | 826 | sector_t offset); |
827 | extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev, | 827 | extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev, |
828 | sector_t offset); | 828 | sector_t offset); |
829 | extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev, | 829 | extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev, |
830 | sector_t offset); | 830 | sector_t offset); |
831 | extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b); | 831 | extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b); |
832 | extern void blk_queue_dma_pad(struct request_queue *, unsigned int); | 832 | extern void blk_queue_dma_pad(struct request_queue *, unsigned int); |
833 | extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int); | 833 | extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int); |
834 | extern int blk_queue_dma_drain(struct request_queue *q, | 834 | extern int blk_queue_dma_drain(struct request_queue *q, |
835 | dma_drain_needed_fn *dma_drain_needed, | 835 | dma_drain_needed_fn *dma_drain_needed, |
836 | void *buf, unsigned int size); | 836 | void *buf, unsigned int size); |
837 | extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn); | 837 | extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn); |
838 | extern void blk_queue_segment_boundary(struct request_queue *, unsigned long); | 838 | extern void blk_queue_segment_boundary(struct request_queue *, unsigned long); |
839 | extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn); | 839 | extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn); |
840 | extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn); | 840 | extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn); |
841 | extern void blk_queue_merge_bvec(struct request_queue *, merge_bvec_fn *); | 841 | extern void blk_queue_merge_bvec(struct request_queue *, merge_bvec_fn *); |
842 | extern void blk_queue_dma_alignment(struct request_queue *, int); | 842 | extern void blk_queue_dma_alignment(struct request_queue *, int); |
843 | extern void blk_queue_update_dma_alignment(struct request_queue *, int); | 843 | extern void blk_queue_update_dma_alignment(struct request_queue *, int); |
844 | extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *); | 844 | extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *); |
845 | extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *); | 845 | extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *); |
846 | extern void blk_queue_rq_timeout(struct request_queue *, unsigned int); | 846 | extern void blk_queue_rq_timeout(struct request_queue *, unsigned int); |
847 | extern void blk_queue_flush(struct request_queue *q, unsigned int flush); | 847 | extern void blk_queue_flush(struct request_queue *q, unsigned int flush); |
848 | extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable); | 848 | extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable); |
849 | extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev); | 849 | extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev); |
850 | 850 | ||
851 | extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *); | 851 | extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *); |
852 | extern void blk_dump_rq_flags(struct request *, char *); | 852 | extern void blk_dump_rq_flags(struct request *, char *); |
853 | extern long nr_blockdev_pages(void); | 853 | extern long nr_blockdev_pages(void); |
854 | 854 | ||
855 | int blk_get_queue(struct request_queue *); | 855 | int blk_get_queue(struct request_queue *); |
856 | struct request_queue *blk_alloc_queue(gfp_t); | 856 | struct request_queue *blk_alloc_queue(gfp_t); |
857 | struct request_queue *blk_alloc_queue_node(gfp_t, int); | 857 | struct request_queue *blk_alloc_queue_node(gfp_t, int); |
858 | extern void blk_put_queue(struct request_queue *); | 858 | extern void blk_put_queue(struct request_queue *); |
859 | 859 | ||
860 | struct blk_plug { | 860 | struct blk_plug { |
861 | unsigned long magic; | 861 | unsigned long magic; |
862 | struct list_head list; | 862 | struct list_head list; |
863 | struct list_head cb_list; | 863 | struct list_head cb_list; |
864 | unsigned int should_sort; | 864 | unsigned int should_sort; |
865 | unsigned int count; | ||
865 | }; | 866 | }; |
867 | #define BLK_MAX_REQUEST_COUNT 16 | ||
868 | |||
866 | struct blk_plug_cb { | 869 | struct blk_plug_cb { |
867 | struct list_head list; | 870 | struct list_head list; |
868 | void (*callback)(struct blk_plug_cb *); | 871 | void (*callback)(struct blk_plug_cb *); |
869 | }; | 872 | }; |
870 | 873 | ||
871 | extern void blk_start_plug(struct blk_plug *); | 874 | extern void blk_start_plug(struct blk_plug *); |
872 | extern void blk_finish_plug(struct blk_plug *); | 875 | extern void blk_finish_plug(struct blk_plug *); |
873 | extern void blk_flush_plug_list(struct blk_plug *, bool); | 876 | extern void blk_flush_plug_list(struct blk_plug *, bool); |
874 | 877 | ||
875 | static inline void blk_flush_plug(struct task_struct *tsk) | 878 | static inline void blk_flush_plug(struct task_struct *tsk) |
876 | { | 879 | { |
877 | struct blk_plug *plug = tsk->plug; | 880 | struct blk_plug *plug = tsk->plug; |
878 | 881 | ||
879 | if (plug) | 882 | if (plug) |
880 | blk_flush_plug_list(plug, false); | 883 | blk_flush_plug_list(plug, false); |
881 | } | 884 | } |
882 | 885 | ||
883 | static inline void blk_schedule_flush_plug(struct task_struct *tsk) | 886 | static inline void blk_schedule_flush_plug(struct task_struct *tsk) |
884 | { | 887 | { |
885 | struct blk_plug *plug = tsk->plug; | 888 | struct blk_plug *plug = tsk->plug; |
886 | 889 | ||
887 | if (plug) | 890 | if (plug) |
888 | blk_flush_plug_list(plug, true); | 891 | blk_flush_plug_list(plug, true); |
889 | } | 892 | } |
890 | 893 | ||
891 | static inline bool blk_needs_flush_plug(struct task_struct *tsk) | 894 | static inline bool blk_needs_flush_plug(struct task_struct *tsk) |
892 | { | 895 | { |
893 | struct blk_plug *plug = tsk->plug; | 896 | struct blk_plug *plug = tsk->plug; |
894 | 897 | ||
895 | return plug && (!list_empty(&plug->list) || !list_empty(&plug->cb_list)); | 898 | return plug && (!list_empty(&plug->list) || !list_empty(&plug->cb_list)); |
896 | } | 899 | } |
897 | 900 | ||
898 | /* | 901 | /* |
899 | * tag stuff | 902 | * tag stuff |
900 | */ | 903 | */ |
901 | #define blk_rq_tagged(rq) ((rq)->cmd_flags & REQ_QUEUED) | 904 | #define blk_rq_tagged(rq) ((rq)->cmd_flags & REQ_QUEUED) |
902 | extern int blk_queue_start_tag(struct request_queue *, struct request *); | 905 | extern int blk_queue_start_tag(struct request_queue *, struct request *); |
903 | extern struct request *blk_queue_find_tag(struct request_queue *, int); | 906 | extern struct request *blk_queue_find_tag(struct request_queue *, int); |
904 | extern void blk_queue_end_tag(struct request_queue *, struct request *); | 907 | extern void blk_queue_end_tag(struct request_queue *, struct request *); |
905 | extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *); | 908 | extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *); |
906 | extern void blk_queue_free_tags(struct request_queue *); | 909 | extern void blk_queue_free_tags(struct request_queue *); |
907 | extern int blk_queue_resize_tags(struct request_queue *, int); | 910 | extern int blk_queue_resize_tags(struct request_queue *, int); |
908 | extern void blk_queue_invalidate_tags(struct request_queue *); | 911 | extern void blk_queue_invalidate_tags(struct request_queue *); |
909 | extern struct blk_queue_tag *blk_init_tags(int); | 912 | extern struct blk_queue_tag *blk_init_tags(int); |
910 | extern void blk_free_tags(struct blk_queue_tag *); | 913 | extern void blk_free_tags(struct blk_queue_tag *); |
911 | 914 | ||
912 | static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt, | 915 | static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt, |
913 | int tag) | 916 | int tag) |
914 | { | 917 | { |
915 | if (unlikely(bqt == NULL || tag >= bqt->real_max_depth)) | 918 | if (unlikely(bqt == NULL || tag >= bqt->real_max_depth)) |
916 | return NULL; | 919 | return NULL; |
917 | return bqt->tag_index[tag]; | 920 | return bqt->tag_index[tag]; |
918 | } | 921 | } |
919 | 922 | ||
920 | #define BLKDEV_DISCARD_SECURE 0x01 /* secure discard */ | 923 | #define BLKDEV_DISCARD_SECURE 0x01 /* secure discard */ |
921 | 924 | ||
922 | extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *); | 925 | extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *); |
923 | extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector, | 926 | extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector, |
924 | sector_t nr_sects, gfp_t gfp_mask, unsigned long flags); | 927 | sector_t nr_sects, gfp_t gfp_mask, unsigned long flags); |
925 | extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, | 928 | extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, |
926 | sector_t nr_sects, gfp_t gfp_mask); | 929 | sector_t nr_sects, gfp_t gfp_mask); |
927 | static inline int sb_issue_discard(struct super_block *sb, sector_t block, | 930 | static inline int sb_issue_discard(struct super_block *sb, sector_t block, |
928 | sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags) | 931 | sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags) |
929 | { | 932 | { |
930 | return blkdev_issue_discard(sb->s_bdev, block << (sb->s_blocksize_bits - 9), | 933 | return blkdev_issue_discard(sb->s_bdev, block << (sb->s_blocksize_bits - 9), |
931 | nr_blocks << (sb->s_blocksize_bits - 9), | 934 | nr_blocks << (sb->s_blocksize_bits - 9), |
932 | gfp_mask, flags); | 935 | gfp_mask, flags); |
933 | } | 936 | } |
934 | static inline int sb_issue_zeroout(struct super_block *sb, sector_t block, | 937 | static inline int sb_issue_zeroout(struct super_block *sb, sector_t block, |
935 | sector_t nr_blocks, gfp_t gfp_mask) | 938 | sector_t nr_blocks, gfp_t gfp_mask) |
936 | { | 939 | { |
937 | return blkdev_issue_zeroout(sb->s_bdev, | 940 | return blkdev_issue_zeroout(sb->s_bdev, |
938 | block << (sb->s_blocksize_bits - 9), | 941 | block << (sb->s_blocksize_bits - 9), |
939 | nr_blocks << (sb->s_blocksize_bits - 9), | 942 | nr_blocks << (sb->s_blocksize_bits - 9), |
940 | gfp_mask); | 943 | gfp_mask); |
941 | } | 944 | } |
942 | 945 | ||
943 | extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm); | 946 | extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm); |
944 | 947 | ||
945 | enum blk_default_limits { | 948 | enum blk_default_limits { |
946 | BLK_MAX_SEGMENTS = 128, | 949 | BLK_MAX_SEGMENTS = 128, |
947 | BLK_SAFE_MAX_SECTORS = 255, | 950 | BLK_SAFE_MAX_SECTORS = 255, |
948 | BLK_DEF_MAX_SECTORS = 1024, | 951 | BLK_DEF_MAX_SECTORS = 1024, |
949 | BLK_MAX_SEGMENT_SIZE = 65536, | 952 | BLK_MAX_SEGMENT_SIZE = 65536, |
950 | BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, | 953 | BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL, |
951 | }; | 954 | }; |
952 | 955 | ||
953 | #define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist) | 956 | #define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist) |
954 | 957 | ||
955 | static inline unsigned long queue_bounce_pfn(struct request_queue *q) | 958 | static inline unsigned long queue_bounce_pfn(struct request_queue *q) |
956 | { | 959 | { |
957 | return q->limits.bounce_pfn; | 960 | return q->limits.bounce_pfn; |
958 | } | 961 | } |
959 | 962 | ||
960 | static inline unsigned long queue_segment_boundary(struct request_queue *q) | 963 | static inline unsigned long queue_segment_boundary(struct request_queue *q) |
961 | { | 964 | { |
962 | return q->limits.seg_boundary_mask; | 965 | return q->limits.seg_boundary_mask; |
963 | } | 966 | } |
964 | 967 | ||
965 | static inline unsigned int queue_max_sectors(struct request_queue *q) | 968 | static inline unsigned int queue_max_sectors(struct request_queue *q) |
966 | { | 969 | { |
967 | return q->limits.max_sectors; | 970 | return q->limits.max_sectors; |
968 | } | 971 | } |
969 | 972 | ||
970 | static inline unsigned int queue_max_hw_sectors(struct request_queue *q) | 973 | static inline unsigned int queue_max_hw_sectors(struct request_queue *q) |
971 | { | 974 | { |
972 | return q->limits.max_hw_sectors; | 975 | return q->limits.max_hw_sectors; |
973 | } | 976 | } |
974 | 977 | ||
975 | static inline unsigned short queue_max_segments(struct request_queue *q) | 978 | static inline unsigned short queue_max_segments(struct request_queue *q) |
976 | { | 979 | { |
977 | return q->limits.max_segments; | 980 | return q->limits.max_segments; |
978 | } | 981 | } |
979 | 982 | ||
980 | static inline unsigned int queue_max_segment_size(struct request_queue *q) | 983 | static inline unsigned int queue_max_segment_size(struct request_queue *q) |
981 | { | 984 | { |
982 | return q->limits.max_segment_size; | 985 | return q->limits.max_segment_size; |
983 | } | 986 | } |
984 | 987 | ||
985 | static inline unsigned short queue_logical_block_size(struct request_queue *q) | 988 | static inline unsigned short queue_logical_block_size(struct request_queue *q) |
986 | { | 989 | { |
987 | int retval = 512; | 990 | int retval = 512; |
988 | 991 | ||
989 | if (q && q->limits.logical_block_size) | 992 | if (q && q->limits.logical_block_size) |
990 | retval = q->limits.logical_block_size; | 993 | retval = q->limits.logical_block_size; |
991 | 994 | ||
992 | return retval; | 995 | return retval; |
993 | } | 996 | } |
994 | 997 | ||
995 | static inline unsigned short bdev_logical_block_size(struct block_device *bdev) | 998 | static inline unsigned short bdev_logical_block_size(struct block_device *bdev) |
996 | { | 999 | { |
997 | return queue_logical_block_size(bdev_get_queue(bdev)); | 1000 | return queue_logical_block_size(bdev_get_queue(bdev)); |
998 | } | 1001 | } |
999 | 1002 | ||
1000 | static inline unsigned int queue_physical_block_size(struct request_queue *q) | 1003 | static inline unsigned int queue_physical_block_size(struct request_queue *q) |
1001 | { | 1004 | { |
1002 | return q->limits.physical_block_size; | 1005 | return q->limits.physical_block_size; |
1003 | } | 1006 | } |
1004 | 1007 | ||
1005 | static inline unsigned int bdev_physical_block_size(struct block_device *bdev) | 1008 | static inline unsigned int bdev_physical_block_size(struct block_device *bdev) |
1006 | { | 1009 | { |
1007 | return queue_physical_block_size(bdev_get_queue(bdev)); | 1010 | return queue_physical_block_size(bdev_get_queue(bdev)); |
1008 | } | 1011 | } |
1009 | 1012 | ||
1010 | static inline unsigned int queue_io_min(struct request_queue *q) | 1013 | static inline unsigned int queue_io_min(struct request_queue *q) |
1011 | { | 1014 | { |
1012 | return q->limits.io_min; | 1015 | return q->limits.io_min; |
1013 | } | 1016 | } |
1014 | 1017 | ||
1015 | static inline int bdev_io_min(struct block_device *bdev) | 1018 | static inline int bdev_io_min(struct block_device *bdev) |
1016 | { | 1019 | { |
1017 | return queue_io_min(bdev_get_queue(bdev)); | 1020 | return queue_io_min(bdev_get_queue(bdev)); |
1018 | } | 1021 | } |
1019 | 1022 | ||
1020 | static inline unsigned int queue_io_opt(struct request_queue *q) | 1023 | static inline unsigned int queue_io_opt(struct request_queue *q) |
1021 | { | 1024 | { |
1022 | return q->limits.io_opt; | 1025 | return q->limits.io_opt; |
1023 | } | 1026 | } |
1024 | 1027 | ||
1025 | static inline int bdev_io_opt(struct block_device *bdev) | 1028 | static inline int bdev_io_opt(struct block_device *bdev) |
1026 | { | 1029 | { |
1027 | return queue_io_opt(bdev_get_queue(bdev)); | 1030 | return queue_io_opt(bdev_get_queue(bdev)); |
1028 | } | 1031 | } |
1029 | 1032 | ||
1030 | static inline int queue_alignment_offset(struct request_queue *q) | 1033 | static inline int queue_alignment_offset(struct request_queue *q) |
1031 | { | 1034 | { |
1032 | if (q->limits.misaligned) | 1035 | if (q->limits.misaligned) |
1033 | return -1; | 1036 | return -1; |
1034 | 1037 | ||
1035 | return q->limits.alignment_offset; | 1038 | return q->limits.alignment_offset; |
1036 | } | 1039 | } |
1037 | 1040 | ||
1038 | static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector) | 1041 | static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector) |
1039 | { | 1042 | { |
1040 | unsigned int granularity = max(lim->physical_block_size, lim->io_min); | 1043 | unsigned int granularity = max(lim->physical_block_size, lim->io_min); |
1041 | unsigned int alignment = (sector << 9) & (granularity - 1); | 1044 | unsigned int alignment = (sector << 9) & (granularity - 1); |
1042 | 1045 | ||
1043 | return (granularity + lim->alignment_offset - alignment) | 1046 | return (granularity + lim->alignment_offset - alignment) |
1044 | & (granularity - 1); | 1047 | & (granularity - 1); |
1045 | } | 1048 | } |
1046 | 1049 | ||
1047 | static inline int bdev_alignment_offset(struct block_device *bdev) | 1050 | static inline int bdev_alignment_offset(struct block_device *bdev) |
1048 | { | 1051 | { |
1049 | struct request_queue *q = bdev_get_queue(bdev); | 1052 | struct request_queue *q = bdev_get_queue(bdev); |
1050 | 1053 | ||
1051 | if (q->limits.misaligned) | 1054 | if (q->limits.misaligned) |
1052 | return -1; | 1055 | return -1; |
1053 | 1056 | ||
1054 | if (bdev != bdev->bd_contains) | 1057 | if (bdev != bdev->bd_contains) |
1055 | return bdev->bd_part->alignment_offset; | 1058 | return bdev->bd_part->alignment_offset; |
1056 | 1059 | ||
1057 | return q->limits.alignment_offset; | 1060 | return q->limits.alignment_offset; |
1058 | } | 1061 | } |
1059 | 1062 | ||
1060 | static inline int queue_discard_alignment(struct request_queue *q) | 1063 | static inline int queue_discard_alignment(struct request_queue *q) |
1061 | { | 1064 | { |
1062 | if (q->limits.discard_misaligned) | 1065 | if (q->limits.discard_misaligned) |
1063 | return -1; | 1066 | return -1; |
1064 | 1067 | ||
1065 | return q->limits.discard_alignment; | 1068 | return q->limits.discard_alignment; |
1066 | } | 1069 | } |
1067 | 1070 | ||
1068 | static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector) | 1071 | static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector) |
1069 | { | 1072 | { |
1070 | unsigned int alignment = (sector << 9) & (lim->discard_granularity - 1); | 1073 | unsigned int alignment = (sector << 9) & (lim->discard_granularity - 1); |
1071 | 1074 | ||
1072 | if (!lim->max_discard_sectors) | 1075 | if (!lim->max_discard_sectors) |
1073 | return 0; | 1076 | return 0; |
1074 | 1077 | ||
1075 | return (lim->discard_granularity + lim->discard_alignment - alignment) | 1078 | return (lim->discard_granularity + lim->discard_alignment - alignment) |
1076 | & (lim->discard_granularity - 1); | 1079 | & (lim->discard_granularity - 1); |
1077 | } | 1080 | } |
1078 | 1081 | ||
1079 | static inline unsigned int queue_discard_zeroes_data(struct request_queue *q) | 1082 | static inline unsigned int queue_discard_zeroes_data(struct request_queue *q) |
1080 | { | 1083 | { |
1081 | if (q->limits.max_discard_sectors && q->limits.discard_zeroes_data == 1) | 1084 | if (q->limits.max_discard_sectors && q->limits.discard_zeroes_data == 1) |
1082 | return 1; | 1085 | return 1; |
1083 | 1086 | ||
1084 | return 0; | 1087 | return 0; |
1085 | } | 1088 | } |
1086 | 1089 | ||
1087 | static inline unsigned int bdev_discard_zeroes_data(struct block_device *bdev) | 1090 | static inline unsigned int bdev_discard_zeroes_data(struct block_device *bdev) |
1088 | { | 1091 | { |
1089 | return queue_discard_zeroes_data(bdev_get_queue(bdev)); | 1092 | return queue_discard_zeroes_data(bdev_get_queue(bdev)); |
1090 | } | 1093 | } |
1091 | 1094 | ||
1092 | static inline int queue_dma_alignment(struct request_queue *q) | 1095 | static inline int queue_dma_alignment(struct request_queue *q) |
1093 | { | 1096 | { |
1094 | return q ? q->dma_alignment : 511; | 1097 | return q ? q->dma_alignment : 511; |
1095 | } | 1098 | } |
1096 | 1099 | ||
1097 | static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr, | 1100 | static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr, |
1098 | unsigned int len) | 1101 | unsigned int len) |
1099 | { | 1102 | { |
1100 | unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask; | 1103 | unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask; |
1101 | return !(addr & alignment) && !(len & alignment); | 1104 | return !(addr & alignment) && !(len & alignment); |
1102 | } | 1105 | } |
1103 | 1106 | ||
1104 | /* assumes size > 256 */ | 1107 | /* assumes size > 256 */ |
1105 | static inline unsigned int blksize_bits(unsigned int size) | 1108 | static inline unsigned int blksize_bits(unsigned int size) |
1106 | { | 1109 | { |
1107 | unsigned int bits = 8; | 1110 | unsigned int bits = 8; |
1108 | do { | 1111 | do { |
1109 | bits++; | 1112 | bits++; |
1110 | size >>= 1; | 1113 | size >>= 1; |
1111 | } while (size > 256); | 1114 | } while (size > 256); |
1112 | return bits; | 1115 | return bits; |
1113 | } | 1116 | } |
1114 | 1117 | ||
1115 | static inline unsigned int block_size(struct block_device *bdev) | 1118 | static inline unsigned int block_size(struct block_device *bdev) |
1116 | { | 1119 | { |
1117 | return bdev->bd_block_size; | 1120 | return bdev->bd_block_size; |
1118 | } | 1121 | } |
1119 | 1122 | ||
1120 | static inline bool queue_flush_queueable(struct request_queue *q) | 1123 | static inline bool queue_flush_queueable(struct request_queue *q) |
1121 | { | 1124 | { |
1122 | return !q->flush_not_queueable; | 1125 | return !q->flush_not_queueable; |
1123 | } | 1126 | } |
1124 | 1127 | ||
1125 | typedef struct {struct page *v;} Sector; | 1128 | typedef struct {struct page *v;} Sector; |
1126 | 1129 | ||
1127 | unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *); | 1130 | unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *); |
1128 | 1131 | ||
1129 | static inline void put_dev_sector(Sector p) | 1132 | static inline void put_dev_sector(Sector p) |
1130 | { | 1133 | { |
1131 | page_cache_release(p.v); | 1134 | page_cache_release(p.v); |
1132 | } | 1135 | } |
1133 | 1136 | ||
1134 | struct work_struct; | 1137 | struct work_struct; |
1135 | int kblockd_schedule_work(struct request_queue *q, struct work_struct *work); | 1138 | int kblockd_schedule_work(struct request_queue *q, struct work_struct *work); |
1136 | 1139 | ||
1137 | #ifdef CONFIG_BLK_CGROUP | 1140 | #ifdef CONFIG_BLK_CGROUP |
1138 | /* | 1141 | /* |
1139 | * This should not be using sched_clock(). A real patch is in progress | 1142 | * This should not be using sched_clock(). A real patch is in progress |
1140 | * to fix this up, until that is in place we need to disable preemption | 1143 | * to fix this up, until that is in place we need to disable preemption |
1141 | * around sched_clock() in this function and set_io_start_time_ns(). | 1144 | * around sched_clock() in this function and set_io_start_time_ns(). |
1142 | */ | 1145 | */ |
1143 | static inline void set_start_time_ns(struct request *req) | 1146 | static inline void set_start_time_ns(struct request *req) |
1144 | { | 1147 | { |
1145 | preempt_disable(); | 1148 | preempt_disable(); |
1146 | req->start_time_ns = sched_clock(); | 1149 | req->start_time_ns = sched_clock(); |
1147 | preempt_enable(); | 1150 | preempt_enable(); |
1148 | } | 1151 | } |
1149 | 1152 | ||
1150 | static inline void set_io_start_time_ns(struct request *req) | 1153 | static inline void set_io_start_time_ns(struct request *req) |
1151 | { | 1154 | { |
1152 | preempt_disable(); | 1155 | preempt_disable(); |
1153 | req->io_start_time_ns = sched_clock(); | 1156 | req->io_start_time_ns = sched_clock(); |
1154 | preempt_enable(); | 1157 | preempt_enable(); |
1155 | } | 1158 | } |
1156 | 1159 | ||
1157 | static inline uint64_t rq_start_time_ns(struct request *req) | 1160 | static inline uint64_t rq_start_time_ns(struct request *req) |
1158 | { | 1161 | { |
1159 | return req->start_time_ns; | 1162 | return req->start_time_ns; |
1160 | } | 1163 | } |
1161 | 1164 | ||
1162 | static inline uint64_t rq_io_start_time_ns(struct request *req) | 1165 | static inline uint64_t rq_io_start_time_ns(struct request *req) |
1163 | { | 1166 | { |
1164 | return req->io_start_time_ns; | 1167 | return req->io_start_time_ns; |
1165 | } | 1168 | } |
1166 | #else | 1169 | #else |
1167 | static inline void set_start_time_ns(struct request *req) {} | 1170 | static inline void set_start_time_ns(struct request *req) {} |
1168 | static inline void set_io_start_time_ns(struct request *req) {} | 1171 | static inline void set_io_start_time_ns(struct request *req) {} |
1169 | static inline uint64_t rq_start_time_ns(struct request *req) | 1172 | static inline uint64_t rq_start_time_ns(struct request *req) |
1170 | { | 1173 | { |
1171 | return 0; | 1174 | return 0; |
1172 | } | 1175 | } |
1173 | static inline uint64_t rq_io_start_time_ns(struct request *req) | 1176 | static inline uint64_t rq_io_start_time_ns(struct request *req) |
1174 | { | 1177 | { |
1175 | return 0; | 1178 | return 0; |
1176 | } | 1179 | } |
1177 | #endif | 1180 | #endif |
1178 | 1181 | ||
1179 | #ifdef CONFIG_BLK_DEV_THROTTLING | 1182 | #ifdef CONFIG_BLK_DEV_THROTTLING |
1180 | extern int blk_throtl_init(struct request_queue *q); | 1183 | extern int blk_throtl_init(struct request_queue *q); |
1181 | extern void blk_throtl_exit(struct request_queue *q); | 1184 | extern void blk_throtl_exit(struct request_queue *q); |
1182 | extern int blk_throtl_bio(struct request_queue *q, struct bio **bio); | 1185 | extern int blk_throtl_bio(struct request_queue *q, struct bio **bio); |
1183 | #else /* CONFIG_BLK_DEV_THROTTLING */ | 1186 | #else /* CONFIG_BLK_DEV_THROTTLING */ |
1184 | static inline int blk_throtl_bio(struct request_queue *q, struct bio **bio) | 1187 | static inline int blk_throtl_bio(struct request_queue *q, struct bio **bio) |
1185 | { | 1188 | { |
1186 | return 0; | 1189 | return 0; |
1187 | } | 1190 | } |
1188 | 1191 | ||
1189 | static inline int blk_throtl_init(struct request_queue *q) { return 0; } | 1192 | static inline int blk_throtl_init(struct request_queue *q) { return 0; } |
1190 | static inline int blk_throtl_exit(struct request_queue *q) { return 0; } | 1193 | static inline int blk_throtl_exit(struct request_queue *q) { return 0; } |
1191 | #endif /* CONFIG_BLK_DEV_THROTTLING */ | 1194 | #endif /* CONFIG_BLK_DEV_THROTTLING */ |
1192 | 1195 | ||
1193 | #define MODULE_ALIAS_BLOCKDEV(major,minor) \ | 1196 | #define MODULE_ALIAS_BLOCKDEV(major,minor) \ |
1194 | MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) | 1197 | MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor)) |
1195 | #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \ | 1198 | #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \ |
1196 | MODULE_ALIAS("block-major-" __stringify(major) "-*") | 1199 | MODULE_ALIAS("block-major-" __stringify(major) "-*") |
1197 | 1200 | ||
1198 | #if defined(CONFIG_BLK_DEV_INTEGRITY) | 1201 | #if defined(CONFIG_BLK_DEV_INTEGRITY) |
1199 | 1202 | ||
1200 | #define INTEGRITY_FLAG_READ 2 /* verify data integrity on read */ | 1203 | #define INTEGRITY_FLAG_READ 2 /* verify data integrity on read */ |
1201 | #define INTEGRITY_FLAG_WRITE 4 /* generate data integrity on write */ | 1204 | #define INTEGRITY_FLAG_WRITE 4 /* generate data integrity on write */ |
1202 | 1205 | ||
1203 | struct blk_integrity_exchg { | 1206 | struct blk_integrity_exchg { |
1204 | void *prot_buf; | 1207 | void *prot_buf; |
1205 | void *data_buf; | 1208 | void *data_buf; |
1206 | sector_t sector; | 1209 | sector_t sector; |
1207 | unsigned int data_size; | 1210 | unsigned int data_size; |
1208 | unsigned short sector_size; | 1211 | unsigned short sector_size; |
1209 | const char *disk_name; | 1212 | const char *disk_name; |
1210 | }; | 1213 | }; |
1211 | 1214 | ||
1212 | typedef void (integrity_gen_fn) (struct blk_integrity_exchg *); | 1215 | typedef void (integrity_gen_fn) (struct blk_integrity_exchg *); |
1213 | typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *); | 1216 | typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *); |
1214 | typedef void (integrity_set_tag_fn) (void *, void *, unsigned int); | 1217 | typedef void (integrity_set_tag_fn) (void *, void *, unsigned int); |
1215 | typedef void (integrity_get_tag_fn) (void *, void *, unsigned int); | 1218 | typedef void (integrity_get_tag_fn) (void *, void *, unsigned int); |
1216 | 1219 | ||
1217 | struct blk_integrity { | 1220 | struct blk_integrity { |
1218 | integrity_gen_fn *generate_fn; | 1221 | integrity_gen_fn *generate_fn; |
1219 | integrity_vrfy_fn *verify_fn; | 1222 | integrity_vrfy_fn *verify_fn; |
1220 | integrity_set_tag_fn *set_tag_fn; | 1223 | integrity_set_tag_fn *set_tag_fn; |
1221 | integrity_get_tag_fn *get_tag_fn; | 1224 | integrity_get_tag_fn *get_tag_fn; |
1222 | 1225 | ||
1223 | unsigned short flags; | 1226 | unsigned short flags; |
1224 | unsigned short tuple_size; | 1227 | unsigned short tuple_size; |
1225 | unsigned short sector_size; | 1228 | unsigned short sector_size; |
1226 | unsigned short tag_size; | 1229 | unsigned short tag_size; |
1227 | 1230 | ||
1228 | const char *name; | 1231 | const char *name; |
1229 | 1232 | ||
1230 | struct kobject kobj; | 1233 | struct kobject kobj; |
1231 | }; | 1234 | }; |
1232 | 1235 | ||
1233 | extern bool blk_integrity_is_initialized(struct gendisk *); | 1236 | extern bool blk_integrity_is_initialized(struct gendisk *); |
1234 | extern int blk_integrity_register(struct gendisk *, struct blk_integrity *); | 1237 | extern int blk_integrity_register(struct gendisk *, struct blk_integrity *); |
1235 | extern void blk_integrity_unregister(struct gendisk *); | 1238 | extern void blk_integrity_unregister(struct gendisk *); |
1236 | extern int blk_integrity_compare(struct gendisk *, struct gendisk *); | 1239 | extern int blk_integrity_compare(struct gendisk *, struct gendisk *); |
1237 | extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *, | 1240 | extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *, |
1238 | struct scatterlist *); | 1241 | struct scatterlist *); |
1239 | extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *); | 1242 | extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *); |
1240 | extern int blk_integrity_merge_rq(struct request_queue *, struct request *, | 1243 | extern int blk_integrity_merge_rq(struct request_queue *, struct request *, |
1241 | struct request *); | 1244 | struct request *); |
1242 | extern int blk_integrity_merge_bio(struct request_queue *, struct request *, | 1245 | extern int blk_integrity_merge_bio(struct request_queue *, struct request *, |
1243 | struct bio *); | 1246 | struct bio *); |
1244 | 1247 | ||
1245 | static inline | 1248 | static inline |
1246 | struct blk_integrity *bdev_get_integrity(struct block_device *bdev) | 1249 | struct blk_integrity *bdev_get_integrity(struct block_device *bdev) |
1247 | { | 1250 | { |
1248 | return bdev->bd_disk->integrity; | 1251 | return bdev->bd_disk->integrity; |
1249 | } | 1252 | } |
1250 | 1253 | ||
1251 | static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk) | 1254 | static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk) |
1252 | { | 1255 | { |
1253 | return disk->integrity; | 1256 | return disk->integrity; |
1254 | } | 1257 | } |
1255 | 1258 | ||
1256 | static inline int blk_integrity_rq(struct request *rq) | 1259 | static inline int blk_integrity_rq(struct request *rq) |
1257 | { | 1260 | { |
1258 | if (rq->bio == NULL) | 1261 | if (rq->bio == NULL) |
1259 | return 0; | 1262 | return 0; |
1260 | 1263 | ||
1261 | return bio_integrity(rq->bio); | 1264 | return bio_integrity(rq->bio); |
1262 | } | 1265 | } |
1263 | 1266 | ||
1264 | static inline void blk_queue_max_integrity_segments(struct request_queue *q, | 1267 | static inline void blk_queue_max_integrity_segments(struct request_queue *q, |
1265 | unsigned int segs) | 1268 | unsigned int segs) |
1266 | { | 1269 | { |
1267 | q->limits.max_integrity_segments = segs; | 1270 | q->limits.max_integrity_segments = segs; |
1268 | } | 1271 | } |
1269 | 1272 | ||
1270 | static inline unsigned short | 1273 | static inline unsigned short |
1271 | queue_max_integrity_segments(struct request_queue *q) | 1274 | queue_max_integrity_segments(struct request_queue *q) |
1272 | { | 1275 | { |
1273 | return q->limits.max_integrity_segments; | 1276 | return q->limits.max_integrity_segments; |
1274 | } | 1277 | } |
1275 | 1278 | ||
1276 | #else /* CONFIG_BLK_DEV_INTEGRITY */ | 1279 | #else /* CONFIG_BLK_DEV_INTEGRITY */ |
1277 | 1280 | ||
1278 | #define blk_integrity_rq(rq) (0) | 1281 | #define blk_integrity_rq(rq) (0) |
1279 | #define blk_rq_count_integrity_sg(a, b) (0) | 1282 | #define blk_rq_count_integrity_sg(a, b) (0) |
1280 | #define blk_rq_map_integrity_sg(a, b, c) (0) | 1283 | #define blk_rq_map_integrity_sg(a, b, c) (0) |
1281 | #define bdev_get_integrity(a) (0) | 1284 | #define bdev_get_integrity(a) (0) |
1282 | #define blk_get_integrity(a) (0) | 1285 | #define blk_get_integrity(a) (0) |
1283 | #define blk_integrity_compare(a, b) (0) | 1286 | #define blk_integrity_compare(a, b) (0) |
1284 | #define blk_integrity_register(a, b) (0) | 1287 | #define blk_integrity_register(a, b) (0) |
1285 | #define blk_integrity_unregister(a) do { } while (0) | 1288 | #define blk_integrity_unregister(a) do { } while (0) |
1286 | #define blk_queue_max_integrity_segments(a, b) do { } while (0) | 1289 | #define blk_queue_max_integrity_segments(a, b) do { } while (0) |
1287 | #define queue_max_integrity_segments(a) (0) | 1290 | #define queue_max_integrity_segments(a) (0) |
1288 | #define blk_integrity_merge_rq(a, b, c) (0) | 1291 | #define blk_integrity_merge_rq(a, b, c) (0) |
1289 | #define blk_integrity_merge_bio(a, b, c) (0) | 1292 | #define blk_integrity_merge_bio(a, b, c) (0) |
1290 | #define blk_integrity_is_initialized(a) (0) | 1293 | #define blk_integrity_is_initialized(a) (0) |
1291 | 1294 | ||
1292 | #endif /* CONFIG_BLK_DEV_INTEGRITY */ | 1295 | #endif /* CONFIG_BLK_DEV_INTEGRITY */ |
1293 | 1296 | ||
1294 | struct block_device_operations { | 1297 | struct block_device_operations { |
1295 | int (*open) (struct block_device *, fmode_t); | 1298 | int (*open) (struct block_device *, fmode_t); |
1296 | int (*release) (struct gendisk *, fmode_t); | 1299 | int (*release) (struct gendisk *, fmode_t); |
1297 | int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); | 1300 | int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); |
1298 | int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); | 1301 | int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long); |
1299 | int (*direct_access) (struct block_device *, sector_t, | 1302 | int (*direct_access) (struct block_device *, sector_t, |
1300 | void **, unsigned long *); | 1303 | void **, unsigned long *); |
1301 | unsigned int (*check_events) (struct gendisk *disk, | 1304 | unsigned int (*check_events) (struct gendisk *disk, |
1302 | unsigned int clearing); | 1305 | unsigned int clearing); |
1303 | /* ->media_changed() is DEPRECATED, use ->check_events() instead */ | 1306 | /* ->media_changed() is DEPRECATED, use ->check_events() instead */ |
1304 | int (*media_changed) (struct gendisk *); | 1307 | int (*media_changed) (struct gendisk *); |
1305 | void (*unlock_native_capacity) (struct gendisk *); | 1308 | void (*unlock_native_capacity) (struct gendisk *); |
1306 | int (*revalidate_disk) (struct gendisk *); | 1309 | int (*revalidate_disk) (struct gendisk *); |
1307 | int (*getgeo)(struct block_device *, struct hd_geometry *); | 1310 | int (*getgeo)(struct block_device *, struct hd_geometry *); |
1308 | /* this callback is with swap_lock and sometimes page table lock held */ | 1311 | /* this callback is with swap_lock and sometimes page table lock held */ |
1309 | void (*swap_slot_free_notify) (struct block_device *, unsigned long); | 1312 | void (*swap_slot_free_notify) (struct block_device *, unsigned long); |
1310 | struct module *owner; | 1313 | struct module *owner; |
1311 | }; | 1314 | }; |
1312 | 1315 | ||
1313 | extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int, | 1316 | extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int, |
1314 | unsigned long); | 1317 | unsigned long); |
1315 | #else /* CONFIG_BLOCK */ | 1318 | #else /* CONFIG_BLOCK */ |
1316 | /* | 1319 | /* |
1317 | * stubs for when the block layer is configured out | 1320 | * stubs for when the block layer is configured out |
1318 | */ | 1321 | */ |
1319 | #define buffer_heads_over_limit 0 | 1322 | #define buffer_heads_over_limit 0 |
1320 | 1323 | ||
1321 | static inline long nr_blockdev_pages(void) | 1324 | static inline long nr_blockdev_pages(void) |
1322 | { | 1325 | { |
1323 | return 0; | 1326 | return 0; |
1324 | } | 1327 | } |
1325 | 1328 | ||
1326 | struct blk_plug { | 1329 | struct blk_plug { |
1327 | }; | 1330 | }; |
1328 | 1331 | ||
1329 | static inline void blk_start_plug(struct blk_plug *plug) | 1332 | static inline void blk_start_plug(struct blk_plug *plug) |
1330 | { | 1333 | { |
1331 | } | 1334 | } |
1332 | 1335 | ||
1333 | static inline void blk_finish_plug(struct blk_plug *plug) | 1336 | static inline void blk_finish_plug(struct blk_plug *plug) |
1334 | { | 1337 | { |
1335 | } | 1338 | } |
1336 | 1339 | ||
1337 | static inline void blk_flush_plug(struct task_struct *task) | 1340 | static inline void blk_flush_plug(struct task_struct *task) |
1338 | { | 1341 | { |
1339 | } | 1342 | } |
1340 | 1343 | ||
1341 | static inline void blk_schedule_flush_plug(struct task_struct *task) | 1344 | static inline void blk_schedule_flush_plug(struct task_struct *task) |
1342 | { | 1345 | { |
1343 | } | 1346 | } |
1344 | 1347 | ||
1345 | 1348 | ||
1346 | static inline bool blk_needs_flush_plug(struct task_struct *tsk) | 1349 | static inline bool blk_needs_flush_plug(struct task_struct *tsk) |
1347 | { | 1350 | { |
1348 | return false; | 1351 | return false; |
1349 | } | 1352 | } |
1350 | 1353 | ||
1351 | #endif /* CONFIG_BLOCK */ | 1354 | #endif /* CONFIG_BLOCK */ |
1352 | 1355 | ||
1353 | #endif | 1356 | #endif |
1354 | 1357 |