Blame view

include/linux/blkdev.h 40.4 KB
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1
2
  #ifndef _LINUX_BLKDEV_H
  #define _LINUX_BLKDEV_H
f5ff8422b   Jens Axboe   Fix warnings with...
3
  #ifdef CONFIG_BLOCK
bcfd8d361   Andrew Morton   [PATCH] CONFIG_BL...
4
  #include <linux/sched.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
5
6
7
8
9
10
11
12
13
14
  #include <linux/major.h>
  #include <linux/genhd.h>
  #include <linux/list.h>
  #include <linux/timer.h>
  #include <linux/workqueue.h>
  #include <linux/pagemap.h>
  #include <linux/backing-dev.h>
  #include <linux/wait.h>
  #include <linux/mempool.h>
  #include <linux/bio.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
15
  #include <linux/stringify.h>
3e6053d76   Hugh Dickins   block: adjust blk...
16
  #include <linux/gfp.h>
d351af01b   FUJITA Tomonori   bsg: bind bsg to ...
17
  #include <linux/bsg.h>
c7c22e4d5   Jens Axboe   block: add suppor...
18
  #include <linux/smp.h>
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
19
20
  
  #include <asm/scatterlist.h>
de4772542   Paul Gortmaker   include: replace ...
21
  struct module;
21b2f0c80   Christoph Hellwig   [SCSI] unify SCSI...
22
  struct scsi_ioctl_command;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
23
  struct request_queue;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
24
  struct elevator_queue;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
25
  struct request_pm_state;
2056a782f   Jens Axboe   [PATCH] Block que...
26
  struct blk_trace;
3d6392cfb   Jens Axboe   bsg: support for ...
27
28
  struct request;
  struct sg_io_hdr;
aa387cc89   Mike Christie   block: add bsg he...
29
  struct bsg_job;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
30
31
32
  
  #define BLKDEV_MIN_RQ	4
  #define BLKDEV_MAX_RQ	128	/* Default maximum */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
33
  struct request;
8ffdc6550   Tejun Heo   [BLOCK] add @upto...
34
  typedef void (rq_end_io_fn)(struct request *, int);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
35
36
  
  struct request_list {
1faa16d22   Jens Axboe   block: change the...
37
38
39
40
  	/*
  	 * count[], starved[], and wait[] are indexed by
  	 * BLK_RW_SYNC/BLK_RW_ASYNC
  	 */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
41
42
  	int count[2];
  	int starved[2];
cb98fc8bb   Tejun Heo   [BLOCK] Reimpleme...
43
  	int elvpriv;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
44
45
  	mempool_t *rq_pool;
  	wait_queue_head_t wait[2];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
46
  };
4aff5e233   Jens Axboe   [PATCH] Split str...
47
48
49
50
51
52
53
54
55
56
  /*
   * request command types
   */
  enum rq_cmd_type_bits {
  	REQ_TYPE_FS		= 1,	/* fs request */
  	REQ_TYPE_BLOCK_PC,		/* scsi command */
  	REQ_TYPE_SENSE,			/* sense request */
  	REQ_TYPE_PM_SUSPEND,		/* suspend request */
  	REQ_TYPE_PM_RESUME,		/* resume request */
  	REQ_TYPE_PM_SHUTDOWN,		/* shutdown request */
4aff5e233   Jens Axboe   [PATCH] Split str...
57
  	REQ_TYPE_SPECIAL,		/* driver defined type */
4aff5e233   Jens Axboe   [PATCH] Split str...
58
59
60
61
62
  	/*
  	 * for ATA/ATAPI devices. this really doesn't belong here, ide should
  	 * use REQ_TYPE_SPECIAL and use rq->cmd[0] with the range of driver
  	 * private REQ_LB opcodes to differentiate what type of request this is
  	 */
4aff5e233   Jens Axboe   [PATCH] Split str...
63
  	REQ_TYPE_ATA_TASKFILE,
cea2885a2   Jens Axboe   [PATCH] ide-cd: f...
64
  	REQ_TYPE_ATA_PC,
4aff5e233   Jens Axboe   [PATCH] Split str...
65
  };
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
66
67
68
  #define BLK_MAX_CDB	16
  
  /*
63a713867   Jens Axboe   block: fixup rq_i...
69
   * try to put the fields that are referenced together in the same cacheline.
4d0d98b60   Wanlong Gao   block:fix the com...
70
   * if you modify this structure, be sure to check block/blk-core.c:blk_rq_init()
63a713867   Jens Axboe   block: fixup rq_i...
71
   * as well!
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
72
73
   */
  struct request {
ff856bad6   Jens Axboe   [BLOCK] ll_rw_blk...
74
  	struct list_head queuelist;
c7c22e4d5   Jens Axboe   block: add suppor...
75
  	struct call_single_data csd;
ff856bad6   Jens Axboe   [BLOCK] ll_rw_blk...
76

165125e1e   Jens Axboe   [BLOCK] Get rid o...
77
  	struct request_queue *q;
e6a1c874a   Jens Axboe   [PATCH] struct re...
78

4aff5e233   Jens Axboe   [PATCH] Split str...
79
80
  	unsigned int cmd_flags;
  	enum rq_cmd_type_bits cmd_type;
242f9dcb8   Jens Axboe   block: unify requ...
81
  	unsigned long atomic_flags;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
82

181fdde3b   Richard Kennedy   block: remove 16 ...
83
  	int cpu;
a2dec7b36   Tejun Heo   block: hide reque...
84
  	/* the following two fields are internal, NEVER access directly */
a2dec7b36   Tejun Heo   block: hide reque...
85
  	unsigned int __data_len;	/* total data len */
181fdde3b   Richard Kennedy   block: remove 16 ...
86
  	sector_t __sector;		/* sector cursor */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
87
88
89
  
  	struct bio *bio;
  	struct bio *biotail;
9817064b6   Jens Axboe   [PATCH] elevator:...
90
  	struct hlist_node hash;	/* merge hash */
e6a1c874a   Jens Axboe   [PATCH] struct re...
91
92
93
  	/*
  	 * The rb_node is only used inside the io scheduler, requests
  	 * are pruned when moved to the dispatch queue. So let the
c186794db   Mike Snitzer   block: share requ...
94
  	 * completion_data share space with the rb_node.
e6a1c874a   Jens Axboe   [PATCH] struct re...
95
96
97
  	 */
  	union {
  		struct rb_node rb_node;	/* sort/lookup */
c186794db   Mike Snitzer   block: share requ...
98
  		void *completion_data;
e6a1c874a   Jens Axboe   [PATCH] struct re...
99
  	};
9817064b6   Jens Axboe   [PATCH] elevator:...
100

ff7d145fd   Jens Axboe   [PATCH] Add one m...
101
  	/*
7f1dc8a2d   Vivek Goyal   blkio: Fix blkio ...
102
  	 * Three pointers are available for the IO schedulers, if they need
c186794db   Mike Snitzer   block: share requ...
103
104
  	 * more they have to dynamically allocate it.  Flush requests are
  	 * never put on the IO scheduler. So let the flush fields share
a612fddf0   Tejun Heo   block, cfq: move ...
105
  	 * space with the elevator data.
ff7d145fd   Jens Axboe   [PATCH] Add one m...
106
  	 */
c186794db   Mike Snitzer   block: share requ...
107
  	union {
a612fddf0   Tejun Heo   block, cfq: move ...
108
109
110
111
  		struct {
  			struct io_cq		*icq;
  			void			*priv[2];
  		} elv;
c186794db   Mike Snitzer   block: share requ...
112
113
114
  		struct {
  			unsigned int		seq;
  			struct list_head	list;
4853abaae   Jeff Moyer   block: fix flush ...
115
  			rq_end_io_fn		*saved_end_io;
c186794db   Mike Snitzer   block: share requ...
116
117
  		} flush;
  	};
ff7d145fd   Jens Axboe   [PATCH] Add one m...
118

8f34ee75d   Jens Axboe   [PATCH] Rearrange...
119
  	struct gendisk *rq_disk;
09e099d4b   Jerome Marchand   block: fix accoun...
120
  	struct hd_struct *part;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
121
  	unsigned long start_time;
9195291e5   Divyesh Shah   blkio: Increment ...
122
123
124
125
  #ifdef CONFIG_BLK_CGROUP
  	unsigned long long start_time_ns;
  	unsigned long long io_start_time_ns;    /* when passed to hardware */
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
126
127
128
129
  	/* Number of scatter-gather DMA addr+len pairs after
  	 * physical address coalescing is performed.
  	 */
  	unsigned short nr_phys_segments;
13f05c8d8   Martin K. Petersen   block/scsi: Provi...
130
131
132
  #if defined(CONFIG_BLK_DEV_INTEGRITY)
  	unsigned short nr_integrity_segments;
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
133

8f34ee75d   Jens Axboe   [PATCH] Rearrange...
134
  	unsigned short ioprio;
181fdde3b   Richard Kennedy   block: remove 16 ...
135
  	int ref_count;
731ec497e   Tejun Heo   block: kill rq->data
136
137
  	void *special;		/* opaque pointer available for LLD use */
  	char *buffer;		/* kaddr of the current segment if available */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
138

cdd602621   Jens Axboe   [PATCH] Remove ->...
139
140
  	int tag;
  	int errors;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
141
142
143
  	/*
  	 * when request is used as a packet command carrier
  	 */
d7e3c3249   FUJITA Tomonori   block: add large ...
144
145
  	unsigned char __cmd[BLK_MAX_CDB];
  	unsigned char *cmd;
181fdde3b   Richard Kennedy   block: remove 16 ...
146
  	unsigned short cmd_len;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
147

7a85f8896   FUJITA Tomonori   block: restore th...
148
  	unsigned int extra_len;	/* length of alignment and padding */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
149
  	unsigned int sense_len;
c3a4d78c5   Tejun Heo   block: add rq->re...
150
  	unsigned int resid_len;	/* residual count */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
151
  	void *sense;
242f9dcb8   Jens Axboe   block: unify requ...
152
153
  	unsigned long deadline;
  	struct list_head timeout_list;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
154
  	unsigned int timeout;
17e01f216   Mike Christie   [SCSI] add retrie...
155
  	int retries;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
156
157
  
  	/*
c00895ab2   Jens Axboe   [PATCH] Remove ->...
158
  	 * completion callback.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
159
160
161
  	 */
  	rq_end_io_fn *end_io;
  	void *end_io_data;
abae1fde6   FUJITA Tomonori   add a struct requ...
162
163
164
  
  	/* for bidi */
  	struct request *next_rq;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
165
  };
766ca4428   Fernando Luis Vázquez Cao   virtio_blk: use a...
166
167
168
169
  static inline unsigned short req_get_ioprio(struct request *req)
  {
  	return req->ioprio;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
170
  /*
4aff5e233   Jens Axboe   [PATCH] Split str...
171
   * State information carried for REQ_TYPE_PM_SUSPEND and REQ_TYPE_PM_RESUME
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
172
173
174
175
176
177
178
179
180
181
182
183
   * requests. Some step values could eventually be made generic.
   */
  struct request_pm_state
  {
  	/* PM state machine step value, currently driver specific */
  	int	pm_step;
  	/* requested PM state value (S1, S2, S3, S4, ...) */
  	u32	pm_state;
  	void*	data;		/* for driver use */
  };
  
  #include <linux/elevator.h>
165125e1e   Jens Axboe   [BLOCK] Get rid o...
184
  typedef void (request_fn_proc) (struct request_queue *q);
5a7bbad27   Christoph Hellwig   block: remove sup...
185
  typedef void (make_request_fn) (struct request_queue *q, struct bio *bio);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
186
  typedef int (prep_rq_fn) (struct request_queue *, struct request *);
28018c242   James Bottomley   block: implement ...
187
  typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
188
189
  
  struct bio_vec;
cc371e66e   Alasdair G Kergon   Add bvec_merge_da...
190
191
192
193
194
195
196
197
  struct bvec_merge_data {
  	struct block_device *bi_bdev;
  	sector_t bi_sector;
  	unsigned bi_size;
  	unsigned long bi_rw;
  };
  typedef int (merge_bvec_fn) (struct request_queue *, struct bvec_merge_data *,
  			     struct bio_vec *);
ff856bad6   Jens Axboe   [BLOCK] ll_rw_blk...
198
  typedef void (softirq_done_fn)(struct request *);
2fb98e841   Tejun Heo   block: implement ...
199
  typedef int (dma_drain_needed_fn)(struct request *);
ef9e3facd   Kiyoshi Ueda   block: add lld bu...
200
  typedef int (lld_busy_fn) (struct request_queue *q);
aa387cc89   Mike Christie   block: add bsg he...
201
  typedef int (bsg_job_fn) (struct bsg_job *);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
202

242f9dcb8   Jens Axboe   block: unify requ...
203
204
205
206
207
208
209
  enum blk_eh_timer_return {
  	BLK_EH_NOT_HANDLED,
  	BLK_EH_HANDLED,
  	BLK_EH_RESET_TIMER,
  };
  
  typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
210
211
212
213
  enum blk_queue_state {
  	Queue_down,
  	Queue_up,
  };
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
214
215
216
  struct blk_queue_tag {
  	struct request **tag_index;	/* map of busy tags */
  	unsigned long *tag_map;		/* bit map of free/busy tags */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
217
218
  	int busy;			/* current depth */
  	int max_depth;			/* what we will send to device */
ba0250824   Tejun Heo   [PATCH] blk: fix ...
219
  	int real_max_depth;		/* what the array can hold */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
220
221
  	atomic_t refcnt;		/* map can be shared */
  };
abf543937   FUJITA Tomonori   block: move cmdfi...
222
223
  #define BLK_SCSI_MAX_CMDS	(256)
  #define BLK_SCSI_CMD_PER_LONG	(BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))
025146e13   Martin K. Petersen   block: Move queue...
224
225
226
227
228
229
230
  struct queue_limits {
  	unsigned long		bounce_pfn;
  	unsigned long		seg_boundary_mask;
  
  	unsigned int		max_hw_sectors;
  	unsigned int		max_sectors;
  	unsigned int		max_segment_size;
c72758f33   Martin K. Petersen   block: Export I/O...
231
232
233
234
  	unsigned int		physical_block_size;
  	unsigned int		alignment_offset;
  	unsigned int		io_min;
  	unsigned int		io_opt;
67efc9258   Christoph Hellwig   block: allow larg...
235
  	unsigned int		max_discard_sectors;
86b372814   Martin K. Petersen   block: Expose dis...
236
237
  	unsigned int		discard_granularity;
  	unsigned int		discard_alignment;
025146e13   Martin K. Petersen   block: Move queue...
238
239
  
  	unsigned short		logical_block_size;
8a78362c4   Martin K. Petersen   block: Consolidat...
240
  	unsigned short		max_segments;
13f05c8d8   Martin K. Petersen   block/scsi: Provi...
241
  	unsigned short		max_integrity_segments;
025146e13   Martin K. Petersen   block: Move queue...
242

c72758f33   Martin K. Petersen   block: Export I/O...
243
  	unsigned char		misaligned;
86b372814   Martin K. Petersen   block: Expose dis...
244
  	unsigned char		discard_misaligned;
e692cb668   Martin K. Petersen   block: Deprecate ...
245
  	unsigned char		cluster;
a934a00a6   Martin K. Petersen   block: Fix discar...
246
  	unsigned char		discard_zeroes_data;
025146e13   Martin K. Petersen   block: Move queue...
247
  };
d7b763013   Richard Kennedy   block: reorder re...
248
  struct request_queue {
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
249
250
251
252
253
  	/*
  	 * Together with queue_head for cacheline sharing
  	 */
  	struct list_head	queue_head;
  	struct request		*last_merge;
b374d18a4   Jens Axboe   block: get rid of...
254
  	struct elevator_queue	*elevator;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
255
256
257
258
259
260
261
  
  	/*
  	 * the queue request freelist, one for reads and one for writes
  	 */
  	struct request_list	rq;
  
  	request_fn_proc		*request_fn;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
262
263
  	make_request_fn		*make_request_fn;
  	prep_rq_fn		*prep_rq_fn;
28018c242   James Bottomley   block: implement ...
264
  	unprep_rq_fn		*unprep_rq_fn;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
265
  	merge_bvec_fn		*merge_bvec_fn;
ff856bad6   Jens Axboe   [BLOCK] ll_rw_blk...
266
  	softirq_done_fn		*softirq_done_fn;
242f9dcb8   Jens Axboe   block: unify requ...
267
  	rq_timed_out_fn		*rq_timed_out_fn;
2fb98e841   Tejun Heo   block: implement ...
268
  	dma_drain_needed_fn	*dma_drain_needed;
ef9e3facd   Kiyoshi Ueda   block: add lld bu...
269
  	lld_busy_fn		*lld_busy_fn;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
270
271
  
  	/*
8922e16cf   Tejun Heo   [PATCH] 01/05 Imp...
272
273
  	 * Dispatch queue sorting
  	 */
1b47f531e   Jens Axboe   [PATCH] generic d...
274
  	sector_t		end_sector;
8922e16cf   Tejun Heo   [PATCH] 01/05 Imp...
275
  	struct request		*boundary_rq;
8922e16cf   Tejun Heo   [PATCH] 01/05 Imp...
276
277
  
  	/*
3cca6dc1c   Jens Axboe   block: add API fo...
278
  	 * Delayed queue handling
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
279
  	 */
3cca6dc1c   Jens Axboe   block: add API fo...
280
  	struct delayed_work	delay_work;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
281
282
283
284
285
286
287
288
  
  	struct backing_dev_info	backing_dev_info;
  
  	/*
  	 * The queue owner gets to use this for whatever they like.
  	 * ll_rw_blk doesn't touch it.
  	 */
  	void			*queuedata;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
289
  	/*
d7b763013   Richard Kennedy   block: reorder re...
290
  	 * various queue flags, see QUEUE_* below
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
291
  	 */
d7b763013   Richard Kennedy   block: reorder re...
292
  	unsigned long		queue_flags;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
293
294
  
  	/*
a73f730d0   Tejun Heo   block, cfq: move ...
295
296
297
298
299
300
  	 * ida allocated id for this queue.  Used to index queues from
  	 * ioctx.
  	 */
  	int			id;
  
  	/*
d7b763013   Richard Kennedy   block: reorder re...
301
  	 * queue needs bounce pages for pages above this limit
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
302
  	 */
d7b763013   Richard Kennedy   block: reorder re...
303
  	gfp_t			bounce_gfp;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
304
305
  
  	/*
152587deb   Jens Axboe   [PATCH] fix NMI l...
306
307
308
  	 * protects queue structures from reentrancy. ->__queue_lock should
  	 * _never_ be used directly, it is queue private. always use
  	 * ->queue_lock.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
309
  	 */
152587deb   Jens Axboe   [PATCH] fix NMI l...
310
  	spinlock_t		__queue_lock;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
311
312
313
314
315
316
317
318
319
320
321
322
323
324
  	spinlock_t		*queue_lock;
  
  	/*
  	 * queue kobject
  	 */
  	struct kobject kobj;
  
  	/*
  	 * queue settings
  	 */
  	unsigned long		nr_requests;	/* Max # of requests */
  	unsigned int		nr_congestion_on;
  	unsigned int		nr_congestion_off;
  	unsigned int		nr_batching;
fa0ccd837   James Bottomley   block: implement ...
325
  	unsigned int		dma_drain_size;
d7b763013   Richard Kennedy   block: reorder re...
326
  	void			*dma_drain_buffer;
e3790c7d4   Tejun Heo   block: separate o...
327
  	unsigned int		dma_pad_mask;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
328
329
330
  	unsigned int		dma_alignment;
  
  	struct blk_queue_tag	*queue_tags;
6eca9004d   Jens Axboe   [BLOCK] Fix bad s...
331
  	struct list_head	tag_busy_list;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
332

15853af9f   Tejun Heo   [BLOCK] Implement...
333
  	unsigned int		nr_sorted;
0a7ae2ff0   Jens Axboe   block: change the...
334
  	unsigned int		in_flight[2];
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
335

242f9dcb8   Jens Axboe   block: unify requ...
336
337
338
  	unsigned int		rq_timeout;
  	struct timer_list	timeout;
  	struct list_head	timeout_list;
a612fddf0   Tejun Heo   block, cfq: move ...
339
  	struct list_head	icq_list;
025146e13   Martin K. Petersen   block: Move queue...
340
  	struct queue_limits	limits;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
341
342
343
344
345
  	/*
  	 * sg stuff
  	 */
  	unsigned int		sg_timeout;
  	unsigned int		sg_reserved_size;
1946089a1   Christoph Lameter   [PATCH] NUMA awar...
346
  	int			node;
6c5c93415   Alexey Dobriyan   [PATCH] ifdef blk...
347
  #ifdef CONFIG_BLK_DEV_IO_TRACE
2056a782f   Jens Axboe   [PATCH] Block que...
348
  	struct blk_trace	*blk_trace;
6c5c93415   Alexey Dobriyan   [PATCH] ifdef blk...
349
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
350
  	/*
4913efe45   Tejun Heo   block: deprecate ...
351
  	 * for flush operations
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
352
  	 */
4913efe45   Tejun Heo   block: deprecate ...
353
  	unsigned int		flush_flags;
f38769309   shaohua.li@intel.com   block: add a non-...
354
  	unsigned int		flush_not_queueable:1;
3ac0cc450   shaohua.li@intel.com   block: hold queue...
355
  	unsigned int		flush_queue_delayed:1;
ae1b15396   Tejun Heo   block: reimplemen...
356
357
358
359
360
  	unsigned int		flush_pending_idx:1;
  	unsigned int		flush_running_idx:1;
  	unsigned long		flush_pending_since;
  	struct list_head	flush_queue[2];
  	struct list_head	flush_data_in_flight;
dd4c133f3   Tejun Heo   block: rename bar...
361
  	struct request		flush_rq;
483f4afc4   Al Viro   [PATCH] fix sysfs...
362
363
  
  	struct mutex		sysfs_lock;
d351af01b   FUJITA Tomonori   bsg: bind bsg to ...
364
365
  
  #if defined(CONFIG_BLK_DEV_BSG)
aa387cc89   Mike Christie   block: add bsg he...
366
367
  	bsg_job_fn		*bsg_job_fn;
  	int			bsg_job_size;
d351af01b   FUJITA Tomonori   bsg: bind bsg to ...
368
369
  	struct bsg_class_device bsg_dev;
  #endif
e43473b7f   Vivek Goyal   blkio: Core imple...
370
371
372
373
374
  
  #ifdef CONFIG_BLK_DEV_THROTTLING
  	/* Throttle data */
  	struct throtl_data *td;
  #endif
b2efa0526   Tejun Heo   block, cfq: unlin...
375
376
377
  #ifdef CONFIG_LOCKDEP
  	int			ioc_release_depth;
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
378
  };
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
379
380
  #define QUEUE_FLAG_QUEUED	1	/* uses generic tag queueing */
  #define QUEUE_FLAG_STOPPED	2	/* queue is stopped */
1faa16d22   Jens Axboe   block: change the...
381
382
  #define	QUEUE_FLAG_SYNCFULL	3	/* read queue has been filled */
  #define QUEUE_FLAG_ASYNCFULL	4	/* write queue has been filled */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
383
  #define QUEUE_FLAG_DEAD		5	/* queue being torn down */
c21e6beba   Jens Axboe   block: get rid of...
384
385
386
  #define QUEUE_FLAG_ELVSWITCH	6	/* don't use elevator, just do FIFO */
  #define QUEUE_FLAG_BIDI		7	/* queue supports bidi requests */
  #define QUEUE_FLAG_NOMERGES     8	/* disable merge attempts */
5757a6d76   Dan Williams   block: strict rq_...
387
  #define QUEUE_FLAG_SAME_COMP	9	/* complete on same CPU-group */
c21e6beba   Jens Axboe   block: get rid of...
388
389
390
  #define QUEUE_FLAG_FAIL_IO     10	/* fake timeout */
  #define QUEUE_FLAG_STACKABLE   11	/* supports request stacking */
  #define QUEUE_FLAG_NONROT      12	/* non-rotational device (SSD) */
88e740f16   Fernando Luis Vázquez Cao   block: add queue ...
391
  #define QUEUE_FLAG_VIRT        QUEUE_FLAG_NONROT /* paravirt device */
c21e6beba   Jens Axboe   block: get rid of...
392
393
394
395
396
  #define QUEUE_FLAG_IO_STAT     13	/* do IO stats */
  #define QUEUE_FLAG_DISCARD     14	/* supports DISCARD */
  #define QUEUE_FLAG_NOXMERGES   15	/* No extended merges */
  #define QUEUE_FLAG_ADD_RANDOM  16	/* Contributes to random pool */
  #define QUEUE_FLAG_SECDISCARD  17	/* supports SECDISCARD */
5757a6d76   Dan Williams   block: strict rq_...
397
  #define QUEUE_FLAG_SAME_FORCE  18	/* force complete on same CPU */
bc58ba946   Jens Axboe   block: add sysfs ...
398
399
  
  #define QUEUE_FLAG_DEFAULT	((1 << QUEUE_FLAG_IO_STAT) |		\
01e97f6b8   Jens Axboe   block: enable rq ...
400
  				 (1 << QUEUE_FLAG_STACKABLE)	|	\
e2e1a148b   Jens Axboe   block: add sysfs ...
401
402
  				 (1 << QUEUE_FLAG_SAME_COMP)	|	\
  				 (1 << QUEUE_FLAG_ADD_RANDOM))
797e7dbbe   Tejun Heo   [BLOCK] reimpleme...
403

8f45c1a58   Linus Torvalds   block: fix queue ...
404
405
  static inline int queue_is_locked(struct request_queue *q)
  {
7663c1e27   Jens Axboe   Improve queue_is_...
406
  #ifdef CONFIG_SMP
8f45c1a58   Linus Torvalds   block: fix queue ...
407
408
  	spinlock_t *lock = q->queue_lock;
  	return lock && spin_is_locked(lock);
7663c1e27   Jens Axboe   Improve queue_is_...
409
410
411
  #else
  	return 1;
  #endif
8f45c1a58   Linus Torvalds   block: fix queue ...
412
  }
75ad23bc0   Nick Piggin   block: make queue...
413
414
415
416
417
  static inline void queue_flag_set_unlocked(unsigned int flag,
  					   struct request_queue *q)
  {
  	__set_bit(flag, &q->queue_flags);
  }
e48ec6900   Jens Axboe   block: extend que...
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
  static inline int queue_flag_test_and_clear(unsigned int flag,
  					    struct request_queue *q)
  {
  	WARN_ON_ONCE(!queue_is_locked(q));
  
  	if (test_bit(flag, &q->queue_flags)) {
  		__clear_bit(flag, &q->queue_flags);
  		return 1;
  	}
  
  	return 0;
  }
  
  static inline int queue_flag_test_and_set(unsigned int flag,
  					  struct request_queue *q)
  {
  	WARN_ON_ONCE(!queue_is_locked(q));
  
  	if (!test_bit(flag, &q->queue_flags)) {
  		__set_bit(flag, &q->queue_flags);
  		return 0;
  	}
  
  	return 1;
  }
75ad23bc0   Nick Piggin   block: make queue...
443
444
  static inline void queue_flag_set(unsigned int flag, struct request_queue *q)
  {
8f45c1a58   Linus Torvalds   block: fix queue ...
445
  	WARN_ON_ONCE(!queue_is_locked(q));
75ad23bc0   Nick Piggin   block: make queue...
446
447
448
449
450
451
452
453
  	__set_bit(flag, &q->queue_flags);
  }
  
  static inline void queue_flag_clear_unlocked(unsigned int flag,
  					     struct request_queue *q)
  {
  	__clear_bit(flag, &q->queue_flags);
  }
0a7ae2ff0   Jens Axboe   block: change the...
454
455
456
457
  static inline int queue_in_flight(struct request_queue *q)
  {
  	return q->in_flight[0] + q->in_flight[1];
  }
75ad23bc0   Nick Piggin   block: make queue...
458
459
  static inline void queue_flag_clear(unsigned int flag, struct request_queue *q)
  {
8f45c1a58   Linus Torvalds   block: fix queue ...
460
  	WARN_ON_ONCE(!queue_is_locked(q));
75ad23bc0   Nick Piggin   block: make queue...
461
462
  	__clear_bit(flag, &q->queue_flags);
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
463
464
  #define blk_queue_tagged(q)	test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
  #define blk_queue_stopped(q)	test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
34f6055c8   Tejun Heo   block: add blk_qu...
465
  #define blk_queue_dead(q)	test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
ac9fafa12   Alan D. Brunelle   block: Skip I/O m...
466
  #define blk_queue_nomerges(q)	test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
488991e28   Alan D. Brunelle   block: Added in s...
467
468
  #define blk_queue_noxmerges(q)	\
  	test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
a68bbddba   Jens Axboe   block: add queue ...
469
  #define blk_queue_nonrot(q)	test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
bc58ba946   Jens Axboe   block: add sysfs ...
470
  #define blk_queue_io_stat(q)	test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
e2e1a148b   Jens Axboe   block: add sysfs ...
471
  #define blk_queue_add_random(q)	test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
4ee5eaf45   Kiyoshi Ueda   block: add a queu...
472
473
  #define blk_queue_stackable(q)	\
  	test_bit(QUEUE_FLAG_STACKABLE, &(q)->queue_flags)
c15227de1   Christoph Hellwig   block: use normal...
474
  #define blk_queue_discard(q)	test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
8d57a98cc   Adrian Hunter   block: add secure...
475
476
  #define blk_queue_secdiscard(q)	(blk_queue_discard(q) && \
  	test_bit(QUEUE_FLAG_SECDISCARD, &(q)->queue_flags))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
477

33659ebba   Christoph Hellwig   block: remove wra...
478
479
480
481
482
483
484
485
  #define blk_noretry_request(rq) \
  	((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
  			     REQ_FAILFAST_DRIVER))
  
  #define blk_account_rq(rq) \
  	(((rq)->cmd_flags & REQ_STARTED) && \
  	 ((rq)->cmd_type == REQ_TYPE_FS || \
  	  ((rq)->cmd_flags & REQ_DISCARD)))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
486
  #define blk_pm_request(rq)	\
33659ebba   Christoph Hellwig   block: remove wra...
487
488
  	((rq)->cmd_type == REQ_TYPE_PM_SUSPEND || \
  	 (rq)->cmd_type == REQ_TYPE_PM_RESUME)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
489

ab780f1ec   Jens Axboe   block: inherit CP...
490
  #define blk_rq_cpu_valid(rq)	((rq)->cpu != -1)
abae1fde6   FUJITA Tomonori   add a struct requ...
491
  #define blk_bidi_rq(rq)		((rq)->next_rq != NULL)
336cdb400   Kiyoshi Ueda   blk_end_request: ...
492
493
  /* rq->queuelist of dequeued request must be list_empty() */
  #define blk_queued_rq(rq)	(!list_empty(&(rq)->queuelist))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
494
495
  
  #define list_entry_rq(ptr)	list_entry((ptr), struct request, queuelist)
4aff5e233   Jens Axboe   [PATCH] Split str...
496
  #define rq_data_dir(rq)		((rq)->cmd_flags & 1)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
497

e692cb668   Martin K. Petersen   block: Deprecate ...
498
499
500
501
  static inline unsigned int blk_queue_cluster(struct request_queue *q)
  {
  	return q->limits.cluster;
  }
9e2585a8a   Jens Axboe   [PATCH] as-iosche...
502
  /*
1faa16d22   Jens Axboe   block: change the...
503
   * We regard a request as sync, if either a read or a sync write
9e2585a8a   Jens Axboe   [PATCH] as-iosche...
504
   */
1faa16d22   Jens Axboe   block: change the...
505
506
  static inline bool rw_is_sync(unsigned int rw_flags)
  {
7b6d91dae   Christoph Hellwig   block: unify flag...
507
  	return !(rw_flags & REQ_WRITE) || (rw_flags & REQ_SYNC);
1faa16d22   Jens Axboe   block: change the...
508
509
510
511
512
513
  }
  
  static inline bool rq_is_sync(struct request *rq)
  {
  	return rw_is_sync(rq->cmd_flags);
  }
1faa16d22   Jens Axboe   block: change the...
514
  static inline int blk_queue_full(struct request_queue *q, int sync)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
515
  {
1faa16d22   Jens Axboe   block: change the...
516
517
518
  	if (sync)
  		return test_bit(QUEUE_FLAG_SYNCFULL, &q->queue_flags);
  	return test_bit(QUEUE_FLAG_ASYNCFULL, &q->queue_flags);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
519
  }
1faa16d22   Jens Axboe   block: change the...
520
  static inline void blk_set_queue_full(struct request_queue *q, int sync)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
521
  {
1faa16d22   Jens Axboe   block: change the...
522
523
  	if (sync)
  		queue_flag_set(QUEUE_FLAG_SYNCFULL, q);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
524
  	else
1faa16d22   Jens Axboe   block: change the...
525
  		queue_flag_set(QUEUE_FLAG_ASYNCFULL, q);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
526
  }
1faa16d22   Jens Axboe   block: change the...
527
  static inline void blk_clear_queue_full(struct request_queue *q, int sync)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
528
  {
1faa16d22   Jens Axboe   block: change the...
529
530
  	if (sync)
  		queue_flag_clear(QUEUE_FLAG_SYNCFULL, q);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
531
  	else
1faa16d22   Jens Axboe   block: change the...
532
  		queue_flag_clear(QUEUE_FLAG_ASYNCFULL, q);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
533
534
535
536
537
538
539
540
  }
  
  
  /*
   * mergeable request must not have _NOMERGE or _BARRIER bit set, nor may
   * it already be started by driver.
   */
  #define RQ_NOMERGE_FLAGS	\
02e031cbc   Christoph Hellwig   block: remove REQ...
541
  	(REQ_NOMERGE | REQ_STARTED | REQ_SOFTBARRIER | REQ_FLUSH | REQ_FUA)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
542
  #define rq_mergeable(rq)	\
e17fc0a1c   David Woodhouse   Allow elevators t...
543
  	(!((rq)->cmd_flags & RQ_NOMERGE_FLAGS) && \
33659ebba   Christoph Hellwig   block: remove wra...
544
545
  	 (((rq)->cmd_flags & REQ_DISCARD) || \
  	  (rq)->cmd_type == REQ_TYPE_FS))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
546
547
  
  /*
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
   * q->prep_rq_fn return values
   */
  #define BLKPREP_OK		0	/* serve it */
  #define BLKPREP_KILL		1	/* fatal error, kill */
  #define BLKPREP_DEFER		2	/* leave on queue */
  
  extern unsigned long blk_max_low_pfn, blk_max_pfn;
  
  /*
   * standard bounce addresses:
   *
   * BLK_BOUNCE_HIGH	: bounce all highmem pages
   * BLK_BOUNCE_ANY	: don't bounce anything
   * BLK_BOUNCE_ISA	: bounce pages above ISA DMA boundary
   */
2472892a3   Andi Kleen   block: fix memory...
563
564
  
  #if BITS_PER_LONG == 32
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
565
  #define BLK_BOUNCE_HIGH		((u64)blk_max_low_pfn << PAGE_SHIFT)
2472892a3   Andi Kleen   block: fix memory...
566
567
568
569
  #else
  #define BLK_BOUNCE_HIGH		-1ULL
  #endif
  #define BLK_BOUNCE_ANY		(-1ULL)
bfe172310   FUJITA Tomonori   block: kill ISA_D...
570
  #define BLK_BOUNCE_ISA		(DMA_BIT_MASK(24))
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
571

3d6392cfb   Jens Axboe   bsg: support for ...
572
573
574
575
  /*
   * default timeout for SG_IO if none specified
   */
  #define BLK_DEFAULT_SG_TIMEOUT	(60 * HZ)
f2f1fa78a   Linus Torvalds   Enforce a minimum...
576
  #define BLK_MIN_SG_TIMEOUT	(7 * HZ)
3d6392cfb   Jens Axboe   bsg: support for ...
577

2a7326b5b   Christoph Lameter   CONFIG_BOUNCE to ...
578
  #ifdef CONFIG_BOUNCE
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
579
  extern int init_emergency_isa_pool(void);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
580
  extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
581
582
583
584
585
  #else
  static inline int init_emergency_isa_pool(void)
  {
  	return 0;
  }
165125e1e   Jens Axboe   [BLOCK] Get rid o...
586
  static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
587
588
589
  {
  }
  #endif /* CONFIG_MMU */
152e283fd   FUJITA Tomonori   block: introduce ...
590
591
592
593
  struct rq_map_data {
  	struct page **pages;
  	int page_order;
  	int nr_entries;
56c451f4b   FUJITA Tomonori   [SCSI] block: fix...
594
  	unsigned long offset;
97ae77a1c   FUJITA Tomonori   [SCSI] block: mak...
595
  	int null_mapped;
ecb554a84   FUJITA Tomonori   block: fix sg SG_...
596
  	int from_user;
152e283fd   FUJITA Tomonori   block: introduce ...
597
  };
5705f7021   NeilBrown   Introduce rq_for_...
598
599
600
601
602
603
  struct req_iterator {
  	int i;
  	struct bio *bio;
  };
  
  /* This should not be used directly - use rq_for_each_segment */
1e4280791   Jens Axboe   block: reduce sta...
604
605
  #define for_each_bio(_bio)		\
  	for (; _bio; _bio = _bio->bi_next)
5705f7021   NeilBrown   Introduce rq_for_...
606
  #define __rq_for_each_bio(_bio, rq)	\
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
607
608
  	if ((rq->bio))			\
  		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
5705f7021   NeilBrown   Introduce rq_for_...
609
610
611
612
613
614
  #define rq_for_each_segment(bvl, _rq, _iter)			\
  	__rq_for_each_bio(_iter.bio, _rq)			\
  		bio_for_each_segment(bvl, _iter.bio, _iter.i)
  
  #define rq_iter_last(rq, _iter)					\
  		(_iter.bio->bi_next == NULL && _iter.i == _iter.bio->bi_vcnt-1)
2d4dc890b   Ilya Loginov   block: add helper...
615
616
617
618
619
620
621
622
623
624
  #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
  # error	"You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
  #endif
  #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
  extern void rq_flush_dcache_pages(struct request *rq);
  #else
  static inline void rq_flush_dcache_pages(struct request *rq)
  {
  }
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
625
626
  extern int blk_register_queue(struct gendisk *disk);
  extern void blk_unregister_queue(struct gendisk *disk);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
627
  extern void generic_make_request(struct bio *bio);
2a4aa30c5   FUJITA Tomonori   block: rename and...
628
  extern void blk_rq_init(struct request_queue *q, struct request *rq);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
629
  extern void blk_put_request(struct request *);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
630
  extern void __blk_put_request(struct request_queue *, struct request *);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
631
  extern struct request *blk_get_request(struct request_queue *, int, gfp_t);
79eb63e9e   Boaz Harrosh   block: Add blk_ma...
632
633
  extern struct request *blk_make_request(struct request_queue *, struct bio *,
  					gfp_t);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
634
  extern void blk_requeue_request(struct request_queue *, struct request *);
66ac02801   Christoph Hellwig   block: don't allo...
635
636
  extern void blk_add_request_payload(struct request *rq, struct page *page,
  		unsigned int len);
82124d603   Kiyoshi Ueda   block: add reques...
637
  extern int blk_rq_check_limits(struct request_queue *q, struct request *rq);
ef9e3facd   Kiyoshi Ueda   block: add lld bu...
638
  extern int blk_lld_busy(struct request_queue *q);
b0fd271d5   Kiyoshi Ueda   block: add reques...
639
640
641
642
643
  extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
  			     struct bio_set *bs, gfp_t gfp_mask,
  			     int (*bio_ctr)(struct bio *, struct bio *, void *),
  			     void *data);
  extern void blk_rq_unprep_clone(struct request *rq);
82124d603   Kiyoshi Ueda   block: add reques...
644
645
  extern int blk_insert_cloned_request(struct request_queue *q,
  				     struct request *rq);
3cca6dc1c   Jens Axboe   block: add API fo...
646
  extern void blk_delay_queue(struct request_queue *, unsigned long);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
647
  extern void blk_recount_segments(struct request_queue *, struct bio *);
0bfc96cb7   Paolo Bonzini   block: fail SCSI ...
648
  extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
577ebb374   Paolo Bonzini   block: add and us...
649
650
  extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
  			      unsigned int, void __user *);
74f3c8aff   Al Viro   [PATCH] switch sc...
651
652
  extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
  			  unsigned int, void __user *);
e915e872e   Al Viro   [PATCH] switch sg...
653
654
  extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
  			 struct scsi_ioctl_command __user *);
3fcfab16c   Andrew Morton   [PATCH] separate ...
655

5a7bbad27   Christoph Hellwig   block: remove sup...
656
  extern void blk_queue_bio(struct request_queue *q, struct bio *bio);
166e1f901   Christoph Hellwig   block: export __m...
657

3fcfab16c   Andrew Morton   [PATCH] separate ...
658
659
660
661
662
  /*
   * A queue has just exitted congestion.  Note this in the global counter of
   * congested queues, and wake up anyone who was waiting for requests to be
   * put back.
   */
8aa7e847d   Jens Axboe   Fix congestion_wa...
663
  static inline void blk_clear_queue_congested(struct request_queue *q, int sync)
3fcfab16c   Andrew Morton   [PATCH] separate ...
664
  {
8aa7e847d   Jens Axboe   Fix congestion_wa...
665
  	clear_bdi_congested(&q->backing_dev_info, sync);
3fcfab16c   Andrew Morton   [PATCH] separate ...
666
667
668
669
670
671
  }
  
  /*
   * A queue has just entered congestion.  Flag that in the queue's VM-visible
   * state flags and increment the global gounter of congested queues.
   */
8aa7e847d   Jens Axboe   Fix congestion_wa...
672
  static inline void blk_set_queue_congested(struct request_queue *q, int sync)
3fcfab16c   Andrew Morton   [PATCH] separate ...
673
  {
8aa7e847d   Jens Axboe   Fix congestion_wa...
674
  	set_bdi_congested(&q->backing_dev_info, sync);
3fcfab16c   Andrew Morton   [PATCH] separate ...
675
  }
165125e1e   Jens Axboe   [BLOCK] Get rid o...
676
677
  extern void blk_start_queue(struct request_queue *q);
  extern void blk_stop_queue(struct request_queue *q);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
678
  extern void blk_sync_queue(struct request_queue *q);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
679
  extern void __blk_stop_queue(struct request_queue *q);
24ecfbe27   Christoph Hellwig   block: add blk_ru...
680
  extern void __blk_run_queue(struct request_queue *q);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
681
  extern void blk_run_queue(struct request_queue *);
c21e6beba   Jens Axboe   block: get rid of...
682
  extern void blk_run_queue_async(struct request_queue *q);
a3bce90ed   FUJITA Tomonori   block: add gfp_ma...
683
  extern int blk_rq_map_user(struct request_queue *, struct request *,
152e283fd   FUJITA Tomonori   block: introduce ...
684
685
  			   struct rq_map_data *, void __user *, unsigned long,
  			   gfp_t);
8e5cfc45e   Jens Axboe   [PATCH] Fixup blk...
686
  extern int blk_rq_unmap_user(struct bio *);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
687
688
  extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
  extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
152e283fd   FUJITA Tomonori   block: introduce ...
689
690
  			       struct rq_map_data *, struct sg_iovec *, int,
  			       unsigned int, gfp_t);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
691
  extern int blk_execute_rq(struct request_queue *, struct gendisk *,
994ca9a19   James Bottomley   [PATCH] update bl...
692
  			  struct request *, int);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
693
  extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
15fc858a0   Jens Axboe   [BLOCK] Correct b...
694
  				  struct request *, int, rq_end_io_fn *);
6e39b69e7   Mike Christie   [SCSI] export blk...
695

165125e1e   Jens Axboe   [BLOCK] Get rid o...
696
  static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
697
698
699
  {
  	return bdev->bd_disk->queue;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
700
  /*
80a761fd3   Tejun Heo   block: implement ...
701
702
703
704
705
706
   * blk_rq_pos()			: the current sector
   * blk_rq_bytes()		: bytes left in the entire request
   * blk_rq_cur_bytes()		: bytes left in the current segment
   * blk_rq_err_bytes()		: bytes left till the next error boundary
   * blk_rq_sectors()		: sectors left in the entire request
   * blk_rq_cur_sectors()		: sectors left in the current segment
5efccd17c   Tejun Heo   block: reorder re...
707
   */
5b93629b4   Tejun Heo   block: implement ...
708
709
  static inline sector_t blk_rq_pos(const struct request *rq)
  {
a2dec7b36   Tejun Heo   block: hide reque...
710
  	return rq->__sector;
2e46e8b27   Tejun Heo   block: drop reque...
711
712
713
714
  }
  
  static inline unsigned int blk_rq_bytes(const struct request *rq)
  {
a2dec7b36   Tejun Heo   block: hide reque...
715
  	return rq->__data_len;
5b93629b4   Tejun Heo   block: implement ...
716
  }
2e46e8b27   Tejun Heo   block: drop reque...
717
718
719
720
  static inline int blk_rq_cur_bytes(const struct request *rq)
  {
  	return rq->bio ? bio_cur_bytes(rq->bio) : 0;
  }
5efccd17c   Tejun Heo   block: reorder re...
721

80a761fd3   Tejun Heo   block: implement ...
722
  extern unsigned int blk_rq_err_bytes(const struct request *rq);
5b93629b4   Tejun Heo   block: implement ...
723
724
  static inline unsigned int blk_rq_sectors(const struct request *rq)
  {
2e46e8b27   Tejun Heo   block: drop reque...
725
  	return blk_rq_bytes(rq) >> 9;
5b93629b4   Tejun Heo   block: implement ...
726
727
728
729
  }
  
  static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
  {
2e46e8b27   Tejun Heo   block: drop reque...
730
  	return blk_rq_cur_bytes(rq) >> 9;
5b93629b4   Tejun Heo   block: implement ...
731
  }
5efccd17c   Tejun Heo   block: reorder re...
732
  /*
9934c8c04   Tejun Heo   block: implement ...
733
734
735
736
737
738
739
   * Request issue related functions.
   */
  extern struct request *blk_peek_request(struct request_queue *q);
  extern void blk_start_request(struct request *rq);
  extern struct request *blk_fetch_request(struct request_queue *q);
  
  /*
2e60e0229   Tejun Heo   block: clean up r...
740
741
742
743
744
   * Request completion related functions.
   *
   * blk_update_request() completes given number of bytes and updates
   * the request without completing it.
   *
f06d9a2b5   Tejun Heo   block: replace en...
745
746
   * blk_end_request() and friends.  __blk_end_request() must be called
   * with the request queue spinlock acquired.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
747
748
   *
   * Several drivers define their own end_request and call
3bcddeac1   Kiyoshi Ueda   blk_end_request: ...
749
750
   * blk_end_request() for parts of the original function.
   * This prevents code duplication in drivers.
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
751
   */
2e60e0229   Tejun Heo   block: clean up r...
752
753
  extern bool blk_update_request(struct request *rq, int error,
  			       unsigned int nr_bytes);
b1f744937   FUJITA Tomonori   block: move compl...
754
755
756
757
  extern bool blk_end_request(struct request *rq, int error,
  			    unsigned int nr_bytes);
  extern void blk_end_request_all(struct request *rq, int error);
  extern bool blk_end_request_cur(struct request *rq, int error);
80a761fd3   Tejun Heo   block: implement ...
758
  extern bool blk_end_request_err(struct request *rq, int error);
b1f744937   FUJITA Tomonori   block: move compl...
759
760
761
762
  extern bool __blk_end_request(struct request *rq, int error,
  			      unsigned int nr_bytes);
  extern void __blk_end_request_all(struct request *rq, int error);
  extern bool __blk_end_request_cur(struct request *rq, int error);
80a761fd3   Tejun Heo   block: implement ...
763
  extern bool __blk_end_request_err(struct request *rq, int error);
2e60e0229   Tejun Heo   block: clean up r...
764

ff856bad6   Jens Axboe   [BLOCK] ll_rw_blk...
765
  extern void blk_complete_request(struct request *);
242f9dcb8   Jens Axboe   block: unify requ...
766
767
  extern void __blk_complete_request(struct request *);
  extern void blk_abort_request(struct request *);
11914a53d   Mike Anderson   block: Add interf...
768
  extern void blk_abort_queue(struct request_queue *);
28018c242   James Bottomley   block: implement ...
769
  extern void blk_unprep_request(struct request *);
ff856bad6   Jens Axboe   [BLOCK] ll_rw_blk...
770

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
771
  /*
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
772
773
   * Access functions for manipulating queue properties
   */
165125e1e   Jens Axboe   [BLOCK] Get rid o...
774
  extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
1946089a1   Christoph Lameter   [PATCH] NUMA awar...
775
  					spinlock_t *lock, int node_id);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
776
  extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
01effb0dc   Mike Snitzer   block: allow init...
777
778
  extern struct request_queue *blk_init_allocated_queue(struct request_queue *,
  						      request_fn_proc *, spinlock_t *);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
779
780
781
  extern void blk_cleanup_queue(struct request_queue *);
  extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
  extern void blk_queue_bounce_limit(struct request_queue *, u64);
72d4cd9f3   Mike Snitzer   block: max hardwa...
782
  extern void blk_limits_max_hw_sectors(struct queue_limits *, unsigned int);
086fa5ff0   Martin K. Petersen   block: Rename blk...
783
  extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
8a78362c4   Martin K. Petersen   block: Consolidat...
784
  extern void blk_queue_max_segments(struct request_queue *, unsigned short);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
785
  extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
67efc9258   Christoph Hellwig   block: allow larg...
786
787
  extern void blk_queue_max_discard_sectors(struct request_queue *q,
  		unsigned int max_discard_sectors);
e1defc4ff   Martin K. Petersen   block: Do away wi...
788
  extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
892b6f90d   Martin K. Petersen   block: Ensure phy...
789
  extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
c72758f33   Martin K. Petersen   block: Export I/O...
790
791
  extern void blk_queue_alignment_offset(struct request_queue *q,
  				       unsigned int alignment);
7c958e326   Martin K. Petersen   block: Add a wrap...
792
  extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
c72758f33   Martin K. Petersen   block: Export I/O...
793
  extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
3c5820c74   Martin K. Petersen   block: Optimal I/...
794
  extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
c72758f33   Martin K. Petersen   block: Export I/O...
795
  extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
e475bba2f   Martin K. Petersen   block: Introduce ...
796
  extern void blk_set_default_limits(struct queue_limits *lim);
b1bd055d3   Martin K. Petersen   block: Introduce ...
797
  extern void blk_set_stacking_limits(struct queue_limits *lim);
c72758f33   Martin K. Petersen   block: Export I/O...
798
799
  extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
  			    sector_t offset);
17be8c245   Martin K. Petersen   block: bdev_stack...
800
801
  extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
  			    sector_t offset);
c72758f33   Martin K. Petersen   block: Export I/O...
802
803
  extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
  			      sector_t offset);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
804
  extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
e3790c7d4   Tejun Heo   block: separate o...
805
  extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
27f8221af   FUJITA Tomonori   block: add blk_qu...
806
  extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
2fb98e841   Tejun Heo   block: implement ...
807
808
809
  extern int blk_queue_dma_drain(struct request_queue *q,
  			       dma_drain_needed_fn *dma_drain_needed,
  			       void *buf, unsigned int size);
ef9e3facd   Kiyoshi Ueda   block: add lld bu...
810
  extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
811
812
  extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
  extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
28018c242   James Bottomley   block: implement ...
813
  extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
814
815
  extern void blk_queue_merge_bvec(struct request_queue *, merge_bvec_fn *);
  extern void blk_queue_dma_alignment(struct request_queue *, int);
11c3e689f   James Bottomley   [SCSI] block: Int...
816
  extern void blk_queue_update_dma_alignment(struct request_queue *, int);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
817
  extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
242f9dcb8   Jens Axboe   block: unify requ...
818
819
  extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
  extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
4913efe45   Tejun Heo   block: deprecate ...
820
  extern void blk_queue_flush(struct request_queue *q, unsigned int flush);
f38769309   shaohua.li@intel.com   block: add a non-...
821
  extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
822
  extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
823

165125e1e   Jens Axboe   [BLOCK] Get rid o...
824
  extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
825
  extern void blk_dump_rq_flags(struct request *, char *);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
826
  extern long nr_blockdev_pages(void);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
827

09ac46c42   Tejun Heo   block: misc updat...
828
  bool __must_check blk_get_queue(struct request_queue *);
165125e1e   Jens Axboe   [BLOCK] Get rid o...
829
830
831
  struct request_queue *blk_alloc_queue(gfp_t);
  struct request_queue *blk_alloc_queue_node(gfp_t, int);
  extern void blk_put_queue(struct request_queue *);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
832

316cc67d5   Shaohua Li   block: document b...
833
  /*
75df71362   Suresh Jayaraman   block: document b...
834
835
836
837
838
839
840
841
842
843
   * blk_plug permits building a queue of related requests by holding the I/O
   * fragments for a short period. This allows merging of sequential requests
   * into single larger request. As the requests are moved from a per-task list to
   * the device's request_queue in a batch, this results in improved scalability
   * as the lock contention for request_queue lock is reduced.
   *
   * It is ok not to disable preemption when adding the request to the plug list
   * or when attempting a merge, because blk_schedule_flush_list() will only flush
   * the plug list when the task sleeps by itself. For details, please see
   * schedule() where blk_schedule_flush_plug() is called.
316cc67d5   Shaohua Li   block: document b...
844
   */
73c101011   Jens Axboe   block: initial pa...
845
  struct blk_plug {
75df71362   Suresh Jayaraman   block: document b...
846
847
848
849
  	unsigned long magic; /* detect uninitialized use-cases */
  	struct list_head list; /* requests */
  	struct list_head cb_list; /* md requires an unplug callback */
  	unsigned int should_sort; /* list to be sorted before flushing? */
73c101011   Jens Axboe   block: initial pa...
850
  };
55c022bbd   Shaohua Li   block: avoid buil...
851
  #define BLK_MAX_REQUEST_COUNT 16
048c9374a   NeilBrown   block: Enhance ne...
852
853
854
855
  struct blk_plug_cb {
  	struct list_head list;
  	void (*callback)(struct blk_plug_cb *);
  };
73c101011   Jens Axboe   block: initial pa...
856
857
858
  
  extern void blk_start_plug(struct blk_plug *);
  extern void blk_finish_plug(struct blk_plug *);
f6603783f   Jens Axboe   block: only force...
859
  extern void blk_flush_plug_list(struct blk_plug *, bool);
73c101011   Jens Axboe   block: initial pa...
860
861
862
863
  
  static inline void blk_flush_plug(struct task_struct *tsk)
  {
  	struct blk_plug *plug = tsk->plug;
88b996cd0   Christoph Hellwig   block: cleanup th...
864
  	if (plug)
a237c1c5b   Jens Axboe   block: let io_sch...
865
866
867
868
869
870
871
872
  		blk_flush_plug_list(plug, false);
  }
  
  static inline void blk_schedule_flush_plug(struct task_struct *tsk)
  {
  	struct blk_plug *plug = tsk->plug;
  
  	if (plug)
f6603783f   Jens Axboe   block: only force...
873
  		blk_flush_plug_list(plug, true);
73c101011   Jens Axboe   block: initial pa...
874
875
876
877
878
  }
  
  static inline bool blk_needs_flush_plug(struct task_struct *tsk)
  {
  	struct blk_plug *plug = tsk->plug;
048c9374a   NeilBrown   block: Enhance ne...
879
  	return plug && (!list_empty(&plug->list) || !list_empty(&plug->cb_list));
73c101011   Jens Axboe   block: initial pa...
880
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
881
882
883
  /*
   * tag stuff
   */
4aff5e233   Jens Axboe   [PATCH] Split str...
884
  #define blk_rq_tagged(rq)		((rq)->cmd_flags & REQ_QUEUED)
165125e1e   Jens Axboe   [BLOCK] Get rid o...
885
886
887
888
889
890
891
  extern int blk_queue_start_tag(struct request_queue *, struct request *);
  extern struct request *blk_queue_find_tag(struct request_queue *, int);
  extern void blk_queue_end_tag(struct request_queue *, struct request *);
  extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *);
  extern void blk_queue_free_tags(struct request_queue *);
  extern int blk_queue_resize_tags(struct request_queue *, int);
  extern void blk_queue_invalidate_tags(struct request_queue *);
492dfb489   James Bottomley   [SCSI] block: add...
892
893
  extern struct blk_queue_tag *blk_init_tags(int);
  extern void blk_free_tags(struct blk_queue_tag *);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
894

f583f4924   David C Somayajulu   [PATCH] helper fu...
895
896
897
898
899
900
901
  static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
  						int tag)
  {
  	if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
  		return NULL;
  	return bqt->tag_index[tag];
  }
dd3932edd   Christoph Hellwig   block: remove BLK...
902
903
904
905
  
  #define BLKDEV_DISCARD_SECURE  0x01    /* secure discard */
  
  extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
fbd9b09a1   Dmitry Monakhov   blkdev: generaliz...
906
907
  extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
  		sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
3f14d792f   Dmitry Monakhov   blkdev: add blkde...
908
  extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
dd3932edd   Christoph Hellwig   block: remove BLK...
909
  			sector_t nr_sects, gfp_t gfp_mask);
2cf6d26a3   Christoph Hellwig   block: pass gfp_m...
910
911
  static inline int sb_issue_discard(struct super_block *sb, sector_t block,
  		sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
fb2dce862   David Woodhouse   Add 'discard' req...
912
  {
2cf6d26a3   Christoph Hellwig   block: pass gfp_m...
913
914
915
  	return blkdev_issue_discard(sb->s_bdev, block << (sb->s_blocksize_bits - 9),
  				    nr_blocks << (sb->s_blocksize_bits - 9),
  				    gfp_mask, flags);
fb2dce862   David Woodhouse   Add 'discard' req...
916
  }
e6fa0be69   Lukas Czerner   Add helper functi...
917
  static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
a107e5a3a   Theodore Ts'o   Merge branch 'nex...
918
  		sector_t nr_blocks, gfp_t gfp_mask)
e6fa0be69   Lukas Czerner   Add helper functi...
919
920
921
922
  {
  	return blkdev_issue_zeroout(sb->s_bdev,
  				    block << (sb->s_blocksize_bits - 9),
  				    nr_blocks << (sb->s_blocksize_bits - 9),
a107e5a3a   Theodore Ts'o   Merge branch 'nex...
923
  				    gfp_mask);
e6fa0be69   Lukas Czerner   Add helper functi...
924
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
925

018e04468   Jens Axboe   block: get rid of...
926
  extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm);
0b07de85a   Adel Gadllah   allow userspace t...
927

eb28d31bc   Martin K. Petersen   block: Add BLK_ p...
928
929
930
931
932
933
934
  enum blk_default_limits {
  	BLK_MAX_SEGMENTS	= 128,
  	BLK_SAFE_MAX_SECTORS	= 255,
  	BLK_DEF_MAX_SECTORS	= 1024,
  	BLK_MAX_SEGMENT_SIZE	= 65536,
  	BLK_SEG_BOUNDARY_MASK	= 0xFFFFFFFFUL,
  };
0e435ac26   Milan Broz   block: fix settin...
935

1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
936
  #define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)
ae03bf639   Martin K. Petersen   block: Use access...
937
938
  static inline unsigned long queue_bounce_pfn(struct request_queue *q)
  {
025146e13   Martin K. Petersen   block: Move queue...
939
  	return q->limits.bounce_pfn;
ae03bf639   Martin K. Petersen   block: Use access...
940
941
942
943
  }
  
  static inline unsigned long queue_segment_boundary(struct request_queue *q)
  {
025146e13   Martin K. Petersen   block: Move queue...
944
  	return q->limits.seg_boundary_mask;
ae03bf639   Martin K. Petersen   block: Use access...
945
946
947
948
  }
  
  static inline unsigned int queue_max_sectors(struct request_queue *q)
  {
025146e13   Martin K. Petersen   block: Move queue...
949
  	return q->limits.max_sectors;
ae03bf639   Martin K. Petersen   block: Use access...
950
951
952
953
  }
  
  static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
  {
025146e13   Martin K. Petersen   block: Move queue...
954
  	return q->limits.max_hw_sectors;
ae03bf639   Martin K. Petersen   block: Use access...
955
  }
8a78362c4   Martin K. Petersen   block: Consolidat...
956
  static inline unsigned short queue_max_segments(struct request_queue *q)
ae03bf639   Martin K. Petersen   block: Use access...
957
  {
8a78362c4   Martin K. Petersen   block: Consolidat...
958
  	return q->limits.max_segments;
ae03bf639   Martin K. Petersen   block: Use access...
959
960
961
962
  }
  
  static inline unsigned int queue_max_segment_size(struct request_queue *q)
  {
025146e13   Martin K. Petersen   block: Move queue...
963
  	return q->limits.max_segment_size;
ae03bf639   Martin K. Petersen   block: Use access...
964
  }
e1defc4ff   Martin K. Petersen   block: Do away wi...
965
  static inline unsigned short queue_logical_block_size(struct request_queue *q)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
966
967
  {
  	int retval = 512;
025146e13   Martin K. Petersen   block: Move queue...
968
969
  	if (q && q->limits.logical_block_size)
  		retval = q->limits.logical_block_size;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
970
971
972
  
  	return retval;
  }
e1defc4ff   Martin K. Petersen   block: Do away wi...
973
  static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
974
  {
e1defc4ff   Martin K. Petersen   block: Do away wi...
975
  	return queue_logical_block_size(bdev_get_queue(bdev));
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
976
  }
c72758f33   Martin K. Petersen   block: Export I/O...
977
978
979
980
  static inline unsigned int queue_physical_block_size(struct request_queue *q)
  {
  	return q->limits.physical_block_size;
  }
892b6f90d   Martin K. Petersen   block: Ensure phy...
981
  static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
ac481c20e   Martin K. Petersen   block: Topology i...
982
983
984
  {
  	return queue_physical_block_size(bdev_get_queue(bdev));
  }
c72758f33   Martin K. Petersen   block: Export I/O...
985
986
987
988
  static inline unsigned int queue_io_min(struct request_queue *q)
  {
  	return q->limits.io_min;
  }
ac481c20e   Martin K. Petersen   block: Topology i...
989
990
991
992
  static inline int bdev_io_min(struct block_device *bdev)
  {
  	return queue_io_min(bdev_get_queue(bdev));
  }
c72758f33   Martin K. Petersen   block: Export I/O...
993
994
995
996
  static inline unsigned int queue_io_opt(struct request_queue *q)
  {
  	return q->limits.io_opt;
  }
ac481c20e   Martin K. Petersen   block: Topology i...
997
998
999
1000
  static inline int bdev_io_opt(struct block_device *bdev)
  {
  	return queue_io_opt(bdev_get_queue(bdev));
  }
c72758f33   Martin K. Petersen   block: Export I/O...
1001
1002
  static inline int queue_alignment_offset(struct request_queue *q)
  {
ac481c20e   Martin K. Petersen   block: Topology i...
1003
  	if (q->limits.misaligned)
c72758f33   Martin K. Petersen   block: Export I/O...
1004
  		return -1;
ac481c20e   Martin K. Petersen   block: Topology i...
1005
  	return q->limits.alignment_offset;
c72758f33   Martin K. Petersen   block: Export I/O...
1006
  }
e03a72e13   Martin K. Petersen   block: Stop using...
1007
  static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
81744ee44   Martin K. Petersen   block: Fix incorr...
1008
1009
  {
  	unsigned int granularity = max(lim->physical_block_size, lim->io_min);
e03a72e13   Martin K. Petersen   block: Stop using...
1010
  	unsigned int alignment = (sector << 9) & (granularity - 1);
81744ee44   Martin K. Petersen   block: Fix incorr...
1011

e03a72e13   Martin K. Petersen   block: Stop using...
1012
1013
  	return (granularity + lim->alignment_offset - alignment)
  		& (granularity - 1);
c72758f33   Martin K. Petersen   block: Export I/O...
1014
  }
ac481c20e   Martin K. Petersen   block: Topology i...
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
  static inline int bdev_alignment_offset(struct block_device *bdev)
  {
  	struct request_queue *q = bdev_get_queue(bdev);
  
  	if (q->limits.misaligned)
  		return -1;
  
  	if (bdev != bdev->bd_contains)
  		return bdev->bd_part->alignment_offset;
  
  	return q->limits.alignment_offset;
  }
86b372814   Martin K. Petersen   block: Expose dis...
1027
1028
1029
1030
1031
1032
1033
  static inline int queue_discard_alignment(struct request_queue *q)
  {
  	if (q->limits.discard_misaligned)
  		return -1;
  
  	return q->limits.discard_alignment;
  }
e03a72e13   Martin K. Petersen   block: Stop using...
1034
  static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
86b372814   Martin K. Petersen   block: Expose dis...
1035
  {
dd3d145d4   Martin K. Petersen   block: Fix discar...
1036
  	unsigned int alignment = (sector << 9) & (lim->discard_granularity - 1);
a934a00a6   Martin K. Petersen   block: Fix discar...
1037
1038
  	if (!lim->max_discard_sectors)
  		return 0;
dd3d145d4   Martin K. Petersen   block: Fix discar...
1039
1040
  	return (lim->discard_granularity + lim->discard_alignment - alignment)
  		& (lim->discard_granularity - 1);
86b372814   Martin K. Petersen   block: Expose dis...
1041
  }
98262f276   Martin K. Petersen   block: Allow devi...
1042
1043
  static inline unsigned int queue_discard_zeroes_data(struct request_queue *q)
  {
a934a00a6   Martin K. Petersen   block: Fix discar...
1044
  	if (q->limits.max_discard_sectors && q->limits.discard_zeroes_data == 1)
98262f276   Martin K. Petersen   block: Allow devi...
1045
1046
1047
1048
1049
1050
1051
1052
1053
  		return 1;
  
  	return 0;
  }
  
  static inline unsigned int bdev_discard_zeroes_data(struct block_device *bdev)
  {
  	return queue_discard_zeroes_data(bdev_get_queue(bdev));
  }
165125e1e   Jens Axboe   [BLOCK] Get rid o...
1054
  static inline int queue_dma_alignment(struct request_queue *q)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1055
  {
482eb6891   Pete Wyckoff   block: allow queu...
1056
  	return q ? q->dma_alignment : 511;
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1057
  }
144177991   Namhyung Kim   block: fix an add...
1058
  static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
879040742   FUJITA Tomonori   block: add blk_rq...
1059
1060
1061
  				 unsigned int len)
  {
  	unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
144177991   Namhyung Kim   block: fix an add...
1062
  	return !(addr & alignment) && !(len & alignment);
879040742   FUJITA Tomonori   block: add blk_rq...
1063
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
  /* assumes size > 256 */
  static inline unsigned int blksize_bits(unsigned int size)
  {
  	unsigned int bits = 8;
  	do {
  		bits++;
  		size >>= 1;
  	} while (size > 256);
  	return bits;
  }
2befb9e36   Adrian Bunk   [PATCH] include/l...
1074
  static inline unsigned int block_size(struct block_device *bdev)
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1075
1076
1077
  {
  	return bdev->bd_block_size;
  }
f38769309   shaohua.li@intel.com   block: add a non-...
1078
1079
1080
1081
  static inline bool queue_flush_queueable(struct request_queue *q)
  {
  	return !q->flush_not_queueable;
  }
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
  typedef struct {struct page *v;} Sector;
  
  unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);
  
  static inline void put_dev_sector(Sector p)
  {
  	page_cache_release(p.v);
  }
  
  struct work_struct;
18887ad91   Jens Axboe   block: make kbloc...
1092
  int kblockd_schedule_work(struct request_queue *q, struct work_struct *work);
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1093

9195291e5   Divyesh Shah   blkio: Increment ...
1094
  #ifdef CONFIG_BLK_CGROUP
28f4197e5   Jens Axboe   block: disable pr...
1095
1096
1097
1098
1099
  /*
   * This should not be using sched_clock(). A real patch is in progress
   * to fix this up, until that is in place we need to disable preemption
   * around sched_clock() in this function and set_io_start_time_ns().
   */
9195291e5   Divyesh Shah   blkio: Increment ...
1100
1101
  static inline void set_start_time_ns(struct request *req)
  {
28f4197e5   Jens Axboe   block: disable pr...
1102
  	preempt_disable();
9195291e5   Divyesh Shah   blkio: Increment ...
1103
  	req->start_time_ns = sched_clock();
28f4197e5   Jens Axboe   block: disable pr...
1104
  	preempt_enable();
9195291e5   Divyesh Shah   blkio: Increment ...
1105
1106
1107
1108
  }
  
  static inline void set_io_start_time_ns(struct request *req)
  {
28f4197e5   Jens Axboe   block: disable pr...
1109
  	preempt_disable();
9195291e5   Divyesh Shah   blkio: Increment ...
1110
  	req->io_start_time_ns = sched_clock();
28f4197e5   Jens Axboe   block: disable pr...
1111
  	preempt_enable();
9195291e5   Divyesh Shah   blkio: Increment ...
1112
  }
84c124da9   Divyesh Shah   blkio: Changes to...
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
  
  static inline uint64_t rq_start_time_ns(struct request *req)
  {
          return req->start_time_ns;
  }
  
  static inline uint64_t rq_io_start_time_ns(struct request *req)
  {
          return req->io_start_time_ns;
  }
9195291e5   Divyesh Shah   blkio: Increment ...
1123
1124
1125
  #else
  static inline void set_start_time_ns(struct request *req) {}
  static inline void set_io_start_time_ns(struct request *req) {}
84c124da9   Divyesh Shah   blkio: Changes to...
1126
1127
1128
1129
1130
1131
1132
1133
  static inline uint64_t rq_start_time_ns(struct request *req)
  {
  	return 0;
  }
  static inline uint64_t rq_io_start_time_ns(struct request *req)
  {
  	return 0;
  }
9195291e5   Divyesh Shah   blkio: Increment ...
1134
  #endif
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1135
1136
1137
1138
  #define MODULE_ALIAS_BLOCKDEV(major,minor) \
  	MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
  #define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
  	MODULE_ALIAS("block-major-" __stringify(major) "-*")
7ba1ba12e   Martin K. Petersen   block: Block laye...
1139
  #if defined(CONFIG_BLK_DEV_INTEGRITY)
b24498d47   Jens Axboe   block: integrity ...
1140
1141
  #define INTEGRITY_FLAG_READ	2	/* verify data integrity on read */
  #define INTEGRITY_FLAG_WRITE	4	/* generate data integrity on write */
7ba1ba12e   Martin K. Petersen   block: Block laye...
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
  
  struct blk_integrity_exchg {
  	void			*prot_buf;
  	void			*data_buf;
  	sector_t		sector;
  	unsigned int		data_size;
  	unsigned short		sector_size;
  	const char		*disk_name;
  };
  
  typedef void (integrity_gen_fn) (struct blk_integrity_exchg *);
  typedef int (integrity_vrfy_fn) (struct blk_integrity_exchg *);
  typedef void (integrity_set_tag_fn) (void *, void *, unsigned int);
  typedef void (integrity_get_tag_fn) (void *, void *, unsigned int);
  
  struct blk_integrity {
  	integrity_gen_fn	*generate_fn;
  	integrity_vrfy_fn	*verify_fn;
  	integrity_set_tag_fn	*set_tag_fn;
  	integrity_get_tag_fn	*get_tag_fn;
  
  	unsigned short		flags;
  	unsigned short		tuple_size;
  	unsigned short		sector_size;
  	unsigned short		tag_size;
  
  	const char		*name;
  
  	struct kobject		kobj;
  };
a63a5cf84   Mike Snitzer   dm: improve block...
1172
  extern bool blk_integrity_is_initialized(struct gendisk *);
7ba1ba12e   Martin K. Petersen   block: Block laye...
1173
1174
  extern int blk_integrity_register(struct gendisk *, struct blk_integrity *);
  extern void blk_integrity_unregister(struct gendisk *);
ad7fce931   Martin K. Petersen   block: Switch blk...
1175
  extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
13f05c8d8   Martin K. Petersen   block/scsi: Provi...
1176
1177
1178
1179
1180
1181
1182
  extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
  				   struct scatterlist *);
  extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
  extern int blk_integrity_merge_rq(struct request_queue *, struct request *,
  				  struct request *);
  extern int blk_integrity_merge_bio(struct request_queue *, struct request *,
  				   struct bio *);
7ba1ba12e   Martin K. Petersen   block: Block laye...
1183

b04accc42   Jens Axboe   block: revert par...
1184
1185
1186
1187
1188
  static inline
  struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
  {
  	return bdev->bd_disk->integrity;
  }
b02739b01   Martin K. Petersen   block: gendisk in...
1189
1190
1191
1192
  static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
  {
  	return disk->integrity;
  }
7ba1ba12e   Martin K. Petersen   block: Block laye...
1193
1194
  static inline int blk_integrity_rq(struct request *rq)
  {
d442cc44c   Martin K. Petersen   block: Trivial fi...
1195
1196
  	if (rq->bio == NULL)
  		return 0;
7ba1ba12e   Martin K. Petersen   block: Block laye...
1197
1198
  	return bio_integrity(rq->bio);
  }
13f05c8d8   Martin K. Petersen   block/scsi: Provi...
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
  static inline void blk_queue_max_integrity_segments(struct request_queue *q,
  						    unsigned int segs)
  {
  	q->limits.max_integrity_segments = segs;
  }
  
  static inline unsigned short
  queue_max_integrity_segments(struct request_queue *q)
  {
  	return q->limits.max_integrity_segments;
  }
7ba1ba12e   Martin K. Petersen   block: Block laye...
1210
  #else /* CONFIG_BLK_DEV_INTEGRITY */
fd83240a6   Stephen Rothwell   blockdev: convert...
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
  struct bio;
  struct block_device;
  struct gendisk;
  struct blk_integrity;
  
  static inline int blk_integrity_rq(struct request *rq)
  {
  	return 0;
  }
  static inline int blk_rq_count_integrity_sg(struct request_queue *q,
  					    struct bio *b)
  {
  	return 0;
  }
  static inline int blk_rq_map_integrity_sg(struct request_queue *q,
  					  struct bio *b,
  					  struct scatterlist *s)
  {
  	return 0;
  }
  static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
  {
  	return 0;
  }
  static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
  {
  	return NULL;
  }
  static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
  {
  	return 0;
  }
  static inline int blk_integrity_register(struct gendisk *d,
  					 struct blk_integrity *b)
  {
  	return 0;
  }
  static inline void blk_integrity_unregister(struct gendisk *d)
  {
  }
  static inline void blk_queue_max_integrity_segments(struct request_queue *q,
  						    unsigned int segs)
  {
  }
  static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
  {
  	return 0;
  }
  static inline int blk_integrity_merge_rq(struct request_queue *rq,
  					 struct request *r1,
  					 struct request *r2)
  {
  	return 0;
  }
  static inline int blk_integrity_merge_bio(struct request_queue *rq,
  					  struct request *r,
  					  struct bio *b)
  {
  	return 0;
  }
  static inline bool blk_integrity_is_initialized(struct gendisk *g)
  {
  	return 0;
  }
7ba1ba12e   Martin K. Petersen   block: Block laye...
1275
1276
  
  #endif /* CONFIG_BLK_DEV_INTEGRITY */
08f858512   Al Viro   [PATCH] move bloc...
1277
  struct block_device_operations {
d4430d62f   Al Viro   [PATCH] beginning...
1278
1279
  	int (*open) (struct block_device *, fmode_t);
  	int (*release) (struct gendisk *, fmode_t);
d4430d62f   Al Viro   [PATCH] beginning...
1280
1281
  	int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
  	int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
08f858512   Al Viro   [PATCH] move bloc...
1282
1283
  	int (*direct_access) (struct block_device *, sector_t,
  						void **, unsigned long *);
77ea887e4   Tejun Heo   implement in-kern...
1284
1285
1286
  	unsigned int (*check_events) (struct gendisk *disk,
  				      unsigned int clearing);
  	/* ->media_changed() is DEPRECATED, use ->check_events() instead */
08f858512   Al Viro   [PATCH] move bloc...
1287
  	int (*media_changed) (struct gendisk *);
c3e33e043   Tejun Heo   block,ide: simpli...
1288
  	void (*unlock_native_capacity) (struct gendisk *);
08f858512   Al Viro   [PATCH] move bloc...
1289
1290
  	int (*revalidate_disk) (struct gendisk *);
  	int (*getgeo)(struct block_device *, struct hd_geometry *);
b3a27d052   Nitin Gupta   swap: Add swap sl...
1291
1292
  	/* this callback is with swap_lock and sometimes page table lock held */
  	void (*swap_slot_free_notify) (struct block_device *, unsigned long);
08f858512   Al Viro   [PATCH] move bloc...
1293
1294
  	struct module *owner;
  };
633a08b81   Al Viro   [PATCH] introduce...
1295
1296
  extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
  				 unsigned long);
9361401eb   David Howells   [PATCH] BLOCK: Ma...
1297
1298
1299
1300
1301
  #else /* CONFIG_BLOCK */
  /*
   * stubs for when the block layer is configured out
   */
  #define buffer_heads_over_limit 0
9361401eb   David Howells   [PATCH] BLOCK: Ma...
1302
1303
1304
1305
  static inline long nr_blockdev_pages(void)
  {
  	return 0;
  }
1f940bdfc   Jens Axboe   block: fixup plug...
1306
1307
1308
1309
  struct blk_plug {
  };
  
  static inline void blk_start_plug(struct blk_plug *plug)
73c101011   Jens Axboe   block: initial pa...
1310
1311
  {
  }
1f940bdfc   Jens Axboe   block: fixup plug...
1312
  static inline void blk_finish_plug(struct blk_plug *plug)
73c101011   Jens Axboe   block: initial pa...
1313
1314
  {
  }
1f940bdfc   Jens Axboe   block: fixup plug...
1315
  static inline void blk_flush_plug(struct task_struct *task)
73c101011   Jens Axboe   block: initial pa...
1316
1317
  {
  }
a237c1c5b   Jens Axboe   block: let io_sch...
1318
1319
1320
  static inline void blk_schedule_flush_plug(struct task_struct *task)
  {
  }
73c101011   Jens Axboe   block: initial pa...
1321
1322
1323
1324
  static inline bool blk_needs_flush_plug(struct task_struct *tsk)
  {
  	return false;
  }
9361401eb   David Howells   [PATCH] BLOCK: Ma...
1325
  #endif /* CONFIG_BLOCK */
1da177e4c   Linus Torvalds   Linux-2.6.12-rc2
1326
  #endif