Blame view
drivers/md/dm-io.c
12.2 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 |
/* * Copyright (C) 2003 Sistina Software |
891ce2070 dm io: prepare fo... |
3 |
* Copyright (C) 2006 Red Hat GmbH |
1da177e4c Linux-2.6.12-rc2 |
4 5 6 |
* * This file is released under the GPL. */ |
952b35576 dm io: use slab f... |
7 |
#include "dm.h" |
586e80e6e dm: remove dm hea... |
8 |
#include <linux/device-mapper.h> |
1da177e4c Linux-2.6.12-rc2 |
9 10 11 12 13 14 |
#include <linux/bio.h> #include <linux/mempool.h> #include <linux/module.h> #include <linux/sched.h> #include <linux/slab.h> |
a765e20ee dm: move include ... |
15 |
#include <linux/dm-io.h> |
1da177e4c Linux-2.6.12-rc2 |
16 |
|
f1e539874 dm io: remove ext... |
17 18 19 |
#define DM_MSG_PREFIX "io" #define DM_IO_MAX_REGIONS BITS_PER_LONG |
bda8efec5 dm io: use fixed ... |
20 21 |
#define MIN_IOS 16 #define MIN_BIOS 16 |
f1e539874 dm io: remove ext... |
22 |
|
891ce2070 dm io: prepare fo... |
23 24 25 26 |
struct dm_io_client { mempool_t *pool; struct bio_set *bios; }; |
f1e539874 dm io: remove ext... |
27 28 29 30 |
/* * Aligning 'struct io' reduces the number of bits required to store * its address. Refer to store_io_and_region_in_bio() below. */ |
1da177e4c Linux-2.6.12-rc2 |
31 |
struct io { |
e01fd7eeb dm io: rename err... |
32 |
unsigned long error_bits; |
1da177e4c Linux-2.6.12-rc2 |
33 34 |
atomic_t count; struct task_struct *sleeper; |
891ce2070 dm io: prepare fo... |
35 |
struct dm_io_client *client; |
1da177e4c Linux-2.6.12-rc2 |
36 37 |
io_notify_fn callback; void *context; |
bb91bc7ba dm io: flush cpu ... |
38 39 |
void *vma_invalidate_address; unsigned long vma_invalidate_size; |
f1e539874 dm io: remove ext... |
40 |
} __attribute__((aligned(DM_IO_MAX_REGIONS))); |
1da177e4c Linux-2.6.12-rc2 |
41 |
|
952b35576 dm io: use slab f... |
42 |
static struct kmem_cache *_dm_io_cache; |
1da177e4c Linux-2.6.12-rc2 |
43 |
/* |
c8b03afe3 dm io: new interface |
44 45 |
* Create a client with mempool and bioset. */ |
bda8efec5 dm io: use fixed ... |
46 |
struct dm_io_client *dm_io_client_create(void) |
c8b03afe3 dm io: new interface |
47 |
{ |
c8b03afe3 dm io: new interface |
48 49 50 51 52 |
struct dm_io_client *client; client = kmalloc(sizeof(*client), GFP_KERNEL); if (!client) return ERR_PTR(-ENOMEM); |
bda8efec5 dm io: use fixed ... |
53 |
client->pool = mempool_create_slab_pool(MIN_IOS, _dm_io_cache); |
c8b03afe3 dm io: new interface |
54 55 |
if (!client->pool) goto bad; |
bda8efec5 dm io: use fixed ... |
56 |
client->bios = bioset_create(MIN_BIOS, 0); |
c8b03afe3 dm io: new interface |
57 58 59 60 61 62 63 64 65 66 67 68 |
if (!client->bios) goto bad; return client; bad: if (client->pool) mempool_destroy(client->pool); kfree(client); return ERR_PTR(-ENOMEM); } EXPORT_SYMBOL(dm_io_client_create); |
c8b03afe3 dm io: new interface |
69 70 71 72 73 74 75 |
void dm_io_client_destroy(struct dm_io_client *client) { mempool_destroy(client->pool); bioset_free(client->bios); kfree(client); } EXPORT_SYMBOL(dm_io_client_destroy); |
1da177e4c Linux-2.6.12-rc2 |
76 77 |
/*----------------------------------------------------------------- * We need to keep track of which region a bio is doing io for. |
f1e539874 dm io: remove ext... |
78 79 80 81 |
* To avoid a memory allocation to store just 5 or 6 bits, we * ensure the 'struct io' pointer is aligned so enough low bits are * always zero and then combine it with the region number directly in * bi_private. |
1da177e4c Linux-2.6.12-rc2 |
82 |
*---------------------------------------------------------------*/ |
f1e539874 dm io: remove ext... |
83 84 |
static void store_io_and_region_in_bio(struct bio *bio, struct io *io, unsigned region) |
1da177e4c Linux-2.6.12-rc2 |
85 |
{ |
f1e539874 dm io: remove ext... |
86 87 88 89 90 91 |
if (unlikely(!IS_ALIGNED((unsigned long)io, DM_IO_MAX_REGIONS))) { DMCRIT("Unaligned struct io pointer %p", io); BUG(); } bio->bi_private = (void *)((unsigned long)io | region); |
1da177e4c Linux-2.6.12-rc2 |
92 |
} |
f1e539874 dm io: remove ext... |
93 94 |
static void retrieve_io_and_region_from_bio(struct bio *bio, struct io **io, unsigned *region) |
1da177e4c Linux-2.6.12-rc2 |
95 |
{ |
f1e539874 dm io: remove ext... |
96 97 98 99 |
unsigned long val = (unsigned long)bio->bi_private; *io = (void *)(val & -(unsigned long)DM_IO_MAX_REGIONS); *region = val & (DM_IO_MAX_REGIONS - 1); |
1da177e4c Linux-2.6.12-rc2 |
100 101 102 103 104 105 106 107 |
} /*----------------------------------------------------------------- * We need an io object to keep track of the number of bios that * have been dispatched for a particular io. *---------------------------------------------------------------*/ static void dec_count(struct io *io, unsigned int region, int error) { |
d87f4c14f dm: implement REQ... |
108 |
if (error) |
e01fd7eeb dm io: rename err... |
109 |
set_bit(region, &io->error_bits); |
1da177e4c Linux-2.6.12-rc2 |
110 111 |
if (atomic_dec_and_test(&io->count)) { |
bb91bc7ba dm io: flush cpu ... |
112 113 114 |
if (io->vma_invalidate_size) invalidate_kernel_vmap_range(io->vma_invalidate_address, io->vma_invalidate_size); |
1da177e4c Linux-2.6.12-rc2 |
115 116 117 118 |
if (io->sleeper) wake_up_process(io->sleeper); else { |
e01fd7eeb dm io: rename err... |
119 |
unsigned long r = io->error_bits; |
1da177e4c Linux-2.6.12-rc2 |
120 121 |
io_notify_fn fn = io->callback; void *context = io->context; |
bf17ce3a6 dm io: remove old... |
122 |
mempool_free(io, io->client->pool); |
1da177e4c Linux-2.6.12-rc2 |
123 124 125 126 |
fn(r, context); } } } |
6712ecf8f Drop 'size' argum... |
127 |
static void endio(struct bio *bio, int error) |
1da177e4c Linux-2.6.12-rc2 |
128 |
{ |
c897feb3d dm io: delay dec_... |
129 130 |
struct io *io; unsigned region; |
1da177e4c Linux-2.6.12-rc2 |
131 |
|
1da177e4c Linux-2.6.12-rc2 |
132 133 |
if (error && bio_data_dir(bio) == READ) zero_fill_bio(bio); |
c897feb3d dm io: delay dec_... |
134 135 136 |
/* * The bio destructor in bio_put() may use the io object. */ |
f1e539874 dm io: remove ext... |
137 |
retrieve_io_and_region_from_bio(bio, &io, ®ion); |
c897feb3d dm io: delay dec_... |
138 |
|
1da177e4c Linux-2.6.12-rc2 |
139 |
bio_put(bio); |
c897feb3d dm io: delay dec_... |
140 |
dec_count(io, region, error); |
1da177e4c Linux-2.6.12-rc2 |
141 142 143 144 145 146 147 148 149 150 151 152 153 |
} /*----------------------------------------------------------------- * These little objects provide an abstraction for getting a new * destination page for io. *---------------------------------------------------------------*/ struct dpages { void (*get_page)(struct dpages *dp, struct page **p, unsigned long *len, unsigned *offset); void (*next_page)(struct dpages *dp); unsigned context_u; void *context_ptr; |
bb91bc7ba dm io: flush cpu ... |
154 155 156 |
void *vma_invalidate_address; unsigned long vma_invalidate_size; |
1da177e4c Linux-2.6.12-rc2 |
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 |
}; /* * Functions for getting the pages from a list. */ static void list_get_page(struct dpages *dp, struct page **p, unsigned long *len, unsigned *offset) { unsigned o = dp->context_u; struct page_list *pl = (struct page_list *) dp->context_ptr; *p = pl->page; *len = PAGE_SIZE - o; *offset = o; } static void list_next_page(struct dpages *dp) { struct page_list *pl = (struct page_list *) dp->context_ptr; dp->context_ptr = pl->next; dp->context_u = 0; } static void list_dp_init(struct dpages *dp, struct page_list *pl, unsigned offset) { dp->get_page = list_get_page; dp->next_page = list_next_page; dp->context_u = offset; dp->context_ptr = pl; } /* * Functions for getting the pages from a bvec. */ static void bvec_get_page(struct dpages *dp, struct page **p, unsigned long *len, unsigned *offset) { struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; *p = bvec->bv_page; *len = bvec->bv_len; *offset = bvec->bv_offset; } static void bvec_next_page(struct dpages *dp) { struct bio_vec *bvec = (struct bio_vec *) dp->context_ptr; dp->context_ptr = bvec + 1; } static void bvec_dp_init(struct dpages *dp, struct bio_vec *bvec) { dp->get_page = bvec_get_page; dp->next_page = bvec_next_page; dp->context_ptr = bvec; } |
c8b03afe3 dm io: new interface |
212 213 214 |
/* * Functions for getting the pages from a VMA. */ |
1da177e4c Linux-2.6.12-rc2 |
215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 |
static void vm_get_page(struct dpages *dp, struct page **p, unsigned long *len, unsigned *offset) { *p = vmalloc_to_page(dp->context_ptr); *offset = dp->context_u; *len = PAGE_SIZE - dp->context_u; } static void vm_next_page(struct dpages *dp) { dp->context_ptr += PAGE_SIZE - dp->context_u; dp->context_u = 0; } static void vm_dp_init(struct dpages *dp, void *data) { dp->get_page = vm_get_page; dp->next_page = vm_next_page; dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); dp->context_ptr = data; } |
3676347a5 [PATCH] kill bio-... |
236 237 |
static void dm_bio_destructor(struct bio *bio) { |
f1e539874 dm io: remove ext... |
238 239 240 241 |
unsigned region; struct io *io; retrieve_io_and_region_from_bio(bio, &io, ®ion); |
891ce2070 dm io: prepare fo... |
242 |
|
bf17ce3a6 dm io: remove old... |
243 |
bio_free(bio, io->client->bios); |
3676347a5 [PATCH] kill bio-... |
244 |
} |
c8b03afe3 dm io: new interface |
245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 |
/* * Functions for getting the pages from kernel memory. */ static void km_get_page(struct dpages *dp, struct page **p, unsigned long *len, unsigned *offset) { *p = virt_to_page(dp->context_ptr); *offset = dp->context_u; *len = PAGE_SIZE - dp->context_u; } static void km_next_page(struct dpages *dp) { dp->context_ptr += PAGE_SIZE - dp->context_u; dp->context_u = 0; } static void km_dp_init(struct dpages *dp, void *data) { dp->get_page = km_get_page; dp->next_page = km_next_page; dp->context_u = ((unsigned long) data) & (PAGE_SIZE - 1); dp->context_ptr = data; } |
1da177e4c Linux-2.6.12-rc2 |
269 270 271 |
/*----------------------------------------------------------------- * IO routines that accept a list of pages. *---------------------------------------------------------------*/ |
22a1ceb1e dm io: clean inte... |
272 |
static void do_region(int rw, unsigned region, struct dm_io_region *where, |
1da177e4c Linux-2.6.12-rc2 |
273 274 275 276 277 278 279 280 |
struct dpages *dp, struct io *io) { struct bio *bio; struct page *page; unsigned long len; unsigned offset; unsigned num_bvecs; sector_t remaining = where->count; |
12fc0f49d dm io: handle emp... |
281 |
/* |
d87f4c14f dm: implement REQ... |
282 283 |
* where->count may be zero if rw holds a flush and we need to * send a zero-sized flush. |
12fc0f49d dm io: handle emp... |
284 285 |
*/ do { |
1da177e4c Linux-2.6.12-rc2 |
286 |
/* |
f1e539874 dm io: remove ext... |
287 |
* Allocate a suitably sized-bio. |
1da177e4c Linux-2.6.12-rc2 |
288 |
*/ |
596f138ee dm io: fix panic ... |
289 290 |
num_bvecs = dm_sector_div_up(remaining, (PAGE_SIZE >> SECTOR_SHIFT)); |
f1e539874 dm io: remove ext... |
291 |
num_bvecs = min_t(int, bio_get_nr_vecs(where->bdev), num_bvecs); |
bf17ce3a6 dm io: remove old... |
292 |
bio = bio_alloc_bioset(GFP_NOIO, num_bvecs, io->client->bios); |
1da177e4c Linux-2.6.12-rc2 |
293 294 295 |
bio->bi_sector = where->sector + (where->count - remaining); bio->bi_bdev = where->bdev; bio->bi_end_io = endio; |
3676347a5 [PATCH] kill bio-... |
296 |
bio->bi_destructor = dm_bio_destructor; |
f1e539874 dm io: remove ext... |
297 |
store_io_and_region_in_bio(bio, io, region); |
1da177e4c Linux-2.6.12-rc2 |
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 |
/* * Try and add as many pages as possible. */ while (remaining) { dp->get_page(dp, &page, &len, &offset); len = min(len, to_bytes(remaining)); if (!bio_add_page(bio, page, len, offset)) break; offset = 0; remaining -= to_sector(len); dp->next_page(dp); } atomic_inc(&io->count); submit_bio(rw, bio); |
12fc0f49d dm io: handle emp... |
315 |
} while (remaining); |
1da177e4c Linux-2.6.12-rc2 |
316 317 318 |
} static void dispatch_io(int rw, unsigned int num_regions, |
22a1ceb1e dm io: clean inte... |
319 |
struct dm_io_region *where, struct dpages *dp, |
1da177e4c Linux-2.6.12-rc2 |
320 321 322 323 |
struct io *io, int sync) { int i; struct dpages old_pages = *dp; |
f1e539874 dm io: remove ext... |
324 |
BUG_ON(num_regions > DM_IO_MAX_REGIONS); |
1da177e4c Linux-2.6.12-rc2 |
325 |
if (sync) |
721a9602e block: kill off R... |
326 |
rw |= REQ_SYNC; |
1da177e4c Linux-2.6.12-rc2 |
327 328 329 330 331 332 333 |
/* * For multiple regions we need to be careful to rewind * the dp object for each call to do_region. */ for (i = 0; i < num_regions; i++) { *dp = old_pages; |
d87f4c14f dm: implement REQ... |
334 |
if (where[i].count || (rw & REQ_FLUSH)) |
1da177e4c Linux-2.6.12-rc2 |
335 336 337 338 |
do_region(rw, i, where + i, dp, io); } /* |
f00b16ad6 [PATCH] dm io: fi... |
339 |
* Drop the extra reference that we were holding to avoid |
1da177e4c Linux-2.6.12-rc2 |
340 341 342 343 |
* the io being completed too early. */ dec_count(io, 0, 0); } |
891ce2070 dm io: prepare fo... |
344 |
static int sync_io(struct dm_io_client *client, unsigned int num_regions, |
22a1ceb1e dm io: clean inte... |
345 |
struct dm_io_region *where, int rw, struct dpages *dp, |
891ce2070 dm io: prepare fo... |
346 |
unsigned long *error_bits) |
1da177e4c Linux-2.6.12-rc2 |
347 |
{ |
f1e539874 dm io: remove ext... |
348 349 350 351 352 353 354 355 |
/* * gcc <= 4.3 can't do the alignment for stack variables, so we must * align it on our own. * volatile prevents the optimizer from removing or reusing * "io_" field from the stack frame (allowed in ANSI C). */ volatile char io_[sizeof(struct io) + __alignof__(struct io) - 1]; struct io *io = (struct io *)PTR_ALIGN(&io_, __alignof__(struct io)); |
1da177e4c Linux-2.6.12-rc2 |
356 |
|
7ff14a361 dm: unplug queues... |
357 |
if (num_regions > 1 && (rw & RW_MASK) != WRITE) { |
1da177e4c Linux-2.6.12-rc2 |
358 359 360 |
WARN_ON(1); return -EIO; } |
f1e539874 dm io: remove ext... |
361 |
io->error_bits = 0; |
f1e539874 dm io: remove ext... |
362 363 364 |
atomic_set(&io->count, 1); /* see dispatch_io() */ io->sleeper = current; io->client = client; |
1da177e4c Linux-2.6.12-rc2 |
365 |
|
bb91bc7ba dm io: flush cpu ... |
366 367 |
io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; |
f1e539874 dm io: remove ext... |
368 |
dispatch_io(rw, num_regions, where, dp, io, 1); |
1da177e4c Linux-2.6.12-rc2 |
369 370 371 |
while (1) { set_current_state(TASK_UNINTERRUPTIBLE); |
f1e539874 dm io: remove ext... |
372 |
if (!atomic_read(&io->count)) |
1da177e4c Linux-2.6.12-rc2 |
373 374 375 376 377 |
break; io_schedule(); } set_current_state(TASK_RUNNING); |
891ce2070 dm io: prepare fo... |
378 |
if (error_bits) |
f1e539874 dm io: remove ext... |
379 |
*error_bits = io->error_bits; |
891ce2070 dm io: prepare fo... |
380 |
|
f1e539874 dm io: remove ext... |
381 |
return io->error_bits ? -EIO : 0; |
1da177e4c Linux-2.6.12-rc2 |
382 |
} |
891ce2070 dm io: prepare fo... |
383 |
static int async_io(struct dm_io_client *client, unsigned int num_regions, |
22a1ceb1e dm io: clean inte... |
384 |
struct dm_io_region *where, int rw, struct dpages *dp, |
891ce2070 dm io: prepare fo... |
385 |
io_notify_fn fn, void *context) |
1da177e4c Linux-2.6.12-rc2 |
386 387 |
{ struct io *io; |
7ff14a361 dm: unplug queues... |
388 |
if (num_regions > 1 && (rw & RW_MASK) != WRITE) { |
1da177e4c Linux-2.6.12-rc2 |
389 390 391 392 |
WARN_ON(1); fn(1, context); return -EIO; } |
bf17ce3a6 dm io: remove old... |
393 |
io = mempool_alloc(client->pool, GFP_NOIO); |
e01fd7eeb dm io: rename err... |
394 |
io->error_bits = 0; |
1da177e4c Linux-2.6.12-rc2 |
395 396 |
atomic_set(&io->count, 1); /* see dispatch_io() */ io->sleeper = NULL; |
891ce2070 dm io: prepare fo... |
397 |
io->client = client; |
1da177e4c Linux-2.6.12-rc2 |
398 399 |
io->callback = fn; io->context = context; |
bb91bc7ba dm io: flush cpu ... |
400 401 |
io->vma_invalidate_address = dp->vma_invalidate_address; io->vma_invalidate_size = dp->vma_invalidate_size; |
1da177e4c Linux-2.6.12-rc2 |
402 403 404 |
dispatch_io(rw, num_regions, where, dp, io, 0); return 0; } |
bb91bc7ba dm io: flush cpu ... |
405 406 |
static int dp_init(struct dm_io_request *io_req, struct dpages *dp, unsigned long size) |
c8b03afe3 dm io: new interface |
407 408 |
{ /* Set up dpages based on memory type */ |
bb91bc7ba dm io: flush cpu ... |
409 410 411 |
dp->vma_invalidate_address = NULL; dp->vma_invalidate_size = 0; |
c8b03afe3 dm io: new interface |
412 413 414 415 416 417 418 419 420 421 |
switch (io_req->mem.type) { case DM_IO_PAGE_LIST: list_dp_init(dp, io_req->mem.ptr.pl, io_req->mem.offset); break; case DM_IO_BVEC: bvec_dp_init(dp, io_req->mem.ptr.bvec); break; case DM_IO_VMA: |
bb91bc7ba dm io: flush cpu ... |
422 423 424 425 426 |
flush_kernel_vmap_range(io_req->mem.ptr.vma, size); if ((io_req->bi_rw & RW_MASK) == READ) { dp->vma_invalidate_address = io_req->mem.ptr.vma; dp->vma_invalidate_size = size; } |
c8b03afe3 dm io: new interface |
427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 |
vm_dp_init(dp, io_req->mem.ptr.vma); break; case DM_IO_KMEM: km_dp_init(dp, io_req->mem.ptr.addr); break; default: return -EINVAL; } return 0; } /* |
7ff14a361 dm: unplug queues... |
442 443 444 |
* New collapsed (a)synchronous interface. * * If the IO is asynchronous (i.e. it has notify.fn), you must either unplug |
7b6d91dae block: unify flag... |
445 446 |
* the queue with blk_unplug() some time later or set REQ_SYNC in io_req->bi_rw. If you fail to do one of these, the IO will be submitted to |
7ff14a361 dm: unplug queues... |
447 |
* the disk after q->unplug_delay, which defaults to 3ms in blk-settings.c. |
c8b03afe3 dm io: new interface |
448 449 |
*/ int dm_io(struct dm_io_request *io_req, unsigned num_regions, |
22a1ceb1e dm io: clean inte... |
450 |
struct dm_io_region *where, unsigned long *sync_error_bits) |
c8b03afe3 dm io: new interface |
451 452 453 |
{ int r; struct dpages dp; |
bb91bc7ba dm io: flush cpu ... |
454 |
r = dp_init(io_req, &dp, (unsigned long)where->count << SECTOR_SHIFT); |
c8b03afe3 dm io: new interface |
455 456 457 458 459 460 461 462 463 464 465 |
if (r) return r; if (!io_req->notify.fn) return sync_io(io_req->client, num_regions, where, io_req->bi_rw, &dp, sync_error_bits); return async_io(io_req->client, num_regions, where, io_req->bi_rw, &dp, io_req->notify.fn, io_req->notify.context); } EXPORT_SYMBOL(dm_io); |
952b35576 dm io: use slab f... |
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 |
int __init dm_io_init(void) { _dm_io_cache = KMEM_CACHE(io, 0); if (!_dm_io_cache) return -ENOMEM; return 0; } void dm_io_exit(void) { kmem_cache_destroy(_dm_io_cache); _dm_io_cache = NULL; } |