Blame view
block/blk-ioc.c
10.2 KB
86db1e297 block: continue l... |
1 2 3 4 5 6 7 8 |
/* * Functions related to io context handling */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> #include <linux/bio.h> #include <linux/blkdev.h> |
5a0e3ad6a include cleanup: ... |
9 |
#include <linux/slab.h> |
86db1e297 block: continue l... |
10 11 12 13 14 15 16 |
#include "blk.h" /* * For io context allocations */ static struct kmem_cache *iocontext_cachep; |
6e736be7f block: make ioc g... |
17 18 19 20 21 22 23 24 25 26 27 28 |
/** * get_io_context - increment reference count to io_context * @ioc: io_context to get * * Increment reference count to @ioc. */ void get_io_context(struct io_context *ioc) { BUG_ON(atomic_long_read(&ioc->refcount) <= 0); atomic_long_inc(&ioc->refcount); } EXPORT_SYMBOL(get_io_context); |
7e5a87944 block, cfq: move ... |
29 30 31 32 33 34 |
static void icq_free_icq_rcu(struct rcu_head *head) { struct io_cq *icq = container_of(head, struct io_cq, __rcu_head); kmem_cache_free(icq->__rcu_icq_cache, icq); } |
621032ad6 block: exit_io_co... |
35 |
/* Exit an icq. Called with both ioc and q locked. */ |
7e5a87944 block, cfq: move ... |
36 37 |
static void ioc_exit_icq(struct io_cq *icq) { |
621032ad6 block: exit_io_co... |
38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
struct elevator_type *et = icq->q->elevator->type; if (icq->flags & ICQ_EXITED) return; if (et->ops.elevator_exit_icq_fn) et->ops.elevator_exit_icq_fn(icq); icq->flags |= ICQ_EXITED; } /* Release an icq. Called with both ioc and q locked. */ static void ioc_destroy_icq(struct io_cq *icq) { |
7e5a87944 block, cfq: move ... |
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
struct io_context *ioc = icq->ioc; struct request_queue *q = icq->q; struct elevator_type *et = q->elevator->type; lockdep_assert_held(&ioc->lock); lockdep_assert_held(q->queue_lock); radix_tree_delete(&ioc->icq_tree, icq->q->id); hlist_del_init(&icq->ioc_node); list_del_init(&icq->q_node); /* * Both setting lookup hint to and clearing it from @icq are done * under queue_lock. If it's not pointing to @icq now, it never * will. Hint assignment itself can race safely. */ |
ec6c676a0 block: Substitute... |
68 |
if (rcu_access_pointer(ioc->icq_hint) == icq) |
7e5a87944 block, cfq: move ... |
69 |
rcu_assign_pointer(ioc->icq_hint, NULL); |
621032ad6 block: exit_io_co... |
70 |
ioc_exit_icq(icq); |
7e5a87944 block, cfq: move ... |
71 72 73 74 75 76 77 78 |
/* * @icq->q might have gone away by the time RCU callback runs * making it impossible to determine icq_cache. Record it in @icq. */ icq->__rcu_icq_cache = et->icq_cache; call_rcu(&icq->__rcu_head, icq_free_icq_rcu); } |
b2efa0526 block, cfq: unlin... |
79 80 |
/* * Slow path for ioc release in put_io_context(). Performs double-lock |
c58698073 block, cfq: reorg... |
81 |
* dancing to unlink all icq's and then frees ioc. |
b2efa0526 block, cfq: unlin... |
82 83 |
*/ static void ioc_release_fn(struct work_struct *work) |
86db1e297 block: continue l... |
84 |
{ |
b2efa0526 block, cfq: unlin... |
85 86 |
struct io_context *ioc = container_of(work, struct io_context, release_work); |
d8c66c5d5 block: fix lockde... |
87 |
unsigned long flags; |
b2efa0526 block, cfq: unlin... |
88 |
|
d8c66c5d5 block: fix lockde... |
89 90 91 92 93 94 95 |
/* * Exiting icq may call into put_io_context() through elevator * which will trigger lockdep warning. The ioc's are guaranteed to * be different, use a different locking subclass here. Use * irqsave variant as there's no spin_lock_irq_nested(). */ spin_lock_irqsave_nested(&ioc->lock, flags, 1); |
b2efa0526 block, cfq: unlin... |
96 |
|
c58698073 block, cfq: reorg... |
97 98 99 |
while (!hlist_empty(&ioc->icq_list)) { struct io_cq *icq = hlist_entry(ioc->icq_list.first, struct io_cq, ioc_node); |
2274b029f block: simplify i... |
100 101 102 |
struct request_queue *q = icq->q; if (spin_trylock(q->queue_lock)) { |
621032ad6 block: exit_io_co... |
103 |
ioc_destroy_icq(icq); |
2274b029f block: simplify i... |
104 105 106 107 108 |
spin_unlock(q->queue_lock); } else { spin_unlock_irqrestore(&ioc->lock, flags); cpu_relax(); spin_lock_irqsave_nested(&ioc->lock, flags, 1); |
b2efa0526 block, cfq: unlin... |
109 |
} |
b2efa0526 block, cfq: unlin... |
110 |
} |
ffc4e7595 cfq-iosched: add ... |
111 |
|
2274b029f block: simplify i... |
112 |
spin_unlock_irqrestore(&ioc->lock, flags); |
b2efa0526 block, cfq: unlin... |
113 114 |
kmem_cache_free(iocontext_cachep, ioc); |
86db1e297 block: continue l... |
115 |
} |
42ec57a8f block: misc ioc c... |
116 117 118 119 120 |
/** * put_io_context - put a reference of io_context * @ioc: io_context to put * * Decrement reference count of @ioc and release it if the count reaches |
11a3122f6 block: strip out ... |
121 |
* zero. |
86db1e297 block: continue l... |
122 |
*/ |
11a3122f6 block: strip out ... |
123 |
void put_io_context(struct io_context *ioc) |
86db1e297 block: continue l... |
124 |
{ |
b2efa0526 block, cfq: unlin... |
125 |
unsigned long flags; |
ff8c1474c block: fix ioc le... |
126 |
bool free_ioc = false; |
b2efa0526 block, cfq: unlin... |
127 |
|
86db1e297 block: continue l... |
128 |
if (ioc == NULL) |
42ec57a8f block: misc ioc c... |
129 |
return; |
86db1e297 block: continue l... |
130 |
|
42ec57a8f block: misc ioc c... |
131 |
BUG_ON(atomic_long_read(&ioc->refcount) <= 0); |
86db1e297 block: continue l... |
132 |
|
b2efa0526 block, cfq: unlin... |
133 |
/* |
11a3122f6 block: strip out ... |
134 135 |
* Releasing ioc requires reverse order double locking and we may * already be holding a queue_lock. Do it asynchronously from wq. |
b2efa0526 block, cfq: unlin... |
136 |
*/ |
11a3122f6 block: strip out ... |
137 138 139 |
if (atomic_long_dec_and_test(&ioc->refcount)) { spin_lock_irqsave(&ioc->lock, flags); if (!hlist_empty(&ioc->icq_list)) |
695588f94 block: queue work... |
140 141 |
queue_work(system_power_efficient_wq, &ioc->release_work); |
ff8c1474c block: fix ioc le... |
142 143 |
else free_ioc = true; |
11a3122f6 block: strip out ... |
144 |
spin_unlock_irqrestore(&ioc->lock, flags); |
b2efa0526 block, cfq: unlin... |
145 |
} |
ff8c1474c block: fix ioc le... |
146 147 148 |
if (free_ioc) kmem_cache_free(iocontext_cachep, ioc); |
86db1e297 block: continue l... |
149 |
} |
b2efa0526 block, cfq: unlin... |
150 |
EXPORT_SYMBOL(put_io_context); |
86db1e297 block: continue l... |
151 |
|
f6e8d01be block: add io_con... |
152 153 154 155 156 157 158 159 |
/** * put_io_context_active - put active reference on ioc * @ioc: ioc of interest * * Undo get_io_context_active(). If active reference reaches zero after * put, @ioc can never issue further IOs and ioscheds are notified. */ void put_io_context_active(struct io_context *ioc) |
86db1e297 block: continue l... |
160 |
{ |
621032ad6 block: exit_io_co... |
161 |
unsigned long flags; |
f6e8d01be block: add io_con... |
162 |
struct io_cq *icq; |
86db1e297 block: continue l... |
163 |
|
f6e8d01be block: add io_con... |
164 |
if (!atomic_dec_and_test(&ioc->active_ref)) { |
621032ad6 block: exit_io_co... |
165 166 167 168 169 170 171 172 173 174 175 |
put_io_context(ioc); return; } /* * Need ioc lock to walk icq_list and q lock to exit icq. Perform * reverse double locking. Read comment in ioc_release_fn() for * explanation on the nested locking annotation. */ retry: spin_lock_irqsave_nested(&ioc->lock, flags, 1); |
b67bfe0d4 hlist: drop the n... |
176 |
hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) { |
621032ad6 block: exit_io_co... |
177 178 179 180 181 182 183 184 185 186 187 188 |
if (icq->flags & ICQ_EXITED) continue; if (spin_trylock(icq->q->queue_lock)) { ioc_exit_icq(icq); spin_unlock(icq->q->queue_lock); } else { spin_unlock_irqrestore(&ioc->lock, flags); cpu_relax(); goto retry; } } spin_unlock_irqrestore(&ioc->lock, flags); |
11a3122f6 block: strip out ... |
189 |
put_io_context(ioc); |
86db1e297 block: continue l... |
190 |
} |
f6e8d01be block: add io_con... |
191 192 193 194 195 196 197 198 199 200 201 202 203 |
/* Called by the exiting task */ void exit_io_context(struct task_struct *task) { struct io_context *ioc; task_lock(task); ioc = task->io_context; task->io_context = NULL; task_unlock(task); atomic_dec(&ioc->nr_tasks); put_io_context_active(ioc); } |
7e5a87944 block, cfq: move ... |
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 |
/** * ioc_clear_queue - break any ioc association with the specified queue * @q: request_queue being cleared * * Walk @q->icq_list and exit all io_cq's. Must be called with @q locked. */ void ioc_clear_queue(struct request_queue *q) { lockdep_assert_held(q->queue_lock); while (!list_empty(&q->icq_list)) { struct io_cq *icq = list_entry(q->icq_list.next, struct io_cq, q_node); struct io_context *ioc = icq->ioc; spin_lock(&ioc->lock); |
621032ad6 block: exit_io_co... |
220 |
ioc_destroy_icq(icq); |
7e5a87944 block, cfq: move ... |
221 222 223 |
spin_unlock(&ioc->lock); } } |
24acfc34f block: interface ... |
224 |
int create_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node) |
86db1e297 block: continue l... |
225 |
{ |
df4156569 block: rename the... |
226 |
struct io_context *ioc; |
3c9c708c9 block: avoid infi... |
227 |
int ret; |
86db1e297 block: continue l... |
228 |
|
42ec57a8f block: misc ioc c... |
229 230 231 |
ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO, node); if (unlikely(!ioc)) |
24acfc34f block: interface ... |
232 |
return -ENOMEM; |
42ec57a8f block: misc ioc c... |
233 234 235 |
/* initialize */ atomic_long_set(&ioc->refcount, 1); |
4638a83e8 block: uninitiali... |
236 |
atomic_set(&ioc->nr_tasks, 1); |
f6e8d01be block: add io_con... |
237 |
atomic_set(&ioc->active_ref, 1); |
42ec57a8f block: misc ioc c... |
238 |
spin_lock_init(&ioc->lock); |
c58698073 block, cfq: reorg... |
239 240 |
INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC | __GFP_HIGH); INIT_HLIST_HEAD(&ioc->icq_list); |
b2efa0526 block, cfq: unlin... |
241 |
INIT_WORK(&ioc->release_work, ioc_release_fn); |
86db1e297 block: continue l... |
242 |
|
fd6383681 block: an exiting... |
243 244 245 246 247 248 249 |
/* * Try to install. ioc shouldn't be installed if someone else * already did or @task, which isn't %current, is exiting. Note * that we need to allow ioc creation on exiting %current as exit * path may issue IOs from e.g. exit_files(). The exit path is * responsible for not issuing IO after exit_io_context(). */ |
6e736be7f block: make ioc g... |
250 |
task_lock(task); |
fd6383681 block: an exiting... |
251 252 |
if (!task->io_context && (task == current || !(task->flags & PF_EXITING))) |
6e736be7f block: make ioc g... |
253 |
task->io_context = ioc; |
f2dbd76a0 block, cfq: repla... |
254 |
else |
6e736be7f block: make ioc g... |
255 |
kmem_cache_free(iocontext_cachep, ioc); |
3c9c708c9 block: avoid infi... |
256 257 |
ret = task->io_context ? 0 : -EBUSY; |
6e736be7f block: make ioc g... |
258 |
task_unlock(task); |
24acfc34f block: interface ... |
259 |
|
3c9c708c9 block: avoid infi... |
260 |
return ret; |
86db1e297 block: continue l... |
261 |
} |
86db1e297 block: continue l... |
262 |
|
6e736be7f block: make ioc g... |
263 264 265 266 267 268 269 270 271 |
/** * get_task_io_context - get io_context of a task * @task: task of interest * @gfp_flags: allocation flags, used if allocation is necessary * @node: allocation node, used if allocation is necessary * * Return io_context of @task. If it doesn't exist, it is created with * @gfp_flags and @node. The returned io_context has its reference count * incremented. |
86db1e297 block: continue l... |
272 |
* |
6e736be7f block: make ioc g... |
273 |
* This function always goes through task_lock() and it's better to use |
f2dbd76a0 block, cfq: repla... |
274 |
* %current->io_context + get_io_context() for %current. |
86db1e297 block: continue l... |
275 |
*/ |
6e736be7f block: make ioc g... |
276 277 |
struct io_context *get_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node) |
86db1e297 block: continue l... |
278 |
{ |
6e736be7f block: make ioc g... |
279 |
struct io_context *ioc; |
86db1e297 block: continue l... |
280 |
|
6e736be7f block: make ioc g... |
281 |
might_sleep_if(gfp_flags & __GFP_WAIT); |
f2dbd76a0 block, cfq: repla... |
282 283 284 285 286 287 288 289 |
do { task_lock(task); ioc = task->io_context; if (likely(ioc)) { get_io_context(ioc); task_unlock(task); return ioc; } |
6e736be7f block: make ioc g... |
290 |
task_unlock(task); |
24acfc34f block: interface ... |
291 |
} while (!create_task_io_context(task, gfp_flags, node)); |
6e736be7f block: make ioc g... |
292 |
|
f2dbd76a0 block, cfq: repla... |
293 |
return NULL; |
86db1e297 block: continue l... |
294 |
} |
6e736be7f block: make ioc g... |
295 |
EXPORT_SYMBOL(get_task_io_context); |
86db1e297 block: continue l... |
296 |
|
47fdd4ca9 block, cfq: move ... |
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 |
/** * ioc_lookup_icq - lookup io_cq from ioc * @ioc: the associated io_context * @q: the associated request_queue * * Look up io_cq associated with @ioc - @q pair from @ioc. Must be called * with @q->queue_lock held. */ struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q) { struct io_cq *icq; lockdep_assert_held(q->queue_lock); /* * icq's are indexed from @ioc using radix tree and hint pointer, * both of which are protected with RCU. All removals are done * holding both q and ioc locks, and we're holding q lock - if we * find a icq which points to us, it's guaranteed to be valid. */ rcu_read_lock(); icq = rcu_dereference(ioc->icq_hint); if (icq && icq->q == q) goto out; icq = radix_tree_lookup(&ioc->icq_tree, q->id); if (icq && icq->q == q) rcu_assign_pointer(ioc->icq_hint, icq); /* allowed to race */ else icq = NULL; out: rcu_read_unlock(); return icq; } EXPORT_SYMBOL(ioc_lookup_icq); |
f1f8cc946 block, cfq: move ... |
332 333 |
/** * ioc_create_icq - create and link io_cq |
24acfc34f block: interface ... |
334 |
* @ioc: io_context of interest |
f1f8cc946 block, cfq: move ... |
335 336 337 |
* @q: request_queue of interest * @gfp_mask: allocation mask * |
24acfc34f block: interface ... |
338 339 |
* Make sure io_cq linking @ioc and @q exists. If icq doesn't exist, they * will be created using @gfp_mask. |
f1f8cc946 block, cfq: move ... |
340 341 342 343 |
* * The caller is responsible for ensuring @ioc won't go away and @q is * alive and will stay alive until this function returns. */ |
24acfc34f block: interface ... |
344 345 |
struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q, gfp_t gfp_mask) |
f1f8cc946 block, cfq: move ... |
346 347 |
{ struct elevator_type *et = q->elevator->type; |
f1f8cc946 block, cfq: move ... |
348 349 350 |
struct io_cq *icq; /* allocate stuff */ |
f1f8cc946 block, cfq: move ... |
351 352 353 354 |
icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO, q->node); if (!icq) return NULL; |
5e4c0d974 lib/radix-tree.c:... |
355 |
if (radix_tree_maybe_preload(gfp_mask) < 0) { |
f1f8cc946 block, cfq: move ... |
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 |
kmem_cache_free(et->icq_cache, icq); return NULL; } icq->ioc = ioc; icq->q = q; INIT_LIST_HEAD(&icq->q_node); INIT_HLIST_NODE(&icq->ioc_node); /* lock both q and ioc and try to link @icq */ spin_lock_irq(q->queue_lock); spin_lock(&ioc->lock); if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) { hlist_add_head(&icq->ioc_node, &ioc->icq_list); list_add(&icq->q_node, &q->icq_list); if (et->ops.elevator_init_icq_fn) et->ops.elevator_init_icq_fn(icq); } else { kmem_cache_free(et->icq_cache, icq); icq = ioc_lookup_icq(ioc, q); if (!icq) printk(KERN_ERR "cfq: icq link failed! "); } spin_unlock(&ioc->lock); spin_unlock_irq(q->queue_lock); radix_tree_preload_end(); return icq; } |
133415982 make blk_ioc_init... |
387 |
static int __init blk_ioc_init(void) |
86db1e297 block: continue l... |
388 389 390 391 392 393 |
{ iocontext_cachep = kmem_cache_create("blkdev_ioc", sizeof(struct io_context), 0, SLAB_PANIC, NULL); return 0; } subsys_initcall(blk_ioc_init); |