Blame view
net/sunrpc/sched.c
30.3 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 5 6 |
/* * linux/net/sunrpc/sched.c * * Scheduling for synchronous and asynchronous RPC requests. * * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de> |
cca5172a7 [NET] SUNRPC: Fix... |
7 |
* |
1da177e4c Linux-2.6.12-rc2 |
8 9 10 11 12 13 14 15 16 17 18 |
* TCP NFS related read + write fixes * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> */ #include <linux/module.h> #include <linux/sched.h> #include <linux/interrupt.h> #include <linux/slab.h> #include <linux/mempool.h> #include <linux/smp.h> |
1da177e4c Linux-2.6.12-rc2 |
19 |
#include <linux/spinlock.h> |
4a3e2f711 [NET] sem2mutex: ... |
20 |
#include <linux/mutex.h> |
d310310cb Freezer / sunrpc ... |
21 |
#include <linux/freezer.h> |
1da177e4c Linux-2.6.12-rc2 |
22 23 |
#include <linux/sunrpc/clnt.h> |
1da177e4c Linux-2.6.12-rc2 |
24 |
|
6951867b9 nfsd41: sunrpc: m... |
25 |
#include "sunrpc.h" |
f895b252d sunrpc: eliminate... |
26 |
#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
1da177e4c Linux-2.6.12-rc2 |
27 |
#define RPCDBG_FACILITY RPCDBG_SCHED |
1da177e4c Linux-2.6.12-rc2 |
28 |
#endif |
82b0a4c3c SUNRPC: Add trace... |
29 30 |
#define CREATE_TRACE_POINTS #include <trace/events/sunrpc.h> |
1da177e4c Linux-2.6.12-rc2 |
31 32 33 34 35 36 |
/* * RPC slabs and memory pools */ #define RPC_BUFFER_MAXSIZE (2048) #define RPC_BUFFER_POOLSIZE (8) #define RPC_TASK_POOLSIZE (8) |
e18b890bb [PATCH] slab: rem... |
37 38 |
static struct kmem_cache *rpc_task_slabp __read_mostly; static struct kmem_cache *rpc_buffer_slabp __read_mostly; |
ba89966c1 [NET]: use __read... |
39 40 |
static mempool_t *rpc_task_mempool __read_mostly; static mempool_t *rpc_buffer_mempool __read_mostly; |
1da177e4c Linux-2.6.12-rc2 |
41 |
|
65f27f384 WorkStruct: Pass ... |
42 |
static void rpc_async_schedule(struct work_struct *); |
bde8f00ce [PATCH] NFS: Fix ... |
43 |
static void rpc_release_task(struct rpc_task *task); |
36df9aae3 SUNRPC: Add a tim... |
44 |
static void __rpc_queue_timer_fn(unsigned long ptr); |
1da177e4c Linux-2.6.12-rc2 |
45 46 |
/* |
1da177e4c Linux-2.6.12-rc2 |
47 48 |
* RPC tasks sit here while waiting for conditions to improve. */ |
a4a874990 SUNRPC: Cleanup t... |
49 |
static struct rpc_wait_queue delay_queue; |
1da177e4c Linux-2.6.12-rc2 |
50 51 |
/* |
1da177e4c Linux-2.6.12-rc2 |
52 53 |
* rpciod-related stuff */ |
40a5f1b19 SUNRPC: RPC trans... |
54 55 |
struct workqueue_struct *rpciod_workqueue __read_mostly; struct workqueue_struct *xprtiod_workqueue __read_mostly; |
1da177e4c Linux-2.6.12-rc2 |
56 57 |
/* |
1da177e4c Linux-2.6.12-rc2 |
58 59 60 61 |
* Disable the timer for a given RPC task. Should be called with * queue->lock and bh_disabled in order to avoid races within * rpc_run_timer(). */ |
5d00837b9 SUNRPC: Run rpc t... |
62 |
static void |
eb276c0e1 SUNRPC: Switch ta... |
63 |
__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
64 |
{ |
36df9aae3 SUNRPC: Add a tim... |
65 66 |
if (task->tk_timeout == 0) return; |
46121cf7d SUNRPC: fix print... |
67 68 |
dprintk("RPC: %5u disabling timer ", task->tk_pid); |
1da177e4c Linux-2.6.12-rc2 |
69 |
task->tk_timeout = 0; |
36df9aae3 SUNRPC: Add a tim... |
70 |
list_del(&task->u.tk_wait.timer_list); |
eb276c0e1 SUNRPC: Switch ta... |
71 72 |
if (list_empty(&queue->timer_list.list)) del_timer(&queue->timer_list.timer); |
36df9aae3 SUNRPC: Add a tim... |
73 74 75 76 77 78 79 |
} static void rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires) { queue->timer_list.expires = expires; mod_timer(&queue->timer_list.timer, expires); |
1da177e4c Linux-2.6.12-rc2 |
80 81 82 |
} /* |
1da177e4c Linux-2.6.12-rc2 |
83 84 |
* Set up a timer for the current task. */ |
5d00837b9 SUNRPC: Run rpc t... |
85 |
static void |
eb276c0e1 SUNRPC: Switch ta... |
86 |
__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
87 88 89 |
{ if (!task->tk_timeout) return; |
55cc1d780 SUNRPC: fix build... |
90 91 92 |
dprintk("RPC: %5u setting alarm for %u ms ", task->tk_pid, jiffies_to_msecs(task->tk_timeout)); |
1da177e4c Linux-2.6.12-rc2 |
93 |
|
eb276c0e1 SUNRPC: Switch ta... |
94 95 96 97 |
task->u.tk_wait.expires = jiffies + task->tk_timeout; if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires)) rpc_set_queue_timer(queue, task->u.tk_wait.expires); list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list); |
1da177e4c Linux-2.6.12-rc2 |
98 |
} |
edd2e36fe SUNRPC: When chan... |
99 100 101 102 103 104 105 106 107 108 109 |
static void rpc_rotate_queue_owner(struct rpc_wait_queue *queue) { struct list_head *q = &queue->tasks[queue->priority]; struct rpc_task *task; if (!list_empty(q)) { task = list_first_entry(q, struct rpc_task, u.tk_wait.list); if (task->tk_owner == queue->owner) list_move_tail(&task->u.tk_wait.list, q); } } |
c05eecf63 SUNRPC: Don't all... |
110 111 |
static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority) { |
edd2e36fe SUNRPC: When chan... |
112 113 114 115 116 |
if (queue->priority != priority) { /* Fairness: rotate the list when changing priority */ rpc_rotate_queue_owner(queue); queue->priority = priority; } |
c05eecf63 SUNRPC: Don't all... |
117 118 119 120 121 122 123 124 125 126 127 128 129 |
} static void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid) { queue->owner = pid; queue->nr = RPC_BATCH_COUNT; } static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue) { rpc_set_waitqueue_priority(queue, queue->maxpriority); rpc_set_waitqueue_owner(queue, 0); } |
1da177e4c Linux-2.6.12-rc2 |
130 131 132 |
/* * Add new request to a priority queue. */ |
3b27bad7f SUNRPC: Allow cal... |
133 134 135 |
static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task, unsigned char queue_priority) |
1da177e4c Linux-2.6.12-rc2 |
136 137 138 139 140 |
{ struct list_head *q; struct rpc_task *t; INIT_LIST_HEAD(&task->u.tk_wait.links); |
3b27bad7f SUNRPC: Allow cal... |
141 |
if (unlikely(queue_priority > queue->maxpriority)) |
c05eecf63 SUNRPC: Don't all... |
142 143 144 145 |
queue_priority = queue->maxpriority; if (queue_priority > queue->priority) rpc_set_waitqueue_priority(queue, queue_priority); q = &queue->tasks[queue_priority]; |
1da177e4c Linux-2.6.12-rc2 |
146 |
list_for_each_entry(t, q, u.tk_wait.list) { |
3ff7576dd SUNRPC: Clean up ... |
147 |
if (t->tk_owner == task->tk_owner) { |
1da177e4c Linux-2.6.12-rc2 |
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 |
list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links); return; } } list_add_tail(&task->u.tk_wait.list, q); } /* * Add new request to wait queue. * * Swapper tasks always get inserted at the head of the queue. * This should avoid many nasty memory deadlocks and hopefully * improve overall performance. * Everyone else gets appended to the queue to ensure proper FIFO behavior. */ |
3b27bad7f SUNRPC: Allow cal... |
163 164 165 |
static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task, unsigned char queue_priority) |
1da177e4c Linux-2.6.12-rc2 |
166 |
{ |
2bd4eef87 SUNRPC: remove BU... |
167 168 169 |
WARN_ON_ONCE(RPC_IS_QUEUED(task)); if (RPC_IS_QUEUED(task)) return; |
1da177e4c Linux-2.6.12-rc2 |
170 171 |
if (RPC_IS_PRIORITY(queue)) |
3b27bad7f SUNRPC: Allow cal... |
172 |
__rpc_add_wait_queue_priority(queue, task, queue_priority); |
1da177e4c Linux-2.6.12-rc2 |
173 174 175 176 |
else if (RPC_IS_SWAPPER(task)) list_add(&task->u.tk_wait.list, &queue->tasks[0]); else list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]); |
96ef13b28 SUNRPC: Add a new... |
177 |
task->tk_waitqueue = queue; |
e19b63daf SUNRPC: track len... |
178 |
queue->qlen++; |
1166fde6a SUNRPC: Add barri... |
179 180 |
/* barrier matches the read in rpc_wake_up_task_queue_locked() */ smp_wmb(); |
1da177e4c Linux-2.6.12-rc2 |
181 |
rpc_set_queued(task); |
46121cf7d SUNRPC: fix print... |
182 183 184 |
dprintk("RPC: %5u added to queue %p \"%s\" ", task->tk_pid, queue, rpc_qname(queue)); |
1da177e4c Linux-2.6.12-rc2 |
185 186 187 188 189 190 191 192 193 194 195 196 197 198 |
} /* * Remove request from a priority queue. */ static void __rpc_remove_wait_queue_priority(struct rpc_task *task) { struct rpc_task *t; if (!list_empty(&task->u.tk_wait.links)) { t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list); list_move(&t->u.tk_wait.list, &task->u.tk_wait.list); list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links); } |
1da177e4c Linux-2.6.12-rc2 |
199 200 201 202 203 204 |
} /* * Remove request from queue. * Note: must be called with spin lock held. */ |
96ef13b28 SUNRPC: Add a new... |
205 |
static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
206 |
{ |
eb276c0e1 SUNRPC: Switch ta... |
207 |
__rpc_disable_timer(queue, task); |
1da177e4c Linux-2.6.12-rc2 |
208 209 |
if (RPC_IS_PRIORITY(queue)) __rpc_remove_wait_queue_priority(task); |
36df9aae3 SUNRPC: Add a tim... |
210 |
list_del(&task->u.tk_wait.list); |
e19b63daf SUNRPC: track len... |
211 |
queue->qlen--; |
46121cf7d SUNRPC: fix print... |
212 213 214 |
dprintk("RPC: %5u removed from queue %p \"%s\" ", task->tk_pid, queue, rpc_qname(queue)); |
1da177e4c Linux-2.6.12-rc2 |
215 |
} |
3ff7576dd SUNRPC: Clean up ... |
216 |
static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues) |
1da177e4c Linux-2.6.12-rc2 |
217 218 219 220 221 222 |
{ int i; spin_lock_init(&queue->lock); for (i = 0; i < ARRAY_SIZE(queue->tasks); i++) INIT_LIST_HEAD(&queue->tasks[i]); |
3ff7576dd SUNRPC: Clean up ... |
223 |
queue->maxpriority = nr_queues - 1; |
1da177e4c Linux-2.6.12-rc2 |
224 |
rpc_reset_waitqueue_priority(queue); |
36df9aae3 SUNRPC: Add a tim... |
225 226 227 |
queue->qlen = 0; setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue); INIT_LIST_HEAD(&queue->timer_list.list); |
2f09c2421 SUNRPC: Ensure th... |
228 |
rpc_assign_waitqueue_name(queue, qname); |
1da177e4c Linux-2.6.12-rc2 |
229 230 231 232 |
} void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname) { |
3ff7576dd SUNRPC: Clean up ... |
233 |
__rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY); |
1da177e4c Linux-2.6.12-rc2 |
234 |
} |
689cf5c15 nfs: enforce FIFO... |
235 |
EXPORT_SYMBOL_GPL(rpc_init_priority_wait_queue); |
1da177e4c Linux-2.6.12-rc2 |
236 237 238 |
void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname) { |
3ff7576dd SUNRPC: Clean up ... |
239 |
__rpc_init_priority_wait_queue(queue, qname, 1); |
1da177e4c Linux-2.6.12-rc2 |
240 |
} |
e8914c65f SUNRPC: Restrict ... |
241 |
EXPORT_SYMBOL_GPL(rpc_init_wait_queue); |
1da177e4c Linux-2.6.12-rc2 |
242 |
|
f6a1cc893 SUNRPC: Add a (em... |
243 244 |
void rpc_destroy_wait_queue(struct rpc_wait_queue *queue) { |
36df9aae3 SUNRPC: Add a tim... |
245 |
del_timer_sync(&queue->timer_list.timer); |
f6a1cc893 SUNRPC: Add a (em... |
246 247 |
} EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue); |
dfd01f026 sched/wait: Fix t... |
248 |
static int rpc_wait_bit_killable(struct wait_bit_key *key, int mode) |
44c288732 NFSv4: stateful N... |
249 |
{ |
416ad3c9c freezer: add unsa... |
250 |
freezable_schedule_unsafe(); |
dfd01f026 sched/wait: Fix t... |
251 252 |
if (signal_pending_state(mode, current)) return -ERESTARTSYS; |
44c288732 NFSv4: stateful N... |
253 254 |
return 0; } |
1306729b0 sunrpc: eliminate... |
255 |
#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS) |
c44fe7055 SUNRPC: Clean up ... |
256 257 258 |
static void rpc_task_set_debuginfo(struct rpc_task *task) { static atomic_t rpc_pid; |
c44fe7055 SUNRPC: Clean up ... |
259 260 261 262 263 264 265 |
task->tk_pid = atomic_inc_return(&rpc_pid); } #else static inline void rpc_task_set_debuginfo(struct rpc_task *task) { } #endif |
e6b3c4db6 Fix a second pote... |
266 267 |
static void rpc_set_active(struct rpc_task *task) { |
82b0a4c3c SUNRPC: Add trace... |
268 |
trace_rpc_task_begin(task->tk_client, task, NULL); |
c44fe7055 SUNRPC: Clean up ... |
269 |
rpc_task_set_debuginfo(task); |
58f9612c6 SUNRPC: Move rema... |
270 |
set_bit(RPC_TASK_ACTIVE, &task->tk_runstate); |
e6b3c4db6 Fix a second pote... |
271 |
} |
44c288732 NFSv4: stateful N... |
272 273 |
/* * Mark an RPC call as having completed by clearing the 'active' bit |
bf294b41c SUNRPC: Close a r... |
274 |
* and then waking up all tasks that were sleeping. |
44c288732 NFSv4: stateful N... |
275 |
*/ |
bf294b41c SUNRPC: Close a r... |
276 |
static int rpc_complete_task(struct rpc_task *task) |
44c288732 NFSv4: stateful N... |
277 |
{ |
bf294b41c SUNRPC: Close a r... |
278 279 280 281 282 |
void *m = &task->tk_runstate; wait_queue_head_t *wq = bit_waitqueue(m, RPC_TASK_ACTIVE); struct wait_bit_key k = __WAIT_BIT_KEY_INITIALIZER(m, RPC_TASK_ACTIVE); unsigned long flags; int ret; |
82b0a4c3c SUNRPC: Add trace... |
283 |
trace_rpc_task_complete(task->tk_client, task, NULL); |
bf294b41c SUNRPC: Close a r... |
284 |
spin_lock_irqsave(&wq->lock, flags); |
e6b3c4db6 Fix a second pote... |
285 |
clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate); |
bf294b41c SUNRPC: Close a r... |
286 287 |
ret = atomic_dec_and_test(&task->tk_count); if (waitqueue_active(wq)) |
ac5be6b47 userfaultfd: reve... |
288 |
__wake_up_locked_key(wq, TASK_NORMAL, &k); |
bf294b41c SUNRPC: Close a r... |
289 290 |
spin_unlock_irqrestore(&wq->lock, flags); return ret; |
44c288732 NFSv4: stateful N... |
291 292 293 294 |
} /* * Allow callers to wait for completion of an RPC call |
bf294b41c SUNRPC: Close a r... |
295 296 297 298 |
* * Note the use of out_of_line_wait_on_bit() rather than wait_on_bit() * to enforce taking of the wq->lock and hence avoid races with * rpc_complete_task(). |
44c288732 NFSv4: stateful N... |
299 |
*/ |
c1221321b sched: Allow wait... |
300 |
int __rpc_wait_for_completion_task(struct rpc_task *task, wait_bit_action_f *action) |
44c288732 NFSv4: stateful N... |
301 302 |
{ if (action == NULL) |
150030b78 NFS: Switch from ... |
303 |
action = rpc_wait_bit_killable; |
bf294b41c SUNRPC: Close a r... |
304 |
return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE, |
150030b78 NFS: Switch from ... |
305 |
action, TASK_KILLABLE); |
44c288732 NFSv4: stateful N... |
306 |
} |
e8914c65f SUNRPC: Restrict ... |
307 |
EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task); |
44c288732 NFSv4: stateful N... |
308 |
|
1da177e4c Linux-2.6.12-rc2 |
309 310 311 |
/* * Make an RPC task runnable. * |
506026c3e sunrpc: clarify c... |
312 313 314 |
* Note: If the task is ASYNC, and is being made runnable after sitting on an * rpc_wait_queue, this must be called with the queue spinlock held to protect * the wait queue operation. |
a3c3cac5d SUNRPC: Prevent a... |
315 316 317 318 |
* Note the ordering of rpc_test_and_set_running() and rpc_clear_queued(), * which is needed to ensure that __rpc_execute() doesn't loop (due to the * lockless RPC_IS_QUEUED() test) before we've had a chance to test * the RPC_TASK_RUNNING flag. |
1da177e4c Linux-2.6.12-rc2 |
319 |
*/ |
f1dc237c6 SUNRPC: Reduce la... |
320 321 |
static void rpc_make_runnable(struct workqueue_struct *wq, struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
322 |
{ |
a3c3cac5d SUNRPC: Prevent a... |
323 |
bool need_wakeup = !rpc_test_and_set_running(task); |
1da177e4c Linux-2.6.12-rc2 |
324 |
rpc_clear_queued(task); |
a3c3cac5d SUNRPC: Prevent a... |
325 |
if (!need_wakeup) |
cc4dc59e5 Subject: Re: [PAT... |
326 |
return; |
1da177e4c Linux-2.6.12-rc2 |
327 |
if (RPC_IS_ASYNC(task)) { |
65f27f384 WorkStruct: Pass ... |
328 |
INIT_WORK(&task->u.tk_work, rpc_async_schedule); |
f1dc237c6 SUNRPC: Reduce la... |
329 |
queue_work(wq, &task->u.tk_work); |
1da177e4c Linux-2.6.12-rc2 |
330 |
} else |
96651ab34 [PATCH] RPC: Shri... |
331 |
wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED); |
1da177e4c Linux-2.6.12-rc2 |
332 333 334 |
} /* |
1da177e4c Linux-2.6.12-rc2 |
335 336 337 338 339 |
* Prepare for sleeping on a wait queue. * By always appending tasks to the list we ensure FIFO behavior. * NB: An RPC task will only receive interrupt-driven events as long * as it's on a wait queue. */ |
3b27bad7f SUNRPC: Allow cal... |
340 341 342 343 |
static void __rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task, rpc_action action, unsigned char queue_priority) |
1da177e4c Linux-2.6.12-rc2 |
344 |
{ |
46121cf7d SUNRPC: fix print... |
345 346 347 |
dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu) ", task->tk_pid, rpc_qname(q), jiffies); |
1da177e4c Linux-2.6.12-rc2 |
348 |
|
82b0a4c3c SUNRPC: Add trace... |
349 |
trace_rpc_task_sleep(task->tk_client, task, q); |
3b27bad7f SUNRPC: Allow cal... |
350 |
__rpc_add_wait_queue(q, task, queue_priority); |
1da177e4c Linux-2.6.12-rc2 |
351 |
|
f50ad4283 SUNRPC: remove BU... |
352 |
WARN_ON_ONCE(task->tk_callback != NULL); |
1da177e4c Linux-2.6.12-rc2 |
353 |
task->tk_callback = action; |
eb276c0e1 SUNRPC: Switch ta... |
354 |
__rpc_add_timer(q, task); |
1da177e4c Linux-2.6.12-rc2 |
355 356 357 |
} void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, |
5d00837b9 SUNRPC: Run rpc t... |
358 |
rpc_action action) |
1da177e4c Linux-2.6.12-rc2 |
359 |
{ |
58f9612c6 SUNRPC: Move rema... |
360 |
/* We shouldn't ever put an inactive task to sleep */ |
e454a7a83 SUNRPC: remove BU... |
361 362 363 364 365 366 |
WARN_ON_ONCE(!RPC_IS_ACTIVATED(task)); if (!RPC_IS_ACTIVATED(task)) { task->tk_status = -EIO; rpc_put_task_async(task); return; } |
e6b3c4db6 Fix a second pote... |
367 |
|
1da177e4c Linux-2.6.12-rc2 |
368 369 370 371 |
/* * Protect the queue operations. */ spin_lock_bh(&q->lock); |
3b27bad7f SUNRPC: Allow cal... |
372 |
__rpc_sleep_on_priority(q, task, action, task->tk_priority); |
1da177e4c Linux-2.6.12-rc2 |
373 374 |
spin_unlock_bh(&q->lock); } |
e8914c65f SUNRPC: Restrict ... |
375 |
EXPORT_SYMBOL_GPL(rpc_sleep_on); |
1da177e4c Linux-2.6.12-rc2 |
376 |
|
3b27bad7f SUNRPC: Allow cal... |
377 378 379 380 |
void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task, rpc_action action, int priority) { /* We shouldn't ever put an inactive task to sleep */ |
e454a7a83 SUNRPC: remove BU... |
381 382 383 384 385 386 |
WARN_ON_ONCE(!RPC_IS_ACTIVATED(task)); if (!RPC_IS_ACTIVATED(task)) { task->tk_status = -EIO; rpc_put_task_async(task); return; } |
3b27bad7f SUNRPC: Allow cal... |
387 388 389 390 391 392 393 394 |
/* * Protect the queue operations. */ spin_lock_bh(&q->lock); __rpc_sleep_on_priority(q, task, action, priority - RPC_PRIORITY_LOW); spin_unlock_bh(&q->lock); } |
1e1093c7f NFSv4.1: Don't me... |
395 |
EXPORT_SYMBOL_GPL(rpc_sleep_on_priority); |
3b27bad7f SUNRPC: Allow cal... |
396 |
|
1da177e4c Linux-2.6.12-rc2 |
397 |
/** |
f1dc237c6 SUNRPC: Reduce la... |
398 399 |
* __rpc_do_wake_up_task_on_wq - wake up a single rpc_task * @wq: workqueue on which to run task |
96ef13b28 SUNRPC: Add a new... |
400 |
* @queue: wait queue |
1da177e4c Linux-2.6.12-rc2 |
401 402 403 404 |
* @task: task to be woken up * * Caller must hold queue->lock, and have cleared the task queued flag. */ |
f1dc237c6 SUNRPC: Reduce la... |
405 406 407 |
static void __rpc_do_wake_up_task_on_wq(struct workqueue_struct *wq, struct rpc_wait_queue *queue, struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
408 |
{ |
46121cf7d SUNRPC: fix print... |
409 410 411 |
dprintk("RPC: %5u __rpc_wake_up_task (now %lu) ", task->tk_pid, jiffies); |
1da177e4c Linux-2.6.12-rc2 |
412 |
|
1da177e4c Linux-2.6.12-rc2 |
413 414 415 416 417 418 |
/* Has the task been executed yet? If not, we cannot wake it up! */ if (!RPC_IS_ACTIVATED(task)) { printk(KERN_ERR "RPC: Inactive task (%p) being woken up! ", task); return; } |
82b0a4c3c SUNRPC: Add trace... |
419 |
trace_rpc_task_wakeup(task->tk_client, task, queue); |
96ef13b28 SUNRPC: Add a new... |
420 |
__rpc_remove_wait_queue(queue, task); |
1da177e4c Linux-2.6.12-rc2 |
421 |
|
f1dc237c6 SUNRPC: Reduce la... |
422 |
rpc_make_runnable(wq, task); |
1da177e4c Linux-2.6.12-rc2 |
423 |
|
46121cf7d SUNRPC: fix print... |
424 425 |
dprintk("RPC: __rpc_wake_up_task done "); |
1da177e4c Linux-2.6.12-rc2 |
426 427 428 |
} /* |
96ef13b28 SUNRPC: Add a new... |
429 |
* Wake up a queued task while the queue lock is being held |
1da177e4c Linux-2.6.12-rc2 |
430 |
*/ |
f1dc237c6 SUNRPC: Reduce la... |
431 432 |
static void rpc_wake_up_task_on_wq_queue_locked(struct workqueue_struct *wq, struct rpc_wait_queue *queue, struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
433 |
{ |
1166fde6a SUNRPC: Add barri... |
434 435 436 |
if (RPC_IS_QUEUED(task)) { smp_rmb(); if (task->tk_waitqueue == queue) |
f1dc237c6 SUNRPC: Reduce la... |
437 |
__rpc_do_wake_up_task_on_wq(wq, queue, task); |
1166fde6a SUNRPC: Add barri... |
438 |
} |
1da177e4c Linux-2.6.12-rc2 |
439 440 441 |
} /* |
f1dc237c6 SUNRPC: Reduce la... |
442 443 444 445 446 447 448 449 |
* Wake up a queued task while the queue lock is being held */ static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task) { rpc_wake_up_task_on_wq_queue_locked(rpciod_workqueue, queue, task); } /* |
96ef13b28 SUNRPC: Add a new... |
450 |
* Wake up a task on a specific queue |
1da177e4c Linux-2.6.12-rc2 |
451 |
*/ |
96ef13b28 SUNRPC: Add a new... |
452 |
void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
453 |
{ |
5e4424af9 SUNRPC: Remove no... |
454 |
spin_lock_bh(&queue->lock); |
96ef13b28 SUNRPC: Add a new... |
455 |
rpc_wake_up_task_queue_locked(queue, task); |
5e4424af9 SUNRPC: Remove no... |
456 |
spin_unlock_bh(&queue->lock); |
1da177e4c Linux-2.6.12-rc2 |
457 |
} |
96ef13b28 SUNRPC: Add a new... |
458 459 460 |
EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task); /* |
1da177e4c Linux-2.6.12-rc2 |
461 462 |
* Wake up the next task on a priority queue. */ |
961a828df SUNRPC: Fix poten... |
463 |
static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue) |
1da177e4c Linux-2.6.12-rc2 |
464 465 466 467 468 |
{ struct list_head *q; struct rpc_task *task; /* |
3ff7576dd SUNRPC: Clean up ... |
469 |
* Service a batch of tasks from a single owner. |
1da177e4c Linux-2.6.12-rc2 |
470 471 472 473 |
*/ q = &queue->tasks[queue->priority]; if (!list_empty(q)) { task = list_entry(q->next, struct rpc_task, u.tk_wait.list); |
3ff7576dd SUNRPC: Clean up ... |
474 |
if (queue->owner == task->tk_owner) { |
1da177e4c Linux-2.6.12-rc2 |
475 476 477 478 479 480 481 |
if (--queue->nr) goto out; list_move_tail(&task->u.tk_wait.list, q); } /* * Check if we need to switch queues. */ |
c05eecf63 SUNRPC: Don't all... |
482 |
goto new_owner; |
1da177e4c Linux-2.6.12-rc2 |
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 |
} /* * Service the next queue. */ do { if (q == &queue->tasks[0]) q = &queue->tasks[queue->maxpriority]; else q = q - 1; if (!list_empty(q)) { task = list_entry(q->next, struct rpc_task, u.tk_wait.list); goto new_queue; } } while (q != &queue->tasks[queue->priority]); rpc_reset_waitqueue_priority(queue); return NULL; new_queue: rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0])); |
3ff7576dd SUNRPC: Clean up ... |
504 505 |
new_owner: rpc_set_waitqueue_owner(queue, task->tk_owner); |
1da177e4c Linux-2.6.12-rc2 |
506 |
out: |
1da177e4c Linux-2.6.12-rc2 |
507 508 |
return task; } |
961a828df SUNRPC: Fix poten... |
509 510 511 512 513 514 515 516 |
static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue) { if (RPC_IS_PRIORITY(queue)) return __rpc_find_next_queued_priority(queue); if (!list_empty(&queue->tasks[0])) return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list); return NULL; } |
1da177e4c Linux-2.6.12-rc2 |
517 |
/* |
961a828df SUNRPC: Fix poten... |
518 |
* Wake up the first task on the wait queue. |
1da177e4c Linux-2.6.12-rc2 |
519 |
*/ |
f1dc237c6 SUNRPC: Reduce la... |
520 521 |
struct rpc_task *rpc_wake_up_first_on_wq(struct workqueue_struct *wq, struct rpc_wait_queue *queue, |
961a828df SUNRPC: Fix poten... |
522 |
bool (*func)(struct rpc_task *, void *), void *data) |
1da177e4c Linux-2.6.12-rc2 |
523 524 |
{ struct rpc_task *task = NULL; |
961a828df SUNRPC: Fix poten... |
525 526 |
dprintk("RPC: wake_up_first(%p \"%s\") ", |
46121cf7d SUNRPC: fix print... |
527 |
queue, rpc_qname(queue)); |
5e4424af9 SUNRPC: Remove no... |
528 |
spin_lock_bh(&queue->lock); |
961a828df SUNRPC: Fix poten... |
529 530 531 |
task = __rpc_find_next_queued(queue); if (task != NULL) { if (func(task, data)) |
f1dc237c6 SUNRPC: Reduce la... |
532 |
rpc_wake_up_task_on_wq_queue_locked(wq, queue, task); |
961a828df SUNRPC: Fix poten... |
533 534 |
else task = NULL; |
1da177e4c Linux-2.6.12-rc2 |
535 |
} |
5e4424af9 SUNRPC: Remove no... |
536 |
spin_unlock_bh(&queue->lock); |
1da177e4c Linux-2.6.12-rc2 |
537 538 539 |
return task; } |
f1dc237c6 SUNRPC: Reduce la... |
540 541 542 543 544 545 546 547 548 |
/* * Wake up the first task on the wait queue. */ struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue, bool (*func)(struct rpc_task *, void *), void *data) { return rpc_wake_up_first_on_wq(rpciod_workqueue, queue, func, data); } |
961a828df SUNRPC: Fix poten... |
549 550 551 552 553 554 555 556 557 558 559 560 561 562 |
EXPORT_SYMBOL_GPL(rpc_wake_up_first); static bool rpc_wake_up_next_func(struct rpc_task *task, void *data) { return true; } /* * Wake up the next task on the wait queue. */ struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue) { return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL); } |
e8914c65f SUNRPC: Restrict ... |
563 |
EXPORT_SYMBOL_GPL(rpc_wake_up_next); |
1da177e4c Linux-2.6.12-rc2 |
564 565 566 567 568 569 570 571 572 |
/** * rpc_wake_up - wake up all rpc_tasks * @queue: rpc_wait_queue on which the tasks are sleeping * * Grabs queue->lock */ void rpc_wake_up(struct rpc_wait_queue *queue) { |
1da177e4c Linux-2.6.12-rc2 |
573 |
struct list_head *head; |
e6d83d556 [PATCH] SUNRPC: F... |
574 |
|
5e4424af9 SUNRPC: Remove no... |
575 |
spin_lock_bh(&queue->lock); |
1da177e4c Linux-2.6.12-rc2 |
576 577 |
head = &queue->tasks[queue->maxpriority]; for (;;) { |
540a0f758 SUNRPC: We must n... |
578 579 580 581 582 |
while (!list_empty(head)) { struct rpc_task *task; task = list_first_entry(head, struct rpc_task, u.tk_wait.list); |
96ef13b28 SUNRPC: Add a new... |
583 |
rpc_wake_up_task_queue_locked(queue, task); |
540a0f758 SUNRPC: We must n... |
584 |
} |
1da177e4c Linux-2.6.12-rc2 |
585 586 587 588 |
if (head == &queue->tasks[0]) break; head--; } |
5e4424af9 SUNRPC: Remove no... |
589 |
spin_unlock_bh(&queue->lock); |
1da177e4c Linux-2.6.12-rc2 |
590 |
} |
e8914c65f SUNRPC: Restrict ... |
591 |
EXPORT_SYMBOL_GPL(rpc_wake_up); |
1da177e4c Linux-2.6.12-rc2 |
592 593 594 595 596 597 598 599 600 601 602 |
/** * rpc_wake_up_status - wake up all rpc_tasks and set their status value. * @queue: rpc_wait_queue on which the tasks are sleeping * @status: status value to set * * Grabs queue->lock */ void rpc_wake_up_status(struct rpc_wait_queue *queue, int status) { struct list_head *head; |
1da177e4c Linux-2.6.12-rc2 |
603 |
|
5e4424af9 SUNRPC: Remove no... |
604 |
spin_lock_bh(&queue->lock); |
1da177e4c Linux-2.6.12-rc2 |
605 606 |
head = &queue->tasks[queue->maxpriority]; for (;;) { |
540a0f758 SUNRPC: We must n... |
607 608 609 610 611 |
while (!list_empty(head)) { struct rpc_task *task; task = list_first_entry(head, struct rpc_task, u.tk_wait.list); |
1da177e4c Linux-2.6.12-rc2 |
612 |
task->tk_status = status; |
96ef13b28 SUNRPC: Add a new... |
613 |
rpc_wake_up_task_queue_locked(queue, task); |
1da177e4c Linux-2.6.12-rc2 |
614 615 616 617 618 |
} if (head == &queue->tasks[0]) break; head--; } |
5e4424af9 SUNRPC: Remove no... |
619 |
spin_unlock_bh(&queue->lock); |
1da177e4c Linux-2.6.12-rc2 |
620 |
} |
e8914c65f SUNRPC: Restrict ... |
621 |
EXPORT_SYMBOL_GPL(rpc_wake_up_status); |
1da177e4c Linux-2.6.12-rc2 |
622 |
|
36df9aae3 SUNRPC: Add a tim... |
623 624 625 626 627 628 629 630 631 632 633 |
static void __rpc_queue_timer_fn(unsigned long ptr) { struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr; struct rpc_task *task, *n; unsigned long expires, now, timeo; spin_lock(&queue->lock); expires = now = jiffies; list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) { timeo = task->u.tk_wait.expires; if (time_after_eq(now, timeo)) { |
36df9aae3 SUNRPC: Add a tim... |
634 635 636 637 638 639 640 641 642 643 644 645 646 |
dprintk("RPC: %5u timeout ", task->tk_pid); task->tk_status = -ETIMEDOUT; rpc_wake_up_task_queue_locked(queue, task); continue; } if (expires == now || time_after(expires, timeo)) expires = timeo; } if (!list_empty(&queue->timer_list.list)) rpc_set_queue_timer(queue, expires); spin_unlock(&queue->lock); } |
8014793b1 SUNRPC: rpc_delay... |
647 648 |
static void __rpc_atrun(struct rpc_task *task) { |
6bd144160 SUNRPC: Don't let... |
649 650 |
if (task->tk_status == -ETIMEDOUT) task->tk_status = 0; |
8014793b1 SUNRPC: rpc_delay... |
651 |
} |
1da177e4c Linux-2.6.12-rc2 |
652 653 654 |
/* * Run a task at a later time */ |
8014793b1 SUNRPC: rpc_delay... |
655 |
void rpc_delay(struct rpc_task *task, unsigned long delay) |
1da177e4c Linux-2.6.12-rc2 |
656 657 |
{ task->tk_timeout = delay; |
5d00837b9 SUNRPC: Run rpc t... |
658 |
rpc_sleep_on(&delay_queue, task, __rpc_atrun); |
1da177e4c Linux-2.6.12-rc2 |
659 |
} |
e8914c65f SUNRPC: Restrict ... |
660 |
EXPORT_SYMBOL_GPL(rpc_delay); |
1da177e4c Linux-2.6.12-rc2 |
661 |
|
1da177e4c Linux-2.6.12-rc2 |
662 |
/* |
4ce70ada1 SUNRPC: Further c... |
663 664 |
* Helper to call task->tk_ops->rpc_call_prepare */ |
aae2006e9 nfs41: sunrpc: Ex... |
665 |
void rpc_prepare_task(struct rpc_task *task) |
4ce70ada1 SUNRPC: Further c... |
666 667 668 |
{ task->tk_ops->rpc_call_prepare(task, task->tk_calldata); } |
7fdcf13b2 SUNRPC: Fix the e... |
669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 |
static void rpc_init_task_statistics(struct rpc_task *task) { /* Initialize retry counters */ task->tk_garb_retry = 2; task->tk_cred_retry = 2; task->tk_rebind_retry = 2; /* starting timestamp */ task->tk_start = ktime_get(); } static void rpc_reset_task_statistics(struct rpc_task *task) { task->tk_timeouts = 0; task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_KILLED|RPC_TASK_SENT); rpc_init_task_statistics(task); } |
4ce70ada1 SUNRPC: Further c... |
689 |
/* |
963d8fe53 RPC: Clean up RPC... |
690 |
* Helper that calls task->tk_ops->rpc_call_done if it exists |
d05fdb0ce [PATCH] RPC: Fix ... |
691 |
*/ |
abbcf28f2 SUNRPC: Yet more ... |
692 |
void rpc_exit_task(struct rpc_task *task) |
d05fdb0ce [PATCH] RPC: Fix ... |
693 |
{ |
abbcf28f2 SUNRPC: Yet more ... |
694 |
task->tk_action = NULL; |
963d8fe53 RPC: Clean up RPC... |
695 696 |
if (task->tk_ops->rpc_call_done != NULL) { task->tk_ops->rpc_call_done(task, task->tk_calldata); |
d05fdb0ce [PATCH] RPC: Fix ... |
697 |
if (task->tk_action != NULL) { |
abbcf28f2 SUNRPC: Yet more ... |
698 699 700 |
WARN_ON(RPC_ASSASSINATED(task)); /* Always release the RPC slot and buffer memory */ xprt_release(task); |
7fdcf13b2 SUNRPC: Fix the e... |
701 |
rpc_reset_task_statistics(task); |
d05fdb0ce [PATCH] RPC: Fix ... |
702 703 |
} } |
d05fdb0ce [PATCH] RPC: Fix ... |
704 |
} |
d9b6cd946 SUNRPC: Ensure th... |
705 706 707 708 709 710 711 712 713 |
void rpc_exit(struct rpc_task *task, int status) { task->tk_status = status; task->tk_action = rpc_exit_task; if (RPC_IS_QUEUED(task)) rpc_wake_up_queued_task(task->tk_waitqueue, task); } EXPORT_SYMBOL_GPL(rpc_exit); |
d05fdb0ce [PATCH] RPC: Fix ... |
714 |
|
bbd5a1f9f SUNRPC: Fix up mi... |
715 716 |
void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata) { |
a86dc496b SUNRPC: Remove th... |
717 |
if (ops->rpc_release != NULL) |
bbd5a1f9f SUNRPC: Fix up mi... |
718 |
ops->rpc_release(calldata); |
bbd5a1f9f SUNRPC: Fix up mi... |
719 |
} |
d05fdb0ce [PATCH] RPC: Fix ... |
720 |
/* |
1da177e4c Linux-2.6.12-rc2 |
721 722 |
* This is the RPC `scheduler' (or rather, the finite state machine). */ |
2efef837f RPC: Clean up rpc... |
723 |
static void __rpc_execute(struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
724 |
{ |
eb9b55ab4 SUNRPC: Tighten u... |
725 726 727 |
struct rpc_wait_queue *queue; int task_is_async = RPC_IS_ASYNC(task); int status = 0; |
1da177e4c Linux-2.6.12-rc2 |
728 |
|
46121cf7d SUNRPC: fix print... |
729 730 731 |
dprintk("RPC: %5u __rpc_execute flags=0x%x ", task->tk_pid, task->tk_flags); |
1da177e4c Linux-2.6.12-rc2 |
732 |
|
2bd4eef87 SUNRPC: remove BU... |
733 734 735 |
WARN_ON_ONCE(RPC_IS_QUEUED(task)); if (RPC_IS_QUEUED(task)) return; |
1da177e4c Linux-2.6.12-rc2 |
736 |
|
d05fdb0ce [PATCH] RPC: Fix ... |
737 |
for (;;) { |
b55c59892 SUNRPC: Fix a rac... |
738 |
void (*do_action)(struct rpc_task *); |
1da177e4c Linux-2.6.12-rc2 |
739 740 |
/* |
b55c59892 SUNRPC: Fix a rac... |
741 |
* Execute any pending callback first. |
1da177e4c Linux-2.6.12-rc2 |
742 |
*/ |
b55c59892 SUNRPC: Fix a rac... |
743 744 745 |
do_action = task->tk_callback; task->tk_callback = NULL; if (do_action == NULL) { |
e020c6800 SUNRPC: Ensure we... |
746 747 |
/* * Perform the next FSM step. |
b55c59892 SUNRPC: Fix a rac... |
748 749 750 |
* tk_action may be NULL if the task has been killed. * In particular, note that rpc_killall_tasks may * do this at any time, so beware when dereferencing. |
e020c6800 SUNRPC: Ensure we... |
751 |
*/ |
b55c59892 SUNRPC: Fix a rac... |
752 753 |
do_action = task->tk_action; if (do_action == NULL) |
1da177e4c Linux-2.6.12-rc2 |
754 |
break; |
1da177e4c Linux-2.6.12-rc2 |
755 |
} |
82b0a4c3c SUNRPC: Add trace... |
756 |
trace_rpc_task_run_action(task->tk_client, task, task->tk_action); |
b55c59892 SUNRPC: Fix a rac... |
757 |
do_action(task); |
1da177e4c Linux-2.6.12-rc2 |
758 759 760 761 762 763 |
/* * Lockless check for whether task is sleeping or not. */ if (!RPC_IS_QUEUED(task)) continue; |
eb9b55ab4 SUNRPC: Tighten u... |
764 765 766 767 768 769 770 771 772 773 774 775 776 |
/* * The queue->lock protects against races with * rpc_make_runnable(). * * Note that once we clear RPC_TASK_RUNNING on an asynchronous * rpc_task, rpc_make_runnable() can assign it to a * different workqueue. We therefore cannot assume that the * rpc_task pointer may still be dereferenced. */ queue = task->tk_waitqueue; spin_lock_bh(&queue->lock); if (!RPC_IS_QUEUED(task)) { spin_unlock_bh(&queue->lock); |
1da177e4c Linux-2.6.12-rc2 |
777 778 |
continue; } |
eb9b55ab4 SUNRPC: Tighten u... |
779 780 781 782 |
rpc_clear_running(task); spin_unlock_bh(&queue->lock); if (task_is_async) return; |
1da177e4c Linux-2.6.12-rc2 |
783 784 |
/* sync task: sleep here */ |
46121cf7d SUNRPC: fix print... |
785 786 |
dprintk("RPC: %5u sync task going to sleep ", task->tk_pid); |
96651ab34 [PATCH] RPC: Shri... |
787 |
status = out_of_line_wait_on_bit(&task->tk_runstate, |
150030b78 NFS: Switch from ... |
788 789 |
RPC_TASK_QUEUED, rpc_wait_bit_killable, TASK_KILLABLE); |
96651ab34 [PATCH] RPC: Shri... |
790 |
if (status == -ERESTARTSYS) { |
1da177e4c Linux-2.6.12-rc2 |
791 792 793 794 795 796 |
/* * When a sync task receives a signal, it exits with * -ERESTARTSYS. In order to catch any callbacks that * clean up after sleeping on some queue, we don't * break the loop here, but go around once more. */ |
46121cf7d SUNRPC: fix print... |
797 798 |
dprintk("RPC: %5u got signal ", task->tk_pid); |
96651ab34 [PATCH] RPC: Shri... |
799 800 |
task->tk_flags |= RPC_TASK_KILLED; rpc_exit(task, -ERESTARTSYS); |
1da177e4c Linux-2.6.12-rc2 |
801 |
} |
46121cf7d SUNRPC: fix print... |
802 803 |
dprintk("RPC: %5u sync task resuming ", task->tk_pid); |
1da177e4c Linux-2.6.12-rc2 |
804 |
} |
46121cf7d SUNRPC: fix print... |
805 806 807 |
dprintk("RPC: %5u return %d, status %d ", task->tk_pid, status, task->tk_status); |
1da177e4c Linux-2.6.12-rc2 |
808 809 |
/* Release all resources associated with the task */ rpc_release_task(task); |
1da177e4c Linux-2.6.12-rc2 |
810 811 812 813 814 815 816 817 818 819 820 |
} /* * User-visible entry point to the scheduler. * * This may be called recursively if e.g. an async NFS task updates * the attributes and finds that dirty pages must be flushed. * NOTE: Upon exit of this function the task is guaranteed to be * released. In particular note that tk_release() will have * been called, so your task memory may have been freed. */ |
2efef837f RPC: Clean up rpc... |
821 |
void rpc_execute(struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
822 |
{ |
a76580fbf SUNRPC: Fix a pot... |
823 |
bool is_async = RPC_IS_ASYNC(task); |
44c288732 NFSv4: stateful N... |
824 |
rpc_set_active(task); |
f1dc237c6 SUNRPC: Reduce la... |
825 |
rpc_make_runnable(rpciod_workqueue, task); |
a76580fbf SUNRPC: Fix a pot... |
826 |
if (!is_async) |
d6a1ed08c SUNRPC: Reduce as... |
827 |
__rpc_execute(task); |
1da177e4c Linux-2.6.12-rc2 |
828 |
} |
65f27f384 WorkStruct: Pass ... |
829 |
static void rpc_async_schedule(struct work_struct *work) |
1da177e4c Linux-2.6.12-rc2 |
830 |
{ |
65f27f384 WorkStruct: Pass ... |
831 |
__rpc_execute(container_of(work, struct rpc_task, u.tk_work)); |
1da177e4c Linux-2.6.12-rc2 |
832 |
} |
021071483 SUNRPC: switchabl... |
833 |
/** |
5fe6eaa1f SUNRPC: Generaliz... |
834 835 836 837 838 839 |
* rpc_malloc - allocate RPC buffer resources * @task: RPC task * * A single memory region is allocated, which is split between the * RPC call and RPC reply that this task is being used for. When * this RPC is retired, the memory is released by calling rpc_free. |
1da177e4c Linux-2.6.12-rc2 |
840 |
* |
c5a4dd8b7 SUNRPC: Eliminate... |
841 |
* To prevent rpciod from hanging, this allocator never sleeps, |
5fe6eaa1f SUNRPC: Generaliz... |
842 843 844 |
* returning -ENOMEM and suppressing warning if the request cannot * be serviced immediately. The caller can arrange to sleep in a * way that is safe for rpciod. |
c5a4dd8b7 SUNRPC: Eliminate... |
845 846 847 848 849 |
* * Most requests are 'small' (under 2KiB) and can be serviced from a * mempool, ensuring that NFS reads and writes can always proceed, * and that there is good locality of reference for these buffers. * |
1da177e4c Linux-2.6.12-rc2 |
850 |
* In order to avoid memory starvation triggering more writebacks of |
c5a4dd8b7 SUNRPC: Eliminate... |
851 |
* NFS requests, we avoid using GFP_KERNEL. |
1da177e4c Linux-2.6.12-rc2 |
852 |
*/ |
5fe6eaa1f SUNRPC: Generaliz... |
853 |
int rpc_malloc(struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
854 |
{ |
5fe6eaa1f SUNRPC: Generaliz... |
855 856 |
struct rpc_rqst *rqst = task->tk_rqstp; size_t size = rqst->rq_callsize + rqst->rq_rcvsize; |
aa3d1faeb SUNRPC: Fix point... |
857 |
struct rpc_buffer *buf; |
c4a7ca774 SUNRPC: Allow wai... |
858 |
gfp_t gfp = GFP_NOIO | __GFP_NOWARN; |
a564b8f03 nfs: enable swap ... |
859 860 |
if (RPC_IS_SWAPPER(task)) |
c4a7ca774 SUNRPC: Allow wai... |
861 |
gfp = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN; |
1da177e4c Linux-2.6.12-rc2 |
862 |
|
aa3d1faeb SUNRPC: Fix point... |
863 |
size += sizeof(struct rpc_buffer); |
c5a4dd8b7 SUNRPC: Eliminate... |
864 865 |
if (size <= RPC_BUFFER_MAXSIZE) buf = mempool_alloc(rpc_buffer_mempool, gfp); |
1da177e4c Linux-2.6.12-rc2 |
866 |
else |
c5a4dd8b7 SUNRPC: Eliminate... |
867 |
buf = kmalloc(size, gfp); |
ddce40df6 sunrpc: fix crash... |
868 869 |
if (!buf) |
5fe6eaa1f SUNRPC: Generaliz... |
870 |
return -ENOMEM; |
ddce40df6 sunrpc: fix crash... |
871 |
|
aa3d1faeb SUNRPC: Fix point... |
872 |
buf->len = size; |
215d06780 Fix sunrpc warnin... |
873 874 |
dprintk("RPC: %5u allocated buffer of size %zu at %p ", |
c5a4dd8b7 SUNRPC: Eliminate... |
875 |
task->tk_pid, size, buf); |
5fe6eaa1f SUNRPC: Generaliz... |
876 |
rqst->rq_buffer = buf->data; |
68778945e SUNRPC: Separate ... |
877 |
rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize; |
5fe6eaa1f SUNRPC: Generaliz... |
878 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
879 |
} |
124448097 SUNRPC: add EXPOR... |
880 |
EXPORT_SYMBOL_GPL(rpc_malloc); |
1da177e4c Linux-2.6.12-rc2 |
881 |
|
021071483 SUNRPC: switchabl... |
882 |
/** |
3435c74ae SUNRPC: Generaliz... |
883 884 |
* rpc_free - free RPC buffer resources allocated via rpc_malloc * @task: RPC task |
021071483 SUNRPC: switchabl... |
885 886 |
* */ |
3435c74ae SUNRPC: Generaliz... |
887 |
void rpc_free(struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
888 |
{ |
3435c74ae SUNRPC: Generaliz... |
889 |
void *buffer = task->tk_rqstp->rq_buffer; |
aa3d1faeb SUNRPC: Fix point... |
890 891 |
size_t size; struct rpc_buffer *buf; |
021071483 SUNRPC: switchabl... |
892 |
|
aa3d1faeb SUNRPC: Fix point... |
893 894 |
buf = container_of(buffer, struct rpc_buffer, data); size = buf->len; |
c5a4dd8b7 SUNRPC: Eliminate... |
895 |
|
215d06780 Fix sunrpc warnin... |
896 897 |
dprintk("RPC: freeing buffer of size %zu at %p ", |
c5a4dd8b7 SUNRPC: Eliminate... |
898 |
size, buf); |
aa3d1faeb SUNRPC: Fix point... |
899 |
|
c5a4dd8b7 SUNRPC: Eliminate... |
900 901 902 903 |
if (size <= RPC_BUFFER_MAXSIZE) mempool_free(buf, rpc_buffer_mempool); else kfree(buf); |
1da177e4c Linux-2.6.12-rc2 |
904 |
} |
124448097 SUNRPC: add EXPOR... |
905 |
EXPORT_SYMBOL_GPL(rpc_free); |
1da177e4c Linux-2.6.12-rc2 |
906 907 908 909 |
/* * Creation and deletion of RPC task structures */ |
47fe06483 SUNRPC: Unexport ... |
910 |
static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data) |
1da177e4c Linux-2.6.12-rc2 |
911 912 |
{ memset(task, 0, sizeof(*task)); |
44c288732 NFSv4: stateful N... |
913 |
atomic_set(&task->tk_count, 1); |
84115e1cd SUNRPC: Cleanup o... |
914 915 916 |
task->tk_flags = task_setup_data->flags; task->tk_ops = task_setup_data->callback_ops; task->tk_calldata = task_setup_data->callback_data; |
6529eba08 SUNRPC: Move rpc_... |
917 |
INIT_LIST_HEAD(&task->tk_task); |
1da177e4c Linux-2.6.12-rc2 |
918 |
|
3ff7576dd SUNRPC: Clean up ... |
919 920 |
task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW; task->tk_owner = current->tgid; |
1da177e4c Linux-2.6.12-rc2 |
921 922 |
/* Initialize workqueue for async tasks */ |
32bfb5c0f SUNRPC: Allow the... |
923 |
task->tk_workqueue = task_setup_data->workqueue; |
1da177e4c Linux-2.6.12-rc2 |
924 |
|
9d61498d5 SUNRPC: Allow cal... |
925 |
task->tk_xprt = xprt_get(task_setup_data->rpc_xprt); |
84115e1cd SUNRPC: Cleanup o... |
926 927 |
if (task->tk_ops->rpc_call_prepare != NULL) task->tk_action = rpc_prepare_task; |
963d8fe53 RPC: Clean up RPC... |
928 |
|
7fdcf13b2 SUNRPC: Fix the e... |
929 |
rpc_init_task_statistics(task); |
ef759a2e5 SUNRPC: introduce... |
930 |
|
46121cf7d SUNRPC: fix print... |
931 932 |
dprintk("RPC: new task initialized, procpid %u ", |
ba25f9dcc Use helpers to ob... |
933 |
task_pid_nr(current)); |
1da177e4c Linux-2.6.12-rc2 |
934 935 936 937 938 |
} static struct rpc_task * rpc_alloc_task(void) { |
a564b8f03 nfs: enable swap ... |
939 |
return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOIO); |
1da177e4c Linux-2.6.12-rc2 |
940 |
} |
1da177e4c Linux-2.6.12-rc2 |
941 |
/* |
90c5755ff SUNRPC: Kill rpc_... |
942 |
* Create a new task for the specified client. |
1da177e4c Linux-2.6.12-rc2 |
943 |
*/ |
84115e1cd SUNRPC: Cleanup o... |
944 |
struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data) |
1da177e4c Linux-2.6.12-rc2 |
945 |
{ |
e8f5d77c8 SUNRPC: allow the... |
946 947 948 949 950 |
struct rpc_task *task = setup_data->task; unsigned short flags = 0; if (task == NULL) { task = rpc_alloc_task(); |
19445b99b SUNRPC: Cleanup -... |
951 952 953 954 955 |
if (task == NULL) { rpc_release_calldata(setup_data->callback_ops, setup_data->callback_data); return ERR_PTR(-ENOMEM); } |
e8f5d77c8 SUNRPC: allow the... |
956 957 |
flags = RPC_TASK_DYNAMIC; } |
1da177e4c Linux-2.6.12-rc2 |
958 |
|
84115e1cd SUNRPC: Cleanup o... |
959 |
rpc_init_task(task, setup_data); |
e8f5d77c8 SUNRPC: allow the... |
960 |
task->tk_flags |= flags; |
46121cf7d SUNRPC: fix print... |
961 962 |
dprintk("RPC: allocated task %p ", task); |
1da177e4c Linux-2.6.12-rc2 |
963 |
return task; |
1da177e4c Linux-2.6.12-rc2 |
964 |
} |
c6567ed14 SUNRPC: Ensure th... |
965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 |
/* * rpc_free_task - release rpc task and perform cleanups * * Note that we free up the rpc_task _after_ rpc_release_calldata() * in order to work around a workqueue dependency issue. * * Tejun Heo states: * "Workqueue currently considers two work items to be the same if they're * on the same address and won't execute them concurrently - ie. it * makes a work item which is queued again while being executed wait * for the previous execution to complete. * * If a work function frees the work item, and then waits for an event * which should be performed by another work item and *that* work item * recycles the freed work item, it can create a false dependency loop. * There really is no reliable way to detect this short of verifying * every memory free." * */ |
32bfb5c0f SUNRPC: Allow the... |
984 |
static void rpc_free_task(struct rpc_task *task) |
1da177e4c Linux-2.6.12-rc2 |
985 |
{ |
c6567ed14 SUNRPC: Ensure th... |
986 987 988 |
unsigned short tk_flags = task->tk_flags; rpc_release_calldata(task->tk_ops, task->tk_calldata); |
1da177e4c Linux-2.6.12-rc2 |
989 |
|
c6567ed14 SUNRPC: Ensure th... |
990 |
if (tk_flags & RPC_TASK_DYNAMIC) { |
5e4424af9 SUNRPC: Remove no... |
991 992 993 994 |
dprintk("RPC: %5u freeing task ", task->tk_pid); mempool_free(task, rpc_task_mempool); } |
32bfb5c0f SUNRPC: Allow the... |
995 996 997 998 999 1000 |
} static void rpc_async_release(struct work_struct *work) { rpc_free_task(container_of(work, struct rpc_task, u.tk_work)); } |
bf294b41c SUNRPC: Close a r... |
1001 |
static void rpc_release_resources_task(struct rpc_task *task) |
32bfb5c0f SUNRPC: Allow the... |
1002 |
{ |
87ed50036 SUNRPC: Ensure we... |
1003 |
xprt_release(task); |
a271c5a0d NFS: Ensure that ... |
1004 |
if (task->tk_msg.rpc_cred) { |
a17c2153d SUNRPC: Move the ... |
1005 |
put_rpccred(task->tk_msg.rpc_cred); |
a271c5a0d NFS: Ensure that ... |
1006 1007 |
task->tk_msg.rpc_cred = NULL; } |
58f9612c6 SUNRPC: Move rema... |
1008 |
rpc_task_release_client(task); |
bf294b41c SUNRPC: Close a r... |
1009 1010 1011 1012 1013 1014 |
} static void rpc_final_put_task(struct rpc_task *task, struct workqueue_struct *q) { if (q != NULL) { |
32bfb5c0f SUNRPC: Allow the... |
1015 |
INIT_WORK(&task->u.tk_work, rpc_async_release); |
bf294b41c SUNRPC: Close a r... |
1016 |
queue_work(q, &task->u.tk_work); |
32bfb5c0f SUNRPC: Allow the... |
1017 1018 |
} else rpc_free_task(task); |
e6b3c4db6 Fix a second pote... |
1019 |
} |
bf294b41c SUNRPC: Close a r... |
1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 |
static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q) { if (atomic_dec_and_test(&task->tk_count)) { rpc_release_resources_task(task); rpc_final_put_task(task, q); } } void rpc_put_task(struct rpc_task *task) { rpc_do_put_task(task, NULL); } |
e8914c65f SUNRPC: Restrict ... |
1033 |
EXPORT_SYMBOL_GPL(rpc_put_task); |
e6b3c4db6 Fix a second pote... |
1034 |
|
bf294b41c SUNRPC: Close a r... |
1035 1036 1037 1038 1039 |
void rpc_put_task_async(struct rpc_task *task) { rpc_do_put_task(task, task->tk_workqueue); } EXPORT_SYMBOL_GPL(rpc_put_task_async); |
bde8f00ce [PATCH] NFS: Fix ... |
1040 |
static void rpc_release_task(struct rpc_task *task) |
e6b3c4db6 Fix a second pote... |
1041 |
{ |
46121cf7d SUNRPC: fix print... |
1042 1043 |
dprintk("RPC: %5u release task ", task->tk_pid); |
1da177e4c Linux-2.6.12-rc2 |
1044 |
|
0a0c2a57b SUNRPC: remove BU... |
1045 |
WARN_ON_ONCE(RPC_IS_QUEUED(task)); |
1da177e4c Linux-2.6.12-rc2 |
1046 |
|
bf294b41c SUNRPC: Close a r... |
1047 |
rpc_release_resources_task(task); |
e6b3c4db6 Fix a second pote... |
1048 |
|
bf294b41c SUNRPC: Close a r... |
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 |
/* * Note: at this point we have been removed from rpc_clnt->cl_tasks, * so it should be safe to use task->tk_count as a test for whether * or not any other processes still hold references to our rpc_task. */ if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) { /* Wake up anyone who may be waiting for task completion */ if (!rpc_complete_task(task)) return; } else { if (!atomic_dec_and_test(&task->tk_count)) return; } rpc_final_put_task(task, task->tk_workqueue); |
1da177e4c Linux-2.6.12-rc2 |
1063 |
} |
b247bbf1d SUNRPC: Fix a rac... |
1064 1065 1066 1067 1068 1069 1070 1071 1072 |
int rpciod_up(void) { return try_module_get(THIS_MODULE) ? 0 : -EINVAL; } void rpciod_down(void) { module_put(THIS_MODULE); } |
1da177e4c Linux-2.6.12-rc2 |
1073 |
/* |
b247bbf1d SUNRPC: Fix a rac... |
1074 |
* Start up the rpciod workqueue. |
1da177e4c Linux-2.6.12-rc2 |
1075 |
*/ |
b247bbf1d SUNRPC: Fix a rac... |
1076 |
static int rpciod_start(void) |
1da177e4c Linux-2.6.12-rc2 |
1077 1078 |
{ struct workqueue_struct *wq; |
ab418d70e SUNRPC: Optimise ... |
1079 |
|
1da177e4c Linux-2.6.12-rc2 |
1080 1081 1082 |
/* * Create the rpciod thread and wait for it to start. */ |
ab418d70e SUNRPC: Optimise ... |
1083 1084 |
dprintk("RPC: creating workqueue rpciod "); |
40a5f1b19 SUNRPC: RPC trans... |
1085 1086 1087 |
wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM, 0); if (!wq) goto out_failed; |
1da177e4c Linux-2.6.12-rc2 |
1088 |
rpciod_workqueue = wq; |
40a5f1b19 SUNRPC: RPC trans... |
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 |
/* Note: highpri because network receive is latency sensitive */ wq = alloc_workqueue("xprtiod", WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); if (!wq) goto free_rpciod; xprtiod_workqueue = wq; return 1; free_rpciod: wq = rpciod_workqueue; rpciod_workqueue = NULL; destroy_workqueue(wq); out_failed: return 0; |
1da177e4c Linux-2.6.12-rc2 |
1101 |
} |
b247bbf1d SUNRPC: Fix a rac... |
1102 |
static void rpciod_stop(void) |
1da177e4c Linux-2.6.12-rc2 |
1103 |
{ |
b247bbf1d SUNRPC: Fix a rac... |
1104 |
struct workqueue_struct *wq = NULL; |
ab418d70e SUNRPC: Optimise ... |
1105 |
|
b247bbf1d SUNRPC: Fix a rac... |
1106 1107 |
if (rpciod_workqueue == NULL) return; |
ab418d70e SUNRPC: Optimise ... |
1108 1109 |
dprintk("RPC: destroying workqueue rpciod "); |
1da177e4c Linux-2.6.12-rc2 |
1110 |
|
b247bbf1d SUNRPC: Fix a rac... |
1111 1112 1113 |
wq = rpciod_workqueue; rpciod_workqueue = NULL; destroy_workqueue(wq); |
40a5f1b19 SUNRPC: RPC trans... |
1114 1115 1116 |
wq = xprtiod_workqueue; xprtiod_workqueue = NULL; destroy_workqueue(wq); |
1da177e4c Linux-2.6.12-rc2 |
1117 |
} |
1da177e4c Linux-2.6.12-rc2 |
1118 1119 1120 |
void rpc_destroy_mempool(void) { |
b247bbf1d SUNRPC: Fix a rac... |
1121 |
rpciod_stop(); |
17a9618e9 SUNRPC: drop null... |
1122 1123 1124 1125 |
mempool_destroy(rpc_buffer_mempool); mempool_destroy(rpc_task_mempool); kmem_cache_destroy(rpc_task_slabp); kmem_cache_destroy(rpc_buffer_slabp); |
f6a1cc893 SUNRPC: Add a (em... |
1126 |
rpc_destroy_wait_queue(&delay_queue); |
1da177e4c Linux-2.6.12-rc2 |
1127 1128 1129 1130 1131 |
} int rpc_init_mempool(void) { |
f6a1cc893 SUNRPC: Add a (em... |
1132 1133 1134 1135 1136 1137 1138 |
/* * The following is not strictly a mempool initialisation, * but there is no harm in doing it here */ rpc_init_wait_queue(&delay_queue, "delayq"); if (!rpciod_start()) goto err_nomem; |
1da177e4c Linux-2.6.12-rc2 |
1139 1140 1141 |
rpc_task_slabp = kmem_cache_create("rpc_tasks", sizeof(struct rpc_task), 0, SLAB_HWCACHE_ALIGN, |
20c2df83d mm: Remove slab d... |
1142 |
NULL); |
1da177e4c Linux-2.6.12-rc2 |
1143 1144 1145 1146 1147 |
if (!rpc_task_slabp) goto err_nomem; rpc_buffer_slabp = kmem_cache_create("rpc_buffers", RPC_BUFFER_MAXSIZE, 0, SLAB_HWCACHE_ALIGN, |
20c2df83d mm: Remove slab d... |
1148 |
NULL); |
1da177e4c Linux-2.6.12-rc2 |
1149 1150 |
if (!rpc_buffer_slabp) goto err_nomem; |
93d2341c7 [PATCH] mempool: ... |
1151 1152 |
rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE, rpc_task_slabp); |
1da177e4c Linux-2.6.12-rc2 |
1153 1154 |
if (!rpc_task_mempool) goto err_nomem; |
93d2341c7 [PATCH] mempool: ... |
1155 1156 |
rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE, rpc_buffer_slabp); |
1da177e4c Linux-2.6.12-rc2 |
1157 1158 1159 1160 1161 1162 1163 |
if (!rpc_buffer_mempool) goto err_nomem; return 0; err_nomem: rpc_destroy_mempool(); return -ENOMEM; } |