Blame view
net/sunrpc/xprt.c
51.3 KB
457c89965
|
1 |
// SPDX-License-Identifier: GPL-2.0-only |
1da177e4c
|
2 3 4 5 6 7 8 9 10 11 12 13 |
/* * linux/net/sunrpc/xprt.c * * This is a generic RPC call interface supporting congestion avoidance, * and asynchronous calls. * * The interface works like this: * * - When a process places a call, it allocates a request slot if * one is available. Otherwise, it sleeps on the backlog queue * (xprt_reserve). * - Next, the caller puts together the RPC message, stuffs it into |
55aa4f58a
|
14 15 |
* the request struct, and calls xprt_transmit(). * - xprt_transmit sends the message and installs the caller on the |
55ae1aabf
|
16 17 18 |
* transport's wait list. At the same time, if a reply is expected, * it installs a timer that is run after the packet's timeout has * expired. |
1da177e4c
|
19 |
* - When a packet arrives, the data_ready handler walks the list of |
55aa4f58a
|
20 |
* pending requests for that transport. If a matching XID is found, the |
1da177e4c
|
21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
* caller is woken up, and the timer removed. * - When no reply arrives within the timeout interval, the timer is * fired by the kernel and runs xprt_timer(). It either adjusts the * timeout values (minor timeout) or wakes up the caller with a status * of -ETIMEDOUT. * - When the caller receives a notification from RPC that a reply arrived, * it should release the RPC slot, and process the reply. * If the call timed out, it may choose to retry the operation by * adjusting the initial timeout value, and simply calling rpc_call * again. * * Support for async RPC is done through a set of RPC-specific scheduling * primitives that `transparently' work for processes as well as async * tasks that rely on callbacks. * * Copyright (C) 1995-1997, Olaf Kirch <okir@monad.swb.de> |
55aa4f58a
|
37 38 |
* * Transport switch API copyright (C) 2005, Chuck Lever <cel@netapp.com> |
1da177e4c
|
39 |
*/ |
a246b0105
|
40 |
#include <linux/module.h> |
1da177e4c
|
41 |
#include <linux/types.h> |
a246b0105
|
42 |
#include <linux/interrupt.h> |
1da177e4c
|
43 |
#include <linux/workqueue.h> |
bf3fcf895
|
44 |
#include <linux/net.h> |
ff8399709
|
45 |
#include <linux/ktime.h> |
1da177e4c
|
46 |
|
a246b0105
|
47 |
#include <linux/sunrpc/clnt.h> |
11c556b3d
|
48 |
#include <linux/sunrpc/metrics.h> |
c9acb42ef
|
49 |
#include <linux/sunrpc/bc_xprt.h> |
fda1bfef9
|
50 |
#include <linux/rcupdate.h> |
a1231fda7
|
51 |
#include <linux/sched/mm.h> |
1da177e4c
|
52 |
|
3705ad64f
|
53 |
#include <trace/events/sunrpc.h> |
55ae1aabf
|
54 |
#include "sunrpc.h" |
1da177e4c
|
55 56 57 |
/* * Local variables */ |
f895b252d
|
58 |
#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
1da177e4c
|
59 60 |
# define RPCDBG_FACILITY RPCDBG_XPRT #endif |
1da177e4c
|
61 62 63 |
/* * Local functions */ |
21de0a955
|
64 |
static void xprt_init(struct rpc_xprt *xprt, struct net *net); |
37ac86c3a
|
65 |
static __be32 xprt_alloc_xid(struct rpc_xprt *xprt); |
4e0038b6b
|
66 |
static void xprt_destroy(struct rpc_xprt *xprt); |
1da177e4c
|
67 |
|
5ba03e82b
|
68 |
static DEFINE_SPINLOCK(xprt_list_lock); |
81c098af3
|
69 |
static LIST_HEAD(xprt_list); |
9e910bff7
|
70 71 72 73 74 75 76 77 |
static unsigned long xprt_request_timeout(const struct rpc_rqst *req) { unsigned long timeout = jiffies + req->rq_timeout; if (time_before(timeout, req->rq_majortimeo)) return timeout; return req->rq_majortimeo; } |
12a804698
|
78 |
/** |
81c098af3
|
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
* xprt_register_transport - register a transport implementation * @transport: transport to register * * If a transport implementation is loaded as a kernel module, it can * call this interface to make itself known to the RPC client. * * Returns: * 0: transport successfully registered * -EEXIST: transport already registered * -EINVAL: transport module being unloaded */ int xprt_register_transport(struct xprt_class *transport) { struct xprt_class *t; int result; result = -EEXIST; spin_lock(&xprt_list_lock); list_for_each_entry(t, &xprt_list, list) { /* don't register the same transport class twice */ |
4fa016eb2
|
99 |
if (t->ident == transport->ident) |
81c098af3
|
100 101 |
goto out; } |
c9f6cde6e
|
102 103 104 105 106 |
list_add_tail(&transport->list, &xprt_list); printk(KERN_INFO "RPC: Registered %s transport module. ", transport->name); result = 0; |
81c098af3
|
107 108 109 110 111 112 113 114 115 |
out: spin_unlock(&xprt_list_lock); return result; } EXPORT_SYMBOL_GPL(xprt_register_transport); /** * xprt_unregister_transport - unregister a transport implementation |
65b6e42cd
|
116 |
* @transport: transport to unregister |
81c098af3
|
117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 |
* * Returns: * 0: transport successfully unregistered * -ENOENT: transport never registered */ int xprt_unregister_transport(struct xprt_class *transport) { struct xprt_class *t; int result; result = 0; spin_lock(&xprt_list_lock); list_for_each_entry(t, &xprt_list, list) { if (t == transport) { printk(KERN_INFO "RPC: Unregistered %s transport module. ", transport->name); list_del_init(&transport->list); |
81c098af3
|
136 137 138 139 140 141 142 143 144 145 146 147 |
goto out; } } result = -ENOENT; out: spin_unlock(&xprt_list_lock); return result; } EXPORT_SYMBOL_GPL(xprt_unregister_transport); /** |
441e3e242
|
148 149 150 151 152 153 154 155 156 157 |
* xprt_load_transport - load a transport implementation * @transport_name: transport to load * * Returns: * 0: transport successfully loaded * -ENOENT: transport module not available */ int xprt_load_transport(const char *transport_name) { struct xprt_class *t; |
441e3e242
|
158 159 160 161 162 163 164 165 166 167 168 |
int result; result = 0; spin_lock(&xprt_list_lock); list_for_each_entry(t, &xprt_list, list) { if (strcmp(t->name, transport_name) == 0) { spin_unlock(&xprt_list_lock); goto out; } } spin_unlock(&xprt_list_lock); |
ef7ffe8f0
|
169 |
result = request_module("xprt%s", transport_name); |
441e3e242
|
170 171 172 173 |
out: return result; } EXPORT_SYMBOL_GPL(xprt_load_transport); |
c544577da
|
174 175 176 177 178 179 180 181 182 183 |
static void xprt_clear_locked(struct rpc_xprt *xprt) { xprt->snd_task = NULL; if (!test_bit(XPRT_CLOSE_WAIT, &xprt->state)) { smp_mb__before_atomic(); clear_bit(XPRT_LOCKED, &xprt->state); smp_mb__after_atomic(); } else queue_work(xprtiod_workqueue, &xprt->task_cleanup); } |
441e3e242
|
184 |
/** |
12a804698
|
185 186 |
* xprt_reserve_xprt - serialize write access to transports * @task: task that is requesting access to the transport |
177c27bf0
|
187 |
* @xprt: pointer to the target transport |
12a804698
|
188 189 190 191 192 |
* * This prevents mixing the payload of separate requests, and prevents * transport connects from colliding with writes. No congestion control * is provided. */ |
43cedbf0e
|
193 |
int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task) |
12a804698
|
194 |
{ |
12a804698
|
195 196 197 198 199 |
struct rpc_rqst *req = task->tk_rqstp; if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) { if (task == xprt->snd_task) return 1; |
12a804698
|
200 201 |
goto out_sleep; } |
c544577da
|
202 203 |
if (test_bit(XPRT_WRITE_SPACE, &xprt->state)) goto out_unlock; |
12a804698
|
204 |
xprt->snd_task = task; |
4d4a76f33
|
205 |
|
12a804698
|
206 |
return 1; |
c544577da
|
207 208 |
out_unlock: xprt_clear_locked(xprt); |
12a804698
|
209 |
out_sleep: |
46121cf7d
|
210 211 |
dprintk("RPC: %5u failed to lock transport %p ", |
12a804698
|
212 |
task->tk_pid, xprt); |
12a804698
|
213 |
task->tk_status = -EAGAIN; |
6b2e68562
|
214 215 |
if (RPC_IS_SOFT(task)) rpc_sleep_on_timeout(&xprt->sending, task, NULL, |
9e910bff7
|
216 |
xprt_request_timeout(req)); |
6b2e68562
|
217 218 |
else rpc_sleep_on(&xprt->sending, task, NULL); |
12a804698
|
219 220 |
return 0; } |
124448097
|
221 |
EXPORT_SYMBOL_GPL(xprt_reserve_xprt); |
12a804698
|
222 |
|
75891f502
|
223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 |
static bool xprt_need_congestion_window_wait(struct rpc_xprt *xprt) { return test_bit(XPRT_CWND_WAIT, &xprt->state); } static void xprt_set_congestion_window_wait(struct rpc_xprt *xprt) { if (!list_empty(&xprt->xmit_queue)) { /* Peek at head of queue to see if it can make progress */ if (list_first_entry(&xprt->xmit_queue, struct rpc_rqst, rq_xmit)->rq_cong) return; } set_bit(XPRT_CWND_WAIT, &xprt->state); } static void xprt_test_and_clear_congestion_window_wait(struct rpc_xprt *xprt) { if (!RPCXPRT_CONGESTED(xprt)) clear_bit(XPRT_CWND_WAIT, &xprt->state); } |
1da177e4c
|
247 |
/* |
12a804698
|
248 249 250 251 252 253 |
* xprt_reserve_xprt_cong - serialize write access to transports * @task: task that is requesting access to the transport * * Same as xprt_reserve_xprt, but Van Jacobson congestion control is * integrated into the decision of whether a request is allowed to be * woken up and given access to the transport. |
75891f502
|
254 |
* Note that the lock is only granted if we know there are free slots. |
1da177e4c
|
255 |
*/ |
43cedbf0e
|
256 |
int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task) |
1da177e4c
|
257 258 |
{ struct rpc_rqst *req = task->tk_rqstp; |
2226feb6b
|
259 |
if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) { |
1da177e4c
|
260 261 |
if (task == xprt->snd_task) return 1; |
1da177e4c
|
262 263 |
goto out_sleep; } |
43cedbf0e
|
264 265 266 267 |
if (req == NULL) { xprt->snd_task = task; return 1; } |
c544577da
|
268 269 |
if (test_bit(XPRT_WRITE_SPACE, &xprt->state)) goto out_unlock; |
75891f502
|
270 |
if (!xprt_need_congestion_window_wait(xprt)) { |
1da177e4c
|
271 |
xprt->snd_task = task; |
1da177e4c
|
272 273 |
return 1; } |
c544577da
|
274 |
out_unlock: |
632e3bdc5
|
275 |
xprt_clear_locked(xprt); |
1da177e4c
|
276 |
out_sleep: |
46121cf7d
|
277 278 |
dprintk("RPC: %5u failed to lock transport %p ", task->tk_pid, xprt); |
1da177e4c
|
279 |
task->tk_status = -EAGAIN; |
6b2e68562
|
280 281 |
if (RPC_IS_SOFT(task)) rpc_sleep_on_timeout(&xprt->sending, task, NULL, |
9e910bff7
|
282 |
xprt_request_timeout(req)); |
6b2e68562
|
283 284 |
else rpc_sleep_on(&xprt->sending, task, NULL); |
1da177e4c
|
285 286 |
return 0; } |
124448097
|
287 |
EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong); |
1da177e4c
|
288 |
|
12a804698
|
289 |
static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task) |
1da177e4c
|
290 291 |
{ int retval; |
bd79bc579
|
292 293 |
if (test_bit(XPRT_LOCKED, &xprt->state) && xprt->snd_task == task) return 1; |
b5e924191
|
294 |
spin_lock(&xprt->transport_lock); |
43cedbf0e
|
295 |
retval = xprt->ops->reserve_xprt(xprt, task); |
b5e924191
|
296 |
spin_unlock(&xprt->transport_lock); |
1da177e4c
|
297 298 |
return retval; } |
961a828df
|
299 |
static bool __xprt_lock_write_func(struct rpc_task *task, void *data) |
1da177e4c
|
300 |
{ |
961a828df
|
301 |
struct rpc_xprt *xprt = data; |
49e9a8908
|
302 |
|
49e9a8908
|
303 |
xprt->snd_task = task; |
961a828df
|
304 305 |
return true; } |
49e9a8908
|
306 |
|
961a828df
|
307 308 309 310 |
static void __xprt_lock_write_next(struct rpc_xprt *xprt) { if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) return; |
c544577da
|
311 312 |
if (test_bit(XPRT_WRITE_SPACE, &xprt->state)) goto out_unlock; |
f1dc237c6
|
313 314 |
if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending, __xprt_lock_write_func, xprt)) |
961a828df
|
315 |
return; |
c544577da
|
316 |
out_unlock: |
632e3bdc5
|
317 |
xprt_clear_locked(xprt); |
49e9a8908
|
318 |
} |
961a828df
|
319 320 321 322 |
static void __xprt_lock_write_next_cong(struct rpc_xprt *xprt) { if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) return; |
c544577da
|
323 324 |
if (test_bit(XPRT_WRITE_SPACE, &xprt->state)) goto out_unlock; |
75891f502
|
325 |
if (xprt_need_congestion_window_wait(xprt)) |
961a828df
|
326 |
goto out_unlock; |
f1dc237c6
|
327 |
if (rpc_wake_up_first_on_wq(xprtiod_workqueue, &xprt->sending, |
75891f502
|
328 |
__xprt_lock_write_func, xprt)) |
961a828df
|
329 |
return; |
1da177e4c
|
330 |
out_unlock: |
632e3bdc5
|
331 |
xprt_clear_locked(xprt); |
1da177e4c
|
332 |
} |
49e9a8908
|
333 334 335 336 337 338 |
/** * xprt_release_xprt - allow other requests to use a transport * @xprt: transport with other tasks potentially waiting * @task: task that is releasing access to the transport * * Note that "task" can be NULL. No congestion control is provided. |
1da177e4c
|
339 |
*/ |
49e9a8908
|
340 |
void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task) |
1da177e4c
|
341 342 |
{ if (xprt->snd_task == task) { |
632e3bdc5
|
343 |
xprt_clear_locked(xprt); |
1da177e4c
|
344 345 346 |
__xprt_lock_write_next(xprt); } } |
124448097
|
347 |
EXPORT_SYMBOL_GPL(xprt_release_xprt); |
1da177e4c
|
348 |
|
49e9a8908
|
349 350 351 352 353 354 355 356 357 358 359 |
/** * xprt_release_xprt_cong - allow other requests to use a transport * @xprt: transport with other tasks potentially waiting * @task: task that is releasing access to the transport * * Note that "task" can be NULL. Another task is awoken to use the * transport if the transport's congestion window allows it. */ void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task) { if (xprt->snd_task == task) { |
632e3bdc5
|
360 |
xprt_clear_locked(xprt); |
49e9a8908
|
361 362 363 |
__xprt_lock_write_next_cong(xprt); } } |
124448097
|
364 |
EXPORT_SYMBOL_GPL(xprt_release_xprt_cong); |
49e9a8908
|
365 366 |
static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task) |
1da177e4c
|
367 |
{ |
bd79bc579
|
368 369 |
if (xprt->snd_task != task) return; |
b5e924191
|
370 |
spin_lock(&xprt->transport_lock); |
49e9a8908
|
371 |
xprt->ops->release_xprt(xprt, task); |
b5e924191
|
372 |
spin_unlock(&xprt->transport_lock); |
1da177e4c
|
373 374 375 |
} /* |
1da177e4c
|
376 377 378 379 |
* Van Jacobson congestion avoidance. Check if the congestion window * overflowed. Put the task to sleep if this is the case. */ static int |
75891f502
|
380 |
__xprt_get_cong(struct rpc_xprt *xprt, struct rpc_rqst *req) |
1da177e4c
|
381 |
{ |
1da177e4c
|
382 383 |
if (req->rq_cong) return 1; |
46121cf7d
|
384 385 |
dprintk("RPC: %5u xprt_cwnd_limited cong = %lu cwnd = %lu ", |
75891f502
|
386 387 388 |
req->rq_task->tk_pid, xprt->cong, xprt->cwnd); if (RPCXPRT_CONGESTED(xprt)) { xprt_set_congestion_window_wait(xprt); |
1da177e4c
|
389 |
return 0; |
75891f502
|
390 |
} |
1da177e4c
|
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 |
req->rq_cong = 1; xprt->cong += RPC_CWNDSCALE; return 1; } /* * Adjust the congestion window, and wake up the next task * that has been sleeping due to congestion */ static void __xprt_put_cong(struct rpc_xprt *xprt, struct rpc_rqst *req) { if (!req->rq_cong) return; req->rq_cong = 0; xprt->cong -= RPC_CWNDSCALE; |
75891f502
|
407 |
xprt_test_and_clear_congestion_window_wait(xprt); |
49e9a8908
|
408 |
__xprt_lock_write_next_cong(xprt); |
1da177e4c
|
409 |
} |
46c0ee8bc
|
410 |
/** |
75891f502
|
411 412 413 414 415 416 417 418 419 420 421 422 423 |
* xprt_request_get_cong - Request congestion control credits * @xprt: pointer to transport * @req: pointer to RPC request * * Useful for transports that require congestion control. */ bool xprt_request_get_cong(struct rpc_xprt *xprt, struct rpc_rqst *req) { bool ret = false; if (req->rq_cong) return true; |
b5e924191
|
424 |
spin_lock(&xprt->transport_lock); |
75891f502
|
425 |
ret = __xprt_get_cong(xprt, req) != 0; |
b5e924191
|
426 |
spin_unlock(&xprt->transport_lock); |
75891f502
|
427 428 429 430 431 |
return ret; } EXPORT_SYMBOL_GPL(xprt_request_get_cong); /** |
a58dd398f
|
432 433 434 435 436 437 438 |
* xprt_release_rqst_cong - housekeeping when request is complete * @task: RPC request that recently completed * * Useful for transports that require congestion control. */ void xprt_release_rqst_cong(struct rpc_task *task) { |
a4f0835c6
|
439 440 441 |
struct rpc_rqst *req = task->tk_rqstp; __xprt_put_cong(req->rq_xprt, req); |
a58dd398f
|
442 |
} |
124448097
|
443 |
EXPORT_SYMBOL_GPL(xprt_release_rqst_cong); |
a58dd398f
|
444 |
|
8593e0107
|
445 446 447 448 449 |
static void xprt_clear_congestion_window_wait_locked(struct rpc_xprt *xprt) { if (test_and_clear_bit(XPRT_CWND_WAIT, &xprt->state)) __xprt_lock_write_next_cong(xprt); } |
75891f502
|
450 451 452 453 454 455 456 457 |
/* * Clear the congestion window wait flag and wake up the next * entry on xprt->sending */ static void xprt_clear_congestion_window_wait(struct rpc_xprt *xprt) { if (test_and_clear_bit(XPRT_CWND_WAIT, &xprt->state)) { |
b5e924191
|
458 |
spin_lock(&xprt->transport_lock); |
75891f502
|
459 |
__xprt_lock_write_next_cong(xprt); |
b5e924191
|
460 |
spin_unlock(&xprt->transport_lock); |
75891f502
|
461 462 |
} } |
a58dd398f
|
463 |
/** |
46c0ee8bc
|
464 |
* xprt_adjust_cwnd - adjust transport congestion window |
6a24dfb64
|
465 |
* @xprt: pointer to xprt |
46c0ee8bc
|
466 467 468 |
* @task: recently completed RPC request used to adjust window * @result: result code of completed RPC request * |
4f4cf5ad6
|
469 470 471 472 473 474 475 476 477 |
* The transport code maintains an estimate on the maximum number of out- * standing RPC requests, using a smoothed version of the congestion * avoidance implemented in 44BSD. This is basically the Van Jacobson * congestion algorithm: If a retransmit occurs, the congestion window is * halved; otherwise, it is incremented by 1/cwnd when * * - a reply is received and * - a full number of requests are outstanding and * - the congestion window hasn't been updated recently. |
1da177e4c
|
478 |
*/ |
6a24dfb64
|
479 |
void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result) |
1da177e4c
|
480 |
{ |
46c0ee8bc
|
481 |
struct rpc_rqst *req = task->tk_rqstp; |
46c0ee8bc
|
482 |
unsigned long cwnd = xprt->cwnd; |
1da177e4c
|
483 |
|
1da177e4c
|
484 485 486 487 488 489 |
if (result >= 0 && cwnd <= xprt->cong) { /* The (cwnd >> 1) term makes sure * the result gets rounded properly. */ cwnd += (RPC_CWNDSCALE * RPC_CWNDSCALE + (cwnd >> 1)) / cwnd; if (cwnd > RPC_MAXCWND(xprt)) cwnd = RPC_MAXCWND(xprt); |
49e9a8908
|
490 |
__xprt_lock_write_next_cong(xprt); |
1da177e4c
|
491 492 493 494 495 |
} else if (result == -ETIMEDOUT) { cwnd >>= 1; if (cwnd < RPC_CWNDSCALE) cwnd = RPC_CWNDSCALE; } |
46121cf7d
|
496 497 |
dprintk("RPC: cong %ld, cwnd was %ld, now %ld ", |
1da177e4c
|
498 499 |
xprt->cong, xprt->cwnd, cwnd); xprt->cwnd = cwnd; |
46c0ee8bc
|
500 |
__xprt_put_cong(xprt, req); |
1da177e4c
|
501 |
} |
124448097
|
502 |
EXPORT_SYMBOL_GPL(xprt_adjust_cwnd); |
1da177e4c
|
503 |
|
44fbac228
|
504 505 506 507 508 509 510 511 512 513 514 515 516 |
/** * xprt_wake_pending_tasks - wake all tasks on a transport's pending queue * @xprt: transport with waiting tasks * @status: result code to plant in each task before waking it * */ void xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status) { if (status < 0) rpc_wake_up_status(&xprt->pending, status); else rpc_wake_up(&xprt->pending); } |
124448097
|
517 |
EXPORT_SYMBOL_GPL(xprt_wake_pending_tasks); |
44fbac228
|
518 |
|
c7b2cae8a
|
519 520 |
/** * xprt_wait_for_buffer_space - wait for transport output buffer to clear |
c544577da
|
521 |
* @xprt: transport |
a9a6b52ee
|
522 523 524 525 |
* * Note that we only set the timer for the case of RPC_IS_SOFT(), since * we don't in general want to force a socket disconnection due to * an incomplete RPC call transmission. |
c7b2cae8a
|
526 |
*/ |
c544577da
|
527 |
void xprt_wait_for_buffer_space(struct rpc_xprt *xprt) |
c7b2cae8a
|
528 |
{ |
c544577da
|
529 |
set_bit(XPRT_WRITE_SPACE, &xprt->state); |
c7b2cae8a
|
530 |
} |
124448097
|
531 |
EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space); |
c7b2cae8a
|
532 |
|
c544577da
|
533 534 535 536 537 538 539 540 541 542 543 544 |
static bool xprt_clear_write_space_locked(struct rpc_xprt *xprt) { if (test_and_clear_bit(XPRT_WRITE_SPACE, &xprt->state)) { __xprt_lock_write_next(xprt); dprintk("RPC: write space: waking waiting task on " "xprt %p ", xprt); return true; } return false; } |
c7b2cae8a
|
545 546 547 548 549 550 |
/** * xprt_write_space - wake the task waiting for transport output buffer space * @xprt: transport with waiting tasks * * Can be called in a soft IRQ context, so xprt_write_space never sleeps. */ |
c544577da
|
551 |
bool xprt_write_space(struct rpc_xprt *xprt) |
c7b2cae8a
|
552 |
{ |
c544577da
|
553 554 555 556 |
bool ret; if (!test_bit(XPRT_WRITE_SPACE, &xprt->state)) return false; |
b5e924191
|
557 |
spin_lock(&xprt->transport_lock); |
c544577da
|
558 |
ret = xprt_clear_write_space_locked(xprt); |
b5e924191
|
559 |
spin_unlock(&xprt->transport_lock); |
c544577da
|
560 |
return ret; |
c7b2cae8a
|
561 |
} |
124448097
|
562 |
EXPORT_SYMBOL_GPL(xprt_write_space); |
c7b2cae8a
|
563 |
|
da953063b
|
564 565 566 567 568 569 570 571 572 |
static unsigned long xprt_abs_ktime_to_jiffies(ktime_t abstime) { s64 delta = ktime_to_ns(ktime_get() - abstime); return likely(delta >= 0) ? jiffies - nsecs_to_jiffies(delta) : jiffies + nsecs_to_jiffies(-delta); } static unsigned long xprt_calc_majortimeo(struct rpc_rqst *req) |
1da177e4c
|
573 |
{ |
ba7392bb3
|
574 |
const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout; |
da953063b
|
575 |
unsigned long majortimeo = req->rq_timeout; |
1da177e4c
|
576 |
|
1da177e4c
|
577 |
if (to->to_exponential) |
da953063b
|
578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 |
majortimeo <<= to->to_retries; else majortimeo += to->to_increment * to->to_retries; if (majortimeo > to->to_maxval || majortimeo == 0) majortimeo = to->to_maxval; return majortimeo; } static void xprt_reset_majortimeo(struct rpc_rqst *req) { req->rq_majortimeo += xprt_calc_majortimeo(req); } static void xprt_init_majortimeo(struct rpc_task *task, struct rpc_rqst *req) { unsigned long time_init; struct rpc_xprt *xprt = req->rq_xprt; if (likely(xprt && xprt_connected(xprt))) time_init = jiffies; |
1da177e4c
|
598 |
else |
da953063b
|
599 600 601 |
time_init = xprt_abs_ktime_to_jiffies(task->tk_start); req->rq_timeout = task->tk_client->cl_timeout->to_initval; req->rq_majortimeo = time_init + xprt_calc_majortimeo(req); |
1da177e4c
|
602 |
} |
9903cd1c2
|
603 604 605 606 |
/** * xprt_adjust_timeout - adjust timeout values for next retransmit * @req: RPC request containing parameters to use for the adjustment * |
1da177e4c
|
607 608 609 610 |
*/ int xprt_adjust_timeout(struct rpc_rqst *req) { struct rpc_xprt *xprt = req->rq_xprt; |
ba7392bb3
|
611 |
const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout; |
1da177e4c
|
612 613 614 615 616 617 618 619 620 621 |
int status = 0; if (time_before(jiffies, req->rq_majortimeo)) { if (to->to_exponential) req->rq_timeout <<= 1; else req->rq_timeout += to->to_increment; if (to->to_maxval && req->rq_timeout >= to->to_maxval) req->rq_timeout = to->to_maxval; req->rq_retries++; |
1da177e4c
|
622 623 624 625 626 |
} else { req->rq_timeout = to->to_initval; req->rq_retries = 0; xprt_reset_majortimeo(req); /* Reset the RTT counters == "slow start" */ |
b5e924191
|
627 |
spin_lock(&xprt->transport_lock); |
1da177e4c
|
628 |
rpc_init_rtt(req->rq_task->tk_client->cl_rtt, to->to_initval); |
b5e924191
|
629 |
spin_unlock(&xprt->transport_lock); |
1da177e4c
|
630 631 632 633 634 635 636 637 638 639 |
status = -ETIMEDOUT; } if (req->rq_timeout == 0) { printk(KERN_WARNING "xprt_adjust_timeout: rq_timeout = 0! "); req->rq_timeout = 5 * HZ; } return status; } |
65f27f384
|
640 |
static void xprt_autoclose(struct work_struct *work) |
1da177e4c
|
641 |
{ |
65f27f384
|
642 643 |
struct rpc_xprt *xprt = container_of(work, struct rpc_xprt, task_cleanup); |
a1231fda7
|
644 |
unsigned int pflags = memalloc_nofs_save(); |
1da177e4c
|
645 |
|
66af1e558
|
646 |
clear_bit(XPRT_CLOSE_WAIT, &xprt->state); |
4876cc779
|
647 |
xprt->ops->close(xprt); |
1da177e4c
|
648 |
xprt_release_write(xprt, NULL); |
79234c3db
|
649 |
wake_up_bit(&xprt->state, XPRT_LOCKED); |
a1231fda7
|
650 |
memalloc_nofs_restore(pflags); |
1da177e4c
|
651 |
} |
9903cd1c2
|
652 |
/** |
62da3b248
|
653 |
* xprt_disconnect_done - mark a transport as disconnected |
9903cd1c2
|
654 655 |
* @xprt: transport to flag for disconnect * |
1da177e4c
|
656 |
*/ |
62da3b248
|
657 |
void xprt_disconnect_done(struct rpc_xprt *xprt) |
1da177e4c
|
658 |
{ |
46121cf7d
|
659 660 |
dprintk("RPC: disconnected transport %p ", xprt); |
b5e924191
|
661 |
spin_lock(&xprt->transport_lock); |
1da177e4c
|
662 |
xprt_clear_connected(xprt); |
c544577da
|
663 |
xprt_clear_write_space_locked(xprt); |
8593e0107
|
664 |
xprt_clear_congestion_window_wait_locked(xprt); |
27adc7859
|
665 |
xprt_wake_pending_tasks(xprt, -ENOTCONN); |
b5e924191
|
666 |
spin_unlock(&xprt->transport_lock); |
1da177e4c
|
667 |
} |
62da3b248
|
668 |
EXPORT_SYMBOL_GPL(xprt_disconnect_done); |
1da177e4c
|
669 |
|
66af1e558
|
670 671 672 673 674 675 676 677 |
/** * xprt_force_disconnect - force a transport to disconnect * @xprt: transport to disconnect * */ void xprt_force_disconnect(struct rpc_xprt *xprt) { /* Don't race with the test_bit() in xprt_clear_locked() */ |
b5e924191
|
678 |
spin_lock(&xprt->transport_lock); |
66af1e558
|
679 680 681 |
set_bit(XPRT_CLOSE_WAIT, &xprt->state); /* Try to schedule an autoclose RPC call */ if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0) |
40a5f1b19
|
682 |
queue_work(xprtiod_workqueue, &xprt->task_cleanup); |
0445f92c5
|
683 684 685 |
else if (xprt->snd_task) rpc_wake_up_queued_task_set_status(&xprt->pending, xprt->snd_task, -ENOTCONN); |
b5e924191
|
686 |
spin_unlock(&xprt->transport_lock); |
66af1e558
|
687 |
} |
e2a4f4fbe
|
688 |
EXPORT_SYMBOL_GPL(xprt_force_disconnect); |
66af1e558
|
689 |
|
7f3a1d1e1
|
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 |
static unsigned int xprt_connect_cookie(struct rpc_xprt *xprt) { return READ_ONCE(xprt->connect_cookie); } static bool xprt_request_retransmit_after_disconnect(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; return req->rq_connect_cookie != xprt_connect_cookie(xprt) || !xprt_connected(xprt); } |
7c1d71cf5
|
705 706 707 708 709 710 711 712 713 714 715 716 717 718 |
/** * xprt_conditional_disconnect - force a transport to disconnect * @xprt: transport to disconnect * @cookie: 'connection cookie' * * This attempts to break the connection if and only if 'cookie' matches * the current transport 'connection cookie'. It ensures that we don't * try to break the connection more than once when we need to retransmit * a batch of RPC requests. * */ void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie) { /* Don't race with the test_bit() in xprt_clear_locked() */ |
b5e924191
|
719 |
spin_lock(&xprt->transport_lock); |
7c1d71cf5
|
720 721 |
if (cookie != xprt->connect_cookie) goto out; |
2c2ee6d20
|
722 |
if (test_bit(XPRT_CLOSING, &xprt->state)) |
7c1d71cf5
|
723 724 725 726 |
goto out; set_bit(XPRT_CLOSE_WAIT, &xprt->state); /* Try to schedule an autoclose RPC call */ if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0) |
40a5f1b19
|
727 |
queue_work(xprtiod_workqueue, &xprt->task_cleanup); |
2a4919919
|
728 |
xprt_wake_pending_tasks(xprt, -EAGAIN); |
7c1d71cf5
|
729 |
out: |
b5e924191
|
730 |
spin_unlock(&xprt->transport_lock); |
7c1d71cf5
|
731 |
} |
ad3331acb
|
732 733 734 735 736 737 738 739 740 741 |
static bool xprt_has_timer(const struct rpc_xprt *xprt) { return xprt->idle_timeout != 0; } static void xprt_schedule_autodisconnect(struct rpc_xprt *xprt) __must_hold(&xprt->transport_lock) { |
80d3c45fd
|
742 |
xprt->last_used = jiffies; |
95f7691da
|
743 |
if (RB_EMPTY_ROOT(&xprt->recv_queue) && xprt_has_timer(xprt)) |
ad3331acb
|
744 745 |
mod_timer(&xprt->timer, xprt->last_used + xprt->idle_timeout); } |
1da177e4c
|
746 |
static void |
ff861c4d6
|
747 |
xprt_init_autodisconnect(struct timer_list *t) |
1da177e4c
|
748 |
{ |
ff861c4d6
|
749 |
struct rpc_xprt *xprt = from_timer(xprt, t, timer); |
1da177e4c
|
750 |
|
95f7691da
|
751 |
if (!RB_EMPTY_ROOT(&xprt->recv_queue)) |
b5e924191
|
752 |
return; |
ad3331acb
|
753 754 |
/* Reset xprt->last_used to avoid connect/autodisconnect cycling */ xprt->last_used = jiffies; |
2226feb6b
|
755 |
if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) |
b5e924191
|
756 |
return; |
40a5f1b19
|
757 |
queue_work(xprtiod_workqueue, &xprt->task_cleanup); |
1da177e4c
|
758 |
} |
718ba5b87
|
759 760 761 762 763 |
bool xprt_lock_connect(struct rpc_xprt *xprt, struct rpc_task *task, void *cookie) { bool ret = false; |
b5e924191
|
764 |
spin_lock(&xprt->transport_lock); |
718ba5b87
|
765 766 767 768 769 770 771 |
if (!test_bit(XPRT_LOCKED, &xprt->state)) goto out; if (xprt->snd_task != task) goto out; xprt->snd_task = cookie; ret = true; out: |
b5e924191
|
772 |
spin_unlock(&xprt->transport_lock); |
718ba5b87
|
773 774 775 776 777 |
return ret; } void xprt_unlock_connect(struct rpc_xprt *xprt, void *cookie) { |
b5e924191
|
778 |
spin_lock(&xprt->transport_lock); |
718ba5b87
|
779 780 781 782 783 784 |
if (xprt->snd_task != cookie) goto out; if (!test_bit(XPRT_LOCKED, &xprt->state)) goto out; xprt->snd_task =NULL; xprt->ops->release_xprt(xprt, NULL); |
ad3331acb
|
785 |
xprt_schedule_autodisconnect(xprt); |
718ba5b87
|
786 |
out: |
b5e924191
|
787 |
spin_unlock(&xprt->transport_lock); |
79234c3db
|
788 |
wake_up_bit(&xprt->state, XPRT_LOCKED); |
718ba5b87
|
789 |
} |
9903cd1c2
|
790 791 792 |
/** * xprt_connect - schedule a transport connect operation * @task: RPC task that is requesting the connect |
1da177e4c
|
793 794 795 796 |
* */ void xprt_connect(struct rpc_task *task) { |
ad2368d6f
|
797 |
struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt; |
1da177e4c
|
798 |
|
46121cf7d
|
799 800 |
dprintk("RPC: %5u xprt_connect xprt %p %s connected ", task->tk_pid, |
1da177e4c
|
801 |
xprt, (xprt_connected(xprt) ? "is" : "is not")); |
ec739ef03
|
802 |
if (!xprt_bound(xprt)) { |
01d37c428
|
803 |
task->tk_status = -EAGAIN; |
1da177e4c
|
804 805 806 807 |
return; } if (!xprt_lock_write(xprt, task)) return; |
feb8ca37c
|
808 809 810 |
if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state)) xprt->ops->close(xprt); |
718ba5b87
|
811 |
if (!xprt_connected(xprt)) { |
2c2ee6d20
|
812 |
task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie; |
6b2e68562
|
813 |
rpc_sleep_on_timeout(&xprt->pending, task, NULL, |
9e910bff7
|
814 |
xprt_request_timeout(task->tk_rqstp)); |
0b9e79431
|
815 816 817 818 819 |
if (test_bit(XPRT_CLOSING, &xprt->state)) return; if (xprt_test_and_set_connecting(xprt)) return; |
0a9a4304f
|
820 821 822 823 824 825 826 827 828 |
/* Race breaker */ if (!xprt_connected(xprt)) { xprt->stat.connect_start = jiffies; xprt->ops->connect(xprt, task); } else { xprt_clear_connecting(xprt); task->tk_status = 0; rpc_wake_up_queued_task(&xprt->pending, task); } |
1da177e4c
|
829 |
} |
718ba5b87
|
830 |
xprt_release_write(xprt, task); |
1da177e4c
|
831 |
} |
675dd90ad
|
832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 |
/** * xprt_reconnect_delay - compute the wait before scheduling a connect * @xprt: transport instance * */ unsigned long xprt_reconnect_delay(const struct rpc_xprt *xprt) { unsigned long start, now = jiffies; start = xprt->stat.connect_start + xprt->reestablish_timeout; if (time_after(start, now)) return start - now; return 0; } EXPORT_SYMBOL_GPL(xprt_reconnect_delay); /** * xprt_reconnect_backoff - compute the new re-establish timeout * @xprt: transport instance * @init_to: initial reestablish timeout * */ void xprt_reconnect_backoff(struct rpc_xprt *xprt, unsigned long init_to) { xprt->reestablish_timeout <<= 1; if (xprt->reestablish_timeout > xprt->max_reconnect_timeout) xprt->reestablish_timeout = xprt->max_reconnect_timeout; if (xprt->reestablish_timeout < init_to) xprt->reestablish_timeout = init_to; } EXPORT_SYMBOL_GPL(xprt_reconnect_backoff); |
95f7691da
|
863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 |
enum xprt_xid_rb_cmp { XID_RB_EQUAL, XID_RB_LEFT, XID_RB_RIGHT, }; static enum xprt_xid_rb_cmp xprt_xid_cmp(__be32 xid1, __be32 xid2) { if (xid1 == xid2) return XID_RB_EQUAL; if ((__force u32)xid1 < (__force u32)xid2) return XID_RB_LEFT; return XID_RB_RIGHT; } static struct rpc_rqst * xprt_request_rb_find(struct rpc_xprt *xprt, __be32 xid) { struct rb_node *n = xprt->recv_queue.rb_node; struct rpc_rqst *req; while (n != NULL) { req = rb_entry(n, struct rpc_rqst, rq_recv); switch (xprt_xid_cmp(xid, req->rq_xid)) { case XID_RB_LEFT: n = n->rb_left; break; case XID_RB_RIGHT: n = n->rb_right; break; case XID_RB_EQUAL: return req; } } return NULL; } static void xprt_request_rb_insert(struct rpc_xprt *xprt, struct rpc_rqst *new) { struct rb_node **p = &xprt->recv_queue.rb_node; struct rb_node *n = NULL; struct rpc_rqst *req; while (*p != NULL) { n = *p; req = rb_entry(n, struct rpc_rqst, rq_recv); switch(xprt_xid_cmp(new->rq_xid, req->rq_xid)) { case XID_RB_LEFT: p = &n->rb_left; break; case XID_RB_RIGHT: p = &n->rb_right; break; case XID_RB_EQUAL: WARN_ON_ONCE(new != req); return; } } rb_link_node(&new->rq_recv, n, p); rb_insert_color(&new->rq_recv, &xprt->recv_queue); } static void xprt_request_rb_remove(struct rpc_xprt *xprt, struct rpc_rqst *req) { rb_erase(&req->rq_recv, &xprt->recv_queue); } |
9903cd1c2
|
931 932 933 934 935 |
/** * xprt_lookup_rqst - find an RPC request corresponding to an XID * @xprt: transport on which the original request was transmitted * @xid: RPC XID of incoming reply * |
75c84151a
|
936 |
* Caller holds xprt->queue_lock. |
1da177e4c
|
937 |
*/ |
d8ed029d6
|
938 |
struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid) |
1da177e4c
|
939 |
{ |
8f3a6de31
|
940 |
struct rpc_rqst *entry; |
1da177e4c
|
941 |
|
95f7691da
|
942 943 944 945 946 947 |
entry = xprt_request_rb_find(xprt, xid); if (entry != NULL) { trace_xprt_lookup_rqst(xprt, xid, 0); entry->rq_rtt = ktime_sub(ktime_get(), entry->rq_xtime); return entry; } |
46121cf7d
|
948 949 950 951 |
dprintk("RPC: xprt_lookup_rqst did not find xid %08x ", ntohl(xid)); |
3705ad64f
|
952 |
trace_xprt_lookup_rqst(xprt, xid, -ENOENT); |
262ca07de
|
953 954 |
xprt->stat.bad_xids++; return NULL; |
1da177e4c
|
955 |
} |
124448097
|
956 |
EXPORT_SYMBOL_GPL(xprt_lookup_rqst); |
1da177e4c
|
957 |
|
cf9946cd6
|
958 959 960 961 962 |
static bool xprt_is_pinned_rqst(struct rpc_rqst *req) { return atomic_read(&req->rq_pin) != 0; } |
729749bb8
|
963 964 965 966 967 |
/** * xprt_pin_rqst - Pin a request on the transport receive list * @req: Request to pin * * Caller must ensure this is atomic with the call to xprt_lookup_rqst() |
1f7d1c73c
|
968 |
* so should be holding xprt->queue_lock. |
729749bb8
|
969 970 971 |
*/ void xprt_pin_rqst(struct rpc_rqst *req) { |
cf9946cd6
|
972 |
atomic_inc(&req->rq_pin); |
729749bb8
|
973 |
} |
9590d083c
|
974 |
EXPORT_SYMBOL_GPL(xprt_pin_rqst); |
729749bb8
|
975 976 977 978 979 |
/** * xprt_unpin_rqst - Unpin a request on the transport receive list * @req: Request to pin * |
1f7d1c73c
|
980 |
* Caller should be holding xprt->queue_lock. |
729749bb8
|
981 982 983 |
*/ void xprt_unpin_rqst(struct rpc_rqst *req) { |
cf9946cd6
|
984 985 986 987 988 989 |
if (!test_bit(RPC_TASK_MSG_PIN_WAIT, &req->rq_task->tk_runstate)) { atomic_dec(&req->rq_pin); return; } if (atomic_dec_and_test(&req->rq_pin)) wake_up_var(&req->rq_pin); |
729749bb8
|
990 |
} |
9590d083c
|
991 |
EXPORT_SYMBOL_GPL(xprt_unpin_rqst); |
729749bb8
|
992 993 |
static void xprt_wait_on_pinned_rqst(struct rpc_rqst *req) |
729749bb8
|
994 |
{ |
cf9946cd6
|
995 |
wait_var_event(&req->rq_pin, !xprt_is_pinned_rqst(req)); |
729749bb8
|
996 |
} |
edc81dcd5
|
997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 |
static bool xprt_request_data_received(struct rpc_task *task) { return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) && READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) != 0; } static bool xprt_request_need_enqueue_receive(struct rpc_task *task, struct rpc_rqst *req) { return !test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) && READ_ONCE(task->tk_rqstp->rq_reply_bytes_recvd) == 0; } /** * xprt_request_enqueue_receive - Add an request to the receive queue * @task: RPC task * */ void xprt_request_enqueue_receive(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; if (!xprt_request_need_enqueue_receive(task, req)) return; |
753690898
|
1024 1025 |
xprt_request_prepare(task->tk_rqstp); |
edc81dcd5
|
1026 1027 1028 1029 1030 1031 1032 |
spin_lock(&xprt->queue_lock); /* Update the softirq receive buffer */ memcpy(&req->rq_private_buf, &req->rq_rcv_buf, sizeof(req->rq_private_buf)); /* Add request to the receive list */ |
95f7691da
|
1033 |
xprt_request_rb_insert(xprt, req); |
edc81dcd5
|
1034 1035 |
set_bit(RPC_TASK_NEED_RECV, &task->tk_runstate); spin_unlock(&xprt->queue_lock); |
edc81dcd5
|
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 |
/* Turn off autodisconnect */ del_singleshot_timer_sync(&xprt->timer); } /** * xprt_request_dequeue_receive_locked - Remove a request from the receive queue * @task: RPC task * * Caller must hold xprt->queue_lock. */ static void xprt_request_dequeue_receive_locked(struct rpc_task *task) { |
95f7691da
|
1049 |
struct rpc_rqst *req = task->tk_rqstp; |
edc81dcd5
|
1050 |
if (test_and_clear_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) |
95f7691da
|
1051 |
xprt_request_rb_remove(req->rq_xprt, req); |
edc81dcd5
|
1052 |
} |
ecd465ee8
|
1053 1054 1055 1056 |
/** * xprt_update_rtt - Update RPC RTT statistics * @task: RPC request that recently completed * |
75c84151a
|
1057 |
* Caller holds xprt->queue_lock. |
ecd465ee8
|
1058 1059 |
*/ void xprt_update_rtt(struct rpc_task *task) |
1570c1e41
|
1060 1061 1062 |
{ struct rpc_rqst *req = task->tk_rqstp; struct rpc_rtt *rtt = task->tk_client->cl_rtt; |
95c961747
|
1063 |
unsigned int timer = task->tk_msg.rpc_proc->p_timer; |
d60dbb20a
|
1064 |
long m = usecs_to_jiffies(ktime_to_us(req->rq_rtt)); |
1570c1e41
|
1065 1066 1067 |
if (timer) { if (req->rq_ntrans == 1) |
ff8399709
|
1068 |
rpc_update_rtt(rtt, timer, m); |
1570c1e41
|
1069 1070 1071 |
rpc_set_timeo(rtt, timer, req->rq_ntrans - 1); } } |
ecd465ee8
|
1072 |
EXPORT_SYMBOL_GPL(xprt_update_rtt); |
1570c1e41
|
1073 1074 |
/** |
9903cd1c2
|
1075 |
* xprt_complete_rqst - called when reply processing is complete |
1570c1e41
|
1076 |
* @task: RPC request that recently completed |
9903cd1c2
|
1077 1078 |
* @copied: actual number of bytes received from the transport * |
75c84151a
|
1079 |
* Caller holds xprt->queue_lock. |
1da177e4c
|
1080 |
*/ |
1570c1e41
|
1081 |
void xprt_complete_rqst(struct rpc_task *task, int copied) |
1da177e4c
|
1082 |
{ |
1570c1e41
|
1083 |
struct rpc_rqst *req = task->tk_rqstp; |
fda139393
|
1084 |
struct rpc_xprt *xprt = req->rq_xprt; |
1da177e4c
|
1085 |
|
1570c1e41
|
1086 1087 1088 |
dprintk("RPC: %5u xid %08x complete (%d bytes received) ", task->tk_pid, ntohl(req->rq_xid), copied); |
3705ad64f
|
1089 |
trace_xprt_complete_rqst(xprt, req->rq_xid, copied); |
1da177e4c
|
1090 |
|
fda139393
|
1091 |
xprt->stat.recvs++; |
ef759a2e5
|
1092 |
|
1e799b673
|
1093 |
req->rq_private_buf.len = copied; |
dd2b63d04
|
1094 1095 |
/* Ensure all writes are done before we update */ /* req->rq_reply_bytes_recvd */ |
43ac3f296
|
1096 |
smp_wmb(); |
dd2b63d04
|
1097 |
req->rq_reply_bytes_recvd = copied; |
edc81dcd5
|
1098 |
xprt_request_dequeue_receive_locked(task); |
fda139393
|
1099 |
rpc_wake_up_queued_task(&xprt->pending, task); |
1da177e4c
|
1100 |
} |
124448097
|
1101 |
EXPORT_SYMBOL_GPL(xprt_complete_rqst); |
1da177e4c
|
1102 |
|
46c0ee8bc
|
1103 |
static void xprt_timer(struct rpc_task *task) |
1da177e4c
|
1104 |
{ |
46c0ee8bc
|
1105 |
struct rpc_rqst *req = task->tk_rqstp; |
1da177e4c
|
1106 |
struct rpc_xprt *xprt = req->rq_xprt; |
5d00837b9
|
1107 1108 |
if (task->tk_status != -ETIMEDOUT) return; |
1da177e4c
|
1109 |
|
82476d9f9
|
1110 |
trace_xprt_timer(xprt, req->rq_xid, task->tk_status); |
dd2b63d04
|
1111 |
if (!req->rq_reply_bytes_recvd) { |
46c0ee8bc
|
1112 |
if (xprt->ops->timer) |
6a24dfb64
|
1113 |
xprt->ops->timer(xprt, task); |
5d00837b9
|
1114 1115 |
} else task->tk_status = 0; |
1da177e4c
|
1116 |
} |
9903cd1c2
|
1117 |
/** |
8ba6a92d0
|
1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 |
* xprt_wait_for_reply_request_def - wait for reply * @task: pointer to rpc_task * * Set a request's retransmit timeout based on the transport's * default timeout parameters. Used by transports that don't adjust * the retransmit timeout based on round-trip time estimation, * and put the task to sleep on the pending queue. */ void xprt_wait_for_reply_request_def(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; |
6b2e68562
|
1129 |
rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer, |
9e910bff7
|
1130 |
xprt_request_timeout(req)); |
8ba6a92d0
|
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 |
} EXPORT_SYMBOL_GPL(xprt_wait_for_reply_request_def); /** * xprt_wait_for_reply_request_rtt - wait for reply using RTT estimator * @task: pointer to rpc_task * * Set a request's retransmit timeout using the RTT estimator, * and put the task to sleep on the pending queue. */ void xprt_wait_for_reply_request_rtt(struct rpc_task *task) { int timer = task->tk_msg.rpc_proc->p_timer; struct rpc_clnt *clnt = task->tk_client; struct rpc_rtt *rtt = clnt->cl_rtt; struct rpc_rqst *req = task->tk_rqstp; unsigned long max_timeout = clnt->cl_timeout->to_maxval; |
6b2e68562
|
1148 |
unsigned long timeout; |
8ba6a92d0
|
1149 |
|
6b2e68562
|
1150 1151 1152 1153 1154 1155 |
timeout = rpc_calc_rto(rtt, timer); timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries; if (timeout > max_timeout || timeout == 0) timeout = max_timeout; rpc_sleep_on_timeout(&req->rq_xprt->pending, task, xprt_timer, jiffies + timeout); |
8ba6a92d0
|
1156 1157 1158 1159 |
} EXPORT_SYMBOL_GPL(xprt_wait_for_reply_request_rtt); /** |
7f3a1d1e1
|
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 |
* xprt_request_wait_receive - wait for the reply to an RPC request * @task: RPC task about to send a request * */ void xprt_request_wait_receive(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; if (!test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) return; /* * Sleep on the pending queue if we're expecting a reply. * The spinlock ensures atomicity between the test of * req->rq_reply_bytes_recvd, and the call to rpc_sleep_on(). */ spin_lock(&xprt->queue_lock); if (test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate)) { |
8ba6a92d0
|
1178 |
xprt->ops->wait_for_reply_request(task); |
7f3a1d1e1
|
1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 |
/* * Send an extra queue wakeup call if the * connection was dropped in case the call to * rpc_sleep_on() raced. */ if (xprt_request_retransmit_after_disconnect(task)) rpc_wake_up_queued_task_set_status(&xprt->pending, task, -ENOTCONN); } spin_unlock(&xprt->queue_lock); } |
944b04292
|
1190 |
static bool |
944b04292
|
1191 1192 |
xprt_request_need_enqueue_transmit(struct rpc_task *task, struct rpc_rqst *req) { |
762e4e67b
|
1193 |
return !test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate); |
944b04292
|
1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 |
} /** * xprt_request_enqueue_transmit - queue a task for transmission * @task: pointer to rpc_task * * Add a task to the transmission queue. */ void xprt_request_enqueue_transmit(struct rpc_task *task) { |
918f3c1fe
|
1205 |
struct rpc_rqst *pos, *req = task->tk_rqstp; |
944b04292
|
1206 1207 1208 |
struct rpc_xprt *xprt = req->rq_xprt; if (xprt_request_need_enqueue_transmit(task, req)) { |
e66721f04
|
1209 |
req->rq_bytes_sent = 0; |
944b04292
|
1210 |
spin_lock(&xprt->queue_lock); |
75891f502
|
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 |
/* * Requests that carry congestion control credits are added * to the head of the list to avoid starvation issues. */ if (req->rq_cong) { xprt_clear_congestion_window_wait(xprt); list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) { if (pos->rq_cong) continue; /* Note: req is added _before_ pos */ list_add_tail(&req->rq_xmit, &pos->rq_xmit); INIT_LIST_HEAD(&req->rq_xmit2); |
0c77668dd
|
1223 |
trace_xprt_enq_xmit(task, 1); |
75891f502
|
1224 1225 |
goto out; } |
86aeee0eb
|
1226 1227 1228 1229 1230 1231 1232 1233 1234 |
} else if (RPC_IS_SWAPPER(task)) { list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) { if (pos->rq_cong || pos->rq_bytes_sent) continue; if (RPC_IS_SWAPPER(pos->rq_task)) continue; /* Note: req is added _before_ pos */ list_add_tail(&req->rq_xmit, &pos->rq_xmit); INIT_LIST_HEAD(&req->rq_xmit2); |
0c77668dd
|
1235 |
trace_xprt_enq_xmit(task, 2); |
86aeee0eb
|
1236 1237 |
goto out; } |
deaa5c96c
|
1238 |
} else if (!req->rq_seqno) { |
75891f502
|
1239 1240 1241 1242 1243 |
list_for_each_entry(pos, &xprt->xmit_queue, rq_xmit) { if (pos->rq_task->tk_owner != task->tk_owner) continue; list_add_tail(&req->rq_xmit2, &pos->rq_xmit2); INIT_LIST_HEAD(&req->rq_xmit); |
0c77668dd
|
1244 |
trace_xprt_enq_xmit(task, 3); |
75891f502
|
1245 1246 |
goto out; } |
918f3c1fe
|
1247 |
} |
944b04292
|
1248 |
list_add_tail(&req->rq_xmit, &xprt->xmit_queue); |
918f3c1fe
|
1249 |
INIT_LIST_HEAD(&req->rq_xmit2); |
0c77668dd
|
1250 |
trace_xprt_enq_xmit(task, 4); |
918f3c1fe
|
1251 |
out: |
944b04292
|
1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 |
set_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate); spin_unlock(&xprt->queue_lock); } } /** * xprt_request_dequeue_transmit_locked - remove a task from the transmission queue * @task: pointer to rpc_task * * Remove a task from the transmission queue * Caller must hold xprt->queue_lock */ static void xprt_request_dequeue_transmit_locked(struct rpc_task *task) { |
918f3c1fe
|
1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 |
struct rpc_rqst *req = task->tk_rqstp; if (!test_and_clear_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) return; if (!list_empty(&req->rq_xmit)) { list_del(&req->rq_xmit); if (!list_empty(&req->rq_xmit2)) { struct rpc_rqst *next = list_first_entry(&req->rq_xmit2, struct rpc_rqst, rq_xmit2); list_del(&req->rq_xmit2); list_add_tail(&next->rq_xmit, &next->rq_xprt->xmit_queue); } } else list_del(&req->rq_xmit2); |
944b04292
|
1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 |
} /** * xprt_request_dequeue_transmit - remove a task from the transmission queue * @task: pointer to rpc_task * * Remove a task from the transmission queue */ static void xprt_request_dequeue_transmit(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; spin_lock(&xprt->queue_lock); xprt_request_dequeue_transmit_locked(task); spin_unlock(&xprt->queue_lock); } |
7f3a1d1e1
|
1299 |
/** |
cc204d012
|
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 |
* xprt_request_dequeue_xprt - remove a task from the transmit+receive queue * @task: pointer to rpc_task * * Remove a task from the transmit and receive queues, and ensure that * it is not pinned by the receive work item. */ void xprt_request_dequeue_xprt(struct rpc_task *task) { struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate) || test_bit(RPC_TASK_NEED_RECV, &task->tk_runstate) || xprt_is_pinned_rqst(req)) { spin_lock(&xprt->queue_lock); xprt_request_dequeue_transmit_locked(task); xprt_request_dequeue_receive_locked(task); while (xprt_is_pinned_rqst(req)) { set_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate); spin_unlock(&xprt->queue_lock); xprt_wait_on_pinned_rqst(req); spin_lock(&xprt->queue_lock); clear_bit(RPC_TASK_MSG_PIN_WAIT, &task->tk_runstate); } spin_unlock(&xprt->queue_lock); } } /** |
9d96acbc7
|
1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 |
* xprt_request_prepare - prepare an encoded request for transport * @req: pointer to rpc_rqst * * Calls into the transport layer to do whatever is needed to prepare * the request for transmission or receive. */ void xprt_request_prepare(struct rpc_rqst *req) { struct rpc_xprt *xprt = req->rq_xprt; if (xprt->ops->prepare_request) xprt->ops->prepare_request(req); } /** |
762e4e67b
|
1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 |
* xprt_request_need_retransmit - Test if a task needs retransmission * @task: pointer to rpc_task * * Test for whether a connection breakage requires the task to retransmit */ bool xprt_request_need_retransmit(struct rpc_task *task) { return xprt_request_retransmit_after_disconnect(task); } /** |
9903cd1c2
|
1358 1359 1360 |
* xprt_prepare_transmit - reserve the transport before sending a request * @task: RPC task about to send a request * |
1da177e4c
|
1361 |
*/ |
90051ea77
|
1362 |
bool xprt_prepare_transmit(struct rpc_task *task) |
1da177e4c
|
1363 1364 1365 |
{ struct rpc_rqst *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; |
1da177e4c
|
1366 |
|
46121cf7d
|
1367 1368 |
dprintk("RPC: %5u xprt_prepare_transmit ", task->tk_pid); |
1da177e4c
|
1369 |
|
5f2f6bd98
|
1370 1371 |
if (!xprt_lock_write(xprt, task)) { /* Race breaker: someone may have transmitted us */ |
944b04292
|
1372 |
if (!test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) |
5f2f6bd98
|
1373 1374 1375 |
rpc_wake_up_queued_task_set_status(&xprt->sending, task, 0); return false; |
90051ea77
|
1376 |
} |
5f2f6bd98
|
1377 |
return true; |
1da177e4c
|
1378 |
} |
e0ab53dea
|
1379 |
void xprt_end_transmit(struct rpc_task *task) |
5e5ce5be6
|
1380 |
{ |
343952fa5
|
1381 |
xprt_release_write(task->tk_rqstp->rq_xprt, task); |
5e5ce5be6
|
1382 |
} |
9903cd1c2
|
1383 |
/** |
89f90fe1a
|
1384 1385 1386 |
* xprt_request_transmit - send an RPC request on a transport * @req: pointer to request to transmit * @snd_task: RPC task that owns the transport lock |
9903cd1c2
|
1387 |
* |
89f90fe1a
|
1388 1389 1390 1391 |
* This performs the transmission of a single request. * Note that if the request is not the same as snd_task, then it * does need to be pinned. * Returns '0' on success. |
9903cd1c2
|
1392 |
*/ |
89f90fe1a
|
1393 1394 |
static int xprt_request_transmit(struct rpc_rqst *req, struct rpc_task *snd_task) |
1da177e4c
|
1395 |
{ |
89f90fe1a
|
1396 1397 |
struct rpc_xprt *xprt = req->rq_xprt; struct rpc_task *task = req->rq_task; |
90d91b0cd
|
1398 |
unsigned int connect_cookie; |
dcbbeda83
|
1399 |
int is_retrans = RPC_WAS_SENT(task); |
ff699ea82
|
1400 |
int status; |
1da177e4c
|
1401 |
|
edc81dcd5
|
1402 |
if (!req->rq_bytes_sent) { |
89f90fe1a
|
1403 1404 |
if (xprt_request_data_received(task)) { status = 0; |
944b04292
|
1405 |
goto out_dequeue; |
89f90fe1a
|
1406 |
} |
3021a5bbb
|
1407 |
/* Verify that our message lies in the RPCSEC_GSS window */ |
edc81dcd5
|
1408 |
if (rpcauth_xmit_need_reencode(task)) { |
89f90fe1a
|
1409 |
status = -EBADMSG; |
944b04292
|
1410 |
goto out_dequeue; |
a79f194aa
|
1411 |
} |
ae67bd382
|
1412 1413 1414 |
if (RPC_SIGNALLED(task)) { status = -ERESTARTSYS; goto out_dequeue; |
3021a5bbb
|
1415 |
} |
edc81dcd5
|
1416 |
} |
1da177e4c
|
1417 |
|
dcbbeda83
|
1418 1419 1420 1421 1422 1423 |
/* * Update req->rq_ntrans before transmitting to avoid races with * xprt_update_rtt(), which needs to know that it is recording a * reply to the first transmission. */ req->rq_ntrans++; |
90d91b0cd
|
1424 |
connect_cookie = xprt->connect_cookie; |
adfa71446
|
1425 |
status = xprt->ops->send_request(req); |
c8485e4d6
|
1426 |
if (status != 0) { |
dcbbeda83
|
1427 |
req->rq_ntrans--; |
0c77668dd
|
1428 |
trace_xprt_transmit(req, status); |
89f90fe1a
|
1429 |
return status; |
c8485e4d6
|
1430 |
} |
7ebbbc6e7
|
1431 |
|
dcbbeda83
|
1432 1433 |
if (is_retrans) task->tk_client->cl_stats->rpcretrans++; |
4a0682583
|
1434 |
xprt_inject_disconnect(xprt); |
262ca07de
|
1435 |
|
468f86134
|
1436 |
task->tk_flags |= RPC_TASK_SENT; |
b5e924191
|
1437 |
spin_lock(&xprt->transport_lock); |
262ca07de
|
1438 |
|
c8485e4d6
|
1439 1440 1441 |
xprt->stat.sends++; xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs; xprt->stat.bklog_u += xprt->backlog.qlen; |
15a452062
|
1442 1443 |
xprt->stat.sending_u += xprt->sending.qlen; xprt->stat.pending_u += xprt->pending.qlen; |
b5e924191
|
1444 |
spin_unlock(&xprt->transport_lock); |
1da177e4c
|
1445 |
|
90d91b0cd
|
1446 |
req->rq_connect_cookie = connect_cookie; |
944b04292
|
1447 |
out_dequeue: |
0c77668dd
|
1448 |
trace_xprt_transmit(req, status); |
944b04292
|
1449 |
xprt_request_dequeue_transmit(task); |
89f90fe1a
|
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 |
rpc_wake_up_queued_task_set_status(&xprt->sending, task, status); return status; } /** * xprt_transmit - send an RPC request on a transport * @task: controlling RPC task * * Attempts to drain the transmit queue. On exit, either the transport * signalled an error that needs to be handled before transmission can * resume, or @task finished transmitting, and detected that it already * received a reply. */ void xprt_transmit(struct rpc_task *task) { struct rpc_rqst *next, *req = task->tk_rqstp; struct rpc_xprt *xprt = req->rq_xprt; int status; spin_lock(&xprt->queue_lock); while (!list_empty(&xprt->xmit_queue)) { next = list_first_entry(&xprt->xmit_queue, struct rpc_rqst, rq_xmit); xprt_pin_rqst(next); spin_unlock(&xprt->queue_lock); status = xprt_request_transmit(next, task); if (status == -EBADMSG && next != req) status = 0; cond_resched(); spin_lock(&xprt->queue_lock); xprt_unpin_rqst(next); if (status == 0) { if (!xprt_request_data_received(task) || test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) continue; |
c544577da
|
1486 |
} else if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) |
89f90fe1a
|
1487 1488 1489 1490 |
task->tk_status = status; break; } spin_unlock(&xprt->queue_lock); |
1da177e4c
|
1491 |
} |
ba60eb25f
|
1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 |
static void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task) { set_bit(XPRT_CONGESTED, &xprt->state); rpc_sleep_on(&xprt->backlog, task, NULL); } static void xprt_wake_up_backlog(struct rpc_xprt *xprt) { if (rpc_wake_up_next(&xprt->backlog) == NULL) clear_bit(XPRT_CONGESTED, &xprt->state); } static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task) { bool ret = false; if (!test_bit(XPRT_CONGESTED, &xprt->state)) goto out; spin_lock(&xprt->reserve_lock); if (test_bit(XPRT_CONGESTED, &xprt->state)) { rpc_sleep_on(&xprt->backlog, task, NULL); ret = true; } spin_unlock(&xprt->reserve_lock); out: return ret; } |
92ea011f7
|
1519 |
static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt) |
d9ba131d8
|
1520 1521 |
{ struct rpc_rqst *req = ERR_PTR(-EAGAIN); |
ff699ea82
|
1522 |
if (xprt->num_reqs >= xprt->max_reqs) |
d9ba131d8
|
1523 |
goto out; |
ff699ea82
|
1524 |
++xprt->num_reqs; |
92ea011f7
|
1525 1526 1527 |
spin_unlock(&xprt->reserve_lock); req = kzalloc(sizeof(struct rpc_rqst), GFP_NOFS); spin_lock(&xprt->reserve_lock); |
d9ba131d8
|
1528 1529 |
if (req != NULL) goto out; |
ff699ea82
|
1530 |
--xprt->num_reqs; |
d9ba131d8
|
1531 1532 1533 1534 1535 1536 1537 |
req = ERR_PTR(-ENOMEM); out: return req; } static bool xprt_dynamic_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req) { |
ff699ea82
|
1538 1539 |
if (xprt->num_reqs > xprt->min_reqs) { --xprt->num_reqs; |
d9ba131d8
|
1540 1541 1542 1543 1544 |
kfree(req); return true; } return false; } |
f39c1bfb5
|
1545 |
void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task) |
1da177e4c
|
1546 |
{ |
d9ba131d8
|
1547 |
struct rpc_rqst *req; |
1da177e4c
|
1548 |
|
f39c1bfb5
|
1549 |
spin_lock(&xprt->reserve_lock); |
1da177e4c
|
1550 |
if (!list_empty(&xprt->free)) { |
d9ba131d8
|
1551 1552 1553 1554 |
req = list_entry(xprt->free.next, struct rpc_rqst, rq_list); list_del(&req->rq_list); goto out_init_req; } |
92ea011f7
|
1555 |
req = xprt_dynamic_alloc_slot(xprt); |
d9ba131d8
|
1556 1557 1558 1559 |
if (!IS_ERR(req)) goto out_init_req; switch (PTR_ERR(req)) { case -ENOMEM: |
d9ba131d8
|
1560 1561 1562 |
dprintk("RPC: dynamic allocation of request slot " "failed! Retrying "); |
1afeaf5c2
|
1563 |
task->tk_status = -ENOMEM; |
d9ba131d8
|
1564 1565 |
break; case -EAGAIN: |
ba60eb25f
|
1566 |
xprt_add_backlog(xprt, task); |
d9ba131d8
|
1567 1568 |
dprintk("RPC: waiting for request slot "); |
e9d476393
|
1569 |
/* fall through */ |
1afeaf5c2
|
1570 1571 |
default: task->tk_status = -EAGAIN; |
1da177e4c
|
1572 |
} |
f39c1bfb5
|
1573 |
spin_unlock(&xprt->reserve_lock); |
d9ba131d8
|
1574 1575 |
return; out_init_req: |
ff699ea82
|
1576 1577 |
xprt->stat.max_slots = max_t(unsigned int, xprt->stat.max_slots, xprt->num_reqs); |
37ac86c3a
|
1578 |
spin_unlock(&xprt->reserve_lock); |
d9ba131d8
|
1579 1580 |
task->tk_status = 0; task->tk_rqstp = req; |
f39c1bfb5
|
1581 1582 |
} EXPORT_SYMBOL_GPL(xprt_alloc_slot); |
a9cde23ab
|
1583 |
void xprt_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req) |
ee5ebe851
|
1584 |
{ |
ee5ebe851
|
1585 |
spin_lock(&xprt->reserve_lock); |
c25573b51
|
1586 1587 1588 1589 |
if (!xprt_dynamic_free_slot(xprt, req)) { memset(req, 0, sizeof(*req)); /* mark unused */ list_add(&req->rq_list, &xprt->free); } |
ba60eb25f
|
1590 |
xprt_wake_up_backlog(xprt); |
ee5ebe851
|
1591 1592 |
spin_unlock(&xprt->reserve_lock); } |
a9cde23ab
|
1593 |
EXPORT_SYMBOL_GPL(xprt_free_slot); |
ee5ebe851
|
1594 |
|
21de0a955
|
1595 1596 1597 1598 1599 1600 1601 1602 1603 |
static void xprt_free_all_slots(struct rpc_xprt *xprt) { struct rpc_rqst *req; while (!list_empty(&xprt->free)) { req = list_first_entry(&xprt->free, struct rpc_rqst, rq_list); list_del(&req->rq_list); kfree(req); } } |
d9ba131d8
|
1604 1605 1606 |
struct rpc_xprt *xprt_alloc(struct net *net, size_t size, unsigned int num_prealloc, unsigned int max_alloc) |
bd1722d43
|
1607 1608 |
{ struct rpc_xprt *xprt; |
21de0a955
|
1609 1610 |
struct rpc_rqst *req; int i; |
bd1722d43
|
1611 1612 1613 1614 |
xprt = kzalloc(size, GFP_KERNEL); if (xprt == NULL) goto out; |
21de0a955
|
1615 1616 1617 1618 1619 |
xprt_init(xprt, net); for (i = 0; i < num_prealloc; i++) { req = kzalloc(sizeof(struct rpc_rqst), GFP_KERNEL); if (!req) |
8313164c3
|
1620 |
goto out_free; |
21de0a955
|
1621 1622 |
list_add(&req->rq_list, &xprt->free); } |
d9ba131d8
|
1623 1624 1625 1626 1627 |
if (max_alloc > num_prealloc) xprt->max_reqs = max_alloc; else xprt->max_reqs = num_prealloc; xprt->min_reqs = num_prealloc; |
ff699ea82
|
1628 |
xprt->num_reqs = num_prealloc; |
bd1722d43
|
1629 1630 1631 1632 |
return xprt; out_free: |
21de0a955
|
1633 |
xprt_free(xprt); |
bd1722d43
|
1634 1635 1636 1637 |
out: return NULL; } EXPORT_SYMBOL_GPL(xprt_alloc); |
e204e621b
|
1638 1639 |
void xprt_free(struct rpc_xprt *xprt) { |
37aa21337
|
1640 |
put_net(xprt->xprt_net); |
21de0a955
|
1641 |
xprt_free_all_slots(xprt); |
fda1bfef9
|
1642 |
kfree_rcu(xprt, rcu); |
e204e621b
|
1643 1644 |
} EXPORT_SYMBOL_GPL(xprt_free); |
902c58872
|
1645 1646 1647 1648 1649 |
static void xprt_init_connect_cookie(struct rpc_rqst *req, struct rpc_xprt *xprt) { req->rq_connect_cookie = xprt_connect_cookie(xprt) - 1; } |
9dc6edcf6
|
1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 |
static __be32 xprt_alloc_xid(struct rpc_xprt *xprt) { __be32 xid; spin_lock(&xprt->reserve_lock); xid = (__force __be32)xprt->xid++; spin_unlock(&xprt->reserve_lock); return xid; } static void xprt_init_xid(struct rpc_xprt *xprt) { xprt->xid = prandom_u32(); } static void xprt_request_init(struct rpc_task *task) { struct rpc_xprt *xprt = task->tk_xprt; struct rpc_rqst *req = task->tk_rqstp; |
9dc6edcf6
|
1672 1673 1674 1675 |
req->rq_task = task; req->rq_xprt = xprt; req->rq_buffer = NULL; req->rq_xid = xprt_alloc_xid(xprt); |
902c58872
|
1676 |
xprt_init_connect_cookie(req, xprt); |
9dc6edcf6
|
1677 1678 1679 1680 |
req->rq_snd_buf.len = 0; req->rq_snd_buf.buflen = 0; req->rq_rcv_buf.len = 0; req->rq_rcv_buf.buflen = 0; |
71700bb96
|
1681 1682 |
req->rq_snd_buf.bvec = NULL; req->rq_rcv_buf.bvec = NULL; |
9dc6edcf6
|
1683 |
req->rq_release_snd_buf = NULL; |
da953063b
|
1684 |
xprt_init_majortimeo(task, req); |
9dc6edcf6
|
1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 |
dprintk("RPC: %5u reserved req %p xid %08x ", task->tk_pid, req, ntohl(req->rq_xid)); } static void xprt_do_reserve(struct rpc_xprt *xprt, struct rpc_task *task) { xprt->ops->alloc_slot(xprt, task); if (task->tk_rqstp != NULL) xprt_request_init(task); } |
9903cd1c2
|
1697 1698 1699 1700 |
/** * xprt_reserve - allocate an RPC request slot * @task: RPC task requesting a slot allocation * |
ba60eb25f
|
1701 1702 |
* If the transport is marked as being congested, or if no more * slots are available, place the task on the transport's |
9903cd1c2
|
1703 1704 1705 |
* backlog queue. */ void xprt_reserve(struct rpc_task *task) |
1da177e4c
|
1706 |
{ |
fb43d1721
|
1707 |
struct rpc_xprt *xprt = task->tk_xprt; |
1da177e4c
|
1708 |
|
43cedbf0e
|
1709 1710 1711 |
task->tk_status = 0; if (task->tk_rqstp != NULL) return; |
43cedbf0e
|
1712 |
task->tk_status = -EAGAIN; |
ba60eb25f
|
1713 |
if (!xprt_throttle_congested(xprt, task)) |
9dc6edcf6
|
1714 |
xprt_do_reserve(xprt, task); |
ba60eb25f
|
1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 |
} /** * xprt_retry_reserve - allocate an RPC request slot * @task: RPC task requesting a slot allocation * * If no more slots are available, place the task on the transport's * backlog queue. * Note that the only difference with xprt_reserve is that we now * ignore the value of the XPRT_CONGESTED flag. */ void xprt_retry_reserve(struct rpc_task *task) { |
fb43d1721
|
1728 |
struct rpc_xprt *xprt = task->tk_xprt; |
ba60eb25f
|
1729 1730 1731 1732 |
task->tk_status = 0; if (task->tk_rqstp != NULL) return; |
ba60eb25f
|
1733 |
task->tk_status = -EAGAIN; |
9dc6edcf6
|
1734 |
xprt_do_reserve(xprt, task); |
1da177e4c
|
1735 |
} |
9903cd1c2
|
1736 1737 1738 1739 |
/** * xprt_release - release an RPC request slot * @task: task which is finished with the slot * |
1da177e4c
|
1740 |
*/ |
9903cd1c2
|
1741 |
void xprt_release(struct rpc_task *task) |
1da177e4c
|
1742 |
{ |
55ae1aabf
|
1743 |
struct rpc_xprt *xprt; |
87ed50036
|
1744 |
struct rpc_rqst *req = task->tk_rqstp; |
1da177e4c
|
1745 |
|
87ed50036
|
1746 1747 |
if (req == NULL) { if (task->tk_client) { |
fb43d1721
|
1748 |
xprt = task->tk_xprt; |
bd79bc579
|
1749 |
xprt_release_write(xprt, task); |
87ed50036
|
1750 |
} |
1da177e4c
|
1751 |
return; |
87ed50036
|
1752 |
} |
55ae1aabf
|
1753 |
|
55ae1aabf
|
1754 |
xprt = req->rq_xprt; |
cc204d012
|
1755 |
xprt_request_dequeue_xprt(task); |
b5e924191
|
1756 |
spin_lock(&xprt->transport_lock); |
49e9a8908
|
1757 |
xprt->ops->release_xprt(xprt, task); |
a58dd398f
|
1758 1759 |
if (xprt->ops->release_request) xprt->ops->release_request(task); |
ad3331acb
|
1760 |
xprt_schedule_autodisconnect(xprt); |
b5e924191
|
1761 |
spin_unlock(&xprt->transport_lock); |
ee5ebe851
|
1762 |
if (req->rq_buffer) |
3435c74ae
|
1763 |
xprt->ops->buf_free(task); |
4a0682583
|
1764 |
xprt_inject_disconnect(xprt); |
9d96acbc7
|
1765 |
xdr_free_bvec(&req->rq_rcv_buf); |
0472e4766
|
1766 |
xdr_free_bvec(&req->rq_snd_buf); |
a17c2153d
|
1767 1768 |
if (req->rq_cred != NULL) put_rpccred(req->rq_cred); |
1da177e4c
|
1769 |
task->tk_rqstp = NULL; |
ead5e1c26
|
1770 1771 |
if (req->rq_release_snd_buf) req->rq_release_snd_buf(req); |
55ae1aabf
|
1772 |
|
46121cf7d
|
1773 1774 |
dprintk("RPC: %5u release request %p ", task->tk_pid, req); |
ee5ebe851
|
1775 |
if (likely(!bc_prealloc(req))) |
a9cde23ab
|
1776 |
xprt->ops->free_slot(xprt, req); |
ee5ebe851
|
1777 |
else |
c9acb42ef
|
1778 |
xprt_free_bc_request(req); |
1da177e4c
|
1779 |
} |
902c58872
|
1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 |
#ifdef CONFIG_SUNRPC_BACKCHANNEL void xprt_init_bc_request(struct rpc_rqst *req, struct rpc_task *task) { struct xdr_buf *xbufp = &req->rq_snd_buf; task->tk_rqstp = req; req->rq_task = task; xprt_init_connect_cookie(req, req->rq_xprt); /* * Set up the xdr_buf length. * This also indicates that the buffer is XDR encoded already. */ xbufp->len = xbufp->head[0].iov_len + xbufp->page_len + xbufp->tail[0].iov_len; |
902c58872
|
1795 1796 |
} #endif |
21de0a955
|
1797 |
static void xprt_init(struct rpc_xprt *xprt, struct net *net) |
c2866763b
|
1798 |
{ |
30c5116b1
|
1799 |
kref_init(&xprt->kref); |
c2866763b
|
1800 1801 1802 |
spin_lock_init(&xprt->transport_lock); spin_lock_init(&xprt->reserve_lock); |
75c84151a
|
1803 |
spin_lock_init(&xprt->queue_lock); |
c2866763b
|
1804 1805 |
INIT_LIST_HEAD(&xprt->free); |
95f7691da
|
1806 |
xprt->recv_queue = RB_ROOT; |
944b04292
|
1807 |
INIT_LIST_HEAD(&xprt->xmit_queue); |
9e00abc3c
|
1808 |
#if defined(CONFIG_SUNRPC_BACKCHANNEL) |
f9acac1a4
|
1809 1810 |
spin_lock_init(&xprt->bc_pa_lock); INIT_LIST_HEAD(&xprt->bc_pa_list); |
9e00abc3c
|
1811 |
#endif /* CONFIG_SUNRPC_BACKCHANNEL */ |
80b14d5e6
|
1812 |
INIT_LIST_HEAD(&xprt->xprt_switch); |
f9acac1a4
|
1813 |
|
c2866763b
|
1814 1815 |
xprt->last_used = jiffies; xprt->cwnd = RPC_INITCWND; |
a509050bd
|
1816 |
xprt->bind_index = 0; |
c2866763b
|
1817 1818 1819 |
rpc_init_wait_queue(&xprt->binding, "xprt_binding"); rpc_init_wait_queue(&xprt->pending, "xprt_pending"); |
79c99152a
|
1820 |
rpc_init_wait_queue(&xprt->sending, "xprt_sending"); |
c2866763b
|
1821 |
rpc_init_priority_wait_queue(&xprt->backlog, "xprt_backlog"); |
c2866763b
|
1822 |
xprt_init_xid(xprt); |
21de0a955
|
1823 |
xprt->xprt_net = get_net(net); |
8d9266ffe
|
1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 |
} /** * xprt_create_transport - create an RPC transport * @args: rpc transport creation arguments * */ struct rpc_xprt *xprt_create_transport(struct xprt_create *args) { struct rpc_xprt *xprt; struct xprt_class *t; spin_lock(&xprt_list_lock); list_for_each_entry(t, &xprt_list, list) { if (t->ident == args->ident) { spin_unlock(&xprt_list_lock); goto found; } } spin_unlock(&xprt_list_lock); |
3c45ddf82
|
1844 1845 |
dprintk("RPC: transport (%d) not supported ", args->ident); |
8d9266ffe
|
1846 1847 1848 1849 1850 1851 1852 1853 |
return ERR_PTR(-EIO); found: xprt = t->setup(args); if (IS_ERR(xprt)) { dprintk("RPC: xprt_create_transport: failed, %ld ", -PTR_ERR(xprt)); |
21de0a955
|
1854 |
goto out; |
8d9266ffe
|
1855 |
} |
33d90ac05
|
1856 1857 |
if (args->flags & XPRT_CREATE_NO_IDLE_TIMEOUT) xprt->idle_timeout = 0; |
21de0a955
|
1858 1859 |
INIT_WORK(&xprt->task_cleanup, xprt_autoclose); if (xprt_has_timer(xprt)) |
502980e84
|
1860 |
timer_setup(&xprt->timer, xprt_init_autodisconnect, 0); |
21de0a955
|
1861 |
else |
ff861c4d6
|
1862 |
timer_setup(&xprt->timer, NULL, 0); |
4e0038b6b
|
1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 |
if (strlen(args->servername) > RPC_MAXNETNAMELEN) { xprt_destroy(xprt); return ERR_PTR(-EINVAL); } xprt->servername = kstrdup(args->servername, GFP_KERNEL); if (xprt->servername == NULL) { xprt_destroy(xprt); return ERR_PTR(-ENOMEM); } |
3f9400981
|
1873 |
rpc_xprt_debugfs_register(xprt); |
388f0c776
|
1874 |
|
46121cf7d
|
1875 1876 |
dprintk("RPC: created transport %p with %u slots ", xprt, |
c2866763b
|
1877 |
xprt->max_reqs); |
21de0a955
|
1878 |
out: |
c2866763b
|
1879 1880 |
return xprt; } |
528fd3547
|
1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 |
static void xprt_destroy_cb(struct work_struct *work) { struct rpc_xprt *xprt = container_of(work, struct rpc_xprt, task_cleanup); rpc_xprt_debugfs_unregister(xprt); rpc_destroy_wait_queue(&xprt->binding); rpc_destroy_wait_queue(&xprt->pending); rpc_destroy_wait_queue(&xprt->sending); rpc_destroy_wait_queue(&xprt->backlog); kfree(xprt->servername); /* |
669996add
|
1893 1894 1895 1896 1897 |
* Destroy any existing back channel */ xprt_destroy_backchannel(xprt, UINT_MAX); /* |
528fd3547
|
1898 1899 1900 1901 |
* Tear down transport state and free the rpc_xprt */ xprt->ops->destroy(xprt); } |
9903cd1c2
|
1902 1903 |
/** * xprt_destroy - destroy an RPC transport, killing off all requests. |
a8de240a9
|
1904 |
* @xprt: transport to destroy |
9903cd1c2
|
1905 |
* |
1da177e4c
|
1906 |
*/ |
a8de240a9
|
1907 |
static void xprt_destroy(struct rpc_xprt *xprt) |
1da177e4c
|
1908 |
{ |
46121cf7d
|
1909 1910 |
dprintk("RPC: destroying transport %p ", xprt); |
79234c3db
|
1911 |
|
528fd3547
|
1912 1913 1914 |
/* * Exclude transport connect/disconnect handlers and autoclose */ |
79234c3db
|
1915 |
wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_UNINTERRUPTIBLE); |
0065db328
|
1916 |
del_timer_sync(&xprt->timer); |
c8541ecdd
|
1917 1918 |
/* |
528fd3547
|
1919 1920 |
* Destroy sockets etc from the system workqueue so they can * safely flush receive work running on rpciod. |
c8541ecdd
|
1921 |
*/ |
528fd3547
|
1922 1923 |
INIT_WORK(&xprt->task_cleanup, xprt_destroy_cb); schedule_work(&xprt->task_cleanup); |
6b6ca86b7
|
1924 |
} |
1da177e4c
|
1925 |
|
30c5116b1
|
1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 |
static void xprt_destroy_kref(struct kref *kref) { xprt_destroy(container_of(kref, struct rpc_xprt, kref)); } /** * xprt_get - return a reference to an RPC transport. * @xprt: pointer to the transport * */ struct rpc_xprt *xprt_get(struct rpc_xprt *xprt) { if (xprt != NULL && kref_get_unless_zero(&xprt->kref)) return xprt; return NULL; } EXPORT_SYMBOL_GPL(xprt_get); |
6b6ca86b7
|
1943 1944 1945 1946 1947 1948 1949 |
/** * xprt_put - release a reference to an RPC transport. * @xprt: pointer to the transport * */ void xprt_put(struct rpc_xprt *xprt) { |
30c5116b1
|
1950 1951 |
if (xprt != NULL) kref_put(&xprt->kref, xprt_destroy_kref); |
6b6ca86b7
|
1952 |
} |
5d252f90a
|
1953 |
EXPORT_SYMBOL_GPL(xprt_put); |