Blame view

net/rxrpc/output.c 17.7 KB
2874c5fd2   Thomas Gleixner   treewide: Replace...
1
  // SPDX-License-Identifier: GPL-2.0-or-later
17926a793   David Howells   [AF_RXRPC]: Provi...
2
3
4
5
  /* RxRPC packet transmission
   *
   * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   * Written by David Howells (dhowells@redhat.com)
17926a793   David Howells   [AF_RXRPC]: Provi...
6
   */
9b6d53985   Joe Perches   rxrpc: Use pr_<le...
7
  #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
17926a793   David Howells   [AF_RXRPC]: Provi...
8
  #include <linux/net.h>
5a0e3ad6a   Tejun Heo   include cleanup: ...
9
  #include <linux/gfp.h>
17926a793   David Howells   [AF_RXRPC]: Provi...
10
  #include <linux/skbuff.h>
bc3b2d7fb   Paul Gortmaker   net: Add export.h...
11
  #include <linux/export.h>
17926a793   David Howells   [AF_RXRPC]: Provi...
12
13
14
  #include <net/sock.h>
  #include <net/af_rxrpc.h>
  #include "ar-internal.h"
26cb02aa6   David Howells   rxrpc: Fix warnin...
15
  struct rxrpc_ack_buffer {
8d94aa381   David Howells   rxrpc: Calls shou...
16
  	struct rxrpc_wire_header whdr;
26cb02aa6   David Howells   rxrpc: Fix warnin...
17
18
19
  	struct rxrpc_ackpacket ack;
  	u8 acks[255];
  	u8 pad[3];
8d94aa381   David Howells   rxrpc: Calls shou...
20
21
  	struct rxrpc_ackinfo ackinfo;
  };
26cb02aa6   David Howells   rxrpc: Fix warnin...
22
23
24
25
  struct rxrpc_abort_buffer {
  	struct rxrpc_wire_header whdr;
  	__be32 abort_code;
  };
ace45bec6   David Howells   rxrpc: Fix firewa...
26
  static const char rxrpc_keepalive_string[] = "";
8d94aa381   David Howells   rxrpc: Calls shou...
27
  /*
c7e86acfc   David Howells   rxrpc: Fix lockup...
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
   * Increase Tx backoff on transmission failure and clear it on success.
   */
  static void rxrpc_tx_backoff(struct rxrpc_call *call, int ret)
  {
  	if (ret < 0) {
  		u16 tx_backoff = READ_ONCE(call->tx_backoff);
  
  		if (tx_backoff < HZ)
  			WRITE_ONCE(call->tx_backoff, tx_backoff + 1);
  	} else {
  		WRITE_ONCE(call->tx_backoff, 0);
  	}
  }
  
  /*
415f44e43   David Howells   rxrpc: Add keepal...
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
   * Arrange for a keepalive ping a certain time after we last transmitted.  This
   * lets the far side know we're still interested in this call and helps keep
   * the route through any intervening firewall open.
   *
   * Receiving a response to the ping will prevent the ->expect_rx_by timer from
   * expiring.
   */
  static void rxrpc_set_keepalive(struct rxrpc_call *call)
  {
  	unsigned long now = jiffies, keepalive_at = call->next_rx_timo / 6;
  
  	keepalive_at += now;
  	WRITE_ONCE(call->keepalive_at, keepalive_at);
  	rxrpc_reduce_call_timer(call, keepalive_at, now,
  				rxrpc_timer_set_for_keepalive);
  }
  
  /*
8d94aa381   David Howells   rxrpc: Calls shou...
61
62
   * Fill out an ACK packet.
   */
1457cc4cf   David Howells   rxrpc: Fix a null...
63
64
  static size_t rxrpc_fill_out_ack(struct rxrpc_connection *conn,
  				 struct rxrpc_call *call,
26cb02aa6   David Howells   rxrpc: Fix warnin...
65
  				 struct rxrpc_ack_buffer *pkt,
805b21b92   David Howells   rxrpc: Send an AC...
66
  				 rxrpc_seq_t *_hard_ack,
a5af7e1fc   David Howells   rxrpc: Fix loss o...
67
68
  				 rxrpc_seq_t *_top,
  				 u8 reason)
8d94aa381   David Howells   rxrpc: Calls shou...
69
  {
f3639df2d   David Howells   rxrpc: Add a trac...
70
  	rxrpc_serial_t serial;
248f219cb   David Howells   rxrpc: Rewrite th...
71
72
  	rxrpc_seq_t hard_ack, top, seq;
  	int ix;
8d94aa381   David Howells   rxrpc: Calls shou...
73
74
  	u32 mtu, jmax;
  	u8 *ackp = pkt->acks;
248f219cb   David Howells   rxrpc: Rewrite th...
75
  	/* Barrier against rxrpc_input_data(). */
f3639df2d   David Howells   rxrpc: Add a trac...
76
  	serial = call->ackr_serial;
248f219cb   David Howells   rxrpc: Rewrite th...
77
78
  	hard_ack = READ_ONCE(call->rx_hard_ack);
  	top = smp_load_acquire(&call->rx_top);
805b21b92   David Howells   rxrpc: Send an AC...
79
80
  	*_hard_ack = hard_ack;
  	*_top = top;
248f219cb   David Howells   rxrpc: Rewrite th...
81

8d94aa381   David Howells   rxrpc: Calls shou...
82
  	pkt->ack.bufferSpace	= htons(8);
e8c3af6bb   David Howells   rxrpc: Don't both...
83
  	pkt->ack.maxSkew	= htons(0);
248f219cb   David Howells   rxrpc: Rewrite th...
84
  	pkt->ack.firstPacket	= htonl(hard_ack + 1);
8d94aa381   David Howells   rxrpc: Calls shou...
85
  	pkt->ack.previousPacket	= htonl(call->ackr_prev_seq);
f3639df2d   David Howells   rxrpc: Add a trac...
86
  	pkt->ack.serial		= htonl(serial);
a5af7e1fc   David Howells   rxrpc: Fix loss o...
87
  	pkt->ack.reason		= reason;
248f219cb   David Howells   rxrpc: Rewrite th...
88
  	pkt->ack.nAcks		= top - hard_ack;
a5af7e1fc   David Howells   rxrpc: Fix loss o...
89
  	if (reason == RXRPC_ACK_PING)
8e83134db   David Howells   rxrpc: Send pings...
90
  		pkt->whdr.flags |= RXRPC_REQUEST_ACK;
248f219cb   David Howells   rxrpc: Rewrite th...
91
92
93
94
95
96
97
98
99
100
101
  	if (after(top, hard_ack)) {
  		seq = hard_ack + 1;
  		do {
  			ix = seq & RXRPC_RXTX_BUFF_MASK;
  			if (call->rxtx_buffer[ix])
  				*ackp++ = RXRPC_ACK_TYPE_ACK;
  			else
  				*ackp++ = RXRPC_ACK_TYPE_NACK;
  			seq++;
  		} while (before_eq(seq, top));
  	}
8d94aa381   David Howells   rxrpc: Calls shou...
102

1457cc4cf   David Howells   rxrpc: Fix a null...
103
104
  	mtu = conn->params.peer->if_mtu;
  	mtu -= conn->params.peer->hdrsize;
75e421263   David Howells   rxrpc: Correctly ...
105
  	jmax = (call->nr_jumbo_bad > 3) ? 1 : rxrpc_rx_jumbo_max;
8d94aa381   David Howells   rxrpc: Calls shou...
106
107
  	pkt->ackinfo.rxMTU	= htonl(rxrpc_rx_mtu);
  	pkt->ackinfo.maxMTU	= htonl(mtu);
75e421263   David Howells   rxrpc: Correctly ...
108
  	pkt->ackinfo.rwind	= htonl(call->rx_winsize);
8d94aa381   David Howells   rxrpc: Calls shou...
109
110
111
112
113
  	pkt->ackinfo.jumbo_max	= htonl(jmax);
  
  	*ackp++ = 0;
  	*ackp++ = 0;
  	*ackp++ = 0;
248f219cb   David Howells   rxrpc: Rewrite th...
114
  	return top - hard_ack + 3;
8d94aa381   David Howells   rxrpc: Calls shou...
115
116
117
  }
  
  /*
4700c4d80   David Howells   rxrpc: Fix loss o...
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
   * Record the beginning of an RTT probe.
   */
  static int rxrpc_begin_rtt_probe(struct rxrpc_call *call, rxrpc_serial_t serial,
  				 enum rxrpc_rtt_tx_trace why)
  {
  	unsigned long avail = call->rtt_avail;
  	int rtt_slot = 9;
  
  	if (!(avail & RXRPC_CALL_RTT_AVAIL_MASK))
  		goto no_slot;
  
  	rtt_slot = __ffs(avail & RXRPC_CALL_RTT_AVAIL_MASK);
  	if (!test_and_clear_bit(rtt_slot, &call->rtt_avail))
  		goto no_slot;
  
  	call->rtt_serial[rtt_slot] = serial;
  	call->rtt_sent_at[rtt_slot] = ktime_get_real();
  	smp_wmb(); /* Write data before avail bit */
  	set_bit(rtt_slot + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);
  
  	trace_rxrpc_rtt_tx(call, why, rtt_slot, serial);
  	return rtt_slot;
  
  no_slot:
  	trace_rxrpc_rtt_tx(call, rxrpc_rtt_tx_no_slot, rtt_slot, serial);
  	return -1;
  }
  
  /*
   * Cancel an RTT probe.
   */
  static void rxrpc_cancel_rtt_probe(struct rxrpc_call *call,
  				   rxrpc_serial_t serial, int rtt_slot)
  {
  	if (rtt_slot != -1) {
  		clear_bit(rtt_slot + RXRPC_CALL_RTT_PEND_SHIFT, &call->rtt_avail);
  		smp_wmb(); /* Clear pending bit before setting slot */
  		set_bit(rtt_slot, &call->rtt_avail);
  		trace_rxrpc_rtt_tx(call, rxrpc_rtt_tx_cancel, rtt_slot, serial);
  	}
  }
  
  /*
26cb02aa6   David Howells   rxrpc: Fix warnin...
161
   * Send an ACK call packet.
8d94aa381   David Howells   rxrpc: Calls shou...
162
   */
bd1fdf8cf   David Howells   rxrpc: Add a time...
163
164
  int rxrpc_send_ack_packet(struct rxrpc_call *call, bool ping,
  			  rxrpc_serial_t *_serial)
8d94aa381   David Howells   rxrpc: Calls shou...
165
  {
5273a191d   David Howells   rxrpc: Fix NULL p...
166
  	struct rxrpc_connection *conn;
26cb02aa6   David Howells   rxrpc: Fix warnin...
167
  	struct rxrpc_ack_buffer *pkt;
8d94aa381   David Howells   rxrpc: Calls shou...
168
169
170
  	struct msghdr msg;
  	struct kvec iov[2];
  	rxrpc_serial_t serial;
805b21b92   David Howells   rxrpc: Send an AC...
171
  	rxrpc_seq_t hard_ack, top;
8d94aa381   David Howells   rxrpc: Calls shou...
172
  	size_t len, n;
4700c4d80   David Howells   rxrpc: Fix loss o...
173
  	int ret, rtt_slot = -1;
a5af7e1fc   David Howells   rxrpc: Fix loss o...
174
  	u8 reason;
8d94aa381   David Howells   rxrpc: Calls shou...
175

5273a191d   David Howells   rxrpc: Fix NULL p...
176
  	if (test_bit(RXRPC_CALL_DISCONNECTED, &call->flags))
8d94aa381   David Howells   rxrpc: Calls shou...
177
178
179
  		return -ECONNRESET;
  
  	pkt = kzalloc(sizeof(*pkt), GFP_KERNEL);
5273a191d   David Howells   rxrpc: Fix NULL p...
180
  	if (!pkt)
8d94aa381   David Howells   rxrpc: Calls shou...
181
  		return -ENOMEM;
5273a191d   David Howells   rxrpc: Fix NULL p...
182
183
  
  	conn = call->conn;
8d94aa381   David Howells   rxrpc: Calls shou...
184

8d94aa381   David Howells   rxrpc: Calls shou...
185
186
187
188
189
190
191
192
193
194
  	msg.msg_name	= &call->peer->srx.transport;
  	msg.msg_namelen	= call->peer->srx.transport_len;
  	msg.msg_control	= NULL;
  	msg.msg_controllen = 0;
  	msg.msg_flags	= 0;
  
  	pkt->whdr.epoch		= htonl(conn->proto.epoch);
  	pkt->whdr.cid		= htonl(call->cid);
  	pkt->whdr.callNumber	= htonl(call->call_id);
  	pkt->whdr.seq		= 0;
26cb02aa6   David Howells   rxrpc: Fix warnin...
195
196
  	pkt->whdr.type		= RXRPC_PACKET_TYPE_ACK;
  	pkt->whdr.flags		= RXRPC_SLOW_START_OK | conn->out_clientflag;
8d94aa381   David Howells   rxrpc: Calls shou...
197
198
199
200
  	pkt->whdr.userStatus	= 0;
  	pkt->whdr.securityIndex	= call->security_ix;
  	pkt->whdr._rsvd		= 0;
  	pkt->whdr.serviceId	= htons(call->service_id);
26cb02aa6   David Howells   rxrpc: Fix warnin...
201
  	spin_lock_bh(&call->lock);
a5af7e1fc   David Howells   rxrpc: Fix loss o...
202
203
204
205
206
207
208
209
210
211
  	if (ping) {
  		reason = RXRPC_ACK_PING;
  	} else {
  		reason = call->ackr_reason;
  		if (!call->ackr_reason) {
  			spin_unlock_bh(&call->lock);
  			ret = 0;
  			goto out;
  		}
  		call->ackr_reason = 0;
8d94aa381   David Howells   rxrpc: Calls shou...
212
  	}
1457cc4cf   David Howells   rxrpc: Fix a null...
213
  	n = rxrpc_fill_out_ack(conn, call, pkt, &hard_ack, &top, reason);
26cb02aa6   David Howells   rxrpc: Fix warnin...
214
215
216
217
218
219
220
221
  
  	spin_unlock_bh(&call->lock);
  
  	iov[0].iov_base	= pkt;
  	iov[0].iov_len	= sizeof(pkt->whdr) + sizeof(pkt->ack) + n;
  	iov[1].iov_base = &pkt->ackinfo;
  	iov[1].iov_len	= sizeof(pkt->ackinfo);
  	len = iov[0].iov_len + iov[1].iov_len;
8d94aa381   David Howells   rxrpc: Calls shou...
222

b86e218e0   David Howells   rxrpc: Don't call...
223
224
  	serial = atomic_inc_return(&conn->serial);
  	pkt->whdr.serial = htonl(serial);
4764c0da6   David Howells   rxrpc: Trace pack...
225
  	trace_rxrpc_tx_ack(call->debug_id, serial,
26cb02aa6   David Howells   rxrpc: Fix warnin...
226
227
228
  			   ntohl(pkt->ack.firstPacket),
  			   ntohl(pkt->ack.serial),
  			   pkt->ack.reason, pkt->ack.nAcks);
bd1fdf8cf   David Howells   rxrpc: Add a time...
229
230
  	if (_serial)
  		*_serial = serial;
b86e218e0   David Howells   rxrpc: Don't call...
231

4700c4d80   David Howells   rxrpc: Fix loss o...
232
233
  	if (ping)
  		rtt_slot = rxrpc_begin_rtt_probe(call, serial, rxrpc_rtt_tx_ping);
26cb02aa6   David Howells   rxrpc: Fix warnin...
234
235
  
  	ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 2, len);
330bdcfad   David Howells   rxrpc: Fix the ke...
236
  	conn->params.peer->last_tx_at = ktime_get_seconds();
6b47fe1d1   David Howells   rxrpc: Trace UDP ...
237
238
  	if (ret < 0)
  		trace_rxrpc_tx_fail(call->debug_id, serial, ret,
4764c0da6   David Howells   rxrpc: Trace pack...
239
240
241
242
  				    rxrpc_tx_point_call_ack);
  	else
  		trace_rxrpc_tx_packet(call->debug_id, &pkt->whdr,
  				      rxrpc_tx_point_call_ack);
c7e86acfc   David Howells   rxrpc: Fix lockup...
243
  	rxrpc_tx_backoff(call, ret);
8d94aa381   David Howells   rxrpc: Calls shou...
244

26cb02aa6   David Howells   rxrpc: Fix warnin...
245
  	if (call->state < RXRPC_CALL_COMPLETE) {
805b21b92   David Howells   rxrpc: Send an AC...
246
  		if (ret < 0) {
4700c4d80   David Howells   rxrpc: Fix loss o...
247
  			rxrpc_cancel_rtt_probe(call, serial, rtt_slot);
248f219cb   David Howells   rxrpc: Rewrite th...
248
  			rxrpc_propose_ACK(call, pkt->ack.reason,
248f219cb   David Howells   rxrpc: Rewrite th...
249
  					  ntohl(pkt->ack.serial),
c7e86acfc   David Howells   rxrpc: Fix lockup...
250
  					  false, true,
9c7ad4344   David Howells   rxrpc: Add tracep...
251
  					  rxrpc_propose_ack_retry_tx);
805b21b92   David Howells   rxrpc: Send an AC...
252
253
254
255
256
257
258
  		} else {
  			spin_lock_bh(&call->lock);
  			if (after(hard_ack, call->ackr_consumed))
  				call->ackr_consumed = hard_ack;
  			if (after(top, call->ackr_seen))
  				call->ackr_seen = top;
  			spin_unlock_bh(&call->lock);
248f219cb   David Howells   rxrpc: Rewrite th...
259
  		}
415f44e43   David Howells   rxrpc: Add keepal...
260
261
  
  		rxrpc_set_keepalive(call);
248f219cb   David Howells   rxrpc: Rewrite th...
262
  	}
8d94aa381   David Howells   rxrpc: Calls shou...
263
  out:
8d94aa381   David Howells   rxrpc: Calls shou...
264
265
266
  	kfree(pkt);
  	return ret;
  }
5873c0834   David Howells   af_rxrpc: Add sys...
267
  /*
26cb02aa6   David Howells   rxrpc: Fix warnin...
268
269
270
271
   * Send an ABORT call packet.
   */
  int rxrpc_send_abort_packet(struct rxrpc_call *call)
  {
5273a191d   David Howells   rxrpc: Fix NULL p...
272
  	struct rxrpc_connection *conn;
26cb02aa6   David Howells   rxrpc: Fix warnin...
273
274
275
276
277
  	struct rxrpc_abort_buffer pkt;
  	struct msghdr msg;
  	struct kvec iov[1];
  	rxrpc_serial_t serial;
  	int ret;
dcbefc30f   David Howells   rxrpc: Fix call e...
278
279
280
281
282
283
284
285
286
  	/* Don't bother sending aborts for a client call once the server has
  	 * hard-ACK'd all of its request data.  After that point, we're not
  	 * going to stop the operation proceeding, and whilst we might limit
  	 * the reply, it's not worth it if we can send a new call on the same
  	 * channel instead, thereby closing off this call.
  	 */
  	if (rxrpc_is_client_call(call) &&
  	    test_bit(RXRPC_CALL_TX_LAST, &call->flags))
  		return 0;
5273a191d   David Howells   rxrpc: Fix NULL p...
287
  	if (test_bit(RXRPC_CALL_DISCONNECTED, &call->flags))
26cb02aa6   David Howells   rxrpc: Fix warnin...
288
  		return -ECONNRESET;
5273a191d   David Howells   rxrpc: Fix NULL p...
289
  	conn = call->conn;
26cb02aa6   David Howells   rxrpc: Fix warnin...
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
  	msg.msg_name	= &call->peer->srx.transport;
  	msg.msg_namelen	= call->peer->srx.transport_len;
  	msg.msg_control	= NULL;
  	msg.msg_controllen = 0;
  	msg.msg_flags	= 0;
  
  	pkt.whdr.epoch		= htonl(conn->proto.epoch);
  	pkt.whdr.cid		= htonl(call->cid);
  	pkt.whdr.callNumber	= htonl(call->call_id);
  	pkt.whdr.seq		= 0;
  	pkt.whdr.type		= RXRPC_PACKET_TYPE_ABORT;
  	pkt.whdr.flags		= conn->out_clientflag;
  	pkt.whdr.userStatus	= 0;
  	pkt.whdr.securityIndex	= call->security_ix;
  	pkt.whdr._rsvd		= 0;
  	pkt.whdr.serviceId	= htons(call->service_id);
  	pkt.abort_code		= htonl(call->abort_code);
  
  	iov[0].iov_base	= &pkt;
  	iov[0].iov_len	= sizeof(pkt);
  
  	serial = atomic_inc_return(&conn->serial);
  	pkt.whdr.serial = htonl(serial);
  
  	ret = kernel_sendmsg(conn->params.local->socket,
  			     &msg, iov, 1, sizeof(pkt));
330bdcfad   David Howells   rxrpc: Fix the ke...
316
  	conn->params.peer->last_tx_at = ktime_get_seconds();
6b47fe1d1   David Howells   rxrpc: Trace UDP ...
317
318
  	if (ret < 0)
  		trace_rxrpc_tx_fail(call->debug_id, serial, ret,
4764c0da6   David Howells   rxrpc: Trace pack...
319
320
321
322
  				    rxrpc_tx_point_call_abort);
  	else
  		trace_rxrpc_tx_packet(call->debug_id, &pkt.whdr,
  				      rxrpc_tx_point_call_abort);
c7e86acfc   David Howells   rxrpc: Fix lockup...
323
  	rxrpc_tx_backoff(call, ret);
26cb02aa6   David Howells   rxrpc: Fix warnin...
324
325
326
327
  	return ret;
  }
  
  /*
17926a793   David Howells   [AF_RXRPC]: Provi...
328
329
   * send a packet through the transport endpoint
   */
a1767077b   David Howells   rxrpc: Make Tx lo...
330
331
  int rxrpc_send_data_packet(struct rxrpc_call *call, struct sk_buff *skb,
  			   bool retrans)
17926a793   David Howells   [AF_RXRPC]: Provi...
332
  {
5a924b895   David Howells   rxrpc: Don't stor...
333
334
335
  	struct rxrpc_connection *conn = call->conn;
  	struct rxrpc_wire_header whdr;
  	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
17926a793   David Howells   [AF_RXRPC]: Provi...
336
  	struct msghdr msg;
5a924b895   David Howells   rxrpc: Don't stor...
337
338
339
  	struct kvec iov[2];
  	rxrpc_serial_t serial;
  	size_t len;
4700c4d80   David Howells   rxrpc: Fix loss o...
340
  	int ret, rtt_slot = -1;
17926a793   David Howells   [AF_RXRPC]: Provi...
341
342
  
  	_enter(",{%d}", skb->len);
245500d85   David Howells   rxrpc: Rewrite th...
343
344
345
346
347
  	if (hlist_unhashed(&call->error_link)) {
  		spin_lock_bh(&call->peer->lock);
  		hlist_add_head_rcu(&call->error_link, &call->peer->error_targets);
  		spin_unlock_bh(&call->peer->lock);
  	}
5a924b895   David Howells   rxrpc: Don't stor...
348
349
  	/* Each transmission of a Tx packet needs a new serial number */
  	serial = atomic_inc_return(&conn->serial);
17926a793   David Howells   [AF_RXRPC]: Provi...
350

5a924b895   David Howells   rxrpc: Don't stor...
351
352
353
354
355
356
357
358
359
360
361
  	whdr.epoch	= htonl(conn->proto.epoch);
  	whdr.cid	= htonl(call->cid);
  	whdr.callNumber	= htonl(call->call_id);
  	whdr.seq	= htonl(sp->hdr.seq);
  	whdr.serial	= htonl(serial);
  	whdr.type	= RXRPC_PACKET_TYPE_DATA;
  	whdr.flags	= sp->hdr.flags;
  	whdr.userStatus	= 0;
  	whdr.securityIndex = call->security_ix;
  	whdr._rsvd	= htons(sp->hdr._rsvd);
  	whdr.serviceId	= htons(call->service_id);
4e255721d   David Howells   rxrpc: Add servic...
362
363
364
  	if (test_bit(RXRPC_CONN_PROBING_FOR_UPGRADE, &conn->flags) &&
  	    sp->hdr.seq == 1)
  		whdr.userStatus	= RXRPC_USERSTATUS_SERVICE_UPGRADE;
5a924b895   David Howells   rxrpc: Don't stor...
365
366
367
368
369
370
371
372
  	iov[0].iov_base = &whdr;
  	iov[0].iov_len = sizeof(whdr);
  	iov[1].iov_base = skb->head;
  	iov[1].iov_len = skb->len;
  	len = iov[0].iov_len + iov[1].iov_len;
  
  	msg.msg_name = &call->peer->srx.transport;
  	msg.msg_namelen = call->peer->srx.transport_len;
17926a793   David Howells   [AF_RXRPC]: Provi...
373
374
375
  	msg.msg_control = NULL;
  	msg.msg_controllen = 0;
  	msg.msg_flags = 0;
57494343c   David Howells   rxrpc: Implement ...
376
377
  	/* If our RTT cache needs working on, request an ACK.  Also request
  	 * ACKs if a DATA packet appears to have been lost.
b604dd988   David Howells   rxrpc: Fix RTT ga...
378
379
380
381
  	 *
  	 * However, we mustn't request an ACK on the last reply packet of a
  	 * service call, lest OpenAFS incorrectly send us an ACK with some
  	 * soft-ACKs in it and then never follow up with a proper hard ACK.
57494343c   David Howells   rxrpc: Implement ...
382
  	 */
b604dd988   David Howells   rxrpc: Fix RTT ga...
383
384
385
  	if ((!(sp->hdr.flags & RXRPC_LAST_PACKET) ||
  	     rxrpc_to_server(sp)
  	     ) &&
bd1fdf8cf   David Howells   rxrpc: Add a time...
386
387
  	    (test_and_clear_bit(RXRPC_CALL_EV_ACK_LOST, &call->events) ||
  	     retrans ||
bf7d620ab   David Howells   rxrpc: Don't requ...
388
  	     call->cong_mode == RXRPC_CALL_SLOW_START ||
c410bf019   David Howells   rxrpc: Fix the ex...
389
  	     (call->peer->rtt_count < 3 && sp->hdr.seq & 1) ||
bf7d620ab   David Howells   rxrpc: Don't requ...
390
391
  	     ktime_before(ktime_add_ms(call->peer->rtt_last_req, 1000),
  			  ktime_get_real())))
0d4b103c0   David Howells   rxrpc: Reduce the...
392
  		whdr.flags |= RXRPC_REQUEST_ACK;
8a681c360   David Howells   rxrpc: Add config...
393
394
395
  	if (IS_ENABLED(CONFIG_AF_RXRPC_INJECT_LOSS)) {
  		static int lose;
  		if ((lose++ & 7) == 7) {
a1767077b   David Howells   rxrpc: Make Tx lo...
396
  			ret = 0;
526949e87   Arnd Bergmann   rxrpc: avoid clan...
397
398
399
  			trace_rxrpc_tx_data(call, sp->hdr.seq, serial,
  					    whdr.flags, retrans, true);
  			goto done;
8a681c360   David Howells   rxrpc: Add config...
400
401
  		}
  	}
526949e87   Arnd Bergmann   rxrpc: avoid clan...
402
403
  	trace_rxrpc_tx_data(call, sp->hdr.seq, serial, whdr.flags, retrans,
  			    false);
5a924b895   David Howells   rxrpc: Don't stor...
404

17926a793   David Howells   [AF_RXRPC]: Provi...
405
406
  	/* send the packet with the don't fragment bit set if we currently
  	 * think it's small enough */
5a924b895   David Howells   rxrpc: Don't stor...
407
408
409
410
  	if (iov[1].iov_len >= call->peer->maxdata)
  		goto send_fragmentable;
  
  	down_read(&conn->params.local->defrag_sem);
b604dd988   David Howells   rxrpc: Fix RTT ga...
411
412
413
414
  
  	sp->hdr.serial = serial;
  	smp_wmb(); /* Set serial before timestamp */
  	skb->tstamp = ktime_get_real();
4700c4d80   David Howells   rxrpc: Fix loss o...
415
416
  	if (whdr.flags & RXRPC_REQUEST_ACK)
  		rtt_slot = rxrpc_begin_rtt_probe(call, serial, rxrpc_rtt_tx_data);
b604dd988   David Howells   rxrpc: Fix RTT ga...
417

5a924b895   David Howells   rxrpc: Don't stor...
418
419
420
421
422
423
424
  	/* send the packet by UDP
  	 * - returns -EMSGSIZE if UDP would have to fragment the packet
  	 *   to go out of the interface
  	 *   - in which case, we'll have processed the ICMP error
  	 *     message and update the peer record
  	 */
  	ret = kernel_sendmsg(conn->params.local->socket, &msg, iov, 2, len);
330bdcfad   David Howells   rxrpc: Fix the ke...
425
  	conn->params.peer->last_tx_at = ktime_get_seconds();
5a924b895   David Howells   rxrpc: Don't stor...
426
427
  
  	up_read(&conn->params.local->defrag_sem);
4700c4d80   David Howells   rxrpc: Fix loss o...
428
429
  	if (ret < 0) {
  		rxrpc_cancel_rtt_probe(call, serial, rtt_slot);
6b47fe1d1   David Howells   rxrpc: Trace UDP ...
430
  		trace_rxrpc_tx_fail(call->debug_id, serial, ret,
4764c0da6   David Howells   rxrpc: Trace pack...
431
  				    rxrpc_tx_point_call_data_nofrag);
4700c4d80   David Howells   rxrpc: Fix loss o...
432
  	} else {
4764c0da6   David Howells   rxrpc: Trace pack...
433
434
  		trace_rxrpc_tx_packet(call->debug_id, &whdr,
  				      rxrpc_tx_point_call_data_nofrag);
4700c4d80   David Howells   rxrpc: Fix loss o...
435
  	}
c7e86acfc   David Howells   rxrpc: Fix lockup...
436
  	rxrpc_tx_backoff(call, ret);
5a924b895   David Howells   rxrpc: Don't stor...
437
438
439
440
  	if (ret == -EMSGSIZE)
  		goto send_fragmentable;
  
  done:
50235c4b5   David Howells   rxrpc: Obtain RTT...
441
  	if (ret >= 0) {
0d4b103c0   David Howells   rxrpc: Reduce the...
442
  		if (whdr.flags & RXRPC_REQUEST_ACK) {
b604dd988   David Howells   rxrpc: Fix RTT ga...
443
  			call->peer->rtt_last_req = skb->tstamp;
c410bf019   David Howells   rxrpc: Fix the ex...
444
  			if (call->peer->rtt_count > 1) {
bd1fdf8cf   David Howells   rxrpc: Add a time...
445
  				unsigned long nowj = jiffies, ack_lost_at;
c410bf019   David Howells   rxrpc: Fix the ex...
446
  				ack_lost_at = rxrpc_get_rto_backoff(call->peer, retrans);
bd1fdf8cf   David Howells   rxrpc: Add a time...
447
448
449
450
451
  				ack_lost_at += nowj;
  				WRITE_ONCE(call->ack_lost_at, ack_lost_at);
  				rxrpc_reduce_call_timer(call, ack_lost_at, nowj,
  							rxrpc_timer_set_for_lost_ack);
  			}
0d4b103c0   David Howells   rxrpc: Reduce the...
452
  		}
c54e43d75   David Howells   rxrpc: Fix missin...
453
454
455
456
457
458
459
460
461
462
463
  
  		if (sp->hdr.seq == 1 &&
  		    !test_and_set_bit(RXRPC_CALL_BEGAN_RX_TIMER,
  				      &call->flags)) {
  			unsigned long nowj = jiffies, expect_rx_by;
  
  			expect_rx_by = nowj + call->next_rx_timo;
  			WRITE_ONCE(call->expect_rx_by, expect_rx_by);
  			rxrpc_reduce_call_timer(call, expect_rx_by, nowj,
  						rxrpc_timer_set_for_normal);
  		}
415f44e43   David Howells   rxrpc: Add keepal...
464

c7e86acfc   David Howells   rxrpc: Fix lockup...
465
466
467
468
469
470
471
472
473
474
475
  		rxrpc_set_keepalive(call);
  	} else {
  		/* Cancel the call if the initial transmission fails,
  		 * particularly if that's due to network routing issues that
  		 * aren't going away anytime soon.  The layer above can arrange
  		 * the retransmission.
  		 */
  		if (!test_and_set_bit(RXRPC_CALL_BEGAN_RX_TIMER, &call->flags))
  			rxrpc_set_call_completion(call, RXRPC_CALL_LOCAL_ERROR,
  						  RX_USER_ABORT, ret);
  	}
415f44e43   David Howells   rxrpc: Add keepal...
476

5a924b895   David Howells   rxrpc: Don't stor...
477
478
  	_leave(" = %d [%u]", ret, call->peer->maxdata);
  	return ret;
17926a793   David Howells   [AF_RXRPC]: Provi...
479
480
481
482
  
  send_fragmentable:
  	/* attempt to send this message with fragmentation enabled */
  	_debug("send fragment");
985a5c824   David Howells   rxrpc: Make rxrpc...
483
  	down_write(&conn->params.local->defrag_sem);
b604dd988   David Howells   rxrpc: Fix RTT ga...
484
485
486
  	sp->hdr.serial = serial;
  	smp_wmb(); /* Set serial before timestamp */
  	skb->tstamp = ktime_get_real();
4700c4d80   David Howells   rxrpc: Fix loss o...
487
488
  	if (whdr.flags & RXRPC_REQUEST_ACK)
  		rtt_slot = rxrpc_begin_rtt_probe(call, serial, rxrpc_rtt_tx_data);
b604dd988   David Howells   rxrpc: Fix RTT ga...
489

985a5c824   David Howells   rxrpc: Make rxrpc...
490
  	switch (conn->params.local->srx.transport.family) {
0e631eee1   David Howells   rxrpc: Fix DATA T...
491
  	case AF_INET6:
985a5c824   David Howells   rxrpc: Make rxrpc...
492
  	case AF_INET:
2de569bda   Christoph Hellwig   ipv4: add ip_sock...
493
494
  		ip_sock_set_mtu_discover(conn->params.local->socket->sk,
  				IP_PMTUDISC_DONT);
0e631eee1   David Howells   rxrpc: Fix DATA T...
495
496
497
  		ret = kernel_sendmsg(conn->params.local->socket, &msg,
  				     iov, 2, len);
  		conn->params.peer->last_tx_at = ktime_get_seconds();
2de569bda   Christoph Hellwig   ipv4: add ip_sock...
498
499
  		ip_sock_set_mtu_discover(conn->params.local->socket->sk,
  				IP_PMTUDISC_DO);
75b54cb57   David Howells   rxrpc: Add IPv6 s...
500
  		break;
3427beb63   David Howells   rxrpc: Fix uninit...
501
502
503
  
  	default:
  		BUG();
17926a793   David Howells   [AF_RXRPC]: Provi...
504
  	}
4700c4d80   David Howells   rxrpc: Fix loss o...
505
506
  	if (ret < 0) {
  		rxrpc_cancel_rtt_probe(call, serial, rtt_slot);
6b47fe1d1   David Howells   rxrpc: Trace UDP ...
507
  		trace_rxrpc_tx_fail(call->debug_id, serial, ret,
4764c0da6   David Howells   rxrpc: Trace pack...
508
  				    rxrpc_tx_point_call_data_frag);
4700c4d80   David Howells   rxrpc: Fix loss o...
509
  	} else {
4764c0da6   David Howells   rxrpc: Trace pack...
510
511
  		trace_rxrpc_tx_packet(call->debug_id, &whdr,
  				      rxrpc_tx_point_call_data_frag);
4700c4d80   David Howells   rxrpc: Fix loss o...
512
  	}
c7e86acfc   David Howells   rxrpc: Fix lockup...
513
  	rxrpc_tx_backoff(call, ret);
6b47fe1d1   David Howells   rxrpc: Trace UDP ...
514

985a5c824   David Howells   rxrpc: Make rxrpc...
515
  	up_write(&conn->params.local->defrag_sem);
5a924b895   David Howells   rxrpc: Don't stor...
516
  	goto done;
17926a793   David Howells   [AF_RXRPC]: Provi...
517
  }
248f219cb   David Howells   rxrpc: Rewrite th...
518
519
520
521
522
523
  
  /*
   * reject packets through the local endpoint
   */
  void rxrpc_reject_packets(struct rxrpc_local *local)
  {
1c2bc7b94   David Howells   rxrpc: Use rxrpc_...
524
  	struct sockaddr_rxrpc srx;
248f219cb   David Howells   rxrpc: Rewrite th...
525
526
527
528
529
530
531
  	struct rxrpc_skb_priv *sp;
  	struct rxrpc_wire_header whdr;
  	struct sk_buff *skb;
  	struct msghdr msg;
  	struct kvec iov[2];
  	size_t size;
  	__be32 code;
ece64fec1   David Howells   rxrpc: Emit BUSY ...
532
  	int ret, ioc;
248f219cb   David Howells   rxrpc: Rewrite th...
533
534
535
536
537
538
539
  
  	_enter("%d", local->debug_id);
  
  	iov[0].iov_base = &whdr;
  	iov[0].iov_len = sizeof(whdr);
  	iov[1].iov_base = &code;
  	iov[1].iov_len = sizeof(code);
248f219cb   David Howells   rxrpc: Rewrite th...
540

1c2bc7b94   David Howells   rxrpc: Use rxrpc_...
541
  	msg.msg_name = &srx.transport;
248f219cb   David Howells   rxrpc: Rewrite th...
542
543
544
  	msg.msg_control = NULL;
  	msg.msg_controllen = 0;
  	msg.msg_flags = 0;
248f219cb   David Howells   rxrpc: Rewrite th...
545
  	memset(&whdr, 0, sizeof(whdr));
248f219cb   David Howells   rxrpc: Rewrite th...
546
547
  
  	while ((skb = skb_dequeue(&local->reject_queue))) {
987db9f7c   David Howells   rxrpc: Use the tx...
548
  		rxrpc_see_skb(skb, rxrpc_skb_seen);
248f219cb   David Howells   rxrpc: Rewrite th...
549
  		sp = rxrpc_skb(skb);
1c2bc7b94   David Howells   rxrpc: Use rxrpc_...
550

ece64fec1   David Howells   rxrpc: Emit BUSY ...
551
552
553
554
555
556
557
558
559
560
561
562
563
  		switch (skb->mark) {
  		case RXRPC_SKB_MARK_REJECT_BUSY:
  			whdr.type = RXRPC_PACKET_TYPE_BUSY;
  			size = sizeof(whdr);
  			ioc = 1;
  			break;
  		case RXRPC_SKB_MARK_REJECT_ABORT:
  			whdr.type = RXRPC_PACKET_TYPE_ABORT;
  			code = htonl(skb->priority);
  			size = sizeof(whdr) + sizeof(code);
  			ioc = 2;
  			break;
  		default:
987db9f7c   David Howells   rxrpc: Use the tx...
564
  			rxrpc_free_skb(skb, rxrpc_skb_freed);
ece64fec1   David Howells   rxrpc: Emit BUSY ...
565
566
  			continue;
  		}
5a790b737   David Howells   rxrpc: Drop the l...
567
  		if (rxrpc_extract_addr_from_skb(&srx, skb) == 0) {
1c2bc7b94   David Howells   rxrpc: Use rxrpc_...
568
  			msg.msg_namelen = srx.transport_len;
248f219cb   David Howells   rxrpc: Rewrite th...
569
570
571
572
573
574
575
  			whdr.epoch	= htonl(sp->hdr.epoch);
  			whdr.cid	= htonl(sp->hdr.cid);
  			whdr.callNumber	= htonl(sp->hdr.callNumber);
  			whdr.serviceId	= htons(sp->hdr.serviceId);
  			whdr.flags	= sp->hdr.flags;
  			whdr.flags	^= RXRPC_CLIENT_INITIATED;
  			whdr.flags	&= RXRPC_CLIENT_INITIATED;
d6672a5a9   YueHaibing   rxrpc: use correc...
576
577
  			ret = kernel_sendmsg(local->socket, &msg,
  					     iov, ioc, size);
6b47fe1d1   David Howells   rxrpc: Trace UDP ...
578
579
  			if (ret < 0)
  				trace_rxrpc_tx_fail(local->debug_id, 0, ret,
4764c0da6   David Howells   rxrpc: Trace pack...
580
581
582
583
  						    rxrpc_tx_point_reject);
  			else
  				trace_rxrpc_tx_packet(local->debug_id, &whdr,
  						      rxrpc_tx_point_reject);
248f219cb   David Howells   rxrpc: Rewrite th...
584
  		}
987db9f7c   David Howells   rxrpc: Use the tx...
585
  		rxrpc_free_skb(skb, rxrpc_skb_freed);
248f219cb   David Howells   rxrpc: Rewrite th...
586
587
588
589
  	}
  
  	_leave("");
  }
ace45bec6   David Howells   rxrpc: Fix firewa...
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
  
  /*
   * Send a VERSION reply to a peer as a keepalive.
   */
  void rxrpc_send_keepalive(struct rxrpc_peer *peer)
  {
  	struct rxrpc_wire_header whdr;
  	struct msghdr msg;
  	struct kvec iov[2];
  	size_t len;
  	int ret;
  
  	_enter("");
  
  	msg.msg_name	= &peer->srx.transport;
  	msg.msg_namelen	= peer->srx.transport_len;
  	msg.msg_control	= NULL;
  	msg.msg_controllen = 0;
  	msg.msg_flags	= 0;
  
  	whdr.epoch	= htonl(peer->local->rxnet->epoch);
  	whdr.cid	= 0;
  	whdr.callNumber	= 0;
  	whdr.seq	= 0;
  	whdr.serial	= 0;
  	whdr.type	= RXRPC_PACKET_TYPE_VERSION; /* Not client-initiated */
  	whdr.flags	= RXRPC_LAST_PACKET;
  	whdr.userStatus	= 0;
  	whdr.securityIndex = 0;
  	whdr._rsvd	= 0;
  	whdr.serviceId	= 0;
  
  	iov[0].iov_base	= &whdr;
  	iov[0].iov_len	= sizeof(whdr);
  	iov[1].iov_base	= (char *)rxrpc_keepalive_string;
  	iov[1].iov_len	= sizeof(rxrpc_keepalive_string);
  
  	len = iov[0].iov_len + iov[1].iov_len;
  
  	_proto("Tx VERSION (keepalive)");
  
  	ret = kernel_sendmsg(peer->local->socket, &msg, iov, 2, len);
  	if (ret < 0)
6b47fe1d1   David Howells   rxrpc: Trace UDP ...
633
  		trace_rxrpc_tx_fail(peer->debug_id, 0, ret,
4764c0da6   David Howells   rxrpc: Trace pack...
634
635
636
637
  				    rxrpc_tx_point_version_keepalive);
  	else
  		trace_rxrpc_tx_packet(peer->debug_id, &whdr,
  				      rxrpc_tx_point_version_keepalive);
ace45bec6   David Howells   rxrpc: Fix firewa...
638

330bdcfad   David Howells   rxrpc: Fix the ke...
639
  	peer->last_tx_at = ktime_get_seconds();
ace45bec6   David Howells   rxrpc: Fix firewa...
640
641
  	_leave("");
  }