Blame view
net/ipv4/udp.c
77.9 KB
2874c5fd2
|
1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
1da177e4c
|
2 3 4 5 6 7 8 |
/* * INET An implementation of the TCP/IP protocol suite for the LINUX * operating system. INET is implemented using the BSD Socket * interface as the means of communication with the user level. * * The User Datagram Protocol (UDP). * |
02c30a84e
|
9 |
* Authors: Ross Biro |
1da177e4c
|
10 11 |
* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> * Arnt Gulbrandsen, <agulbra@nvg.unit.no> |
113aa838e
|
12 |
* Alan Cox, <alan@lxorguk.ukuu.org.uk> |
1da177e4c
|
13 14 15 16 17 18 19 20 21 |
* Hirokazu Takahashi, <taka@valinux.co.jp> * * Fixes: * Alan Cox : verify_area() calls * Alan Cox : stopped close while in use off icmp * messages. Not a fix but a botch that * for udp at least is 'valid'. * Alan Cox : Fixed icmp handling properly * Alan Cox : Correct error for oversized datagrams |
e905a9eda
|
22 23 |
* Alan Cox : Tidied select() semantics. * Alan Cox : udp_err() fixed properly, also now |
1da177e4c
|
24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 |
* select and read wake correctly on errors * Alan Cox : udp_send verify_area moved to avoid mem leak * Alan Cox : UDP can count its memory * Alan Cox : send to an unknown connection causes * an ECONNREFUSED off the icmp, but * does NOT close. * Alan Cox : Switched to new sk_buff handlers. No more backlog! * Alan Cox : Using generic datagram code. Even smaller and the PEEK * bug no longer crashes it. * Fred Van Kempen : Net2e support for sk->broadcast. * Alan Cox : Uses skb_free_datagram * Alan Cox : Added get/set sockopt support. * Alan Cox : Broadcasting without option set returns EACCES. * Alan Cox : No wakeup calls. Instead we now use the callbacks. * Alan Cox : Use ip_tos and ip_ttl * Alan Cox : SNMP Mibs * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support. * Matt Dillon : UDP length checks. * Alan Cox : Smarter af_inet used properly. * Alan Cox : Use new kernel side addressing. * Alan Cox : Incorrect return on truncated datagram receive. * Arnt Gulbrandsen : New udp_send and stuff * Alan Cox : Cache last socket * Alan Cox : Route cache * Jon Peatfield : Minor efficiency fix to sendto(). * Mike Shaver : RFC1122 checks. * Alan Cox : Nonblocking error fix. * Willy Konynenberg : Transparent proxying support. * Mike McLagan : Routing by source * David S. Miller : New socket lookup architecture. * Last socket cache retained as it * does have a high hit rate. * Olaf Kirch : Don't linearise iovec on sendmsg. * Andi Kleen : Some cleanups, cache destination entry |
e905a9eda
|
58 |
* for connect. |
1da177e4c
|
59 60 61 62 63 64 65 66 67 68 69 70 71 |
* Vitaly E. Lavrov : Transparent proxy revived after year coma. * Melvin Smith : Check msg_name not msg_namelen in sendto(), * return ENOTCONN for unconnected sockets (POSIX) * Janos Farkas : don't deliver multi/broadcasts to a different * bound-to-device socket * Hirokazu Takahashi : HW checksumming for outgoing UDP * datagrams. * Hirokazu Takahashi : sendfile() on UDP works now. * Arnaldo C. Melo : convert /proc/net/udp to seq_file * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind * a single port at the same time. * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support |
342f0234c
|
72 |
* James Chapman : Add L2TP encapsulation type. |
1da177e4c
|
73 |
*/ |
e905a9eda
|
74 |
|
afd465030
|
75 |
#define pr_fmt(fmt) "UDP: " fmt |
7c0f6ba68
|
76 |
#include <linux/uaccess.h> |
1da177e4c
|
77 |
#include <asm/ioctls.h> |
57c8a661d
|
78 |
#include <linux/memblock.h> |
8203efb3c
|
79 80 |
#include <linux/highmem.h> #include <linux/swap.h> |
1da177e4c
|
81 82 83 84 85 |
#include <linux/types.h> #include <linux/fcntl.h> #include <linux/module.h> #include <linux/socket.h> #include <linux/sockios.h> |
14c850212
|
86 |
#include <linux/igmp.h> |
6e5403093
|
87 |
#include <linux/inetdevice.h> |
1da177e4c
|
88 89 90 91 |
#include <linux/in.h> #include <linux/errno.h> #include <linux/timer.h> #include <linux/mm.h> |
1da177e4c
|
92 |
#include <linux/inet.h> |
1da177e4c
|
93 |
#include <linux/netdevice.h> |
5a0e3ad6a
|
94 |
#include <linux/slab.h> |
c752f0739
|
95 |
#include <net/tcp_states.h> |
1da177e4c
|
96 97 98 |
#include <linux/skbuff.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> |
457c4cbc5
|
99 |
#include <net/net_namespace.h> |
1da177e4c
|
100 |
#include <net/icmp.h> |
421b3885b
|
101 |
#include <net/inet_hashtables.h> |
e7cc08245
|
102 |
#include <net/ip_tunnels.h> |
1da177e4c
|
103 |
#include <net/route.h> |
1da177e4c
|
104 105 |
#include <net/checksum.h> #include <net/xfrm.h> |
296f7ea75
|
106 |
#include <trace/events/udp.h> |
447167bf5
|
107 |
#include <linux/static_key.h> |
22911fc58
|
108 |
#include <trace/events/skb.h> |
076bb0c82
|
109 |
#include <net/busy_poll.h> |
ba4e58eca
|
110 |
#include "udp_impl.h" |
e32ea7e74
|
111 |
#include <net/sock_reuseport.h> |
217375a0c
|
112 |
#include <net/addrconf.h> |
60fb9567b
|
113 |
#include <net/udp_tunnel.h> |
1da177e4c
|
114 |
|
f86dcc5aa
|
115 |
struct udp_table udp_table __read_mostly; |
645ca708f
|
116 |
EXPORT_SYMBOL(udp_table); |
1da177e4c
|
117 |
|
8d987e5c7
|
118 |
long sysctl_udp_mem[3] __read_mostly; |
95766fff6
|
119 |
EXPORT_SYMBOL(sysctl_udp_mem); |
c482c5685
|
120 |
|
8d987e5c7
|
121 |
atomic_long_t udp_memory_allocated; |
95766fff6
|
122 |
EXPORT_SYMBOL(udp_memory_allocated); |
f86dcc5aa
|
123 124 |
#define MAX_UDP_PORTS 65536 #define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN) |
98322f22e
|
125 |
|
f24d43c07
|
126 |
static int udp_lib_lport_inuse(struct net *net, __u16 num, |
645ca708f
|
127 |
const struct udp_hslot *hslot, |
98322f22e
|
128 |
unsigned long *bitmap, |
fe38d2a1c
|
129 |
struct sock *sk, unsigned int log) |
1da177e4c
|
130 |
{ |
f24d43c07
|
131 |
struct sock *sk2; |
ba418fa35
|
132 |
kuid_t uid = sock_i_uid(sk); |
25030a7f9
|
133 |
|
ca065d0cf
|
134 |
sk_for_each(sk2, &hslot->head) { |
9d4fb27db
|
135 136 |
if (net_eq(sock_net(sk2), net) && sk2 != sk && |
d4cada4ae
|
137 |
(bitmap || udp_sk(sk2)->udp_port_hash == num) && |
9d4fb27db
|
138 139 140 |
(!sk2->sk_reuse || !sk->sk_reuse) && (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && |
fe38d2a1c
|
141 |
inet_rcv_saddr_equal(sk, sk2, true)) { |
df560056d
|
142 143 144 145 146 147 148 149 150 151 152 |
if (sk2->sk_reuseport && sk->sk_reuseport && !rcu_access_pointer(sk->sk_reuseport_cb) && uid_eq(uid, sock_i_uid(sk2))) { if (!bitmap) return 0; } else { if (!bitmap) return 1; __set_bit(udp_sk(sk2)->udp_port_hash >> log, bitmap); } |
98322f22e
|
153 |
} |
4243cdc2c
|
154 |
} |
25030a7f9
|
155 156 |
return 0; } |
30fff9231
|
157 158 159 160 161 |
/* * Note: we still hold spinlock of primary hash chain, so no other writer * can insert/delete a socket with local_port == num */ static int udp_lib_lport_inuse2(struct net *net, __u16 num, |
4243cdc2c
|
162 |
struct udp_hslot *hslot2, |
fe38d2a1c
|
163 |
struct sock *sk) |
30fff9231
|
164 165 |
{ struct sock *sk2; |
ba418fa35
|
166 |
kuid_t uid = sock_i_uid(sk); |
30fff9231
|
167 168 169 |
int res = 0; spin_lock(&hslot2->lock); |
ca065d0cf
|
170 |
udp_portaddr_for_each_entry(sk2, &hslot2->head) { |
9d4fb27db
|
171 172 173 174 175 176 |
if (net_eq(sock_net(sk2), net) && sk2 != sk && (udp_sk(sk2)->udp_port_hash == num) && (!sk2->sk_reuse || !sk->sk_reuse) && (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && |
fe38d2a1c
|
177 |
inet_rcv_saddr_equal(sk, sk2, true)) { |
df560056d
|
178 179 180 181 182 183 184 |
if (sk2->sk_reuseport && sk->sk_reuseport && !rcu_access_pointer(sk->sk_reuseport_cb) && uid_eq(uid, sock_i_uid(sk2))) { res = 0; } else { res = 1; } |
30fff9231
|
185 186 |
break; } |
4243cdc2c
|
187 |
} |
30fff9231
|
188 189 190 |
spin_unlock(&hslot2->lock); return res; } |
fe38d2a1c
|
191 |
static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot) |
e32ea7e74
|
192 193 |
{ struct net *net = sock_net(sk); |
e32ea7e74
|
194 195 |
kuid_t uid = sock_i_uid(sk); struct sock *sk2; |
ca065d0cf
|
196 |
sk_for_each(sk2, &hslot->head) { |
e32ea7e74
|
197 198 199 200 201 202 203 |
if (net_eq(sock_net(sk2), net) && sk2 != sk && sk2->sk_family == sk->sk_family && ipv6_only_sock(sk2) == ipv6_only_sock(sk) && (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) && (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && |
fe38d2a1c
|
204 |
inet_rcv_saddr_equal(sk, sk2, false)) { |
2dbb9b9e6
|
205 206 |
return reuseport_add_sock(sk, sk2, inet_rcv_saddr_any(sk)); |
e32ea7e74
|
207 208 |
} } |
2dbb9b9e6
|
209 |
return reuseport_alloc(sk, inet_rcv_saddr_any(sk)); |
e32ea7e74
|
210 |
} |
25030a7f9
|
211 |
/** |
6ba5a3c52
|
212 |
* udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6 |
25030a7f9
|
213 214 215 |
* * @sk: socket struct in question * @snum: port number to look up |
25985edce
|
216 |
* @hash2_nulladdr: AF-dependent hash value in secondary hash chains, |
30fff9231
|
217 |
* with NULL address |
25030a7f9
|
218 |
*/ |
6ba5a3c52
|
219 |
int udp_lib_get_port(struct sock *sk, unsigned short snum, |
30fff9231
|
220 |
unsigned int hash2_nulladdr) |
25030a7f9
|
221 |
{ |
512615b6b
|
222 |
struct udp_hslot *hslot, *hslot2; |
645ca708f
|
223 |
struct udp_table *udptable = sk->sk_prot->h.udp_table; |
25030a7f9
|
224 |
int error = 1; |
3b1e0a655
|
225 |
struct net *net = sock_net(sk); |
1da177e4c
|
226 |
|
32c1da708
|
227 |
if (!snum) { |
9088c5609
|
228 |
int low, high, remaining; |
95c961747
|
229 |
unsigned int rand; |
98322f22e
|
230 231 |
unsigned short first, last; DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN); |
32c1da708
|
232 |
|
0bbf87d85
|
233 |
inet_get_local_port_range(net, &low, &high); |
a25de534f
|
234 |
remaining = (high - low) + 1; |
227b60f51
|
235 |
|
63862b5be
|
236 |
rand = prandom_u32(); |
8fc54f689
|
237 |
first = reciprocal_scale(rand, remaining) + low; |
98322f22e
|
238 239 240 |
/* * force rand to be an odd multiple of UDP_HTABLE_SIZE */ |
f86dcc5aa
|
241 |
rand = (rand | 1) * (udptable->mask + 1); |
5781b2356
|
242 243 |
last = first + udptable->mask + 1; do { |
f86dcc5aa
|
244 |
hslot = udp_hashslot(udptable, net, first); |
98322f22e
|
245 |
bitmap_zero(bitmap, PORTS_PER_CHAIN); |
645ca708f
|
246 |
spin_lock_bh(&hslot->lock); |
98322f22e
|
247 |
udp_lib_lport_inuse(net, snum, hslot, bitmap, sk, |
fe38d2a1c
|
248 |
udptable->log); |
98322f22e
|
249 250 251 252 253 254 255 |
snum = first; /* * Iterate on all possible values of snum for this hash. * Using steps of an odd multiple of UDP_HTABLE_SIZE * give us randomization and full range coverage. */ |
9088c5609
|
256 |
do { |
98322f22e
|
257 |
if (low <= snum && snum <= high && |
e3826f1e9
|
258 |
!test_bit(snum >> udptable->log, bitmap) && |
122ff243f
|
259 |
!inet_is_local_reserved_port(net, snum)) |
98322f22e
|
260 261 262 263 |
goto found; snum += rand; } while (snum != first); spin_unlock_bh(&hslot->lock); |
df560056d
|
264 |
cond_resched(); |
5781b2356
|
265 |
} while (++first != last); |
98322f22e
|
266 |
goto fail; |
645ca708f
|
267 |
} else { |
f86dcc5aa
|
268 |
hslot = udp_hashslot(udptable, net, snum); |
645ca708f
|
269 |
spin_lock_bh(&hslot->lock); |
30fff9231
|
270 271 272 273 274 275 276 277 278 279 |
if (hslot->count > 10) { int exist; unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; slot2 &= udptable->mask; hash2_nulladdr &= udptable->mask; hslot2 = udp_hashslot2(udptable, slot2); if (hslot->count < hslot2->count) goto scan_primary_hash; |
fe38d2a1c
|
280 |
exist = udp_lib_lport_inuse2(net, snum, hslot2, sk); |
30fff9231
|
281 282 283 |
if (!exist && (hash2_nulladdr != slot2)) { hslot2 = udp_hashslot2(udptable, hash2_nulladdr); exist = udp_lib_lport_inuse2(net, snum, hslot2, |
fe38d2a1c
|
284 |
sk); |
30fff9231
|
285 286 287 288 289 290 291 |
} if (exist) goto fail_unlock; else goto found; } scan_primary_hash: |
fe38d2a1c
|
292 |
if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0)) |
645ca708f
|
293 294 |
goto fail_unlock; } |
98322f22e
|
295 |
found: |
c720c7e83
|
296 |
inet_sk(sk)->inet_num = snum; |
d4cada4ae
|
297 298 |
udp_sk(sk)->udp_port_hash = snum; udp_sk(sk)->udp_portaddr_hash ^= snum; |
1da177e4c
|
299 |
if (sk_unhashed(sk)) { |
e32ea7e74
|
300 |
if (sk->sk_reuseport && |
fe38d2a1c
|
301 |
udp_reuseport_add_sock(sk, hslot)) { |
e32ea7e74
|
302 303 304 305 306 |
inet_sk(sk)->inet_num = 0; udp_sk(sk)->udp_port_hash = 0; udp_sk(sk)->udp_portaddr_hash ^= snum; goto fail_unlock; } |
ca065d0cf
|
307 |
sk_add_node_rcu(sk, &hslot->head); |
fdcc8aa95
|
308 |
hslot->count++; |
c29a0bc4d
|
309 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); |
512615b6b
|
310 311 312 |
hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); spin_lock(&hslot2->lock); |
d894ba18d
|
313 |
if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && |
1602f49b5
|
314 315 316 |
sk->sk_family == AF_INET6) hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node, &hslot2->head); |
d894ba18d
|
317 |
else |
1602f49b5
|
318 319 |
hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, &hslot2->head); |
512615b6b
|
320 321 |
hslot2->count++; spin_unlock(&hslot2->lock); |
1da177e4c
|
322 |
} |
ca065d0cf
|
323 |
sock_set_flag(sk, SOCK_RCU_FREE); |
25030a7f9
|
324 |
error = 0; |
645ca708f
|
325 326 |
fail_unlock: spin_unlock_bh(&hslot->lock); |
1da177e4c
|
327 |
fail: |
25030a7f9
|
328 329 |
return error; } |
c482c5685
|
330 |
EXPORT_SYMBOL(udp_lib_get_port); |
25030a7f9
|
331 |
|
6ba5a3c52
|
332 |
int udp_v4_get_port(struct sock *sk, unsigned short snum) |
db8dac20d
|
333 |
{ |
30fff9231
|
334 |
unsigned int hash2_nulladdr = |
f0b1e64c1
|
335 |
ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum); |
30fff9231
|
336 |
unsigned int hash2_partial = |
f0b1e64c1
|
337 |
ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0); |
30fff9231
|
338 |
|
d4cada4ae
|
339 |
/* precompute partial secondary hash */ |
30fff9231
|
340 |
udp_sk(sk)->udp_portaddr_hash = hash2_partial; |
fe38d2a1c
|
341 |
return udp_lib_get_port(sk, snum, hash2_nulladdr); |
db8dac20d
|
342 |
} |
d1e37288c
|
343 344 |
static int compute_score(struct sock *sk, struct net *net, __be32 saddr, __be16 sport, |
fb74c2773
|
345 |
__be32 daddr, unsigned short hnum, |
735453730
|
346 |
int dif, int sdif) |
645ca708f
|
347 |
{ |
60c04aecd
|
348 349 |
int score; struct inet_sock *inet; |
6da5b0f02
|
350 |
bool dev_match; |
645ca708f
|
351 |
|
60c04aecd
|
352 353 354 355 |
if (!net_eq(sock_net(sk), net) || udp_sk(sk)->udp_port_hash != hnum || ipv6_only_sock(sk)) return -1; |
645ca708f
|
356 |
|
4cdeeee92
|
357 358 |
if (sk->sk_rcv_saddr != daddr) return -1; |
60c04aecd
|
359 |
|
4cdeeee92
|
360 |
score = (sk->sk_family == PF_INET) ? 2 : 1; |
60c04aecd
|
361 |
|
4cdeeee92
|
362 |
inet = inet_sk(sk); |
60c04aecd
|
363 364 365 366 367 368 369 370 371 372 373 |
if (inet->inet_daddr) { if (inet->inet_daddr != saddr) return -1; score += 4; } if (inet->inet_dport) { if (inet->inet_dport != sport) return -1; score += 4; } |
6da5b0f02
|
374 375 376 377 378 |
dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif); if (!dev_match) return -1; score += 4; |
fb74c2773
|
379 |
|
7170a9777
|
380 |
if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) |
70da268b5
|
381 |
score++; |
645ca708f
|
382 383 |
return score; } |
6eada0110
|
384 385 386 |
static u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport, const __be32 faddr, const __be16 fport) |
65cd8033f
|
387 |
{ |
1bbdceef1
|
388 389 390 |
static u32 udp_ehash_secret __read_mostly; net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret)); |
65cd8033f
|
391 |
return __inet_ehashfn(laddr, lport, faddr, fport, |
1bbdceef1
|
392 |
udp_ehash_secret + net_hash_mix(net)); |
65cd8033f
|
393 |
} |
d1e37288c
|
394 |
/* called with rcu_read_lock() */ |
5051ebd27
|
395 |
static struct sock *udp4_lib_lookup2(struct net *net, |
fb74c2773
|
396 397 |
__be32 saddr, __be16 sport, __be32 daddr, unsigned int hnum, |
735453730
|
398 |
int dif, int sdif, |
fb74c2773
|
399 400 |
struct udp_hslot *hslot2, struct sk_buff *skb) |
5051ebd27
|
401 402 |
{ struct sock *sk, *result; |
e94a62f50
|
403 |
int score, badness; |
ba418fa35
|
404 |
u32 hash = 0; |
5051ebd27
|
405 |
|
5051ebd27
|
406 |
result = NULL; |
ba418fa35
|
407 |
badness = 0; |
ca065d0cf
|
408 |
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { |
d1e37288c
|
409 |
score = compute_score(sk, net, saddr, sport, |
735453730
|
410 |
daddr, hnum, dif, sdif); |
5051ebd27
|
411 |
if (score > badness) { |
acdcecc61
|
412 413 |
if (sk->sk_reuseport && sk->sk_state != TCP_ESTABLISHED) { |
65cd8033f
|
414 415 |
hash = udp_ehashfn(net, daddr, hnum, saddr, sport); |
ca065d0cf
|
416 |
result = reuseport_select_sock(sk, hash, skb, |
ed0dfffd7
|
417 |
sizeof(struct udphdr)); |
acdcecc61
|
418 |
if (result && !reuseport_has_conns(sk, false)) |
ca065d0cf
|
419 |
return result; |
ba418fa35
|
420 |
} |
ca065d0cf
|
421 422 |
badness = score; result = sk; |
5051ebd27
|
423 424 |
} } |
5051ebd27
|
425 426 |
return result; } |
db8dac20d
|
427 428 429 |
/* UDP is nearly always wildcards out the wazoo, it makes no sense to try * harder than this. -DaveM */ |
fce823381
|
430 |
struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, |
fb74c2773
|
431 432 |
__be16 sport, __be32 daddr, __be16 dport, int dif, int sdif, struct udp_table *udptable, struct sk_buff *skb) |
db8dac20d
|
433 |
{ |
4cdeeee92
|
434 |
struct sock *result; |
db8dac20d
|
435 |
unsigned short hnum = ntohs(dport); |
4cdeeee92
|
436 437 |
unsigned int hash2, slot2; struct udp_hslot *hslot2; |
645ca708f
|
438 |
|
4cdeeee92
|
439 440 441 442 443 444 |
hash2 = ipv4_portaddr_hash(net, daddr, hnum); slot2 = hash2 & udptable->mask; hslot2 = &udptable->hash2[slot2]; result = udp4_lib_lookup2(net, saddr, sport, daddr, hnum, dif, sdif, |
735453730
|
445 |
hslot2, skb); |
4cdeeee92
|
446 447 |
if (!result) { hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum); |
5051ebd27
|
448 449 |
slot2 = hash2 & udptable->mask; hslot2 = &udptable->hash2[slot2]; |
5051ebd27
|
450 451 |
result = udp4_lib_lookup2(net, saddr, sport, |
4cdeeee92
|
452 |
htonl(INADDR_ANY), hnum, dif, sdif, |
735453730
|
453 |
hslot2, skb); |
db8dac20d
|
454 |
} |
88e235b80
|
455 |
if (IS_ERR(result)) |
4cdeeee92
|
456 |
return NULL; |
db8dac20d
|
457 458 |
return result; } |
fce823381
|
459 |
EXPORT_SYMBOL_GPL(__udp4_lib_lookup); |
db8dac20d
|
460 |
|
607c4aaf0
|
461 462 |
static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb, __be16 sport, __be16 dport, |
645ca708f
|
463 |
struct udp_table *udptable) |
607c4aaf0
|
464 465 |
{ const struct iphdr *iph = ip_hdr(skb); |
ed7cbbce5
|
466 |
return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport, |
8afdd99a1
|
467 |
iph->daddr, dport, inet_iif(skb), |
fb74c2773
|
468 |
inet_sdif(skb), udptable, skb); |
607c4aaf0
|
469 |
} |
63058308c
|
470 471 472 |
struct sock *udp4_lib_lookup_skb(struct sk_buff *skb, __be16 sport, __be16 dport) { |
257a525fe
|
473 474 475 476 477 |
const struct iphdr *iph = ip_hdr(skb); return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport, iph->daddr, dport, inet_iif(skb), inet_sdif(skb), &udp_table, NULL); |
63058308c
|
478 479 |
} EXPORT_SYMBOL_GPL(udp4_lib_lookup_skb); |
ca065d0cf
|
480 481 482 |
/* Must be called under rcu_read_lock(). * Does increment socket refcount. */ |
6e86000c2
|
483 |
#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4) |
bcd41303f
|
484 485 486 |
struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport, __be32 daddr, __be16 dport, int dif) { |
ca065d0cf
|
487 488 489 |
struct sock *sk; sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport, |
fb74c2773
|
490 |
dif, 0, &udp_table, NULL); |
41c6d650f
|
491 |
if (sk && !refcount_inc_not_zero(&sk->sk_refcnt)) |
ca065d0cf
|
492 493 |
sk = NULL; return sk; |
bcd41303f
|
494 495 |
} EXPORT_SYMBOL_GPL(udp4_lib_lookup); |
ca065d0cf
|
496 |
#endif |
bcd41303f
|
497 |
|
421b3885b
|
498 499 500 |
static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk, __be16 loc_port, __be32 loc_addr, __be16 rmt_port, __be32 rmt_addr, |
fb74c2773
|
501 |
int dif, int sdif, unsigned short hnum) |
421b3885b
|
502 503 504 505 506 507 508 509 510 |
{ struct inet_sock *inet = inet_sk(sk); if (!net_eq(sock_net(sk), net) || udp_sk(sk)->udp_port_hash != hnum || (inet->inet_daddr && inet->inet_daddr != rmt_addr) || (inet->inet_dport != rmt_port && inet->inet_dport) || (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) || ipv6_only_sock(sk) || |
82ba25c6d
|
511 |
!udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) |
421b3885b
|
512 |
return false; |
60d9b0314
|
513 |
if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif)) |
421b3885b
|
514 515 516 |
return false; return true; } |
a36e185e8
|
517 518 519 |
DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key); void udp_encap_enable(void) { |
9c4806014
|
520 |
static_branch_inc(&udp_encap_needed_key); |
a36e185e8
|
521 522 |
} EXPORT_SYMBOL(udp_encap_enable); |
e7cc08245
|
523 524 525 526 527 528 529 530 531 |
/* Handler for tunnels with arbitrary destination ports: no socket lookup, go * through error handlers in encapsulations looking for a match. */ static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info) { int i; for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) { int (*handler)(struct sk_buff *skb, u32 info); |
92b953642
|
532 |
const struct ip_tunnel_encap_ops *encap; |
e7cc08245
|
533 |
|
92b953642
|
534 535 |
encap = rcu_dereference(iptun_encaps[i]); if (!encap) |
e7cc08245
|
536 |
continue; |
92b953642
|
537 |
handler = encap->err_handler; |
e7cc08245
|
538 539 540 541 542 543 |
if (handler && !handler(skb, info)) return 0; } return -ENOENT; } |
a36e185e8
|
544 545 546 547 548 549 550 |
/* Try to match ICMP errors to UDP tunnels by looking up a socket without * reversing source and destination port: this will match tunnels that force the * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that * lwtunnels might actually break this assumption by being configured with * different destination ports on endpoints, in this case we won't be able to * trace ICMP messages back to them. * |
e7cc08245
|
551 552 553 554 |
* If this doesn't match any socket, probe tunnels with arbitrary destination * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port * we've sent packets to won't necessarily match the local destination port. * |
a36e185e8
|
555 556 557 |
* Then ask the tunnel implementation to match the error against a valid * association. * |
e7cc08245
|
558 559 |
* Return an error if we can't find a match, the socket if we need further * processing, zero otherwise. |
a36e185e8
|
560 561 562 563 564 |
*/ static struct sock *__udp4_lib_err_encap(struct net *net, const struct iphdr *iph, struct udphdr *uh, struct udp_table *udptable, |
e7cc08245
|
565 |
struct sk_buff *skb, u32 info) |
a36e185e8
|
566 |
{ |
a36e185e8
|
567 |
int network_offset, transport_offset; |
a36e185e8
|
568 |
struct sock *sk; |
a36e185e8
|
569 570 571 572 573 574 575 576 |
network_offset = skb_network_offset(skb); transport_offset = skb_transport_offset(skb); /* Network header needs to point to the outer IPv4 header inside ICMP */ skb_reset_network_header(skb); /* Transport header needs to point to the UDP header */ skb_set_transport_header(skb, iph->ihl << 2); |
e7cc08245
|
577 578 579 580 581 582 583 584 585 586 587 588 589 590 |
sk = __udp4_lib_lookup(net, iph->daddr, uh->source, iph->saddr, uh->dest, skb->dev->ifindex, 0, udptable, NULL); if (sk) { int (*lookup)(struct sock *sk, struct sk_buff *skb); struct udp_sock *up = udp_sk(sk); lookup = READ_ONCE(up->encap_err_lookup); if (!lookup || lookup(sk, skb)) sk = NULL; } if (!sk) sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info)); |
a36e185e8
|
591 592 593 594 595 596 |
skb_set_transport_header(skb, transport_offset); skb_set_network_header(skb, network_offset); return sk; } |
db8dac20d
|
597 598 599 600 601 602 603 604 605 606 |
/* * This routine is called by the ICMP module when it gets some * sort of error condition. If err < 0 then the socket should * be closed and the error returned to the user. If err > 0 * it's just the icmp type << 8 | icmp code. * Header points to the ip header of the error packet. We move * on past this. Then (as it used to claim before adjustment) * header points to the first 8 bytes of the udp header. We need * to find the appropriate port. */ |
32bbd8793
|
607 |
int __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable) |
db8dac20d
|
608 609 |
{ struct inet_sock *inet; |
b71d1d426
|
610 |
const struct iphdr *iph = (const struct iphdr *)skb->data; |
c482c5685
|
611 |
struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2)); |
db8dac20d
|
612 613 |
const int type = icmp_hdr(skb)->type; const int code = icmp_hdr(skb)->code; |
a36e185e8
|
614 |
bool tunnel = false; |
db8dac20d
|
615 616 617 |
struct sock *sk; int harderr; int err; |
fd54d716b
|
618 |
struct net *net = dev_net(skb->dev); |
db8dac20d
|
619 |
|
fd54d716b
|
620 |
sk = __udp4_lib_lookup(net, iph->daddr, uh->dest, |
f64bf6b8a
|
621 622 |
iph->saddr, uh->source, skb->dev->ifindex, inet_sdif(skb), udptable, NULL); |
51456b291
|
623 |
if (!sk) { |
a36e185e8
|
624 |
/* No socket for error: try tunnels before discarding */ |
e7cc08245
|
625 626 627 628 629 630 631 |
sk = ERR_PTR(-ENOENT); if (static_branch_unlikely(&udp_encap_needed_key)) { sk = __udp4_lib_err_encap(net, iph, uh, udptable, skb, info); if (!sk) return 0; } |
a36e185e8
|
632 |
|
e7cc08245
|
633 |
if (IS_ERR(sk)) { |
a36e185e8
|
634 |
__ICMP_INC_STATS(net, ICMP_MIB_INERRORS); |
e7cc08245
|
635 |
return PTR_ERR(sk); |
a36e185e8
|
636 |
} |
e7cc08245
|
637 |
|
a36e185e8
|
638 |
tunnel = true; |
db8dac20d
|
639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 |
} err = 0; harderr = 0; inet = inet_sk(sk); switch (type) { default: case ICMP_TIME_EXCEEDED: err = EHOSTUNREACH; break; case ICMP_SOURCE_QUENCH: goto out; case ICMP_PARAMETERPROB: err = EPROTO; harderr = 1; break; case ICMP_DEST_UNREACH: if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */ |
363933955
|
658 |
ipv4_sk_update_pmtu(skb, sk, info); |
db8dac20d
|
659 660 661 662 663 664 665 666 667 668 669 670 671 |
if (inet->pmtudisc != IP_PMTUDISC_DONT) { err = EMSGSIZE; harderr = 1; break; } goto out; } err = EHOSTUNREACH; if (code <= NR_ICMP_UNREACH) { harderr = icmp_err_convert[code].fatal; err = icmp_err_convert[code].errno; } break; |
55be7a9c6
|
672 673 |
case ICMP_REDIRECT: ipv4_sk_redirect(skb, sk); |
1a462d189
|
674 |
goto out; |
db8dac20d
|
675 676 677 678 679 680 |
} /* * RFC1122: OK. Passes ICMP errors back to application, as per * 4.1.3.3. */ |
a36e185e8
|
681 682 683 684 |
if (tunnel) { /* ...not for tunnels though: we don't have a sending socket */ goto out; } |
db8dac20d
|
685 686 687 |
if (!inet->recverr) { if (!harderr || sk->sk_state != TCP_ESTABLISHED) goto out; |
b1faf5666
|
688 |
} else |
c482c5685
|
689 |
ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1)); |
b1faf5666
|
690 |
|
db8dac20d
|
691 692 693 |
sk->sk_err = err; sk->sk_error_report(sk); out: |
32bbd8793
|
694 |
return 0; |
db8dac20d
|
695 |
} |
32bbd8793
|
696 |
int udp_err(struct sk_buff *skb, u32 info) |
db8dac20d
|
697 |
{ |
32bbd8793
|
698 |
return __udp4_lib_err(skb, info, &udp_table); |
db8dac20d
|
699 700 701 702 703 |
} /* * Throw away all pending data and cancel the corking. Socket is locked. */ |
36d926b94
|
704 |
void udp_flush_pending_frames(struct sock *sk) |
db8dac20d
|
705 706 707 708 709 710 711 712 713 |
{ struct udp_sock *up = udp_sk(sk); if (up->pending) { up->len = 0; up->pending = 0; ip_flush_pending_frames(sk); } } |
36d926b94
|
714 |
EXPORT_SYMBOL(udp_flush_pending_frames); |
db8dac20d
|
715 716 |
/** |
f6b9664f8
|
717 |
* udp4_hwcsum - handle outgoing HW checksumming |
db8dac20d
|
718 719 |
* @skb: sk_buff containing the filled-in UDP header * (checksum field must be zeroed out) |
f6b9664f8
|
720 721 |
* @src: source IP address * @dst: destination IP address |
db8dac20d
|
722 |
*/ |
c26bf4a51
|
723 |
void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst) |
db8dac20d
|
724 |
{ |
db8dac20d
|
725 |
struct udphdr *uh = udp_hdr(skb); |
f6b9664f8
|
726 727 728 |
int offset = skb_transport_offset(skb); int len = skb->len - offset; int hlen = len; |
db8dac20d
|
729 |
__wsum csum = 0; |
ebbe495f1
|
730 |
if (!skb_has_frag_list(skb)) { |
db8dac20d
|
731 732 733 734 735 |
/* * Only one fragment on the socket. */ skb->csum_start = skb_transport_header(skb) - skb->head; skb->csum_offset = offsetof(struct udphdr, check); |
f6b9664f8
|
736 737 |
uh->check = ~csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, 0); |
db8dac20d
|
738 |
} else { |
ebbe495f1
|
739 |
struct sk_buff *frags; |
db8dac20d
|
740 741 742 743 744 |
/* * HW-checksum won't work as there are two or more * fragments on the socket so that all csums of sk_buffs * should be together */ |
ebbe495f1
|
745 |
skb_walk_frags(skb, frags) { |
f6b9664f8
|
746 747 |
csum = csum_add(csum, frags->csum); hlen -= frags->len; |
ebbe495f1
|
748 |
} |
db8dac20d
|
749 |
|
f6b9664f8
|
750 |
csum = skb_checksum(skb, offset, hlen, csum); |
db8dac20d
|
751 |
skb->ip_summed = CHECKSUM_NONE; |
db8dac20d
|
752 753 754 755 756 |
uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum); if (uh->check == 0) uh->check = CSUM_MANGLED_0; } } |
c26bf4a51
|
757 |
EXPORT_SYMBOL_GPL(udp4_hwcsum); |
db8dac20d
|
758 |
|
af5fcba7f
|
759 760 761 762 763 764 765 |
/* Function to set UDP checksum for an IPv4 UDP packet. This is intended * for the simple case like when setting the checksum for a UDP tunnel. */ void udp_set_csum(bool nocheck, struct sk_buff *skb, __be32 saddr, __be32 daddr, int len) { struct udphdr *uh = udp_hdr(skb); |
179bc67f6
|
766 |
if (nocheck) { |
af5fcba7f
|
767 |
uh->check = 0; |
179bc67f6
|
768 |
} else if (skb_is_gso(skb)) { |
af5fcba7f
|
769 |
uh->check = ~udp_v4_check(len, saddr, daddr, 0); |
179bc67f6
|
770 771 772 773 774 |
} else if (skb->ip_summed == CHECKSUM_PARTIAL) { uh->check = 0; uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb)); if (uh->check == 0) uh->check = CSUM_MANGLED_0; |
d75f1306d
|
775 |
} else { |
af5fcba7f
|
776 777 778 779 |
skb->ip_summed = CHECKSUM_PARTIAL; skb->csum_start = skb_transport_header(skb) - skb->head; skb->csum_offset = offsetof(struct udphdr, check); uh->check = ~udp_v4_check(len, saddr, daddr, 0); |
af5fcba7f
|
780 781 782 |
} } EXPORT_SYMBOL(udp_set_csum); |
bec1f6f69
|
783 784 |
static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4, struct inet_cork *cork) |
db8dac20d
|
785 |
{ |
f6b9664f8
|
786 |
struct sock *sk = skb->sk; |
db8dac20d
|
787 |
struct inet_sock *inet = inet_sk(sk); |
db8dac20d
|
788 789 790 |
struct udphdr *uh; int err = 0; int is_udplite = IS_UDPLITE(sk); |
f6b9664f8
|
791 792 |
int offset = skb_transport_offset(skb); int len = skb->len - offset; |
4094871db
|
793 |
int datalen = len - sizeof(*uh); |
db8dac20d
|
794 |
__wsum csum = 0; |
db8dac20d
|
795 796 797 798 |
/* * Create a UDP header */ uh = udp_hdr(skb); |
f6b9664f8
|
799 |
uh->source = inet->inet_sport; |
79ab05314
|
800 |
uh->dest = fl4->fl4_dport; |
f6b9664f8
|
801 |
uh->len = htons(len); |
db8dac20d
|
802 |
uh->check = 0; |
bec1f6f69
|
803 804 805 |
if (cork->gso_size) { const int hlen = skb_network_header_len(skb) + sizeof(struct udphdr); |
0f149c9fe
|
806 807 |
if (hlen + cork->gso_size > cork->fragsize) { kfree_skb(skb); |
bec1f6f69
|
808 |
return -EINVAL; |
0f149c9fe
|
809 810 811 |
} if (skb->len > cork->gso_size * UDP_MAX_SEGMENTS) { kfree_skb(skb); |
bec1f6f69
|
812 |
return -EINVAL; |
0f149c9fe
|
813 814 815 |
} if (sk->sk_no_check_tx) { kfree_skb(skb); |
a8c744a8b
|
816 |
return -EINVAL; |
0f149c9fe
|
817 |
} |
ff06342cb
|
818 |
if (skb->ip_summed != CHECKSUM_PARTIAL || is_udplite || |
0f149c9fe
|
819 820 |
dst_xfrm(skb_dst(skb))) { kfree_skb(skb); |
bec1f6f69
|
821 |
return -EIO; |
0f149c9fe
|
822 |
} |
bec1f6f69
|
823 |
|
4094871db
|
824 825 826 827 828 829 |
if (datalen > cork->gso_size) { skb_shinfo(skb)->gso_size = cork->gso_size; skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4; skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen, cork->gso_size); } |
a8c744a8b
|
830 |
goto csum_partial; |
bec1f6f69
|
831 |
} |
db8dac20d
|
832 |
if (is_udplite) /* UDP-Lite */ |
f6b9664f8
|
833 |
csum = udplite_csum(skb); |
db8dac20d
|
834 |
|
ab2fb7e32
|
835 |
else if (sk->sk_no_check_tx) { /* UDP csum off */ |
db8dac20d
|
836 837 838 839 840 |
skb->ip_summed = CHECKSUM_NONE; goto send; } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */ |
a8c744a8b
|
841 |
csum_partial: |
db8dac20d
|
842 |
|
79ab05314
|
843 |
udp4_hwcsum(skb, fl4->saddr, fl4->daddr); |
db8dac20d
|
844 |
goto send; |
f6b9664f8
|
845 846 |
} else csum = udp_csum(skb); |
db8dac20d
|
847 848 |
/* add protocol-dependent pseudo-header */ |
79ab05314
|
849 |
uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len, |
c482c5685
|
850 |
sk->sk_protocol, csum); |
db8dac20d
|
851 852 853 854 |
if (uh->check == 0) uh->check = CSUM_MANGLED_0; send: |
b5ec8eeac
|
855 |
err = ip_send_skb(sock_net(sk), skb); |
6ce9e7b5f
|
856 857 |
if (err) { if (err == -ENOBUFS && !inet->recverr) { |
6aef70a85
|
858 859 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS, is_udplite); |
6ce9e7b5f
|
860 861 862 |
err = 0; } } else |
6aef70a85
|
863 864 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_OUTDATAGRAMS, is_udplite); |
f6b9664f8
|
865 866 867 868 869 870 |
return err; } /* * Push out all pending data as one UDP datagram. Socket is locked. */ |
8822b64a0
|
871 |
int udp_push_pending_frames(struct sock *sk) |
f6b9664f8
|
872 873 874 |
{ struct udp_sock *up = udp_sk(sk); struct inet_sock *inet = inet_sk(sk); |
b6f21b268
|
875 |
struct flowi4 *fl4 = &inet->cork.fl.u.ip4; |
f6b9664f8
|
876 877 |
struct sk_buff *skb; int err = 0; |
77968b782
|
878 |
skb = ip_finish_skb(sk, fl4); |
f6b9664f8
|
879 880 |
if (!skb) goto out; |
bec1f6f69
|
881 |
err = udp_send_skb(skb, fl4, &inet->cork.base); |
f6b9664f8
|
882 |
|
db8dac20d
|
883 884 885 |
out: up->len = 0; up->pending = 0; |
db8dac20d
|
886 887 |
return err; } |
8822b64a0
|
888 |
EXPORT_SYMBOL(udp_push_pending_frames); |
db8dac20d
|
889 |
|
2e8de8576
|
890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 |
static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size) { switch (cmsg->cmsg_type) { case UDP_SEGMENT: if (cmsg->cmsg_len != CMSG_LEN(sizeof(__u16))) return -EINVAL; *gso_size = *(__u16 *)CMSG_DATA(cmsg); return 0; default: return -EINVAL; } } int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size) { struct cmsghdr *cmsg; bool need_ip = false; int err; for_each_cmsghdr(cmsg, msg) { if (!CMSG_OK(msg, cmsg)) return -EINVAL; if (cmsg->cmsg_level != SOL_UDP) { need_ip = true; continue; } err = __udp_cmsg_send(cmsg, gso_size); if (err) return err; } return need_ip; } EXPORT_SYMBOL_GPL(udp_cmsg_send); |
1b7841404
|
926 |
int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) |
db8dac20d
|
927 928 929 |
{ struct inet_sock *inet = inet_sk(sk); struct udp_sock *up = udp_sk(sk); |
1cedee13d
|
930 |
DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name); |
e474995f2
|
931 |
struct flowi4 fl4_stack; |
b6f21b268
|
932 |
struct flowi4 *fl4; |
db8dac20d
|
933 934 935 936 937 938 939 940 941 942 943 |
int ulen = len; struct ipcm_cookie ipc; struct rtable *rt = NULL; int free = 0; int connected = 0; __be32 daddr, faddr, saddr; __be16 dport; u8 tos; int err, is_udplite = IS_UDPLITE(sk); int corkreq = up->corkflag || msg->msg_flags&MSG_MORE; int (*getfrag)(void *, char *, int, int, int, struct sk_buff *); |
903ab86d1
|
944 |
struct sk_buff *skb; |
f6d8bd051
|
945 |
struct ip_options_data opt_copy; |
db8dac20d
|
946 947 948 949 950 951 952 |
if (len > 0xFFFF) return -EMSGSIZE; /* * Check the flags. */ |
c482c5685
|
953 |
if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */ |
db8dac20d
|
954 |
return -EOPNOTSUPP; |
903ab86d1
|
955 |
getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag; |
f5fca6086
|
956 |
fl4 = &inet->cork.fl.u.ip4; |
db8dac20d
|
957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 |
if (up->pending) { /* * There are pending frames. * The socket lock must be held while it's corked. */ lock_sock(sk); if (likely(up->pending)) { if (unlikely(up->pending != AF_INET)) { release_sock(sk); return -EINVAL; } goto do_append_data; } release_sock(sk); } ulen += sizeof(struct udphdr); /* * Get and verify the address. */ |
1cedee13d
|
977 |
if (usin) { |
db8dac20d
|
978 979 980 981 982 983 984 985 986 987 988 989 990 991 |
if (msg->msg_namelen < sizeof(*usin)) return -EINVAL; if (usin->sin_family != AF_INET) { if (usin->sin_family != AF_UNSPEC) return -EAFNOSUPPORT; } daddr = usin->sin_addr.s_addr; dport = usin->sin_port; if (dport == 0) return -EINVAL; } else { if (sk->sk_state != TCP_ESTABLISHED) return -EDESTADDRREQ; |
c720c7e83
|
992 993 |
daddr = inet->inet_daddr; dport = inet->inet_dport; |
db8dac20d
|
994 995 996 997 998 |
/* Open fast path for connected socket. Route will not be used, if at least one option is set. */ connected = 1; } |
db8dac20d
|
999 |
|
351782067
|
1000 |
ipcm_init_sk(&ipc, inet); |
bec1f6f69
|
1001 |
ipc.gso_size = up->gso_size; |
bf84a0106
|
1002 |
|
db8dac20d
|
1003 |
if (msg->msg_controllen) { |
2e8de8576
|
1004 1005 1006 1007 1008 |
err = udp_cmsg_send(sk, msg, &ipc.gso_size); if (err > 0) err = ip_cmsg_send(sk, msg, &ipc, sk->sk_family == AF_INET6); if (unlikely(err < 0)) { |
919483096
|
1009 |
kfree(ipc.opt); |
db8dac20d
|
1010 |
return err; |
919483096
|
1011 |
} |
db8dac20d
|
1012 1013 1014 1015 |
if (ipc.opt) free = 1; connected = 0; } |
f6d8bd051
|
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 |
if (!ipc.opt) { struct ip_options_rcu *inet_opt; rcu_read_lock(); inet_opt = rcu_dereference(inet->inet_opt); if (inet_opt) { memcpy(&opt_copy, inet_opt, sizeof(*inet_opt) + inet_opt->opt.optlen); ipc.opt = &opt_copy.opt; } rcu_read_unlock(); } |
db8dac20d
|
1028 |
|
1cedee13d
|
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 |
if (cgroup_bpf_enabled && !connected) { err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk, (struct sockaddr *)usin, &ipc.addr); if (err) goto out_free; if (usin) { if (usin->sin_port == 0) { /* BPF program set invalid port. Reject it. */ err = -EINVAL; goto out_free; } daddr = usin->sin_addr.s_addr; dport = usin->sin_port; } } |
db8dac20d
|
1044 1045 |
saddr = ipc.addr; ipc.addr = faddr = daddr; |
f6d8bd051
|
1046 |
if (ipc.opt && ipc.opt->opt.srr) { |
1b97013bf
|
1047 1048 1049 1050 |
if (!daddr) { err = -EINVAL; goto out_free; } |
f6d8bd051
|
1051 |
faddr = ipc.opt->opt.faddr; |
db8dac20d
|
1052 1053 |
connected = 0; } |
aa6615814
|
1054 |
tos = get_rttos(&ipc, inet); |
db8dac20d
|
1055 1056 |
if (sock_flag(sk, SOCK_LOCALROUTE) || (msg->msg_flags & MSG_DONTROUTE) || |
f6d8bd051
|
1057 |
(ipc.opt && ipc.opt->opt.is_strictroute)) { |
db8dac20d
|
1058 1059 1060 1061 1062 |
tos |= RTO_ONLINK; connected = 0; } if (ipv4_is_multicast(daddr)) { |
854da9917
|
1063 |
if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif)) |
db8dac20d
|
1064 1065 1066 1067 |
ipc.oif = inet->mc_index; if (!saddr) saddr = inet->mc_addr; connected = 0; |
9515a2e08
|
1068 |
} else if (!ipc.oif) { |
76e21053b
|
1069 |
ipc.oif = inet->uc_index; |
9515a2e08
|
1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 |
} else if (ipv4_is_lbcast(daddr) && inet->uc_index) { /* oif is set, packet is to local broadcast and * and uc_index is set. oif is most likely set * by sk_bound_dev_if. If uc_index != oif check if the * oif is an L3 master and uc_index is an L3 slave. * If so, we want to allow the send using the uc_index. */ if (ipc.oif != inet->uc_index && ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk), inet->uc_index)) { ipc.oif = inet->uc_index; } } |
db8dac20d
|
1083 1084 |
if (connected) |
c482c5685
|
1085 |
rt = (struct rtable *)sk_dst_check(sk, 0); |
db8dac20d
|
1086 |
|
51456b291
|
1087 |
if (!rt) { |
84a3aa000
|
1088 |
struct net *net = sock_net(sk); |
9a24abfa4
|
1089 |
__u8 flow_flags = inet_sk_flowi_flags(sk); |
84a3aa000
|
1090 |
|
e474995f2
|
1091 |
fl4 = &fl4_stack; |
9a24abfa4
|
1092 |
|
c6af0c227
|
1093 |
flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark, tos, |
c0951cbcf
|
1094 |
RT_SCOPE_UNIVERSE, sk->sk_protocol, |
9a24abfa4
|
1095 |
flow_flags, |
e2d118a1c
|
1096 1097 |
faddr, saddr, dport, inet->inet_sport, sk->sk_uid); |
c0951cbcf
|
1098 |
|
e474995f2
|
1099 1100 |
security_sk_classify_flow(sk, flowi4_to_flowi(fl4)); rt = ip_route_output_flow(net, fl4, sk); |
b23dd4fe4
|
1101 1102 |
if (IS_ERR(rt)) { err = PTR_ERR(rt); |
06dc94b1e
|
1103 |
rt = NULL; |
db8dac20d
|
1104 |
if (err == -ENETUNREACH) |
f1d8cba61
|
1105 |
IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); |
db8dac20d
|
1106 1107 1108 1109 1110 1111 1112 1113 |
goto out; } err = -EACCES; if ((rt->rt_flags & RTCF_BROADCAST) && !sock_flag(sk, SOCK_BROADCAST)) goto out; if (connected) |
d8d1f30b9
|
1114 |
sk_dst_set(sk, dst_clone(&rt->dst)); |
db8dac20d
|
1115 1116 1117 1118 1119 |
} if (msg->msg_flags&MSG_CONFIRM) goto do_confirm; back_from_confirm: |
e474995f2
|
1120 |
saddr = fl4->saddr; |
db8dac20d
|
1121 |
if (!ipc.addr) |
e474995f2
|
1122 |
daddr = ipc.addr = fl4->daddr; |
db8dac20d
|
1123 |
|
903ab86d1
|
1124 1125 |
/* Lockless fast path for the non-corking case. */ if (!corkreq) { |
1cd7884df
|
1126 |
struct inet_cork cork; |
f69e6d131
|
1127 |
skb = ip_make_skb(sk, fl4, getfrag, msg, ulen, |
903ab86d1
|
1128 |
sizeof(struct udphdr), &ipc, &rt, |
1cd7884df
|
1129 |
&cork, msg->msg_flags); |
903ab86d1
|
1130 |
err = PTR_ERR(skb); |
50c3a487d
|
1131 |
if (!IS_ERR_OR_NULL(skb)) |
bec1f6f69
|
1132 |
err = udp_send_skb(skb, fl4, &cork); |
903ab86d1
|
1133 1134 |
goto out; } |
db8dac20d
|
1135 1136 1137 1138 1139 |
lock_sock(sk); if (unlikely(up->pending)) { /* The socket is already corked while preparing it. */ /* ... which is an evident application bug. --ANK */ release_sock(sk); |
197df02cb
|
1140 1141 |
net_dbg_ratelimited("socket already corked "); |
db8dac20d
|
1142 1143 1144 1145 1146 1147 |
err = -EINVAL; goto out; } /* * Now cork the socket to pend data. */ |
b6f21b268
|
1148 1149 1150 |
fl4 = &inet->cork.fl.u.ip4; fl4->daddr = daddr; fl4->saddr = saddr; |
9cce96df5
|
1151 1152 |
fl4->fl4_dport = dport; fl4->fl4_sport = inet->inet_sport; |
db8dac20d
|
1153 1154 1155 1156 |
up->pending = AF_INET; do_append_data: up->len += ulen; |
f69e6d131
|
1157 |
err = ip_append_data(sk, fl4, getfrag, msg, ulen, |
f5fca6086
|
1158 1159 |
sizeof(struct udphdr), &ipc, &rt, corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags); |
db8dac20d
|
1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 |
if (err) udp_flush_pending_frames(sk); else if (!corkreq) err = udp_push_pending_frames(sk); else if (unlikely(skb_queue_empty(&sk->sk_write_queue))) up->pending = 0; release_sock(sk); out: ip_rt_put(rt); |
1b97013bf
|
1170 |
out_free: |
db8dac20d
|
1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 |
if (free) kfree(ipc.opt); if (!err) return len; /* * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting * ENOBUFS might not be good (it's not tunable per se), but otherwise * we don't have a good statistic (IpOutDiscards but it can be too many * things). We could add another new stat but at least for now that * seems like overkill. */ if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { |
6aef70a85
|
1183 1184 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS, is_udplite); |
db8dac20d
|
1185 1186 1187 1188 |
} return err; do_confirm: |
0dec879f6
|
1189 1190 |
if (msg->msg_flags & MSG_PROBE) dst_confirm_neigh(&rt->dst, &fl4->daddr); |
db8dac20d
|
1191 1192 1193 1194 1195 |
if (!(msg->msg_flags&MSG_PROBE) || len) goto back_from_confirm; err = 0; goto out; } |
c482c5685
|
1196 |
EXPORT_SYMBOL(udp_sendmsg); |
db8dac20d
|
1197 1198 1199 1200 |
int udp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, int flags) { |
f5fca6086
|
1201 |
struct inet_sock *inet = inet_sk(sk); |
db8dac20d
|
1202 1203 |
struct udp_sock *up = udp_sk(sk); int ret; |
d3f7d56a7
|
1204 1205 |
if (flags & MSG_SENDPAGE_NOTLAST) flags |= MSG_MORE; |
db8dac20d
|
1206 1207 1208 1209 1210 1211 1212 |
if (!up->pending) { struct msghdr msg = { .msg_flags = flags|MSG_MORE }; /* Call udp_sendmsg to specify destination address which * sendpage interface can't pass. * This will succeed only when the socket is connected. */ |
1b7841404
|
1213 |
ret = udp_sendmsg(sk, &msg, 0); |
db8dac20d
|
1214 1215 1216 1217 1218 1219 1220 1221 |
if (ret < 0) return ret; } lock_sock(sk); if (unlikely(!up->pending)) { release_sock(sk); |
197df02cb
|
1222 1223 |
net_dbg_ratelimited("cork failed "); |
db8dac20d
|
1224 1225 |
return -EINVAL; } |
f5fca6086
|
1226 1227 |
ret = ip_append_page(sk, &inet->cork.fl.u.ip4, page, offset, size, flags); |
db8dac20d
|
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 |
if (ret == -EOPNOTSUPP) { release_sock(sk); return sock_no_sendpage(sk->sk_socket, page, offset, size, flags); } if (ret < 0) { udp_flush_pending_frames(sk); goto out; } up->len += size; if (!(up->corkflag || (flags&MSG_MORE))) ret = udp_push_pending_frames(sk); if (!ret) ret = size; out: release_sock(sk); return ret; } |
dce4551cb
|
1247 |
#define UDP_SKB_IS_STATELESS 0x80000000 |
677bf08cf
|
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 |
/* all head states (dst, sk, nf conntrack) except skb extensions are * cleared by udp_rcv(). * * We need to preserve secpath, if present, to eventually process * IP_CMSG_PASSSEC at recvmsg() time. * * Other extensions can be cleared. */ static bool udp_try_make_stateless(struct sk_buff *skb) { if (!skb_has_extensions(skb)) return true; if (!secpath_exists(skb)) { skb_ext_reset(skb); return true; } return false; } |
b65ac4467
|
1268 1269 |
static void udp_set_dev_scratch(struct sk_buff *skb) { |
dce4551cb
|
1270 |
struct udp_dev_scratch *scratch = udp_skb_scratch(skb); |
b65ac4467
|
1271 1272 |
BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long)); |
dce4551cb
|
1273 1274 |
scratch->_tsize_state = skb->truesize; #if BITS_PER_LONG == 64 |
b65ac4467
|
1275 1276 1277 |
scratch->len = skb->len; scratch->csum_unnecessary = !!skb_csum_unnecessary(skb); scratch->is_linear = !skb_is_nonlinear(skb); |
dce4551cb
|
1278 |
#endif |
677bf08cf
|
1279 |
if (udp_try_make_stateless(skb)) |
dce4551cb
|
1280 |
scratch->_tsize_state |= UDP_SKB_IS_STATELESS; |
b65ac4467
|
1281 |
} |
a793183ca
|
1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 |
static void udp_skb_csum_unnecessary_set(struct sk_buff *skb) { /* We come here after udp_lib_checksum_complete() returned 0. * This means that __skb_checksum_complete() might have * set skb->csum_valid to 1. * On 64bit platforms, we can set csum_unnecessary * to true, but only if the skb is not shared. */ #if BITS_PER_LONG == 64 if (!skb_shared(skb)) udp_skb_scratch(skb)->csum_unnecessary = true; #endif } |
b65ac4467
|
1295 1296 |
static int udp_skb_truesize(struct sk_buff *skb) { |
dce4551cb
|
1297 |
return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS; |
b65ac4467
|
1298 |
} |
dce4551cb
|
1299 |
static bool udp_skb_has_head_state(struct sk_buff *skb) |
b65ac4467
|
1300 |
{ |
dce4551cb
|
1301 |
return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS); |
b65ac4467
|
1302 |
} |
b65ac4467
|
1303 |
|
7c13f97ff
|
1304 |
/* fully reclaim rmem/fwd memory allocated for skb */ |
6dfb4367c
|
1305 1306 |
static void udp_rmem_release(struct sock *sk, int size, int partial, bool rx_queue_lock_held) |
f970bd9e3
|
1307 |
{ |
6b229cf77
|
1308 |
struct udp_sock *up = udp_sk(sk); |
2276f58ac
|
1309 |
struct sk_buff_head *sk_queue; |
f970bd9e3
|
1310 |
int amt; |
6b229cf77
|
1311 1312 1313 |
if (likely(partial)) { up->forward_deficit += size; size = up->forward_deficit; |
41c230ae2
|
1314 1315 |
if (size < (sk->sk_rcvbuf >> 2) && !skb_queue_empty(&up->reader_queue)) |
6b229cf77
|
1316 1317 1318 1319 1320 |
return; } else { size += up->forward_deficit; } up->forward_deficit = 0; |
6dfb4367c
|
1321 1322 1323 |
/* acquire the sk_receive_queue for fwd allocated memory scheduling, * if the called don't held it already */ |
2276f58ac
|
1324 |
sk_queue = &sk->sk_receive_queue; |
6dfb4367c
|
1325 1326 |
if (!rx_queue_lock_held) spin_lock(&sk_queue->lock); |
2276f58ac
|
1327 |
|
f970bd9e3
|
1328 1329 1330 |
sk->sk_forward_alloc += size; amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1); sk->sk_forward_alloc -= amt; |
f970bd9e3
|
1331 1332 1333 |
if (amt) __sk_mem_reduce_allocated(sk, amt >> SK_MEM_QUANTUM_SHIFT); |
02ab0d139
|
1334 1335 |
atomic_sub(size, &sk->sk_rmem_alloc); |
2276f58ac
|
1336 1337 1338 |
/* this can save us from acquiring the rx queue lock on next receive */ skb_queue_splice_tail_init(sk_queue, &up->reader_queue); |
6dfb4367c
|
1339 1340 |
if (!rx_queue_lock_held) spin_unlock(&sk_queue->lock); |
f970bd9e3
|
1341 |
} |
2276f58ac
|
1342 |
/* Note: called with reader_queue.lock held. |
c84d94905
|
1343 1344 1345 1346 |
* Instead of using skb->truesize here, find a copy of it in skb->dev_scratch * This avoids a cache line miss while receive_queue lock is held. * Look at __udp_enqueue_schedule_skb() to find where this copy is done. */ |
7c13f97ff
|
1347 |
void udp_skb_destructor(struct sock *sk, struct sk_buff *skb) |
f970bd9e3
|
1348 |
{ |
b65ac4467
|
1349 1350 |
prefetch(&skb->data); udp_rmem_release(sk, udp_skb_truesize(skb), 1, false); |
f970bd9e3
|
1351 |
} |
7c13f97ff
|
1352 |
EXPORT_SYMBOL(udp_skb_destructor); |
f970bd9e3
|
1353 |
|
6dfb4367c
|
1354 |
/* as above, but the caller held the rx queue lock, too */ |
64f5102dc
|
1355 |
static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb) |
6dfb4367c
|
1356 |
{ |
b65ac4467
|
1357 1358 |
prefetch(&skb->data); udp_rmem_release(sk, udp_skb_truesize(skb), 1, true); |
6dfb4367c
|
1359 |
} |
4b272750d
|
1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 |
/* Idea of busylocks is to let producers grab an extra spinlock * to relieve pressure on the receive_queue spinlock shared by consumer. * Under flood, this means that only one producer can be in line * trying to acquire the receive_queue spinlock. * These busylock can be allocated on a per cpu manner, instead of a * per socket one (that would consume a cache line per socket) */ static int udp_busylocks_log __read_mostly; static spinlock_t *udp_busylocks __read_mostly; static spinlock_t *busylock_acquire(void *ptr) { spinlock_t *busy; busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log); spin_lock(busy); return busy; } static void busylock_release(spinlock_t *busy) { if (busy) spin_unlock(busy); } |
f970bd9e3
|
1384 1385 1386 1387 |
int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb) { struct sk_buff_head *list = &sk->sk_receive_queue; int rmem, delta, amt, err = -ENOMEM; |
4b272750d
|
1388 |
spinlock_t *busy = NULL; |
c8c8b1270
|
1389 |
int size; |
f970bd9e3
|
1390 1391 1392 1393 1394 |
/* try to avoid the costly atomic add/sub pair when the receive * queue is full; always allow at least a packet */ rmem = atomic_read(&sk->sk_rmem_alloc); |
363dc73ac
|
1395 |
if (rmem > sk->sk_rcvbuf) |
f970bd9e3
|
1396 |
goto drop; |
c8c8b1270
|
1397 1398 1399 1400 1401 1402 |
/* Under mem pressure, it might be helpful to help udp_recvmsg() * having linear skbs : * - Reduce memory overhead and thus increase receive queue capacity * - Less cache line misses at copyout() time * - Less work at consume_skb() (less alien page frag freeing) */ |
4b272750d
|
1403 |
if (rmem > (sk->sk_rcvbuf >> 1)) { |
c8c8b1270
|
1404 |
skb_condense(skb); |
4b272750d
|
1405 1406 1407 |
busy = busylock_acquire(sk); } |
c8c8b1270
|
1408 |
size = skb->truesize; |
b65ac4467
|
1409 |
udp_set_dev_scratch(skb); |
c8c8b1270
|
1410 |
|
f970bd9e3
|
1411 1412 1413 1414 |
/* we drop only if the receive buf is full and the receive * queue contains some other skb */ rmem = atomic_add_return(size, &sk->sk_rmem_alloc); |
ff4839120
|
1415 |
if (rmem > (size + (unsigned int)sk->sk_rcvbuf)) |
f970bd9e3
|
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 |
goto uncharge_drop; spin_lock(&list->lock); if (size >= sk->sk_forward_alloc) { amt = sk_mem_pages(size); delta = amt << SK_MEM_QUANTUM_SHIFT; if (!__sk_mem_raise_allocated(sk, delta, amt, SK_MEM_RECV)) { err = -ENOBUFS; spin_unlock(&list->lock); goto uncharge_drop; } sk->sk_forward_alloc += delta; } sk->sk_forward_alloc -= size; |
7c13f97ff
|
1432 1433 1434 |
/* no need to setup a destructor, we will explicitly release the * forward allocated memory on dequeue */ |
f970bd9e3
|
1435 1436 1437 1438 1439 1440 1441 |
sock_skb_set_dropcount(sk, skb); __skb_queue_tail(list, skb); spin_unlock(&list->lock); if (!sock_flag(sk, SOCK_DEAD)) sk->sk_data_ready(sk); |
4b272750d
|
1442 |
busylock_release(busy); |
f970bd9e3
|
1443 1444 1445 1446 1447 1448 1449 |
return 0; uncharge_drop: atomic_sub(skb->truesize, &sk->sk_rmem_alloc); drop: atomic_inc(&sk->sk_drops); |
4b272750d
|
1450 |
busylock_release(busy); |
f970bd9e3
|
1451 1452 1453 |
return err; } EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb); |
c915fe13c
|
1454 |
void udp_destruct_sock(struct sock *sk) |
f970bd9e3
|
1455 1456 |
{ /* reclaim completely the forward allocated memory */ |
2276f58ac
|
1457 |
struct udp_sock *up = udp_sk(sk); |
7c13f97ff
|
1458 1459 |
unsigned int total = 0; struct sk_buff *skb; |
2276f58ac
|
1460 1461 |
skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue); while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) { |
7c13f97ff
|
1462 1463 1464 |
total += skb->truesize; kfree_skb(skb); } |
6dfb4367c
|
1465 |
udp_rmem_release(sk, total, 0, true); |
7c13f97ff
|
1466 |
|
f970bd9e3
|
1467 1468 |
inet_sock_destruct(sk); } |
c915fe13c
|
1469 |
EXPORT_SYMBOL_GPL(udp_destruct_sock); |
f970bd9e3
|
1470 1471 1472 |
int udp_init_sock(struct sock *sk) { |
2276f58ac
|
1473 |
skb_queue_head_init(&udp_sk(sk)->reader_queue); |
f970bd9e3
|
1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 |
sk->sk_destruct = udp_destruct_sock; return 0; } EXPORT_SYMBOL_GPL(udp_init_sock); void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len) { if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) { bool slow = lock_sock_fast(sk); sk_peek_offset_bwd(sk, len); unlock_sock_fast(sk, slow); } |
0a463c78d
|
1487 |
|
ca2c1418e
|
1488 1489 |
if (!skb_unref(skb)) return; |
dce4551cb
|
1490 1491 |
/* In the more common cases we cleared the head states previously, * see __udp_queue_rcv_skb(). |
0ddf3fb2c
|
1492 |
*/ |
dce4551cb
|
1493 |
if (unlikely(udp_skb_has_head_state(skb))) |
0ddf3fb2c
|
1494 |
skb_release_head_state(skb); |
ca2c1418e
|
1495 |
__consume_stateless_skb(skb); |
f970bd9e3
|
1496 1497 |
} EXPORT_SYMBOL_GPL(skb_consume_udp); |
2276f58ac
|
1498 1499 1500 1501 1502 |
static struct sk_buff *__first_packet_length(struct sock *sk, struct sk_buff_head *rcvq, int *total) { struct sk_buff *skb; |
9bd780f5e
|
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 |
while ((skb = skb_peek(rcvq)) != NULL) { if (udp_lib_checksum_complete(skb)) { __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, IS_UDPLITE(sk)); __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, IS_UDPLITE(sk)); atomic_inc(&sk->sk_drops); __skb_unlink(skb, rcvq); *total += skb->truesize; kfree_skb(skb); } else { |
a793183ca
|
1514 |
udp_skb_csum_unnecessary_set(skb); |
9bd780f5e
|
1515 1516 |
break; } |
2276f58ac
|
1517 1518 1519 |
} return skb; } |
855846720
|
1520 1521 1522 1523 1524 |
/** * first_packet_length - return length of first packet in receive queue * @sk: socket * * Drops all bad checksum frames, until a valid one is found. |
e83c6744e
|
1525 |
* Returns the length of found skb, or -1 if none is found. |
855846720
|
1526 |
*/ |
e83c6744e
|
1527 |
static int first_packet_length(struct sock *sk) |
855846720
|
1528 |
{ |
2276f58ac
|
1529 1530 |
struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue; struct sk_buff_head *sk_queue = &sk->sk_receive_queue; |
855846720
|
1531 |
struct sk_buff *skb; |
7c13f97ff
|
1532 |
int total = 0; |
e83c6744e
|
1533 |
int res; |
855846720
|
1534 |
|
855846720
|
1535 |
spin_lock_bh(&rcvq->lock); |
2276f58ac
|
1536 |
skb = __first_packet_length(sk, rcvq, &total); |
137a0dbe3
|
1537 |
if (!skb && !skb_queue_empty_lockless(sk_queue)) { |
2276f58ac
|
1538 1539 1540 1541 1542 |
spin_lock(&sk_queue->lock); skb_queue_splice_tail_init(sk_queue, rcvq); spin_unlock(&sk_queue->lock); skb = __first_packet_length(sk, rcvq, &total); |
855846720
|
1543 |
} |
e83c6744e
|
1544 |
res = skb ? skb->len : -1; |
7c13f97ff
|
1545 |
if (total) |
6dfb4367c
|
1546 |
udp_rmem_release(sk, total, 1, false); |
855846720
|
1547 |
spin_unlock_bh(&rcvq->lock); |
855846720
|
1548 1549 |
return res; } |
1da177e4c
|
1550 1551 1552 |
/* * IOCTL requests applicable to the UDP protocol */ |
e905a9eda
|
1553 |
|
1da177e4c
|
1554 1555 |
int udp_ioctl(struct sock *sk, int cmd, unsigned long arg) { |
6516c6557
|
1556 1557 |
switch (cmd) { case SIOCOUTQ: |
1da177e4c
|
1558 |
{ |
31e6d363a
|
1559 |
int amount = sk_wmem_alloc_get(sk); |
6516c6557
|
1560 1561 |
return put_user(amount, (int __user *)arg); } |
1da177e4c
|
1562 |
|
6516c6557
|
1563 1564 |
case SIOCINQ: { |
e83c6744e
|
1565 |
int amount = max_t(int, 0, first_packet_length(sk)); |
6516c6557
|
1566 |
|
6516c6557
|
1567 1568 |
return put_user(amount, (int __user *)arg); } |
1da177e4c
|
1569 |
|
6516c6557
|
1570 1571 |
default: return -ENOIOCTLCMD; |
1da177e4c
|
1572 |
} |
6516c6557
|
1573 1574 |
return 0; |
1da177e4c
|
1575 |
} |
c482c5685
|
1576 |
EXPORT_SYMBOL(udp_ioctl); |
1da177e4c
|
1577 |
|
2276f58ac
|
1578 |
struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags, |
fd69c399c
|
1579 |
int noblock, int *off, int *err) |
2276f58ac
|
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 |
{ struct sk_buff_head *sk_queue = &sk->sk_receive_queue; struct sk_buff_head *queue; struct sk_buff *last; long timeo; int error; queue = &udp_sk(sk)->reader_queue; flags |= noblock ? MSG_DONTWAIT : 0; timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); do { struct sk_buff *skb; error = sock_error(sk); if (error) break; error = -EAGAIN; |
2276f58ac
|
1598 |
do { |
2276f58ac
|
1599 1600 1601 |
spin_lock_bh(&queue->lock); skb = __skb_try_recv_from_queue(sk, queue, flags, udp_skb_destructor, |
fd69c399c
|
1602 |
off, err, &last); |
2276f58ac
|
1603 1604 |
if (skb) { spin_unlock_bh(&queue->lock); |
2276f58ac
|
1605 1606 |
return skb; } |
137a0dbe3
|
1607 |
if (skb_queue_empty_lockless(sk_queue)) { |
2276f58ac
|
1608 1609 1610 |
spin_unlock_bh(&queue->lock); goto busy_check; } |
6dfb4367c
|
1611 1612 1613 1614 1615 |
/* refill the reader queue and walk it again * keep both queues locked to avoid re-acquiring * the sk_receive_queue lock if fwd memory scheduling * is needed. */ |
2276f58ac
|
1616 1617 |
spin_lock(&sk_queue->lock); skb_queue_splice_tail_init(sk_queue, queue); |
2276f58ac
|
1618 1619 |
skb = __skb_try_recv_from_queue(sk, queue, flags, |
6dfb4367c
|
1620 |
udp_skb_dtor_locked, |
fd69c399c
|
1621 |
off, err, &last); |
6dfb4367c
|
1622 |
spin_unlock(&sk_queue->lock); |
2276f58ac
|
1623 |
spin_unlock_bh(&queue->lock); |
de321ed38
|
1624 |
if (skb) |
2276f58ac
|
1625 |
return skb; |
2276f58ac
|
1626 1627 1628 1629 1630 1631 |
busy_check: if (!sk_can_busy_loop(sk)) break; sk_busy_loop(sk, flags & MSG_DONTWAIT); |
137a0dbe3
|
1632 |
} while (!skb_queue_empty_lockless(sk_queue)); |
2276f58ac
|
1633 1634 1635 1636 1637 1638 1639 1640 1641 |
/* sk_queue is empty, reader_queue may contain peeked packets */ } while (timeo && !__skb_wait_for_more_packets(sk, &error, &timeo, (struct sk_buff *)sk_queue)); *err = error; return NULL; } |
7e823644b
|
1642 |
EXPORT_SYMBOL(__skb_recv_udp); |
2276f58ac
|
1643 |
|
db8dac20d
|
1644 1645 1646 1647 |
/* * This should be easy, if there is something there we * return it, otherwise we block. */ |
1b7841404
|
1648 1649 |
int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock, int flags, int *addr_len) |
db8dac20d
|
1650 1651 |
{ struct inet_sock *inet = inet_sk(sk); |
342dfc306
|
1652 |
DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); |
db8dac20d
|
1653 |
struct sk_buff *skb; |
59c2cdae2
|
1654 |
unsigned int ulen, copied; |
fd69c399c
|
1655 |
int off, err, peeking = flags & MSG_PEEK; |
db8dac20d
|
1656 |
int is_udplite = IS_UDPLITE(sk); |
197c949e7
|
1657 |
bool checksum_valid = false; |
db8dac20d
|
1658 |
|
db8dac20d
|
1659 |
if (flags & MSG_ERRQUEUE) |
85fbaa750
|
1660 |
return ip_recv_error(sk, msg, len, addr_len); |
db8dac20d
|
1661 1662 |
try_again: |
a0917e0bc
|
1663 |
off = sk_peek_offset(sk, flags); |
fd69c399c
|
1664 |
skb = __skb_recv_udp(sk, flags, noblock, &off, &err); |
db8dac20d
|
1665 |
if (!skb) |
627d2d6b5
|
1666 |
return err; |
db8dac20d
|
1667 |
|
b65ac4467
|
1668 |
ulen = udp_skb_len(skb); |
59c2cdae2
|
1669 |
copied = len; |
627d2d6b5
|
1670 1671 |
if (copied > ulen - off) copied = ulen - off; |
59c2cdae2
|
1672 |
else if (copied < ulen) |
db8dac20d
|
1673 1674 1675 1676 1677 1678 1679 |
msg->msg_flags |= MSG_TRUNC; /* * If checksum is needed at all, try to do it while copying the * data. If the data is truncated, or if we only want a partial * coverage checksum (UDP-Lite), do it before the copy. */ |
d21dbdfe0
|
1680 1681 |
if (copied < ulen || peeking || (is_udplite && UDP_SKB_CB(skb)->partial_cov)) { |
b65ac4467
|
1682 1683 |
checksum_valid = udp_skb_csum_unnecessary(skb) || !__udp_lib_checksum_complete(skb); |
197c949e7
|
1684 |
if (!checksum_valid) |
db8dac20d
|
1685 1686 |
goto csum_copy_err; } |
b65ac4467
|
1687 1688 1689 1690 1691 1692 |
if (checksum_valid || udp_skb_csum_unnecessary(skb)) { if (udp_skb_is_linear(skb)) err = copy_linear_skb(skb, copied, off, &msg->msg_iter); else err = skb_copy_datagram_msg(skb, off, msg, copied); } else { |
627d2d6b5
|
1693 |
err = skb_copy_and_csum_datagram_msg(skb, off, msg); |
db8dac20d
|
1694 1695 1696 1697 |
if (err == -EINVAL) goto csum_copy_err; } |
22911fc58
|
1698 |
if (unlikely(err)) { |
fd69c399c
|
1699 |
if (!peeking) { |
979402b16
|
1700 |
atomic_inc(&sk->sk_drops); |
6aef70a85
|
1701 1702 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); |
979402b16
|
1703 |
} |
850cbaddb
|
1704 |
kfree_skb(skb); |
627d2d6b5
|
1705 |
return err; |
22911fc58
|
1706 |
} |
db8dac20d
|
1707 |
|
fd69c399c
|
1708 |
if (!peeking) |
6aef70a85
|
1709 1710 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS, is_udplite); |
db8dac20d
|
1711 |
|
3b885787e
|
1712 |
sock_recv_ts_and_drops(msg, sk, skb); |
db8dac20d
|
1713 1714 |
/* Copy the address. */ |
c482c5685
|
1715 |
if (sin) { |
db8dac20d
|
1716 1717 1718 1719 |
sin->sin_family = AF_INET; sin->sin_port = udp_hdr(skb)->source; sin->sin_addr.s_addr = ip_hdr(skb)->saddr; memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); |
bceaa9024
|
1720 |
*addr_len = sizeof(*sin); |
983695fa6
|
1721 1722 1723 1724 |
if (cgroup_bpf_enabled) BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, (struct sockaddr *)sin); |
db8dac20d
|
1725 |
} |
bcd1665e3
|
1726 1727 1728 |
if (udp_sk(sk)->gro_enabled) udp_cmsg_recv(msg, sk, skb); |
db8dac20d
|
1729 |
if (inet->cmsg_flags) |
ad959036a
|
1730 |
ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off); |
db8dac20d
|
1731 |
|
59c2cdae2
|
1732 |
err = copied; |
db8dac20d
|
1733 1734 |
if (flags & MSG_TRUNC) err = ulen; |
850cbaddb
|
1735 |
skb_consume_udp(sk, skb, peeking ? -err : err); |
db8dac20d
|
1736 1737 1738 |
return err; csum_copy_err: |
2276f58ac
|
1739 1740 |
if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags, udp_skb_destructor)) { |
6aef70a85
|
1741 1742 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); |
6a5dc9e59
|
1743 |
} |
850cbaddb
|
1744 |
kfree_skb(skb); |
db8dac20d
|
1745 |
|
beb39db59
|
1746 1747 |
/* starting over for a new packet, but check if we need to yield */ cond_resched(); |
9cfaa8def
|
1748 |
msg->msg_flags &= ~MSG_TRUNC; |
db8dac20d
|
1749 1750 |
goto try_again; } |
d74bad4e7
|
1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 |
int udp_pre_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) { /* This check is replicated from __ip4_datagram_connect() and * intended to prevent BPF program called below from accessing bytes * that are out of the bound specified by user in addr_len. */ if (addr_len < sizeof(struct sockaddr_in)) return -EINVAL; return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr); } EXPORT_SYMBOL(udp_pre_connect); |
286c72dea
|
1763 |
int __udp_disconnect(struct sock *sk, int flags) |
1da177e4c
|
1764 1765 1766 1767 1768 |
{ struct inet_sock *inet = inet_sk(sk); /* * 1003.1g - break association. */ |
e905a9eda
|
1769 |
|
1da177e4c
|
1770 |
sk->sk_state = TCP_CLOSE; |
c720c7e83
|
1771 1772 |
inet->inet_daddr = 0; inet->inet_dport = 0; |
bdeab9919
|
1773 |
sock_rps_reset_rxhash(sk); |
1da177e4c
|
1774 |
sk->sk_bound_dev_if = 0; |
e72258bf6
|
1775 |
if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) { |
1da177e4c
|
1776 |
inet_reset_saddr(sk); |
e72258bf6
|
1777 1778 1779 1780 |
if (sk->sk_prot->rehash && (sk->sk_userlocks & SOCK_BINDPORT_LOCK)) sk->sk_prot->rehash(sk); } |
1da177e4c
|
1781 1782 1783 |
if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) { sk->sk_prot->unhash(sk); |
c720c7e83
|
1784 |
inet->inet_sport = 0; |
1da177e4c
|
1785 1786 1787 1788 |
} sk_dst_reset(sk); return 0; } |
286c72dea
|
1789 1790 1791 1792 1793 1794 1795 1796 1797 |
EXPORT_SYMBOL(__udp_disconnect); int udp_disconnect(struct sock *sk, int flags) { lock_sock(sk); __udp_disconnect(sk, flags); release_sock(sk); return 0; } |
c482c5685
|
1798 |
EXPORT_SYMBOL(udp_disconnect); |
1da177e4c
|
1799 |
|
645ca708f
|
1800 1801 |
void udp_lib_unhash(struct sock *sk) { |
723b46108
|
1802 1803 |
if (sk_hashed(sk)) { struct udp_table *udptable = sk->sk_prot->h.udp_table; |
512615b6b
|
1804 1805 1806 1807 1808 |
struct udp_hslot *hslot, *hslot2; hslot = udp_hashslot(udptable, sock_net(sk), udp_sk(sk)->udp_port_hash); hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); |
645ca708f
|
1809 |
|
723b46108
|
1810 |
spin_lock_bh(&hslot->lock); |
e32ea7e74
|
1811 1812 |
if (rcu_access_pointer(sk->sk_reuseport_cb)) reuseport_detach_sock(sk); |
ca065d0cf
|
1813 |
if (sk_del_node_init_rcu(sk)) { |
fdcc8aa95
|
1814 |
hslot->count--; |
c720c7e83
|
1815 |
inet_sk(sk)->inet_num = 0; |
723b46108
|
1816 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); |
512615b6b
|
1817 1818 |
spin_lock(&hslot2->lock); |
ca065d0cf
|
1819 |
hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); |
512615b6b
|
1820 1821 |
hslot2->count--; spin_unlock(&hslot2->lock); |
723b46108
|
1822 1823 |
} spin_unlock_bh(&hslot->lock); |
645ca708f
|
1824 |
} |
645ca708f
|
1825 1826 |
} EXPORT_SYMBOL(udp_lib_unhash); |
719f83585
|
1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 |
/* * inet_rcv_saddr was changed, we must rehash secondary hash */ void udp_lib_rehash(struct sock *sk, u16 newhash) { if (sk_hashed(sk)) { struct udp_table *udptable = sk->sk_prot->h.udp_table; struct udp_hslot *hslot, *hslot2, *nhslot2; hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); nhslot2 = udp_hashslot2(udptable, newhash); udp_sk(sk)->udp_portaddr_hash = newhash; |
e32ea7e74
|
1839 1840 1841 |
if (hslot2 != nhslot2 || rcu_access_pointer(sk->sk_reuseport_cb)) { |
719f83585
|
1842 1843 1844 1845 |
hslot = udp_hashslot(udptable, sock_net(sk), udp_sk(sk)->udp_port_hash); /* we must lock primary chain too */ spin_lock_bh(&hslot->lock); |
e32ea7e74
|
1846 1847 1848 1849 1850 |
if (rcu_access_pointer(sk->sk_reuseport_cb)) reuseport_detach_sock(sk); if (hslot2 != nhslot2) { spin_lock(&hslot2->lock); |
ca065d0cf
|
1851 |
hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); |
e32ea7e74
|
1852 1853 1854 1855 |
hslot2->count--; spin_unlock(&hslot2->lock); spin_lock(&nhslot2->lock); |
ca065d0cf
|
1856 |
hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, |
e32ea7e74
|
1857 1858 1859 1860 |
&nhslot2->head); nhslot2->count++; spin_unlock(&nhslot2->lock); } |
719f83585
|
1861 1862 1863 1864 1865 1866 |
spin_unlock_bh(&hslot->lock); } } } EXPORT_SYMBOL(udp_lib_rehash); |
8f6b53928
|
1867 |
void udp_v4_rehash(struct sock *sk) |
719f83585
|
1868 |
{ |
f0b1e64c1
|
1869 |
u16 new_hash = ipv4_portaddr_hash(sock_net(sk), |
719f83585
|
1870 1871 1872 1873 |
inet_sk(sk)->inet_rcv_saddr, inet_sk(sk)->inet_num); udp_lib_rehash(sk, new_hash); } |
a3f96c47c
|
1874 |
static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) |
93821778d
|
1875 |
{ |
fec5e652e
|
1876 |
int rc; |
766e9037c
|
1877 |
|
005ec9743
|
1878 |
if (inet_sk(sk)->inet_daddr) { |
bdeab9919
|
1879 |
sock_rps_save_rxhash(sk, skb); |
005ec9743
|
1880 |
sk_mark_napi_id(sk, skb); |
2c8c56e15
|
1881 |
sk_incoming_cpu_update(sk); |
e68b6e50f
|
1882 1883 |
} else { sk_mark_napi_id_once(sk, skb); |
005ec9743
|
1884 |
} |
fec5e652e
|
1885 |
|
850cbaddb
|
1886 |
rc = __udp_enqueue_schedule_skb(sk, skb); |
766e9037c
|
1887 1888 |
if (rc < 0) { int is_udplite = IS_UDPLITE(sk); |
93821778d
|
1889 |
|
93821778d
|
1890 |
/* Note that an ENOMEM error is charged twice */ |
766e9037c
|
1891 |
if (rc == -ENOMEM) |
e61da9e25
|
1892 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS, |
02c223470
|
1893 |
is_udplite); |
e61da9e25
|
1894 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); |
766e9037c
|
1895 |
kfree_skb(skb); |
296f7ea75
|
1896 |
trace_udp_fail_queue_rcv_skb(rc, sk); |
766e9037c
|
1897 |
return -1; |
93821778d
|
1898 1899 1900 |
} return 0; |
93821778d
|
1901 |
} |
db8dac20d
|
1902 1903 1904 1905 1906 1907 1908 1909 |
/* returns: * -1: error * 0: success * >0: "udp encap" protocol resubmission * * Note that in the success and error cases, the skb is assumed to * have either been requeued or freed. */ |
cf329aa42
|
1910 |
static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb) |
db8dac20d
|
1911 1912 |
{ struct udp_sock *up = udp_sk(sk); |
db8dac20d
|
1913 1914 1915 1916 1917 1918 1919 |
int is_udplite = IS_UDPLITE(sk); /* * Charge it to the socket, dropping if the queue is full. */ if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) goto drop; |
895b5c9f2
|
1920 |
nf_reset_ct(skb); |
db8dac20d
|
1921 |
|
88ab31081
|
1922 |
if (static_branch_unlikely(&udp_encap_needed_key) && up->encap_type) { |
0ad92ad03
|
1923 |
int (*encap_rcv)(struct sock *sk, struct sk_buff *skb); |
db8dac20d
|
1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 |
/* * This is an encapsulation socket so pass the skb to * the socket's udp_encap_rcv() hook. Otherwise, just * fall through and pass this up the UDP socket. * up->encap_rcv() returns the following value: * =0 if skb was successfully passed to the encap * handler or was discarded by it. * >0 if skb should be passed on to UDP. * <0 if skb should be resubmitted as proto -N */ /* if we're overly short, let UDP handle it */ |
6aa7de059
|
1936 |
encap_rcv = READ_ONCE(up->encap_rcv); |
e5aed006b
|
1937 |
if (encap_rcv) { |
db8dac20d
|
1938 |
int ret; |
0a80966b1
|
1939 1940 1941 |
/* Verify checksum before giving to encap */ if (udp_lib_checksum_complete(skb)) goto csum_error; |
0ad92ad03
|
1942 |
ret = encap_rcv(sk, skb); |
db8dac20d
|
1943 |
if (ret <= 0) { |
02c223470
|
1944 1945 1946 |
__UDP_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS, is_udplite); |
db8dac20d
|
1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 |
return -ret; } } /* FALLTHROUGH -- it's a UDP Packet */ } /* * UDP-Lite specific tests, ignored on UDP sockets */ if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) { /* * MIB statistics other than incrementing the error count are * disabled for the following two types of errors: these depend * on the application settings, not on the functioning of the * protocol stack as such. * * RFC 3828 here recommends (sec 3.3): "There should also be a * way ... to ... at least let the receiving application block * delivery of packets with coverage values less than a value * provided by the application." */ if (up->pcrlen == 0) { /* full coverage was set */ |
ba7a46f16
|
1971 1972 1973 |
net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested ", UDP_SKB_CB(skb)->cscov, skb->len); |
db8dac20d
|
1974 1975 1976 1977 1978 1979 1980 1981 1982 |
goto drop; } /* The next case involves violating the min. coverage requested * by the receiver. This is subtle: if receiver wants x and x is * greater than the buffersize/MTU then receiver will complain * that it wants x while sender emits packets of smaller size y. * Therefore the above ...()->partial_cov statement is essential. */ if (UDP_SKB_CB(skb)->cscov < up->pcrlen) { |
ba7a46f16
|
1983 1984 1985 |
net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d ", UDP_SKB_CB(skb)->cscov, up->pcrlen); |
db8dac20d
|
1986 1987 1988 |
goto drop; } } |
dd99e425b
|
1989 |
prefetch(&sk->sk_rmem_alloc); |
ce25d66ad
|
1990 1991 |
if (rcu_access_pointer(sk->sk_filter) && udp_lib_checksum_complete(skb)) |
e6afc8ace
|
1992 |
goto csum_error; |
ce25d66ad
|
1993 |
|
ba66bbe54
|
1994 |
if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr))) |
a61276977
|
1995 |
goto drop; |
db8dac20d
|
1996 |
|
e6afc8ace
|
1997 |
udp_csum_pull_header(skb); |
db8dac20d
|
1998 |
|
fbf8866d6
|
1999 |
ipv4_pktinfo_prepare(sk, skb); |
850cbaddb
|
2000 |
return __udp_queue_rcv_skb(sk, skb); |
db8dac20d
|
2001 |
|
6a5dc9e59
|
2002 |
csum_error: |
02c223470
|
2003 |
__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); |
db8dac20d
|
2004 |
drop: |
02c223470
|
2005 |
__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); |
8edf19c2f
|
2006 |
atomic_inc(&sk->sk_drops); |
db8dac20d
|
2007 2008 2009 |
kfree_skb(skb); return -1; } |
cf329aa42
|
2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 |
static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) { struct sk_buff *next, *segs; int ret; if (likely(!udp_unexpected_gso(sk, skb))) return udp_queue_rcv_one_skb(sk, skb); BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_SGO_CB_OFFSET); __skb_push(skb, -skb_mac_offset(skb)); segs = udp_rcv_segment(sk, skb, true); for (skb = segs; skb; skb = next) { next = skb->next; __skb_pull(skb, skb_transport_offset(skb)); ret = udp_queue_rcv_one_skb(sk, skb); if (ret > 0) ip_protocol_deliver_rcu(dev_net(skb->dev), skb, -ret); } return 0; } |
975022310
|
2030 |
/* For TCP sockets, sk_rx_dst is protected by socket lock |
e47eb5dfb
|
2031 |
* For UDP, we use xchg() to guard against concurrent changes. |
975022310
|
2032 |
*/ |
64f0f5d18
|
2033 |
bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst) |
421b3885b
|
2034 |
{ |
975022310
|
2035 |
struct dst_entry *old; |
d24406c85
|
2036 2037 2038 |
if (dst_hold_safe(dst)) { old = xchg(&sk->sk_rx_dst, dst); dst_release(old); |
64f0f5d18
|
2039 |
return old != dst; |
d24406c85
|
2040 |
} |
64f0f5d18
|
2041 |
return false; |
421b3885b
|
2042 |
} |
c9f2c1ae1
|
2043 |
EXPORT_SYMBOL(udp_sk_rx_dst_set); |
421b3885b
|
2044 |
|
db8dac20d
|
2045 2046 2047 |
/* * Multicasts and broadcasts go to each listener. * |
1240d1373
|
2048 |
* Note: called only from the BH handler context. |
db8dac20d
|
2049 |
*/ |
e31634931
|
2050 |
static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, |
db8dac20d
|
2051 2052 |
struct udphdr *uh, __be32 saddr, __be32 daddr, |
36cbb2452
|
2053 2054 |
struct udp_table *udptable, int proto) |
db8dac20d
|
2055 |
{ |
ca065d0cf
|
2056 |
struct sock *sk, *first = NULL; |
5cf3d4619
|
2057 2058 |
unsigned short hnum = ntohs(uh->dest); struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum); |
2dc41cff7
|
2059 |
unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10); |
ca065d0cf
|
2060 2061 |
unsigned int offset = offsetof(typeof(*sk), sk_node); int dif = skb->dev->ifindex; |
fb74c2773
|
2062 |
int sdif = inet_sdif(skb); |
ca065d0cf
|
2063 2064 |
struct hlist_node *node; struct sk_buff *nskb; |
2dc41cff7
|
2065 2066 |
if (use_hash2) { |
f0b1e64c1
|
2067 |
hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) & |
73e2d5e34
|
2068 |
udptable->mask; |
f0b1e64c1
|
2069 |
hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask; |
2dc41cff7
|
2070 |
start_lookup: |
73e2d5e34
|
2071 |
hslot = &udptable->hash2[hash2]; |
2dc41cff7
|
2072 2073 |
offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node); } |
db8dac20d
|
2074 |
|
ca065d0cf
|
2075 2076 |
sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) { if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr, |
fb74c2773
|
2077 |
uh->source, saddr, dif, sdif, hnum)) |
ca065d0cf
|
2078 2079 2080 2081 2082 |
continue; if (!first) { first = sk; continue; |
1240d1373
|
2083 |
} |
ca065d0cf
|
2084 |
nskb = skb_clone(skb, GFP_ATOMIC); |
1240d1373
|
2085 |
|
ca065d0cf
|
2086 2087 |
if (unlikely(!nskb)) { atomic_inc(&sk->sk_drops); |
02c223470
|
2088 2089 2090 2091 |
__UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS, IS_UDPLITE(sk)); __UDP_INC_STATS(net, UDP_MIB_INERRORS, IS_UDPLITE(sk)); |
ca065d0cf
|
2092 2093 2094 2095 2096 |
continue; } if (udp_queue_rcv_skb(sk, nskb) > 0) consume_skb(nskb); } |
1240d1373
|
2097 |
|
2dc41cff7
|
2098 2099 2100 2101 2102 |
/* Also lookup *:port if we are using hash2 and haven't done so yet. */ if (use_hash2 && hash2 != hash2_any) { hash2 = hash2_any; goto start_lookup; } |
ca065d0cf
|
2103 2104 2105 |
if (first) { if (udp_queue_rcv_skb(first, skb) > 0) consume_skb(skb); |
1240d1373
|
2106 |
} else { |
ca065d0cf
|
2107 |
kfree_skb(skb); |
02c223470
|
2108 2109 |
__UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI, proto == IPPROTO_UDPLITE); |
1240d1373
|
2110 |
} |
db8dac20d
|
2111 2112 2113 2114 2115 |
return 0; } /* Initialize UDP checksum. If exited with zero value (success), * CHECKSUM_UNNECESSARY means, that no more checks are required. |
666a3d6e1
|
2116 |
* Otherwise, csum completion requires checksumming packet body, |
db8dac20d
|
2117 2118 2119 2120 2121 |
* including udp header and folding it to skb->csum. */ static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh, int proto) { |
db8dac20d
|
2122 2123 2124 2125 2126 2127 2128 2129 2130 |
int err; UDP_SKB_CB(skb)->partial_cov = 0; UDP_SKB_CB(skb)->cscov = skb->len; if (proto == IPPROTO_UDPLITE) { err = udplite_checksum_init(skb, uh); if (err) return err; |
15f35d49c
|
2131 2132 2133 2134 2135 |
if (UDP_SKB_CB(skb)->partial_cov) { skb->csum = inet_compute_pseudo(skb, proto); return 0; } |
db8dac20d
|
2136 |
} |
b46d9f625
|
2137 2138 2139 |
/* Note, we are only interested in != 0 or == 0, thus the * force to int. */ |
db4f1be3c
|
2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 |
err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check, inet_compute_pseudo); if (err) return err; if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) { /* If SW calculated the value, we know it's bad */ if (skb->csum_complete_sw) return 1; /* HW says the value is bad. Let's validate that. * skb->csum is no longer the full packet checksum, * so don't treat it as such. */ skb_checksum_complete_unset(skb); } return 0; |
db8dac20d
|
2158 |
} |
2b5a92174
|
2159 2160 2161 2162 2163 2164 2165 2166 2167 |
/* wrapper for udp_queue_rcv_skb tacking care of csum conversion and * return code conversion for ip layer consumption */ static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb, struct udphdr *uh) { int ret; if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk)) |
e4aa33ad5
|
2168 |
skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo); |
2b5a92174
|
2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 |
ret = udp_queue_rcv_skb(sk, skb); /* a return value > 0 means to resubmit the input, but * it wants the return to be -protocol, or 0 */ if (ret > 0) return -ret; return 0; } |
db8dac20d
|
2179 2180 2181 |
/* * All we need to do is get the socket, and then do a checksum. */ |
645ca708f
|
2182 |
int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, |
db8dac20d
|
2183 2184 2185 |
int proto) { struct sock *sk; |
7b5e56f9d
|
2186 |
struct udphdr *uh; |
db8dac20d
|
2187 |
unsigned short ulen; |
adf30907d
|
2188 |
struct rtable *rt = skb_rtable(skb); |
2783ef231
|
2189 |
__be32 saddr, daddr; |
0283328e2
|
2190 |
struct net *net = dev_net(skb->dev); |
db8dac20d
|
2191 2192 2193 2194 2195 2196 |
/* * Validate the packet. */ if (!pskb_may_pull(skb, sizeof(struct udphdr))) goto drop; /* No space for header. */ |
7b5e56f9d
|
2197 |
uh = udp_hdr(skb); |
db8dac20d
|
2198 |
ulen = ntohs(uh->len); |
ccc2d97cb
|
2199 2200 |
saddr = ip_hdr(skb)->saddr; daddr = ip_hdr(skb)->daddr; |
db8dac20d
|
2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 |
if (ulen > skb->len) goto short_packet; if (proto == IPPROTO_UDP) { /* UDP validates ulen. */ if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen)) goto short_packet; uh = udp_hdr(skb); } if (udp4_csum_init(skb, uh, proto)) goto csum_error; |
8afdd99a1
|
2213 2214 |
sk = skb_steal_sock(skb); if (sk) { |
975022310
|
2215 |
struct dst_entry *dst = skb_dst(skb); |
421b3885b
|
2216 |
int ret; |
421b3885b
|
2217 |
|
975022310
|
2218 2219 |
if (unlikely(sk->sk_rx_dst != dst)) udp_sk_rx_dst_set(sk, dst); |
db8dac20d
|
2220 |
|
2b5a92174
|
2221 |
ret = udp_unicast_rcv_skb(sk, skb, uh); |
8afdd99a1
|
2222 |
sock_put(sk); |
2b5a92174
|
2223 |
return ret; |
421b3885b
|
2224 |
} |
db8dac20d
|
2225 |
|
c18450a52
|
2226 2227 |
if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST)) return __udp4_lib_mcast_deliver(net, skb, uh, |
36cbb2452
|
2228 |
saddr, daddr, udptable, proto); |
c18450a52
|
2229 2230 |
sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable); |
2b5a92174
|
2231 2232 |
if (sk) return udp_unicast_rcv_skb(sk, skb, uh); |
db8dac20d
|
2233 2234 2235 |
if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) goto drop; |
895b5c9f2
|
2236 |
nf_reset_ct(skb); |
db8dac20d
|
2237 2238 2239 2240 |
/* No socket. Drop packet silently, if checksum is wrong */ if (udp_lib_checksum_complete(skb)) goto csum_error; |
02c223470
|
2241 |
__UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE); |
db8dac20d
|
2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 |
icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0); /* * Hmm. We got an UDP packet to a port to which we * don't wanna listen. Ignore it. */ kfree_skb(skb); return 0; short_packet: |
ba7a46f16
|
2252 2253 2254 2255 2256 2257 |
net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u ", proto == IPPROTO_UDPLITE ? "Lite" : "", &saddr, ntohs(uh->source), ulen, skb->len, &daddr, ntohs(uh->dest)); |
db8dac20d
|
2258 2259 2260 2261 2262 2263 2264 |
goto drop; csum_error: /* * RFC1122: OK. Discards the bad packet silently (as far as * the network is concerned, anyway) as per 4.1.3.4 (MUST). */ |
ba7a46f16
|
2265 2266 2267 2268 2269 |
net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d ", proto == IPPROTO_UDPLITE ? "Lite" : "", &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest), ulen); |
02c223470
|
2270 |
__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE); |
db8dac20d
|
2271 |
drop: |
02c223470
|
2272 |
__UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE); |
db8dac20d
|
2273 2274 2275 |
kfree_skb(skb); return 0; } |
421b3885b
|
2276 2277 2278 2279 2280 2281 |
/* We can only early demux multicast if there is a single matching socket. * If more than one socket found returns NULL */ static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net, __be16 loc_port, __be32 loc_addr, __be16 rmt_port, __be32 rmt_addr, |
fb74c2773
|
2282 |
int dif, int sdif) |
421b3885b
|
2283 2284 |
{ struct sock *sk, *result; |
421b3885b
|
2285 |
unsigned short hnum = ntohs(loc_port); |
ca065d0cf
|
2286 |
unsigned int slot = udp_hashfn(net, hnum, udp_table.mask); |
421b3885b
|
2287 |
struct udp_hslot *hslot = &udp_table.hash[slot]; |
63c6f81cd
|
2288 2289 2290 |
/* Do not bother scanning a too big list */ if (hslot->count > 10) return NULL; |
421b3885b
|
2291 |
result = NULL; |
ca065d0cf
|
2292 2293 |
sk_for_each_rcu(sk, &hslot->head) { if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr, |
fb74c2773
|
2294 |
rmt_port, rmt_addr, dif, sdif, hnum)) { |
ca065d0cf
|
2295 2296 |
if (result) return NULL; |
421b3885b
|
2297 |
result = sk; |
421b3885b
|
2298 2299 |
} } |
ca065d0cf
|
2300 |
|
421b3885b
|
2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 |
return result; } /* For unicast we should only early demux connected sockets or we can * break forwarding setups. The chains here can be long so only check * if the first socket is an exact match and if not move on. */ static struct sock *__udp4_lib_demux_lookup(struct net *net, __be16 loc_port, __be32 loc_addr, __be16 rmt_port, __be32 rmt_addr, |
3fa6f616a
|
2311 |
int dif, int sdif) |
421b3885b
|
2312 |
{ |
421b3885b
|
2313 |
unsigned short hnum = ntohs(loc_port); |
f0b1e64c1
|
2314 |
unsigned int hash2 = ipv4_portaddr_hash(net, loc_addr, hnum); |
421b3885b
|
2315 2316 |
unsigned int slot2 = hash2 & udp_table.mask; struct udp_hslot *hslot2 = &udp_table.hash2[slot2]; |
c72283174
|
2317 |
INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr); |
421b3885b
|
2318 |
const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum); |
ca065d0cf
|
2319 |
struct sock *sk; |
421b3885b
|
2320 |
|
ca065d0cf
|
2321 2322 |
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { if (INET_MATCH(sk, net, acookie, rmt_addr, |
3fa6f616a
|
2323 |
loc_addr, ports, dif, sdif)) |
ca065d0cf
|
2324 |
return sk; |
421b3885b
|
2325 2326 2327 |
/* Only check first socket in chain */ break; } |
ca065d0cf
|
2328 |
return NULL; |
421b3885b
|
2329 |
} |
7487449c8
|
2330 |
int udp_v4_early_demux(struct sk_buff *skb) |
421b3885b
|
2331 |
{ |
610438b74
|
2332 |
struct net *net = dev_net(skb->dev); |
bc044e8db
|
2333 |
struct in_device *in_dev = NULL; |
610438b74
|
2334 2335 |
const struct iphdr *iph; const struct udphdr *uh; |
ca065d0cf
|
2336 |
struct sock *sk = NULL; |
421b3885b
|
2337 |
struct dst_entry *dst; |
421b3885b
|
2338 |
int dif = skb->dev->ifindex; |
fb74c2773
|
2339 |
int sdif = inet_sdif(skb); |
6e5403093
|
2340 |
int ours; |
421b3885b
|
2341 2342 2343 |
/* validate the packet */ if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr))) |
7487449c8
|
2344 |
return 0; |
421b3885b
|
2345 |
|
610438b74
|
2346 2347 |
iph = ip_hdr(skb); uh = udp_hdr(skb); |
996b44fce
|
2348 |
if (skb->pkt_type == PACKET_MULTICAST) { |
bc044e8db
|
2349 |
in_dev = __in_dev_get_rcu(skb->dev); |
6e5403093
|
2350 2351 |
if (!in_dev) |
7487449c8
|
2352 |
return 0; |
6e5403093
|
2353 |
|
996b44fce
|
2354 2355 2356 2357 |
ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr, iph->protocol); if (!ours) return 0; |
ad0ea1989
|
2358 |
|
421b3885b
|
2359 |
sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr, |
fb74c2773
|
2360 2361 |
uh->source, iph->saddr, dif, sdif); |
6e5403093
|
2362 |
} else if (skb->pkt_type == PACKET_HOST) { |
421b3885b
|
2363 |
sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr, |
3fa6f616a
|
2364 |
uh->source, iph->saddr, dif, sdif); |
6e5403093
|
2365 |
} |
421b3885b
|
2366 |
|
41c6d650f
|
2367 |
if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) |
7487449c8
|
2368 |
return 0; |
421b3885b
|
2369 2370 |
skb->sk = sk; |
82eabd9eb
|
2371 |
skb->destructor = sock_efree; |
10e2eb878
|
2372 |
dst = READ_ONCE(sk->sk_rx_dst); |
421b3885b
|
2373 2374 2375 |
if (dst) dst = dst_check(dst, 0); |
10e2eb878
|
2376 |
if (dst) { |
bc044e8db
|
2377 |
u32 itag = 0; |
d24406c85
|
2378 2379 2380 2381 2382 |
/* set noref for now. * any place which wants to hold dst has to call * dst_hold_safe() */ skb_dst_set_noref(skb, dst); |
bc044e8db
|
2383 2384 2385 2386 2387 2388 2389 2390 |
/* for unconnected multicast sockets we need to validate * the source on each packet */ if (!inet_sk(sk)->inet_daddr && in_dev) return ip_mc_validate_source(skb, iph->daddr, iph->saddr, iph->tos, skb->dev, in_dev, &itag); |
10e2eb878
|
2391 |
} |
7487449c8
|
2392 |
return 0; |
421b3885b
|
2393 |
} |
db8dac20d
|
2394 2395 |
int udp_rcv(struct sk_buff *skb) { |
645ca708f
|
2396 |
return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP); |
db8dac20d
|
2397 |
} |
7d06b2e05
|
2398 |
void udp_destroy_sock(struct sock *sk) |
db8dac20d
|
2399 |
{ |
44046a593
|
2400 |
struct udp_sock *up = udp_sk(sk); |
8a74ad60a
|
2401 |
bool slow = lock_sock_fast(sk); |
db8dac20d
|
2402 |
udp_flush_pending_frames(sk); |
8a74ad60a
|
2403 |
unlock_sock_fast(sk, slow); |
60fb9567b
|
2404 2405 2406 2407 2408 2409 2410 2411 |
if (static_branch_unlikely(&udp_encap_needed_key)) { if (up->encap_type) { void (*encap_destroy)(struct sock *sk); encap_destroy = READ_ONCE(up->encap_destroy); if (encap_destroy) encap_destroy(sk); } if (up->encap_enabled) |
9c4806014
|
2412 |
static_branch_dec(&udp_encap_needed_key); |
44046a593
|
2413 |
} |
db8dac20d
|
2414 |
} |
1da177e4c
|
2415 2416 2417 |
/* * Socket option code for UDP */ |
4c0a6cb0d
|
2418 |
int udp_lib_setsockopt(struct sock *sk, int level, int optname, |
b7058842c
|
2419 |
char __user *optval, unsigned int optlen, |
4c0a6cb0d
|
2420 |
int (*push_pending_frames)(struct sock *)) |
1da177e4c
|
2421 2422 |
{ struct udp_sock *up = udp_sk(sk); |
1c19448c9
|
2423 |
int val, valbool; |
1da177e4c
|
2424 |
int err = 0; |
b2bf1e265
|
2425 |
int is_udplite = IS_UDPLITE(sk); |
1da177e4c
|
2426 |
|
c482c5685
|
2427 |
if (optlen < sizeof(int)) |
1da177e4c
|
2428 2429 2430 2431 |
return -EINVAL; if (get_user(val, (int __user *)optval)) return -EFAULT; |
1c19448c9
|
2432 |
valbool = val ? 1 : 0; |
6516c6557
|
2433 |
switch (optname) { |
1da177e4c
|
2434 2435 2436 2437 2438 2439 |
case UDP_CORK: if (val != 0) { up->corkflag = 1; } else { up->corkflag = 0; lock_sock(sk); |
4243cdc2c
|
2440 |
push_pending_frames(sk); |
1da177e4c
|
2441 2442 2443 |
release_sock(sk); } break; |
e905a9eda
|
2444 |
|
1da177e4c
|
2445 2446 2447 2448 2449 |
case UDP_ENCAP: switch (val) { case 0: case UDP_ENCAP_ESPINUDP: case UDP_ENCAP_ESPINUDP_NON_IKE: |
067b207b2
|
2450 2451 |
up->encap_rcv = xfrm4_udp_encap_rcv; /* FALLTHROUGH */ |
342f0234c
|
2452 |
case UDP_ENCAP_L2TPINUDP: |
1da177e4c
|
2453 |
up->encap_type = val; |
60fb9567b
|
2454 2455 2456 |
lock_sock(sk); udp_tunnel_encap_enable(sk->sk_socket); release_sock(sk); |
1da177e4c
|
2457 2458 2459 2460 2461 2462 |
break; default: err = -ENOPROTOOPT; break; } break; |
1c19448c9
|
2463 2464 2465 2466 2467 2468 2469 |
case UDP_NO_CHECK6_TX: up->no_check6_tx = valbool; break; case UDP_NO_CHECK6_RX: up->no_check6_rx = valbool; break; |
bec1f6f69
|
2470 2471 2472 2473 2474 |
case UDP_SEGMENT: if (val < 0 || val > USHRT_MAX) return -EINVAL; up->gso_size = val; break; |
e20cf8d3f
|
2475 2476 2477 2478 2479 2480 2481 |
case UDP_GRO: lock_sock(sk); if (valbool) udp_tunnel_encap_enable(sk->sk_socket); up->gro_enabled = valbool; release_sock(sk); break; |
ba4e58eca
|
2482 2483 2484 2485 2486 2487 |
/* * UDP-Lite's partial checksum coverage (RFC 3828). */ /* The sender sets actual checksum coverage length via this option. * The case coverage > packet length is handled by send module. */ case UDPLITE_SEND_CSCOV: |
b2bf1e265
|
2488 |
if (!is_udplite) /* Disable the option on UDP sockets */ |
ba4e58eca
|
2489 2490 2491 |
return -ENOPROTOOPT; if (val != 0 && val < 8) /* Illegal coverage: use default (8) */ val = 8; |
4be929be3
|
2492 2493 |
else if (val > USHRT_MAX) val = USHRT_MAX; |
ba4e58eca
|
2494 2495 2496 |
up->pcslen = val; up->pcflag |= UDPLITE_SEND_CC; break; |
e905a9eda
|
2497 2498 |
/* The receiver specifies a minimum checksum coverage value. To make * sense, this should be set to at least 8 (as done below). If zero is |
ba4e58eca
|
2499 2500 |
* used, this again means full checksum coverage. */ case UDPLITE_RECV_CSCOV: |
b2bf1e265
|
2501 |
if (!is_udplite) /* Disable the option on UDP sockets */ |
ba4e58eca
|
2502 2503 2504 |
return -ENOPROTOOPT; if (val != 0 && val < 8) /* Avoid silly minimal values. */ val = 8; |
4be929be3
|
2505 2506 |
else if (val > USHRT_MAX) val = USHRT_MAX; |
ba4e58eca
|
2507 2508 2509 |
up->pcrlen = val; up->pcflag |= UDPLITE_RECV_CC; break; |
1da177e4c
|
2510 2511 2512 |
default: err = -ENOPROTOOPT; break; |
6516c6557
|
2513 |
} |
1da177e4c
|
2514 2515 2516 |
return err; } |
c482c5685
|
2517 |
EXPORT_SYMBOL(udp_lib_setsockopt); |
1da177e4c
|
2518 |
|
db8dac20d
|
2519 |
int udp_setsockopt(struct sock *sk, int level, int optname, |
b7058842c
|
2520 |
char __user *optval, unsigned int optlen) |
db8dac20d
|
2521 2522 2523 2524 2525 2526 2527 2528 2529 |
{ if (level == SOL_UDP || level == SOL_UDPLITE) return udp_lib_setsockopt(sk, level, optname, optval, optlen, udp_push_pending_frames); return ip_setsockopt(sk, level, optname, optval, optlen); } #ifdef CONFIG_COMPAT int compat_udp_setsockopt(struct sock *sk, int level, int optname, |
b7058842c
|
2530 |
char __user *optval, unsigned int optlen) |
db8dac20d
|
2531 2532 2533 2534 2535 2536 2537 |
{ if (level == SOL_UDP || level == SOL_UDPLITE) return udp_lib_setsockopt(sk, level, optname, optval, optlen, udp_push_pending_frames); return compat_ip_setsockopt(sk, level, optname, optval, optlen); } #endif |
4c0a6cb0d
|
2538 2539 |
int udp_lib_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) |
1da177e4c
|
2540 2541 2542 |
{ struct udp_sock *up = udp_sk(sk); int val, len; |
c482c5685
|
2543 |
if (get_user(len, optlen)) |
1da177e4c
|
2544 2545 2546 |
return -EFAULT; len = min_t(unsigned int, len, sizeof(int)); |
e905a9eda
|
2547 |
|
6516c6557
|
2548 |
if (len < 0) |
1da177e4c
|
2549 |
return -EINVAL; |
6516c6557
|
2550 |
switch (optname) { |
1da177e4c
|
2551 2552 2553 2554 2555 2556 2557 |
case UDP_CORK: val = up->corkflag; break; case UDP_ENCAP: val = up->encap_type; break; |
1c19448c9
|
2558 2559 2560 2561 2562 2563 2564 |
case UDP_NO_CHECK6_TX: val = up->no_check6_tx; break; case UDP_NO_CHECK6_RX: val = up->no_check6_rx; break; |
bec1f6f69
|
2565 2566 2567 |
case UDP_SEGMENT: val = up->gso_size; break; |
ba4e58eca
|
2568 2569 2570 2571 2572 2573 2574 2575 2576 |
/* The following two cannot be changed on UDP sockets, the return is * always 0 (which corresponds to the full checksum coverage of UDP). */ case UDPLITE_SEND_CSCOV: val = up->pcslen; break; case UDPLITE_RECV_CSCOV: val = up->pcrlen; break; |
1da177e4c
|
2577 2578 |
default: return -ENOPROTOOPT; |
6516c6557
|
2579 |
} |
1da177e4c
|
2580 |
|
6516c6557
|
2581 |
if (put_user(len, optlen)) |
e905a9eda
|
2582 |
return -EFAULT; |
c482c5685
|
2583 |
if (copy_to_user(optval, &val, len)) |
1da177e4c
|
2584 |
return -EFAULT; |
e905a9eda
|
2585 |
return 0; |
1da177e4c
|
2586 |
} |
c482c5685
|
2587 |
EXPORT_SYMBOL(udp_lib_getsockopt); |
1da177e4c
|
2588 |
|
db8dac20d
|
2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 |
int udp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) { if (level == SOL_UDP || level == SOL_UDPLITE) return udp_lib_getsockopt(sk, level, optname, optval, optlen); return ip_getsockopt(sk, level, optname, optval, optlen); } #ifdef CONFIG_COMPAT int compat_udp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) { if (level == SOL_UDP || level == SOL_UDPLITE) return udp_lib_getsockopt(sk, level, optname, optval, optlen); return compat_ip_getsockopt(sk, level, optname, optval, optlen); } #endif |
1da177e4c
|
2606 2607 2608 2609 |
/** * udp_poll - wait for a UDP event. * @file - file struct * @sock - socket |
a11e1d432
|
2610 |
* @wait - poll table |
1da177e4c
|
2611 |
* |
e905a9eda
|
2612 |
* This is same as datagram poll, except for the special case of |
1da177e4c
|
2613 2614 2615 2616 2617 2618 |
* blocking sockets. If application is using a blocking fd * and a packet with checksum error is in the queue; * then it could get return from select indicating data available * but then block when reading it. Add special case code * to work around these arguably broken applications. */ |
a11e1d432
|
2619 |
__poll_t udp_poll(struct file *file, struct socket *sock, poll_table *wait) |
1da177e4c
|
2620 |
{ |
a11e1d432
|
2621 |
__poll_t mask = datagram_poll(file, sock, wait); |
1da177e4c
|
2622 |
struct sock *sk = sock->sk; |
ba4e58eca
|
2623 |
|
3ef7cf57c
|
2624 |
if (!skb_queue_empty_lockless(&udp_sk(sk)->reader_queue)) |
a9a08845e
|
2625 |
mask |= EPOLLIN | EPOLLRDNORM; |
2276f58ac
|
2626 |
|
1da177e4c
|
2627 |
/* Check for false positives due to checksum errors */ |
a11e1d432
|
2628 |
if ((mask & EPOLLRDNORM) && !(file->f_flags & O_NONBLOCK) && |
e83c6744e
|
2629 |
!(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1) |
a9a08845e
|
2630 |
mask &= ~(EPOLLIN | EPOLLRDNORM); |
1da177e4c
|
2631 2632 |
return mask; |
e905a9eda
|
2633 |
|
1da177e4c
|
2634 |
} |
a11e1d432
|
2635 |
EXPORT_SYMBOL(udp_poll); |
1da177e4c
|
2636 |
|
5d77dca82
|
2637 2638 2639 2640 2641 2642 |
int udp_abort(struct sock *sk, int err) { lock_sock(sk); sk->sk_err = err; sk->sk_error_report(sk); |
286c72dea
|
2643 |
__udp_disconnect(sk, 0); |
5d77dca82
|
2644 2645 2646 2647 2648 2649 |
release_sock(sk); return 0; } EXPORT_SYMBOL_GPL(udp_abort); |
db8dac20d
|
2650 |
struct proto udp_prot = { |
1e8029515
|
2651 2652 2653 |
.name = "UDP", .owner = THIS_MODULE, .close = udp_lib_close, |
d74bad4e7
|
2654 |
.pre_connect = udp_pre_connect, |
1e8029515
|
2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 |
.connect = ip4_datagram_connect, .disconnect = udp_disconnect, .ioctl = udp_ioctl, .init = udp_init_sock, .destroy = udp_destroy_sock, .setsockopt = udp_setsockopt, .getsockopt = udp_getsockopt, .sendmsg = udp_sendmsg, .recvmsg = udp_recvmsg, .sendpage = udp_sendpage, .release_cb = ip4_datagram_release_cb, .hash = udp_lib_hash, .unhash = udp_lib_unhash, .rehash = udp_v4_rehash, .get_port = udp_v4_get_port, .memory_allocated = &udp_memory_allocated, .sysctl_mem = sysctl_udp_mem, .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min), .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min), .obj_size = sizeof(struct udp_sock), .h.udp_table = &udp_table, |
db8dac20d
|
2676 |
#ifdef CONFIG_COMPAT |
1e8029515
|
2677 2678 |
.compat_setsockopt = compat_udp_setsockopt, .compat_getsockopt = compat_udp_getsockopt, |
db8dac20d
|
2679 |
#endif |
1e8029515
|
2680 |
.diag_destroy = udp_abort, |
db8dac20d
|
2681 |
}; |
c482c5685
|
2682 |
EXPORT_SYMBOL(udp_prot); |
1da177e4c
|
2683 2684 2685 |
/* ------------------------------------------------------------------------ */ #ifdef CONFIG_PROC_FS |
645ca708f
|
2686 |
static struct sock *udp_get_first(struct seq_file *seq, int start) |
1da177e4c
|
2687 2688 |
{ struct sock *sk; |
a3d2599b2
|
2689 |
struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file)); |
1da177e4c
|
2690 |
struct udp_iter_state *state = seq->private; |
6f191efe4
|
2691 |
struct net *net = seq_file_net(seq); |
1da177e4c
|
2692 |
|
a3d2599b2
|
2693 |
for (state->bucket = start; state->bucket <= afinfo->udp_table->mask; |
f86dcc5aa
|
2694 |
++state->bucket) { |
a3d2599b2
|
2695 |
struct udp_hslot *hslot = &afinfo->udp_table->hash[state->bucket]; |
f86dcc5aa
|
2696 |
|
ca065d0cf
|
2697 |
if (hlist_empty(&hslot->head)) |
f86dcc5aa
|
2698 |
continue; |
645ca708f
|
2699 |
spin_lock_bh(&hslot->lock); |
ca065d0cf
|
2700 |
sk_for_each(sk, &hslot->head) { |
878628fbf
|
2701 |
if (!net_eq(sock_net(sk), net)) |
a91275eff
|
2702 |
continue; |
a3d2599b2
|
2703 |
if (sk->sk_family == afinfo->family) |
1da177e4c
|
2704 2705 |
goto found; } |
645ca708f
|
2706 |
spin_unlock_bh(&hslot->lock); |
1da177e4c
|
2707 2708 2709 2710 2711 2712 2713 2714 |
} sk = NULL; found: return sk; } static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk) { |
a3d2599b2
|
2715 |
struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file)); |
1da177e4c
|
2716 |
struct udp_iter_state *state = seq->private; |
6f191efe4
|
2717 |
struct net *net = seq_file_net(seq); |
1da177e4c
|
2718 2719 |
do { |
ca065d0cf
|
2720 |
sk = sk_next(sk); |
a3d2599b2
|
2721 |
} while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != afinfo->family)); |
1da177e4c
|
2722 |
|
645ca708f
|
2723 |
if (!sk) { |
a3d2599b2
|
2724 2725 |
if (state->bucket <= afinfo->udp_table->mask) spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock); |
645ca708f
|
2726 |
return udp_get_first(seq, state->bucket + 1); |
1da177e4c
|
2727 2728 2729 2730 2731 2732 |
} return sk; } static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos) { |
645ca708f
|
2733 |
struct sock *sk = udp_get_first(seq, 0); |
1da177e4c
|
2734 2735 |
if (sk) |
6516c6557
|
2736 |
while (pos && (sk = udp_get_next(seq, sk)) != NULL) |
1da177e4c
|
2737 2738 2739 |
--pos; return pos ? NULL : sk; } |
a3d2599b2
|
2740 |
void *udp_seq_start(struct seq_file *seq, loff_t *pos) |
1da177e4c
|
2741 |
{ |
30842f298
|
2742 |
struct udp_iter_state *state = seq->private; |
f86dcc5aa
|
2743 |
state->bucket = MAX_UDP_PORTS; |
30842f298
|
2744 |
|
b50660f1f
|
2745 |
return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN; |
1da177e4c
|
2746 |
} |
a3d2599b2
|
2747 |
EXPORT_SYMBOL(udp_seq_start); |
1da177e4c
|
2748 |
|
a3d2599b2
|
2749 |
void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos) |
1da177e4c
|
2750 2751 |
{ struct sock *sk; |
b50660f1f
|
2752 |
if (v == SEQ_START_TOKEN) |
1da177e4c
|
2753 2754 2755 2756 2757 2758 2759 |
sk = udp_get_idx(seq, 0); else sk = udp_get_next(seq, v); ++*pos; return sk; } |
a3d2599b2
|
2760 |
EXPORT_SYMBOL(udp_seq_next); |
1da177e4c
|
2761 |
|
a3d2599b2
|
2762 |
void udp_seq_stop(struct seq_file *seq, void *v) |
1da177e4c
|
2763 |
{ |
a3d2599b2
|
2764 |
struct udp_seq_afinfo *afinfo = PDE_DATA(file_inode(seq->file)); |
645ca708f
|
2765 |
struct udp_iter_state *state = seq->private; |
a3d2599b2
|
2766 2767 |
if (state->bucket <= afinfo->udp_table->mask) spin_unlock_bh(&afinfo->udp_table->hash[state->bucket].lock); |
1da177e4c
|
2768 |
} |
a3d2599b2
|
2769 |
EXPORT_SYMBOL(udp_seq_stop); |
db8dac20d
|
2770 2771 |
/* ------------------------------------------------------------------------ */ |
5e659e4cb
|
2772 |
static void udp4_format_sock(struct sock *sp, struct seq_file *f, |
652586df9
|
2773 |
int bucket) |
db8dac20d
|
2774 2775 |
{ struct inet_sock *inet = inet_sk(sp); |
c720c7e83
|
2776 2777 2778 2779 |
__be32 dest = inet->inet_daddr; __be32 src = inet->inet_rcv_saddr; __u16 destp = ntohs(inet->inet_dport); __u16 srcp = ntohs(inet->inet_sport); |
db8dac20d
|
2780 |
|
f86dcc5aa
|
2781 |
seq_printf(f, "%5d: %08X:%04X %08X:%04X" |
ea9a03791
|
2782 |
" %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %u", |
db8dac20d
|
2783 |
bucket, src, srcp, dest, destp, sp->sk_state, |
31e6d363a
|
2784 |
sk_wmem_alloc_get(sp), |
6c206b200
|
2785 |
udp_rqueue_get(sp), |
a7cb5a49b
|
2786 2787 2788 |
0, 0L, 0, from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)), 0, sock_i_ino(sp), |
41c6d650f
|
2789 |
refcount_read(&sp->sk_refcnt), sp, |
652586df9
|
2790 |
atomic_read(&sp->sk_drops)); |
db8dac20d
|
2791 2792 2793 2794 |
} int udp4_seq_show(struct seq_file *seq, void *v) { |
652586df9
|
2795 |
seq_setwidth(seq, 127); |
db8dac20d
|
2796 |
if (v == SEQ_START_TOKEN) |
652586df9
|
2797 |
seq_puts(seq, " sl local_address rem_address st tx_queue " |
db8dac20d
|
2798 |
"rx_queue tr tm->when retrnsmt uid timeout " |
cb61cb9b8
|
2799 |
"inode ref pointer drops"); |
db8dac20d
|
2800 |
else { |
db8dac20d
|
2801 |
struct udp_iter_state *state = seq->private; |
652586df9
|
2802 |
udp4_format_sock(v, seq, state->bucket); |
db8dac20d
|
2803 |
} |
652586df9
|
2804 2805 |
seq_pad(seq, ' '); |
db8dac20d
|
2806 2807 |
return 0; } |
c35063722
|
2808 |
const struct seq_operations udp_seq_ops = { |
a3d2599b2
|
2809 2810 2811 2812 2813 |
.start = udp_seq_start, .next = udp_seq_next, .stop = udp_seq_stop, .show = udp4_seq_show, }; |
c35063722
|
2814 |
EXPORT_SYMBOL(udp_seq_ops); |
73cb88ecb
|
2815 |
|
db8dac20d
|
2816 |
static struct udp_seq_afinfo udp4_seq_afinfo = { |
db8dac20d
|
2817 |
.family = AF_INET, |
645ca708f
|
2818 |
.udp_table = &udp_table, |
db8dac20d
|
2819 |
}; |
2c8c1e729
|
2820 |
static int __net_init udp4_proc_init_net(struct net *net) |
15439febb
|
2821 |
{ |
c35063722
|
2822 2823 |
if (!proc_create_net_data("udp", 0444, net->proc_net, &udp_seq_ops, sizeof(struct udp_iter_state), &udp4_seq_afinfo)) |
a3d2599b2
|
2824 2825 |
return -ENOMEM; return 0; |
15439febb
|
2826 |
} |
2c8c1e729
|
2827 |
static void __net_exit udp4_proc_exit_net(struct net *net) |
15439febb
|
2828 |
{ |
a3d2599b2
|
2829 |
remove_proc_entry("udp", net->proc_net); |
15439febb
|
2830 2831 2832 2833 2834 2835 |
} static struct pernet_operations udp4_net_ops = { .init = udp4_proc_init_net, .exit = udp4_proc_exit_net, }; |
db8dac20d
|
2836 2837 |
int __init udp4_proc_init(void) { |
15439febb
|
2838 |
return register_pernet_subsys(&udp4_net_ops); |
db8dac20d
|
2839 2840 2841 2842 |
} void udp4_proc_exit(void) { |
15439febb
|
2843 |
unregister_pernet_subsys(&udp4_net_ops); |
db8dac20d
|
2844 |
} |
1da177e4c
|
2845 |
#endif /* CONFIG_PROC_FS */ |
f86dcc5aa
|
2846 2847 |
static __initdata unsigned long uhash_entries; static int __init set_uhash_entries(char *str) |
645ca708f
|
2848 |
{ |
413c27d86
|
2849 |
ssize_t ret; |
f86dcc5aa
|
2850 2851 |
if (!str) return 0; |
413c27d86
|
2852 2853 2854 2855 |
ret = kstrtoul(str, 0, &uhash_entries); if (ret) return 0; |
f86dcc5aa
|
2856 2857 2858 2859 2860 |
if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN) uhash_entries = UDP_HTABLE_SIZE_MIN; return 1; } __setup("uhash_entries=", set_uhash_entries); |
645ca708f
|
2861 |
|
f86dcc5aa
|
2862 2863 2864 |
void __init udp_table_init(struct udp_table *table, const char *name) { unsigned int i; |
31fe62b95
|
2865 2866 2867 2868 2869 2870 2871 2872 2873 |
table->hash = alloc_large_system_hash(name, 2 * sizeof(struct udp_hslot), uhash_entries, 21, /* one slot per 2 MB */ 0, &table->log, &table->mask, UDP_HTABLE_SIZE_MIN, 64 * 1024); |
512615b6b
|
2874 |
table->hash2 = table->hash + (table->mask + 1); |
f86dcc5aa
|
2875 |
for (i = 0; i <= table->mask; i++) { |
ca065d0cf
|
2876 |
INIT_HLIST_HEAD(&table->hash[i].head); |
fdcc8aa95
|
2877 |
table->hash[i].count = 0; |
645ca708f
|
2878 2879 |
spin_lock_init(&table->hash[i].lock); } |
512615b6b
|
2880 |
for (i = 0; i <= table->mask; i++) { |
ca065d0cf
|
2881 |
INIT_HLIST_HEAD(&table->hash2[i].head); |
512615b6b
|
2882 2883 2884 |
table->hash2[i].count = 0; spin_lock_init(&table->hash2[i].lock); } |
645ca708f
|
2885 |
} |
723b8e460
|
2886 2887 2888 2889 2890 2891 2892 2893 2894 |
u32 udp_flow_hashrnd(void) { static u32 hashrnd __read_mostly; net_get_random_once(&hashrnd, sizeof(hashrnd)); return hashrnd; } EXPORT_SYMBOL(udp_flow_hashrnd); |
1e8029515
|
2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 |
static void __udp_sysctl_init(struct net *net) { net->ipv4.sysctl_udp_rmem_min = SK_MEM_QUANTUM; net->ipv4.sysctl_udp_wmem_min = SK_MEM_QUANTUM; #ifdef CONFIG_NET_L3_MASTER_DEV net->ipv4.sysctl_udp_l3mdev_accept = 0; #endif } static int __net_init udp_sysctl_init(struct net *net) { __udp_sysctl_init(net); return 0; } static struct pernet_operations __net_initdata udp_sysctl_ops = { |
fc18999ed
|
2912 |
.init = udp_sysctl_init, |
1e8029515
|
2913 |
}; |
95766fff6
|
2914 2915 |
void __init udp_init(void) { |
f03d78db6
|
2916 |
unsigned long limit; |
4b272750d
|
2917 |
unsigned int i; |
95766fff6
|
2918 |
|
f86dcc5aa
|
2919 |
udp_table_init(&udp_table, "UDP"); |
f03d78db6
|
2920 |
limit = nr_free_buffer_pages() / 8; |
95766fff6
|
2921 2922 2923 2924 |
limit = max(limit, 128UL); sysctl_udp_mem[0] = limit / 4 * 3; sysctl_udp_mem[1] = limit; sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2; |
1e8029515
|
2925 |
__udp_sysctl_init(&init_net); |
4b272750d
|
2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 |
/* 16 spinlocks per cpu */ udp_busylocks_log = ilog2(nr_cpu_ids) + 4; udp_busylocks = kmalloc(sizeof(spinlock_t) << udp_busylocks_log, GFP_KERNEL); if (!udp_busylocks) panic("UDP: failed to alloc udp_busylocks "); for (i = 0; i < (1U << udp_busylocks_log); i++) spin_lock_init(udp_busylocks + i); |
1e8029515
|
2936 2937 2938 2939 |
if (register_pernet_subsys(&udp_sysctl_ops)) panic("UDP: failed to init sysctl parameters. "); |
95766fff6
|
2940 |
} |