Blame view
net/ipv4/udp.c
73.1 KB
1da177e4c Linux-2.6.12-rc2 |
1 2 3 4 5 6 7 |
/* * INET An implementation of the TCP/IP protocol suite for the LINUX * operating system. INET is implemented using the BSD Socket * interface as the means of communication with the user level. * * The User Datagram Protocol (UDP). * |
02c30a84e [PATCH] update Ro... |
8 |
* Authors: Ross Biro |
1da177e4c Linux-2.6.12-rc2 |
9 10 |
* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> * Arnt Gulbrandsen, <agulbra@nvg.unit.no> |
113aa838e net: Rationalise ... |
11 |
* Alan Cox, <alan@lxorguk.ukuu.org.uk> |
1da177e4c Linux-2.6.12-rc2 |
12 13 14 15 16 17 18 19 20 |
* Hirokazu Takahashi, <taka@valinux.co.jp> * * Fixes: * Alan Cox : verify_area() calls * Alan Cox : stopped close while in use off icmp * messages. Not a fix but a botch that * for udp at least is 'valid'. * Alan Cox : Fixed icmp handling properly * Alan Cox : Correct error for oversized datagrams |
e905a9eda [NET] IPV4: Fix w... |
21 22 |
* Alan Cox : Tidied select() semantics. * Alan Cox : udp_err() fixed properly, also now |
1da177e4c Linux-2.6.12-rc2 |
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 |
* select and read wake correctly on errors * Alan Cox : udp_send verify_area moved to avoid mem leak * Alan Cox : UDP can count its memory * Alan Cox : send to an unknown connection causes * an ECONNREFUSED off the icmp, but * does NOT close. * Alan Cox : Switched to new sk_buff handlers. No more backlog! * Alan Cox : Using generic datagram code. Even smaller and the PEEK * bug no longer crashes it. * Fred Van Kempen : Net2e support for sk->broadcast. * Alan Cox : Uses skb_free_datagram * Alan Cox : Added get/set sockopt support. * Alan Cox : Broadcasting without option set returns EACCES. * Alan Cox : No wakeup calls. Instead we now use the callbacks. * Alan Cox : Use ip_tos and ip_ttl * Alan Cox : SNMP Mibs * Alan Cox : MSG_DONTROUTE, and 0.0.0.0 support. * Matt Dillon : UDP length checks. * Alan Cox : Smarter af_inet used properly. * Alan Cox : Use new kernel side addressing. * Alan Cox : Incorrect return on truncated datagram receive. * Arnt Gulbrandsen : New udp_send and stuff * Alan Cox : Cache last socket * Alan Cox : Route cache * Jon Peatfield : Minor efficiency fix to sendto(). * Mike Shaver : RFC1122 checks. * Alan Cox : Nonblocking error fix. * Willy Konynenberg : Transparent proxying support. * Mike McLagan : Routing by source * David S. Miller : New socket lookup architecture. * Last socket cache retained as it * does have a high hit rate. * Olaf Kirch : Don't linearise iovec on sendmsg. * Andi Kleen : Some cleanups, cache destination entry |
e905a9eda [NET] IPV4: Fix w... |
57 |
* for connect. |
1da177e4c Linux-2.6.12-rc2 |
58 59 60 61 62 63 64 65 66 67 68 69 70 |
* Vitaly E. Lavrov : Transparent proxy revived after year coma. * Melvin Smith : Check msg_name not msg_namelen in sendto(), * return ENOTCONN for unconnected sockets (POSIX) * Janos Farkas : don't deliver multi/broadcasts to a different * bound-to-device socket * Hirokazu Takahashi : HW checksumming for outgoing UDP * datagrams. * Hirokazu Takahashi : sendfile() on UDP works now. * Arnaldo C. Melo : convert /proc/net/udp to seq_file * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which * Alexey Kuznetsov: allow both IPv4 and IPv6 sockets to bind * a single port at the same time. * Derek Atkins <derek@ihtfp.com>: Add Encapulation Support |
342f0234c [UDP]: Introduce ... |
71 |
* James Chapman : Add L2TP encapsulation type. |
1da177e4c Linux-2.6.12-rc2 |
72 73 74 75 76 77 78 |
* * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ |
e905a9eda [NET] IPV4: Fix w... |
79 |
|
afd465030 net: ipv4: Standa... |
80 |
#define pr_fmt(fmt) "UDP: " fmt |
7c0f6ba68 Replace <asm/uacc... |
81 |
#include <linux/uaccess.h> |
1da177e4c Linux-2.6.12-rc2 |
82 |
#include <asm/ioctls.h> |
95766fff6 [UDP]: Add memory... |
83 |
#include <linux/bootmem.h> |
8203efb3c udp: calculate ud... |
84 85 |
#include <linux/highmem.h> #include <linux/swap.h> |
1da177e4c Linux-2.6.12-rc2 |
86 87 88 89 90 |
#include <linux/types.h> #include <linux/fcntl.h> #include <linux/module.h> #include <linux/socket.h> #include <linux/sockios.h> |
14c850212 [INET_SOCK]: Move... |
91 |
#include <linux/igmp.h> |
6e5403093 ipv4/udp: Verify ... |
92 |
#include <linux/inetdevice.h> |
1da177e4c Linux-2.6.12-rc2 |
93 94 95 96 |
#include <linux/in.h> #include <linux/errno.h> #include <linux/timer.h> #include <linux/mm.h> |
1da177e4c Linux-2.6.12-rc2 |
97 |
#include <linux/inet.h> |
1da177e4c Linux-2.6.12-rc2 |
98 |
#include <linux/netdevice.h> |
5a0e3ad6a include cleanup: ... |
99 |
#include <linux/slab.h> |
c752f0739 [TCP]: Move the t... |
100 |
#include <net/tcp_states.h> |
1da177e4c Linux-2.6.12-rc2 |
101 102 103 |
#include <linux/skbuff.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> |
457c4cbc5 [NET]: Make /proc... |
104 |
#include <net/net_namespace.h> |
1da177e4c Linux-2.6.12-rc2 |
105 |
#include <net/icmp.h> |
421b3885b udp: ipv4: Add ud... |
106 |
#include <net/inet_hashtables.h> |
1da177e4c Linux-2.6.12-rc2 |
107 |
#include <net/route.h> |
1da177e4c Linux-2.6.12-rc2 |
108 109 |
#include <net/checksum.h> #include <net/xfrm.h> |
296f7ea75 udp: add tracepoi... |
110 |
#include <trace/events/udp.h> |
447167bf5 udp: intoduce udp... |
111 |
#include <linux/static_key.h> |
22911fc58 net: skb_free_dat... |
112 |
#include <trace/events/skb.h> |
076bb0c82 net: rename inclu... |
113 |
#include <net/busy_poll.h> |
ba4e58eca [NET]: Supporting... |
114 |
#include "udp_impl.h" |
e32ea7e74 soreuseport: fast... |
115 |
#include <net/sock_reuseport.h> |
217375a0c udp: include addr... |
116 |
#include <net/addrconf.h> |
1da177e4c Linux-2.6.12-rc2 |
117 |
|
f86dcc5aa udp: dynamically ... |
118 |
struct udp_table udp_table __read_mostly; |
645ca708f udp: introduce st... |
119 |
EXPORT_SYMBOL(udp_table); |
1da177e4c Linux-2.6.12-rc2 |
120 |
|
8d987e5c7 net: avoid limits... |
121 |
long sysctl_udp_mem[3] __read_mostly; |
95766fff6 [UDP]: Add memory... |
122 |
EXPORT_SYMBOL(sysctl_udp_mem); |
c482c5685 udp: cleanups |
123 124 |
int sysctl_udp_rmem_min __read_mostly; |
95766fff6 [UDP]: Add memory... |
125 |
EXPORT_SYMBOL(sysctl_udp_rmem_min); |
c482c5685 udp: cleanups |
126 127 |
int sysctl_udp_wmem_min __read_mostly; |
95766fff6 [UDP]: Add memory... |
128 |
EXPORT_SYMBOL(sysctl_udp_wmem_min); |
8d987e5c7 net: avoid limits... |
129 |
atomic_long_t udp_memory_allocated; |
95766fff6 [UDP]: Add memory... |
130 |
EXPORT_SYMBOL(udp_memory_allocated); |
f86dcc5aa udp: dynamically ... |
131 132 |
#define MAX_UDP_PORTS 65536 #define PORTS_PER_CHAIN (MAX_UDP_PORTS / UDP_HTABLE_SIZE_MIN) |
98322f22e udp: optimize bin... |
133 |
|
63a6fff35 net: Avoid receiv... |
134 135 136 137 138 139 140 141 142 143 |
/* IPCB reference means this can not be used from early demux */ static bool udp_lib_exact_dif_match(struct net *net, struct sk_buff *skb) { #if IS_ENABLED(CONFIG_NET_L3_MASTER_DEV) if (!net->ipv4.sysctl_udp_l3mdev_accept && skb && ipv4_l3mdev_skb(IPCB(skb)->flags)) return true; #endif return false; } |
f24d43c07 udp: complete por... |
144 |
static int udp_lib_lport_inuse(struct net *net, __u16 num, |
645ca708f udp: introduce st... |
145 |
const struct udp_hslot *hslot, |
98322f22e udp: optimize bin... |
146 |
unsigned long *bitmap, |
fe38d2a1c inet: collapse ip... |
147 |
struct sock *sk, unsigned int log) |
1da177e4c Linux-2.6.12-rc2 |
148 |
{ |
f24d43c07 udp: complete por... |
149 |
struct sock *sk2; |
ba418fa35 soreuseport: UDP/... |
150 |
kuid_t uid = sock_i_uid(sk); |
25030a7f9 [UDP]: Unify UDPv... |
151 |
|
ca065d0cf udp: no longer us... |
152 |
sk_for_each(sk2, &hslot->head) { |
9d4fb27db net/ipv4: Move &&... |
153 154 |
if (net_eq(sock_net(sk2), net) && sk2 != sk && |
d4cada4ae udp: split sk_has... |
155 |
(bitmap || udp_sk(sk2)->udp_port_hash == num) && |
9d4fb27db net/ipv4: Move &&... |
156 157 158 |
(!sk2->sk_reuse || !sk->sk_reuse) && (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && |
fe38d2a1c inet: collapse ip... |
159 |
inet_rcv_saddr_equal(sk, sk2, true)) { |
df560056d udp: inuse checks... |
160 161 162 163 164 165 166 167 168 169 170 |
if (sk2->sk_reuseport && sk->sk_reuseport && !rcu_access_pointer(sk->sk_reuseport_cb) && uid_eq(uid, sock_i_uid(sk2))) { if (!bitmap) return 0; } else { if (!bitmap) return 1; __set_bit(udp_sk(sk2)->udp_port_hash >> log, bitmap); } |
98322f22e udp: optimize bin... |
171 |
} |
4243cdc2c udp: Neaten funct... |
172 |
} |
25030a7f9 [UDP]: Unify UDPv... |
173 174 |
return 0; } |
30fff9231 udp: bind() optim... |
175 176 177 178 179 |
/* * Note: we still hold spinlock of primary hash chain, so no other writer * can insert/delete a socket with local_port == num */ static int udp_lib_lport_inuse2(struct net *net, __u16 num, |
4243cdc2c udp: Neaten funct... |
180 |
struct udp_hslot *hslot2, |
fe38d2a1c inet: collapse ip... |
181 |
struct sock *sk) |
30fff9231 udp: bind() optim... |
182 183 |
{ struct sock *sk2; |
ba418fa35 soreuseport: UDP/... |
184 |
kuid_t uid = sock_i_uid(sk); |
30fff9231 udp: bind() optim... |
185 186 187 |
int res = 0; spin_lock(&hslot2->lock); |
ca065d0cf udp: no longer us... |
188 |
udp_portaddr_for_each_entry(sk2, &hslot2->head) { |
9d4fb27db net/ipv4: Move &&... |
189 190 191 192 193 194 |
if (net_eq(sock_net(sk2), net) && sk2 != sk && (udp_sk(sk2)->udp_port_hash == num) && (!sk2->sk_reuse || !sk->sk_reuse) && (!sk2->sk_bound_dev_if || !sk->sk_bound_dev_if || sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && |
fe38d2a1c inet: collapse ip... |
195 |
inet_rcv_saddr_equal(sk, sk2, true)) { |
df560056d udp: inuse checks... |
196 197 198 199 200 201 202 |
if (sk2->sk_reuseport && sk->sk_reuseport && !rcu_access_pointer(sk->sk_reuseport_cb) && uid_eq(uid, sock_i_uid(sk2))) { res = 0; } else { res = 1; } |
30fff9231 udp: bind() optim... |
203 204 |
break; } |
4243cdc2c udp: Neaten funct... |
205 |
} |
30fff9231 udp: bind() optim... |
206 207 208 |
spin_unlock(&hslot2->lock); return res; } |
fe38d2a1c inet: collapse ip... |
209 |
static int udp_reuseport_add_sock(struct sock *sk, struct udp_hslot *hslot) |
e32ea7e74 soreuseport: fast... |
210 211 |
{ struct net *net = sock_net(sk); |
e32ea7e74 soreuseport: fast... |
212 213 |
kuid_t uid = sock_i_uid(sk); struct sock *sk2; |
ca065d0cf udp: no longer us... |
214 |
sk_for_each(sk2, &hslot->head) { |
e32ea7e74 soreuseport: fast... |
215 216 217 218 219 220 221 |
if (net_eq(sock_net(sk2), net) && sk2 != sk && sk2->sk_family == sk->sk_family && ipv6_only_sock(sk2) == ipv6_only_sock(sk) && (udp_sk(sk2)->udp_port_hash == udp_sk(sk)->udp_port_hash) && (sk2->sk_bound_dev_if == sk->sk_bound_dev_if) && sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && |
fe38d2a1c inet: collapse ip... |
222 |
inet_rcv_saddr_equal(sk, sk2, false)) { |
e32ea7e74 soreuseport: fast... |
223 224 225 |
return reuseport_add_sock(sk, sk2); } } |
1b5f962e7 soreuseport: fix ... |
226 |
return reuseport_alloc(sk); |
e32ea7e74 soreuseport: fast... |
227 |
} |
25030a7f9 [UDP]: Unify UDPv... |
228 |
/** |
6ba5a3c52 [UDP]: Make full ... |
229 |
* udp_lib_get_port - UDP/-Lite port lookup for IPv4 and IPv6 |
25030a7f9 [UDP]: Unify UDPv... |
230 231 232 |
* * @sk: socket struct in question * @snum: port number to look up |
25985edce Fix common misspe... |
233 |
* @hash2_nulladdr: AF-dependent hash value in secondary hash chains, |
30fff9231 udp: bind() optim... |
234 |
* with NULL address |
25030a7f9 [UDP]: Unify UDPv... |
235 |
*/ |
6ba5a3c52 [UDP]: Make full ... |
236 |
int udp_lib_get_port(struct sock *sk, unsigned short snum, |
30fff9231 udp: bind() optim... |
237 |
unsigned int hash2_nulladdr) |
25030a7f9 [UDP]: Unify UDPv... |
238 |
{ |
512615b6b udp: secondary ha... |
239 |
struct udp_hslot *hslot, *hslot2; |
645ca708f udp: introduce st... |
240 |
struct udp_table *udptable = sk->sk_prot->h.udp_table; |
25030a7f9 [UDP]: Unify UDPv... |
241 |
int error = 1; |
3b1e0a655 [NET] NETNS: Omit... |
242 |
struct net *net = sock_net(sk); |
1da177e4c Linux-2.6.12-rc2 |
243 |
|
32c1da708 [UDP]: Randomize ... |
244 |
if (!snum) { |
9088c5609 udp: Improve port... |
245 |
int low, high, remaining; |
95c961747 net: cleanup unsi... |
246 |
unsigned int rand; |
98322f22e udp: optimize bin... |
247 248 |
unsigned short first, last; DECLARE_BITMAP(bitmap, PORTS_PER_CHAIN); |
32c1da708 [UDP]: Randomize ... |
249 |
|
0bbf87d85 net ipv4: Convert... |
250 |
inet_get_local_port_range(net, &low, &high); |
a25de534f [INET]: Justifica... |
251 |
remaining = (high - low) + 1; |
227b60f51 [INET]: local por... |
252 |
|
63862b5be net: replace macr... |
253 |
rand = prandom_u32(); |
8fc54f689 net: use reciproc... |
254 |
first = reciprocal_scale(rand, remaining) + low; |
98322f22e udp: optimize bin... |
255 256 257 |
/* * force rand to be an odd multiple of UDP_HTABLE_SIZE */ |
f86dcc5aa udp: dynamically ... |
258 |
rand = (rand | 1) * (udptable->mask + 1); |
5781b2356 udp: udp_lib_get_... |
259 260 |
last = first + udptable->mask + 1; do { |
f86dcc5aa udp: dynamically ... |
261 |
hslot = udp_hashslot(udptable, net, first); |
98322f22e udp: optimize bin... |
262 |
bitmap_zero(bitmap, PORTS_PER_CHAIN); |
645ca708f udp: introduce st... |
263 |
spin_lock_bh(&hslot->lock); |
98322f22e udp: optimize bin... |
264 |
udp_lib_lport_inuse(net, snum, hslot, bitmap, sk, |
fe38d2a1c inet: collapse ip... |
265 |
udptable->log); |
98322f22e udp: optimize bin... |
266 267 268 269 270 271 272 |
snum = first; /* * Iterate on all possible values of snum for this hash. * Using steps of an odd multiple of UDP_HTABLE_SIZE * give us randomization and full range coverage. */ |
9088c5609 udp: Improve port... |
273 |
do { |
98322f22e udp: optimize bin... |
274 |
if (low <= snum && snum <= high && |
e3826f1e9 net: reserve port... |
275 |
!test_bit(snum >> udptable->log, bitmap) && |
122ff243f ipv4: make ip_loc... |
276 |
!inet_is_local_reserved_port(net, snum)) |
98322f22e udp: optimize bin... |
277 278 279 280 |
goto found; snum += rand; } while (snum != first); spin_unlock_bh(&hslot->lock); |
df560056d udp: inuse checks... |
281 |
cond_resched(); |
5781b2356 udp: udp_lib_get_... |
282 |
} while (++first != last); |
98322f22e udp: optimize bin... |
283 |
goto fail; |
645ca708f udp: introduce st... |
284 |
} else { |
f86dcc5aa udp: dynamically ... |
285 |
hslot = udp_hashslot(udptable, net, snum); |
645ca708f udp: introduce st... |
286 |
spin_lock_bh(&hslot->lock); |
30fff9231 udp: bind() optim... |
287 288 289 290 291 292 293 294 295 296 |
if (hslot->count > 10) { int exist; unsigned int slot2 = udp_sk(sk)->udp_portaddr_hash ^ snum; slot2 &= udptable->mask; hash2_nulladdr &= udptable->mask; hslot2 = udp_hashslot2(udptable, slot2); if (hslot->count < hslot2->count) goto scan_primary_hash; |
fe38d2a1c inet: collapse ip... |
297 |
exist = udp_lib_lport_inuse2(net, snum, hslot2, sk); |
30fff9231 udp: bind() optim... |
298 299 300 |
if (!exist && (hash2_nulladdr != slot2)) { hslot2 = udp_hashslot2(udptable, hash2_nulladdr); exist = udp_lib_lport_inuse2(net, snum, hslot2, |
fe38d2a1c inet: collapse ip... |
301 |
sk); |
30fff9231 udp: bind() optim... |
302 303 304 305 306 307 308 |
} if (exist) goto fail_unlock; else goto found; } scan_primary_hash: |
fe38d2a1c inet: collapse ip... |
309 |
if (udp_lib_lport_inuse(net, snum, hslot, NULL, sk, 0)) |
645ca708f udp: introduce st... |
310 311 |
goto fail_unlock; } |
98322f22e udp: optimize bin... |
312 |
found: |
c720c7e83 inet: rename some... |
313 |
inet_sk(sk)->inet_num = snum; |
d4cada4ae udp: split sk_has... |
314 315 |
udp_sk(sk)->udp_port_hash = snum; udp_sk(sk)->udp_portaddr_hash ^= snum; |
1da177e4c Linux-2.6.12-rc2 |
316 |
if (sk_unhashed(sk)) { |
e32ea7e74 soreuseport: fast... |
317 |
if (sk->sk_reuseport && |
fe38d2a1c inet: collapse ip... |
318 |
udp_reuseport_add_sock(sk, hslot)) { |
e32ea7e74 soreuseport: fast... |
319 320 321 322 323 |
inet_sk(sk)->inet_num = 0; udp_sk(sk)->udp_port_hash = 0; udp_sk(sk)->udp_portaddr_hash ^= snum; goto fail_unlock; } |
ca065d0cf udp: no longer us... |
324 |
sk_add_node_rcu(sk, &hslot->head); |
fdcc8aa95 udp: add a counte... |
325 |
hslot->count++; |
c29a0bc4d [SOCK][NETNS]: Ad... |
326 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); |
512615b6b udp: secondary ha... |
327 328 329 |
hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); spin_lock(&hslot2->lock); |
d894ba18d soreuseport: fix ... |
330 |
if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && |
1602f49b5 Merge git://git.k... |
331 332 333 |
sk->sk_family == AF_INET6) hlist_add_tail_rcu(&udp_sk(sk)->udp_portaddr_node, &hslot2->head); |
d894ba18d soreuseport: fix ... |
334 |
else |
1602f49b5 Merge git://git.k... |
335 336 |
hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, &hslot2->head); |
512615b6b udp: secondary ha... |
337 338 |
hslot2->count++; spin_unlock(&hslot2->lock); |
1da177e4c Linux-2.6.12-rc2 |
339 |
} |
ca065d0cf udp: no longer us... |
340 |
sock_set_flag(sk, SOCK_RCU_FREE); |
25030a7f9 [UDP]: Unify UDPv... |
341 |
error = 0; |
645ca708f udp: introduce st... |
342 343 |
fail_unlock: spin_unlock_bh(&hslot->lock); |
1da177e4c Linux-2.6.12-rc2 |
344 |
fail: |
25030a7f9 [UDP]: Unify UDPv... |
345 346 |
return error; } |
c482c5685 udp: cleanups |
347 |
EXPORT_SYMBOL(udp_lib_get_port); |
25030a7f9 [UDP]: Unify UDPv... |
348 |
|
6eada0110 netns: constify n... |
349 350 |
static u32 udp4_portaddr_hash(const struct net *net, __be32 saddr, unsigned int port) |
d4cada4ae udp: split sk_has... |
351 |
{ |
0eae88f31 net: Fix various ... |
352 |
return jhash_1word((__force u32)saddr, net_hash_mix(net)) ^ port; |
d4cada4ae udp: split sk_has... |
353 |
} |
6ba5a3c52 [UDP]: Make full ... |
354 |
int udp_v4_get_port(struct sock *sk, unsigned short snum) |
db8dac20d [UDP]: Revert udp... |
355 |
{ |
30fff9231 udp: bind() optim... |
356 |
unsigned int hash2_nulladdr = |
0eae88f31 net: Fix various ... |
357 |
udp4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum); |
30fff9231 udp: bind() optim... |
358 359 |
unsigned int hash2_partial = udp4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0); |
d4cada4ae udp: split sk_has... |
360 |
/* precompute partial secondary hash */ |
30fff9231 udp: bind() optim... |
361 |
udp_sk(sk)->udp_portaddr_hash = hash2_partial; |
fe38d2a1c inet: collapse ip... |
362 |
return udp_lib_get_port(sk, snum, hash2_nulladdr); |
db8dac20d [UDP]: Revert udp... |
363 |
} |
d1e37288c udp reuseport: fi... |
364 365 |
static int compute_score(struct sock *sk, struct net *net, __be32 saddr, __be16 sport, |
fb74c2773 net: ipv4: add se... |
366 367 |
__be32 daddr, unsigned short hnum, int dif, int sdif, bool exact_dif) |
645ca708f udp: introduce st... |
368 |
{ |
60c04aecd udp: Neaten and r... |
369 370 |
int score; struct inet_sock *inet; |
645ca708f udp: introduce st... |
371 |
|
60c04aecd udp: Neaten and r... |
372 373 374 375 |
if (!net_eq(sock_net(sk), net) || udp_sk(sk)->udp_port_hash != hnum || ipv6_only_sock(sk)) return -1; |
645ca708f udp: introduce st... |
376 |
|
60c04aecd udp: Neaten and r... |
377 378 379 380 381 382 383 |
score = (sk->sk_family == PF_INET) ? 2 : 1; inet = inet_sk(sk); if (inet->inet_rcv_saddr) { if (inet->inet_rcv_saddr != daddr) return -1; score += 4; |
645ca708f udp: introduce st... |
384 |
} |
60c04aecd udp: Neaten and r... |
385 386 387 388 389 390 391 392 393 394 395 396 |
if (inet->inet_daddr) { if (inet->inet_daddr != saddr) return -1; score += 4; } if (inet->inet_dport) { if (inet->inet_dport != sport) return -1; score += 4; } |
63a6fff35 net: Avoid receiv... |
397 |
if (sk->sk_bound_dev_if || exact_dif) { |
fb74c2773 net: ipv4: add se... |
398 399 |
bool dev_match = (sk->sk_bound_dev_if == dif || sk->sk_bound_dev_if == sdif); |
59afc1841 udp: fix SO_BINDT... |
400 |
if (!dev_match) |
60c04aecd udp: Neaten and r... |
401 |
return -1; |
59afc1841 udp: fix SO_BINDT... |
402 |
if (sk->sk_bound_dev_if) |
fb74c2773 net: ipv4: add se... |
403 |
score += 4; |
60c04aecd udp: Neaten and r... |
404 |
} |
fb74c2773 net: ipv4: add se... |
405 |
|
70da268b5 net: SO_INCOMING_... |
406 407 |
if (sk->sk_incoming_cpu == raw_smp_processor_id()) score++; |
645ca708f udp: introduce st... |
408 409 |
return score; } |
6eada0110 netns: constify n... |
410 411 412 |
static u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport, const __be32 faddr, const __be16 fport) |
65cd8033f ipv4: split inet_... |
413 |
{ |
1bbdceef1 inet: convert ine... |
414 415 416 |
static u32 udp_ehash_secret __read_mostly; net_get_random_once(&udp_ehash_secret, sizeof(udp_ehash_secret)); |
65cd8033f ipv4: split inet_... |
417 |
return __inet_ehashfn(laddr, lport, faddr, fport, |
1bbdceef1 inet: convert ine... |
418 |
udp_ehash_secret + net_hash_mix(net)); |
65cd8033f ipv4: split inet_... |
419 |
} |
d1e37288c udp reuseport: fi... |
420 |
/* called with rcu_read_lock() */ |
5051ebd27 ipv4: udp: optimi... |
421 |
static struct sock *udp4_lib_lookup2(struct net *net, |
fb74c2773 net: ipv4: add se... |
422 423 424 425 426 |
__be32 saddr, __be16 sport, __be32 daddr, unsigned int hnum, int dif, int sdif, bool exact_dif, struct udp_hslot *hslot2, struct sk_buff *skb) |
5051ebd27 ipv4: udp: optimi... |
427 428 |
{ struct sock *sk, *result; |
ba418fa35 soreuseport: UDP/... |
429 430 |
int score, badness, matches = 0, reuseport = 0; u32 hash = 0; |
5051ebd27 ipv4: udp: optimi... |
431 |
|
5051ebd27 ipv4: udp: optimi... |
432 |
result = NULL; |
ba418fa35 soreuseport: UDP/... |
433 |
badness = 0; |
ca065d0cf udp: no longer us... |
434 |
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { |
d1e37288c udp reuseport: fi... |
435 |
score = compute_score(sk, net, saddr, sport, |
fb74c2773 net: ipv4: add se... |
436 |
daddr, hnum, dif, sdif, exact_dif); |
5051ebd27 ipv4: udp: optimi... |
437 |
if (score > badness) { |
ba418fa35 soreuseport: UDP/... |
438 439 |
reuseport = sk->sk_reuseport; if (reuseport) { |
65cd8033f ipv4: split inet_... |
440 441 |
hash = udp_ehashfn(net, daddr, hnum, saddr, sport); |
ca065d0cf udp: no longer us... |
442 |
result = reuseport_select_sock(sk, hash, skb, |
ed0dfffd7 udp: fix potentia... |
443 |
sizeof(struct udphdr)); |
ca065d0cf udp: no longer us... |
444 445 |
if (result) return result; |
ba418fa35 soreuseport: UDP/... |
446 447 |
matches = 1; } |
ca065d0cf udp: no longer us... |
448 449 |
badness = score; result = sk; |
ba418fa35 soreuseport: UDP/... |
450 451 |
} else if (score == badness && reuseport) { matches++; |
8fc54f689 net: use reciproc... |
452 |
if (reciprocal_scale(hash, matches) == 0) |
ba418fa35 soreuseport: UDP/... |
453 454 |
result = sk; hash = next_pseudo_random32(hash); |
5051ebd27 ipv4: udp: optimi... |
455 456 |
} } |
5051ebd27 ipv4: udp: optimi... |
457 458 |
return result; } |
db8dac20d [UDP]: Revert udp... |
459 460 461 |
/* UDP is nearly always wildcards out the wazoo, it makes no sense to try * harder than this. -DaveM */ |
fce823381 udp: Export code ... |
462 |
struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, |
fb74c2773 net: ipv4: add se... |
463 464 |
__be16 sport, __be32 daddr, __be16 dport, int dif, int sdif, struct udp_table *udptable, struct sk_buff *skb) |
db8dac20d [UDP]: Revert udp... |
465 |
{ |
271b72c7f udp: RCU handling... |
466 |
struct sock *sk, *result; |
db8dac20d [UDP]: Revert udp... |
467 |
unsigned short hnum = ntohs(dport); |
5051ebd27 ipv4: udp: optimi... |
468 469 |
unsigned int hash2, slot2, slot = udp_hashfn(net, hnum, udptable->mask); struct udp_hslot *hslot2, *hslot = &udptable->hash[slot]; |
63a6fff35 net: Avoid receiv... |
470 |
bool exact_dif = udp_lib_exact_dif_match(net, skb); |
ba418fa35 soreuseport: UDP/... |
471 472 |
int score, badness, matches = 0, reuseport = 0; u32 hash = 0; |
645ca708f udp: introduce st... |
473 |
|
5051ebd27 ipv4: udp: optimi... |
474 475 476 477 478 479 480 481 |
if (hslot->count > 10) { hash2 = udp4_portaddr_hash(net, daddr, hnum); slot2 = hash2 & udptable->mask; hslot2 = &udptable->hash2[slot2]; if (hslot->count < hslot2->count) goto begin; result = udp4_lib_lookup2(net, saddr, sport, |
fb74c2773 net: ipv4: add se... |
482 |
daddr, hnum, dif, sdif, |
63a6fff35 net: Avoid receiv... |
483 |
exact_dif, hslot2, skb); |
5051ebd27 ipv4: udp: optimi... |
484 |
if (!result) { |
d1e37288c udp reuseport: fi... |
485 |
unsigned int old_slot2 = slot2; |
0eae88f31 net: Fix various ... |
486 |
hash2 = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum); |
5051ebd27 ipv4: udp: optimi... |
487 |
slot2 = hash2 & udptable->mask; |
d1e37288c udp reuseport: fi... |
488 489 490 |
/* avoid searching the same slot again. */ if (unlikely(slot2 == old_slot2)) return result; |
5051ebd27 ipv4: udp: optimi... |
491 492 493 |
hslot2 = &udptable->hash2[slot2]; if (hslot->count < hslot2->count) goto begin; |
1223c67c0 udp: fix for unic... |
494 |
result = udp4_lib_lookup2(net, saddr, sport, |
fb74c2773 net: ipv4: add se... |
495 |
daddr, hnum, dif, sdif, |
63a6fff35 net: Avoid receiv... |
496 |
exact_dif, hslot2, skb); |
5051ebd27 ipv4: udp: optimi... |
497 |
} |
5051ebd27 ipv4: udp: optimi... |
498 499 |
return result; } |
271b72c7f udp: RCU handling... |
500 501 |
begin: result = NULL; |
ba418fa35 soreuseport: UDP/... |
502 |
badness = 0; |
ca065d0cf udp: no longer us... |
503 |
sk_for_each_rcu(sk, &hslot->head) { |
d1e37288c udp reuseport: fi... |
504 |
score = compute_score(sk, net, saddr, sport, |
fb74c2773 net: ipv4: add se... |
505 |
daddr, hnum, dif, sdif, exact_dif); |
645ca708f udp: introduce st... |
506 |
if (score > badness) { |
ba418fa35 soreuseport: UDP/... |
507 508 |
reuseport = sk->sk_reuseport; if (reuseport) { |
65cd8033f ipv4: split inet_... |
509 510 |
hash = udp_ehashfn(net, daddr, hnum, saddr, sport); |
ca065d0cf udp: no longer us... |
511 |
result = reuseport_select_sock(sk, hash, skb, |
538950a1b soreuseport: sets... |
512 |
sizeof(struct udphdr)); |
ca065d0cf udp: no longer us... |
513 514 |
if (result) return result; |
ba418fa35 soreuseport: UDP/... |
515 516 |
matches = 1; } |
ca065d0cf udp: no longer us... |
517 518 |
result = sk; badness = score; |
ba418fa35 soreuseport: UDP/... |
519 520 |
} else if (score == badness && reuseport) { matches++; |
8fc54f689 net: use reciproc... |
521 |
if (reciprocal_scale(hash, matches) == 0) |
ba418fa35 soreuseport: UDP/... |
522 523 |
result = sk; hash = next_pseudo_random32(hash); |
db8dac20d [UDP]: Revert udp... |
524 525 |
} } |
db8dac20d [UDP]: Revert udp... |
526 527 |
return result; } |
fce823381 udp: Export code ... |
528 |
EXPORT_SYMBOL_GPL(__udp4_lib_lookup); |
db8dac20d [UDP]: Revert udp... |
529 |
|
607c4aaf0 inet: Add udplib_... |
530 531 |
static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb, __be16 sport, __be16 dport, |
645ca708f udp: introduce st... |
532 |
struct udp_table *udptable) |
607c4aaf0 inet: Add udplib_... |
533 534 |
{ const struct iphdr *iph = ip_hdr(skb); |
ed7cbbce5 udp: Resolve NULL... |
535 |
return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport, |
8afdd99a1 udp: ipv4: fix an... |
536 |
iph->daddr, dport, inet_iif(skb), |
fb74c2773 net: ipv4: add se... |
537 |
inet_sdif(skb), udptable, skb); |
607c4aaf0 inet: Add udplib_... |
538 |
} |
63058308c udp: Add udp6_lib... |
539 540 541 |
struct sock *udp4_lib_lookup_skb(struct sk_buff *skb, __be16 sport, __be16 dport) { |
ed7cbbce5 udp: Resolve NULL... |
542 |
return __udp4_lib_lookup_skb(skb, sport, dport, &udp_table); |
63058308c udp: Add udp6_lib... |
543 544 |
} EXPORT_SYMBOL_GPL(udp4_lib_lookup_skb); |
ca065d0cf udp: no longer us... |
545 546 547 548 |
/* Must be called under rcu_read_lock(). * Does increment socket refcount. */ #if IS_ENABLED(CONFIG_NETFILTER_XT_MATCH_SOCKET) || \ |
30f581584 udp: provide udp{... |
549 550 |
IS_ENABLED(CONFIG_NETFILTER_XT_TARGET_TPROXY) || \ IS_ENABLED(CONFIG_NF_SOCKET_IPV4) |
bcd41303f udp: Export UDP s... |
551 552 553 |
struct sock *udp4_lib_lookup(struct net *net, __be32 saddr, __be16 sport, __be32 daddr, __be16 dport, int dif) { |
ca065d0cf udp: no longer us... |
554 555 556 |
struct sock *sk; sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport, |
fb74c2773 net: ipv4: add se... |
557 |
dif, 0, &udp_table, NULL); |
41c6d650f net: convert sock... |
558 |
if (sk && !refcount_inc_not_zero(&sk->sk_refcnt)) |
ca065d0cf udp: no longer us... |
559 560 |
sk = NULL; return sk; |
bcd41303f udp: Export UDP s... |
561 562 |
} EXPORT_SYMBOL_GPL(udp4_lib_lookup); |
ca065d0cf udp: no longer us... |
563 |
#endif |
bcd41303f udp: Export UDP s... |
564 |
|
421b3885b udp: ipv4: Add ud... |
565 566 567 |
static inline bool __udp_is_mcast_sock(struct net *net, struct sock *sk, __be16 loc_port, __be32 loc_addr, __be16 rmt_port, __be32 rmt_addr, |
fb74c2773 net: ipv4: add se... |
568 |
int dif, int sdif, unsigned short hnum) |
421b3885b udp: ipv4: Add ud... |
569 570 571 572 573 574 575 576 577 |
{ struct inet_sock *inet = inet_sk(sk); if (!net_eq(sock_net(sk), net) || udp_sk(sk)->udp_port_hash != hnum || (inet->inet_daddr && inet->inet_daddr != rmt_addr) || (inet->inet_dport != rmt_port && inet->inet_dport) || (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) || ipv6_only_sock(sk) || |
fb74c2773 net: ipv4: add se... |
578 579 |
(sk->sk_bound_dev_if && sk->sk_bound_dev_if != dif && sk->sk_bound_dev_if != sdif)) |
421b3885b udp: ipv4: Add ud... |
580 |
return false; |
60d9b0314 net: ipv4: add se... |
581 |
if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif)) |
421b3885b udp: ipv4: Add ud... |
582 583 584 |
return false; return true; } |
db8dac20d [UDP]: Revert udp... |
585 586 587 588 589 590 591 592 593 594 |
/* * This routine is called by the ICMP module when it gets some * sort of error condition. If err < 0 then the socket should * be closed and the error returned to the user. If err > 0 * it's just the icmp type << 8 | icmp code. * Header points to the ip header of the error packet. We move * on past this. Then (as it used to claim before adjustment) * header points to the first 8 bytes of the udp header. We need * to find the appropriate port. */ |
645ca708f udp: introduce st... |
595 |
void __udp4_lib_err(struct sk_buff *skb, u32 info, struct udp_table *udptable) |
db8dac20d [UDP]: Revert udp... |
596 597 |
{ struct inet_sock *inet; |
b71d1d426 inet: constify ip... |
598 |
const struct iphdr *iph = (const struct iphdr *)skb->data; |
c482c5685 udp: cleanups |
599 |
struct udphdr *uh = (struct udphdr *)(skb->data+(iph->ihl<<2)); |
db8dac20d [UDP]: Revert udp... |
600 601 602 603 604 |
const int type = icmp_hdr(skb)->type; const int code = icmp_hdr(skb)->code; struct sock *sk; int harderr; int err; |
fd54d716b inet: toss struct... |
605 |
struct net *net = dev_net(skb->dev); |
db8dac20d [UDP]: Revert udp... |
606 |
|
fd54d716b inet: toss struct... |
607 |
sk = __udp4_lib_lookup(net, iph->daddr, uh->dest, |
fb74c2773 net: ipv4: add se... |
608 609 |
iph->saddr, uh->source, skb->dev->ifindex, 0, udptable, NULL); |
51456b291 ipv4: coding styl... |
610 |
if (!sk) { |
5d3848bc3 net: rename ICMP_... |
611 |
__ICMP_INC_STATS(net, ICMP_MIB_INERRORS); |
db8dac20d [UDP]: Revert udp... |
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 |
return; /* No socket for error */ } err = 0; harderr = 0; inet = inet_sk(sk); switch (type) { default: case ICMP_TIME_EXCEEDED: err = EHOSTUNREACH; break; case ICMP_SOURCE_QUENCH: goto out; case ICMP_PARAMETERPROB: err = EPROTO; harderr = 1; break; case ICMP_DEST_UNREACH: if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */ |
363933955 ipv4: Handle PMTU... |
632 |
ipv4_sk_update_pmtu(skb, sk, info); |
db8dac20d [UDP]: Revert udp... |
633 634 635 636 637 638 639 640 641 642 643 644 645 |
if (inet->pmtudisc != IP_PMTUDISC_DONT) { err = EMSGSIZE; harderr = 1; break; } goto out; } err = EHOSTUNREACH; if (code <= NR_ICMP_UNREACH) { harderr = icmp_err_convert[code].fatal; err = icmp_err_convert[code].errno; } break; |
55be7a9c6 ipv4: Add redirec... |
646 647 |
case ICMP_REDIRECT: ipv4_sk_redirect(skb, sk); |
1a462d189 net: udp: do not ... |
648 |
goto out; |
db8dac20d [UDP]: Revert udp... |
649 650 651 652 653 654 655 656 657 |
} /* * RFC1122: OK. Passes ICMP errors back to application, as per * 4.1.3.3. */ if (!inet->recverr) { if (!harderr || sk->sk_state != TCP_ESTABLISHED) goto out; |
b1faf5666 net: sock_queue_e... |
658 |
} else |
c482c5685 udp: cleanups |
659 |
ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1)); |
b1faf5666 net: sock_queue_e... |
660 |
|
db8dac20d [UDP]: Revert udp... |
661 662 663 |
sk->sk_err = err; sk->sk_error_report(sk); out: |
ca065d0cf udp: no longer us... |
664 |
return; |
db8dac20d [UDP]: Revert udp... |
665 666 667 668 |
} void udp_err(struct sk_buff *skb, u32 info) { |
645ca708f udp: introduce st... |
669 |
__udp4_lib_err(skb, info, &udp_table); |
db8dac20d [UDP]: Revert udp... |
670 671 672 673 674 |
} /* * Throw away all pending data and cancel the corking. Socket is locked. */ |
36d926b94 [IPV6]: inet_sk(s... |
675 |
void udp_flush_pending_frames(struct sock *sk) |
db8dac20d [UDP]: Revert udp... |
676 677 678 679 680 681 682 683 684 |
{ struct udp_sock *up = udp_sk(sk); if (up->pending) { up->len = 0; up->pending = 0; ip_flush_pending_frames(sk); } } |
36d926b94 [IPV6]: inet_sk(s... |
685 |
EXPORT_SYMBOL(udp_flush_pending_frames); |
db8dac20d [UDP]: Revert udp... |
686 687 |
/** |
f6b9664f8 udp: Switch to ip... |
688 |
* udp4_hwcsum - handle outgoing HW checksumming |
db8dac20d [UDP]: Revert udp... |
689 690 |
* @skb: sk_buff containing the filled-in UDP header * (checksum field must be zeroed out) |
f6b9664f8 udp: Switch to ip... |
691 692 |
* @src: source IP address * @dst: destination IP address |
db8dac20d [UDP]: Revert udp... |
693 |
*/ |
c26bf4a51 pktgen: Add UDPCS... |
694 |
void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst) |
db8dac20d [UDP]: Revert udp... |
695 |
{ |
db8dac20d [UDP]: Revert udp... |
696 |
struct udphdr *uh = udp_hdr(skb); |
f6b9664f8 udp: Switch to ip... |
697 698 699 |
int offset = skb_transport_offset(skb); int len = skb->len - offset; int hlen = len; |
db8dac20d [UDP]: Revert udp... |
700 |
__wsum csum = 0; |
ebbe495f1 ipv4: use skb fra... |
701 |
if (!skb_has_frag_list(skb)) { |
db8dac20d [UDP]: Revert udp... |
702 703 704 705 706 |
/* * Only one fragment on the socket. */ skb->csum_start = skb_transport_header(skb) - skb->head; skb->csum_offset = offsetof(struct udphdr, check); |
f6b9664f8 udp: Switch to ip... |
707 708 |
uh->check = ~csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, 0); |
db8dac20d [UDP]: Revert udp... |
709 |
} else { |
ebbe495f1 ipv4: use skb fra... |
710 |
struct sk_buff *frags; |
db8dac20d [UDP]: Revert udp... |
711 712 713 714 715 |
/* * HW-checksum won't work as there are two or more * fragments on the socket so that all csums of sk_buffs * should be together */ |
ebbe495f1 ipv4: use skb fra... |
716 |
skb_walk_frags(skb, frags) { |
f6b9664f8 udp: Switch to ip... |
717 718 |
csum = csum_add(csum, frags->csum); hlen -= frags->len; |
ebbe495f1 ipv4: use skb fra... |
719 |
} |
db8dac20d [UDP]: Revert udp... |
720 |
|
f6b9664f8 udp: Switch to ip... |
721 |
csum = skb_checksum(skb, offset, hlen, csum); |
db8dac20d [UDP]: Revert udp... |
722 |
skb->ip_summed = CHECKSUM_NONE; |
db8dac20d [UDP]: Revert udp... |
723 724 725 726 727 |
uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum); if (uh->check == 0) uh->check = CSUM_MANGLED_0; } } |
c26bf4a51 pktgen: Add UDPCS... |
728 |
EXPORT_SYMBOL_GPL(udp4_hwcsum); |
db8dac20d [UDP]: Revert udp... |
729 |
|
af5fcba7f udp: Generic func... |
730 731 732 733 734 735 736 |
/* Function to set UDP checksum for an IPv4 UDP packet. This is intended * for the simple case like when setting the checksum for a UDP tunnel. */ void udp_set_csum(bool nocheck, struct sk_buff *skb, __be32 saddr, __be32 daddr, int len) { struct udphdr *uh = udp_hdr(skb); |
179bc67f6 net: local checks... |
737 |
if (nocheck) { |
af5fcba7f udp: Generic func... |
738 |
uh->check = 0; |
179bc67f6 net: local checks... |
739 |
} else if (skb_is_gso(skb)) { |
af5fcba7f udp: Generic func... |
740 |
uh->check = ~udp_v4_check(len, saddr, daddr, 0); |
179bc67f6 net: local checks... |
741 742 743 744 745 |
} else if (skb->ip_summed == CHECKSUM_PARTIAL) { uh->check = 0; uh->check = udp_v4_check(len, saddr, daddr, lco_csum(skb)); if (uh->check == 0) uh->check = CSUM_MANGLED_0; |
d75f1306d net: udp: always ... |
746 |
} else { |
af5fcba7f udp: Generic func... |
747 748 749 750 |
skb->ip_summed = CHECKSUM_PARTIAL; skb->csum_start = skb_transport_header(skb) - skb->head; skb->csum_offset = offsetof(struct udphdr, check); uh->check = ~udp_v4_check(len, saddr, daddr, 0); |
af5fcba7f udp: Generic func... |
751 752 753 |
} } EXPORT_SYMBOL(udp_set_csum); |
79ab05314 ipv4: udp: Elimin... |
754 |
static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4) |
db8dac20d [UDP]: Revert udp... |
755 |
{ |
f6b9664f8 udp: Switch to ip... |
756 |
struct sock *sk = skb->sk; |
db8dac20d [UDP]: Revert udp... |
757 |
struct inet_sock *inet = inet_sk(sk); |
db8dac20d [UDP]: Revert udp... |
758 759 760 |
struct udphdr *uh; int err = 0; int is_udplite = IS_UDPLITE(sk); |
f6b9664f8 udp: Switch to ip... |
761 762 |
int offset = skb_transport_offset(skb); int len = skb->len - offset; |
db8dac20d [UDP]: Revert udp... |
763 |
__wsum csum = 0; |
db8dac20d [UDP]: Revert udp... |
764 765 766 767 |
/* * Create a UDP header */ uh = udp_hdr(skb); |
f6b9664f8 udp: Switch to ip... |
768 |
uh->source = inet->inet_sport; |
79ab05314 ipv4: udp: Elimin... |
769 |
uh->dest = fl4->fl4_dport; |
f6b9664f8 udp: Switch to ip... |
770 |
uh->len = htons(len); |
db8dac20d [UDP]: Revert udp... |
771 772 773 |
uh->check = 0; if (is_udplite) /* UDP-Lite */ |
f6b9664f8 udp: Switch to ip... |
774 |
csum = udplite_csum(skb); |
db8dac20d [UDP]: Revert udp... |
775 |
|
ab2fb7e32 udp: remove unrea... |
776 |
else if (sk->sk_no_check_tx) { /* UDP csum off */ |
db8dac20d [UDP]: Revert udp... |
777 778 779 780 781 |
skb->ip_summed = CHECKSUM_NONE; goto send; } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */ |
79ab05314 ipv4: udp: Elimin... |
782 |
udp4_hwcsum(skb, fl4->saddr, fl4->daddr); |
db8dac20d [UDP]: Revert udp... |
783 |
goto send; |
f6b9664f8 udp: Switch to ip... |
784 785 |
} else csum = udp_csum(skb); |
db8dac20d [UDP]: Revert udp... |
786 787 |
/* add protocol-dependent pseudo-header */ |
79ab05314 ipv4: udp: Elimin... |
788 |
uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len, |
c482c5685 udp: cleanups |
789 |
sk->sk_protocol, csum); |
db8dac20d [UDP]: Revert udp... |
790 791 792 793 |
if (uh->check == 0) uh->check = CSUM_MANGLED_0; send: |
b5ec8eeac ipv4: fix ip_send... |
794 |
err = ip_send_skb(sock_net(sk), skb); |
6ce9e7b5f ip: Report qdisc ... |
795 796 |
if (err) { if (err == -ENOBUFS && !inet->recverr) { |
6aef70a85 net: snmp: kill v... |
797 798 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS, is_udplite); |
6ce9e7b5f ip: Report qdisc ... |
799 800 801 |
err = 0; } } else |
6aef70a85 net: snmp: kill v... |
802 803 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_OUTDATAGRAMS, is_udplite); |
f6b9664f8 udp: Switch to ip... |
804 805 806 807 808 809 |
return err; } /* * Push out all pending data as one UDP datagram. Socket is locked. */ |
8822b64a0 ipv6: call udp_pu... |
810 |
int udp_push_pending_frames(struct sock *sk) |
f6b9664f8 udp: Switch to ip... |
811 812 813 |
{ struct udp_sock *up = udp_sk(sk); struct inet_sock *inet = inet_sk(sk); |
b6f21b268 ipv4: Use flowi4 ... |
814 |
struct flowi4 *fl4 = &inet->cork.fl.u.ip4; |
f6b9664f8 udp: Switch to ip... |
815 816 |
struct sk_buff *skb; int err = 0; |
77968b782 ipv4: Pass flow k... |
817 |
skb = ip_finish_skb(sk, fl4); |
f6b9664f8 udp: Switch to ip... |
818 819 |
if (!skb) goto out; |
79ab05314 ipv4: udp: Elimin... |
820 |
err = udp_send_skb(skb, fl4); |
f6b9664f8 udp: Switch to ip... |
821 |
|
db8dac20d [UDP]: Revert udp... |
822 823 824 |
out: up->len = 0; up->pending = 0; |
db8dac20d [UDP]: Revert udp... |
825 826 |
return err; } |
8822b64a0 ipv6: call udp_pu... |
827 |
EXPORT_SYMBOL(udp_push_pending_frames); |
db8dac20d [UDP]: Revert udp... |
828 |
|
1b7841404 net: Remove iocb ... |
829 |
int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) |
db8dac20d [UDP]: Revert udp... |
830 831 832 |
{ struct inet_sock *inet = inet_sk(sk); struct udp_sock *up = udp_sk(sk); |
e474995f2 udp: Use flow key... |
833 |
struct flowi4 fl4_stack; |
b6f21b268 ipv4: Use flowi4 ... |
834 |
struct flowi4 *fl4; |
db8dac20d [UDP]: Revert udp... |
835 836 837 838 839 840 841 842 843 844 845 |
int ulen = len; struct ipcm_cookie ipc; struct rtable *rt = NULL; int free = 0; int connected = 0; __be32 daddr, faddr, saddr; __be16 dport; u8 tos; int err, is_udplite = IS_UDPLITE(sk); int corkreq = up->corkflag || msg->msg_flags&MSG_MORE; int (*getfrag)(void *, char *, int, int, int, struct sk_buff *); |
903ab86d1 udp: Add lockless... |
846 |
struct sk_buff *skb; |
f6d8bd051 inet: add RCU pro... |
847 |
struct ip_options_data opt_copy; |
db8dac20d [UDP]: Revert udp... |
848 849 850 851 852 853 854 |
if (len > 0xFFFF) return -EMSGSIZE; /* * Check the flags. */ |
c482c5685 udp: cleanups |
855 |
if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */ |
db8dac20d [UDP]: Revert udp... |
856 857 858 |
return -EOPNOTSUPP; ipc.opt = NULL; |
2244d07bf net: simplify fla... |
859 |
ipc.tx_flags = 0; |
aa6615814 ipv4: processing ... |
860 861 |
ipc.ttl = 0; ipc.tos = -1; |
db8dac20d [UDP]: Revert udp... |
862 |
|
903ab86d1 udp: Add lockless... |
863 |
getfrag = is_udplite ? udplite_getfrag : ip_generic_getfrag; |
f5fca6086 ipv4: Pass flow k... |
864 |
fl4 = &inet->cork.fl.u.ip4; |
db8dac20d [UDP]: Revert udp... |
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 |
if (up->pending) { /* * There are pending frames. * The socket lock must be held while it's corked. */ lock_sock(sk); if (likely(up->pending)) { if (unlikely(up->pending != AF_INET)) { release_sock(sk); return -EINVAL; } goto do_append_data; } release_sock(sk); } ulen += sizeof(struct udphdr); /* * Get and verify the address. */ if (msg->msg_name) { |
342dfc306 net: add build-ti... |
886 |
DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name); |
db8dac20d [UDP]: Revert udp... |
887 888 889 890 891 892 893 894 895 896 897 898 899 900 |
if (msg->msg_namelen < sizeof(*usin)) return -EINVAL; if (usin->sin_family != AF_INET) { if (usin->sin_family != AF_UNSPEC) return -EAFNOSUPPORT; } daddr = usin->sin_addr.s_addr; dport = usin->sin_port; if (dport == 0) return -EINVAL; } else { if (sk->sk_state != TCP_ESTABLISHED) return -EDESTADDRREQ; |
c720c7e83 inet: rename some... |
901 902 |
daddr = inet->inet_daddr; dport = inet->inet_dport; |
db8dac20d [UDP]: Revert udp... |
903 904 905 906 907 |
/* Open fast path for connected socket. Route will not be used, if at least one option is set. */ connected = 1; } |
db8dac20d [UDP]: Revert udp... |
908 |
|
c14ac9451 sock: enable time... |
909 910 |
ipc.sockc.tsflags = sk->sk_tsflags; ipc.addr = inet->inet_saddr; |
db8dac20d [UDP]: Revert udp... |
911 |
ipc.oif = sk->sk_bound_dev_if; |
bf84a0106 net: sock: make s... |
912 |
|
db8dac20d [UDP]: Revert udp... |
913 |
if (msg->msg_controllen) { |
24025c465 ipv4: process soc... |
914 |
err = ip_cmsg_send(sk, msg, &ipc, sk->sk_family == AF_INET6); |
919483096 ipv4: fix memory ... |
915 916 |
if (unlikely(err)) { kfree(ipc.opt); |
db8dac20d [UDP]: Revert udp... |
917 |
return err; |
919483096 ipv4: fix memory ... |
918 |
} |
db8dac20d [UDP]: Revert udp... |
919 920 921 922 |
if (ipc.opt) free = 1; connected = 0; } |
f6d8bd051 inet: add RCU pro... |
923 924 925 926 927 928 929 930 931 932 933 934 |
if (!ipc.opt) { struct ip_options_rcu *inet_opt; rcu_read_lock(); inet_opt = rcu_dereference(inet->inet_opt); if (inet_opt) { memcpy(&opt_copy, inet_opt, sizeof(*inet_opt) + inet_opt->opt.optlen); ipc.opt = &opt_copy.opt; } rcu_read_unlock(); } |
db8dac20d [UDP]: Revert udp... |
935 936 937 |
saddr = ipc.addr; ipc.addr = faddr = daddr; |
c14ac9451 sock: enable time... |
938 |
sock_tx_timestamp(sk, ipc.sockc.tsflags, &ipc.tx_flags); |
f6d8bd051 inet: add RCU pro... |
939 |
if (ipc.opt && ipc.opt->opt.srr) { |
a7aea8e27 ipv4: fix memory ... |
940 941 942 943 |
if (!daddr) { err = -EINVAL; goto out_free; } |
f6d8bd051 inet: add RCU pro... |
944 |
faddr = ipc.opt->opt.faddr; |
db8dac20d [UDP]: Revert udp... |
945 946 |
connected = 0; } |
aa6615814 ipv4: processing ... |
947 |
tos = get_rttos(&ipc, inet); |
db8dac20d [UDP]: Revert udp... |
948 949 |
if (sock_flag(sk, SOCK_LOCALROUTE) || (msg->msg_flags & MSG_DONTROUTE) || |
f6d8bd051 inet: add RCU pro... |
950 |
(ipc.opt && ipc.opt->opt.is_strictroute)) { |
db8dac20d [UDP]: Revert udp... |
951 952 953 954 955 956 957 958 959 960 |
tos |= RTO_ONLINK; connected = 0; } if (ipv4_is_multicast(daddr)) { if (!ipc.oif) ipc.oif = inet->mc_index; if (!saddr) saddr = inet->mc_addr; connected = 0; |
76e21053b ipv4: Implement I... |
961 962 |
} else if (!ipc.oif) ipc.oif = inet->uc_index; |
db8dac20d [UDP]: Revert udp... |
963 964 |
if (connected) |
c482c5685 udp: cleanups |
965 |
rt = (struct rtable *)sk_dst_check(sk, 0); |
db8dac20d [UDP]: Revert udp... |
966 |
|
51456b291 ipv4: coding styl... |
967 |
if (!rt) { |
84a3aa000 ipv4: prepare net... |
968 |
struct net *net = sock_net(sk); |
9a24abfa4 udp: Handle VRF d... |
969 |
__u8 flow_flags = inet_sk_flowi_flags(sk); |
84a3aa000 ipv4: prepare net... |
970 |
|
e474995f2 udp: Use flow key... |
971 |
fl4 = &fl4_stack; |
9a24abfa4 udp: Handle VRF d... |
972 |
|
e474995f2 udp: Use flow key... |
973 |
flowi4_init_output(fl4, ipc.oif, sk->sk_mark, tos, |
c0951cbcf ipv4: Use flowi4_... |
974 |
RT_SCOPE_UNIVERSE, sk->sk_protocol, |
9a24abfa4 udp: Handle VRF d... |
975 |
flow_flags, |
e2d118a1c net: inet: Suppor... |
976 977 |
faddr, saddr, dport, inet->inet_sport, sk->sk_uid); |
c0951cbcf ipv4: Use flowi4_... |
978 |
|
e474995f2 udp: Use flow key... |
979 980 |
security_sk_classify_flow(sk, flowi4_to_flowi(fl4)); rt = ip_route_output_flow(net, fl4, sk); |
b23dd4fe4 ipv4: Make output... |
981 982 |
if (IS_ERR(rt)) { err = PTR_ERR(rt); |
06dc94b1e ipv4: Fix crash i... |
983 |
rt = NULL; |
db8dac20d [UDP]: Revert udp... |
984 |
if (err == -ENETUNREACH) |
f1d8cba61 inet: fix possibl... |
985 |
IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES); |
db8dac20d [UDP]: Revert udp... |
986 987 988 989 990 991 992 993 |
goto out; } err = -EACCES; if ((rt->rt_flags & RTCF_BROADCAST) && !sock_flag(sk, SOCK_BROADCAST)) goto out; if (connected) |
d8d1f30b9 net-next: remove ... |
994 |
sk_dst_set(sk, dst_clone(&rt->dst)); |
db8dac20d [UDP]: Revert udp... |
995 996 997 998 999 |
} if (msg->msg_flags&MSG_CONFIRM) goto do_confirm; back_from_confirm: |
e474995f2 udp: Use flow key... |
1000 |
saddr = fl4->saddr; |
db8dac20d [UDP]: Revert udp... |
1001 |
if (!ipc.addr) |
e474995f2 udp: Use flow key... |
1002 |
daddr = ipc.addr = fl4->daddr; |
db8dac20d [UDP]: Revert udp... |
1003 |
|
903ab86d1 udp: Add lockless... |
1004 1005 |
/* Lockless fast path for the non-corking case. */ if (!corkreq) { |
f69e6d131 ip_generic_getfra... |
1006 |
skb = ip_make_skb(sk, fl4, getfrag, msg, ulen, |
903ab86d1 udp: Add lockless... |
1007 1008 1009 |
sizeof(struct udphdr), &ipc, &rt, msg->msg_flags); err = PTR_ERR(skb); |
50c3a487d ipv4: Use IS_ERR_... |
1010 |
if (!IS_ERR_OR_NULL(skb)) |
79ab05314 ipv4: udp: Elimin... |
1011 |
err = udp_send_skb(skb, fl4); |
903ab86d1 udp: Add lockless... |
1012 1013 |
goto out; } |
db8dac20d [UDP]: Revert udp... |
1014 1015 1016 1017 1018 |
lock_sock(sk); if (unlikely(up->pending)) { /* The socket is already corked while preparing it. */ /* ... which is an evident application bug. --ANK */ release_sock(sk); |
197df02cb udp: make some me... |
1019 1020 |
net_dbg_ratelimited("socket already corked "); |
db8dac20d [UDP]: Revert udp... |
1021 1022 1023 1024 1025 1026 |
err = -EINVAL; goto out; } /* * Now cork the socket to pend data. */ |
b6f21b268 ipv4: Use flowi4 ... |
1027 1028 1029 |
fl4 = &inet->cork.fl.u.ip4; fl4->daddr = daddr; fl4->saddr = saddr; |
9cce96df5 net: Put fl4_* ma... |
1030 1031 |
fl4->fl4_dport = dport; fl4->fl4_sport = inet->inet_sport; |
db8dac20d [UDP]: Revert udp... |
1032 1033 1034 1035 |
up->pending = AF_INET; do_append_data: up->len += ulen; |
f69e6d131 ip_generic_getfra... |
1036 |
err = ip_append_data(sk, fl4, getfrag, msg, ulen, |
f5fca6086 ipv4: Pass flow k... |
1037 1038 |
sizeof(struct udphdr), &ipc, &rt, corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags); |
db8dac20d [UDP]: Revert udp... |
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 |
if (err) udp_flush_pending_frames(sk); else if (!corkreq) err = udp_push_pending_frames(sk); else if (unlikely(skb_queue_empty(&sk->sk_write_queue))) up->pending = 0; release_sock(sk); out: ip_rt_put(rt); |
a7aea8e27 ipv4: fix memory ... |
1049 |
out_free: |
db8dac20d [UDP]: Revert udp... |
1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 |
if (free) kfree(ipc.opt); if (!err) return len; /* * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting * ENOBUFS might not be good (it's not tunable per se), but otherwise * we don't have a good statistic (IpOutDiscards but it can be too many * things). We could add another new stat but at least for now that * seems like overkill. */ if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) { |
6aef70a85 net: snmp: kill v... |
1062 1063 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS, is_udplite); |
db8dac20d [UDP]: Revert udp... |
1064 1065 1066 1067 |
} return err; do_confirm: |
0dec879f6 net: use dst_conf... |
1068 1069 |
if (msg->msg_flags & MSG_PROBE) dst_confirm_neigh(&rt->dst, &fl4->daddr); |
db8dac20d [UDP]: Revert udp... |
1070 1071 1072 1073 1074 |
if (!(msg->msg_flags&MSG_PROBE) || len) goto back_from_confirm; err = 0; goto out; } |
c482c5685 udp: cleanups |
1075 |
EXPORT_SYMBOL(udp_sendmsg); |
db8dac20d [UDP]: Revert udp... |
1076 1077 1078 1079 |
int udp_sendpage(struct sock *sk, struct page *page, int offset, size_t size, int flags) { |
f5fca6086 ipv4: Pass flow k... |
1080 |
struct inet_sock *inet = inet_sk(sk); |
db8dac20d [UDP]: Revert udp... |
1081 1082 |
struct udp_sock *up = udp_sk(sk); int ret; |
d3f7d56a7 net: update consu... |
1083 1084 |
if (flags & MSG_SENDPAGE_NOTLAST) flags |= MSG_MORE; |
db8dac20d [UDP]: Revert udp... |
1085 1086 1087 1088 1089 1090 1091 |
if (!up->pending) { struct msghdr msg = { .msg_flags = flags|MSG_MORE }; /* Call udp_sendmsg to specify destination address which * sendpage interface can't pass. * This will succeed only when the socket is connected. */ |
1b7841404 net: Remove iocb ... |
1092 |
ret = udp_sendmsg(sk, &msg, 0); |
db8dac20d [UDP]: Revert udp... |
1093 1094 1095 1096 1097 1098 1099 1100 |
if (ret < 0) return ret; } lock_sock(sk); if (unlikely(!up->pending)) { release_sock(sk); |
197df02cb udp: make some me... |
1101 1102 |
net_dbg_ratelimited("cork failed "); |
db8dac20d [UDP]: Revert udp... |
1103 1104 |
return -EINVAL; } |
f5fca6086 ipv4: Pass flow k... |
1105 1106 |
ret = ip_append_page(sk, &inet->cork.fl.u.ip4, page, offset, size, flags); |
db8dac20d [UDP]: Revert udp... |
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 |
if (ret == -EOPNOTSUPP) { release_sock(sk); return sock_no_sendpage(sk->sk_socket, page, offset, size, flags); } if (ret < 0) { udp_flush_pending_frames(sk); goto out; } up->len += size; if (!(up->corkflag || (flags&MSG_MORE))) ret = udp_push_pending_frames(sk); if (!ret) ret = size; out: release_sock(sk); return ret; } |
dce4551cb udp: preserve hea... |
1126 |
#define UDP_SKB_IS_STATELESS 0x80000000 |
b65ac4467 udp: try to avoid... |
1127 1128 |
static void udp_set_dev_scratch(struct sk_buff *skb) { |
dce4551cb udp: preserve hea... |
1129 |
struct udp_dev_scratch *scratch = udp_skb_scratch(skb); |
b65ac4467 udp: try to avoid... |
1130 1131 |
BUILD_BUG_ON(sizeof(struct udp_dev_scratch) > sizeof(long)); |
dce4551cb udp: preserve hea... |
1132 1133 |
scratch->_tsize_state = skb->truesize; #if BITS_PER_LONG == 64 |
b65ac4467 udp: try to avoid... |
1134 1135 1136 |
scratch->len = skb->len; scratch->csum_unnecessary = !!skb_csum_unnecessary(skb); scratch->is_linear = !skb_is_nonlinear(skb); |
dce4551cb udp: preserve hea... |
1137 |
#endif |
3bdefdf9d udp: no need to p... |
1138 1139 1140 1141 1142 |
/* all head states execept sp (dst, sk, nf) are always cleared by * udp_rcv() and we need to preserve secpath, if present, to eventually * process IP_CMSG_PASSSEC at recvmsg() time */ if (likely(!skb_sec_path(skb))) |
dce4551cb udp: preserve hea... |
1143 |
scratch->_tsize_state |= UDP_SKB_IS_STATELESS; |
b65ac4467 udp: try to avoid... |
1144 1145 1146 1147 |
} static int udp_skb_truesize(struct sk_buff *skb) { |
dce4551cb udp: preserve hea... |
1148 |
return udp_skb_scratch(skb)->_tsize_state & ~UDP_SKB_IS_STATELESS; |
b65ac4467 udp: try to avoid... |
1149 |
} |
dce4551cb udp: preserve hea... |
1150 |
static bool udp_skb_has_head_state(struct sk_buff *skb) |
b65ac4467 udp: try to avoid... |
1151 |
{ |
dce4551cb udp: preserve hea... |
1152 |
return !(udp_skb_scratch(skb)->_tsize_state & UDP_SKB_IS_STATELESS); |
b65ac4467 udp: try to avoid... |
1153 |
} |
b65ac4467 udp: try to avoid... |
1154 |
|
7c13f97ff udp: do fwd memor... |
1155 |
/* fully reclaim rmem/fwd memory allocated for skb */ |
6dfb4367c udp: keep the sk_... |
1156 1157 |
static void udp_rmem_release(struct sock *sk, int size, int partial, bool rx_queue_lock_held) |
f970bd9e3 udp: implement me... |
1158 |
{ |
6b229cf77 udp: add batching... |
1159 |
struct udp_sock *up = udp_sk(sk); |
2276f58ac udp: use a separa... |
1160 |
struct sk_buff_head *sk_queue; |
f970bd9e3 udp: implement me... |
1161 |
int amt; |
6b229cf77 udp: add batching... |
1162 1163 1164 1165 |
if (likely(partial)) { up->forward_deficit += size; size = up->forward_deficit; if (size < (sk->sk_rcvbuf >> 2) && |
2276f58ac udp: use a separa... |
1166 |
!skb_queue_empty(&up->reader_queue)) |
6b229cf77 udp: add batching... |
1167 1168 1169 1170 1171 |
return; } else { size += up->forward_deficit; } up->forward_deficit = 0; |
6dfb4367c udp: keep the sk_... |
1172 1173 1174 |
/* acquire the sk_receive_queue for fwd allocated memory scheduling, * if the called don't held it already */ |
2276f58ac udp: use a separa... |
1175 |
sk_queue = &sk->sk_receive_queue; |
6dfb4367c udp: keep the sk_... |
1176 1177 |
if (!rx_queue_lock_held) spin_lock(&sk_queue->lock); |
2276f58ac udp: use a separa... |
1178 |
|
f970bd9e3 udp: implement me... |
1179 1180 1181 |
sk->sk_forward_alloc += size; amt = (sk->sk_forward_alloc - partial) & ~(SK_MEM_QUANTUM - 1); sk->sk_forward_alloc -= amt; |
f970bd9e3 udp: implement me... |
1182 1183 1184 |
if (amt) __sk_mem_reduce_allocated(sk, amt >> SK_MEM_QUANTUM_SHIFT); |
02ab0d139 udp: udp_rmem_rel... |
1185 1186 |
atomic_sub(size, &sk->sk_rmem_alloc); |
2276f58ac udp: use a separa... |
1187 1188 1189 |
/* this can save us from acquiring the rx queue lock on next receive */ skb_queue_splice_tail_init(sk_queue, &up->reader_queue); |
6dfb4367c udp: keep the sk_... |
1190 1191 |
if (!rx_queue_lock_held) spin_unlock(&sk_queue->lock); |
f970bd9e3 udp: implement me... |
1192 |
} |
2276f58ac udp: use a separa... |
1193 |
/* Note: called with reader_queue.lock held. |
c84d94905 udp: copy skb->tr... |
1194 1195 1196 1197 |
* Instead of using skb->truesize here, find a copy of it in skb->dev_scratch * This avoids a cache line miss while receive_queue lock is held. * Look at __udp_enqueue_schedule_skb() to find where this copy is done. */ |
7c13f97ff udp: do fwd memor... |
1198 |
void udp_skb_destructor(struct sock *sk, struct sk_buff *skb) |
f970bd9e3 udp: implement me... |
1199 |
{ |
b65ac4467 udp: try to avoid... |
1200 1201 |
prefetch(&skb->data); udp_rmem_release(sk, udp_skb_truesize(skb), 1, false); |
f970bd9e3 udp: implement me... |
1202 |
} |
7c13f97ff udp: do fwd memor... |
1203 |
EXPORT_SYMBOL(udp_skb_destructor); |
f970bd9e3 udp: implement me... |
1204 |
|
6dfb4367c udp: keep the sk_... |
1205 |
/* as above, but the caller held the rx queue lock, too */ |
64f5102dc udp: make functio... |
1206 |
static void udp_skb_dtor_locked(struct sock *sk, struct sk_buff *skb) |
6dfb4367c udp: keep the sk_... |
1207 |
{ |
b65ac4467 udp: try to avoid... |
1208 1209 |
prefetch(&skb->data); udp_rmem_release(sk, udp_skb_truesize(skb), 1, true); |
6dfb4367c udp: keep the sk_... |
1210 |
} |
4b272750d udp: add busylock... |
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 |
/* Idea of busylocks is to let producers grab an extra spinlock * to relieve pressure on the receive_queue spinlock shared by consumer. * Under flood, this means that only one producer can be in line * trying to acquire the receive_queue spinlock. * These busylock can be allocated on a per cpu manner, instead of a * per socket one (that would consume a cache line per socket) */ static int udp_busylocks_log __read_mostly; static spinlock_t *udp_busylocks __read_mostly; static spinlock_t *busylock_acquire(void *ptr) { spinlock_t *busy; busy = udp_busylocks + hash_ptr(ptr, udp_busylocks_log); spin_lock(busy); return busy; } static void busylock_release(spinlock_t *busy) { if (busy) spin_unlock(busy); } |
f970bd9e3 udp: implement me... |
1235 1236 1237 1238 |
int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb) { struct sk_buff_head *list = &sk->sk_receive_queue; int rmem, delta, amt, err = -ENOMEM; |
4b272750d udp: add busylock... |
1239 |
spinlock_t *busy = NULL; |
c8c8b1270 udp: under rx pre... |
1240 |
int size; |
f970bd9e3 udp: implement me... |
1241 1242 1243 1244 1245 |
/* try to avoid the costly atomic add/sub pair when the receive * queue is full; always allow at least a packet */ rmem = atomic_read(&sk->sk_rmem_alloc); |
363dc73ac udp: be less cons... |
1246 |
if (rmem > sk->sk_rcvbuf) |
f970bd9e3 udp: implement me... |
1247 |
goto drop; |
c8c8b1270 udp: under rx pre... |
1248 1249 1250 1251 1252 1253 |
/* Under mem pressure, it might be helpful to help udp_recvmsg() * having linear skbs : * - Reduce memory overhead and thus increase receive queue capacity * - Less cache line misses at copyout() time * - Less work at consume_skb() (less alien page frag freeing) */ |
4b272750d udp: add busylock... |
1254 |
if (rmem > (sk->sk_rcvbuf >> 1)) { |
c8c8b1270 udp: under rx pre... |
1255 |
skb_condense(skb); |
4b272750d udp: add busylock... |
1256 1257 1258 |
busy = busylock_acquire(sk); } |
c8c8b1270 udp: under rx pre... |
1259 |
size = skb->truesize; |
b65ac4467 udp: try to avoid... |
1260 |
udp_set_dev_scratch(skb); |
c8c8b1270 udp: under rx pre... |
1261 |
|
f970bd9e3 udp: implement me... |
1262 1263 1264 1265 |
/* we drop only if the receive buf is full and the receive * queue contains some other skb */ rmem = atomic_add_return(size, &sk->sk_rmem_alloc); |
363dc73ac udp: be less cons... |
1266 |
if (rmem > (size + sk->sk_rcvbuf)) |
f970bd9e3 udp: implement me... |
1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 |
goto uncharge_drop; spin_lock(&list->lock); if (size >= sk->sk_forward_alloc) { amt = sk_mem_pages(size); delta = amt << SK_MEM_QUANTUM_SHIFT; if (!__sk_mem_raise_allocated(sk, delta, amt, SK_MEM_RECV)) { err = -ENOBUFS; spin_unlock(&list->lock); goto uncharge_drop; } sk->sk_forward_alloc += delta; } sk->sk_forward_alloc -= size; |
7c13f97ff udp: do fwd memor... |
1283 1284 1285 |
/* no need to setup a destructor, we will explicitly release the * forward allocated memory on dequeue */ |
f970bd9e3 udp: implement me... |
1286 1287 1288 1289 1290 1291 1292 |
sock_skb_set_dropcount(sk, skb); __skb_queue_tail(list, skb); spin_unlock(&list->lock); if (!sock_flag(sk, SOCK_DEAD)) sk->sk_data_ready(sk); |
4b272750d udp: add busylock... |
1293 |
busylock_release(busy); |
f970bd9e3 udp: implement me... |
1294 1295 1296 1297 1298 1299 1300 |
return 0; uncharge_drop: atomic_sub(skb->truesize, &sk->sk_rmem_alloc); drop: atomic_inc(&sk->sk_drops); |
4b272750d udp: add busylock... |
1301 |
busylock_release(busy); |
f970bd9e3 udp: implement me... |
1302 1303 1304 |
return err; } EXPORT_SYMBOL_GPL(__udp_enqueue_schedule_skb); |
c915fe13c udplite: fix NULL... |
1305 |
void udp_destruct_sock(struct sock *sk) |
f970bd9e3 udp: implement me... |
1306 1307 |
{ /* reclaim completely the forward allocated memory */ |
2276f58ac udp: use a separa... |
1308 |
struct udp_sock *up = udp_sk(sk); |
7c13f97ff udp: do fwd memor... |
1309 1310 |
unsigned int total = 0; struct sk_buff *skb; |
2276f58ac udp: use a separa... |
1311 1312 |
skb_queue_splice_tail_init(&sk->sk_receive_queue, &up->reader_queue); while ((skb = __skb_dequeue(&up->reader_queue)) != NULL) { |
7c13f97ff udp: do fwd memor... |
1313 1314 1315 |
total += skb->truesize; kfree_skb(skb); } |
6dfb4367c udp: keep the sk_... |
1316 |
udp_rmem_release(sk, total, 0, true); |
7c13f97ff udp: do fwd memor... |
1317 |
|
f970bd9e3 udp: implement me... |
1318 1319 |
inet_sock_destruct(sk); } |
c915fe13c udplite: fix NULL... |
1320 |
EXPORT_SYMBOL_GPL(udp_destruct_sock); |
f970bd9e3 udp: implement me... |
1321 1322 1323 |
int udp_init_sock(struct sock *sk) { |
2276f58ac udp: use a separa... |
1324 |
skb_queue_head_init(&udp_sk(sk)->reader_queue); |
f970bd9e3 udp: implement me... |
1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 |
sk->sk_destruct = udp_destruct_sock; return 0; } EXPORT_SYMBOL_GPL(udp_init_sock); void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len) { if (unlikely(READ_ONCE(sk->sk_peek_off) >= 0)) { bool slow = lock_sock_fast(sk); sk_peek_offset_bwd(sk, len); unlock_sock_fast(sk, slow); } |
0a463c78d udp: avoid a cach... |
1338 |
|
ca2c1418e udp: drop head st... |
1339 1340 |
if (!skb_unref(skb)) return; |
dce4551cb udp: preserve hea... |
1341 1342 |
/* In the more common cases we cleared the head states previously, * see __udp_queue_rcv_skb(). |
0ddf3fb2c udp: preserve skb... |
1343 |
*/ |
dce4551cb udp: preserve hea... |
1344 |
if (unlikely(udp_skb_has_head_state(skb))) |
0ddf3fb2c udp: preserve skb... |
1345 |
skb_release_head_state(skb); |
ca2c1418e udp: drop head st... |
1346 |
__consume_stateless_skb(skb); |
f970bd9e3 udp: implement me... |
1347 1348 |
} EXPORT_SYMBOL_GPL(skb_consume_udp); |
2276f58ac udp: use a separa... |
1349 1350 1351 1352 1353 |
static struct sk_buff *__first_packet_length(struct sock *sk, struct sk_buff_head *rcvq, int *total) { struct sk_buff *skb; |
9bd780f5e udp: fix poll() |
1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 |
while ((skb = skb_peek(rcvq)) != NULL) { if (udp_lib_checksum_complete(skb)) { __UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, IS_UDPLITE(sk)); __UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, IS_UDPLITE(sk)); atomic_inc(&sk->sk_drops); __skb_unlink(skb, rcvq); *total += skb->truesize; kfree_skb(skb); } else { /* the csum related bits could be changed, refresh * the scratch area */ udp_set_dev_scratch(skb); break; } |
2276f58ac udp: use a separa... |
1371 1372 1373 |
} return skb; } |
855846720 udp: Fix udp_poll... |
1374 1375 1376 1377 1378 |
/** * first_packet_length - return length of first packet in receive queue * @sk: socket * * Drops all bad checksum frames, until a valid one is found. |
e83c6744e udp: fix poll() i... |
1379 |
* Returns the length of found skb, or -1 if none is found. |
855846720 udp: Fix udp_poll... |
1380 |
*/ |
e83c6744e udp: fix poll() i... |
1381 |
static int first_packet_length(struct sock *sk) |
855846720 udp: Fix udp_poll... |
1382 |
{ |
2276f58ac udp: use a separa... |
1383 1384 |
struct sk_buff_head *rcvq = &udp_sk(sk)->reader_queue; struct sk_buff_head *sk_queue = &sk->sk_receive_queue; |
855846720 udp: Fix udp_poll... |
1385 |
struct sk_buff *skb; |
7c13f97ff udp: do fwd memor... |
1386 |
int total = 0; |
e83c6744e udp: fix poll() i... |
1387 |
int res; |
855846720 udp: Fix udp_poll... |
1388 |
|
855846720 udp: Fix udp_poll... |
1389 |
spin_lock_bh(&rcvq->lock); |
2276f58ac udp: use a separa... |
1390 1391 1392 1393 1394 1395 1396 |
skb = __first_packet_length(sk, rcvq, &total); if (!skb && !skb_queue_empty(sk_queue)) { spin_lock(&sk_queue->lock); skb_queue_splice_tail_init(sk_queue, rcvq); spin_unlock(&sk_queue->lock); skb = __first_packet_length(sk, rcvq, &total); |
855846720 udp: Fix udp_poll... |
1397 |
} |
e83c6744e udp: fix poll() i... |
1398 |
res = skb ? skb->len : -1; |
7c13f97ff udp: do fwd memor... |
1399 |
if (total) |
6dfb4367c udp: keep the sk_... |
1400 |
udp_rmem_release(sk, total, 1, false); |
855846720 udp: Fix udp_poll... |
1401 |
spin_unlock_bh(&rcvq->lock); |
855846720 udp: Fix udp_poll... |
1402 1403 |
return res; } |
1da177e4c Linux-2.6.12-rc2 |
1404 1405 1406 |
/* * IOCTL requests applicable to the UDP protocol */ |
e905a9eda [NET] IPV4: Fix w... |
1407 |
|
1da177e4c Linux-2.6.12-rc2 |
1408 1409 |
int udp_ioctl(struct sock *sk, int cmd, unsigned long arg) { |
6516c6557 [UDP]: ipv4 white... |
1410 1411 |
switch (cmd) { case SIOCOUTQ: |
1da177e4c Linux-2.6.12-rc2 |
1412 |
{ |
31e6d363a net: correct off-... |
1413 |
int amount = sk_wmem_alloc_get(sk); |
6516c6557 [UDP]: ipv4 white... |
1414 1415 |
return put_user(amount, (int __user *)arg); } |
1da177e4c Linux-2.6.12-rc2 |
1416 |
|
6516c6557 [UDP]: ipv4 white... |
1417 1418 |
case SIOCINQ: { |
e83c6744e udp: fix poll() i... |
1419 |
int amount = max_t(int, 0, first_packet_length(sk)); |
6516c6557 [UDP]: ipv4 white... |
1420 |
|
6516c6557 [UDP]: ipv4 white... |
1421 1422 |
return put_user(amount, (int __user *)arg); } |
1da177e4c Linux-2.6.12-rc2 |
1423 |
|
6516c6557 [UDP]: ipv4 white... |
1424 1425 |
default: return -ENOIOCTLCMD; |
1da177e4c Linux-2.6.12-rc2 |
1426 |
} |
6516c6557 [UDP]: ipv4 white... |
1427 1428 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
1429 |
} |
c482c5685 udp: cleanups |
1430 |
EXPORT_SYMBOL(udp_ioctl); |
1da177e4c Linux-2.6.12-rc2 |
1431 |
|
2276f58ac udp: use a separa... |
1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 |
struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags, int noblock, int *peeked, int *off, int *err) { struct sk_buff_head *sk_queue = &sk->sk_receive_queue; struct sk_buff_head *queue; struct sk_buff *last; long timeo; int error; queue = &udp_sk(sk)->reader_queue; flags |= noblock ? MSG_DONTWAIT : 0; timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); do { struct sk_buff *skb; error = sock_error(sk); if (error) break; error = -EAGAIN; *peeked = 0; do { |
2276f58ac udp: use a separa... |
1454 1455 1456 |
spin_lock_bh(&queue->lock); skb = __skb_try_recv_from_queue(sk, queue, flags, udp_skb_destructor, |
de321ed38 net: fix __skb_tr... |
1457 |
peeked, off, err, |
2276f58ac udp: use a separa... |
1458 1459 1460 |
&last); if (skb) { spin_unlock_bh(&queue->lock); |
2276f58ac udp: use a separa... |
1461 1462 1463 1464 1465 1466 1467 |
return skb; } if (skb_queue_empty(sk_queue)) { spin_unlock_bh(&queue->lock); goto busy_check; } |
6dfb4367c udp: keep the sk_... |
1468 1469 1470 1471 1472 |
/* refill the reader queue and walk it again * keep both queues locked to avoid re-acquiring * the sk_receive_queue lock if fwd memory scheduling * is needed. */ |
2276f58ac udp: use a separa... |
1473 1474 |
spin_lock(&sk_queue->lock); skb_queue_splice_tail_init(sk_queue, queue); |
2276f58ac udp: use a separa... |
1475 1476 |
skb = __skb_try_recv_from_queue(sk, queue, flags, |
6dfb4367c udp: keep the sk_... |
1477 |
udp_skb_dtor_locked, |
de321ed38 net: fix __skb_tr... |
1478 |
peeked, off, err, |
2276f58ac udp: use a separa... |
1479 |
&last); |
6dfb4367c udp: keep the sk_... |
1480 |
spin_unlock(&sk_queue->lock); |
2276f58ac udp: use a separa... |
1481 |
spin_unlock_bh(&queue->lock); |
de321ed38 net: fix __skb_tr... |
1482 |
if (skb) |
2276f58ac udp: use a separa... |
1483 |
return skb; |
2276f58ac udp: use a separa... |
1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 |
busy_check: if (!sk_can_busy_loop(sk)) break; sk_busy_loop(sk, flags & MSG_DONTWAIT); } while (!skb_queue_empty(sk_queue)); /* sk_queue is empty, reader_queue may contain peeked packets */ } while (timeo && !__skb_wait_for_more_packets(sk, &error, &timeo, (struct sk_buff *)sk_queue)); *err = error; return NULL; } |
7976e6b70 udp: Unbreak modu... |
1500 |
EXPORT_SYMBOL(__skb_recv_udp); |
2276f58ac udp: use a separa... |
1501 |
|
db8dac20d [UDP]: Revert udp... |
1502 1503 1504 1505 |
/* * This should be easy, if there is something there we * return it, otherwise we block. */ |
1b7841404 net: Remove iocb ... |
1506 1507 |
int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int noblock, int flags, int *addr_len) |
db8dac20d [UDP]: Revert udp... |
1508 1509 |
{ struct inet_sock *inet = inet_sk(sk); |
342dfc306 net: add build-ti... |
1510 |
DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name); |
db8dac20d [UDP]: Revert udp... |
1511 |
struct sk_buff *skb; |
59c2cdae2 Revert "udp: remo... |
1512 |
unsigned int ulen, copied; |
627d2d6b5 udp: enable MSG_P... |
1513 |
int peeked, peeking, off; |
db8dac20d [UDP]: Revert udp... |
1514 1515 |
int err; int is_udplite = IS_UDPLITE(sk); |
197c949e7 udp: properly sup... |
1516 |
bool checksum_valid = false; |
db8dac20d [UDP]: Revert udp... |
1517 |
|
db8dac20d [UDP]: Revert udp... |
1518 |
if (flags & MSG_ERRQUEUE) |
85fbaa750 inet: fix addr_le... |
1519 |
return ip_recv_error(sk, msg, len, addr_len); |
db8dac20d [UDP]: Revert udp... |
1520 1521 |
try_again: |
a0917e0bc datagram: When pe... |
1522 1523 |
peeking = flags & MSG_PEEK; off = sk_peek_offset(sk, flags); |
7c13f97ff udp: do fwd memor... |
1524 |
skb = __skb_recv_udp(sk, flags, noblock, &peeked, &off, &err); |
db8dac20d [UDP]: Revert udp... |
1525 |
if (!skb) |
627d2d6b5 udp: enable MSG_P... |
1526 |
return err; |
db8dac20d [UDP]: Revert udp... |
1527 |
|
b65ac4467 udp: try to avoid... |
1528 |
ulen = udp_skb_len(skb); |
59c2cdae2 Revert "udp: remo... |
1529 |
copied = len; |
627d2d6b5 udp: enable MSG_P... |
1530 1531 |
if (copied > ulen - off) copied = ulen - off; |
59c2cdae2 Revert "udp: remo... |
1532 |
else if (copied < ulen) |
db8dac20d [UDP]: Revert udp... |
1533 1534 1535 1536 1537 1538 1539 |
msg->msg_flags |= MSG_TRUNC; /* * If checksum is needed at all, try to do it while copying the * data. If the data is truncated, or if we only want a partial * coverage checksum (UDP-Lite), do it before the copy. */ |
d21dbdfe0 udp: avoid one ca... |
1540 1541 |
if (copied < ulen || peeking || (is_udplite && UDP_SKB_CB(skb)->partial_cov)) { |
b65ac4467 udp: try to avoid... |
1542 1543 |
checksum_valid = udp_skb_csum_unnecessary(skb) || !__udp_lib_checksum_complete(skb); |
197c949e7 udp: properly sup... |
1544 |
if (!checksum_valid) |
db8dac20d [UDP]: Revert udp... |
1545 1546 |
goto csum_copy_err; } |
b65ac4467 udp: try to avoid... |
1547 1548 1549 1550 1551 1552 |
if (checksum_valid || udp_skb_csum_unnecessary(skb)) { if (udp_skb_is_linear(skb)) err = copy_linear_skb(skb, copied, off, &msg->msg_iter); else err = skb_copy_datagram_msg(skb, off, msg, copied); } else { |
627d2d6b5 udp: enable MSG_P... |
1553 |
err = skb_copy_and_csum_datagram_msg(skb, off, msg); |
db8dac20d [UDP]: Revert udp... |
1554 1555 1556 1557 |
if (err == -EINVAL) goto csum_copy_err; } |
22911fc58 net: skb_free_dat... |
1558 |
if (unlikely(err)) { |
979402b16 udp: increment UD... |
1559 1560 |
if (!peeked) { atomic_inc(&sk->sk_drops); |
6aef70a85 net: snmp: kill v... |
1561 1562 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); |
979402b16 udp: increment UD... |
1563 |
} |
850cbaddb udp: use it's own... |
1564 |
kfree_skb(skb); |
627d2d6b5 udp: enable MSG_P... |
1565 |
return err; |
22911fc58 net: skb_free_dat... |
1566 |
} |
db8dac20d [UDP]: Revert udp... |
1567 1568 |
if (!peeked) |
6aef70a85 net: snmp: kill v... |
1569 1570 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS, is_udplite); |
db8dac20d [UDP]: Revert udp... |
1571 |
|
3b885787e net: Generalize s... |
1572 |
sock_recv_ts_and_drops(msg, sk, skb); |
db8dac20d [UDP]: Revert udp... |
1573 1574 |
/* Copy the address. */ |
c482c5685 udp: cleanups |
1575 |
if (sin) { |
db8dac20d [UDP]: Revert udp... |
1576 1577 1578 1579 |
sin->sin_family = AF_INET; sin->sin_port = udp_hdr(skb)->source; sin->sin_addr.s_addr = ip_hdr(skb)->saddr; memset(sin->sin_zero, 0, sizeof(sin->sin_zero)); |
bceaa9024 inet: prevent lea... |
1580 |
*addr_len = sizeof(*sin); |
db8dac20d [UDP]: Revert udp... |
1581 1582 |
} if (inet->cmsg_flags) |
ad959036a net/sock: add an ... |
1583 |
ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off); |
db8dac20d [UDP]: Revert udp... |
1584 |
|
59c2cdae2 Revert "udp: remo... |
1585 |
err = copied; |
db8dac20d [UDP]: Revert udp... |
1586 1587 |
if (flags & MSG_TRUNC) err = ulen; |
850cbaddb udp: use it's own... |
1588 |
skb_consume_udp(sk, skb, peeking ? -err : err); |
db8dac20d [UDP]: Revert udp... |
1589 1590 1591 |
return err; csum_copy_err: |
2276f58ac udp: use a separa... |
1592 1593 |
if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags, udp_skb_destructor)) { |
6aef70a85 net: snmp: kill v... |
1594 1595 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); |
6a5dc9e59 net: Add MIB coun... |
1596 |
} |
850cbaddb udp: use it's own... |
1597 |
kfree_skb(skb); |
db8dac20d [UDP]: Revert udp... |
1598 |
|
beb39db59 udp: fix behavior... |
1599 1600 |
/* starting over for a new packet, but check if we need to yield */ cond_resched(); |
9cfaa8def udp/recvmsg: Clea... |
1601 |
msg->msg_flags &= ~MSG_TRUNC; |
db8dac20d [UDP]: Revert udp... |
1602 1603 |
goto try_again; } |
286c72dea udp: must lock th... |
1604 |
int __udp_disconnect(struct sock *sk, int flags) |
1da177e4c Linux-2.6.12-rc2 |
1605 1606 1607 1608 1609 |
{ struct inet_sock *inet = inet_sk(sk); /* * 1003.1g - break association. */ |
e905a9eda [NET] IPV4: Fix w... |
1610 |
|
1da177e4c Linux-2.6.12-rc2 |
1611 |
sk->sk_state = TCP_CLOSE; |
c720c7e83 inet: rename some... |
1612 1613 |
inet->inet_daddr = 0; inet->inet_dport = 0; |
bdeab9919 rps: Add flag to ... |
1614 |
sock_rps_reset_rxhash(sk); |
1da177e4c Linux-2.6.12-rc2 |
1615 1616 1617 1618 1619 1620 |
sk->sk_bound_dev_if = 0; if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK)) inet_reset_saddr(sk); if (!(sk->sk_userlocks & SOCK_BINDPORT_LOCK)) { sk->sk_prot->unhash(sk); |
c720c7e83 inet: rename some... |
1621 |
inet->inet_sport = 0; |
1da177e4c Linux-2.6.12-rc2 |
1622 1623 1624 1625 |
} sk_dst_reset(sk); return 0; } |
286c72dea udp: must lock th... |
1626 1627 1628 1629 1630 1631 1632 1633 1634 |
EXPORT_SYMBOL(__udp_disconnect); int udp_disconnect(struct sock *sk, int flags) { lock_sock(sk); __udp_disconnect(sk, flags); release_sock(sk); return 0; } |
c482c5685 udp: cleanups |
1635 |
EXPORT_SYMBOL(udp_disconnect); |
1da177e4c Linux-2.6.12-rc2 |
1636 |
|
645ca708f udp: introduce st... |
1637 1638 |
void udp_lib_unhash(struct sock *sk) { |
723b46108 net: udp_unhash()... |
1639 1640 |
if (sk_hashed(sk)) { struct udp_table *udptable = sk->sk_prot->h.udp_table; |
512615b6b udp: secondary ha... |
1641 1642 1643 1644 1645 |
struct udp_hslot *hslot, *hslot2; hslot = udp_hashslot(udptable, sock_net(sk), udp_sk(sk)->udp_port_hash); hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); |
645ca708f udp: introduce st... |
1646 |
|
723b46108 net: udp_unhash()... |
1647 |
spin_lock_bh(&hslot->lock); |
e32ea7e74 soreuseport: fast... |
1648 1649 |
if (rcu_access_pointer(sk->sk_reuseport_cb)) reuseport_detach_sock(sk); |
ca065d0cf udp: no longer us... |
1650 |
if (sk_del_node_init_rcu(sk)) { |
fdcc8aa95 udp: add a counte... |
1651 |
hslot->count--; |
c720c7e83 inet: rename some... |
1652 |
inet_sk(sk)->inet_num = 0; |
723b46108 net: udp_unhash()... |
1653 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); |
512615b6b udp: secondary ha... |
1654 1655 |
spin_lock(&hslot2->lock); |
ca065d0cf udp: no longer us... |
1656 |
hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); |
512615b6b udp: secondary ha... |
1657 1658 |
hslot2->count--; spin_unlock(&hslot2->lock); |
723b46108 net: udp_unhash()... |
1659 1660 |
} spin_unlock_bh(&hslot->lock); |
645ca708f udp: introduce st... |
1661 |
} |
645ca708f udp: introduce st... |
1662 1663 |
} EXPORT_SYMBOL(udp_lib_unhash); |
719f83585 udp: add rehash o... |
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 |
/* * inet_rcv_saddr was changed, we must rehash secondary hash */ void udp_lib_rehash(struct sock *sk, u16 newhash) { if (sk_hashed(sk)) { struct udp_table *udptable = sk->sk_prot->h.udp_table; struct udp_hslot *hslot, *hslot2, *nhslot2; hslot2 = udp_hashslot2(udptable, udp_sk(sk)->udp_portaddr_hash); nhslot2 = udp_hashslot2(udptable, newhash); udp_sk(sk)->udp_portaddr_hash = newhash; |
e32ea7e74 soreuseport: fast... |
1676 1677 1678 |
if (hslot2 != nhslot2 || rcu_access_pointer(sk->sk_reuseport_cb)) { |
719f83585 udp: add rehash o... |
1679 1680 1681 1682 |
hslot = udp_hashslot(udptable, sock_net(sk), udp_sk(sk)->udp_port_hash); /* we must lock primary chain too */ spin_lock_bh(&hslot->lock); |
e32ea7e74 soreuseport: fast... |
1683 1684 1685 1686 1687 |
if (rcu_access_pointer(sk->sk_reuseport_cb)) reuseport_detach_sock(sk); if (hslot2 != nhslot2) { spin_lock(&hslot2->lock); |
ca065d0cf udp: no longer us... |
1688 |
hlist_del_init_rcu(&udp_sk(sk)->udp_portaddr_node); |
e32ea7e74 soreuseport: fast... |
1689 1690 1691 1692 |
hslot2->count--; spin_unlock(&hslot2->lock); spin_lock(&nhslot2->lock); |
ca065d0cf udp: no longer us... |
1693 |
hlist_add_head_rcu(&udp_sk(sk)->udp_portaddr_node, |
e32ea7e74 soreuseport: fast... |
1694 1695 1696 1697 |
&nhslot2->head); nhslot2->count++; spin_unlock(&nhslot2->lock); } |
719f83585 udp: add rehash o... |
1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 |
spin_unlock_bh(&hslot->lock); } } } EXPORT_SYMBOL(udp_lib_rehash); static void udp_v4_rehash(struct sock *sk) { u16 new_hash = udp4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, inet_sk(sk)->inet_num); udp_lib_rehash(sk, new_hash); } |
a3f96c47c udp: make *udp*_q... |
1712 |
static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) |
93821778d udp: Fix rcv sock... |
1713 |
{ |
fec5e652e rfs: Receive Flow... |
1714 |
int rc; |
766e9037c net: sk_drops con... |
1715 |
|
005ec9743 udp: Only allow b... |
1716 |
if (inet_sk(sk)->inet_daddr) { |
bdeab9919 rps: Add flag to ... |
1717 |
sock_rps_save_rxhash(sk, skb); |
005ec9743 udp: Only allow b... |
1718 |
sk_mark_napi_id(sk, skb); |
2c8c56e15 net: introduce SO... |
1719 |
sk_incoming_cpu_update(sk); |
e68b6e50f udp: enable busy ... |
1720 1721 |
} else { sk_mark_napi_id_once(sk, skb); |
005ec9743 udp: Only allow b... |
1722 |
} |
fec5e652e rfs: Receive Flow... |
1723 |
|
850cbaddb udp: use it's own... |
1724 |
rc = __udp_enqueue_schedule_skb(sk, skb); |
766e9037c net: sk_drops con... |
1725 1726 |
if (rc < 0) { int is_udplite = IS_UDPLITE(sk); |
93821778d udp: Fix rcv sock... |
1727 |
|
93821778d udp: Fix rcv sock... |
1728 |
/* Note that an ENOMEM error is charged twice */ |
766e9037c net: sk_drops con... |
1729 |
if (rc == -ENOMEM) |
e61da9e25 udp: prepare for ... |
1730 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_RCVBUFERRORS, |
02c223470 net: udp: rename ... |
1731 |
is_udplite); |
e61da9e25 udp: prepare for ... |
1732 |
UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); |
766e9037c net: sk_drops con... |
1733 |
kfree_skb(skb); |
296f7ea75 udp: add tracepoi... |
1734 |
trace_udp_fail_queue_rcv_skb(rc, sk); |
766e9037c net: sk_drops con... |
1735 |
return -1; |
93821778d udp: Fix rcv sock... |
1736 1737 1738 |
} return 0; |
93821778d udp: Fix rcv sock... |
1739 |
} |
447167bf5 udp: intoduce udp... |
1740 1741 1742 |
static struct static_key udp_encap_needed __read_mostly; void udp_encap_enable(void) { |
7a34bcb8b jump_label: Do no... |
1743 |
static_key_enable(&udp_encap_needed); |
447167bf5 udp: intoduce udp... |
1744 1745 |
} EXPORT_SYMBOL(udp_encap_enable); |
db8dac20d [UDP]: Revert udp... |
1746 1747 1748 1749 1750 1751 1752 1753 |
/* returns: * -1: error * 0: success * >0: "udp encap" protocol resubmission * * Note that in the success and error cases, the skb is assumed to * have either been requeued or freed. */ |
a3f96c47c udp: make *udp*_q... |
1754 |
static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb) |
db8dac20d [UDP]: Revert udp... |
1755 1756 |
{ struct udp_sock *up = udp_sk(sk); |
db8dac20d [UDP]: Revert udp... |
1757 1758 1759 1760 1761 1762 1763 1764 |
int is_udplite = IS_UDPLITE(sk); /* * Charge it to the socket, dropping if the queue is full. */ if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) goto drop; nf_reset(skb); |
447167bf5 udp: intoduce udp... |
1765 |
if (static_key_false(&udp_encap_needed) && up->encap_type) { |
0ad92ad03 udp: fix a race i... |
1766 |
int (*encap_rcv)(struct sock *sk, struct sk_buff *skb); |
db8dac20d [UDP]: Revert udp... |
1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 |
/* * This is an encapsulation socket so pass the skb to * the socket's udp_encap_rcv() hook. Otherwise, just * fall through and pass this up the UDP socket. * up->encap_rcv() returns the following value: * =0 if skb was successfully passed to the encap * handler or was discarded by it. * >0 if skb should be passed on to UDP. * <0 if skb should be resubmitted as proto -N */ /* if we're overly short, let UDP handle it */ |
0ad92ad03 udp: fix a race i... |
1779 |
encap_rcv = ACCESS_ONCE(up->encap_rcv); |
e5aed006b udp: prevent skbs... |
1780 |
if (encap_rcv) { |
db8dac20d [UDP]: Revert udp... |
1781 |
int ret; |
0a80966b1 net: Verify UDP c... |
1782 1783 1784 |
/* Verify checksum before giving to encap */ if (udp_lib_checksum_complete(skb)) goto csum_error; |
0ad92ad03 udp: fix a race i... |
1785 |
ret = encap_rcv(sk, skb); |
db8dac20d [UDP]: Revert udp... |
1786 |
if (ret <= 0) { |
02c223470 net: udp: rename ... |
1787 1788 1789 |
__UDP_INC_STATS(sock_net(sk), UDP_MIB_INDATAGRAMS, is_udplite); |
db8dac20d [UDP]: Revert udp... |
1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 |
return -ret; } } /* FALLTHROUGH -- it's a UDP Packet */ } /* * UDP-Lite specific tests, ignored on UDP sockets */ if ((is_udplite & UDPLITE_RECV_CC) && UDP_SKB_CB(skb)->partial_cov) { /* * MIB statistics other than incrementing the error count are * disabled for the following two types of errors: these depend * on the application settings, not on the functioning of the * protocol stack as such. * * RFC 3828 here recommends (sec 3.3): "There should also be a * way ... to ... at least let the receiving application block * delivery of packets with coverage values less than a value * provided by the application." */ if (up->pcrlen == 0) { /* full coverage was set */ |
ba7a46f16 net: Convert LIMI... |
1814 1815 1816 |
net_dbg_ratelimited("UDPLite: partial coverage %d while full coverage %d requested ", UDP_SKB_CB(skb)->cscov, skb->len); |
db8dac20d [UDP]: Revert udp... |
1817 1818 1819 1820 1821 1822 1823 1824 1825 |
goto drop; } /* The next case involves violating the min. coverage requested * by the receiver. This is subtle: if receiver wants x and x is * greater than the buffersize/MTU then receiver will complain * that it wants x while sender emits packets of smaller size y. * Therefore the above ...()->partial_cov statement is essential. */ if (UDP_SKB_CB(skb)->cscov < up->pcrlen) { |
ba7a46f16 net: Convert LIMI... |
1826 1827 1828 |
net_dbg_ratelimited("UDPLite: coverage %d too small, need min %d ", UDP_SKB_CB(skb)->cscov, up->pcrlen); |
db8dac20d [UDP]: Revert udp... |
1829 1830 1831 |
goto drop; } } |
dd99e425b udp: prefetch rme... |
1832 |
prefetch(&sk->sk_rmem_alloc); |
ce25d66ad Possible problem ... |
1833 1834 |
if (rcu_access_pointer(sk->sk_filter) && udp_lib_checksum_complete(skb)) |
e6afc8ace udp: remove heade... |
1835 |
goto csum_error; |
ce25d66ad Possible problem ... |
1836 |
|
ba66bbe54 udp: use sk_filte... |
1837 |
if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr))) |
a61276977 udp: prevent bugc... |
1838 |
goto drop; |
db8dac20d [UDP]: Revert udp... |
1839 |
|
e6afc8ace udp: remove heade... |
1840 |
udp_csum_pull_header(skb); |
db8dac20d [UDP]: Revert udp... |
1841 |
|
fbf8866d6 net: ipv4 only po... |
1842 |
ipv4_pktinfo_prepare(sk, skb); |
850cbaddb udp: use it's own... |
1843 |
return __udp_queue_rcv_skb(sk, skb); |
db8dac20d [UDP]: Revert udp... |
1844 |
|
6a5dc9e59 net: Add MIB coun... |
1845 |
csum_error: |
02c223470 net: udp: rename ... |
1846 |
__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite); |
db8dac20d [UDP]: Revert udp... |
1847 |
drop: |
02c223470 net: udp: rename ... |
1848 |
__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite); |
8edf19c2f net: sk_drops con... |
1849 |
atomic_inc(&sk->sk_drops); |
db8dac20d [UDP]: Revert udp... |
1850 1851 1852 |
kfree_skb(skb); return -1; } |
975022310 udp: ipv4: must a... |
1853 |
/* For TCP sockets, sk_rx_dst is protected by socket lock |
e47eb5dfb udp: ipv4: do not... |
1854 |
* For UDP, we use xchg() to guard against concurrent changes. |
975022310 udp: ipv4: must a... |
1855 |
*/ |
64f0f5d18 udp6: set rx_dst_... |
1856 |
bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst) |
421b3885b udp: ipv4: Add ud... |
1857 |
{ |
975022310 udp: ipv4: must a... |
1858 |
struct dst_entry *old; |
d24406c85 udp: call dst_hol... |
1859 1860 1861 |
if (dst_hold_safe(dst)) { old = xchg(&sk->sk_rx_dst, dst); dst_release(old); |
64f0f5d18 udp6: set rx_dst_... |
1862 |
return old != dst; |
d24406c85 udp: call dst_hol... |
1863 |
} |
64f0f5d18 udp6: set rx_dst_... |
1864 |
return false; |
421b3885b udp: ipv4: Add ud... |
1865 |
} |
c9f2c1ae1 udp6: fix socket ... |
1866 |
EXPORT_SYMBOL(udp_sk_rx_dst_set); |
421b3885b udp: ipv4: Add ud... |
1867 |
|
db8dac20d [UDP]: Revert udp... |
1868 1869 1870 |
/* * Multicasts and broadcasts go to each listener. * |
1240d1373 ipv4: udp: Optimi... |
1871 |
* Note: called only from the BH handler context. |
db8dac20d [UDP]: Revert udp... |
1872 |
*/ |
e31634931 udp: provide a st... |
1873 |
static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb, |
db8dac20d [UDP]: Revert udp... |
1874 1875 |
struct udphdr *uh, __be32 saddr, __be32 daddr, |
36cbb2452 udp: Increment UD... |
1876 1877 |
struct udp_table *udptable, int proto) |
db8dac20d [UDP]: Revert udp... |
1878 |
{ |
ca065d0cf udp: no longer us... |
1879 |
struct sock *sk, *first = NULL; |
5cf3d4619 udp: Simplify __u... |
1880 1881 |
unsigned short hnum = ntohs(uh->dest); struct udp_hslot *hslot = udp_hashslot(udptable, net, hnum); |
2dc41cff7 udp: Use hash2 fo... |
1882 |
unsigned int hash2 = 0, hash2_any = 0, use_hash2 = (hslot->count > 10); |
ca065d0cf udp: no longer us... |
1883 1884 |
unsigned int offset = offsetof(typeof(*sk), sk_node); int dif = skb->dev->ifindex; |
fb74c2773 net: ipv4: add se... |
1885 |
int sdif = inet_sdif(skb); |
ca065d0cf udp: no longer us... |
1886 1887 |
struct hlist_node *node; struct sk_buff *nskb; |
2dc41cff7 udp: Use hash2 fo... |
1888 1889 1890 |
if (use_hash2) { hash2_any = udp4_portaddr_hash(net, htonl(INADDR_ANY), hnum) & |
73e2d5e34 udp: restore UDPl... |
1891 1892 |
udptable->mask; hash2 = udp4_portaddr_hash(net, daddr, hnum) & udptable->mask; |
2dc41cff7 udp: Use hash2 fo... |
1893 |
start_lookup: |
73e2d5e34 udp: restore UDPl... |
1894 |
hslot = &udptable->hash2[hash2]; |
2dc41cff7 udp: Use hash2 fo... |
1895 1896 |
offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node); } |
db8dac20d [UDP]: Revert udp... |
1897 |
|
ca065d0cf udp: no longer us... |
1898 1899 |
sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) { if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr, |
fb74c2773 net: ipv4: add se... |
1900 |
uh->source, saddr, dif, sdif, hnum)) |
ca065d0cf udp: no longer us... |
1901 1902 1903 1904 1905 |
continue; if (!first) { first = sk; continue; |
1240d1373 ipv4: udp: Optimi... |
1906 |
} |
ca065d0cf udp: no longer us... |
1907 |
nskb = skb_clone(skb, GFP_ATOMIC); |
1240d1373 ipv4: udp: Optimi... |
1908 |
|
ca065d0cf udp: no longer us... |
1909 1910 |
if (unlikely(!nskb)) { atomic_inc(&sk->sk_drops); |
02c223470 net: udp: rename ... |
1911 1912 1913 1914 |
__UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS, IS_UDPLITE(sk)); __UDP_INC_STATS(net, UDP_MIB_INERRORS, IS_UDPLITE(sk)); |
ca065d0cf udp: no longer us... |
1915 1916 1917 1918 1919 |
continue; } if (udp_queue_rcv_skb(sk, nskb) > 0) consume_skb(nskb); } |
1240d1373 ipv4: udp: Optimi... |
1920 |
|
2dc41cff7 udp: Use hash2 fo... |
1921 1922 1923 1924 1925 |
/* Also lookup *:port if we are using hash2 and haven't done so yet. */ if (use_hash2 && hash2 != hash2_any) { hash2 = hash2_any; goto start_lookup; } |
ca065d0cf udp: no longer us... |
1926 1927 1928 |
if (first) { if (udp_queue_rcv_skb(first, skb) > 0) consume_skb(skb); |
1240d1373 ipv4: udp: Optimi... |
1929 |
} else { |
ca065d0cf udp: no longer us... |
1930 |
kfree_skb(skb); |
02c223470 net: udp: rename ... |
1931 1932 |
__UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI, proto == IPPROTO_UDPLITE); |
1240d1373 ipv4: udp: Optimi... |
1933 |
} |
db8dac20d [UDP]: Revert udp... |
1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 |
return 0; } /* Initialize UDP checksum. If exited with zero value (success), * CHECKSUM_UNNECESSARY means, that no more checks are required. * Otherwise, csum completion requires chacksumming packet body, * including udp header and folding it to skb->csum. */ static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh, int proto) { |
db8dac20d [UDP]: Revert udp... |
1945 1946 1947 1948 1949 1950 1951 1952 1953 |
int err; UDP_SKB_CB(skb)->partial_cov = 0; UDP_SKB_CB(skb)->cscov = skb->len; if (proto == IPPROTO_UDPLITE) { err = udplite_checksum_init(skb, uh); if (err) return err; |
fecb84a83 udplite: fix part... |
1954 1955 1956 1957 1958 |
if (UDP_SKB_CB(skb)->partial_cov) { skb->csum = inet_compute_pseudo(skb, proto); return 0; } |
db8dac20d [UDP]: Revert udp... |
1959 |
} |
b46d9f625 ipv4: fix checksu... |
1960 1961 1962 |
/* Note, we are only interested in != 0 or == 0, thus the * force to int. */ |
0ecebdfb2 net: udp: fix han... |
1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 |
err = (__force int)skb_checksum_init_zero_check(skb, proto, uh->check, inet_compute_pseudo); if (err) return err; if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) { /* If SW calculated the value, we know it's bad */ if (skb->csum_complete_sw) return 1; /* HW says the value is bad. Let's validate that. * skb->csum is no longer the full packet checksum, * so don't treat it as such. */ skb_checksum_complete_unset(skb); } return 0; |
db8dac20d [UDP]: Revert udp... |
1981 |
} |
0f6f77f3b udp4: fix IP_CMSG... |
1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 |
/* wrapper for udp_queue_rcv_skb tacking care of csum conversion and * return code conversion for ip layer consumption */ static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb, struct udphdr *uh) { int ret; if (inet_get_convert_csum(sk) && uh->check && !IS_UDPLITE(sk)) skb_checksum_try_convert(skb, IPPROTO_UDP, uh->check, inet_compute_pseudo); ret = udp_queue_rcv_skb(sk, skb); /* a return value > 0 means to resubmit the input, but * it wants the return to be -protocol, or 0 */ if (ret > 0) return -ret; return 0; } |
db8dac20d [UDP]: Revert udp... |
2003 2004 2005 |
/* * All we need to do is get the socket, and then do a checksum. */ |
645ca708f udp: introduce st... |
2006 |
int __udp4_lib_rcv(struct sk_buff *skb, struct udp_table *udptable, |
db8dac20d [UDP]: Revert udp... |
2007 2008 2009 |
int proto) { struct sock *sk; |
7b5e56f9d udp: Fix UDP shor... |
2010 |
struct udphdr *uh; |
db8dac20d [UDP]: Revert udp... |
2011 |
unsigned short ulen; |
adf30907d net: skb->dst acc... |
2012 |
struct rtable *rt = skb_rtable(skb); |
2783ef231 udp: Fix potentia... |
2013 |
__be32 saddr, daddr; |
0283328e2 MIB: add struct n... |
2014 |
struct net *net = dev_net(skb->dev); |
db8dac20d [UDP]: Revert udp... |
2015 2016 2017 2018 2019 2020 |
/* * Validate the packet. */ if (!pskb_may_pull(skb, sizeof(struct udphdr))) goto drop; /* No space for header. */ |
7b5e56f9d udp: Fix UDP shor... |
2021 |
uh = udp_hdr(skb); |
db8dac20d [UDP]: Revert udp... |
2022 |
ulen = ntohs(uh->len); |
ccc2d97cb ipv4: udp: fix sh... |
2023 2024 |
saddr = ip_hdr(skb)->saddr; daddr = ip_hdr(skb)->daddr; |
db8dac20d [UDP]: Revert udp... |
2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 |
if (ulen > skb->len) goto short_packet; if (proto == IPPROTO_UDP) { /* UDP validates ulen. */ if (ulen < sizeof(*uh) || pskb_trim_rcsum(skb, ulen)) goto short_packet; uh = udp_hdr(skb); } if (udp4_csum_init(skb, uh, proto)) goto csum_error; |
8afdd99a1 udp: ipv4: fix an... |
2037 2038 |
sk = skb_steal_sock(skb); if (sk) { |
975022310 udp: ipv4: must a... |
2039 |
struct dst_entry *dst = skb_dst(skb); |
421b3885b udp: ipv4: Add ud... |
2040 |
int ret; |
421b3885b udp: ipv4: Add ud... |
2041 |
|
975022310 udp: ipv4: must a... |
2042 2043 |
if (unlikely(sk->sk_rx_dst != dst)) udp_sk_rx_dst_set(sk, dst); |
db8dac20d [UDP]: Revert udp... |
2044 |
|
0f6f77f3b udp4: fix IP_CMSG... |
2045 |
ret = udp_unicast_rcv_skb(sk, skb, uh); |
8afdd99a1 udp: ipv4: fix an... |
2046 |
sock_put(sk); |
0f6f77f3b udp4: fix IP_CMSG... |
2047 |
return ret; |
421b3885b udp: ipv4: Add ud... |
2048 |
} |
db8dac20d [UDP]: Revert udp... |
2049 |
|
c18450a52 udp: remove else ... |
2050 2051 |
if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST)) return __udp4_lib_mcast_deliver(net, skb, uh, |
36cbb2452 udp: Increment UD... |
2052 |
saddr, daddr, udptable, proto); |
c18450a52 udp: remove else ... |
2053 2054 |
sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest, udptable); |
0f6f77f3b udp4: fix IP_CMSG... |
2055 2056 |
if (sk) return udp_unicast_rcv_skb(sk, skb, uh); |
db8dac20d [UDP]: Revert udp... |
2057 2058 2059 2060 2061 2062 2063 2064 |
if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) goto drop; nf_reset(skb); /* No socket. Drop packet silently, if checksum is wrong */ if (udp_lib_checksum_complete(skb)) goto csum_error; |
02c223470 net: udp: rename ... |
2065 |
__UDP_INC_STATS(net, UDP_MIB_NOPORTS, proto == IPPROTO_UDPLITE); |
db8dac20d [UDP]: Revert udp... |
2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 |
icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0); /* * Hmm. We got an UDP packet to a port to which we * don't wanna listen. Ignore it. */ kfree_skb(skb); return 0; short_packet: |
ba7a46f16 net: Convert LIMI... |
2076 2077 2078 2079 2080 2081 |
net_dbg_ratelimited("UDP%s: short packet: From %pI4:%u %d/%d to %pI4:%u ", proto == IPPROTO_UDPLITE ? "Lite" : "", &saddr, ntohs(uh->source), ulen, skb->len, &daddr, ntohs(uh->dest)); |
db8dac20d [UDP]: Revert udp... |
2082 2083 2084 2085 2086 2087 2088 |
goto drop; csum_error: /* * RFC1122: OK. Discards the bad packet silently (as far as * the network is concerned, anyway) as per 4.1.3.4 (MUST). */ |
ba7a46f16 net: Convert LIMI... |
2089 2090 2091 2092 2093 |
net_dbg_ratelimited("UDP%s: bad checksum. From %pI4:%u to %pI4:%u ulen %d ", proto == IPPROTO_UDPLITE ? "Lite" : "", &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest), ulen); |
02c223470 net: udp: rename ... |
2094 |
__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, proto == IPPROTO_UDPLITE); |
db8dac20d [UDP]: Revert udp... |
2095 |
drop: |
02c223470 net: udp: rename ... |
2096 |
__UDP_INC_STATS(net, UDP_MIB_INERRORS, proto == IPPROTO_UDPLITE); |
db8dac20d [UDP]: Revert udp... |
2097 2098 2099 |
kfree_skb(skb); return 0; } |
421b3885b udp: ipv4: Add ud... |
2100 2101 2102 2103 2104 2105 |
/* We can only early demux multicast if there is a single matching socket. * If more than one socket found returns NULL */ static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net, __be16 loc_port, __be32 loc_addr, __be16 rmt_port, __be32 rmt_addr, |
fb74c2773 net: ipv4: add se... |
2106 |
int dif, int sdif) |
421b3885b udp: ipv4: Add ud... |
2107 2108 |
{ struct sock *sk, *result; |
421b3885b udp: ipv4: Add ud... |
2109 |
unsigned short hnum = ntohs(loc_port); |
ca065d0cf udp: no longer us... |
2110 |
unsigned int slot = udp_hashfn(net, hnum, udp_table.mask); |
421b3885b udp: ipv4: Add ud... |
2111 |
struct udp_hslot *hslot = &udp_table.hash[slot]; |
63c6f81cd udp: ipv4: do not... |
2112 2113 2114 |
/* Do not bother scanning a too big list */ if (hslot->count > 10) return NULL; |
421b3885b udp: ipv4: Add ud... |
2115 |
result = NULL; |
ca065d0cf udp: no longer us... |
2116 2117 |
sk_for_each_rcu(sk, &hslot->head) { if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr, |
fb74c2773 net: ipv4: add se... |
2118 |
rmt_port, rmt_addr, dif, sdif, hnum)) { |
ca065d0cf udp: no longer us... |
2119 2120 |
if (result) return NULL; |
421b3885b udp: ipv4: Add ud... |
2121 |
result = sk; |
421b3885b udp: ipv4: Add ud... |
2122 2123 |
} } |
ca065d0cf udp: no longer us... |
2124 |
|
421b3885b udp: ipv4: Add ud... |
2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 |
return result; } /* For unicast we should only early demux connected sockets or we can * break forwarding setups. The chains here can be long so only check * if the first socket is an exact match and if not move on. */ static struct sock *__udp4_lib_demux_lookup(struct net *net, __be16 loc_port, __be32 loc_addr, __be16 rmt_port, __be32 rmt_addr, |
3fa6f616a net: ipv4: add se... |
2135 |
int dif, int sdif) |
421b3885b udp: ipv4: Add ud... |
2136 |
{ |
421b3885b udp: ipv4: Add ud... |
2137 2138 2139 2140 |
unsigned short hnum = ntohs(loc_port); unsigned int hash2 = udp4_portaddr_hash(net, loc_addr, hnum); unsigned int slot2 = hash2 & udp_table.mask; struct udp_hslot *hslot2 = &udp_table.hash2[slot2]; |
c72283174 net: Use a more s... |
2141 |
INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr); |
421b3885b udp: ipv4: Add ud... |
2142 |
const __portpair ports = INET_COMBINED_PORTS(rmt_port, hnum); |
ca065d0cf udp: no longer us... |
2143 |
struct sock *sk; |
421b3885b udp: ipv4: Add ud... |
2144 |
|
ca065d0cf udp: no longer us... |
2145 2146 |
udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) { if (INET_MATCH(sk, net, acookie, rmt_addr, |
3fa6f616a net: ipv4: add se... |
2147 |
loc_addr, ports, dif, sdif)) |
ca065d0cf udp: no longer us... |
2148 |
return sk; |
421b3885b udp: ipv4: Add ud... |
2149 2150 2151 |
/* Only check first socket in chain */ break; } |
ca065d0cf udp: no longer us... |
2152 |
return NULL; |
421b3885b udp: ipv4: Add ud... |
2153 |
} |
7487449c8 IPv4: early demux... |
2154 |
int udp_v4_early_demux(struct sk_buff *skb) |
421b3885b udp: ipv4: Add ud... |
2155 |
{ |
610438b74 udp: ipv4: fix po... |
2156 |
struct net *net = dev_net(skb->dev); |
bc044e8db udp: perform sour... |
2157 |
struct in_device *in_dev = NULL; |
610438b74 udp: ipv4: fix po... |
2158 2159 |
const struct iphdr *iph; const struct udphdr *uh; |
ca065d0cf udp: no longer us... |
2160 |
struct sock *sk = NULL; |
421b3885b udp: ipv4: Add ud... |
2161 |
struct dst_entry *dst; |
421b3885b udp: ipv4: Add ud... |
2162 |
int dif = skb->dev->ifindex; |
fb74c2773 net: ipv4: add se... |
2163 |
int sdif = inet_sdif(skb); |
6e5403093 ipv4/udp: Verify ... |
2164 |
int ours; |
421b3885b udp: ipv4: Add ud... |
2165 2166 2167 |
/* validate the packet */ if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr))) |
7487449c8 IPv4: early demux... |
2168 |
return 0; |
421b3885b udp: ipv4: Add ud... |
2169 |
|
610438b74 udp: ipv4: fix po... |
2170 2171 |
iph = ip_hdr(skb); uh = udp_hdr(skb); |
996b44fce udp: fix bcast pa... |
2172 |
if (skb->pkt_type == PACKET_MULTICAST) { |
bc044e8db udp: perform sour... |
2173 |
in_dev = __in_dev_get_rcu(skb->dev); |
6e5403093 ipv4/udp: Verify ... |
2174 2175 |
if (!in_dev) |
7487449c8 IPv4: early demux... |
2176 |
return 0; |
6e5403093 ipv4/udp: Verify ... |
2177 |
|
996b44fce udp: fix bcast pa... |
2178 2179 2180 2181 |
ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr, iph->protocol); if (!ours) return 0; |
ad0ea1989 ipv4: fix broadca... |
2182 |
|
421b3885b udp: ipv4: Add ud... |
2183 |
sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr, |
fb74c2773 net: ipv4: add se... |
2184 2185 |
uh->source, iph->saddr, dif, sdif); |
6e5403093 ipv4/udp: Verify ... |
2186 |
} else if (skb->pkt_type == PACKET_HOST) { |
421b3885b udp: ipv4: Add ud... |
2187 |
sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr, |
3fa6f616a net: ipv4: add se... |
2188 |
uh->source, iph->saddr, dif, sdif); |
6e5403093 ipv4/udp: Verify ... |
2189 |
} |
421b3885b udp: ipv4: Add ud... |
2190 |
|
41c6d650f net: convert sock... |
2191 |
if (!sk || !refcount_inc_not_zero(&sk->sk_refcnt)) |
7487449c8 IPv4: early demux... |
2192 |
return 0; |
421b3885b udp: ipv4: Add ud... |
2193 2194 |
skb->sk = sk; |
82eabd9eb net: merge cases ... |
2195 |
skb->destructor = sock_efree; |
10e2eb878 udp: fix dst race... |
2196 |
dst = READ_ONCE(sk->sk_rx_dst); |
421b3885b udp: ipv4: Add ud... |
2197 2198 2199 |
if (dst) dst = dst_check(dst, 0); |
10e2eb878 udp: fix dst race... |
2200 |
if (dst) { |
bc044e8db udp: perform sour... |
2201 |
u32 itag = 0; |
d24406c85 udp: call dst_hol... |
2202 2203 2204 2205 2206 |
/* set noref for now. * any place which wants to hold dst has to call * dst_hold_safe() */ skb_dst_set_noref(skb, dst); |
bc044e8db udp: perform sour... |
2207 2208 2209 2210 2211 2212 2213 2214 |
/* for unconnected multicast sockets we need to validate * the source on each packet */ if (!inet_sk(sk)->inet_daddr && in_dev) return ip_mc_validate_source(skb, iph->daddr, iph->saddr, iph->tos, skb->dev, in_dev, &itag); |
10e2eb878 udp: fix dst race... |
2215 |
} |
7487449c8 IPv4: early demux... |
2216 |
return 0; |
421b3885b udp: ipv4: Add ud... |
2217 |
} |
db8dac20d [UDP]: Revert udp... |
2218 2219 |
int udp_rcv(struct sk_buff *skb) { |
645ca708f udp: introduce st... |
2220 |
return __udp4_lib_rcv(skb, &udp_table, IPPROTO_UDP); |
db8dac20d [UDP]: Revert udp... |
2221 |
} |
7d06b2e05 net: change proto... |
2222 |
void udp_destroy_sock(struct sock *sk) |
db8dac20d [UDP]: Revert udp... |
2223 |
{ |
44046a593 udp: add encap_de... |
2224 |
struct udp_sock *up = udp_sk(sk); |
8a74ad60a net: fix lock_soc... |
2225 |
bool slow = lock_sock_fast(sk); |
db8dac20d [UDP]: Revert udp... |
2226 |
udp_flush_pending_frames(sk); |
8a74ad60a net: fix lock_soc... |
2227 |
unlock_sock_fast(sk, slow); |
44046a593 udp: add encap_de... |
2228 2229 2230 2231 2232 2233 |
if (static_key_false(&udp_encap_needed) && up->encap_type) { void (*encap_destroy)(struct sock *sk); encap_destroy = ACCESS_ONCE(up->encap_destroy); if (encap_destroy) encap_destroy(sk); } |
db8dac20d [UDP]: Revert udp... |
2234 |
} |
1da177e4c Linux-2.6.12-rc2 |
2235 2236 2237 |
/* * Socket option code for UDP */ |
4c0a6cb0d [UDP(-Lite)]: con... |
2238 |
int udp_lib_setsockopt(struct sock *sk, int level, int optname, |
b7058842c net: Make setsock... |
2239 |
char __user *optval, unsigned int optlen, |
4c0a6cb0d [UDP(-Lite)]: con... |
2240 |
int (*push_pending_frames)(struct sock *)) |
1da177e4c Linux-2.6.12-rc2 |
2241 2242 |
{ struct udp_sock *up = udp_sk(sk); |
1c19448c9 net: Make enablin... |
2243 |
int val, valbool; |
1da177e4c Linux-2.6.12-rc2 |
2244 |
int err = 0; |
b2bf1e265 [UDP]: Clean up f... |
2245 |
int is_udplite = IS_UDPLITE(sk); |
1da177e4c Linux-2.6.12-rc2 |
2246 |
|
c482c5685 udp: cleanups |
2247 |
if (optlen < sizeof(int)) |
1da177e4c Linux-2.6.12-rc2 |
2248 2249 2250 2251 |
return -EINVAL; if (get_user(val, (int __user *)optval)) return -EFAULT; |
1c19448c9 net: Make enablin... |
2252 |
valbool = val ? 1 : 0; |
6516c6557 [UDP]: ipv4 white... |
2253 |
switch (optname) { |
1da177e4c Linux-2.6.12-rc2 |
2254 2255 2256 2257 2258 2259 |
case UDP_CORK: if (val != 0) { up->corkflag = 1; } else { up->corkflag = 0; lock_sock(sk); |
4243cdc2c udp: Neaten funct... |
2260 |
push_pending_frames(sk); |
1da177e4c Linux-2.6.12-rc2 |
2261 2262 2263 |
release_sock(sk); } break; |
e905a9eda [NET] IPV4: Fix w... |
2264 |
|
1da177e4c Linux-2.6.12-rc2 |
2265 2266 2267 2268 2269 |
case UDP_ENCAP: switch (val) { case 0: case UDP_ENCAP_ESPINUDP: case UDP_ENCAP_ESPINUDP_NON_IKE: |
067b207b2 [UDP]: Cleanup UD... |
2270 2271 |
up->encap_rcv = xfrm4_udp_encap_rcv; /* FALLTHROUGH */ |
342f0234c [UDP]: Introduce ... |
2272 |
case UDP_ENCAP_L2TPINUDP: |
1da177e4c Linux-2.6.12-rc2 |
2273 |
up->encap_type = val; |
447167bf5 udp: intoduce udp... |
2274 |
udp_encap_enable(); |
1da177e4c Linux-2.6.12-rc2 |
2275 2276 2277 2278 2279 2280 |
break; default: err = -ENOPROTOOPT; break; } break; |
1c19448c9 net: Make enablin... |
2281 2282 2283 2284 2285 2286 2287 |
case UDP_NO_CHECK6_TX: up->no_check6_tx = valbool; break; case UDP_NO_CHECK6_RX: up->no_check6_rx = valbool; break; |
ba4e58eca [NET]: Supporting... |
2288 2289 2290 2291 2292 2293 |
/* * UDP-Lite's partial checksum coverage (RFC 3828). */ /* The sender sets actual checksum coverage length via this option. * The case coverage > packet length is handled by send module. */ case UDPLITE_SEND_CSCOV: |
b2bf1e265 [UDP]: Clean up f... |
2294 |
if (!is_udplite) /* Disable the option on UDP sockets */ |
ba4e58eca [NET]: Supporting... |
2295 2296 2297 |
return -ENOPROTOOPT; if (val != 0 && val < 8) /* Illegal coverage: use default (8) */ val = 8; |
4be929be3 kernel-wide: repl... |
2298 2299 |
else if (val > USHRT_MAX) val = USHRT_MAX; |
ba4e58eca [NET]: Supporting... |
2300 2301 2302 |
up->pcslen = val; up->pcflag |= UDPLITE_SEND_CC; break; |
e905a9eda [NET] IPV4: Fix w... |
2303 2304 |
/* The receiver specifies a minimum checksum coverage value. To make * sense, this should be set to at least 8 (as done below). If zero is |
ba4e58eca [NET]: Supporting... |
2305 2306 |
* used, this again means full checksum coverage. */ case UDPLITE_RECV_CSCOV: |
b2bf1e265 [UDP]: Clean up f... |
2307 |
if (!is_udplite) /* Disable the option on UDP sockets */ |
ba4e58eca [NET]: Supporting... |
2308 2309 2310 |
return -ENOPROTOOPT; if (val != 0 && val < 8) /* Avoid silly minimal values. */ val = 8; |
4be929be3 kernel-wide: repl... |
2311 2312 |
else if (val > USHRT_MAX) val = USHRT_MAX; |
ba4e58eca [NET]: Supporting... |
2313 2314 2315 |
up->pcrlen = val; up->pcflag |= UDPLITE_RECV_CC; break; |
1da177e4c Linux-2.6.12-rc2 |
2316 2317 2318 |
default: err = -ENOPROTOOPT; break; |
6516c6557 [UDP]: ipv4 white... |
2319 |
} |
1da177e4c Linux-2.6.12-rc2 |
2320 2321 2322 |
return err; } |
c482c5685 udp: cleanups |
2323 |
EXPORT_SYMBOL(udp_lib_setsockopt); |
1da177e4c Linux-2.6.12-rc2 |
2324 |
|
db8dac20d [UDP]: Revert udp... |
2325 |
int udp_setsockopt(struct sock *sk, int level, int optname, |
b7058842c net: Make setsock... |
2326 |
char __user *optval, unsigned int optlen) |
db8dac20d [UDP]: Revert udp... |
2327 2328 2329 2330 2331 2332 2333 2334 2335 |
{ if (level == SOL_UDP || level == SOL_UDPLITE) return udp_lib_setsockopt(sk, level, optname, optval, optlen, udp_push_pending_frames); return ip_setsockopt(sk, level, optname, optval, optlen); } #ifdef CONFIG_COMPAT int compat_udp_setsockopt(struct sock *sk, int level, int optname, |
b7058842c net: Make setsock... |
2336 |
char __user *optval, unsigned int optlen) |
db8dac20d [UDP]: Revert udp... |
2337 2338 2339 2340 2341 2342 2343 |
{ if (level == SOL_UDP || level == SOL_UDPLITE) return udp_lib_setsockopt(sk, level, optname, optval, optlen, udp_push_pending_frames); return compat_ip_setsockopt(sk, level, optname, optval, optlen); } #endif |
4c0a6cb0d [UDP(-Lite)]: con... |
2344 2345 |
int udp_lib_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) |
1da177e4c Linux-2.6.12-rc2 |
2346 2347 2348 |
{ struct udp_sock *up = udp_sk(sk); int val, len; |
c482c5685 udp: cleanups |
2349 |
if (get_user(len, optlen)) |
1da177e4c Linux-2.6.12-rc2 |
2350 2351 2352 |
return -EFAULT; len = min_t(unsigned int, len, sizeof(int)); |
e905a9eda [NET] IPV4: Fix w... |
2353 |
|
6516c6557 [UDP]: ipv4 white... |
2354 |
if (len < 0) |
1da177e4c Linux-2.6.12-rc2 |
2355 |
return -EINVAL; |
6516c6557 [UDP]: ipv4 white... |
2356 |
switch (optname) { |
1da177e4c Linux-2.6.12-rc2 |
2357 2358 2359 2360 2361 2362 2363 |
case UDP_CORK: val = up->corkflag; break; case UDP_ENCAP: val = up->encap_type; break; |
1c19448c9 net: Make enablin... |
2364 2365 2366 2367 2368 2369 2370 |
case UDP_NO_CHECK6_TX: val = up->no_check6_tx; break; case UDP_NO_CHECK6_RX: val = up->no_check6_rx; break; |
ba4e58eca [NET]: Supporting... |
2371 2372 2373 2374 2375 2376 2377 2378 2379 |
/* The following two cannot be changed on UDP sockets, the return is * always 0 (which corresponds to the full checksum coverage of UDP). */ case UDPLITE_SEND_CSCOV: val = up->pcslen; break; case UDPLITE_RECV_CSCOV: val = up->pcrlen; break; |
1da177e4c Linux-2.6.12-rc2 |
2380 2381 |
default: return -ENOPROTOOPT; |
6516c6557 [UDP]: ipv4 white... |
2382 |
} |
1da177e4c Linux-2.6.12-rc2 |
2383 |
|
6516c6557 [UDP]: ipv4 white... |
2384 |
if (put_user(len, optlen)) |
e905a9eda [NET] IPV4: Fix w... |
2385 |
return -EFAULT; |
c482c5685 udp: cleanups |
2386 |
if (copy_to_user(optval, &val, len)) |
1da177e4c Linux-2.6.12-rc2 |
2387 |
return -EFAULT; |
e905a9eda [NET] IPV4: Fix w... |
2388 |
return 0; |
1da177e4c Linux-2.6.12-rc2 |
2389 |
} |
c482c5685 udp: cleanups |
2390 |
EXPORT_SYMBOL(udp_lib_getsockopt); |
1da177e4c Linux-2.6.12-rc2 |
2391 |
|
db8dac20d [UDP]: Revert udp... |
2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 |
int udp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) { if (level == SOL_UDP || level == SOL_UDPLITE) return udp_lib_getsockopt(sk, level, optname, optval, optlen); return ip_getsockopt(sk, level, optname, optval, optlen); } #ifdef CONFIG_COMPAT int compat_udp_getsockopt(struct sock *sk, int level, int optname, char __user *optval, int __user *optlen) { if (level == SOL_UDP || level == SOL_UDPLITE) return udp_lib_getsockopt(sk, level, optname, optval, optlen); return compat_ip_getsockopt(sk, level, optname, optval, optlen); } #endif |
1da177e4c Linux-2.6.12-rc2 |
2409 2410 2411 2412 2413 2414 |
/** * udp_poll - wait for a UDP event. * @file - file struct * @sock - socket * @wait - poll table * |
e905a9eda [NET] IPV4: Fix w... |
2415 |
* This is same as datagram poll, except for the special case of |
1da177e4c Linux-2.6.12-rc2 |
2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 |
* blocking sockets. If application is using a blocking fd * and a packet with checksum error is in the queue; * then it could get return from select indicating data available * but then block when reading it. Add special case code * to work around these arguably broken applications. */ unsigned int udp_poll(struct file *file, struct socket *sock, poll_table *wait) { unsigned int mask = datagram_poll(file, sock, wait); struct sock *sk = sock->sk; |
ba4e58eca [NET]: Supporting... |
2426 |
|
2276f58ac udp: use a separa... |
2427 2428 |
if (!skb_queue_empty(&udp_sk(sk)->reader_queue)) mask |= POLLIN | POLLRDNORM; |
c3f1dbaf6 net: Update RFS t... |
2429 |
sock_rps_record_flow(sk); |
1da177e4c Linux-2.6.12-rc2 |
2430 |
/* Check for false positives due to checksum errors */ |
855846720 udp: Fix udp_poll... |
2431 |
if ((mask & POLLRDNORM) && !(file->f_flags & O_NONBLOCK) && |
e83c6744e udp: fix poll() i... |
2432 |
!(sk->sk_shutdown & RCV_SHUTDOWN) && first_packet_length(sk) == -1) |
855846720 udp: Fix udp_poll... |
2433 |
mask &= ~(POLLIN | POLLRDNORM); |
1da177e4c Linux-2.6.12-rc2 |
2434 2435 |
return mask; |
e905a9eda [NET] IPV4: Fix w... |
2436 |
|
1da177e4c Linux-2.6.12-rc2 |
2437 |
} |
c482c5685 udp: cleanups |
2438 |
EXPORT_SYMBOL(udp_poll); |
1da177e4c Linux-2.6.12-rc2 |
2439 |
|
5d77dca82 net: diag: suppor... |
2440 2441 2442 2443 2444 2445 |
int udp_abort(struct sock *sk, int err) { lock_sock(sk); sk->sk_err = err; sk->sk_error_report(sk); |
286c72dea udp: must lock th... |
2446 |
__udp_disconnect(sk, 0); |
5d77dca82 net: diag: suppor... |
2447 2448 2449 2450 2451 2452 |
release_sock(sk); return 0; } EXPORT_SYMBOL_GPL(udp_abort); |
db8dac20d [UDP]: Revert udp... |
2453 2454 2455 2456 2457 2458 2459 |
struct proto udp_prot = { .name = "UDP", .owner = THIS_MODULE, .close = udp_lib_close, .connect = ip4_datagram_connect, .disconnect = udp_disconnect, .ioctl = udp_ioctl, |
850cbaddb udp: use it's own... |
2460 |
.init = udp_init_sock, |
db8dac20d [UDP]: Revert udp... |
2461 2462 2463 2464 2465 2466 |
.destroy = udp_destroy_sock, .setsockopt = udp_setsockopt, .getsockopt = udp_getsockopt, .sendmsg = udp_sendmsg, .recvmsg = udp_recvmsg, .sendpage = udp_sendpage, |
8141ed9fc ipv4: Add a socke... |
2467 |
.release_cb = ip4_datagram_release_cb, |
db8dac20d [UDP]: Revert udp... |
2468 2469 |
.hash = udp_lib_hash, .unhash = udp_lib_unhash, |
719f83585 udp: add rehash o... |
2470 |
.rehash = udp_v4_rehash, |
db8dac20d [UDP]: Revert udp... |
2471 2472 2473 2474 2475 2476 |
.get_port = udp_v4_get_port, .memory_allocated = &udp_memory_allocated, .sysctl_mem = sysctl_udp_mem, .sysctl_wmem = &sysctl_udp_wmem_min, .sysctl_rmem = &sysctl_udp_rmem_min, .obj_size = sizeof(struct udp_sock), |
645ca708f udp: introduce st... |
2477 |
.h.udp_table = &udp_table, |
db8dac20d [UDP]: Revert udp... |
2478 2479 2480 2481 |
#ifdef CONFIG_COMPAT .compat_setsockopt = compat_udp_setsockopt, .compat_getsockopt = compat_udp_getsockopt, #endif |
5d77dca82 net: diag: suppor... |
2482 |
.diag_destroy = udp_abort, |
db8dac20d [UDP]: Revert udp... |
2483 |
}; |
c482c5685 udp: cleanups |
2484 |
EXPORT_SYMBOL(udp_prot); |
1da177e4c Linux-2.6.12-rc2 |
2485 2486 2487 |
/* ------------------------------------------------------------------------ */ #ifdef CONFIG_PROC_FS |
645ca708f udp: introduce st... |
2488 |
static struct sock *udp_get_first(struct seq_file *seq, int start) |
1da177e4c Linux-2.6.12-rc2 |
2489 2490 2491 |
{ struct sock *sk; struct udp_iter_state *state = seq->private; |
6f191efe4 [UDP]: Replace st... |
2492 |
struct net *net = seq_file_net(seq); |
1da177e4c Linux-2.6.12-rc2 |
2493 |
|
f86dcc5aa udp: dynamically ... |
2494 2495 |
for (state->bucket = start; state->bucket <= state->udp_table->mask; ++state->bucket) { |
645ca708f udp: introduce st... |
2496 |
struct udp_hslot *hslot = &state->udp_table->hash[state->bucket]; |
f86dcc5aa udp: dynamically ... |
2497 |
|
ca065d0cf udp: no longer us... |
2498 |
if (hlist_empty(&hslot->head)) |
f86dcc5aa udp: dynamically ... |
2499 |
continue; |
645ca708f udp: introduce st... |
2500 |
spin_lock_bh(&hslot->lock); |
ca065d0cf udp: no longer us... |
2501 |
sk_for_each(sk, &hslot->head) { |
878628fbf [NET] NETNS: Omit... |
2502 |
if (!net_eq(sock_net(sk), net)) |
a91275eff [NETNS][IPV6] udp... |
2503 |
continue; |
1da177e4c Linux-2.6.12-rc2 |
2504 2505 2506 |
if (sk->sk_family == state->family) goto found; } |
645ca708f udp: introduce st... |
2507 |
spin_unlock_bh(&hslot->lock); |
1da177e4c Linux-2.6.12-rc2 |
2508 2509 2510 2511 2512 2513 2514 2515 2516 |
} sk = NULL; found: return sk; } static struct sock *udp_get_next(struct seq_file *seq, struct sock *sk) { struct udp_iter_state *state = seq->private; |
6f191efe4 [UDP]: Replace st... |
2517 |
struct net *net = seq_file_net(seq); |
1da177e4c Linux-2.6.12-rc2 |
2518 2519 |
do { |
ca065d0cf udp: no longer us... |
2520 |
sk = sk_next(sk); |
878628fbf [NET] NETNS: Omit... |
2521 |
} while (sk && (!net_eq(sock_net(sk), net) || sk->sk_family != state->family)); |
1da177e4c Linux-2.6.12-rc2 |
2522 |
|
645ca708f udp: introduce st... |
2523 |
if (!sk) { |
f86dcc5aa udp: dynamically ... |
2524 |
if (state->bucket <= state->udp_table->mask) |
30842f298 udp: Wrong lockin... |
2525 |
spin_unlock_bh(&state->udp_table->hash[state->bucket].lock); |
645ca708f udp: introduce st... |
2526 |
return udp_get_first(seq, state->bucket + 1); |
1da177e4c Linux-2.6.12-rc2 |
2527 2528 2529 2530 2531 2532 |
} return sk; } static struct sock *udp_get_idx(struct seq_file *seq, loff_t pos) { |
645ca708f udp: introduce st... |
2533 |
struct sock *sk = udp_get_first(seq, 0); |
1da177e4c Linux-2.6.12-rc2 |
2534 2535 |
if (sk) |
6516c6557 [UDP]: ipv4 white... |
2536 |
while (pos && (sk = udp_get_next(seq, sk)) != NULL) |
1da177e4c Linux-2.6.12-rc2 |
2537 2538 2539 2540 2541 2542 |
--pos; return pos ? NULL : sk; } static void *udp_seq_start(struct seq_file *seq, loff_t *pos) { |
30842f298 udp: Wrong lockin... |
2543 |
struct udp_iter_state *state = seq->private; |
f86dcc5aa udp: dynamically ... |
2544 |
state->bucket = MAX_UDP_PORTS; |
30842f298 udp: Wrong lockin... |
2545 |
|
b50660f1f [IP] UDP: Use SEQ... |
2546 |
return *pos ? udp_get_idx(seq, *pos-1) : SEQ_START_TOKEN; |
1da177e4c Linux-2.6.12-rc2 |
2547 2548 2549 2550 2551 |
} static void *udp_seq_next(struct seq_file *seq, void *v, loff_t *pos) { struct sock *sk; |
b50660f1f [IP] UDP: Use SEQ... |
2552 |
if (v == SEQ_START_TOKEN) |
1da177e4c Linux-2.6.12-rc2 |
2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 |
sk = udp_get_idx(seq, 0); else sk = udp_get_next(seq, v); ++*pos; return sk; } static void udp_seq_stop(struct seq_file *seq, void *v) { |
645ca708f udp: introduce st... |
2563 |
struct udp_iter_state *state = seq->private; |
f86dcc5aa udp: dynamically ... |
2564 |
if (state->bucket <= state->udp_table->mask) |
645ca708f udp: introduce st... |
2565 |
spin_unlock_bh(&state->udp_table->hash[state->bucket].lock); |
1da177e4c Linux-2.6.12-rc2 |
2566 |
} |
73cb88ecb net: make the tcp... |
2567 |
int udp_seq_open(struct inode *inode, struct file *file) |
1da177e4c Linux-2.6.12-rc2 |
2568 |
{ |
d9dda78ba procfs: new helpe... |
2569 |
struct udp_seq_afinfo *afinfo = PDE_DATA(inode); |
a2be75c18 [UDP]: Cleanup /p... |
2570 2571 |
struct udp_iter_state *s; int err; |
a91275eff [NETNS][IPV6] udp... |
2572 |
|
a2be75c18 [UDP]: Cleanup /p... |
2573 2574 2575 2576 |
err = seq_open_net(inode, file, &afinfo->seq_ops, sizeof(struct udp_iter_state)); if (err < 0) return err; |
a91275eff [NETNS][IPV6] udp... |
2577 |
|
a2be75c18 [UDP]: Cleanup /p... |
2578 |
s = ((struct seq_file *)file->private_data)->private; |
1da177e4c Linux-2.6.12-rc2 |
2579 |
s->family = afinfo->family; |
645ca708f udp: introduce st... |
2580 |
s->udp_table = afinfo->udp_table; |
a2be75c18 [UDP]: Cleanup /p... |
2581 |
return err; |
a91275eff [NETNS][IPV6] udp... |
2582 |
} |
73cb88ecb net: make the tcp... |
2583 |
EXPORT_SYMBOL(udp_seq_open); |
a91275eff [NETNS][IPV6] udp... |
2584 |
|
1da177e4c Linux-2.6.12-rc2 |
2585 |
/* ------------------------------------------------------------------------ */ |
0c96d8c50 [NETNS][IPV6] udp... |
2586 |
int udp_proc_register(struct net *net, struct udp_seq_afinfo *afinfo) |
1da177e4c Linux-2.6.12-rc2 |
2587 2588 2589 |
{ struct proc_dir_entry *p; int rc = 0; |
dda61925f [UDP]: Move seq_o... |
2590 2591 2592 |
afinfo->seq_ops.start = udp_seq_start; afinfo->seq_ops.next = udp_seq_next; afinfo->seq_ops.stop = udp_seq_stop; |
84841c3c6 ipv4: assign PDE-... |
2593 |
p = proc_create_data(afinfo->name, S_IRUGO, net->proc_net, |
73cb88ecb net: make the tcp... |
2594 |
afinfo->seq_fops, afinfo); |
84841c3c6 ipv4: assign PDE-... |
2595 |
if (!p) |
1da177e4c Linux-2.6.12-rc2 |
2596 2597 2598 |
rc = -ENOMEM; return rc; } |
c482c5685 udp: cleanups |
2599 |
EXPORT_SYMBOL(udp_proc_register); |
1da177e4c Linux-2.6.12-rc2 |
2600 |
|
0c96d8c50 [NETNS][IPV6] udp... |
2601 |
void udp_proc_unregister(struct net *net, struct udp_seq_afinfo *afinfo) |
1da177e4c Linux-2.6.12-rc2 |
2602 |
{ |
ece31ffd5 net: proc: change... |
2603 |
remove_proc_entry(afinfo->name, net->proc_net); |
1da177e4c Linux-2.6.12-rc2 |
2604 |
} |
c482c5685 udp: cleanups |
2605 |
EXPORT_SYMBOL(udp_proc_unregister); |
db8dac20d [UDP]: Revert udp... |
2606 2607 |
/* ------------------------------------------------------------------------ */ |
5e659e4cb [NET]: Fix heavy ... |
2608 |
static void udp4_format_sock(struct sock *sp, struct seq_file *f, |
652586df9 seq_file: remove ... |
2609 |
int bucket) |
db8dac20d [UDP]: Revert udp... |
2610 2611 |
{ struct inet_sock *inet = inet_sk(sp); |
c720c7e83 inet: rename some... |
2612 2613 2614 2615 |
__be32 dest = inet->inet_daddr; __be32 src = inet->inet_rcv_saddr; __u16 destp = ntohs(inet->inet_dport); __u16 srcp = ntohs(inet->inet_sport); |
db8dac20d [UDP]: Revert udp... |
2616 |
|
f86dcc5aa udp: dynamically ... |
2617 |
seq_printf(f, "%5d: %08X:%04X %08X:%04X" |
652586df9 seq_file: remove ... |
2618 |
" %02X %08X:%08X %02X:%08lX %08X %5u %8d %lu %d %pK %d", |
db8dac20d [UDP]: Revert udp... |
2619 |
bucket, src, srcp, dest, destp, sp->sk_state, |
31e6d363a net: correct off-... |
2620 |
sk_wmem_alloc_get(sp), |
2e5d31688 udp: fix rx queue... |
2621 |
udp_rqueue_get(sp), |
a7cb5a49b userns: Print out... |
2622 2623 2624 |
0, 0L, 0, from_kuid_munged(seq_user_ns(f), sock_i_uid(sp)), 0, sock_i_ino(sp), |
41c6d650f net: convert sock... |
2625 |
refcount_read(&sp->sk_refcnt), sp, |
652586df9 seq_file: remove ... |
2626 |
atomic_read(&sp->sk_drops)); |
db8dac20d [UDP]: Revert udp... |
2627 2628 2629 2630 |
} int udp4_seq_show(struct seq_file *seq, void *v) { |
652586df9 seq_file: remove ... |
2631 |
seq_setwidth(seq, 127); |
db8dac20d [UDP]: Revert udp... |
2632 |
if (v == SEQ_START_TOKEN) |
652586df9 seq_file: remove ... |
2633 |
seq_puts(seq, " sl local_address rem_address st tx_queue " |
db8dac20d [UDP]: Revert udp... |
2634 |
"rx_queue tr tm->when retrnsmt uid timeout " |
cb61cb9b8 udp: sk_drops han... |
2635 |
"inode ref pointer drops"); |
db8dac20d [UDP]: Revert udp... |
2636 |
else { |
db8dac20d [UDP]: Revert udp... |
2637 |
struct udp_iter_state *state = seq->private; |
652586df9 seq_file: remove ... |
2638 |
udp4_format_sock(v, seq, state->bucket); |
db8dac20d [UDP]: Revert udp... |
2639 |
} |
652586df9 seq_file: remove ... |
2640 2641 |
seq_pad(seq, ' '); |
db8dac20d [UDP]: Revert udp... |
2642 2643 |
return 0; } |
73cb88ecb net: make the tcp... |
2644 2645 2646 2647 2648 2649 2650 |
static const struct file_operations udp_afinfo_seq_fops = { .owner = THIS_MODULE, .open = udp_seq_open, .read = seq_read, .llseek = seq_lseek, .release = seq_release_net }; |
db8dac20d [UDP]: Revert udp... |
2651 |
/* ------------------------------------------------------------------------ */ |
db8dac20d [UDP]: Revert udp... |
2652 |
static struct udp_seq_afinfo udp4_seq_afinfo = { |
db8dac20d [UDP]: Revert udp... |
2653 2654 |
.name = "udp", .family = AF_INET, |
645ca708f udp: introduce st... |
2655 |
.udp_table = &udp_table, |
73cb88ecb net: make the tcp... |
2656 |
.seq_fops = &udp_afinfo_seq_fops, |
dda61925f [UDP]: Move seq_o... |
2657 2658 2659 |
.seq_ops = { .show = udp4_seq_show, }, |
db8dac20d [UDP]: Revert udp... |
2660 |
}; |
2c8c1e729 net: spread __net... |
2661 |
static int __net_init udp4_proc_init_net(struct net *net) |
15439febb [NETNS][UDP]: Reg... |
2662 2663 2664 |
{ return udp_proc_register(net, &udp4_seq_afinfo); } |
2c8c1e729 net: spread __net... |
2665 |
static void __net_exit udp4_proc_exit_net(struct net *net) |
15439febb [NETNS][UDP]: Reg... |
2666 2667 2668 2669 2670 2671 2672 2673 |
{ udp_proc_unregister(net, &udp4_seq_afinfo); } static struct pernet_operations udp4_net_ops = { .init = udp4_proc_init_net, .exit = udp4_proc_exit_net, }; |
db8dac20d [UDP]: Revert udp... |
2674 2675 |
int __init udp4_proc_init(void) { |
15439febb [NETNS][UDP]: Reg... |
2676 |
return register_pernet_subsys(&udp4_net_ops); |
db8dac20d [UDP]: Revert udp... |
2677 2678 2679 2680 |
} void udp4_proc_exit(void) { |
15439febb [NETNS][UDP]: Reg... |
2681 |
unregister_pernet_subsys(&udp4_net_ops); |
db8dac20d [UDP]: Revert udp... |
2682 |
} |
1da177e4c Linux-2.6.12-rc2 |
2683 |
#endif /* CONFIG_PROC_FS */ |
f86dcc5aa udp: dynamically ... |
2684 2685 |
static __initdata unsigned long uhash_entries; static int __init set_uhash_entries(char *str) |
645ca708f udp: introduce st... |
2686 |
{ |
413c27d86 net/ipv4: replace... |
2687 |
ssize_t ret; |
f86dcc5aa udp: dynamically ... |
2688 2689 |
if (!str) return 0; |
413c27d86 net/ipv4: replace... |
2690 2691 2692 2693 |
ret = kstrtoul(str, 0, &uhash_entries); if (ret) return 0; |
f86dcc5aa udp: dynamically ... |
2694 2695 2696 2697 2698 |
if (uhash_entries && uhash_entries < UDP_HTABLE_SIZE_MIN) uhash_entries = UDP_HTABLE_SIZE_MIN; return 1; } __setup("uhash_entries=", set_uhash_entries); |
645ca708f udp: introduce st... |
2699 |
|
f86dcc5aa udp: dynamically ... |
2700 2701 2702 |
void __init udp_table_init(struct udp_table *table, const char *name) { unsigned int i; |
31fe62b95 mm: add a low lim... |
2703 2704 2705 2706 2707 2708 2709 2710 2711 |
table->hash = alloc_large_system_hash(name, 2 * sizeof(struct udp_hslot), uhash_entries, 21, /* one slot per 2 MB */ 0, &table->log, &table->mask, UDP_HTABLE_SIZE_MIN, 64 * 1024); |
512615b6b udp: secondary ha... |
2712 |
table->hash2 = table->hash + (table->mask + 1); |
f86dcc5aa udp: dynamically ... |
2713 |
for (i = 0; i <= table->mask; i++) { |
ca065d0cf udp: no longer us... |
2714 |
INIT_HLIST_HEAD(&table->hash[i].head); |
fdcc8aa95 udp: add a counte... |
2715 |
table->hash[i].count = 0; |
645ca708f udp: introduce st... |
2716 2717 |
spin_lock_init(&table->hash[i].lock); } |
512615b6b udp: secondary ha... |
2718 |
for (i = 0; i <= table->mask; i++) { |
ca065d0cf udp: no longer us... |
2719 |
INIT_HLIST_HEAD(&table->hash2[i].head); |
512615b6b udp: secondary ha... |
2720 2721 2722 |
table->hash2[i].count = 0; spin_lock_init(&table->hash2[i].lock); } |
645ca708f udp: introduce st... |
2723 |
} |
723b8e460 udp: In udp_flow_... |
2724 2725 2726 2727 2728 2729 2730 2731 2732 |
u32 udp_flow_hashrnd(void) { static u32 hashrnd __read_mostly; net_get_random_once(&hashrnd, sizeof(hashrnd)); return hashrnd; } EXPORT_SYMBOL(udp_flow_hashrnd); |
95766fff6 [UDP]: Add memory... |
2733 2734 |
void __init udp_init(void) { |
f03d78db6 net: refine {udp|... |
2735 |
unsigned long limit; |
4b272750d udp: add busylock... |
2736 |
unsigned int i; |
95766fff6 [UDP]: Add memory... |
2737 |
|
f86dcc5aa udp: dynamically ... |
2738 |
udp_table_init(&udp_table, "UDP"); |
f03d78db6 net: refine {udp|... |
2739 |
limit = nr_free_buffer_pages() / 8; |
95766fff6 [UDP]: Add memory... |
2740 2741 2742 2743 2744 2745 2746 |
limit = max(limit, 128UL); sysctl_udp_mem[0] = limit / 4 * 3; sysctl_udp_mem[1] = limit; sysctl_udp_mem[2] = sysctl_udp_mem[0] * 2; sysctl_udp_rmem_min = SK_MEM_QUANTUM; sysctl_udp_wmem_min = SK_MEM_QUANTUM; |
4b272750d udp: add busylock... |
2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 |
/* 16 spinlocks per cpu */ udp_busylocks_log = ilog2(nr_cpu_ids) + 4; udp_busylocks = kmalloc(sizeof(spinlock_t) << udp_busylocks_log, GFP_KERNEL); if (!udp_busylocks) panic("UDP: failed to alloc udp_busylocks "); for (i = 0; i < (1U << udp_busylocks_log); i++) spin_lock_init(udp_busylocks + i); |
95766fff6 [UDP]: Add memory... |
2757 |
} |