Blame view
net/ipv4/inet_hashtables.c
18.2 KB
77d8bf9c6 [INET]: Move the ... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 |
/* * INET An implementation of the TCP/IP protocol suite for the LINUX * operating system. INET is implemented using the BSD Socket * interface as the means of communication with the user level. * * Generic INET transport hashtables * * Authors: Lotsa people, from code originally in tcp * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ |
2d8c4ce51 [INET]: Generalis... |
15 |
#include <linux/module.h> |
a7f5e7f16 [INET]: Generalis... |
16 |
#include <linux/random.h> |
f3f05f704 [INET]: Generalis... |
17 |
#include <linux/sched.h> |
77d8bf9c6 [INET]: Move the ... |
18 |
#include <linux/slab.h> |
f3f05f704 [INET]: Generalis... |
19 |
#include <linux/wait.h> |
095dc8e0c tcp: fix/cleanup ... |
20 |
#include <linux/vmalloc.h> |
77d8bf9c6 [INET]: Move the ... |
21 |
|
c125e80b8 soreuseport: fast... |
22 |
#include <net/addrconf.h> |
463c84b97 [NET]: Introduce ... |
23 |
#include <net/inet_connection_sock.h> |
77d8bf9c6 [INET]: Move the ... |
24 |
#include <net/inet_hashtables.h> |
6e5714eaf net: Compute prot... |
25 |
#include <net/secure_seq.h> |
a7f5e7f16 [INET]: Generalis... |
26 |
#include <net/ip.h> |
a04a480d4 net: Require exac... |
27 |
#include <net/tcp.h> |
c125e80b8 soreuseport: fast... |
28 |
#include <net/sock_reuseport.h> |
77d8bf9c6 [INET]: Move the ... |
29 |
|
6eada0110 netns: constify n... |
30 31 32 |
static u32 inet_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport, const __be32 faddr, const __be16 fport) |
65cd8033f ipv4: split inet_... |
33 |
{ |
1bbdceef1 inet: convert ine... |
34 35 36 |
static u32 inet_ehash_secret __read_mostly; net_get_random_once(&inet_ehash_secret, sizeof(inet_ehash_secret)); |
65cd8033f ipv4: split inet_... |
37 38 39 |
return __inet_ehashfn(laddr, lport, faddr, fport, inet_ehash_secret + net_hash_mix(net)); } |
d1e559d0b inet: add IPv6 su... |
40 41 42 |
/* This function handles inet_sock, but also timewait and request sockets * for IPv4/IPv6. */ |
784c372a8 net: make sk_ehas... |
43 |
static u32 sk_ehashfn(const struct sock *sk) |
65cd8033f ipv4: split inet_... |
44 |
{ |
d1e559d0b inet: add IPv6 su... |
45 46 47 48 49 50 51 |
#if IS_ENABLED(CONFIG_IPV6) if (sk->sk_family == AF_INET6 && !ipv6_addr_v4mapped(&sk->sk_v6_daddr)) return inet6_ehashfn(sock_net(sk), &sk->sk_v6_rcv_saddr, sk->sk_num, &sk->sk_v6_daddr, sk->sk_dport); #endif |
5b441f76f net: introduce sk... |
52 53 54 |
return inet_ehashfn(sock_net(sk), sk->sk_rcv_saddr, sk->sk_num, sk->sk_daddr, sk->sk_dport); |
65cd8033f ipv4: split inet_... |
55 |
} |
77d8bf9c6 [INET]: Move the ... |
56 57 58 59 |
/* * Allocate and initialize a new local port bind bucket. * The bindhash mutex for snum's hash chain must be held here. */ |
e18b890bb [PATCH] slab: rem... |
60 |
struct inet_bind_bucket *inet_bind_bucket_create(struct kmem_cache *cachep, |
941b1d22c [NETNS]: Make bin... |
61 |
struct net *net, |
77d8bf9c6 [INET]: Move the ... |
62 63 64 |
struct inet_bind_hashbucket *head, const unsigned short snum) { |
54e6ecb23 [PATCH] slab: rem... |
65 |
struct inet_bind_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC); |
77d8bf9c6 [INET]: Move the ... |
66 |
|
00db41243 ipv4: coding styl... |
67 |
if (tb) { |
efd7ef1c1 net: Kill hold_ne... |
68 |
write_pnet(&tb->ib_net, net); |
77d8bf9c6 [INET]: Move the ... |
69 70 |
tb->port = snum; tb->fastreuse = 0; |
da5e36308 soreuseport: TCP/... |
71 |
tb->fastreuseport = 0; |
77d8bf9c6 [INET]: Move the ... |
72 73 74 75 76 |
INIT_HLIST_HEAD(&tb->owners); hlist_add_head(&tb->node, &head->chain); } return tb; } |
77d8bf9c6 [INET]: Move the ... |
77 78 79 |
/* * Caller must hold hashbucket lock for this tb with local BH disabled */ |
e18b890bb [PATCH] slab: rem... |
80 |
void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket *tb) |
77d8bf9c6 [INET]: Move the ... |
81 82 83 84 85 86 |
{ if (hlist_empty(&tb->owners)) { __hlist_del(&tb->node); kmem_cache_free(cachep, tb); } } |
2d8c4ce51 [INET]: Generalis... |
87 88 89 90 |
void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb, const unsigned short snum) { |
c720c7e83 inet: rename some... |
91 |
inet_sk(sk)->inet_num = snum; |
2d8c4ce51 [INET]: Generalis... |
92 |
sk_add_bind_node(sk, &tb->owners); |
463c84b97 [NET]: Introduce ... |
93 |
inet_csk(sk)->icsk_bind_hash = tb; |
2d8c4ce51 [INET]: Generalis... |
94 |
} |
2d8c4ce51 [INET]: Generalis... |
95 96 97 |
/* * Get rid of any references to a local port held by the given sock. */ |
ab1e0a13d [SOCK] proto: Add... |
98 |
static void __inet_put_port(struct sock *sk) |
2d8c4ce51 [INET]: Generalis... |
99 |
{ |
39d8cda76 [SOCK]: Add udp_h... |
100 |
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; |
c720c7e83 inet: rename some... |
101 |
const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num, |
7f635ab71 inet: add struct ... |
102 |
hashinfo->bhash_size); |
2d8c4ce51 [INET]: Generalis... |
103 104 105 106 |
struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash]; struct inet_bind_bucket *tb; spin_lock(&head->lock); |
463c84b97 [NET]: Introduce ... |
107 |
tb = inet_csk(sk)->icsk_bind_hash; |
2d8c4ce51 [INET]: Generalis... |
108 |
__sk_del_bind_node(sk); |
463c84b97 [NET]: Introduce ... |
109 |
inet_csk(sk)->icsk_bind_hash = NULL; |
c720c7e83 inet: rename some... |
110 |
inet_sk(sk)->inet_num = 0; |
2d8c4ce51 [INET]: Generalis... |
111 112 113 |
inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb); spin_unlock(&head->lock); } |
ab1e0a13d [SOCK] proto: Add... |
114 |
void inet_put_port(struct sock *sk) |
2d8c4ce51 [INET]: Generalis... |
115 116 |
{ local_bh_disable(); |
ab1e0a13d [SOCK] proto: Add... |
117 |
__inet_put_port(sk); |
2d8c4ce51 [INET]: Generalis... |
118 119 |
local_bh_enable(); } |
2d8c4ce51 [INET]: Generalis... |
120 |
EXPORT_SYMBOL(inet_put_port); |
f3f05f704 [INET]: Generalis... |
121 |
|
1ce31c9e0 inet: constify __... |
122 |
int __inet_inherit_port(const struct sock *sk, struct sock *child) |
53083773d [INET]: Uninline ... |
123 124 |
{ struct inet_hashinfo *table = sk->sk_prot->h.hashinfo; |
093d28232 tproxy: fix hash ... |
125 126 |
unsigned short port = inet_sk(child)->inet_num; const int bhash = inet_bhashfn(sock_net(sk), port, |
7f635ab71 inet: add struct ... |
127 |
table->bhash_size); |
53083773d [INET]: Uninline ... |
128 129 130 131 132 |
struct inet_bind_hashbucket *head = &table->bhash[bhash]; struct inet_bind_bucket *tb; spin_lock(&head->lock); tb = inet_csk(sk)->icsk_bind_hash; |
c2f34a65a tcp/dccp: fix pot... |
133 134 135 136 |
if (unlikely(!tb)) { spin_unlock(&head->lock); return -ENOENT; } |
093d28232 tproxy: fix hash ... |
137 138 139 140 141 142 |
if (tb->port != port) { /* NOTE: using tproxy and redirecting skbs to a proxy * on a different listener port breaks the assumption * that the listener socket's icsk_bind_hash is the same * as that of the child socket. We have to look up or * create a new bind bucket for the child here. */ |
b67bfe0d4 hlist: drop the n... |
143 |
inet_bind_bucket_for_each(tb, &head->chain) { |
093d28232 tproxy: fix hash ... |
144 145 146 147 |
if (net_eq(ib_net(tb), sock_net(sk)) && tb->port == port) break; } |
b67bfe0d4 hlist: drop the n... |
148 |
if (!tb) { |
093d28232 tproxy: fix hash ... |
149 150 151 152 153 154 155 156 |
tb = inet_bind_bucket_create(table->bind_bucket_cachep, sock_net(sk), head, port); if (!tb) { spin_unlock(&head->lock); return -ENOMEM; } } } |
b4ff3c90e inet: Fix __inet_... |
157 |
inet_bind_hash(child, tb, port); |
53083773d [INET]: Uninline ... |
158 |
spin_unlock(&head->lock); |
093d28232 tproxy: fix hash ... |
159 160 |
return 0; |
53083773d [INET]: Uninline ... |
161 |
} |
53083773d [INET]: Uninline ... |
162 |
EXPORT_SYMBOL_GPL(__inet_inherit_port); |
c25eb3bfb net: Convert TCP/... |
163 164 |
static inline int compute_score(struct sock *sk, struct net *net, const unsigned short hnum, const __be32 daddr, |
3fa6f616a net: ipv4: add se... |
165 |
const int dif, const int sdif, bool exact_dif) |
c25eb3bfb net: Convert TCP/... |
166 167 168 |
{ int score = -1; struct inet_sock *inet = inet_sk(sk); |
c720c7e83 inet: rename some... |
169 |
if (net_eq(sock_net(sk), net) && inet->inet_num == hnum && |
c25eb3bfb net: Convert TCP/... |
170 |
!ipv6_only_sock(sk)) { |
c720c7e83 inet: rename some... |
171 |
__be32 rcv_saddr = inet->inet_rcv_saddr; |
da5e36308 soreuseport: TCP/... |
172 |
score = sk->sk_family == PF_INET ? 2 : 1; |
c25eb3bfb net: Convert TCP/... |
173 174 175 |
if (rcv_saddr) { if (rcv_saddr != daddr) return -1; |
da5e36308 soreuseport: TCP/... |
176 |
score += 4; |
c25eb3bfb net: Convert TCP/... |
177 |
} |
a04a480d4 net: Require exac... |
178 |
if (sk->sk_bound_dev_if || exact_dif) { |
3fa6f616a net: ipv4: add se... |
179 180 |
bool dev_match = (sk->sk_bound_dev_if == dif || sk->sk_bound_dev_if == sdif); |
35e324ebe net/tcp: Fix sock... |
181 |
if (!dev_match) |
c25eb3bfb net: Convert TCP/... |
182 |
return -1; |
35e324ebe net/tcp: Fix sock... |
183 |
if (sk->sk_bound_dev_if) |
3fa6f616a net: ipv4: add se... |
184 |
score += 4; |
c25eb3bfb net: Convert TCP/... |
185 |
} |
70da268b5 net: SO_INCOMING_... |
186 187 |
if (sk->sk_incoming_cpu == raw_smp_processor_id()) score++; |
c25eb3bfb net: Convert TCP/... |
188 189 190 |
} return score; } |
f3f05f704 [INET]: Generalis... |
191 |
/* |
3b24d854c tcp/dccp: do not ... |
192 193 |
* Here are some nice properties to exploit here. The BSD API * does not allow a listening sock to specify the remote port nor the |
33b622319 [INET]: Generalis... |
194 195 196 |
* remote address for the connection. So always assume those are both * wildcarded during the search since they can never be otherwise. */ |
e48c414ee [INET]: Generalis... |
197 |
|
3b24d854c tcp/dccp: do not ... |
198 |
/* called with rcu_read_lock() : No refcount taken on the socket */ |
c67499c0e [NETNS]: Tcp-v4 s... |
199 200 |
struct sock *__inet_lookup_listener(struct net *net, struct inet_hashinfo *hashinfo, |
a583636a8 inet: refactor in... |
201 |
struct sk_buff *skb, int doff, |
da5e36308 soreuseport: TCP/... |
202 |
const __be32 saddr, __be16 sport, |
fb99c848e [IPV4]: annotate ... |
203 |
const __be32 daddr, const unsigned short hnum, |
3fa6f616a net: ipv4: add se... |
204 |
const int dif, const int sdif) |
99a92ff50 [IPV4]: Uninline ... |
205 |
{ |
c25eb3bfb net: Convert TCP/... |
206 207 |
unsigned int hash = inet_lhashfn(net, hnum); struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; |
3b24d854c tcp/dccp: do not ... |
208 |
int score, hiscore = 0, matches = 0, reuseport = 0; |
a04a480d4 net: Require exac... |
209 |
bool exact_dif = inet_exact_dif_match(net, skb); |
3b24d854c tcp/dccp: do not ... |
210 |
struct sock *sk, *result = NULL; |
da5e36308 soreuseport: TCP/... |
211 |
u32 phash = 0; |
99a92ff50 [IPV4]: Uninline ... |
212 |
|
3b24d854c tcp/dccp: do not ... |
213 |
sk_for_each_rcu(sk, &ilb->head) { |
3fa6f616a net: ipv4: add se... |
214 215 |
score = compute_score(sk, net, hnum, daddr, dif, sdif, exact_dif); |
c25eb3bfb net: Convert TCP/... |
216 |
if (score > hiscore) { |
da5e36308 soreuseport: TCP/... |
217 218 219 220 |
reuseport = sk->sk_reuseport; if (reuseport) { phash = inet_ehashfn(net, daddr, hnum, saddr, sport); |
3b24d854c tcp/dccp: do not ... |
221 222 223 224 |
result = reuseport_select_sock(sk, phash, skb, doff); if (result) return result; |
da5e36308 soreuseport: TCP/... |
225 226 |
matches = 1; } |
3b24d854c tcp/dccp: do not ... |
227 228 |
result = sk; hiscore = score; |
da5e36308 soreuseport: TCP/... |
229 230 |
} else if (score == hiscore && reuseport) { matches++; |
8fc54f689 net: use reciproc... |
231 |
if (reciprocal_scale(phash, matches) == 0) |
da5e36308 soreuseport: TCP/... |
232 233 |
result = sk; phash = next_pseudo_random32(phash); |
c25eb3bfb net: Convert TCP/... |
234 |
} |
99a92ff50 [IPV4]: Uninline ... |
235 |
} |
c25eb3bfb net: Convert TCP/... |
236 |
return result; |
99a92ff50 [IPV4]: Uninline ... |
237 |
} |
8f491069b [IPV4]: Use netwo... |
238 |
EXPORT_SYMBOL_GPL(__inet_lookup_listener); |
a7f5e7f16 [INET]: Generalis... |
239 |
|
05dbc7b59 tcp/dccp: remove ... |
240 241 242 |
/* All sockets share common refcount, but have different destructors */ void sock_gen_put(struct sock *sk) { |
41c6d650f net: convert sock... |
243 |
if (!refcount_dec_and_test(&sk->sk_refcnt)) |
05dbc7b59 tcp/dccp: remove ... |
244 245 246 247 |
return; if (sk->sk_state == TCP_TIME_WAIT) inet_twsk_free(inet_twsk(sk)); |
41b822c59 inet: prepare soc... |
248 249 |
else if (sk->sk_state == TCP_NEW_SYN_RECV) reqsk_free(inet_reqsk(sk)); |
05dbc7b59 tcp/dccp: remove ... |
250 251 252 253 |
else sk_free(sk); } EXPORT_SYMBOL_GPL(sock_gen_put); |
2c13270b4 inet: factorize s... |
254 255 256 257 258 |
void sock_edemux(struct sk_buff *skb) { sock_gen_put(skb->sk); } EXPORT_SYMBOL(sock_edemux); |
5e73ea1a3 ipv4: fix checkpa... |
259 |
struct sock *__inet_lookup_established(struct net *net, |
c67499c0e [NETNS]: Tcp-v4 s... |
260 |
struct inet_hashinfo *hashinfo, |
77a5ba55d [INET]: Uninline ... |
261 262 |
const __be32 saddr, const __be16 sport, const __be32 daddr, const u16 hnum, |
3fa6f616a net: ipv4: add se... |
263 |
const int dif, const int sdif) |
77a5ba55d [INET]: Uninline ... |
264 |
{ |
c72283174 net: Use a more s... |
265 |
INET_ADDR_COOKIE(acookie, saddr, daddr); |
77a5ba55d [INET]: Uninline ... |
266 267 |
const __portpair ports = INET_COMBINED_PORTS(sport, hnum); struct sock *sk; |
3ab5aee7f net: Convert TCP ... |
268 |
const struct hlist_nulls_node *node; |
77a5ba55d [INET]: Uninline ... |
269 270 271 |
/* Optimize here for direct hit, only listening connections can * have wildcards anyways. */ |
9f26b3add inet: add struct ... |
272 |
unsigned int hash = inet_ehashfn(net, daddr, hnum, saddr, sport); |
f373b53b5 tcp: replace ehas... |
273 |
unsigned int slot = hash & hashinfo->ehash_mask; |
3ab5aee7f net: Convert TCP ... |
274 |
struct inet_ehash_bucket *head = &hashinfo->ehash[slot]; |
77a5ba55d [INET]: Uninline ... |
275 |
|
3ab5aee7f net: Convert TCP ... |
276 277 |
begin: sk_nulls_for_each_rcu(sk, node, &head->chain) { |
ce43b03e8 net: move inet_dp... |
278 279 280 |
if (sk->sk_hash != hash) continue; if (likely(INET_MATCH(sk, net, acookie, |
3fa6f616a net: ipv4: add se... |
281 |
saddr, daddr, ports, dif, sdif))) { |
41c6d650f net: convert sock... |
282 |
if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) |
05dbc7b59 tcp/dccp: remove ... |
283 |
goto out; |
ce43b03e8 net: move inet_dp... |
284 |
if (unlikely(!INET_MATCH(sk, net, acookie, |
3fa6f616a net: ipv4: add se... |
285 286 |
saddr, daddr, ports, dif, sdif))) { |
05dbc7b59 tcp/dccp: remove ... |
287 |
sock_gen_put(sk); |
3ab5aee7f net: Convert TCP ... |
288 289 |
goto begin; } |
05dbc7b59 tcp/dccp: remove ... |
290 |
goto found; |
3ab5aee7f net: Convert TCP ... |
291 |
} |
77a5ba55d [INET]: Uninline ... |
292 |
} |
3ab5aee7f net: Convert TCP ... |
293 294 295 296 297 298 299 |
/* * if the nulls value we got at the end of this lookup is * not the expected one, we must restart lookup. * We probably met an item that was moved to another chain. */ if (get_nulls_value(node) != slot) goto begin; |
77a5ba55d [INET]: Uninline ... |
300 |
out: |
05dbc7b59 tcp/dccp: remove ... |
301 302 |
sk = NULL; found: |
77a5ba55d [INET]: Uninline ... |
303 |
return sk; |
77a5ba55d [INET]: Uninline ... |
304 305 |
} EXPORT_SYMBOL_GPL(__inet_lookup_established); |
a7f5e7f16 [INET]: Generalis... |
306 307 308 309 310 311 312 |
/* called with local bh disabled */ static int __inet_check_established(struct inet_timewait_death_row *death_row, struct sock *sk, __u16 lport, struct inet_timewait_sock **twp) { struct inet_hashinfo *hinfo = death_row->hashinfo; struct inet_sock *inet = inet_sk(sk); |
c720c7e83 inet: rename some... |
313 314 |
__be32 daddr = inet->inet_rcv_saddr; __be32 saddr = inet->inet_daddr; |
a7f5e7f16 [INET]: Generalis... |
315 |
int dif = sk->sk_bound_dev_if; |
3fa6f616a net: ipv4: add se... |
316 317 |
struct net *net = sock_net(sk); int sdif = l3mdev_master_ifindex_by_index(net, dif); |
c72283174 net: Use a more s... |
318 |
INET_ADDR_COOKIE(acookie, saddr, daddr); |
c720c7e83 inet: rename some... |
319 |
const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport); |
c720c7e83 inet: rename some... |
320 321 |
unsigned int hash = inet_ehashfn(net, daddr, lport, saddr, inet->inet_dport); |
a7f5e7f16 [INET]: Generalis... |
322 |
struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash); |
9db66bdcc net: convert TCP/... |
323 |
spinlock_t *lock = inet_ehash_lockp(hinfo, hash); |
a7f5e7f16 [INET]: Generalis... |
324 |
struct sock *sk2; |
3ab5aee7f net: Convert TCP ... |
325 |
const struct hlist_nulls_node *node; |
05dbc7b59 tcp/dccp: remove ... |
326 |
struct inet_timewait_sock *tw = NULL; |
a7f5e7f16 [INET]: Generalis... |
327 |
|
9db66bdcc net: convert TCP/... |
328 |
spin_lock(lock); |
a7f5e7f16 [INET]: Generalis... |
329 |
|
3ab5aee7f net: Convert TCP ... |
330 |
sk_nulls_for_each(sk2, node, &head->chain) { |
ce43b03e8 net: move inet_dp... |
331 332 |
if (sk2->sk_hash != hash) continue; |
05dbc7b59 tcp/dccp: remove ... |
333 |
|
ce43b03e8 net: move inet_dp... |
334 |
if (likely(INET_MATCH(sk2, net, acookie, |
3fa6f616a net: ipv4: add se... |
335 |
saddr, daddr, ports, dif, sdif))) { |
05dbc7b59 tcp/dccp: remove ... |
336 337 338 339 340 |
if (sk2->sk_state == TCP_TIME_WAIT) { tw = inet_twsk(sk2); if (twsk_unique(sk, sk2, twp)) break; } |
a7f5e7f16 [INET]: Generalis... |
341 |
goto not_unique; |
05dbc7b59 tcp/dccp: remove ... |
342 |
} |
a7f5e7f16 [INET]: Generalis... |
343 |
} |
a7f5e7f16 [INET]: Generalis... |
344 |
/* Must record num and sport now. Otherwise we will see |
05dbc7b59 tcp/dccp: remove ... |
345 346 |
* in hash table socket with a funny identity. */ |
c720c7e83 inet: rename some... |
347 348 |
inet->inet_num = lport; inet->inet_sport = htons(lport); |
a7f5e7f16 [INET]: Generalis... |
349 |
sk->sk_hash = hash; |
547b792ca net: convert BUG_... |
350 |
WARN_ON(!sk_unhashed(sk)); |
3ab5aee7f net: Convert TCP ... |
351 |
__sk_nulls_add_node_rcu(sk, &head->chain); |
13475a30b tcp: connect() ra... |
352 |
if (tw) { |
fc01538f9 inet: simplify ti... |
353 |
sk_nulls_del_node_init_rcu((struct sock *)tw); |
02a1d6e7a net: rename NET_{... |
354 |
__NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED); |
13475a30b tcp: connect() ra... |
355 |
} |
9db66bdcc net: convert TCP/... |
356 |
spin_unlock(lock); |
c29a0bc4d [SOCK][NETNS]: Ad... |
357 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); |
a7f5e7f16 [INET]: Generalis... |
358 359 360 |
if (twp) { *twp = tw; |
a7f5e7f16 [INET]: Generalis... |
361 362 |
} else if (tw) { /* Silly. Should hash-dance instead... */ |
dbe7faa40 inet: inet_twsk_d... |
363 |
inet_twsk_deschedule_put(tw); |
a7f5e7f16 [INET]: Generalis... |
364 |
} |
a7f5e7f16 [INET]: Generalis... |
365 366 367 |
return 0; not_unique: |
9db66bdcc net: convert TCP/... |
368 |
spin_unlock(lock); |
a7f5e7f16 [INET]: Generalis... |
369 370 |
return -EADDRNOTAVAIL; } |
e2baad9e4 tcp: connect() fr... |
371 |
static u32 inet_sk_port_offset(const struct sock *sk) |
a7f5e7f16 [INET]: Generalis... |
372 373 |
{ const struct inet_sock *inet = inet_sk(sk); |
e2baad9e4 tcp: connect() fr... |
374 |
|
c720c7e83 inet: rename some... |
375 376 377 |
return secure_ipv4_port_ephemeral(inet->inet_rcv_saddr, inet->inet_daddr, inet->inet_dport); |
a7f5e7f16 [INET]: Generalis... |
378 |
} |
079096f10 tcp/dccp: install... |
379 380 381 |
/* insert a socket into ehash, and eventually remove another one * (The another one can be a SYN_RECV or TIMEWAIT */ |
5e0724d02 tcp/dccp: fix has... |
382 |
bool inet_ehash_insert(struct sock *sk, struct sock *osk) |
152da81de [INET]: Uninline ... |
383 |
{ |
39d8cda76 [SOCK]: Add udp_h... |
384 |
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; |
3ab5aee7f net: Convert TCP ... |
385 |
struct hlist_nulls_head *list; |
152da81de [INET]: Uninline ... |
386 |
struct inet_ehash_bucket *head; |
5b441f76f net: introduce sk... |
387 |
spinlock_t *lock; |
5e0724d02 tcp/dccp: fix has... |
388 |
bool ret = true; |
152da81de [INET]: Uninline ... |
389 |
|
079096f10 tcp/dccp: install... |
390 |
WARN_ON_ONCE(!sk_unhashed(sk)); |
152da81de [INET]: Uninline ... |
391 |
|
5b441f76f net: introduce sk... |
392 |
sk->sk_hash = sk_ehashfn(sk); |
152da81de [INET]: Uninline ... |
393 394 395 |
head = inet_ehash_bucket(hashinfo, sk->sk_hash); list = &head->chain; lock = inet_ehash_lockp(hashinfo, sk->sk_hash); |
9db66bdcc net: convert TCP/... |
396 |
spin_lock(lock); |
fc01538f9 inet: simplify ti... |
397 |
if (osk) { |
5e0724d02 tcp/dccp: fix has... |
398 399 |
WARN_ON_ONCE(sk->sk_hash != osk->sk_hash); ret = sk_nulls_del_node_init_rcu(osk); |
9327f7053 tcp: Fix a connec... |
400 |
} |
5e0724d02 tcp/dccp: fix has... |
401 402 |
if (ret) __sk_nulls_add_node_rcu(sk, list); |
9db66bdcc net: convert TCP/... |
403 |
spin_unlock(lock); |
079096f10 tcp/dccp: install... |
404 405 |
return ret; } |
5e0724d02 tcp/dccp: fix has... |
406 |
bool inet_ehash_nolisten(struct sock *sk, struct sock *osk) |
079096f10 tcp/dccp: install... |
407 |
{ |
5e0724d02 tcp/dccp: fix has... |
408 409 410 411 412 413 414 415 416 417 418 |
bool ok = inet_ehash_insert(sk, osk); if (ok) { sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); } else { percpu_counter_inc(sk->sk_prot->orphan_count); sk->sk_state = TCP_CLOSE; sock_set_flag(sk, SOCK_DEAD); inet_csk_destroy_sock(sk); } return ok; |
152da81de [INET]: Uninline ... |
419 |
} |
5e0724d02 tcp/dccp: fix has... |
420 |
EXPORT_SYMBOL_GPL(inet_ehash_nolisten); |
152da81de [INET]: Uninline ... |
421 |
|
c125e80b8 soreuseport: fast... |
422 |
static int inet_reuseport_add_sock(struct sock *sk, |
fe38d2a1c inet: collapse ip... |
423 |
struct inet_listen_hashbucket *ilb) |
c125e80b8 soreuseport: fast... |
424 |
{ |
90e5d0db2 soreuseport: Fix ... |
425 |
struct inet_bind_bucket *tb = inet_csk(sk)->icsk_bind_hash; |
c125e80b8 soreuseport: fast... |
426 |
struct sock *sk2; |
c125e80b8 soreuseport: fast... |
427 |
kuid_t uid = sock_i_uid(sk); |
850178692 tcp/dccp: fix ine... |
428 |
sk_for_each_rcu(sk2, &ilb->head) { |
c125e80b8 soreuseport: fast... |
429 430 431 432 |
if (sk2 != sk && sk2->sk_family == sk->sk_family && ipv6_only_sock(sk2) == ipv6_only_sock(sk) && sk2->sk_bound_dev_if == sk->sk_bound_dev_if && |
90e5d0db2 soreuseport: Fix ... |
433 |
inet_csk(sk2)->icsk_bind_hash == tb && |
c125e80b8 soreuseport: fast... |
434 |
sk2->sk_reuseport && uid_eq(uid, sock_i_uid(sk2)) && |
fe38d2a1c inet: collapse ip... |
435 |
inet_rcv_saddr_equal(sk, sk2, false)) |
c125e80b8 soreuseport: fast... |
436 437 |
return reuseport_add_sock(sk, sk2); } |
1b5f962e7 soreuseport: fix ... |
438 |
return reuseport_alloc(sk); |
c125e80b8 soreuseport: fast... |
439 |
} |
fe38d2a1c inet: collapse ip... |
440 |
int __inet_hash(struct sock *sk, struct sock *osk) |
152da81de [INET]: Uninline ... |
441 |
{ |
39d8cda76 [SOCK]: Add udp_h... |
442 |
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; |
5caea4ea7 net: listening_ha... |
443 |
struct inet_listen_hashbucket *ilb; |
c125e80b8 soreuseport: fast... |
444 |
int err = 0; |
152da81de [INET]: Uninline ... |
445 |
|
5e0724d02 tcp/dccp: fix has... |
446 447 |
if (sk->sk_state != TCP_LISTEN) { inet_ehash_nolisten(sk, osk); |
c125e80b8 soreuseport: fast... |
448 |
return 0; |
5e0724d02 tcp/dccp: fix has... |
449 |
} |
547b792ca net: convert BUG_... |
450 |
WARN_ON(!sk_unhashed(sk)); |
5caea4ea7 net: listening_ha... |
451 |
ilb = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)]; |
152da81de [INET]: Uninline ... |
452 |
|
5caea4ea7 net: listening_ha... |
453 |
spin_lock(&ilb->lock); |
c125e80b8 soreuseport: fast... |
454 |
if (sk->sk_reuseport) { |
fe38d2a1c inet: collapse ip... |
455 |
err = inet_reuseport_add_sock(sk, ilb); |
c125e80b8 soreuseport: fast... |
456 457 458 |
if (err) goto unlock; } |
d296ba60d soreuseport: Reso... |
459 460 461 462 463 |
if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport && sk->sk_family == AF_INET6) hlist_add_tail_rcu(&sk->sk_node, &ilb->head); else hlist_add_head_rcu(&sk->sk_node, &ilb->head); |
3b24d854c tcp/dccp: do not ... |
464 |
sock_set_flag(sk, SOCK_RCU_FREE); |
c29a0bc4d [SOCK][NETNS]: Ad... |
465 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); |
c125e80b8 soreuseport: fast... |
466 |
unlock: |
5caea4ea7 net: listening_ha... |
467 |
spin_unlock(&ilb->lock); |
c125e80b8 soreuseport: fast... |
468 469 |
return err; |
152da81de [INET]: Uninline ... |
470 |
} |
77a6a471b ipv6: get rid of ... |
471 |
EXPORT_SYMBOL(__inet_hash); |
ab1e0a13d [SOCK] proto: Add... |
472 |
|
086c653f5 sock: struct prot... |
473 |
int inet_hash(struct sock *sk) |
ab1e0a13d [SOCK] proto: Add... |
474 |
{ |
c125e80b8 soreuseport: fast... |
475 |
int err = 0; |
ab1e0a13d [SOCK] proto: Add... |
476 477 |
if (sk->sk_state != TCP_CLOSE) { local_bh_disable(); |
fe38d2a1c inet: collapse ip... |
478 |
err = __inet_hash(sk, NULL); |
ab1e0a13d [SOCK] proto: Add... |
479 480 |
local_bh_enable(); } |
086c653f5 sock: struct prot... |
481 |
|
c125e80b8 soreuseport: fast... |
482 |
return err; |
ab1e0a13d [SOCK] proto: Add... |
483 484 485 486 487 |
} EXPORT_SYMBOL_GPL(inet_hash); void inet_unhash(struct sock *sk) { |
39d8cda76 [SOCK]: Add udp_h... |
488 |
struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo; |
c25eb3bfb net: Convert TCP/... |
489 |
spinlock_t *lock; |
3b24d854c tcp/dccp: do not ... |
490 |
bool listener = false; |
c25eb3bfb net: Convert TCP/... |
491 |
int done; |
ab1e0a13d [SOCK] proto: Add... |
492 493 |
if (sk_unhashed(sk)) |
5caea4ea7 net: listening_ha... |
494 |
return; |
ab1e0a13d [SOCK] proto: Add... |
495 |
|
3b24d854c tcp/dccp: do not ... |
496 |
if (sk->sk_state == TCP_LISTEN) { |
c25eb3bfb net: Convert TCP/... |
497 |
lock = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)].lock; |
3b24d854c tcp/dccp: do not ... |
498 499 |
listener = true; } else { |
c25eb3bfb net: Convert TCP/... |
500 |
lock = inet_ehash_lockp(hashinfo, sk->sk_hash); |
3b24d854c tcp/dccp: do not ... |
501 |
} |
c25eb3bfb net: Convert TCP/... |
502 |
spin_lock_bh(lock); |
c125e80b8 soreuseport: fast... |
503 504 |
if (rcu_access_pointer(sk->sk_reuseport_cb)) reuseport_detach_sock(sk); |
3b24d854c tcp/dccp: do not ... |
505 506 507 508 |
if (listener) done = __sk_del_node_init(sk); else done = __sk_nulls_del_node_init_rcu(sk); |
c25eb3bfb net: Convert TCP/... |
509 510 |
if (done) sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); |
920de804b net: Make sure BH... |
511 |
spin_unlock_bh(lock); |
ab1e0a13d [SOCK] proto: Add... |
512 513 |
} EXPORT_SYMBOL_GPL(inet_unhash); |
152da81de [INET]: Uninline ... |
514 |
|
5ee31fc1e [INET]: Consolida... |
515 |
int __inet_hash_connect(struct inet_timewait_death_row *death_row, |
5d8c0aa94 [INET]: Fix accid... |
516 |
struct sock *sk, u32 port_offset, |
5ee31fc1e [INET]: Consolida... |
517 |
int (*check_established)(struct inet_timewait_death_row *, |
b4d6444ea inet: get rid of ... |
518 |
struct sock *, __u16, struct inet_timewait_sock **)) |
a7f5e7f16 [INET]: Generalis... |
519 520 |
{ struct inet_hashinfo *hinfo = death_row->hashinfo; |
1580ab63f tcp/dccp: better ... |
521 |
struct inet_timewait_sock *tw = NULL; |
e905a9eda [NET] IPV4: Fix w... |
522 |
struct inet_bind_hashbucket *head; |
1580ab63f tcp/dccp: better ... |
523 |
int port = inet_sk(sk)->inet_num; |
3b1e0a655 [NET] NETNS: Omit... |
524 |
struct net *net = sock_net(sk); |
1580ab63f tcp/dccp: better ... |
525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 |
struct inet_bind_bucket *tb; u32 remaining, offset; int ret, i, low, high; static u32 hint; if (port) { head = &hinfo->bhash[inet_bhashfn(net, port, hinfo->bhash_size)]; tb = inet_csk(sk)->icsk_bind_hash; spin_lock_bh(&head->lock); if (sk_head(&tb->owners) == sk && !sk->sk_bind_node.next) { inet_ehash_nolisten(sk, NULL); spin_unlock_bh(&head->lock); return 0; } spin_unlock(&head->lock); /* No definite answer... Walk to established hash table */ ret = check_established(death_row, sk, port, NULL); local_bh_enable(); return ret; } |
a7f5e7f16 [INET]: Generalis... |
546 |
|
1580ab63f tcp/dccp: better ... |
547 548 549 550 551 |
inet_get_local_port_range(net, &low, &high); high++; /* [32768, 60999] -> [32768, 61000[ */ remaining = high - low; if (likely(remaining > 1)) remaining &= ~1U; |
a7f5e7f16 [INET]: Generalis... |
552 |
|
1580ab63f tcp/dccp: better ... |
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 |
offset = (hint + port_offset) % remaining; /* In first pass we try ports of @low parity. * inet_csk_get_port() does the opposite choice. */ offset &= ~1U; other_parity_scan: port = low + offset; for (i = 0; i < remaining; i += 2, port += 2) { if (unlikely(port >= high)) port -= remaining; if (inet_is_local_reserved_port(net, port)) continue; head = &hinfo->bhash[inet_bhashfn(net, port, hinfo->bhash_size)]; spin_lock_bh(&head->lock); |
227b60f51 [INET]: local por... |
568 |
|
1580ab63f tcp/dccp: better ... |
569 570 |
/* Does not bother with rcv_saddr checks, because * the established check is already unique enough. |
07f4c9006 tcp/dccp: try to ... |
571 |
*/ |
1580ab63f tcp/dccp: better ... |
572 573 574 575 |
inet_bind_bucket_for_each(tb, &head->chain) { if (net_eq(ib_net(tb), net) && tb->port == port) { if (tb->fastreuse >= 0 || tb->fastreuseport >= 0) |
e905a9eda [NET] IPV4: Fix w... |
576 |
goto next_port; |
1580ab63f tcp/dccp: better ... |
577 578 579 580 581 |
WARN_ON(hlist_empty(&tb->owners)); if (!check_established(death_row, sk, port, &tw)) goto ok; goto next_port; |
e905a9eda [NET] IPV4: Fix w... |
582 |
} |
e905a9eda [NET] IPV4: Fix w... |
583 |
} |
a7f5e7f16 [INET]: Generalis... |
584 |
|
1580ab63f tcp/dccp: better ... |
585 586 587 588 589 |
tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, net, head, port); if (!tb) { spin_unlock_bh(&head->lock); return -ENOMEM; |
e905a9eda [NET] IPV4: Fix w... |
590 |
} |
1580ab63f tcp/dccp: better ... |
591 592 593 594 595 596 597 |
tb->fastreuse = -1; tb->fastreuseport = -1; goto ok; next_port: spin_unlock_bh(&head->lock); cond_resched(); } |
a7f5e7f16 [INET]: Generalis... |
598 |
|
1580ab63f tcp/dccp: better ... |
599 600 601 |
offset++; if ((offset & 1) && remaining > 1) goto other_parity_scan; |
a7f5e7f16 [INET]: Generalis... |
602 |
|
1580ab63f tcp/dccp: better ... |
603 |
return -EADDRNOTAVAIL; |
a7f5e7f16 [INET]: Generalis... |
604 |
|
1580ab63f tcp/dccp: better ... |
605 606 607 608 609 610 611 612 |
ok: hint += i + 2; /* Head lock still held and bh's disabled */ inet_bind_hash(sk, tb, port); if (sk_unhashed(sk)) { inet_sk(sk)->inet_sport = htons(port); inet_ehash_nolisten(sk, (struct sock *)tw); |
a7f5e7f16 [INET]: Generalis... |
613 |
} |
1580ab63f tcp/dccp: better ... |
614 615 616 617 618 619 620 |
if (tw) inet_twsk_bind_unhash(tw, hinfo); spin_unlock(&head->lock); if (tw) inet_twsk_deschedule_put(tw); local_bh_enable(); return 0; |
a7f5e7f16 [INET]: Generalis... |
621 |
} |
5ee31fc1e [INET]: Consolida... |
622 623 624 625 626 627 628 |
/* * Bind a port for a connect operation and hash it. */ int inet_hash_connect(struct inet_timewait_death_row *death_row, struct sock *sk) { |
e2baad9e4 tcp: connect() fr... |
629 630 631 632 633 |
u32 port_offset = 0; if (!inet_sk(sk)->inet_num) port_offset = inet_sk_port_offset(sk); return __inet_hash_connect(death_row, sk, port_offset, |
b4d6444ea inet: get rid of ... |
634 |
__inet_check_established); |
5ee31fc1e [INET]: Consolida... |
635 |
} |
a7f5e7f16 [INET]: Generalis... |
636 |
EXPORT_SYMBOL_GPL(inet_hash_connect); |
5caea4ea7 net: listening_ha... |
637 638 639 640 |
void inet_hashinfo_init(struct inet_hashinfo *h) { int i; |
c25eb3bfb net: Convert TCP/... |
641 |
for (i = 0; i < INET_LHTABLE_SIZE; i++) { |
5caea4ea7 net: listening_ha... |
642 |
spin_lock_init(&h->listening_hash[i].lock); |
3b24d854c tcp/dccp: do not ... |
643 644 |
INIT_HLIST_HEAD(&h->listening_hash[i].head); } |
5caea4ea7 net: listening_ha... |
645 |
} |
5caea4ea7 net: listening_ha... |
646 |
EXPORT_SYMBOL_GPL(inet_hashinfo_init); |
095dc8e0c tcp: fix/cleanup ... |
647 648 649 |
int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo) { |
89e478a2a tcp: suppress a d... |
650 |
unsigned int locksz = sizeof(spinlock_t); |
095dc8e0c tcp: fix/cleanup ... |
651 |
unsigned int i, nblocks = 1; |
89e478a2a tcp: suppress a d... |
652 |
if (locksz != 0) { |
095dc8e0c tcp: fix/cleanup ... |
653 |
/* allocate 2 cache lines or at least one spinlock per cpu */ |
89e478a2a tcp: suppress a d... |
654 |
nblocks = max(2U * L1_CACHE_BYTES / locksz, 1U); |
095dc8e0c tcp: fix/cleanup ... |
655 656 657 658 |
nblocks = roundup_pow_of_two(nblocks * num_possible_cpus()); /* no more locks than number of hash buckets */ nblocks = min(nblocks, hashinfo->ehash_mask + 1); |
752ade68c treewide: use kv[... |
659 |
hashinfo->ehash_locks = kvmalloc_array(nblocks, locksz, GFP_KERNEL); |
095dc8e0c tcp: fix/cleanup ... |
660 661 662 663 664 665 666 667 668 669 |
if (!hashinfo->ehash_locks) return -ENOMEM; for (i = 0; i < nblocks; i++) spin_lock_init(&hashinfo->ehash_locks[i]); } hashinfo->ehash_locks_mask = nblocks - 1; return 0; } EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc); |