Blame view
net/ipv6/inet6_hashtables.c
9.17 KB
2874c5fd2 treewide: Replace... |
1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
5324a040c [INET6_HASHTABLES... |
2 3 4 5 6 7 8 |
/* * INET An implementation of the TCP/IP protocol suite for the LINUX * operating system. INET is implemented using the BSD Socket * interface as the means of communication with the user level. * * Generic INET6 transport hashtables * |
d8313f5ca [INET6]: Generali... |
9 |
* Authors: Lotsa people, from code originally in tcp, generalised here |
67ba4152e ipv6: White-space... |
10 |
* by Arnaldo Carvalho de Melo <acme@mandriva.com> |
5324a040c [INET6_HASHTABLES... |
11 |
*/ |
5324a040c [INET6_HASHTABLES... |
12 |
#include <linux/module.h> |
d8313f5ca [INET6]: Generali... |
13 |
#include <linux/random.h> |
5324a040c [INET6_HASHTABLES... |
14 |
|
c125e80b8 soreuseport: fast... |
15 |
#include <net/addrconf.h> |
5324a040c [INET6_HASHTABLES... |
16 17 18 |
#include <net/inet_connection_sock.h> #include <net/inet_hashtables.h> #include <net/inet6_hashtables.h> |
6e5714eaf net: Compute prot... |
19 |
#include <net/secure_seq.h> |
d8313f5ca [INET6]: Generali... |
20 |
#include <net/ip.h> |
c125e80b8 soreuseport: fast... |
21 |
#include <net/sock_reuseport.h> |
5324a040c [INET6_HASHTABLES... |
22 |
|
1122702f0 inet6: Run SK_LOO... |
23 |
extern struct inet_hashinfo tcp_hashinfo; |
d1e559d0b inet: add IPv6 su... |
24 25 26 |
u32 inet6_ehashfn(const struct net *net, const struct in6_addr *laddr, const u16 lport, const struct in6_addr *faddr, const __be16 fport) |
b50026b5a ipv6: split inet6... |
27 |
{ |
1bbdceef1 inet: convert ine... |
28 29 30 31 32 33 34 35 36 37 |
static u32 inet6_ehash_secret __read_mostly; static u32 ipv6_hash_secret __read_mostly; u32 lhash, fhash; net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret)); net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret)); lhash = (__force u32)laddr->s6_addr32[3]; fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret); |
b50026b5a ipv6: split inet6... |
38 |
return __inet6_ehashfn(lhash, lport, fhash, fport, |
1bbdceef1 inet: convert ine... |
39 |
inet6_ehash_secret + net_hash_mix(net)); |
b50026b5a ipv6: split inet6... |
40 |
} |
b1a7ffcb7 [IPV6]: Deinline ... |
41 42 43 44 45 46 |
/* * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM * * The sockhash lock must be held as a reader here. */ |
d86e0dac2 [NETNS]: Tcp-v6 s... |
47 48 |
struct sock *__inet6_lookup_established(struct net *net, struct inet_hashinfo *hashinfo, |
b1a7ffcb7 [IPV6]: Deinline ... |
49 |
const struct in6_addr *saddr, |
d2ecd9ccd [IPV6]: annotate ... |
50 |
const __be16 sport, |
b1a7ffcb7 [IPV6]: Deinline ... |
51 52 |
const struct in6_addr *daddr, const u16 hnum, |
4297a0ef0 net: ipv6: add se... |
53 |
const int dif, const int sdif) |
b1a7ffcb7 [IPV6]: Deinline ... |
54 55 |
{ struct sock *sk; |
3ab5aee7f net: Convert TCP ... |
56 |
const struct hlist_nulls_node *node; |
4f765d842 [IPV4]: INET_MATC... |
57 |
const __portpair ports = INET_COMBINED_PORTS(sport, hnum); |
b1a7ffcb7 [IPV6]: Deinline ... |
58 59 60 |
/* Optimize here for direct hit, only listening connections can * have wildcards anyways. */ |
33de014c6 inet6: add struct... |
61 |
unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport); |
f373b53b5 tcp: replace ehas... |
62 |
unsigned int slot = hash & hashinfo->ehash_mask; |
3ab5aee7f net: Convert TCP ... |
63 |
struct inet_ehash_bucket *head = &hashinfo->ehash[slot]; |
b1a7ffcb7 [IPV6]: Deinline ... |
64 |
|
3ab5aee7f net: Convert TCP ... |
65 |
|
3ab5aee7f net: Convert TCP ... |
66 67 |
begin: sk_nulls_for_each_rcu(sk, node, &head->chain) { |
ce43b03e8 net: move inet_dp... |
68 69 |
if (sk->sk_hash != hash) continue; |
4297a0ef0 net: ipv6: add se... |
70 |
if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif)) |
efe4208f4 ipv6: make lookup... |
71 |
continue; |
41c6d650f net: convert sock... |
72 |
if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) |
05dbc7b59 tcp/dccp: remove ... |
73 |
goto out; |
4297a0ef0 net: ipv6: add se... |
74 |
if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))) { |
efe4208f4 ipv6: make lookup... |
75 76 |
sock_gen_put(sk); goto begin; |
3ab5aee7f net: Convert TCP ... |
77 |
} |
efe4208f4 ipv6: make lookup... |
78 |
goto found; |
b1a7ffcb7 [IPV6]: Deinline ... |
79 |
} |
3ab5aee7f net: Convert TCP ... |
80 81 |
if (get_nulls_value(node) != slot) goto begin; |
3ab5aee7f net: Convert TCP ... |
82 |
out: |
05dbc7b59 tcp/dccp: remove ... |
83 84 |
sk = NULL; found: |
b1a7ffcb7 [IPV6]: Deinline ... |
85 86 87 |
return sk; } EXPORT_SYMBOL(__inet6_lookup_established); |
42b16b3fb Kill off warning:... |
88 |
static inline int compute_score(struct sock *sk, struct net *net, |
c25eb3bfb net: Convert TCP/... |
89 90 |
const unsigned short hnum, const struct in6_addr *daddr, |
3f7d820ba net: ipv6: remove... |
91 |
const int dif, const int sdif) |
c25eb3bfb net: Convert TCP/... |
92 93 |
{ int score = -1; |
c720c7e83 inet: rename some... |
94 |
if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum && |
c25eb3bfb net: Convert TCP/... |
95 |
sk->sk_family == PF_INET6) { |
0ee58dad5 net: tcp6: prefer... |
96 97 |
if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return -1; |
c25eb3bfb net: Convert TCP/... |
98 |
|
0ee58dad5 net: tcp6: prefer... |
99 |
if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) |
e78190581 net: ensure unbou... |
100 |
return -1; |
4297a0ef0 net: ipv6: add se... |
101 |
|
0ee58dad5 net: tcp6: prefer... |
102 |
score = 1; |
7170a9777 net: annotate acc... |
103 |
if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) |
70da268b5 net: SO_INCOMING_... |
104 |
score++; |
c25eb3bfb net: Convert TCP/... |
105 106 107 |
} return score; } |
5df653129 inet6: Extract he... |
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 |
static inline struct sock *lookup_reuseport(struct net *net, struct sock *sk, struct sk_buff *skb, int doff, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, unsigned short hnum) { struct sock *reuse_sk = NULL; u32 phash; if (sk->sk_reuseport) { phash = inet6_ehashfn(net, daddr, hnum, saddr, sport); reuse_sk = reuseport_select_sock(sk, phash, skb, doff); } return reuse_sk; } |
3b24d854c tcp/dccp: do not ... |
124 |
/* called with rcu_read_lock() */ |
61b7c691c inet: Add a 2nd l... |
125 126 127 128 129 130 131 |
static struct sock *inet6_lhash2_lookup(struct net *net, struct inet_listen_hashbucket *ilb2, struct sk_buff *skb, int doff, const struct in6_addr *saddr, const __be16 sport, const struct in6_addr *daddr, const unsigned short hnum, const int dif, const int sdif) { |
61b7c691c inet: Add a 2nd l... |
132 133 134 |
struct inet_connection_sock *icsk; struct sock *sk, *result = NULL; int score, hiscore = 0; |
61b7c691c inet: Add a 2nd l... |
135 136 137 |
inet_lhash2_for_each_icsk_rcu(icsk, &ilb2->head) { sk = (struct sock *)icsk; |
3f7d820ba net: ipv6: remove... |
138 |
score = compute_score(sk, net, hnum, daddr, dif, sdif); |
61b7c691c inet: Add a 2nd l... |
139 |
if (score > hiscore) { |
5df653129 inet6: Extract he... |
140 141 142 143 |
result = lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum); if (result) return result; |
61b7c691c inet: Add a 2nd l... |
144 145 146 147 148 149 150 |
result = sk; hiscore = score; } } return result; } |
1122702f0 inet6: Run SK_LOO... |
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 |
static inline struct sock *inet6_lookup_run_bpf(struct net *net, struct inet_hashinfo *hashinfo, struct sk_buff *skb, int doff, const struct in6_addr *saddr, const __be16 sport, const struct in6_addr *daddr, const u16 hnum) { struct sock *sk, *reuse_sk; bool no_reuseport; if (hashinfo != &tcp_hashinfo) return NULL; /* only TCP is supported */ no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, saddr, sport, daddr, hnum, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; reuse_sk = lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum); if (reuse_sk) sk = reuse_sk; return sk; } |
d86e0dac2 [NETNS]: Tcp-v6 s... |
175 |
struct sock *inet6_lookup_listener(struct net *net, |
a583636a8 inet: refactor in... |
176 177 178 |
struct inet_hashinfo *hashinfo, struct sk_buff *skb, int doff, const struct in6_addr *saddr, |
5ba24953e soreuseport: TCP/... |
179 |
const __be16 sport, const struct in6_addr *daddr, |
4297a0ef0 net: ipv6: add se... |
180 |
const unsigned short hnum, const int dif, const int sdif) |
5324a040c [INET6_HASHTABLES... |
181 |
{ |
61b7c691c inet: Add a 2nd l... |
182 |
struct inet_listen_hashbucket *ilb2; |
0ee58dad5 net: tcp6: prefer... |
183 |
struct sock *result = NULL; |
61b7c691c inet: Add a 2nd l... |
184 |
unsigned int hash2; |
61b7c691c inet: Add a 2nd l... |
185 |
|
1122702f0 inet6: Run SK_LOO... |
186 187 188 189 190 191 192 |
/* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { result = inet6_lookup_run_bpf(net, hashinfo, skb, doff, saddr, sport, daddr, hnum); if (result) goto done; } |
61b7c691c inet: Add a 2nd l... |
193 194 |
hash2 = ipv6_portaddr_hash(net, daddr, hnum); ilb2 = inet_lhash2_bucket(hashinfo, hash2); |
61b7c691c inet: Add a 2nd l... |
195 196 197 198 199 |
result = inet6_lhash2_lookup(net, ilb2, skb, doff, saddr, sport, daddr, hnum, dif, sdif); if (result) |
8217ca653 bpf: Enable BPF_P... |
200 |
goto done; |
61b7c691c inet: Add a 2nd l... |
201 202 |
/* Lookup lhash2 with in6addr_any */ |
61b7c691c inet: Add a 2nd l... |
203 204 |
hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum); ilb2 = inet_lhash2_bucket(hashinfo, hash2); |
61b7c691c inet: Add a 2nd l... |
205 |
|
8217ca653 bpf: Enable BPF_P... |
206 |
result = inet6_lhash2_lookup(net, ilb2, skb, doff, |
0ee58dad5 net: tcp6: prefer... |
207 |
saddr, sport, &in6addr_any, hnum, |
8217ca653 bpf: Enable BPF_P... |
208 |
dif, sdif); |
8217ca653 bpf: Enable BPF_P... |
209 |
done: |
26f8113cc net: ipv6: drop u... |
210 |
if (IS_ERR(result)) |
8217ca653 bpf: Enable BPF_P... |
211 |
return NULL; |
5324a040c [INET6_HASHTABLES... |
212 213 |
return result; } |
5324a040c [INET6_HASHTABLES... |
214 |
EXPORT_SYMBOL_GPL(inet6_lookup_listener); |
d86e0dac2 [NETNS]: Tcp-v6 s... |
215 |
struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo, |
a583636a8 inet: refactor in... |
216 |
struct sk_buff *skb, int doff, |
d2ecd9ccd [IPV6]: annotate ... |
217 218 |
const struct in6_addr *saddr, const __be16 sport, const struct in6_addr *daddr, const __be16 dport, |
5324a040c [INET6_HASHTABLES... |
219 220 221 |
const int dif) { struct sock *sk; |
3b24d854c tcp/dccp: do not ... |
222 |
bool refcounted; |
5324a040c [INET6_HASHTABLES... |
223 |
|
a583636a8 inet: refactor in... |
224 |
sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr, |
4297a0ef0 net: ipv6: add se... |
225 |
ntohs(dport), dif, 0, &refcounted); |
41c6d650f net: convert sock... |
226 |
if (sk && !refcounted && !refcount_inc_not_zero(&sk->sk_refcnt)) |
3b24d854c tcp/dccp: do not ... |
227 |
sk = NULL; |
5324a040c [INET6_HASHTABLES... |
228 229 |
return sk; } |
5324a040c [INET6_HASHTABLES... |
230 |
EXPORT_SYMBOL_GPL(inet6_lookup); |
d8313f5ca [INET6]: Generali... |
231 232 233 234 235 236 |
static int __inet6_check_established(struct inet_timewait_death_row *death_row, struct sock *sk, const __u16 lport, struct inet_timewait_sock **twp) { struct inet_hashinfo *hinfo = death_row->hashinfo; |
3759fa9c5 [TCP]: Fix zero p... |
237 |
struct inet_sock *inet = inet_sk(sk); |
efe4208f4 ipv6: make lookup... |
238 239 |
const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr; const struct in6_addr *saddr = &sk->sk_v6_daddr; |
d8313f5ca [INET6]: Generali... |
240 |
const int dif = sk->sk_bound_dev_if; |
33de014c6 inet6: add struct... |
241 |
struct net *net = sock_net(sk); |
4297a0ef0 net: ipv6: add se... |
242 243 |
const int sdif = l3mdev_master_ifindex_by_index(net, dif); const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport); |
33de014c6 inet6: add struct... |
244 |
const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr, |
c720c7e83 inet: rename some... |
245 |
inet->inet_dport); |
d8313f5ca [INET6]: Generali... |
246 |
struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash); |
9db66bdcc net: convert TCP/... |
247 |
spinlock_t *lock = inet_ehash_lockp(hinfo, hash); |
d8313f5ca [INET6]: Generali... |
248 |
struct sock *sk2; |
3ab5aee7f net: Convert TCP ... |
249 |
const struct hlist_nulls_node *node; |
05dbc7b59 tcp/dccp: remove ... |
250 |
struct inet_timewait_sock *tw = NULL; |
d8313f5ca [INET6]: Generali... |
251 |
|
9db66bdcc net: convert TCP/... |
252 |
spin_lock(lock); |
d8313f5ca [INET6]: Generali... |
253 |
|
05dbc7b59 tcp/dccp: remove ... |
254 |
sk_nulls_for_each(sk2, node, &head->chain) { |
ce43b03e8 net: move inet_dp... |
255 256 |
if (sk2->sk_hash != hash) continue; |
d8313f5ca [INET6]: Generali... |
257 |
|
4297a0ef0 net: ipv6: add se... |
258 259 |
if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif, sdif))) { |
efe4208f4 ipv6: make lookup... |
260 |
if (sk2->sk_state == TCP_TIME_WAIT) { |
05dbc7b59 tcp/dccp: remove ... |
261 262 |
tw = inet_twsk(sk2); if (twsk_unique(sk, sk2, twp)) |
efe4208f4 ipv6: make lookup... |
263 |
break; |
05dbc7b59 tcp/dccp: remove ... |
264 |
} |
d8313f5ca [INET6]: Generali... |
265 |
goto not_unique; |
efe4208f4 ipv6: make lookup... |
266 |
} |
d8313f5ca [INET6]: Generali... |
267 |
} |
3759fa9c5 [TCP]: Fix zero p... |
268 |
/* Must record num and sport now. Otherwise we will see |
efe4208f4 ipv6: make lookup... |
269 270 |
* in hash table socket with a funny identity. */ |
c720c7e83 inet: rename some... |
271 272 |
inet->inet_num = lport; inet->inet_sport = htons(lport); |
13475a30b tcp: connect() ra... |
273 |
sk->sk_hash = hash; |
547b792ca net: convert BUG_... |
274 |
WARN_ON(!sk_unhashed(sk)); |
3ab5aee7f net: Convert TCP ... |
275 |
__sk_nulls_add_node_rcu(sk, &head->chain); |
13475a30b tcp: connect() ra... |
276 |
if (tw) { |
fc01538f9 inet: simplify ti... |
277 |
sk_nulls_del_node_init_rcu((struct sock *)tw); |
02a1d6e7a net: rename NET_{... |
278 |
__NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED); |
13475a30b tcp: connect() ra... |
279 |
} |
9db66bdcc net: convert TCP/... |
280 |
spin_unlock(lock); |
c29a0bc4d [SOCK][NETNS]: Ad... |
281 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); |
d8313f5ca [INET6]: Generali... |
282 |
|
13475a30b tcp: connect() ra... |
283 |
if (twp) { |
d8313f5ca [INET6]: Generali... |
284 |
*twp = tw; |
13475a30b tcp: connect() ra... |
285 |
} else if (tw) { |
d8313f5ca [INET6]: Generali... |
286 |
/* Silly. Should hash-dance instead... */ |
dbe7faa40 inet: inet_twsk_d... |
287 |
inet_twsk_deschedule_put(tw); |
d8313f5ca [INET6]: Generali... |
288 289 290 291 |
} return 0; not_unique: |
9db66bdcc net: convert TCP/... |
292 |
spin_unlock(lock); |
d8313f5ca [INET6]: Generali... |
293 294 |
return -EADDRNOTAVAIL; } |
e2baad9e4 tcp: connect() fr... |
295 |
static u32 inet6_sk_port_offset(const struct sock *sk) |
d8313f5ca [INET6]: Generali... |
296 297 |
{ const struct inet_sock *inet = inet_sk(sk); |
efe4208f4 ipv6: make lookup... |
298 299 300 |
return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32, sk->sk_v6_daddr.s6_addr32, |
c720c7e83 inet: rename some... |
301 |
inet->inet_dport); |
d8313f5ca [INET6]: Generali... |
302 303 304 305 306 |
} int inet6_hash_connect(struct inet_timewait_death_row *death_row, struct sock *sk) { |
e2baad9e4 tcp: connect() fr... |
307 308 309 310 311 |
u32 port_offset = 0; if (!inet_sk(sk)->inet_num) port_offset = inet6_sk_port_offset(sk); return __inet_hash_connect(death_row, sk, port_offset, |
b4d6444ea inet: get rid of ... |
312 |
__inet6_check_established); |
d8313f5ca [INET6]: Generali... |
313 |
} |
d8313f5ca [INET6]: Generali... |
314 |
EXPORT_SYMBOL_GPL(inet6_hash_connect); |
496611d7b inet: create IPv6... |
315 316 317 |
int inet6_hash(struct sock *sk) { |
e4cabca54 inet: Fix missing... |
318 |
int err = 0; |
496611d7b inet: create IPv6... |
319 320 |
if (sk->sk_state != TCP_CLOSE) { local_bh_disable(); |
fe38d2a1c inet: collapse ip... |
321 |
err = __inet_hash(sk, NULL); |
496611d7b inet: create IPv6... |
322 323 |
local_bh_enable(); } |
e4cabca54 inet: Fix missing... |
324 |
return err; |
496611d7b inet: create IPv6... |
325 326 |
} EXPORT_SYMBOL_GPL(inet6_hash); |