Blame view
net/ipv6/inet6_hashtables.c
9.26 KB
2874c5fd2
|
1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
5324a040c
|
2 3 4 5 6 7 8 |
/* * INET An implementation of the TCP/IP protocol suite for the LINUX * operating system. INET is implemented using the BSD Socket * interface as the means of communication with the user level. * * Generic INET6 transport hashtables * |
d8313f5ca
|
9 |
* Authors: Lotsa people, from code originally in tcp, generalised here |
67ba4152e
|
10 |
* by Arnaldo Carvalho de Melo <acme@mandriva.com> |
5324a040c
|
11 |
*/ |
5324a040c
|
12 |
#include <linux/module.h> |
d8313f5ca
|
13 |
#include <linux/random.h> |
5324a040c
|
14 |
|
c125e80b8
|
15 |
#include <net/addrconf.h> |
5324a040c
|
16 17 18 |
#include <net/inet_connection_sock.h> #include <net/inet_hashtables.h> #include <net/inet6_hashtables.h> |
6e5714eaf
|
19 |
#include <net/secure_seq.h> |
d8313f5ca
|
20 |
#include <net/ip.h> |
c125e80b8
|
21 |
#include <net/sock_reuseport.h> |
5324a040c
|
22 |
|
1122702f0
|
23 |
extern struct inet_hashinfo tcp_hashinfo; |
d1e559d0b
|
24 25 26 |
u32 inet6_ehashfn(const struct net *net, const struct in6_addr *laddr, const u16 lport, const struct in6_addr *faddr, const __be16 fport) |
b50026b5a
|
27 |
{ |
1bbdceef1
|
28 29 30 31 32 33 34 35 36 37 |
static u32 inet6_ehash_secret __read_mostly; static u32 ipv6_hash_secret __read_mostly; u32 lhash, fhash; net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret)); net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret)); lhash = (__force u32)laddr->s6_addr32[3]; fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret); |
b50026b5a
|
38 |
return __inet6_ehashfn(lhash, lport, fhash, fport, |
1bbdceef1
|
39 |
inet6_ehash_secret + net_hash_mix(net)); |
b50026b5a
|
40 |
} |
b1a7ffcb7
|
41 42 43 44 45 46 |
/* * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM * * The sockhash lock must be held as a reader here. */ |
d86e0dac2
|
47 48 |
struct sock *__inet6_lookup_established(struct net *net, struct inet_hashinfo *hashinfo, |
b1a7ffcb7
|
49 |
const struct in6_addr *saddr, |
d2ecd9ccd
|
50 |
const __be16 sport, |
b1a7ffcb7
|
51 52 |
const struct in6_addr *daddr, const u16 hnum, |
4297a0ef0
|
53 |
const int dif, const int sdif) |
b1a7ffcb7
|
54 55 |
{ struct sock *sk; |
3ab5aee7f
|
56 |
const struct hlist_nulls_node *node; |
4f765d842
|
57 |
const __portpair ports = INET_COMBINED_PORTS(sport, hnum); |
b1a7ffcb7
|
58 59 60 |
/* Optimize here for direct hit, only listening connections can * have wildcards anyways. */ |
33de014c6
|
61 |
unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport); |
f373b53b5
|
62 |
unsigned int slot = hash & hashinfo->ehash_mask; |
3ab5aee7f
|
63 |
struct inet_ehash_bucket *head = &hashinfo->ehash[slot]; |
b1a7ffcb7
|
64 |
|
3ab5aee7f
|
65 |
|
3ab5aee7f
|
66 67 |
begin: sk_nulls_for_each_rcu(sk, node, &head->chain) { |
ce43b03e8
|
68 69 |
if (sk->sk_hash != hash) continue; |
4297a0ef0
|
70 |
if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif)) |
efe4208f4
|
71 |
continue; |
41c6d650f
|
72 |
if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) |
05dbc7b59
|
73 |
goto out; |
4297a0ef0
|
74 |
if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))) { |
efe4208f4
|
75 76 |
sock_gen_put(sk); goto begin; |
3ab5aee7f
|
77 |
} |
efe4208f4
|
78 |
goto found; |
b1a7ffcb7
|
79 |
} |
3ab5aee7f
|
80 81 |
if (get_nulls_value(node) != slot) goto begin; |
3ab5aee7f
|
82 |
out: |
05dbc7b59
|
83 84 |
sk = NULL; found: |
b1a7ffcb7
|
85 86 87 |
return sk; } EXPORT_SYMBOL(__inet6_lookup_established); |
42b16b3fb
|
88 |
static inline int compute_score(struct sock *sk, struct net *net, |
c25eb3bfb
|
89 90 |
const unsigned short hnum, const struct in6_addr *daddr, |
4297a0ef0
|
91 |
const int dif, const int sdif, bool exact_dif) |
c25eb3bfb
|
92 93 |
{ int score = -1; |
c720c7e83
|
94 |
if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum && |
c25eb3bfb
|
95 |
sk->sk_family == PF_INET6) { |
0ee58dad5
|
96 97 |
if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) return -1; |
c25eb3bfb
|
98 |
|
0ee58dad5
|
99 |
if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) |
e78190581
|
100 |
return -1; |
4297a0ef0
|
101 |
|
0ee58dad5
|
102 |
score = 1; |
7170a9777
|
103 |
if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) |
70da268b5
|
104 |
score++; |
c25eb3bfb
|
105 106 107 |
} return score; } |
5df653129
|
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 |
static inline struct sock *lookup_reuseport(struct net *net, struct sock *sk, struct sk_buff *skb, int doff, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, unsigned short hnum) { struct sock *reuse_sk = NULL; u32 phash; if (sk->sk_reuseport) { phash = inet6_ehashfn(net, daddr, hnum, saddr, sport); reuse_sk = reuseport_select_sock(sk, phash, skb, doff); } return reuse_sk; } |
3b24d854c
|
124 |
/* called with rcu_read_lock() */ |
61b7c691c
|
125 126 127 128 129 130 131 132 133 134 135 |
static struct sock *inet6_lhash2_lookup(struct net *net, struct inet_listen_hashbucket *ilb2, struct sk_buff *skb, int doff, const struct in6_addr *saddr, const __be16 sport, const struct in6_addr *daddr, const unsigned short hnum, const int dif, const int sdif) { bool exact_dif = inet6_exact_dif_match(net, skb); struct inet_connection_sock *icsk; struct sock *sk, *result = NULL; int score, hiscore = 0; |
61b7c691c
|
136 137 138 139 140 141 |
inet_lhash2_for_each_icsk_rcu(icsk, &ilb2->head) { sk = (struct sock *)icsk; score = compute_score(sk, net, hnum, daddr, dif, sdif, exact_dif); if (score > hiscore) { |
5df653129
|
142 143 144 145 |
result = lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum); if (result) return result; |
61b7c691c
|
146 147 148 149 150 151 152 |
result = sk; hiscore = score; } } return result; } |
1122702f0
|
153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 |
static inline struct sock *inet6_lookup_run_bpf(struct net *net, struct inet_hashinfo *hashinfo, struct sk_buff *skb, int doff, const struct in6_addr *saddr, const __be16 sport, const struct in6_addr *daddr, const u16 hnum) { struct sock *sk, *reuse_sk; bool no_reuseport; if (hashinfo != &tcp_hashinfo) return NULL; /* only TCP is supported */ no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, saddr, sport, daddr, hnum, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; reuse_sk = lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum); if (reuse_sk) sk = reuse_sk; return sk; } |
d86e0dac2
|
177 |
struct sock *inet6_lookup_listener(struct net *net, |
a583636a8
|
178 179 180 |
struct inet_hashinfo *hashinfo, struct sk_buff *skb, int doff, const struct in6_addr *saddr, |
5ba24953e
|
181 |
const __be16 sport, const struct in6_addr *daddr, |
4297a0ef0
|
182 |
const unsigned short hnum, const int dif, const int sdif) |
5324a040c
|
183 |
{ |
61b7c691c
|
184 |
struct inet_listen_hashbucket *ilb2; |
0ee58dad5
|
185 |
struct sock *result = NULL; |
61b7c691c
|
186 |
unsigned int hash2; |
61b7c691c
|
187 |
|
1122702f0
|
188 189 190 191 192 193 194 |
/* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { result = inet6_lookup_run_bpf(net, hashinfo, skb, doff, saddr, sport, daddr, hnum); if (result) goto done; } |
61b7c691c
|
195 196 |
hash2 = ipv6_portaddr_hash(net, daddr, hnum); ilb2 = inet_lhash2_bucket(hashinfo, hash2); |
61b7c691c
|
197 198 199 200 201 |
result = inet6_lhash2_lookup(net, ilb2, skb, doff, saddr, sport, daddr, hnum, dif, sdif); if (result) |
8217ca653
|
202 |
goto done; |
61b7c691c
|
203 204 |
/* Lookup lhash2 with in6addr_any */ |
61b7c691c
|
205 206 |
hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum); ilb2 = inet_lhash2_bucket(hashinfo, hash2); |
61b7c691c
|
207 |
|
8217ca653
|
208 |
result = inet6_lhash2_lookup(net, ilb2, skb, doff, |
0ee58dad5
|
209 |
saddr, sport, &in6addr_any, hnum, |
8217ca653
|
210 |
dif, sdif); |
8217ca653
|
211 |
done: |
26f8113cc
|
212 |
if (IS_ERR(result)) |
8217ca653
|
213 |
return NULL; |
5324a040c
|
214 215 |
return result; } |
5324a040c
|
216 |
EXPORT_SYMBOL_GPL(inet6_lookup_listener); |
d86e0dac2
|
217 |
struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo, |
a583636a8
|
218 |
struct sk_buff *skb, int doff, |
d2ecd9ccd
|
219 220 |
const struct in6_addr *saddr, const __be16 sport, const struct in6_addr *daddr, const __be16 dport, |
5324a040c
|
221 222 223 |
const int dif) { struct sock *sk; |
3b24d854c
|
224 |
bool refcounted; |
5324a040c
|
225 |
|
a583636a8
|
226 |
sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr, |
4297a0ef0
|
227 |
ntohs(dport), dif, 0, &refcounted); |
41c6d650f
|
228 |
if (sk && !refcounted && !refcount_inc_not_zero(&sk->sk_refcnt)) |
3b24d854c
|
229 |
sk = NULL; |
5324a040c
|
230 231 |
return sk; } |
5324a040c
|
232 |
EXPORT_SYMBOL_GPL(inet6_lookup); |
d8313f5ca
|
233 234 235 236 237 238 |
static int __inet6_check_established(struct inet_timewait_death_row *death_row, struct sock *sk, const __u16 lport, struct inet_timewait_sock **twp) { struct inet_hashinfo *hinfo = death_row->hashinfo; |
3759fa9c5
|
239 |
struct inet_sock *inet = inet_sk(sk); |
efe4208f4
|
240 241 |
const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr; const struct in6_addr *saddr = &sk->sk_v6_daddr; |
d8313f5ca
|
242 |
const int dif = sk->sk_bound_dev_if; |
33de014c6
|
243 |
struct net *net = sock_net(sk); |
4297a0ef0
|
244 245 |
const int sdif = l3mdev_master_ifindex_by_index(net, dif); const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport); |
33de014c6
|
246 |
const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr, |
c720c7e83
|
247 |
inet->inet_dport); |
d8313f5ca
|
248 |
struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash); |
9db66bdcc
|
249 |
spinlock_t *lock = inet_ehash_lockp(hinfo, hash); |
d8313f5ca
|
250 |
struct sock *sk2; |
3ab5aee7f
|
251 |
const struct hlist_nulls_node *node; |
05dbc7b59
|
252 |
struct inet_timewait_sock *tw = NULL; |
d8313f5ca
|
253 |
|
9db66bdcc
|
254 |
spin_lock(lock); |
d8313f5ca
|
255 |
|
05dbc7b59
|
256 |
sk_nulls_for_each(sk2, node, &head->chain) { |
ce43b03e8
|
257 258 |
if (sk2->sk_hash != hash) continue; |
d8313f5ca
|
259 |
|
4297a0ef0
|
260 261 |
if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif, sdif))) { |
efe4208f4
|
262 |
if (sk2->sk_state == TCP_TIME_WAIT) { |
05dbc7b59
|
263 264 |
tw = inet_twsk(sk2); if (twsk_unique(sk, sk2, twp)) |
efe4208f4
|
265 |
break; |
05dbc7b59
|
266 |
} |
d8313f5ca
|
267 |
goto not_unique; |
efe4208f4
|
268 |
} |
d8313f5ca
|
269 |
} |
3759fa9c5
|
270 |
/* Must record num and sport now. Otherwise we will see |
efe4208f4
|
271 272 |
* in hash table socket with a funny identity. */ |
c720c7e83
|
273 274 |
inet->inet_num = lport; inet->inet_sport = htons(lport); |
13475a30b
|
275 |
sk->sk_hash = hash; |
547b792ca
|
276 |
WARN_ON(!sk_unhashed(sk)); |
3ab5aee7f
|
277 |
__sk_nulls_add_node_rcu(sk, &head->chain); |
13475a30b
|
278 |
if (tw) { |
fc01538f9
|
279 |
sk_nulls_del_node_init_rcu((struct sock *)tw); |
02a1d6e7a
|
280 |
__NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED); |
13475a30b
|
281 |
} |
9db66bdcc
|
282 |
spin_unlock(lock); |
c29a0bc4d
|
283 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); |
d8313f5ca
|
284 |
|
13475a30b
|
285 |
if (twp) { |
d8313f5ca
|
286 |
*twp = tw; |
13475a30b
|
287 |
} else if (tw) { |
d8313f5ca
|
288 |
/* Silly. Should hash-dance instead... */ |
dbe7faa40
|
289 |
inet_twsk_deschedule_put(tw); |
d8313f5ca
|
290 291 292 293 |
} return 0; not_unique: |
9db66bdcc
|
294 |
spin_unlock(lock); |
d8313f5ca
|
295 296 |
return -EADDRNOTAVAIL; } |
e2baad9e4
|
297 |
static u32 inet6_sk_port_offset(const struct sock *sk) |
d8313f5ca
|
298 299 |
{ const struct inet_sock *inet = inet_sk(sk); |
efe4208f4
|
300 301 302 |
return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32, sk->sk_v6_daddr.s6_addr32, |
c720c7e83
|
303 |
inet->inet_dport); |
d8313f5ca
|
304 305 306 307 308 |
} int inet6_hash_connect(struct inet_timewait_death_row *death_row, struct sock *sk) { |
e2baad9e4
|
309 310 311 312 313 |
u32 port_offset = 0; if (!inet_sk(sk)->inet_num) port_offset = inet6_sk_port_offset(sk); return __inet_hash_connect(death_row, sk, port_offset, |
b4d6444ea
|
314 |
__inet6_check_established); |
d8313f5ca
|
315 |
} |
d8313f5ca
|
316 |
EXPORT_SYMBOL_GPL(inet6_hash_connect); |
496611d7b
|
317 318 319 |
int inet6_hash(struct sock *sk) { |
e4cabca54
|
320 |
int err = 0; |
496611d7b
|
321 322 |
if (sk->sk_state != TCP_CLOSE) { local_bh_disable(); |
fe38d2a1c
|
323 |
err = __inet_hash(sk, NULL); |
496611d7b
|
324 325 |
local_bh_enable(); } |
e4cabca54
|
326 |
return err; |
496611d7b
|
327 328 |
} EXPORT_SYMBOL_GPL(inet6_hash); |