Blame view
net/ipv6/inet6_hashtables.c
7.77 KB
5324a040c [INET6_HASHTABLES... |
1 2 3 4 5 6 7 |
/* * INET An implementation of the TCP/IP protocol suite for the LINUX * operating system. INET is implemented using the BSD Socket * interface as the means of communication with the user level. * * Generic INET6 transport hashtables * |
d8313f5ca [INET6]: Generali... |
8 |
* Authors: Lotsa people, from code originally in tcp, generalised here |
67ba4152e ipv6: White-space... |
9 |
* by Arnaldo Carvalho de Melo <acme@mandriva.com> |
5324a040c [INET6_HASHTABLES... |
10 11 12 13 14 15 |
* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License * as published by the Free Software Foundation; either version * 2 of the License, or (at your option) any later version. */ |
5324a040c [INET6_HASHTABLES... |
16 |
#include <linux/module.h> |
d8313f5ca [INET6]: Generali... |
17 |
#include <linux/random.h> |
5324a040c [INET6_HASHTABLES... |
18 |
|
c125e80b8 soreuseport: fast... |
19 |
#include <net/addrconf.h> |
5324a040c [INET6_HASHTABLES... |
20 21 22 |
#include <net/inet_connection_sock.h> #include <net/inet_hashtables.h> #include <net/inet6_hashtables.h> |
6e5714eaf net: Compute prot... |
23 |
#include <net/secure_seq.h> |
d8313f5ca [INET6]: Generali... |
24 |
#include <net/ip.h> |
c125e80b8 soreuseport: fast... |
25 |
#include <net/sock_reuseport.h> |
5324a040c [INET6_HASHTABLES... |
26 |
|
d1e559d0b inet: add IPv6 su... |
27 28 29 |
u32 inet6_ehashfn(const struct net *net, const struct in6_addr *laddr, const u16 lport, const struct in6_addr *faddr, const __be16 fport) |
b50026b5a ipv6: split inet6... |
30 |
{ |
1bbdceef1 inet: convert ine... |
31 32 33 34 35 36 37 38 39 40 |
static u32 inet6_ehash_secret __read_mostly; static u32 ipv6_hash_secret __read_mostly; u32 lhash, fhash; net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret)); net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret)); lhash = (__force u32)laddr->s6_addr32[3]; fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret); |
b50026b5a ipv6: split inet6... |
41 |
return __inet6_ehashfn(lhash, lport, fhash, fport, |
1bbdceef1 inet: convert ine... |
42 |
inet6_ehash_secret + net_hash_mix(net)); |
b50026b5a ipv6: split inet6... |
43 |
} |
b1a7ffcb7 [IPV6]: Deinline ... |
44 45 46 47 48 49 |
/* * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM * * The sockhash lock must be held as a reader here. */ |
d86e0dac2 [NETNS]: Tcp-v6 s... |
50 51 |
struct sock *__inet6_lookup_established(struct net *net, struct inet_hashinfo *hashinfo, |
b1a7ffcb7 [IPV6]: Deinline ... |
52 |
const struct in6_addr *saddr, |
d2ecd9ccd [IPV6]: annotate ... |
53 |
const __be16 sport, |
b1a7ffcb7 [IPV6]: Deinline ... |
54 55 |
const struct in6_addr *daddr, const u16 hnum, |
4297a0ef0 net: ipv6: add se... |
56 |
const int dif, const int sdif) |
b1a7ffcb7 [IPV6]: Deinline ... |
57 58 |
{ struct sock *sk; |
3ab5aee7f net: Convert TCP ... |
59 |
const struct hlist_nulls_node *node; |
4f765d842 [IPV4]: INET_MATC... |
60 |
const __portpair ports = INET_COMBINED_PORTS(sport, hnum); |
b1a7ffcb7 [IPV6]: Deinline ... |
61 62 63 |
/* Optimize here for direct hit, only listening connections can * have wildcards anyways. */ |
33de014c6 inet6: add struct... |
64 |
unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport); |
f373b53b5 tcp: replace ehas... |
65 |
unsigned int slot = hash & hashinfo->ehash_mask; |
3ab5aee7f net: Convert TCP ... |
66 |
struct inet_ehash_bucket *head = &hashinfo->ehash[slot]; |
b1a7ffcb7 [IPV6]: Deinline ... |
67 |
|
3ab5aee7f net: Convert TCP ... |
68 |
|
3ab5aee7f net: Convert TCP ... |
69 70 |
begin: sk_nulls_for_each_rcu(sk, node, &head->chain) { |
ce43b03e8 net: move inet_dp... |
71 72 |
if (sk->sk_hash != hash) continue; |
4297a0ef0 net: ipv6: add se... |
73 |
if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif)) |
efe4208f4 ipv6: make lookup... |
74 |
continue; |
41c6d650f net: convert sock... |
75 |
if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) |
05dbc7b59 tcp/dccp: remove ... |
76 |
goto out; |
4297a0ef0 net: ipv6: add se... |
77 |
if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))) { |
efe4208f4 ipv6: make lookup... |
78 79 |
sock_gen_put(sk); goto begin; |
3ab5aee7f net: Convert TCP ... |
80 |
} |
efe4208f4 ipv6: make lookup... |
81 |
goto found; |
b1a7ffcb7 [IPV6]: Deinline ... |
82 |
} |
3ab5aee7f net: Convert TCP ... |
83 84 |
if (get_nulls_value(node) != slot) goto begin; |
3ab5aee7f net: Convert TCP ... |
85 |
out: |
05dbc7b59 tcp/dccp: remove ... |
86 87 |
sk = NULL; found: |
b1a7ffcb7 [IPV6]: Deinline ... |
88 89 90 |
return sk; } EXPORT_SYMBOL(__inet6_lookup_established); |
42b16b3fb Kill off warning:... |
91 |
static inline int compute_score(struct sock *sk, struct net *net, |
c25eb3bfb net: Convert TCP/... |
92 93 |
const unsigned short hnum, const struct in6_addr *daddr, |
4297a0ef0 net: ipv6: add se... |
94 |
const int dif, const int sdif, bool exact_dif) |
c25eb3bfb net: Convert TCP/... |
95 96 |
{ int score = -1; |
c720c7e83 inet: rename some... |
97 |
if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum && |
c25eb3bfb net: Convert TCP/... |
98 |
sk->sk_family == PF_INET6) { |
c25eb3bfb net: Convert TCP/... |
99 100 |
score = 1; |
efe4208f4 ipv6: make lookup... |
101 102 |
if (!ipv6_addr_any(&sk->sk_v6_rcv_saddr)) { if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) |
c25eb3bfb net: Convert TCP/... |
103 104 105 |
return -1; score++; } |
a04a480d4 net: Require exac... |
106 |
if (sk->sk_bound_dev_if || exact_dif) { |
4297a0ef0 net: ipv6: add se... |
107 108 |
bool dev_match = (sk->sk_bound_dev_if == dif || sk->sk_bound_dev_if == sdif); |
35e324ebe net/tcp: Fix sock... |
109 |
if (!dev_match) |
c25eb3bfb net: Convert TCP/... |
110 |
return -1; |
35e324ebe net/tcp: Fix sock... |
111 |
if (sk->sk_bound_dev_if) |
4297a0ef0 net: ipv6: add se... |
112 |
score++; |
c25eb3bfb net: Convert TCP/... |
113 |
} |
70da268b5 net: SO_INCOMING_... |
114 115 |
if (sk->sk_incoming_cpu == raw_smp_processor_id()) score++; |
c25eb3bfb net: Convert TCP/... |
116 117 118 |
} return score; } |
3b24d854c tcp/dccp: do not ... |
119 |
/* called with rcu_read_lock() */ |
d86e0dac2 [NETNS]: Tcp-v6 s... |
120 |
struct sock *inet6_lookup_listener(struct net *net, |
a583636a8 inet: refactor in... |
121 122 123 |
struct inet_hashinfo *hashinfo, struct sk_buff *skb, int doff, const struct in6_addr *saddr, |
5ba24953e soreuseport: TCP/... |
124 |
const __be16 sport, const struct in6_addr *daddr, |
4297a0ef0 net: ipv6: add se... |
125 |
const unsigned short hnum, const int dif, const int sdif) |
5324a040c [INET6_HASHTABLES... |
126 |
{ |
c25eb3bfb net: Convert TCP/... |
127 128 |
unsigned int hash = inet_lhashfn(net, hnum); struct inet_listen_hashbucket *ilb = &hashinfo->listening_hash[hash]; |
3b24d854c tcp/dccp: do not ... |
129 |
int score, hiscore = 0, matches = 0, reuseport = 0; |
a04a480d4 net: Require exac... |
130 |
bool exact_dif = inet6_exact_dif_match(net, skb); |
3b24d854c tcp/dccp: do not ... |
131 132 |
struct sock *sk, *result = NULL; u32 phash = 0; |
c25eb3bfb net: Convert TCP/... |
133 |
|
3b24d854c tcp/dccp: do not ... |
134 |
sk_for_each(sk, &ilb->head) { |
4297a0ef0 net: ipv6: add se... |
135 |
score = compute_score(sk, net, hnum, daddr, dif, sdif, exact_dif); |
c25eb3bfb net: Convert TCP/... |
136 |
if (score > hiscore) { |
03c5b5341 ipv6: fix inet6_l... |
137 |
reuseport = sk->sk_reuseport; |
5ba24953e soreuseport: TCP/... |
138 139 140 |
if (reuseport) { phash = inet6_ehashfn(net, daddr, hnum, saddr, sport); |
3b24d854c tcp/dccp: do not ... |
141 142 143 144 |
result = reuseport_select_sock(sk, phash, skb, doff); if (result) return result; |
5ba24953e soreuseport: TCP/... |
145 146 |
matches = 1; } |
3b24d854c tcp/dccp: do not ... |
147 |
result = sk; |
03c5b5341 ipv6: fix inet6_l... |
148 |
hiscore = score; |
5ba24953e soreuseport: TCP/... |
149 150 |
} else if (score == hiscore && reuseport) { matches++; |
8fc54f689 net: use reciproc... |
151 |
if (reciprocal_scale(phash, matches) == 0) |
5ba24953e soreuseport: TCP/... |
152 153 |
result = sk; phash = next_pseudo_random32(phash); |
5324a040c [INET6_HASHTABLES... |
154 155 |
} } |
5324a040c [INET6_HASHTABLES... |
156 157 |
return result; } |
5324a040c [INET6_HASHTABLES... |
158 |
EXPORT_SYMBOL_GPL(inet6_lookup_listener); |
d86e0dac2 [NETNS]: Tcp-v6 s... |
159 |
struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo, |
a583636a8 inet: refactor in... |
160 |
struct sk_buff *skb, int doff, |
d2ecd9ccd [IPV6]: annotate ... |
161 162 |
const struct in6_addr *saddr, const __be16 sport, const struct in6_addr *daddr, const __be16 dport, |
5324a040c [INET6_HASHTABLES... |
163 164 165 |
const int dif) { struct sock *sk; |
3b24d854c tcp/dccp: do not ... |
166 |
bool refcounted; |
5324a040c [INET6_HASHTABLES... |
167 |
|
a583636a8 inet: refactor in... |
168 |
sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr, |
4297a0ef0 net: ipv6: add se... |
169 |
ntohs(dport), dif, 0, &refcounted); |
41c6d650f net: convert sock... |
170 |
if (sk && !refcounted && !refcount_inc_not_zero(&sk->sk_refcnt)) |
3b24d854c tcp/dccp: do not ... |
171 |
sk = NULL; |
5324a040c [INET6_HASHTABLES... |
172 173 |
return sk; } |
5324a040c [INET6_HASHTABLES... |
174 |
EXPORT_SYMBOL_GPL(inet6_lookup); |
d8313f5ca [INET6]: Generali... |
175 176 177 178 179 180 |
static int __inet6_check_established(struct inet_timewait_death_row *death_row, struct sock *sk, const __u16 lport, struct inet_timewait_sock **twp) { struct inet_hashinfo *hinfo = death_row->hashinfo; |
3759fa9c5 [TCP]: Fix zero p... |
181 |
struct inet_sock *inet = inet_sk(sk); |
efe4208f4 ipv6: make lookup... |
182 183 |
const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr; const struct in6_addr *saddr = &sk->sk_v6_daddr; |
d8313f5ca [INET6]: Generali... |
184 |
const int dif = sk->sk_bound_dev_if; |
33de014c6 inet6: add struct... |
185 |
struct net *net = sock_net(sk); |
4297a0ef0 net: ipv6: add se... |
186 187 |
const int sdif = l3mdev_master_ifindex_by_index(net, dif); const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport); |
33de014c6 inet6: add struct... |
188 |
const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr, |
c720c7e83 inet: rename some... |
189 |
inet->inet_dport); |
d8313f5ca [INET6]: Generali... |
190 |
struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash); |
9db66bdcc net: convert TCP/... |
191 |
spinlock_t *lock = inet_ehash_lockp(hinfo, hash); |
d8313f5ca [INET6]: Generali... |
192 |
struct sock *sk2; |
3ab5aee7f net: Convert TCP ... |
193 |
const struct hlist_nulls_node *node; |
05dbc7b59 tcp/dccp: remove ... |
194 |
struct inet_timewait_sock *tw = NULL; |
d8313f5ca [INET6]: Generali... |
195 |
|
9db66bdcc net: convert TCP/... |
196 |
spin_lock(lock); |
d8313f5ca [INET6]: Generali... |
197 |
|
05dbc7b59 tcp/dccp: remove ... |
198 |
sk_nulls_for_each(sk2, node, &head->chain) { |
ce43b03e8 net: move inet_dp... |
199 200 |
if (sk2->sk_hash != hash) continue; |
d8313f5ca [INET6]: Generali... |
201 |
|
4297a0ef0 net: ipv6: add se... |
202 203 |
if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports, dif, sdif))) { |
efe4208f4 ipv6: make lookup... |
204 |
if (sk2->sk_state == TCP_TIME_WAIT) { |
05dbc7b59 tcp/dccp: remove ... |
205 206 |
tw = inet_twsk(sk2); if (twsk_unique(sk, sk2, twp)) |
efe4208f4 ipv6: make lookup... |
207 |
break; |
05dbc7b59 tcp/dccp: remove ... |
208 |
} |
d8313f5ca [INET6]: Generali... |
209 |
goto not_unique; |
efe4208f4 ipv6: make lookup... |
210 |
} |
d8313f5ca [INET6]: Generali... |
211 |
} |
3759fa9c5 [TCP]: Fix zero p... |
212 |
/* Must record num and sport now. Otherwise we will see |
efe4208f4 ipv6: make lookup... |
213 214 |
* in hash table socket with a funny identity. */ |
c720c7e83 inet: rename some... |
215 216 |
inet->inet_num = lport; inet->inet_sport = htons(lport); |
13475a30b tcp: connect() ra... |
217 |
sk->sk_hash = hash; |
547b792ca net: convert BUG_... |
218 |
WARN_ON(!sk_unhashed(sk)); |
3ab5aee7f net: Convert TCP ... |
219 |
__sk_nulls_add_node_rcu(sk, &head->chain); |
13475a30b tcp: connect() ra... |
220 |
if (tw) { |
fc01538f9 inet: simplify ti... |
221 |
sk_nulls_del_node_init_rcu((struct sock *)tw); |
02a1d6e7a net: rename NET_{... |
222 |
__NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED); |
13475a30b tcp: connect() ra... |
223 |
} |
9db66bdcc net: convert TCP/... |
224 |
spin_unlock(lock); |
c29a0bc4d [SOCK][NETNS]: Ad... |
225 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); |
d8313f5ca [INET6]: Generali... |
226 |
|
13475a30b tcp: connect() ra... |
227 |
if (twp) { |
d8313f5ca [INET6]: Generali... |
228 |
*twp = tw; |
13475a30b tcp: connect() ra... |
229 |
} else if (tw) { |
d8313f5ca [INET6]: Generali... |
230 |
/* Silly. Should hash-dance instead... */ |
dbe7faa40 inet: inet_twsk_d... |
231 |
inet_twsk_deschedule_put(tw); |
d8313f5ca [INET6]: Generali... |
232 233 234 235 |
} return 0; not_unique: |
9db66bdcc net: convert TCP/... |
236 |
spin_unlock(lock); |
d8313f5ca [INET6]: Generali... |
237 238 |
return -EADDRNOTAVAIL; } |
e2baad9e4 tcp: connect() fr... |
239 |
static u32 inet6_sk_port_offset(const struct sock *sk) |
d8313f5ca [INET6]: Generali... |
240 241 |
{ const struct inet_sock *inet = inet_sk(sk); |
efe4208f4 ipv6: make lookup... |
242 243 244 |
return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32, sk->sk_v6_daddr.s6_addr32, |
c720c7e83 inet: rename some... |
245 |
inet->inet_dport); |
d8313f5ca [INET6]: Generali... |
246 247 248 249 250 |
} int inet6_hash_connect(struct inet_timewait_death_row *death_row, struct sock *sk) { |
e2baad9e4 tcp: connect() fr... |
251 252 253 254 255 |
u32 port_offset = 0; if (!inet_sk(sk)->inet_num) port_offset = inet6_sk_port_offset(sk); return __inet_hash_connect(death_row, sk, port_offset, |
b4d6444ea inet: get rid of ... |
256 |
__inet6_check_established); |
d8313f5ca [INET6]: Generali... |
257 |
} |
d8313f5ca [INET6]: Generali... |
258 |
EXPORT_SYMBOL_GPL(inet6_hash_connect); |
496611d7b inet: create IPv6... |
259 260 261 |
int inet6_hash(struct sock *sk) { |
e4cabca54 inet: Fix missing... |
262 |
int err = 0; |
496611d7b inet: create IPv6... |
263 264 |
if (sk->sk_state != TCP_CLOSE) { local_bh_disable(); |
fe38d2a1c inet: collapse ip... |
265 |
err = __inet_hash(sk, NULL); |
496611d7b inet: create IPv6... |
266 267 |
local_bh_enable(); } |
e4cabca54 inet: Fix missing... |
268 |
return err; |
496611d7b inet: create IPv6... |
269 270 |
} EXPORT_SYMBOL_GPL(inet6_hash); |