Blame view

net/ipv6/inet6_hashtables.c 8.95 KB
5324a040c   Arnaldo Carvalho de Melo   [INET6_HASHTABLES...
1
2
3
4
5
6
7
  /*
   * INET		An implementation of the TCP/IP protocol suite for the LINUX
   *		operating system.  INET is implemented using the BSD Socket
   *		interface as the means of communication with the user level.
   *
   *		Generic INET6 transport hashtables
   *
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
8
9
   * Authors:	Lotsa people, from code originally in tcp, generalised here
   * 		by Arnaldo Carvalho de Melo <acme@mandriva.com>
5324a040c   Arnaldo Carvalho de Melo   [INET6_HASHTABLES...
10
11
12
13
14
15
   *
   *	This program is free software; you can redistribute it and/or
   *      modify it under the terms of the GNU General Public License
   *      as published by the Free Software Foundation; either version
   *      2 of the License, or (at your option) any later version.
   */
5324a040c   Arnaldo Carvalho de Melo   [INET6_HASHTABLES...
16
  #include <linux/module.h>
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
17
  #include <linux/random.h>
5324a040c   Arnaldo Carvalho de Melo   [INET6_HASHTABLES...
18
19
20
21
  
  #include <net/inet_connection_sock.h>
  #include <net/inet_hashtables.h>
  #include <net/inet6_hashtables.h>
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
22
  #include <net/ip.h>
5324a040c   Arnaldo Carvalho de Melo   [INET6_HASHTABLES...
23

b1a7ffcb7   Denis Vlasenko   [IPV6]: Deinline ...
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
  void __inet6_hash(struct inet_hashinfo *hashinfo,
  				struct sock *sk)
  {
  	struct hlist_head *list;
  	rwlock_t *lock;
  
  	BUG_TRAP(sk_unhashed(sk));
  
  	if (sk->sk_state == TCP_LISTEN) {
  		list = &hashinfo->listening_hash[inet_sk_listen_hashfn(sk)];
  		lock = &hashinfo->lhash_lock;
  		inet_listen_wlock(hashinfo);
  	} else {
  		unsigned int hash;
  		sk->sk_hash = hash = inet6_sk_ehashfn(sk);
  		hash &= (hashinfo->ehash_size - 1);
  		list = &hashinfo->ehash[hash].chain;
  		lock = &hashinfo->ehash[hash].lock;
  		write_lock(lock);
  	}
  
  	__sk_add_node(sk, list);
  	sock_prot_inc_use(sk->sk_prot);
  	write_unlock(lock);
  }
  EXPORT_SYMBOL(__inet6_hash);
  
  /*
   * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
   * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
   *
   * The sockhash lock must be held as a reader here.
   */
  struct sock *__inet6_lookup_established(struct inet_hashinfo *hashinfo,
  					   const struct in6_addr *saddr,
d2ecd9ccd   Al Viro   [IPV6]: annotate ...
59
  					   const __be16 sport,
b1a7ffcb7   Denis Vlasenko   [IPV6]: Deinline ...
60
61
62
63
64
65
  					   const struct in6_addr *daddr,
  					   const u16 hnum,
  					   const int dif)
  {
  	struct sock *sk;
  	const struct hlist_node *node;
4f765d842   Al Viro   [IPV4]: INET_MATC...
66
  	const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
b1a7ffcb7   Denis Vlasenko   [IPV6]: Deinline ...
67
68
69
70
71
72
73
74
75
76
77
78
79
80
  	/* Optimize here for direct hit, only listening connections can
  	 * have wildcards anyways.
  	 */
  	unsigned int hash = inet6_ehashfn(daddr, hnum, saddr, sport);
  	struct inet_ehash_bucket *head = inet_ehash_bucket(hashinfo, hash);
  
  	prefetch(head->chain.first);
  	read_lock(&head->lock);
  	sk_for_each(sk, node, &head->chain) {
  		/* For IPV6 do the cheaper port and family tests first. */
  		if (INET6_MATCH(sk, hash, saddr, daddr, ports, dif))
  			goto hit; /* You sunk my battleship! */
  	}
  	/* Must check for a TIME_WAIT'er before going to listener hash. */
dbca9b275   Eric Dumazet   [NET]: change lay...
81
  	sk_for_each(sk, node, &head->twchain) {
b1a7ffcb7   Denis Vlasenko   [IPV6]: Deinline ...
82
  		const struct inet_timewait_sock *tw = inet_twsk(sk);
4f765d842   Al Viro   [IPV4]: INET_MATC...
83
  		if(*((__portpair *)&(tw->tw_dport))	== ports	&&
b1a7ffcb7   Denis Vlasenko   [IPV6]: Deinline ...
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
  		   sk->sk_family		== PF_INET6) {
  			const struct inet6_timewait_sock *tw6 = inet6_twsk(sk);
  
  			if (ipv6_addr_equal(&tw6->tw_v6_daddr, saddr)	&&
  			    ipv6_addr_equal(&tw6->tw_v6_rcv_saddr, daddr)	&&
  			    (!sk->sk_bound_dev_if || sk->sk_bound_dev_if == dif))
  				goto hit;
  		}
  	}
  	read_unlock(&head->lock);
  	return NULL;
  
  hit:
  	sock_hold(sk);
  	read_unlock(&head->lock);
  	return sk;
  }
  EXPORT_SYMBOL(__inet6_lookup_established);
5324a040c   Arnaldo Carvalho de Melo   [INET6_HASHTABLES...
102
103
104
105
106
107
108
109
110
111
112
113
114
  struct sock *inet6_lookup_listener(struct inet_hashinfo *hashinfo,
  				   const struct in6_addr *daddr,
  				   const unsigned short hnum, const int dif)
  {
  	struct sock *sk;
  	const struct hlist_node *node;
  	struct sock *result = NULL;
  	int score, hiscore = 0;
  
  	read_lock(&hashinfo->lhash_lock);
  	sk_for_each(sk, node, &hashinfo->listening_hash[inet_lhashfn(hnum)]) {
  		if (inet_sk(sk)->num == hnum && sk->sk_family == PF_INET6) {
  			const struct ipv6_pinfo *np = inet6_sk(sk);
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
115

5324a040c   Arnaldo Carvalho de Melo   [INET6_HASHTABLES...
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
  			score = 1;
  			if (!ipv6_addr_any(&np->rcv_saddr)) {
  				if (!ipv6_addr_equal(&np->rcv_saddr, daddr))
  					continue;
  				score++;
  			}
  			if (sk->sk_bound_dev_if) {
  				if (sk->sk_bound_dev_if != dif)
  					continue;
  				score++;
  			}
  			if (score == 3) {
  				result = sk;
  				break;
  			}
  			if (score > hiscore) {
  				hiscore = score;
  				result = sk;
  			}
  		}
  	}
  	if (result)
  		sock_hold(result);
  	read_unlock(&hashinfo->lhash_lock);
  	return result;
  }
  
  EXPORT_SYMBOL_GPL(inet6_lookup_listener);
  
  struct sock *inet6_lookup(struct inet_hashinfo *hashinfo,
d2ecd9ccd   Al Viro   [IPV6]: annotate ...
146
147
  			  const struct in6_addr *saddr, const __be16 sport,
  			  const struct in6_addr *daddr, const __be16 dport,
5324a040c   Arnaldo Carvalho de Melo   [INET6_HASHTABLES...
148
149
150
151
152
153
154
155
156
157
158
159
  			  const int dif)
  {
  	struct sock *sk;
  
  	local_bh_disable();
  	sk = __inet6_lookup(hashinfo, saddr, sport, daddr, ntohs(dport), dif);
  	local_bh_enable();
  
  	return sk;
  }
  
  EXPORT_SYMBOL_GPL(inet6_lookup);
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
160
161
162
163
164
165
  
  static int __inet6_check_established(struct inet_timewait_death_row *death_row,
  				     struct sock *sk, const __u16 lport,
  				     struct inet_timewait_sock **twp)
  {
  	struct inet_hashinfo *hinfo = death_row->hashinfo;
3759fa9c5   Herbert Xu   [TCP]: Fix zero p...
166
  	struct inet_sock *inet = inet_sk(sk);
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
167
168
169
170
  	const struct ipv6_pinfo *np = inet6_sk(sk);
  	const struct in6_addr *daddr = &np->rcv_saddr;
  	const struct in6_addr *saddr = &np->daddr;
  	const int dif = sk->sk_bound_dev_if;
4f765d842   Al Viro   [IPV4]: INET_MATC...
171
  	const __portpair ports = INET_COMBINED_PORTS(inet->dport, lport);
6e1d9d04c   YOSHIFUJI Hideaki   [IPV6] HASHTABLES...
172
  	const unsigned int hash = inet6_ehashfn(daddr, lport, saddr,
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
173
174
175
176
177
178
179
180
181
182
  						inet->dport);
  	struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
  	struct sock *sk2;
  	const struct hlist_node *node;
  	struct inet_timewait_sock *tw;
  
  	prefetch(head->chain.first);
  	write_lock(&head->lock);
  
  	/* Check TIME-WAIT sockets first. */
dbca9b275   Eric Dumazet   [NET]: change lay...
183
  	sk_for_each(sk2, node, &head->twchain) {
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
184
185
186
  		const struct inet6_timewait_sock *tw6 = inet6_twsk(sk2);
  
  		tw = inet_twsk(sk2);
4f765d842   Al Viro   [IPV4]: INET_MATC...
187
  		if(*((__portpair *)&(tw->tw_dport)) == ports		 &&
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
  		   sk2->sk_family	       == PF_INET6	 &&
  		   ipv6_addr_equal(&tw6->tw_v6_daddr, saddr)	 &&
  		   ipv6_addr_equal(&tw6->tw_v6_rcv_saddr, daddr) &&
  		   sk2->sk_bound_dev_if == sk->sk_bound_dev_if) {
  			if (twsk_unique(sk, sk2, twp))
  				goto unique;
  			else
  				goto not_unique;
  		}
  	}
  	tw = NULL;
  
  	/* And established part... */
  	sk_for_each(sk2, node, &head->chain) {
  		if (INET6_MATCH(sk2, hash, saddr, daddr, ports, dif))
  			goto not_unique;
  	}
  
  unique:
3759fa9c5   Herbert Xu   [TCP]: Fix zero p...
207
208
209
210
  	/* Must record num and sport now. Otherwise we will see
  	 * in hash table socket with a funny identity. */
  	inet->num = lport;
  	inet->sport = htons(lport);
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
  	BUG_TRAP(sk_unhashed(sk));
  	__sk_add_node(sk, &head->chain);
  	sk->sk_hash = hash;
  	sock_prot_inc_use(sk->sk_prot);
  	write_unlock(&head->lock);
  
  	if (twp != NULL) {
  		*twp = tw;
  		NET_INC_STATS_BH(LINUX_MIB_TIMEWAITRECYCLED);
  	} else if (tw != NULL) {
  		/* Silly. Should hash-dance instead... */
  		inet_twsk_deschedule(tw, death_row);
  		NET_INC_STATS_BH(LINUX_MIB_TIMEWAITRECYCLED);
  
  		inet_twsk_put(tw);
  	}
  	return 0;
  
  not_unique:
  	write_unlock(&head->lock);
  	return -EADDRNOTAVAIL;
  }
  
  static inline u32 inet6_sk_port_offset(const struct sock *sk)
  {
  	const struct inet_sock *inet = inet_sk(sk);
  	const struct ipv6_pinfo *np = inet6_sk(sk);
  	return secure_ipv6_port_ephemeral(np->rcv_saddr.s6_addr32,
  					  np->daddr.s6_addr32,
  					  inet->dport);
  }
  
  int inet6_hash_connect(struct inet_timewait_death_row *death_row,
  		       struct sock *sk)
  {
  	struct inet_hashinfo *hinfo = death_row->hashinfo;
  	const unsigned short snum = inet_sk(sk)->num;
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
248
249
  	struct inet_bind_hashbucket *head;
  	struct inet_bind_bucket *tb;
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
250
  	int ret;
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
251
252
253
  	if (snum == 0) {
  		const int low = sysctl_local_port_range[0];
  		const int high = sysctl_local_port_range[1];
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
254
  		const int range = high - low;
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
255
  		int i, port;
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
256
257
258
  		static u32 hint;
  		const u32 offset = hint + inet6_sk_port_offset(sk);
  		struct hlist_node *node;
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
259
  		struct inet_timewait_sock *tw = NULL;
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
260

1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
261
  		local_bh_disable();
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
262
263
  		for (i = 1; i <= range; i++) {
  			port = low + (i + offset) % range;
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
264
265
  			head = &hinfo->bhash[inet_bhashfn(port, hinfo->bhash_size)];
  			spin_lock(&head->lock);
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
266

1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
267
268
269
270
  			/* Does not bother with rcv_saddr checks,
  			 * because the established check is already
  			 * unique enough.
  			 */
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
271
  			inet_bind_bucket_for_each(tb, node, &head->chain) {
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
272
273
274
275
276
  				if (tb->port == port) {
  					BUG_TRAP(!hlist_empty(&tb->owners));
  					if (tb->fastreuse >= 0)
  						goto next_port;
  					if (!__inet6_check_established(death_row,
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
277
278
  								       sk, port,
  								       &tw))
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
279
280
281
282
  						goto ok;
  					goto next_port;
  				}
  			}
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
283

1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
284
  			tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
285
  						     head, port);
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
286
287
288
289
290
291
  			if (!tb) {
  				spin_unlock(&head->lock);
  				break;
  			}
  			tb->fastreuse = -1;
  			goto ok;
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
292

1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
293
294
295
296
  		next_port:
  			spin_unlock(&head->lock);
  		}
  		local_bh_enable();
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
297

1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
298
  		return -EADDRNOTAVAIL;
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
299
300
301
  
  ok:
  		hint += i;
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
302
303
  		/* Head lock still held and bh's disabled */
  		inet_bind_hash(sk, tb, port);
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
304
  		if (sk_unhashed(sk)) {
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
305
306
307
308
  			inet_sk(sk)->sport = htons(port);
  			__inet6_hash(hinfo, sk);
  		}
  		spin_unlock(&head->lock);
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
309

1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
310
311
312
313
  		if (tw) {
  			inet_twsk_deschedule(tw, death_row);
  			inet_twsk_put(tw);
  		}
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
314
315
316
  
  		ret = 0;
  		goto out;
1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
317
  	}
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
318

1ab1457c4   YOSHIFUJI Hideaki   [NET] IPV6: Fix w...
319
320
  	head = &hinfo->bhash[inet_bhashfn(snum, hinfo->bhash_size)];
  	tb   = inet_csk(sk)->icsk_bind_hash;
d8313f5ca   Arnaldo Carvalho de Melo   [INET6]: Generali...
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
  	spin_lock_bh(&head->lock);
  
  	if (sk_head(&tb->owners) == sk && sk->sk_bind_node.next == NULL) {
  		__inet6_hash(hinfo, sk);
  		spin_unlock_bh(&head->lock);
  		return 0;
  	} else {
  		spin_unlock(&head->lock);
  		/* No definite answer... Walk to established hash table */
  		ret = __inet6_check_established(death_row, sk, snum, NULL);
  out:
  		local_bh_enable();
  		return ret;
  	}
  }
  
  EXPORT_SYMBOL_GPL(inet6_hash_connect);