Commit ddbe503203855939946430e39bae58de11b70b69
Committed by
David S. Miller
1 parent
dc9059512c
Exists in
smarc-l5.0.0_1.0.0-ga
and in
5 other branches
ipv6: add ipv6_addr_hash() helper
Introduce ipv6_addr_hash() helper doing a XOR on all bits of an IPv6 address, with an optimized x86_64 version. Use it in flow dissector, as suggested by Andrew McGregor, to reduce hash collision probabilities in fq_codel (and other users of flow dissector) Use it in ip6_tunnel.c and use more bit shuffling, as suggested by David Laight, as existing hash was ignoring most of them. Use it in sunrpc and use more bit shuffling, using hash_32(). Use it in net/ipv6/addrconf.c, using hash_32() as well. As a cleanup, use it in net/ipv4/tcp_metrics.c Signed-off-by: Eric Dumazet <edumazet@google.com> Reported-by: Andrew McGregor <andrewmcgr@gmail.com> Cc: Dave Taht <dave.taht@gmail.com> Cc: Tom Herbert <therbert@google.com> Cc: David Laight <David.Laight@ACULAB.COM> Cc: Joe Perches <joe@perches.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Showing 7 changed files with 45 additions and 54 deletions Side-by-side Diff
include/net/addrconf.h
include/net/ipv6.h
... | ... | @@ -419,6 +419,19 @@ |
419 | 419 | #endif |
420 | 420 | } |
421 | 421 | |
422 | +static inline u32 ipv6_addr_hash(const struct in6_addr *a) | |
423 | +{ | |
424 | +#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64 | |
425 | + const unsigned long *ul = (const unsigned long *)a; | |
426 | + unsigned long x = ul[0] ^ ul[1]; | |
427 | + | |
428 | + return (u32)(x ^ (x >> 32)); | |
429 | +#else | |
430 | + return (__force u32)(a->s6_addr32[0] ^ a->s6_addr32[1] ^ | |
431 | + a->s6_addr32[2] ^ a->s6_addr32[3]); | |
432 | +#endif | |
433 | +} | |
434 | + | |
422 | 435 | static inline bool ipv6_addr_loopback(const struct in6_addr *a) |
423 | 436 | { |
424 | 437 | return (a->s6_addr32[0] | a->s6_addr32[1] | |
net/core/flow_dissector.c
... | ... | @@ -4,6 +4,7 @@ |
4 | 4 | #include <linux/ipv6.h> |
5 | 5 | #include <linux/if_vlan.h> |
6 | 6 | #include <net/ip.h> |
7 | +#include <net/ipv6.h> | |
7 | 8 | #include <linux/if_tunnel.h> |
8 | 9 | #include <linux/if_pppox.h> |
9 | 10 | #include <linux/ppp_defs.h> |
... | ... | @@ -55,8 +56,8 @@ |
55 | 56 | return false; |
56 | 57 | |
57 | 58 | ip_proto = iph->nexthdr; |
58 | - flow->src = iph->saddr.s6_addr32[3]; | |
59 | - flow->dst = iph->daddr.s6_addr32[3]; | |
59 | + flow->src = (__force __be32)ipv6_addr_hash(&iph->saddr); | |
60 | + flow->dst = (__force __be32)ipv6_addr_hash(&iph->daddr); | |
60 | 61 | nhoff += sizeof(struct ipv6hdr); |
61 | 62 | break; |
62 | 63 | } |
net/ipv4/tcp_metrics.c
... | ... | @@ -211,10 +211,7 @@ |
211 | 211 | break; |
212 | 212 | case AF_INET6: |
213 | 213 | *(struct in6_addr *)addr.addr.a6 = inet6_rsk(req)->rmt_addr; |
214 | - hash = ((__force unsigned int) addr.addr.a6[0] ^ | |
215 | - (__force unsigned int) addr.addr.a6[1] ^ | |
216 | - (__force unsigned int) addr.addr.a6[2] ^ | |
217 | - (__force unsigned int) addr.addr.a6[3]); | |
214 | + hash = ipv6_addr_hash(&inet6_rsk(req)->rmt_addr); | |
218 | 215 | break; |
219 | 216 | default: |
220 | 217 | return NULL; |
... | ... | @@ -251,10 +248,7 @@ |
251 | 248 | case AF_INET6: |
252 | 249 | tw6 = inet6_twsk((struct sock *)tw); |
253 | 250 | *(struct in6_addr *)addr.addr.a6 = tw6->tw_v6_daddr; |
254 | - hash = ((__force unsigned int) addr.addr.a6[0] ^ | |
255 | - (__force unsigned int) addr.addr.a6[1] ^ | |
256 | - (__force unsigned int) addr.addr.a6[2] ^ | |
257 | - (__force unsigned int) addr.addr.a6[3]); | |
251 | + hash = ipv6_addr_hash(&tw6->tw_v6_daddr); | |
258 | 252 | break; |
259 | 253 | default: |
260 | 254 | return NULL; |
... | ... | @@ -291,10 +285,7 @@ |
291 | 285 | break; |
292 | 286 | case AF_INET6: |
293 | 287 | *(struct in6_addr *)addr.addr.a6 = inet6_sk(sk)->daddr; |
294 | - hash = ((__force unsigned int) addr.addr.a6[0] ^ | |
295 | - (__force unsigned int) addr.addr.a6[1] ^ | |
296 | - (__force unsigned int) addr.addr.a6[2] ^ | |
297 | - (__force unsigned int) addr.addr.a6[3]); | |
288 | + hash = ipv6_addr_hash(&inet6_sk(sk)->daddr); | |
298 | 289 | break; |
299 | 290 | default: |
300 | 291 | return NULL; |
net/ipv6/addrconf.c
... | ... | @@ -63,6 +63,7 @@ |
63 | 63 | #include <linux/delay.h> |
64 | 64 | #include <linux/notifier.h> |
65 | 65 | #include <linux/string.h> |
66 | +#include <linux/hash.h> | |
66 | 67 | |
67 | 68 | #include <net/net_namespace.h> |
68 | 69 | #include <net/sock.h> |
69 | 70 | |
... | ... | @@ -579,15 +580,9 @@ |
579 | 580 | list_add_tail(&ifp->if_list, p); |
580 | 581 | } |
581 | 582 | |
582 | -static u32 ipv6_addr_hash(const struct in6_addr *addr) | |
583 | +static u32 inet6_addr_hash(const struct in6_addr *addr) | |
583 | 584 | { |
584 | - /* | |
585 | - * We perform the hash function over the last 64 bits of the address | |
586 | - * This will include the IEEE address token on links that support it. | |
587 | - */ | |
588 | - return jhash_2words((__force u32)addr->s6_addr32[2], | |
589 | - (__force u32)addr->s6_addr32[3], 0) | |
590 | - & (IN6_ADDR_HSIZE - 1); | |
585 | + return hash_32(ipv6_addr_hash(addr), IN6_ADDR_HSIZE_SHIFT); | |
591 | 586 | } |
592 | 587 | |
593 | 588 | /* On success it returns ifp with increased reference count */ |
... | ... | @@ -662,7 +657,7 @@ |
662 | 657 | in6_ifa_hold(ifa); |
663 | 658 | |
664 | 659 | /* Add to big hash table */ |
665 | - hash = ipv6_addr_hash(addr); | |
660 | + hash = inet6_addr_hash(addr); | |
666 | 661 | |
667 | 662 | hlist_add_head_rcu(&ifa->addr_lst, &inet6_addr_lst[hash]); |
668 | 663 | spin_unlock(&addrconf_hash_lock); |
... | ... | @@ -1270,7 +1265,7 @@ |
1270 | 1265 | { |
1271 | 1266 | struct inet6_ifaddr *ifp; |
1272 | 1267 | struct hlist_node *node; |
1273 | - unsigned int hash = ipv6_addr_hash(addr); | |
1268 | + unsigned int hash = inet6_addr_hash(addr); | |
1274 | 1269 | |
1275 | 1270 | rcu_read_lock_bh(); |
1276 | 1271 | hlist_for_each_entry_rcu(ifp, node, &inet6_addr_lst[hash], addr_lst) { |
... | ... | @@ -1293,7 +1288,7 @@ |
1293 | 1288 | static bool ipv6_chk_same_addr(struct net *net, const struct in6_addr *addr, |
1294 | 1289 | struct net_device *dev) |
1295 | 1290 | { |
1296 | - unsigned int hash = ipv6_addr_hash(addr); | |
1291 | + unsigned int hash = inet6_addr_hash(addr); | |
1297 | 1292 | struct inet6_ifaddr *ifp; |
1298 | 1293 | struct hlist_node *node; |
1299 | 1294 | |
... | ... | @@ -1336,7 +1331,7 @@ |
1336 | 1331 | struct net_device *dev, int strict) |
1337 | 1332 | { |
1338 | 1333 | struct inet6_ifaddr *ifp, *result = NULL; |
1339 | - unsigned int hash = ipv6_addr_hash(addr); | |
1334 | + unsigned int hash = inet6_addr_hash(addr); | |
1340 | 1335 | struct hlist_node *node; |
1341 | 1336 | |
1342 | 1337 | rcu_read_lock_bh(); |
... | ... | @@ -3223,7 +3218,7 @@ |
3223 | 3218 | int ret = 0; |
3224 | 3219 | struct inet6_ifaddr *ifp = NULL; |
3225 | 3220 | struct hlist_node *n; |
3226 | - unsigned int hash = ipv6_addr_hash(addr); | |
3221 | + unsigned int hash = inet6_addr_hash(addr); | |
3227 | 3222 | |
3228 | 3223 | rcu_read_lock_bh(); |
3229 | 3224 | hlist_for_each_entry_rcu_bh(ifp, n, &inet6_addr_lst[hash], addr_lst) { |
net/ipv6/ip6_tunnel.c
... | ... | @@ -40,6 +40,7 @@ |
40 | 40 | #include <linux/rtnetlink.h> |
41 | 41 | #include <linux/netfilter_ipv6.h> |
42 | 42 | #include <linux/slab.h> |
43 | +#include <linux/hash.h> | |
43 | 44 | |
44 | 45 | #include <asm/uaccess.h> |
45 | 46 | #include <linux/atomic.h> |
46 | 47 | |
47 | 48 | |
... | ... | @@ -70,12 +71,16 @@ |
70 | 71 | #define IPV6_TCLASS_MASK (IPV6_FLOWINFO_MASK & ~IPV6_FLOWLABEL_MASK) |
71 | 72 | #define IPV6_TCLASS_SHIFT 20 |
72 | 73 | |
73 | -#define HASH_SIZE 32 | |
74 | +#define HASH_SIZE_SHIFT 5 | |
75 | +#define HASH_SIZE (1 << HASH_SIZE_SHIFT) | |
74 | 76 | |
75 | -#define HASH(addr) ((__force u32)((addr)->s6_addr32[0] ^ (addr)->s6_addr32[1] ^ \ | |
76 | - (addr)->s6_addr32[2] ^ (addr)->s6_addr32[3]) & \ | |
77 | - (HASH_SIZE - 1)) | |
77 | +static u32 HASH(const struct in6_addr *addr1, const struct in6_addr *addr2) | |
78 | +{ | |
79 | + u32 hash = ipv6_addr_hash(addr1) ^ ipv6_addr_hash(addr2); | |
78 | 80 | |
81 | + return hash_32(hash, HASH_SIZE_SHIFT); | |
82 | +} | |
83 | + | |
79 | 84 | static int ip6_tnl_dev_init(struct net_device *dev); |
80 | 85 | static void ip6_tnl_dev_setup(struct net_device *dev); |
81 | 86 | |
82 | 87 | |
... | ... | @@ -166,12 +171,11 @@ |
166 | 171 | static struct ip6_tnl * |
167 | 172 | ip6_tnl_lookup(struct net *net, const struct in6_addr *remote, const struct in6_addr *local) |
168 | 173 | { |
169 | - unsigned int h0 = HASH(remote); | |
170 | - unsigned int h1 = HASH(local); | |
174 | + unsigned int hash = HASH(remote, local); | |
171 | 175 | struct ip6_tnl *t; |
172 | 176 | struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id); |
173 | 177 | |
174 | - for_each_ip6_tunnel_rcu(ip6n->tnls_r_l[h0 ^ h1]) { | |
178 | + for_each_ip6_tunnel_rcu(ip6n->tnls_r_l[hash]) { | |
175 | 179 | if (ipv6_addr_equal(local, &t->parms.laddr) && |
176 | 180 | ipv6_addr_equal(remote, &t->parms.raddr) && |
177 | 181 | (t->dev->flags & IFF_UP)) |
... | ... | @@ -205,7 +209,7 @@ |
205 | 209 | |
206 | 210 | if (!ipv6_addr_any(remote) || !ipv6_addr_any(local)) { |
207 | 211 | prio = 1; |
208 | - h = HASH(remote) ^ HASH(local); | |
212 | + h = HASH(remote, local); | |
209 | 213 | } |
210 | 214 | return &ip6n->tnls[prio][h]; |
211 | 215 | } |
net/sunrpc/svcauth_unix.c
... | ... | @@ -104,24 +104,10 @@ |
104 | 104 | kfree(im); |
105 | 105 | } |
106 | 106 | |
107 | -#if IP_HASHBITS == 8 | |
108 | -/* hash_long on a 64 bit machine is currently REALLY BAD for | |
109 | - * IP addresses in reverse-endian (i.e. on a little-endian machine). | |
110 | - * So use a trivial but reliable hash instead | |
111 | - */ | |
112 | -static inline int hash_ip(__be32 ip) | |
107 | +static inline int hash_ip6(const struct in6_addr *ip) | |
113 | 108 | { |
114 | - int hash = (__force u32)ip ^ ((__force u32)ip>>16); | |
115 | - return (hash ^ (hash>>8)) & 0xff; | |
109 | + return hash_32(ipv6_addr_hash(ip), IP_HASHBITS); | |
116 | 110 | } |
117 | -#endif | |
118 | -static inline int hash_ip6(struct in6_addr ip) | |
119 | -{ | |
120 | - return (hash_ip(ip.s6_addr32[0]) ^ | |
121 | - hash_ip(ip.s6_addr32[1]) ^ | |
122 | - hash_ip(ip.s6_addr32[2]) ^ | |
123 | - hash_ip(ip.s6_addr32[3])); | |
124 | -} | |
125 | 111 | static int ip_map_match(struct cache_head *corig, struct cache_head *cnew) |
126 | 112 | { |
127 | 113 | struct ip_map *orig = container_of(corig, struct ip_map, h); |
... | ... | @@ -301,7 +287,7 @@ |
301 | 287 | ip.m_addr = *addr; |
302 | 288 | ch = sunrpc_cache_lookup(cd, &ip.h, |
303 | 289 | hash_str(class, IP_HASHBITS) ^ |
304 | - hash_ip6(*addr)); | |
290 | + hash_ip6(addr)); | |
305 | 291 | |
306 | 292 | if (ch) |
307 | 293 | return container_of(ch, struct ip_map, h); |
... | ... | @@ -331,7 +317,7 @@ |
331 | 317 | ip.h.expiry_time = expiry; |
332 | 318 | ch = sunrpc_cache_update(cd, &ip.h, &ipm->h, |
333 | 319 | hash_str(ipm->m_class, IP_HASHBITS) ^ |
334 | - hash_ip6(ipm->m_addr)); | |
320 | + hash_ip6(&ipm->m_addr)); | |
335 | 321 | if (!ch) |
336 | 322 | return -ENOMEM; |
337 | 323 | cache_put(ch, cd); |