Commit ddbe503203855939946430e39bae58de11b70b69

Authored by Eric Dumazet
Committed by David S. Miller
1 parent dc9059512c

ipv6: add ipv6_addr_hash() helper

Introduce ipv6_addr_hash() helper doing a XOR on all bits
of an IPv6 address, with an optimized x86_64 version.

Use it in flow dissector, as suggested by Andrew McGregor,
to reduce hash collision probabilities in fq_codel (and other
users of flow dissector)

Use it in ip6_tunnel.c and use more bit shuffling, as suggested
by David Laight, as existing hash was ignoring most of them.

Use it in sunrpc and use more bit shuffling, using hash_32().

Use it in net/ipv6/addrconf.c, using hash_32() as well.

As a cleanup, use it in net/ipv4/tcp_metrics.c

Signed-off-by: Eric Dumazet <edumazet@google.com>
Reported-by: Andrew McGregor <andrewmcgr@gmail.com>
Cc: Dave Taht <dave.taht@gmail.com>
Cc: Tom Herbert <therbert@google.com>
Cc: David Laight <David.Laight@ACULAB.COM>
Cc: Joe Perches <joe@perches.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

Showing 7 changed files with 45 additions and 54 deletions Side-by-side Diff

include/net/addrconf.h
... ... @@ -46,7 +46,8 @@
46 46 #include <net/if_inet6.h>
47 47 #include <net/ipv6.h>
48 48  
49   -#define IN6_ADDR_HSIZE 16
  49 +#define IN6_ADDR_HSIZE_SHIFT 4
  50 +#define IN6_ADDR_HSIZE (1 << IN6_ADDR_HSIZE_SHIFT)
50 51  
51 52 extern int addrconf_init(void);
52 53 extern void addrconf_cleanup(void);
... ... @@ -419,6 +419,19 @@
419 419 #endif
420 420 }
421 421  
  422 +static inline u32 ipv6_addr_hash(const struct in6_addr *a)
  423 +{
  424 +#if defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && BITS_PER_LONG == 64
  425 + const unsigned long *ul = (const unsigned long *)a;
  426 + unsigned long x = ul[0] ^ ul[1];
  427 +
  428 + return (u32)(x ^ (x >> 32));
  429 +#else
  430 + return (__force u32)(a->s6_addr32[0] ^ a->s6_addr32[1] ^
  431 + a->s6_addr32[2] ^ a->s6_addr32[3]);
  432 +#endif
  433 +}
  434 +
422 435 static inline bool ipv6_addr_loopback(const struct in6_addr *a)
423 436 {
424 437 return (a->s6_addr32[0] | a->s6_addr32[1] |
net/core/flow_dissector.c
... ... @@ -4,6 +4,7 @@
4 4 #include <linux/ipv6.h>
5 5 #include <linux/if_vlan.h>
6 6 #include <net/ip.h>
  7 +#include <net/ipv6.h>
7 8 #include <linux/if_tunnel.h>
8 9 #include <linux/if_pppox.h>
9 10 #include <linux/ppp_defs.h>
... ... @@ -55,8 +56,8 @@
55 56 return false;
56 57  
57 58 ip_proto = iph->nexthdr;
58   - flow->src = iph->saddr.s6_addr32[3];
59   - flow->dst = iph->daddr.s6_addr32[3];
  59 + flow->src = (__force __be32)ipv6_addr_hash(&iph->saddr);
  60 + flow->dst = (__force __be32)ipv6_addr_hash(&iph->daddr);
60 61 nhoff += sizeof(struct ipv6hdr);
61 62 break;
62 63 }
net/ipv4/tcp_metrics.c
... ... @@ -211,10 +211,7 @@
211 211 break;
212 212 case AF_INET6:
213 213 *(struct in6_addr *)addr.addr.a6 = inet6_rsk(req)->rmt_addr;
214   - hash = ((__force unsigned int) addr.addr.a6[0] ^
215   - (__force unsigned int) addr.addr.a6[1] ^
216   - (__force unsigned int) addr.addr.a6[2] ^
217   - (__force unsigned int) addr.addr.a6[3]);
  214 + hash = ipv6_addr_hash(&inet6_rsk(req)->rmt_addr);
218 215 break;
219 216 default:
220 217 return NULL;
... ... @@ -251,10 +248,7 @@
251 248 case AF_INET6:
252 249 tw6 = inet6_twsk((struct sock *)tw);
253 250 *(struct in6_addr *)addr.addr.a6 = tw6->tw_v6_daddr;
254   - hash = ((__force unsigned int) addr.addr.a6[0] ^
255   - (__force unsigned int) addr.addr.a6[1] ^
256   - (__force unsigned int) addr.addr.a6[2] ^
257   - (__force unsigned int) addr.addr.a6[3]);
  251 + hash = ipv6_addr_hash(&tw6->tw_v6_daddr);
258 252 break;
259 253 default:
260 254 return NULL;
... ... @@ -291,10 +285,7 @@
291 285 break;
292 286 case AF_INET6:
293 287 *(struct in6_addr *)addr.addr.a6 = inet6_sk(sk)->daddr;
294   - hash = ((__force unsigned int) addr.addr.a6[0] ^
295   - (__force unsigned int) addr.addr.a6[1] ^
296   - (__force unsigned int) addr.addr.a6[2] ^
297   - (__force unsigned int) addr.addr.a6[3]);
  288 + hash = ipv6_addr_hash(&inet6_sk(sk)->daddr);
298 289 break;
299 290 default:
300 291 return NULL;
... ... @@ -63,6 +63,7 @@
63 63 #include <linux/delay.h>
64 64 #include <linux/notifier.h>
65 65 #include <linux/string.h>
  66 +#include <linux/hash.h>
66 67  
67 68 #include <net/net_namespace.h>
68 69 #include <net/sock.h>
69 70  
... ... @@ -579,15 +580,9 @@
579 580 list_add_tail(&ifp->if_list, p);
580 581 }
581 582  
582   -static u32 ipv6_addr_hash(const struct in6_addr *addr)
  583 +static u32 inet6_addr_hash(const struct in6_addr *addr)
583 584 {
584   - /*
585   - * We perform the hash function over the last 64 bits of the address
586   - * This will include the IEEE address token on links that support it.
587   - */
588   - return jhash_2words((__force u32)addr->s6_addr32[2],
589   - (__force u32)addr->s6_addr32[3], 0)
590   - & (IN6_ADDR_HSIZE - 1);
  585 + return hash_32(ipv6_addr_hash(addr), IN6_ADDR_HSIZE_SHIFT);
591 586 }
592 587  
593 588 /* On success it returns ifp with increased reference count */
... ... @@ -662,7 +657,7 @@
662 657 in6_ifa_hold(ifa);
663 658  
664 659 /* Add to big hash table */
665   - hash = ipv6_addr_hash(addr);
  660 + hash = inet6_addr_hash(addr);
666 661  
667 662 hlist_add_head_rcu(&ifa->addr_lst, &inet6_addr_lst[hash]);
668 663 spin_unlock(&addrconf_hash_lock);
... ... @@ -1270,7 +1265,7 @@
1270 1265 {
1271 1266 struct inet6_ifaddr *ifp;
1272 1267 struct hlist_node *node;
1273   - unsigned int hash = ipv6_addr_hash(addr);
  1268 + unsigned int hash = inet6_addr_hash(addr);
1274 1269  
1275 1270 rcu_read_lock_bh();
1276 1271 hlist_for_each_entry_rcu(ifp, node, &inet6_addr_lst[hash], addr_lst) {
... ... @@ -1293,7 +1288,7 @@
1293 1288 static bool ipv6_chk_same_addr(struct net *net, const struct in6_addr *addr,
1294 1289 struct net_device *dev)
1295 1290 {
1296   - unsigned int hash = ipv6_addr_hash(addr);
  1291 + unsigned int hash = inet6_addr_hash(addr);
1297 1292 struct inet6_ifaddr *ifp;
1298 1293 struct hlist_node *node;
1299 1294  
... ... @@ -1336,7 +1331,7 @@
1336 1331 struct net_device *dev, int strict)
1337 1332 {
1338 1333 struct inet6_ifaddr *ifp, *result = NULL;
1339   - unsigned int hash = ipv6_addr_hash(addr);
  1334 + unsigned int hash = inet6_addr_hash(addr);
1340 1335 struct hlist_node *node;
1341 1336  
1342 1337 rcu_read_lock_bh();
... ... @@ -3223,7 +3218,7 @@
3223 3218 int ret = 0;
3224 3219 struct inet6_ifaddr *ifp = NULL;
3225 3220 struct hlist_node *n;
3226   - unsigned int hash = ipv6_addr_hash(addr);
  3221 + unsigned int hash = inet6_addr_hash(addr);
3227 3222  
3228 3223 rcu_read_lock_bh();
3229 3224 hlist_for_each_entry_rcu_bh(ifp, n, &inet6_addr_lst[hash], addr_lst) {
net/ipv6/ip6_tunnel.c
... ... @@ -40,6 +40,7 @@
40 40 #include <linux/rtnetlink.h>
41 41 #include <linux/netfilter_ipv6.h>
42 42 #include <linux/slab.h>
  43 +#include <linux/hash.h>
43 44  
44 45 #include <asm/uaccess.h>
45 46 #include <linux/atomic.h>
46 47  
47 48  
... ... @@ -70,12 +71,16 @@
70 71 #define IPV6_TCLASS_MASK (IPV6_FLOWINFO_MASK & ~IPV6_FLOWLABEL_MASK)
71 72 #define IPV6_TCLASS_SHIFT 20
72 73  
73   -#define HASH_SIZE 32
  74 +#define HASH_SIZE_SHIFT 5
  75 +#define HASH_SIZE (1 << HASH_SIZE_SHIFT)
74 76  
75   -#define HASH(addr) ((__force u32)((addr)->s6_addr32[0] ^ (addr)->s6_addr32[1] ^ \
76   - (addr)->s6_addr32[2] ^ (addr)->s6_addr32[3]) & \
77   - (HASH_SIZE - 1))
  77 +static u32 HASH(const struct in6_addr *addr1, const struct in6_addr *addr2)
  78 +{
  79 + u32 hash = ipv6_addr_hash(addr1) ^ ipv6_addr_hash(addr2);
78 80  
  81 + return hash_32(hash, HASH_SIZE_SHIFT);
  82 +}
  83 +
79 84 static int ip6_tnl_dev_init(struct net_device *dev);
80 85 static void ip6_tnl_dev_setup(struct net_device *dev);
81 86  
82 87  
... ... @@ -166,12 +171,11 @@
166 171 static struct ip6_tnl *
167 172 ip6_tnl_lookup(struct net *net, const struct in6_addr *remote, const struct in6_addr *local)
168 173 {
169   - unsigned int h0 = HASH(remote);
170   - unsigned int h1 = HASH(local);
  174 + unsigned int hash = HASH(remote, local);
171 175 struct ip6_tnl *t;
172 176 struct ip6_tnl_net *ip6n = net_generic(net, ip6_tnl_net_id);
173 177  
174   - for_each_ip6_tunnel_rcu(ip6n->tnls_r_l[h0 ^ h1]) {
  178 + for_each_ip6_tunnel_rcu(ip6n->tnls_r_l[hash]) {
175 179 if (ipv6_addr_equal(local, &t->parms.laddr) &&
176 180 ipv6_addr_equal(remote, &t->parms.raddr) &&
177 181 (t->dev->flags & IFF_UP))
... ... @@ -205,7 +209,7 @@
205 209  
206 210 if (!ipv6_addr_any(remote) || !ipv6_addr_any(local)) {
207 211 prio = 1;
208   - h = HASH(remote) ^ HASH(local);
  212 + h = HASH(remote, local);
209 213 }
210 214 return &ip6n->tnls[prio][h];
211 215 }
net/sunrpc/svcauth_unix.c
... ... @@ -104,24 +104,10 @@
104 104 kfree(im);
105 105 }
106 106  
107   -#if IP_HASHBITS == 8
108   -/* hash_long on a 64 bit machine is currently REALLY BAD for
109   - * IP addresses in reverse-endian (i.e. on a little-endian machine).
110   - * So use a trivial but reliable hash instead
111   - */
112   -static inline int hash_ip(__be32 ip)
  107 +static inline int hash_ip6(const struct in6_addr *ip)
113 108 {
114   - int hash = (__force u32)ip ^ ((__force u32)ip>>16);
115   - return (hash ^ (hash>>8)) & 0xff;
  109 + return hash_32(ipv6_addr_hash(ip), IP_HASHBITS);
116 110 }
117   -#endif
118   -static inline int hash_ip6(struct in6_addr ip)
119   -{
120   - return (hash_ip(ip.s6_addr32[0]) ^
121   - hash_ip(ip.s6_addr32[1]) ^
122   - hash_ip(ip.s6_addr32[2]) ^
123   - hash_ip(ip.s6_addr32[3]));
124   -}
125 111 static int ip_map_match(struct cache_head *corig, struct cache_head *cnew)
126 112 {
127 113 struct ip_map *orig = container_of(corig, struct ip_map, h);
... ... @@ -301,7 +287,7 @@
301 287 ip.m_addr = *addr;
302 288 ch = sunrpc_cache_lookup(cd, &ip.h,
303 289 hash_str(class, IP_HASHBITS) ^
304   - hash_ip6(*addr));
  290 + hash_ip6(addr));
305 291  
306 292 if (ch)
307 293 return container_of(ch, struct ip_map, h);
... ... @@ -331,7 +317,7 @@
331 317 ip.h.expiry_time = expiry;
332 318 ch = sunrpc_cache_update(cd, &ip.h, &ipm->h,
333 319 hash_str(ipm->m_class, IP_HASHBITS) ^
334   - hash_ip6(ipm->m_addr));
  320 + hash_ip6(&ipm->m_addr));
335 321 if (!ch)
336 322 return -ENOMEM;
337 323 cache_put(ch, cd);