Eric Lee / smarc-fsl-linux-kernel

Commit 08733a0cb7decce40bbbd0331a0449465f13c444

Authored by Pablo Neira Ayuso 2016-11-03 17:56:43 +0800

netfilter: handle NF_REPEAT from nf_conntrack_in()

NF_REPEAT is only needed from nf_conntrack_in() under a very specific
case required by the TCP protocol tracker, we can handle this case
without returning to the core hook path. Handling of NF_REPEAT from the
nf_reinject() is left untouched.

Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>

Showing 3 changed files with 8 additions and 13 deletions Inline Diff

net/netfilter/core.c
net/netfilter/nf_conntrack_core.c
net/openvswitch/conntrack.c

net/netfilter/core.c

Diff comments View file @ 08733a0

 /* netfilter.c: look after the filters for various protocols.
  * Heavily influenced by the old firewall.c by David Bonn and Alan Cox.
  *
  * Thanks to Rob `CmdrTaco' Malda for not influencing this code in any
  * way.
  *
  * Rusty Russell (C)2000 -- This code is GPL.
  * Patrick McHardy (c) 2006-2012
  */
 #include <linux/kernel.h>
 #include <linux/netfilter.h>
 #include <net/protocol.h>
 #include <linux/init.h>
 #include <linux/skbuff.h>
 #include <linux/wait.h>
 #include <linux/module.h>
 #include <linux/interrupt.h>
 #include <linux/if.h>
 #include <linux/netdevice.h>
 #include <linux/netfilter_ipv6.h>
 #include <linux/inetdevice.h>
 #include <linux/proc_fs.h>
 #include <linux/mutex.h>
 #include <linux/slab.h>
 #include <linux/rcupdate.h>
 #include <net/net_namespace.h>
 #include <net/sock.h>
 #include "nf_internals.h"
 static DEFINE_MUTEX(afinfo_mutex);
 const struct nf_afinfo __rcu *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly;
 EXPORT_SYMBOL(nf_afinfo);
 const struct nf_ipv6_ops __rcu *nf_ipv6_ops __read_mostly;
 EXPORT_SYMBOL_GPL(nf_ipv6_ops);
 DEFINE_PER_CPU(bool, nf_skb_duplicated);
 EXPORT_SYMBOL_GPL(nf_skb_duplicated);
 int nf_register_afinfo(const struct nf_afinfo *afinfo)
 {
 	mutex_lock(&afinfo_mutex);
 	RCU_INIT_POINTER(nf_afinfo[afinfo->family], afinfo);
 	mutex_unlock(&afinfo_mutex);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nf_register_afinfo);
 void nf_unregister_afinfo(const struct nf_afinfo *afinfo)
 {
 	mutex_lock(&afinfo_mutex);
 	RCU_INIT_POINTER(nf_afinfo[afinfo->family], NULL);
 	mutex_unlock(&afinfo_mutex);
 	synchronize_rcu();
 }
 EXPORT_SYMBOL_GPL(nf_unregister_afinfo);
 #ifdef HAVE_JUMP_LABEL
 struct static_key nf_hooks_needed[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
 EXPORT_SYMBOL(nf_hooks_needed);
 #endif
 static DEFINE_MUTEX(nf_hook_mutex);
 #define nf_entry_dereference(e) \
 	rcu_dereference_protected(e, lockdep_is_held(&nf_hook_mutex))
 static struct nf_hook_entry __rcu **nf_hook_entry_head(struct net *net, const struct nf_hook_ops *reg)
 {
 	if (reg->pf != NFPROTO_NETDEV)
 		return net->nf.hooks[reg->pf]+reg->hooknum;
 #ifdef CONFIG_NETFILTER_INGRESS
 	if (reg->hooknum == NF_NETDEV_INGRESS) {
 		if (reg->dev && dev_net(reg->dev) == net)
 			return &reg->dev->nf_hooks_ingress;
 	}
 #endif
 	return NULL;
 }
 int nf_register_net_hook(struct net *net, const struct nf_hook_ops *reg)
 {
 	struct nf_hook_entry __rcu **pp;
 	struct nf_hook_entry *entry, *p;
 	if (reg->pf == NFPROTO_NETDEV) {
 #ifndef CONFIG_NETFILTER_INGRESS
 		if (reg->hooknum == NF_NETDEV_INGRESS)
 			return -EOPNOTSUPP;
 #endif
 		if (reg->hooknum != NF_NETDEV_INGRESS ||
 		    !reg->dev || dev_net(reg->dev) != net)
 			return -EINVAL;
 	}
 	pp = nf_hook_entry_head(net, reg);
 	if (!pp)
 		return -EINVAL;
 	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
 	if (!entry)
 		return -ENOMEM;
 	entry->orig_ops	= reg;
 	entry->ops	= *reg;
 	entry->next	= NULL;
 	mutex_lock(&nf_hook_mutex);
 	/* Find the spot in the list */
 	while ((p = nf_entry_dereference(*pp)) != NULL) {
 		if (reg->priority < p->orig_ops->priority)
 			break;
 		pp = &p->next;
 	}
 	rcu_assign_pointer(entry->next, p);
 	rcu_assign_pointer(*pp, entry);
 	mutex_unlock(&nf_hook_mutex);
 #ifdef CONFIG_NETFILTER_INGRESS
 	if (reg->pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS)
 		net_inc_ingress_queue();
 #endif
 #ifdef HAVE_JUMP_LABEL
 	static_key_slow_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
 #endif
 	return 0;
 }
 EXPORT_SYMBOL(nf_register_net_hook);
 void nf_unregister_net_hook(struct net *net, const struct nf_hook_ops *reg)
 {
 	struct nf_hook_entry __rcu **pp;
 	struct nf_hook_entry *p;
 	pp = nf_hook_entry_head(net, reg);
 	if (WARN_ON_ONCE(!pp))
 		return;
 	mutex_lock(&nf_hook_mutex);
 	while ((p = nf_entry_dereference(*pp)) != NULL) {
 		if (p->orig_ops == reg) {
 			rcu_assign_pointer(*pp, p->next);
 			break;
 		}
 		pp = &p->next;
 	}
 	mutex_unlock(&nf_hook_mutex);
 	if (!p) {
 		WARN(1, "nf_unregister_net_hook: hook not found!\n");
 		return;
 	}
 #ifdef CONFIG_NETFILTER_INGRESS
 	if (reg->pf == NFPROTO_NETDEV && reg->hooknum == NF_NETDEV_INGRESS)
 		net_dec_ingress_queue();
 #endif
 #ifdef HAVE_JUMP_LABEL
 	static_key_slow_dec(&nf_hooks_needed[reg->pf][reg->hooknum]);
 #endif
 	synchronize_net();
 	nf_queue_nf_hook_drop(net, p);
 	/* other cpu might still process nfqueue verdict that used reg */
 	synchronize_net();
 	kfree(p);
 }
 EXPORT_SYMBOL(nf_unregister_net_hook);
 int nf_register_net_hooks(struct net *net, const struct nf_hook_ops *reg,
 			  unsigned int n)
 {
 	unsigned int i;
 	int err = 0;
 	for (i = 0; i < n; i++) {
 		err = nf_register_net_hook(net, &reg[i]);
 		if (err)
 			goto err;
 	}
 	return err;
 err:
 	if (i > 0)
 		nf_unregister_net_hooks(net, reg, i);
 	return err;
 }
 EXPORT_SYMBOL(nf_register_net_hooks);
 void nf_unregister_net_hooks(struct net *net, const struct nf_hook_ops *reg,
 			     unsigned int n)
 {
 	while (n-- > 0)
 		nf_unregister_net_hook(net, &reg[n]);
 }
 EXPORT_SYMBOL(nf_unregister_net_hooks);
 static LIST_HEAD(nf_hook_list);
 static int _nf_register_hook(struct nf_hook_ops *reg)
 {
 	struct net *net, *last;
 	int ret;
 	for_each_net(net) {
 		ret = nf_register_net_hook(net, reg);
 		if (ret && ret != -ENOENT)
 			goto rollback;
 	}
 	list_add_tail(&reg->list, &nf_hook_list);
 	return 0;
 rollback:
 	last = net;
 	for_each_net(net) {
 		if (net == last)
 			break;
 		nf_unregister_net_hook(net, reg);
 	}
 	return ret;
 }
 int nf_register_hook(struct nf_hook_ops *reg)
 {
 	int ret;
 	rtnl_lock();
 	ret = _nf_register_hook(reg);
 	rtnl_unlock();
 	return ret;
 }
 EXPORT_SYMBOL(nf_register_hook);
 static void _nf_unregister_hook(struct nf_hook_ops *reg)
 {
 	struct net *net;
 	list_del(&reg->list);
 	for_each_net(net)
 		nf_unregister_net_hook(net, reg);
 }
 void nf_unregister_hook(struct nf_hook_ops *reg)
 {
 	rtnl_lock();
 	_nf_unregister_hook(reg);
 	rtnl_unlock();
 }
 EXPORT_SYMBOL(nf_unregister_hook);
 int nf_register_hooks(struct nf_hook_ops *reg, unsigned int n)
 {
 	unsigned int i;
 	int err = 0;
 	for (i = 0; i < n; i++) {
 		err = nf_register_hook(&reg[i]);
 		if (err)
 			goto err;
 	}
 	return err;
 err:
 	if (i > 0)
 		nf_unregister_hooks(reg, i);
 	return err;
 }
 EXPORT_SYMBOL(nf_register_hooks);
 /* Caller MUST take rtnl_lock() */
 int _nf_register_hooks(struct nf_hook_ops *reg, unsigned int n)
 {
 	unsigned int i;
 	int err = 0;
 	for (i = 0; i < n; i++) {
 		err = _nf_register_hook(&reg[i]);
 		if (err)
 			goto err;
 	}
 	return err;
 err:
 	if (i > 0)
 		_nf_unregister_hooks(reg, i);
 	return err;
 }
 EXPORT_SYMBOL(_nf_register_hooks);
 void nf_unregister_hooks(struct nf_hook_ops *reg, unsigned int n)
 {
 	while (n-- > 0)
 		nf_unregister_hook(&reg[n]);
 }
 EXPORT_SYMBOL(nf_unregister_hooks);
 /* Caller MUST take rtnl_lock */
 void _nf_unregister_hooks(struct nf_hook_ops *reg, unsigned int n)
 {
 	while (n-- > 0)
 		_nf_unregister_hook(&reg[n]);
 }
 EXPORT_SYMBOL(_nf_unregister_hooks);
 /* Returns 1 if okfn() needs to be executed by the caller,
  * -EPERM for NF_DROP, 0 otherwise.  Caller must hold rcu_read_lock. */
 int nf_hook_slow(struct sk_buff *skb, struct nf_hook_state *state,
 		 struct nf_hook_entry *entry)
 {
 	unsigned int verdict;
 	int ret;
 	do {
 		verdict = entry->ops.hook(entry->ops.priv, skb, state);
 		switch (verdict & NF_VERDICT_MASK) {
 		case NF_ACCEPT:
 			entry = rcu_dereference(entry->next);
 			break;
 		case NF_DROP:
 			kfree_skb(skb);
 			ret = NF_DROP_GETERR(verdict);
 			if (ret == 0)
 				ret = -EPERM;
 			return ret;
-		case NF_REPEAT:
-			continue;
 		case NF_QUEUE:
 			ret = nf_queue(skb, state, &entry, verdict);
 			if (ret == 1 && entry)
 				continue;
 			return ret;
 		default:
 			/* Implicit handling for NF_STOLEN, as well as any other
 			 * non conventional verdicts.
 			 */
 			return 0;
 		}
 	} while (entry);
 	return 1;
 }
 EXPORT_SYMBOL(nf_hook_slow);
 int skb_make_writable(struct sk_buff *skb, unsigned int writable_len)
 {
 	if (writable_len > skb->len)
 		return 0;
 	/* Not exclusive use of packet?  Must copy. */
 	if (!skb_cloned(skb)) {
 		if (writable_len <= skb_headlen(skb))
 			return 1;
 	} else if (skb_clone_writable(skb, writable_len))
 		return 1;
 	if (writable_len <= skb_headlen(skb))
 		writable_len = 0;
 	else
 		writable_len -= skb_headlen(skb);
 	return !!__pskb_pull_tail(skb, writable_len);
 }
 EXPORT_SYMBOL(skb_make_writable);
 /* This needs to be compiled in any case to avoid dependencies between the
  * nfnetlink_queue code and nf_conntrack.
  */
 struct nfnl_ct_hook __rcu *nfnl_ct_hook __read_mostly;
 EXPORT_SYMBOL_GPL(nfnl_ct_hook);
 #if IS_ENABLED(CONFIG_NF_CONNTRACK)
 /* This does not belong here, but locally generated errors need it if connection
    tracking in use: without this, connection may not be in hash table, and hence
    manufactured ICMP or RST packets will not be associated with it. */
 void (*ip_ct_attach)(struct sk_buff *, const struct sk_buff *)
 		__rcu __read_mostly;
 EXPORT_SYMBOL(ip_ct_attach);
 void nf_ct_attach(struct sk_buff *new, const struct sk_buff *skb)
 {
 	void (*attach)(struct sk_buff *, const struct sk_buff *);
 	if (skb->nfct) {
 		rcu_read_lock();
 		attach = rcu_dereference(ip_ct_attach);
 		if (attach)
 			attach(new, skb);
 		rcu_read_unlock();
 	}
 }
 EXPORT_SYMBOL(nf_ct_attach);
 void (*nf_ct_destroy)(struct nf_conntrack *) __rcu __read_mostly;
 EXPORT_SYMBOL(nf_ct_destroy);
 void nf_conntrack_destroy(struct nf_conntrack *nfct)
 {
 	void (*destroy)(struct nf_conntrack *);
 	rcu_read_lock();
 	destroy = rcu_dereference(nf_ct_destroy);
 	BUG_ON(destroy == NULL);
 	destroy(nfct);
 	rcu_read_unlock();
 }
 EXPORT_SYMBOL(nf_conntrack_destroy);
 /* Built-in default zone used e.g. by modules. */
 const struct nf_conntrack_zone nf_ct_zone_dflt = {
 	.id	= NF_CT_DEFAULT_ZONE_ID,
 	.dir	= NF_CT_DEFAULT_ZONE_DIR,
 };
 EXPORT_SYMBOL_GPL(nf_ct_zone_dflt);
 #endif /* CONFIG_NF_CONNTRACK */
 #ifdef CONFIG_NF_NAT_NEEDED
 void (*nf_nat_decode_session_hook)(struct sk_buff *, struct flowi *);
 EXPORT_SYMBOL(nf_nat_decode_session_hook);
 #endif
 static int nf_register_hook_list(struct net *net)
 {
 	struct nf_hook_ops *elem;
 	int ret;
 	rtnl_lock();
 	list_for_each_entry(elem, &nf_hook_list, list) {
 		ret = nf_register_net_hook(net, elem);
 		if (ret && ret != -ENOENT)
 			goto out_undo;
 	}
 	rtnl_unlock();
 	return 0;
 out_undo:
 	list_for_each_entry_continue_reverse(elem, &nf_hook_list, list)
 		nf_unregister_net_hook(net, elem);
 	rtnl_unlock();
 	return ret;
 }
 static void nf_unregister_hook_list(struct net *net)
 {
 	struct nf_hook_ops *elem;
 	rtnl_lock();
 	list_for_each_entry(elem, &nf_hook_list, list)
 		nf_unregister_net_hook(net, elem);
 	rtnl_unlock();
 }
 static int __net_init netfilter_net_init(struct net *net)
 {
 	int i, h, ret;
 	for (i = 0; i < ARRAY_SIZE(net->nf.hooks); i++) {
 		for (h = 0; h < NF_MAX_HOOKS; h++)
 			RCU_INIT_POINTER(net->nf.hooks[i][h], NULL);
 	}
 #ifdef CONFIG_PROC_FS
 	net->nf.proc_netfilter = proc_net_mkdir(net, "netfilter",
 						net->proc_net);
 	if (!net->nf.proc_netfilter) {
 		if (!net_eq(net, &init_net))
 			pr_err("cannot create netfilter proc entry");
 		return -ENOMEM;
 	}
 #endif
 	ret = nf_register_hook_list(net);
 	if (ret)
 		remove_proc_entry("netfilter", net->proc_net);
 	return ret;
 }
 static void __net_exit netfilter_net_exit(struct net *net)
 {
 	nf_unregister_hook_list(net);
 	remove_proc_entry("netfilter", net->proc_net);
 }
 static struct pernet_operations netfilter_net_ops = {
 	.init = netfilter_net_init,
 	.exit = netfilter_net_exit,
 };
 int __init netfilter_init(void)
 {
 	int ret;
 	ret = register_pernet_subsys(&netfilter_net_ops);
 	if (ret < 0)
 		goto err;
 	ret = netfilter_log_init();
 	if (ret < 0)
 		goto err_pernet;
 	return 0;
 err_pernet:
 	unregister_pernet_subsys(&netfilter_net_ops);
 err:
 	return ret;
 }

net/netfilter/nf_conntrack_core.c

Diff comments View file @ 08733a0

1	/* Connection state tracking for netfilter. This is separated from,	1	/* Connection state tracking for netfilter. This is separated from,
2	but required by, the NAT layer; it can also be used by an iptables	2	but required by, the NAT layer; it can also be used by an iptables
3	extension. */	3	extension. */
4		4
5	/* (C) 1999-2001 Paul `Rusty' Russell	5	/* (C) 1999-2001 Paul `Rusty' Russell
6	* (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>	6	* (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
7	* (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>	7	* (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
8	* (C) 2005-2012 Patrick McHardy <kaber@trash.net>	8	* (C) 2005-2012 Patrick McHardy <kaber@trash.net>
9	*	9	*
10	* This program is free software; you can redistribute it and/or modify	10	* This program is free software; you can redistribute it and/or modify
11	* it under the terms of the GNU General Public License version 2 as	11	* it under the terms of the GNU General Public License version 2 as
12	* published by the Free Software Foundation.	12	* published by the Free Software Foundation.
13	*/	13	*/
14		14
15	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt	15	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16		16
17	#include <linux/types.h>	17	#include <linux/types.h>
18	#include <linux/netfilter.h>	18	#include <linux/netfilter.h>
19	#include <linux/module.h>	19	#include <linux/module.h>
20	#include <linux/sched.h>	20	#include <linux/sched.h>
21	#include <linux/skbuff.h>	21	#include <linux/skbuff.h>
22	#include <linux/proc_fs.h>	22	#include <linux/proc_fs.h>
23	#include <linux/vmalloc.h>	23	#include <linux/vmalloc.h>
24	#include <linux/stddef.h>	24	#include <linux/stddef.h>
25	#include <linux/slab.h>	25	#include <linux/slab.h>
26	#include <linux/random.h>	26	#include <linux/random.h>
27	#include <linux/jhash.h>	27	#include <linux/jhash.h>
28	#include <linux/err.h>	28	#include <linux/err.h>
29	#include <linux/percpu.h>	29	#include <linux/percpu.h>
30	#include <linux/moduleparam.h>	30	#include <linux/moduleparam.h>
31	#include <linux/notifier.h>	31	#include <linux/notifier.h>
32	#include <linux/kernel.h>	32	#include <linux/kernel.h>
33	#include <linux/netdevice.h>	33	#include <linux/netdevice.h>
34	#include <linux/socket.h>	34	#include <linux/socket.h>
35	#include <linux/mm.h>	35	#include <linux/mm.h>
36	#include <linux/nsproxy.h>	36	#include <linux/nsproxy.h>
37	#include <linux/rculist_nulls.h>	37	#include <linux/rculist_nulls.h>
38		38
39	#include <net/netfilter/nf_conntrack.h>	39	#include <net/netfilter/nf_conntrack.h>
40	#include <net/netfilter/nf_conntrack_l3proto.h>	40	#include <net/netfilter/nf_conntrack_l3proto.h>
41	#include <net/netfilter/nf_conntrack_l4proto.h>	41	#include <net/netfilter/nf_conntrack_l4proto.h>
42	#include <net/netfilter/nf_conntrack_expect.h>	42	#include <net/netfilter/nf_conntrack_expect.h>
43	#include <net/netfilter/nf_conntrack_helper.h>	43	#include <net/netfilter/nf_conntrack_helper.h>
44	#include <net/netfilter/nf_conntrack_seqadj.h>	44	#include <net/netfilter/nf_conntrack_seqadj.h>
45	#include <net/netfilter/nf_conntrack_core.h>	45	#include <net/netfilter/nf_conntrack_core.h>
46	#include <net/netfilter/nf_conntrack_extend.h>	46	#include <net/netfilter/nf_conntrack_extend.h>
47	#include <net/netfilter/nf_conntrack_acct.h>	47	#include <net/netfilter/nf_conntrack_acct.h>
48	#include <net/netfilter/nf_conntrack_ecache.h>	48	#include <net/netfilter/nf_conntrack_ecache.h>
49	#include <net/netfilter/nf_conntrack_zones.h>	49	#include <net/netfilter/nf_conntrack_zones.h>
50	#include <net/netfilter/nf_conntrack_timestamp.h>	50	#include <net/netfilter/nf_conntrack_timestamp.h>
51	#include <net/netfilter/nf_conntrack_timeout.h>	51	#include <net/netfilter/nf_conntrack_timeout.h>
52	#include <net/netfilter/nf_conntrack_labels.h>	52	#include <net/netfilter/nf_conntrack_labels.h>
53	#include <net/netfilter/nf_conntrack_synproxy.h>	53	#include <net/netfilter/nf_conntrack_synproxy.h>
54	#include <net/netfilter/nf_nat.h>	54	#include <net/netfilter/nf_nat.h>
55	#include <net/netfilter/nf_nat_core.h>	55	#include <net/netfilter/nf_nat_core.h>
56	#include <net/netfilter/nf_nat_helper.h>	56	#include <net/netfilter/nf_nat_helper.h>
57	#include <net/netns/hash.h>	57	#include <net/netns/hash.h>
58		58
59	#define NF_CONNTRACK_VERSION "0.5.0"	59	#define NF_CONNTRACK_VERSION "0.5.0"
60		60
61	int (nfnetlink_parse_nat_setup_hook)(struct nf_conn ct,	61	int (nfnetlink_parse_nat_setup_hook)(struct nf_conn ct,
62	enum nf_nat_manip_type manip,	62	enum nf_nat_manip_type manip,
63	const struct nlattr *attr) __read_mostly;	63	const struct nlattr *attr) __read_mostly;
64	EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);	64	EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
65		65
66	__cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];	66	__cacheline_aligned_in_smp spinlock_t nf_conntrack_locks[CONNTRACK_LOCKS];
67	EXPORT_SYMBOL_GPL(nf_conntrack_locks);	67	EXPORT_SYMBOL_GPL(nf_conntrack_locks);
68		68
69	__cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);	69	__cacheline_aligned_in_smp DEFINE_SPINLOCK(nf_conntrack_expect_lock);
70	EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);	70	EXPORT_SYMBOL_GPL(nf_conntrack_expect_lock);
71		71
72	struct hlist_nulls_head *nf_conntrack_hash __read_mostly;	72	struct hlist_nulls_head *nf_conntrack_hash __read_mostly;
73	EXPORT_SYMBOL_GPL(nf_conntrack_hash);	73	EXPORT_SYMBOL_GPL(nf_conntrack_hash);
74		74
75	struct conntrack_gc_work {	75	struct conntrack_gc_work {
76	struct delayed_work dwork;	76	struct delayed_work dwork;
77	u32 last_bucket;	77	u32 last_bucket;
78	bool exiting;	78	bool exiting;
79	};	79	};
80		80
81	static __read_mostly struct kmem_cache *nf_conntrack_cachep;	81	static __read_mostly struct kmem_cache *nf_conntrack_cachep;
82	static __read_mostly spinlock_t nf_conntrack_locks_all_lock;	82	static __read_mostly spinlock_t nf_conntrack_locks_all_lock;
83	static __read_mostly DEFINE_SPINLOCK(nf_conntrack_locks_all_lock);	83	static __read_mostly DEFINE_SPINLOCK(nf_conntrack_locks_all_lock);
84	static __read_mostly bool nf_conntrack_locks_all;	84	static __read_mostly bool nf_conntrack_locks_all;
85		85
86	#define GC_MAX_BUCKETS_DIV 64u	86	#define GC_MAX_BUCKETS_DIV 64u
87	#define GC_MAX_BUCKETS 8192u	87	#define GC_MAX_BUCKETS 8192u
88	#define GC_INTERVAL (5 * HZ)	88	#define GC_INTERVAL (5 * HZ)
89	#define GC_MAX_EVICTS 256u	89	#define GC_MAX_EVICTS 256u
90		90
91	static struct conntrack_gc_work conntrack_gc_work;	91	static struct conntrack_gc_work conntrack_gc_work;
92		92
93	void nf_conntrack_lock(spinlock_t *lock) __acquires(lock)	93	void nf_conntrack_lock(spinlock_t *lock) __acquires(lock)
94	{	94	{
95	spin_lock(lock);	95	spin_lock(lock);
96	while (unlikely(nf_conntrack_locks_all)) {	96	while (unlikely(nf_conntrack_locks_all)) {
97	spin_unlock(lock);	97	spin_unlock(lock);
98		98
99	/*	99	/*
100	* Order the 'nf_conntrack_locks_all' load vs. the	100	* Order the 'nf_conntrack_locks_all' load vs. the
101	* spin_unlock_wait() loads below, to ensure	101	* spin_unlock_wait() loads below, to ensure
102	* that 'nf_conntrack_locks_all_lock' is indeed held:	102	* that 'nf_conntrack_locks_all_lock' is indeed held:
103	*/	103	*/
104	smp_rmb(); /* spin_lock(&nf_conntrack_locks_all_lock) */	104	smp_rmb(); /* spin_lock(&nf_conntrack_locks_all_lock) */
105	spin_unlock_wait(&nf_conntrack_locks_all_lock);	105	spin_unlock_wait(&nf_conntrack_locks_all_lock);
106	spin_lock(lock);	106	spin_lock(lock);
107	}	107	}
108	}	108	}
109	EXPORT_SYMBOL_GPL(nf_conntrack_lock);	109	EXPORT_SYMBOL_GPL(nf_conntrack_lock);
110		110
111	static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)	111	static void nf_conntrack_double_unlock(unsigned int h1, unsigned int h2)
112	{	112	{
113	h1 %= CONNTRACK_LOCKS;	113	h1 %= CONNTRACK_LOCKS;
114	h2 %= CONNTRACK_LOCKS;	114	h2 %= CONNTRACK_LOCKS;
115	spin_unlock(&nf_conntrack_locks[h1]);	115	spin_unlock(&nf_conntrack_locks[h1]);
116	if (h1 != h2)	116	if (h1 != h2)
117	spin_unlock(&nf_conntrack_locks[h2]);	117	spin_unlock(&nf_conntrack_locks[h2]);
118	}	118	}
119		119
120	/* return true if we need to recompute hashes (in case hash table was resized) */	120	/* return true if we need to recompute hashes (in case hash table was resized) */
121	static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,	121	static bool nf_conntrack_double_lock(struct net *net, unsigned int h1,
122	unsigned int h2, unsigned int sequence)	122	unsigned int h2, unsigned int sequence)
123	{	123	{
124	h1 %= CONNTRACK_LOCKS;	124	h1 %= CONNTRACK_LOCKS;
125	h2 %= CONNTRACK_LOCKS;	125	h2 %= CONNTRACK_LOCKS;
126	if (h1 <= h2) {	126	if (h1 <= h2) {
127	nf_conntrack_lock(&nf_conntrack_locks[h1]);	127	nf_conntrack_lock(&nf_conntrack_locks[h1]);
128	if (h1 != h2)	128	if (h1 != h2)
129	spin_lock_nested(&nf_conntrack_locks[h2],	129	spin_lock_nested(&nf_conntrack_locks[h2],
130	SINGLE_DEPTH_NESTING);	130	SINGLE_DEPTH_NESTING);
131	} else {	131	} else {
132	nf_conntrack_lock(&nf_conntrack_locks[h2]);	132	nf_conntrack_lock(&nf_conntrack_locks[h2]);
133	spin_lock_nested(&nf_conntrack_locks[h1],	133	spin_lock_nested(&nf_conntrack_locks[h1],
134	SINGLE_DEPTH_NESTING);	134	SINGLE_DEPTH_NESTING);
135	}	135	}
136	if (read_seqcount_retry(&nf_conntrack_generation, sequence)) {	136	if (read_seqcount_retry(&nf_conntrack_generation, sequence)) {
137	nf_conntrack_double_unlock(h1, h2);	137	nf_conntrack_double_unlock(h1, h2);
138	return true;	138	return true;
139	}	139	}
140	return false;	140	return false;
141	}	141	}
142		142
143	static void nf_conntrack_all_lock(void)	143	static void nf_conntrack_all_lock(void)
144	{	144	{
145	int i;	145	int i;
146		146
147	spin_lock(&nf_conntrack_locks_all_lock);	147	spin_lock(&nf_conntrack_locks_all_lock);
148	nf_conntrack_locks_all = true;	148	nf_conntrack_locks_all = true;
149		149
150	/*	150	/*
151	* Order the above store of 'nf_conntrack_locks_all' against	151	* Order the above store of 'nf_conntrack_locks_all' against
152	* the spin_unlock_wait() loads below, such that if	152	* the spin_unlock_wait() loads below, such that if
153	* nf_conntrack_lock() observes 'nf_conntrack_locks_all'	153	* nf_conntrack_lock() observes 'nf_conntrack_locks_all'
154	* we must observe nf_conntrack_locks[] held:	154	* we must observe nf_conntrack_locks[] held:
155	*/	155	*/
156	smp_mb(); /* spin_lock(&nf_conntrack_locks_all_lock) */	156	smp_mb(); /* spin_lock(&nf_conntrack_locks_all_lock) */
157		157
158	for (i = 0; i < CONNTRACK_LOCKS; i++) {	158	for (i = 0; i < CONNTRACK_LOCKS; i++) {
159	spin_unlock_wait(&nf_conntrack_locks[i]);	159	spin_unlock_wait(&nf_conntrack_locks[i]);
160	}	160	}
161	}	161	}
162		162
163	static void nf_conntrack_all_unlock(void)	163	static void nf_conntrack_all_unlock(void)
164	{	164	{
165	/*	165	/*
166	* All prior stores must be complete before we clear	166	* All prior stores must be complete before we clear
167	* 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock()	167	* 'nf_conntrack_locks_all'. Otherwise nf_conntrack_lock()
168	* might observe the false value but not the entire	168	* might observe the false value but not the entire
169	* critical section:	169	* critical section:
170	*/	170	*/
171	smp_store_release(&nf_conntrack_locks_all, false);	171	smp_store_release(&nf_conntrack_locks_all, false);
172	spin_unlock(&nf_conntrack_locks_all_lock);	172	spin_unlock(&nf_conntrack_locks_all_lock);
173	}	173	}
174		174
175	unsigned int nf_conntrack_htable_size __read_mostly;	175	unsigned int nf_conntrack_htable_size __read_mostly;
176	EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);	176	EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
177		177
178	unsigned int nf_conntrack_max __read_mostly;	178	unsigned int nf_conntrack_max __read_mostly;
179	seqcount_t nf_conntrack_generation __read_mostly;	179	seqcount_t nf_conntrack_generation __read_mostly;
180		180
181	DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);	181	DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
182	EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);	182	EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
183		183
184	static unsigned int nf_conntrack_hash_rnd __read_mostly;	184	static unsigned int nf_conntrack_hash_rnd __read_mostly;
185		185
186	static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple,	186	static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple,
187	const struct net *net)	187	const struct net *net)
188	{	188	{
189	unsigned int n;	189	unsigned int n;
190	u32 seed;	190	u32 seed;
191		191
192	get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd));	192	get_random_once(&nf_conntrack_hash_rnd, sizeof(nf_conntrack_hash_rnd));
193		193
194	/* The direction must be ignored, so we hash everything up to the	194	/* The direction must be ignored, so we hash everything up to the
195	* destination ports (which is a multiple of 4) and treat the last	195	* destination ports (which is a multiple of 4) and treat the last
196	* three bytes manually.	196	* three bytes manually.
197	*/	197	*/
198	seed = nf_conntrack_hash_rnd ^ net_hash_mix(net);	198	seed = nf_conntrack_hash_rnd ^ net_hash_mix(net);
199	n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);	199	n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
200	return jhash2((u32 *)tuple, n, seed ^	200	return jhash2((u32 *)tuple, n, seed ^
201	(((__force __u16)tuple->dst.u.all << 16) \|	201	(((__force __u16)tuple->dst.u.all << 16) \|
202	tuple->dst.protonum));	202	tuple->dst.protonum));
203	}	203	}
204		204
205	static u32 scale_hash(u32 hash)	205	static u32 scale_hash(u32 hash)
206	{	206	{
207	return reciprocal_scale(hash, nf_conntrack_htable_size);	207	return reciprocal_scale(hash, nf_conntrack_htable_size);
208	}	208	}
209		209
210	static u32 __hash_conntrack(const struct net *net,	210	static u32 __hash_conntrack(const struct net *net,
211	const struct nf_conntrack_tuple *tuple,	211	const struct nf_conntrack_tuple *tuple,
212	unsigned int size)	212	unsigned int size)
213	{	213	{
214	return reciprocal_scale(hash_conntrack_raw(tuple, net), size);	214	return reciprocal_scale(hash_conntrack_raw(tuple, net), size);
215	}	215	}
216		216
217	static u32 hash_conntrack(const struct net *net,	217	static u32 hash_conntrack(const struct net *net,
218	const struct nf_conntrack_tuple *tuple)	218	const struct nf_conntrack_tuple *tuple)
219	{	219	{
220	return scale_hash(hash_conntrack_raw(tuple, net));	220	return scale_hash(hash_conntrack_raw(tuple, net));
221	}	221	}
222		222
223	bool	223	bool
224	nf_ct_get_tuple(const struct sk_buff *skb,	224	nf_ct_get_tuple(const struct sk_buff *skb,
225	unsigned int nhoff,	225	unsigned int nhoff,
226	unsigned int dataoff,	226	unsigned int dataoff,
227	u_int16_t l3num,	227	u_int16_t l3num,
228	u_int8_t protonum,	228	u_int8_t protonum,
229	struct net *net,	229	struct net *net,
230	struct nf_conntrack_tuple *tuple,	230	struct nf_conntrack_tuple *tuple,
231	const struct nf_conntrack_l3proto *l3proto,	231	const struct nf_conntrack_l3proto *l3proto,
232	const struct nf_conntrack_l4proto *l4proto)	232	const struct nf_conntrack_l4proto *l4proto)
233	{	233	{
234	memset(tuple, 0, sizeof(*tuple));	234	memset(tuple, 0, sizeof(*tuple));
235		235
236	tuple->src.l3num = l3num;	236	tuple->src.l3num = l3num;
237	if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)	237	if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
238	return false;	238	return false;
239		239
240	tuple->dst.protonum = protonum;	240	tuple->dst.protonum = protonum;
241	tuple->dst.dir = IP_CT_DIR_ORIGINAL;	241	tuple->dst.dir = IP_CT_DIR_ORIGINAL;
242		242
243	return l4proto->pkt_to_tuple(skb, dataoff, net, tuple);	243	return l4proto->pkt_to_tuple(skb, dataoff, net, tuple);
244	}	244	}
245	EXPORT_SYMBOL_GPL(nf_ct_get_tuple);	245	EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
246		246
247	bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,	247	bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
248	u_int16_t l3num,	248	u_int16_t l3num,
249	struct net net, struct nf_conntrack_tuple tuple)	249	struct net net, struct nf_conntrack_tuple tuple)
250	{	250	{
251	struct nf_conntrack_l3proto *l3proto;	251	struct nf_conntrack_l3proto *l3proto;
252	struct nf_conntrack_l4proto *l4proto;	252	struct nf_conntrack_l4proto *l4proto;
253	unsigned int protoff;	253	unsigned int protoff;
254	u_int8_t protonum;	254	u_int8_t protonum;
255	int ret;	255	int ret;
256		256
257	rcu_read_lock();	257	rcu_read_lock();
258		258
259	l3proto = __nf_ct_l3proto_find(l3num);	259	l3proto = __nf_ct_l3proto_find(l3num);
260	ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);	260	ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
261	if (ret != NF_ACCEPT) {	261	if (ret != NF_ACCEPT) {
262	rcu_read_unlock();	262	rcu_read_unlock();
263	return false;	263	return false;
264	}	264	}
265		265
266	l4proto = __nf_ct_l4proto_find(l3num, protonum);	266	l4proto = __nf_ct_l4proto_find(l3num, protonum);
267		267
268	ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple,	268	ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, net, tuple,
269	l3proto, l4proto);	269	l3proto, l4proto);
270		270
271	rcu_read_unlock();	271	rcu_read_unlock();
272	return ret;	272	return ret;
273	}	273	}
274	EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);	274	EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
275		275
276	bool	276	bool
277	nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,	277	nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
278	const struct nf_conntrack_tuple *orig,	278	const struct nf_conntrack_tuple *orig,
279	const struct nf_conntrack_l3proto *l3proto,	279	const struct nf_conntrack_l3proto *l3proto,
280	const struct nf_conntrack_l4proto *l4proto)	280	const struct nf_conntrack_l4proto *l4proto)
281	{	281	{
282	memset(inverse, 0, sizeof(*inverse));	282	memset(inverse, 0, sizeof(*inverse));
283		283
284	inverse->src.l3num = orig->src.l3num;	284	inverse->src.l3num = orig->src.l3num;
285	if (l3proto->invert_tuple(inverse, orig) == 0)	285	if (l3proto->invert_tuple(inverse, orig) == 0)
286	return false;	286	return false;
287		287
288	inverse->dst.dir = !orig->dst.dir;	288	inverse->dst.dir = !orig->dst.dir;
289		289
290	inverse->dst.protonum = orig->dst.protonum;	290	inverse->dst.protonum = orig->dst.protonum;
291	return l4proto->invert_tuple(inverse, orig);	291	return l4proto->invert_tuple(inverse, orig);
292	}	292	}
293	EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);	293	EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
294		294
295	static void	295	static void
296	clean_from_lists(struct nf_conn *ct)	296	clean_from_lists(struct nf_conn *ct)
297	{	297	{
298	pr_debug("clean_from_lists(%p)\n", ct);	298	pr_debug("clean_from_lists(%p)\n", ct);
299	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);	299	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
300	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);	300	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
301		301
302	/* Destroy all pending expectations */	302	/* Destroy all pending expectations */
303	nf_ct_remove_expectations(ct);	303	nf_ct_remove_expectations(ct);
304	}	304	}
305		305
306	/* must be called with local_bh_disable */	306	/* must be called with local_bh_disable */
307	static void nf_ct_add_to_dying_list(struct nf_conn *ct)	307	static void nf_ct_add_to_dying_list(struct nf_conn *ct)
308	{	308	{
309	struct ct_pcpu *pcpu;	309	struct ct_pcpu *pcpu;
310		310
311	/* add this conntrack to the (per cpu) dying list */	311	/* add this conntrack to the (per cpu) dying list */
312	ct->cpu = smp_processor_id();	312	ct->cpu = smp_processor_id();
313	pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);	313	pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
314		314
315	spin_lock(&pcpu->lock);	315	spin_lock(&pcpu->lock);
316	hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,	316	hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
317	&pcpu->dying);	317	&pcpu->dying);
318	spin_unlock(&pcpu->lock);	318	spin_unlock(&pcpu->lock);
319	}	319	}
320		320
321	/* must be called with local_bh_disable */	321	/* must be called with local_bh_disable */
322	static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct)	322	static void nf_ct_add_to_unconfirmed_list(struct nf_conn *ct)
323	{	323	{
324	struct ct_pcpu *pcpu;	324	struct ct_pcpu *pcpu;
325		325
326	/* add this conntrack to the (per cpu) unconfirmed list */	326	/* add this conntrack to the (per cpu) unconfirmed list */
327	ct->cpu = smp_processor_id();	327	ct->cpu = smp_processor_id();
328	pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);	328	pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
329		329
330	spin_lock(&pcpu->lock);	330	spin_lock(&pcpu->lock);
331	hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,	331	hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
332	&pcpu->unconfirmed);	332	&pcpu->unconfirmed);
333	spin_unlock(&pcpu->lock);	333	spin_unlock(&pcpu->lock);
334	}	334	}
335		335
336	/* must be called with local_bh_disable */	336	/* must be called with local_bh_disable */
337	static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)	337	static void nf_ct_del_from_dying_or_unconfirmed_list(struct nf_conn *ct)
338	{	338	{
339	struct ct_pcpu *pcpu;	339	struct ct_pcpu *pcpu;
340		340
341	/* We overload first tuple to link into unconfirmed or dying list.*/	341	/* We overload first tuple to link into unconfirmed or dying list.*/
342	pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);	342	pcpu = per_cpu_ptr(nf_ct_net(ct)->ct.pcpu_lists, ct->cpu);
343		343
344	spin_lock(&pcpu->lock);	344	spin_lock(&pcpu->lock);
345	BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));	345	BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
346	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);	346	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
347	spin_unlock(&pcpu->lock);	347	spin_unlock(&pcpu->lock);
348	}	348	}
349		349
350	/* Released via destroy_conntrack() */	350	/* Released via destroy_conntrack() */
351	struct nf_conn nf_ct_tmpl_alloc(struct net net,	351	struct nf_conn nf_ct_tmpl_alloc(struct net net,
352	const struct nf_conntrack_zone *zone,	352	const struct nf_conntrack_zone *zone,
353	gfp_t flags)	353	gfp_t flags)
354	{	354	{
355	struct nf_conn *tmpl;	355	struct nf_conn *tmpl;
356		356
357	tmpl = kzalloc(sizeof(*tmpl), flags);	357	tmpl = kzalloc(sizeof(*tmpl), flags);
358	if (tmpl == NULL)	358	if (tmpl == NULL)
359	return NULL;	359	return NULL;
360		360
361	tmpl->status = IPS_TEMPLATE;	361	tmpl->status = IPS_TEMPLATE;
362	write_pnet(&tmpl->ct_net, net);	362	write_pnet(&tmpl->ct_net, net);
363	nf_ct_zone_add(tmpl, zone);	363	nf_ct_zone_add(tmpl, zone);
364	atomic_set(&tmpl->ct_general.use, 0);	364	atomic_set(&tmpl->ct_general.use, 0);
365		365
366	return tmpl;	366	return tmpl;
367	}	367	}
368	EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc);	368	EXPORT_SYMBOL_GPL(nf_ct_tmpl_alloc);
369		369
370	void nf_ct_tmpl_free(struct nf_conn *tmpl)	370	void nf_ct_tmpl_free(struct nf_conn *tmpl)
371	{	371	{
372	nf_ct_ext_destroy(tmpl);	372	nf_ct_ext_destroy(tmpl);
373	nf_ct_ext_free(tmpl);	373	nf_ct_ext_free(tmpl);
374	kfree(tmpl);	374	kfree(tmpl);
375	}	375	}
376	EXPORT_SYMBOL_GPL(nf_ct_tmpl_free);	376	EXPORT_SYMBOL_GPL(nf_ct_tmpl_free);
377		377
378	static void	378	static void
379	destroy_conntrack(struct nf_conntrack *nfct)	379	destroy_conntrack(struct nf_conntrack *nfct)
380	{	380	{
381	struct nf_conn ct = (struct nf_conn )nfct;	381	struct nf_conn ct = (struct nf_conn )nfct;
382	struct nf_conntrack_l4proto *l4proto;	382	struct nf_conntrack_l4proto *l4proto;
383		383
384	pr_debug("destroy_conntrack(%p)\n", ct);	384	pr_debug("destroy_conntrack(%p)\n", ct);
385	NF_CT_ASSERT(atomic_read(&nfct->use) == 0);	385	NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
386		386
387	if (unlikely(nf_ct_is_template(ct))) {	387	if (unlikely(nf_ct_is_template(ct))) {
388	nf_ct_tmpl_free(ct);	388	nf_ct_tmpl_free(ct);
389	return;	389	return;
390	}	390	}
391	rcu_read_lock();	391	rcu_read_lock();
392	l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));	392	l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
393	if (l4proto->destroy)	393	if (l4proto->destroy)
394	l4proto->destroy(ct);	394	l4proto->destroy(ct);
395		395
396	rcu_read_unlock();	396	rcu_read_unlock();
397		397
398	local_bh_disable();	398	local_bh_disable();
399	/* Expectations will have been removed in clean_from_lists,	399	/* Expectations will have been removed in clean_from_lists,
400	* except TFTP can create an expectation on the first packet,	400	* except TFTP can create an expectation on the first packet,
401	* before connection is in the list, so we need to clean here,	401	* before connection is in the list, so we need to clean here,
402	* too.	402	* too.
403	*/	403	*/
404	nf_ct_remove_expectations(ct);	404	nf_ct_remove_expectations(ct);
405		405
406	nf_ct_del_from_dying_or_unconfirmed_list(ct);	406	nf_ct_del_from_dying_or_unconfirmed_list(ct);
407		407
408	local_bh_enable();	408	local_bh_enable();
409		409
410	if (ct->master)	410	if (ct->master)
411	nf_ct_put(ct->master);	411	nf_ct_put(ct->master);
412		412
413	pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);	413	pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
414	nf_conntrack_free(ct);	414	nf_conntrack_free(ct);
415	}	415	}
416		416
417	static void nf_ct_delete_from_lists(struct nf_conn *ct)	417	static void nf_ct_delete_from_lists(struct nf_conn *ct)
418	{	418	{
419	struct net *net = nf_ct_net(ct);	419	struct net *net = nf_ct_net(ct);
420	unsigned int hash, reply_hash;	420	unsigned int hash, reply_hash;
421	unsigned int sequence;	421	unsigned int sequence;
422		422
423	nf_ct_helper_destroy(ct);	423	nf_ct_helper_destroy(ct);
424		424
425	local_bh_disable();	425	local_bh_disable();
426	do {	426	do {
427	sequence = read_seqcount_begin(&nf_conntrack_generation);	427	sequence = read_seqcount_begin(&nf_conntrack_generation);
428	hash = hash_conntrack(net,	428	hash = hash_conntrack(net,
429	&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);	429	&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
430	reply_hash = hash_conntrack(net,	430	reply_hash = hash_conntrack(net,
431	&ct->tuplehash[IP_CT_DIR_REPLY].tuple);	431	&ct->tuplehash[IP_CT_DIR_REPLY].tuple);
432	} while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));	432	} while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
433		433
434	clean_from_lists(ct);	434	clean_from_lists(ct);
435	nf_conntrack_double_unlock(hash, reply_hash);	435	nf_conntrack_double_unlock(hash, reply_hash);
436		436
437	nf_ct_add_to_dying_list(ct);	437	nf_ct_add_to_dying_list(ct);
438		438
439	local_bh_enable();	439	local_bh_enable();
440	}	440	}
441		441
442	bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)	442	bool nf_ct_delete(struct nf_conn *ct, u32 portid, int report)
443	{	443	{
444	struct nf_conn_tstamp *tstamp;	444	struct nf_conn_tstamp *tstamp;
445		445
446	if (test_and_set_bit(IPS_DYING_BIT, &ct->status))	446	if (test_and_set_bit(IPS_DYING_BIT, &ct->status))
447	return false;	447	return false;
448		448
449	tstamp = nf_conn_tstamp_find(ct);	449	tstamp = nf_conn_tstamp_find(ct);
450	if (tstamp && tstamp->stop == 0)	450	if (tstamp && tstamp->stop == 0)
451	tstamp->stop = ktime_get_real_ns();	451	tstamp->stop = ktime_get_real_ns();
452		452
453	if (nf_conntrack_event_report(IPCT_DESTROY, ct,	453	if (nf_conntrack_event_report(IPCT_DESTROY, ct,
454	portid, report) < 0) {	454	portid, report) < 0) {
455	/* destroy event was not delivered. nf_ct_put will	455	/* destroy event was not delivered. nf_ct_put will
456	* be done by event cache worker on redelivery.	456	* be done by event cache worker on redelivery.
457	*/	457	*/
458	nf_ct_delete_from_lists(ct);	458	nf_ct_delete_from_lists(ct);
459	nf_conntrack_ecache_delayed_work(nf_ct_net(ct));	459	nf_conntrack_ecache_delayed_work(nf_ct_net(ct));
460	return false;	460	return false;
461	}	461	}
462		462
463	nf_conntrack_ecache_work(nf_ct_net(ct));	463	nf_conntrack_ecache_work(nf_ct_net(ct));
464	nf_ct_delete_from_lists(ct);	464	nf_ct_delete_from_lists(ct);
465	nf_ct_put(ct);	465	nf_ct_put(ct);
466	return true;	466	return true;
467	}	467	}
468	EXPORT_SYMBOL_GPL(nf_ct_delete);	468	EXPORT_SYMBOL_GPL(nf_ct_delete);
469		469
470	static inline bool	470	static inline bool
471	nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,	471	nf_ct_key_equal(struct nf_conntrack_tuple_hash *h,
472	const struct nf_conntrack_tuple *tuple,	472	const struct nf_conntrack_tuple *tuple,
473	const struct nf_conntrack_zone *zone,	473	const struct nf_conntrack_zone *zone,
474	const struct net *net)	474	const struct net *net)
475	{	475	{
476	struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);	476	struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
477		477
478	/* A conntrack can be recreated with the equal tuple,	478	/* A conntrack can be recreated with the equal tuple,
479	* so we need to check that the conntrack is confirmed	479	* so we need to check that the conntrack is confirmed
480	*/	480	*/
481	return nf_ct_tuple_equal(tuple, &h->tuple) &&	481	return nf_ct_tuple_equal(tuple, &h->tuple) &&
482	nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) &&	482	nf_ct_zone_equal(ct, zone, NF_CT_DIRECTION(h)) &&
483	nf_ct_is_confirmed(ct) &&	483	nf_ct_is_confirmed(ct) &&
484	net_eq(net, nf_ct_net(ct));	484	net_eq(net, nf_ct_net(ct));
485	}	485	}
486		486
487	/* caller must hold rcu readlock and none of the nf_conntrack_locks */	487	/* caller must hold rcu readlock and none of the nf_conntrack_locks */
488	static void nf_ct_gc_expired(struct nf_conn *ct)	488	static void nf_ct_gc_expired(struct nf_conn *ct)
489	{	489	{
490	if (!atomic_inc_not_zero(&ct->ct_general.use))	490	if (!atomic_inc_not_zero(&ct->ct_general.use))
491	return;	491	return;
492		492
493	if (nf_ct_should_gc(ct))	493	if (nf_ct_should_gc(ct))
494	nf_ct_kill(ct);	494	nf_ct_kill(ct);
495		495
496	nf_ct_put(ct);	496	nf_ct_put(ct);
497	}	497	}
498		498
499	/*	499	/*
500	* Warning :	500	* Warning :
501	* - Caller must take a reference on returned object	501	* - Caller must take a reference on returned object
502	* and recheck nf_ct_tuple_equal(tuple, &h->tuple)	502	* and recheck nf_ct_tuple_equal(tuple, &h->tuple)
503	*/	503	*/
504	static struct nf_conntrack_tuple_hash *	504	static struct nf_conntrack_tuple_hash *
505	____nf_conntrack_find(struct net net, const struct nf_conntrack_zone zone,	505	____nf_conntrack_find(struct net net, const struct nf_conntrack_zone zone,
506	const struct nf_conntrack_tuple *tuple, u32 hash)	506	const struct nf_conntrack_tuple *tuple, u32 hash)
507	{	507	{
508	struct nf_conntrack_tuple_hash *h;	508	struct nf_conntrack_tuple_hash *h;
509	struct hlist_nulls_head *ct_hash;	509	struct hlist_nulls_head *ct_hash;
510	struct hlist_nulls_node *n;	510	struct hlist_nulls_node *n;
511	unsigned int bucket, hsize;	511	unsigned int bucket, hsize;
512		512
513	begin:	513	begin:
514	nf_conntrack_get_ht(&ct_hash, &hsize);	514	nf_conntrack_get_ht(&ct_hash, &hsize);
515	bucket = reciprocal_scale(hash, hsize);	515	bucket = reciprocal_scale(hash, hsize);
516		516
517	hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) {	517	hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[bucket], hnnode) {
518	struct nf_conn *ct;	518	struct nf_conn *ct;
519		519
520	ct = nf_ct_tuplehash_to_ctrack(h);	520	ct = nf_ct_tuplehash_to_ctrack(h);
521	if (nf_ct_is_expired(ct)) {	521	if (nf_ct_is_expired(ct)) {
522	nf_ct_gc_expired(ct);	522	nf_ct_gc_expired(ct);
523	continue;	523	continue;
524	}	524	}
525		525
526	if (nf_ct_is_dying(ct))	526	if (nf_ct_is_dying(ct))
527	continue;	527	continue;
528		528
529	if (nf_ct_key_equal(h, tuple, zone, net))	529	if (nf_ct_key_equal(h, tuple, zone, net))
530	return h;	530	return h;
531	}	531	}
532	/*	532	/*
533	* if the nulls value we got at the end of this lookup is	533	* if the nulls value we got at the end of this lookup is
534	* not the expected one, we must restart lookup.	534	* not the expected one, we must restart lookup.
535	* We probably met an item that was moved to another chain.	535	* We probably met an item that was moved to another chain.
536	*/	536	*/
537	if (get_nulls_value(n) != bucket) {	537	if (get_nulls_value(n) != bucket) {
538	NF_CT_STAT_INC_ATOMIC(net, search_restart);	538	NF_CT_STAT_INC_ATOMIC(net, search_restart);
539	goto begin;	539	goto begin;
540	}	540	}
541		541
542	return NULL;	542	return NULL;
543	}	543	}
544		544
545	/* Find a connection corresponding to a tuple. */	545	/* Find a connection corresponding to a tuple. */
546	static struct nf_conntrack_tuple_hash *	546	static struct nf_conntrack_tuple_hash *
547	__nf_conntrack_find_get(struct net net, const struct nf_conntrack_zone zone,	547	__nf_conntrack_find_get(struct net net, const struct nf_conntrack_zone zone,
548	const struct nf_conntrack_tuple *tuple, u32 hash)	548	const struct nf_conntrack_tuple *tuple, u32 hash)
549	{	549	{
550	struct nf_conntrack_tuple_hash *h;	550	struct nf_conntrack_tuple_hash *h;
551	struct nf_conn *ct;	551	struct nf_conn *ct;
552		552
553	rcu_read_lock();	553	rcu_read_lock();
554	begin:	554	begin:
555	h = ____nf_conntrack_find(net, zone, tuple, hash);	555	h = ____nf_conntrack_find(net, zone, tuple, hash);
556	if (h) {	556	if (h) {
557	ct = nf_ct_tuplehash_to_ctrack(h);	557	ct = nf_ct_tuplehash_to_ctrack(h);
558	if (unlikely(nf_ct_is_dying(ct) \|\|	558	if (unlikely(nf_ct_is_dying(ct) \|\|
559	!atomic_inc_not_zero(&ct->ct_general.use)))	559	!atomic_inc_not_zero(&ct->ct_general.use)))
560	h = NULL;	560	h = NULL;
561	else {	561	else {
562	if (unlikely(!nf_ct_key_equal(h, tuple, zone, net))) {	562	if (unlikely(!nf_ct_key_equal(h, tuple, zone, net))) {
563	nf_ct_put(ct);	563	nf_ct_put(ct);
564	goto begin;	564	goto begin;
565	}	565	}
566	}	566	}
567	}	567	}
568	rcu_read_unlock();	568	rcu_read_unlock();
569		569
570	return h;	570	return h;
571	}	571	}
572		572
573	struct nf_conntrack_tuple_hash *	573	struct nf_conntrack_tuple_hash *
574	nf_conntrack_find_get(struct net net, const struct nf_conntrack_zone zone,	574	nf_conntrack_find_get(struct net net, const struct nf_conntrack_zone zone,
575	const struct nf_conntrack_tuple *tuple)	575	const struct nf_conntrack_tuple *tuple)
576	{	576	{
577	return __nf_conntrack_find_get(net, zone, tuple,	577	return __nf_conntrack_find_get(net, zone, tuple,
578	hash_conntrack_raw(tuple, net));	578	hash_conntrack_raw(tuple, net));
579	}	579	}
580	EXPORT_SYMBOL_GPL(nf_conntrack_find_get);	580	EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
581		581
582	static void __nf_conntrack_hash_insert(struct nf_conn *ct,	582	static void __nf_conntrack_hash_insert(struct nf_conn *ct,
583	unsigned int hash,	583	unsigned int hash,
584	unsigned int reply_hash)	584	unsigned int reply_hash)
585	{	585	{
586	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,	586	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
587	&nf_conntrack_hash[hash]);	587	&nf_conntrack_hash[hash]);
588	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,	588	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
589	&nf_conntrack_hash[reply_hash]);	589	&nf_conntrack_hash[reply_hash]);
590	}	590	}
591		591
592	int	592	int
593	nf_conntrack_hash_check_insert(struct nf_conn *ct)	593	nf_conntrack_hash_check_insert(struct nf_conn *ct)
594	{	594	{
595	const struct nf_conntrack_zone *zone;	595	const struct nf_conntrack_zone *zone;
596	struct net *net = nf_ct_net(ct);	596	struct net *net = nf_ct_net(ct);
597	unsigned int hash, reply_hash;	597	unsigned int hash, reply_hash;
598	struct nf_conntrack_tuple_hash *h;	598	struct nf_conntrack_tuple_hash *h;
599	struct hlist_nulls_node *n;	599	struct hlist_nulls_node *n;
600	unsigned int sequence;	600	unsigned int sequence;
601		601
602	zone = nf_ct_zone(ct);	602	zone = nf_ct_zone(ct);
603		603
604	local_bh_disable();	604	local_bh_disable();
605	do {	605	do {
606	sequence = read_seqcount_begin(&nf_conntrack_generation);	606	sequence = read_seqcount_begin(&nf_conntrack_generation);
607	hash = hash_conntrack(net,	607	hash = hash_conntrack(net,
608	&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);	608	&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
609	reply_hash = hash_conntrack(net,	609	reply_hash = hash_conntrack(net,
610	&ct->tuplehash[IP_CT_DIR_REPLY].tuple);	610	&ct->tuplehash[IP_CT_DIR_REPLY].tuple);
611	} while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));	611	} while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
612		612
613	/* See if there's one in the list already, including reverse */	613	/* See if there's one in the list already, including reverse */
614	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)	614	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
615	if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,	615	if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
616	zone, net))	616	zone, net))
617	goto out;	617	goto out;
618		618
619	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)	619	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
620	if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,	620	if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
621	zone, net))	621	zone, net))
622	goto out;	622	goto out;
623		623
624	smp_wmb();	624	smp_wmb();
625	/* The caller holds a reference to this object */	625	/* The caller holds a reference to this object */
626	atomic_set(&ct->ct_general.use, 2);	626	atomic_set(&ct->ct_general.use, 2);
627	__nf_conntrack_hash_insert(ct, hash, reply_hash);	627	__nf_conntrack_hash_insert(ct, hash, reply_hash);
628	nf_conntrack_double_unlock(hash, reply_hash);	628	nf_conntrack_double_unlock(hash, reply_hash);
629	NF_CT_STAT_INC(net, insert);	629	NF_CT_STAT_INC(net, insert);
630	local_bh_enable();	630	local_bh_enable();
631	return 0;	631	return 0;
632		632
633	out:	633	out:
634	nf_conntrack_double_unlock(hash, reply_hash);	634	nf_conntrack_double_unlock(hash, reply_hash);
635	NF_CT_STAT_INC(net, insert_failed);	635	NF_CT_STAT_INC(net, insert_failed);
636	local_bh_enable();	636	local_bh_enable();
637	return -EEXIST;	637	return -EEXIST;
638	}	638	}
639	EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);	639	EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
640		640
641	static inline void nf_ct_acct_update(struct nf_conn *ct,	641	static inline void nf_ct_acct_update(struct nf_conn *ct,
642	enum ip_conntrack_info ctinfo,	642	enum ip_conntrack_info ctinfo,
643	unsigned int len)	643	unsigned int len)
644	{	644	{
645	struct nf_conn_acct *acct;	645	struct nf_conn_acct *acct;
646		646
647	acct = nf_conn_acct_find(ct);	647	acct = nf_conn_acct_find(ct);
648	if (acct) {	648	if (acct) {
649	struct nf_conn_counter *counter = acct->counter;	649	struct nf_conn_counter *counter = acct->counter;
650		650
651	atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets);	651	atomic64_inc(&counter[CTINFO2DIR(ctinfo)].packets);
652	atomic64_add(len, &counter[CTINFO2DIR(ctinfo)].bytes);	652	atomic64_add(len, &counter[CTINFO2DIR(ctinfo)].bytes);
653	}	653	}
654	}	654	}
655		655
656	static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,	656	static void nf_ct_acct_merge(struct nf_conn *ct, enum ip_conntrack_info ctinfo,
657	const struct nf_conn *loser_ct)	657	const struct nf_conn *loser_ct)
658	{	658	{
659	struct nf_conn_acct *acct;	659	struct nf_conn_acct *acct;
660		660
661	acct = nf_conn_acct_find(loser_ct);	661	acct = nf_conn_acct_find(loser_ct);
662	if (acct) {	662	if (acct) {
663	struct nf_conn_counter *counter = acct->counter;	663	struct nf_conn_counter *counter = acct->counter;
664	unsigned int bytes;	664	unsigned int bytes;
665		665
666	/* u32 should be fine since we must have seen one packet. */	666	/* u32 should be fine since we must have seen one packet. */
667	bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes);	667	bytes = atomic64_read(&counter[CTINFO2DIR(ctinfo)].bytes);
668	nf_ct_acct_update(ct, ctinfo, bytes);	668	nf_ct_acct_update(ct, ctinfo, bytes);
669	}	669	}
670	}	670	}
671		671
672	/* Resolve race on insertion if this protocol allows this. */	672	/* Resolve race on insertion if this protocol allows this. */
673	static int nf_ct_resolve_clash(struct net net, struct sk_buff skb,	673	static int nf_ct_resolve_clash(struct net net, struct sk_buff skb,
674	enum ip_conntrack_info ctinfo,	674	enum ip_conntrack_info ctinfo,
675	struct nf_conntrack_tuple_hash *h)	675	struct nf_conntrack_tuple_hash *h)
676	{	676	{
677	/* This is the conntrack entry already in hashes that won race. */	677	/* This is the conntrack entry already in hashes that won race. */
678	struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);	678	struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
679	struct nf_conntrack_l4proto *l4proto;	679	struct nf_conntrack_l4proto *l4proto;
680		680
681	l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));	681	l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
682	if (l4proto->allow_clash &&	682	if (l4proto->allow_clash &&
683	!nfct_nat(ct) &&	683	!nfct_nat(ct) &&
684	!nf_ct_is_dying(ct) &&	684	!nf_ct_is_dying(ct) &&
685	atomic_inc_not_zero(&ct->ct_general.use)) {	685	atomic_inc_not_zero(&ct->ct_general.use)) {
686	nf_ct_acct_merge(ct, ctinfo, (struct nf_conn *)skb->nfct);	686	nf_ct_acct_merge(ct, ctinfo, (struct nf_conn *)skb->nfct);
687	nf_conntrack_put(skb->nfct);	687	nf_conntrack_put(skb->nfct);
688	/* Assign conntrack already in hashes to this skbuff. Don't	688	/* Assign conntrack already in hashes to this skbuff. Don't
689	* modify skb->nfctinfo to ensure consistent stateful filtering.	689	* modify skb->nfctinfo to ensure consistent stateful filtering.
690	*/	690	*/
691	skb->nfct = &ct->ct_general;	691	skb->nfct = &ct->ct_general;
692	return NF_ACCEPT;	692	return NF_ACCEPT;
693	}	693	}
694	NF_CT_STAT_INC(net, drop);	694	NF_CT_STAT_INC(net, drop);
695	return NF_DROP;	695	return NF_DROP;
696	}	696	}
697		697
698	/* Confirm a connection given skb; places it in hash table */	698	/* Confirm a connection given skb; places it in hash table */
699	int	699	int
700	__nf_conntrack_confirm(struct sk_buff *skb)	700	__nf_conntrack_confirm(struct sk_buff *skb)
701	{	701	{
702	const struct nf_conntrack_zone *zone;	702	const struct nf_conntrack_zone *zone;
703	unsigned int hash, reply_hash;	703	unsigned int hash, reply_hash;
704	struct nf_conntrack_tuple_hash *h;	704	struct nf_conntrack_tuple_hash *h;
705	struct nf_conn *ct;	705	struct nf_conn *ct;
706	struct nf_conn_help *help;	706	struct nf_conn_help *help;
707	struct nf_conn_tstamp *tstamp;	707	struct nf_conn_tstamp *tstamp;
708	struct hlist_nulls_node *n;	708	struct hlist_nulls_node *n;
709	enum ip_conntrack_info ctinfo;	709	enum ip_conntrack_info ctinfo;
710	struct net *net;	710	struct net *net;
711	unsigned int sequence;	711	unsigned int sequence;
712	int ret = NF_DROP;	712	int ret = NF_DROP;
713		713
714	ct = nf_ct_get(skb, &ctinfo);	714	ct = nf_ct_get(skb, &ctinfo);
715	net = nf_ct_net(ct);	715	net = nf_ct_net(ct);
716		716
717	/* ipt_REJECT uses nf_conntrack_attach to attach related	717	/* ipt_REJECT uses nf_conntrack_attach to attach related
718	ICMP/TCP RST packets in other direction. Actual packet	718	ICMP/TCP RST packets in other direction. Actual packet
719	which created connection will be IP_CT_NEW or for an	719	which created connection will be IP_CT_NEW or for an
720	expected connection, IP_CT_RELATED. */	720	expected connection, IP_CT_RELATED. */
721	if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)	721	if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
722	return NF_ACCEPT;	722	return NF_ACCEPT;
723		723
724	zone = nf_ct_zone(ct);	724	zone = nf_ct_zone(ct);
725	local_bh_disable();	725	local_bh_disable();
726		726
727	do {	727	do {
728	sequence = read_seqcount_begin(&nf_conntrack_generation);	728	sequence = read_seqcount_begin(&nf_conntrack_generation);
729	/* reuse the hash saved before */	729	/* reuse the hash saved before */
730	hash = (unsigned long )&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;	730	hash = (unsigned long )&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
731	hash = scale_hash(hash);	731	hash = scale_hash(hash);
732	reply_hash = hash_conntrack(net,	732	reply_hash = hash_conntrack(net,
733	&ct->tuplehash[IP_CT_DIR_REPLY].tuple);	733	&ct->tuplehash[IP_CT_DIR_REPLY].tuple);
734		734
735	} while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));	735	} while (nf_conntrack_double_lock(net, hash, reply_hash, sequence));
736		736
737	/* We're not in hash table, and we refuse to set up related	737	/* We're not in hash table, and we refuse to set up related
738	* connections for unconfirmed conns. But packet copies and	738	* connections for unconfirmed conns. But packet copies and
739	* REJECT will give spurious warnings here.	739	* REJECT will give spurious warnings here.
740	*/	740	*/
741	/* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */	741	/* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
742		742
743	/* No external references means no one else could have	743	/* No external references means no one else could have
744	* confirmed us.	744	* confirmed us.
745	*/	745	*/
746	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));	746	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
747	pr_debug("Confirming conntrack %p\n", ct);	747	pr_debug("Confirming conntrack %p\n", ct);
748	/* We have to check the DYING flag after unlink to prevent	748	/* We have to check the DYING flag after unlink to prevent
749	* a race against nf_ct_get_next_corpse() possibly called from	749	* a race against nf_ct_get_next_corpse() possibly called from
750	* user context, else we insert an already 'dead' hash, blocking	750	* user context, else we insert an already 'dead' hash, blocking
751	* further use of that particular connection -JM.	751	* further use of that particular connection -JM.
752	*/	752	*/
753	nf_ct_del_from_dying_or_unconfirmed_list(ct);	753	nf_ct_del_from_dying_or_unconfirmed_list(ct);
754		754
755	if (unlikely(nf_ct_is_dying(ct))) {	755	if (unlikely(nf_ct_is_dying(ct))) {
756	nf_ct_add_to_dying_list(ct);	756	nf_ct_add_to_dying_list(ct);
757	goto dying;	757	goto dying;
758	}	758	}
759		759
760	/* See if there's one in the list already, including reverse:	760	/* See if there's one in the list already, including reverse:
761	NAT could have grabbed it without realizing, since we're	761	NAT could have grabbed it without realizing, since we're
762	not in the hash. If there is, we lost race. */	762	not in the hash. If there is, we lost race. */
763	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)	763	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[hash], hnnode)
764	if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,	764	if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
765	zone, net))	765	zone, net))
766	goto out;	766	goto out;
767		767
768	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)	768	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[reply_hash], hnnode)
769	if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,	769	if (nf_ct_key_equal(h, &ct->tuplehash[IP_CT_DIR_REPLY].tuple,
770	zone, net))	770	zone, net))
771	goto out;	771	goto out;
772		772
773	/* Timer relative to confirmation time, not original	773	/* Timer relative to confirmation time, not original
774	setting time, otherwise we'd get timer wrap in	774	setting time, otherwise we'd get timer wrap in
775	weird delay cases. */	775	weird delay cases. */
776	ct->timeout += nfct_time_stamp;	776	ct->timeout += nfct_time_stamp;
777	atomic_inc(&ct->ct_general.use);	777	atomic_inc(&ct->ct_general.use);
778	ct->status \|= IPS_CONFIRMED;	778	ct->status \|= IPS_CONFIRMED;
779		779
780	/* set conntrack timestamp, if enabled. */	780	/* set conntrack timestamp, if enabled. */
781	tstamp = nf_conn_tstamp_find(ct);	781	tstamp = nf_conn_tstamp_find(ct);
782	if (tstamp) {	782	if (tstamp) {
783	if (skb->tstamp.tv64 == 0)	783	if (skb->tstamp.tv64 == 0)
784	__net_timestamp(skb);	784	__net_timestamp(skb);
785		785
786	tstamp->start = ktime_to_ns(skb->tstamp);	786	tstamp->start = ktime_to_ns(skb->tstamp);
787	}	787	}
788	/* Since the lookup is lockless, hash insertion must be done after	788	/* Since the lookup is lockless, hash insertion must be done after
789	* starting the timer and setting the CONFIRMED bit. The RCU barriers	789	* starting the timer and setting the CONFIRMED bit. The RCU barriers
790	* guarantee that no other CPU can find the conntrack before the above	790	* guarantee that no other CPU can find the conntrack before the above
791	* stores are visible.	791	* stores are visible.
792	*/	792	*/
793	__nf_conntrack_hash_insert(ct, hash, reply_hash);	793	__nf_conntrack_hash_insert(ct, hash, reply_hash);
794	nf_conntrack_double_unlock(hash, reply_hash);	794	nf_conntrack_double_unlock(hash, reply_hash);
795	local_bh_enable();	795	local_bh_enable();
796		796
797	help = nfct_help(ct);	797	help = nfct_help(ct);
798	if (help && help->helper)	798	if (help && help->helper)
799	nf_conntrack_event_cache(IPCT_HELPER, ct);	799	nf_conntrack_event_cache(IPCT_HELPER, ct);
800		800
801	nf_conntrack_event_cache(master_ct(ct) ?	801	nf_conntrack_event_cache(master_ct(ct) ?
802	IPCT_RELATED : IPCT_NEW, ct);	802	IPCT_RELATED : IPCT_NEW, ct);
803	return NF_ACCEPT;	803	return NF_ACCEPT;
804		804
805	out:	805	out:
806	nf_ct_add_to_dying_list(ct);	806	nf_ct_add_to_dying_list(ct);
807	ret = nf_ct_resolve_clash(net, skb, ctinfo, h);	807	ret = nf_ct_resolve_clash(net, skb, ctinfo, h);
808	dying:	808	dying:
809	nf_conntrack_double_unlock(hash, reply_hash);	809	nf_conntrack_double_unlock(hash, reply_hash);
810	NF_CT_STAT_INC(net, insert_failed);	810	NF_CT_STAT_INC(net, insert_failed);
811	local_bh_enable();	811	local_bh_enable();
812	return ret;	812	return ret;
813	}	813	}
814	EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);	814	EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
815		815
816	/* Returns true if a connection correspondings to the tuple (required	816	/* Returns true if a connection correspondings to the tuple (required
817	for NAT). */	817	for NAT). */
818	int	818	int
819	nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,	819	nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
820	const struct nf_conn *ignored_conntrack)	820	const struct nf_conn *ignored_conntrack)
821	{	821	{
822	struct net *net = nf_ct_net(ignored_conntrack);	822	struct net *net = nf_ct_net(ignored_conntrack);
823	const struct nf_conntrack_zone *zone;	823	const struct nf_conntrack_zone *zone;
824	struct nf_conntrack_tuple_hash *h;	824	struct nf_conntrack_tuple_hash *h;
825	struct hlist_nulls_head *ct_hash;	825	struct hlist_nulls_head *ct_hash;
826	unsigned int hash, hsize;	826	unsigned int hash, hsize;
827	struct hlist_nulls_node *n;	827	struct hlist_nulls_node *n;
828	struct nf_conn *ct;	828	struct nf_conn *ct;
829		829
830	zone = nf_ct_zone(ignored_conntrack);	830	zone = nf_ct_zone(ignored_conntrack);
831		831
832	rcu_read_lock();	832	rcu_read_lock();
833	begin:	833	begin:
834	nf_conntrack_get_ht(&ct_hash, &hsize);	834	nf_conntrack_get_ht(&ct_hash, &hsize);
835	hash = __hash_conntrack(net, tuple, hsize);	835	hash = __hash_conntrack(net, tuple, hsize);
836		836
837	hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) {	837	hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[hash], hnnode) {
838	ct = nf_ct_tuplehash_to_ctrack(h);	838	ct = nf_ct_tuplehash_to_ctrack(h);
839		839
840	if (ct == ignored_conntrack)	840	if (ct == ignored_conntrack)
841	continue;	841	continue;
842		842
843	if (nf_ct_is_expired(ct)) {	843	if (nf_ct_is_expired(ct)) {
844	nf_ct_gc_expired(ct);	844	nf_ct_gc_expired(ct);
845	continue;	845	continue;
846	}	846	}
847		847
848	if (nf_ct_key_equal(h, tuple, zone, net)) {	848	if (nf_ct_key_equal(h, tuple, zone, net)) {
849	NF_CT_STAT_INC_ATOMIC(net, found);	849	NF_CT_STAT_INC_ATOMIC(net, found);
850	rcu_read_unlock();	850	rcu_read_unlock();
851	return 1;	851	return 1;
852	}	852	}
853	}	853	}
854		854
855	if (get_nulls_value(n) != hash) {	855	if (get_nulls_value(n) != hash) {
856	NF_CT_STAT_INC_ATOMIC(net, search_restart);	856	NF_CT_STAT_INC_ATOMIC(net, search_restart);
857	goto begin;	857	goto begin;
858	}	858	}
859		859
860	rcu_read_unlock();	860	rcu_read_unlock();
861		861
862	return 0;	862	return 0;
863	}	863	}
864	EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);	864	EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
865		865
866	#define NF_CT_EVICTION_RANGE 8	866	#define NF_CT_EVICTION_RANGE 8
867		867
868	/* There's a small race here where we may free a just-assured	868	/* There's a small race here where we may free a just-assured
869	connection. Too bad: we're in trouble anyway. */	869	connection. Too bad: we're in trouble anyway. */
870	static unsigned int early_drop_list(struct net *net,	870	static unsigned int early_drop_list(struct net *net,
871	struct hlist_nulls_head *head)	871	struct hlist_nulls_head *head)
872	{	872	{
873	struct nf_conntrack_tuple_hash *h;	873	struct nf_conntrack_tuple_hash *h;
874	struct hlist_nulls_node *n;	874	struct hlist_nulls_node *n;
875	unsigned int drops = 0;	875	unsigned int drops = 0;
876	struct nf_conn *tmp;	876	struct nf_conn *tmp;
877		877
878	hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) {	878	hlist_nulls_for_each_entry_rcu(h, n, head, hnnode) {
879	tmp = nf_ct_tuplehash_to_ctrack(h);	879	tmp = nf_ct_tuplehash_to_ctrack(h);
880		880
881	if (nf_ct_is_expired(tmp)) {	881	if (nf_ct_is_expired(tmp)) {
882	nf_ct_gc_expired(tmp);	882	nf_ct_gc_expired(tmp);
883	continue;	883	continue;
884	}	884	}
885		885
886	if (test_bit(IPS_ASSURED_BIT, &tmp->status) \|\|	886	if (test_bit(IPS_ASSURED_BIT, &tmp->status) \|\|
887	!net_eq(nf_ct_net(tmp), net) \|\|	887	!net_eq(nf_ct_net(tmp), net) \|\|
888	nf_ct_is_dying(tmp))	888	nf_ct_is_dying(tmp))
889	continue;	889	continue;
890		890
891	if (!atomic_inc_not_zero(&tmp->ct_general.use))	891	if (!atomic_inc_not_zero(&tmp->ct_general.use))
892	continue;	892	continue;
893		893
894	/* kill only if still in same netns -- might have moved due to	894	/* kill only if still in same netns -- might have moved due to
895	* SLAB_DESTROY_BY_RCU rules.	895	* SLAB_DESTROY_BY_RCU rules.
896	*	896	*
897	* We steal the timer reference. If that fails timer has	897	* We steal the timer reference. If that fails timer has
898	* already fired or someone else deleted it. Just drop ref	898	* already fired or someone else deleted it. Just drop ref
899	* and move to next entry.	899	* and move to next entry.
900	*/	900	*/
901	if (net_eq(nf_ct_net(tmp), net) &&	901	if (net_eq(nf_ct_net(tmp), net) &&
902	nf_ct_is_confirmed(tmp) &&	902	nf_ct_is_confirmed(tmp) &&
903	nf_ct_delete(tmp, 0, 0))	903	nf_ct_delete(tmp, 0, 0))
904	drops++;	904	drops++;
905		905
906	nf_ct_put(tmp);	906	nf_ct_put(tmp);
907	}	907	}
908		908
909	return drops;	909	return drops;
910	}	910	}
911		911
912	static noinline int early_drop(struct net *net, unsigned int _hash)	912	static noinline int early_drop(struct net *net, unsigned int _hash)
913	{	913	{
914	unsigned int i;	914	unsigned int i;
915		915
916	for (i = 0; i < NF_CT_EVICTION_RANGE; i++) {	916	for (i = 0; i < NF_CT_EVICTION_RANGE; i++) {
917	struct hlist_nulls_head *ct_hash;	917	struct hlist_nulls_head *ct_hash;
918	unsigned int hash, hsize, drops;	918	unsigned int hash, hsize, drops;
919		919
920	rcu_read_lock();	920	rcu_read_lock();
921	nf_conntrack_get_ht(&ct_hash, &hsize);	921	nf_conntrack_get_ht(&ct_hash, &hsize);
922	hash = reciprocal_scale(_hash++, hsize);	922	hash = reciprocal_scale(_hash++, hsize);
923		923
924	drops = early_drop_list(net, &ct_hash[hash]);	924	drops = early_drop_list(net, &ct_hash[hash]);
925	rcu_read_unlock();	925	rcu_read_unlock();
926		926
927	if (drops) {	927	if (drops) {
928	NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops);	928	NF_CT_STAT_ADD_ATOMIC(net, early_drop, drops);
929	return true;	929	return true;
930	}	930	}
931	}	931	}
932		932
933	return false;	933	return false;
934	}	934	}
935		935
936	static void gc_worker(struct work_struct *work)	936	static void gc_worker(struct work_struct *work)
937	{	937	{
938	unsigned int i, goal, buckets = 0, expired_count = 0;	938	unsigned int i, goal, buckets = 0, expired_count = 0;
939	unsigned long next_run = GC_INTERVAL;	939	unsigned long next_run = GC_INTERVAL;
940	unsigned int ratio, scanned = 0;	940	unsigned int ratio, scanned = 0;
941	struct conntrack_gc_work *gc_work;	941	struct conntrack_gc_work *gc_work;
942		942
943	gc_work = container_of(work, struct conntrack_gc_work, dwork.work);	943	gc_work = container_of(work, struct conntrack_gc_work, dwork.work);
944		944
945	goal = min(nf_conntrack_htable_size / GC_MAX_BUCKETS_DIV, GC_MAX_BUCKETS);	945	goal = min(nf_conntrack_htable_size / GC_MAX_BUCKETS_DIV, GC_MAX_BUCKETS);
946	i = gc_work->last_bucket;	946	i = gc_work->last_bucket;
947		947
948	do {	948	do {
949	struct nf_conntrack_tuple_hash *h;	949	struct nf_conntrack_tuple_hash *h;
950	struct hlist_nulls_head *ct_hash;	950	struct hlist_nulls_head *ct_hash;
951	struct hlist_nulls_node *n;	951	struct hlist_nulls_node *n;
952	unsigned int hashsz;	952	unsigned int hashsz;
953	struct nf_conn *tmp;	953	struct nf_conn *tmp;
954		954
955	i++;	955	i++;
956	rcu_read_lock();	956	rcu_read_lock();
957		957
958	nf_conntrack_get_ht(&ct_hash, &hashsz);	958	nf_conntrack_get_ht(&ct_hash, &hashsz);
959	if (i >= hashsz)	959	if (i >= hashsz)
960	i = 0;	960	i = 0;
961		961
962	hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) {	962	hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) {
963	tmp = nf_ct_tuplehash_to_ctrack(h);	963	tmp = nf_ct_tuplehash_to_ctrack(h);
964		964
965	scanned++;	965	scanned++;
966	if (nf_ct_is_expired(tmp)) {	966	if (nf_ct_is_expired(tmp)) {
967	nf_ct_gc_expired(tmp);	967	nf_ct_gc_expired(tmp);
968	expired_count++;	968	expired_count++;
969	continue;	969	continue;
970	}	970	}
971	}	971	}
972		972
973	/* could check get_nulls_value() here and restart if ct	973	/* could check get_nulls_value() here and restart if ct
974	* was moved to another chain. But given gc is best-effort	974	* was moved to another chain. But given gc is best-effort
975	* we will just continue with next hash slot.	975	* we will just continue with next hash slot.
976	*/	976	*/
977	rcu_read_unlock();	977	rcu_read_unlock();
978	cond_resched_rcu_qs();	978	cond_resched_rcu_qs();
979	} while (++buckets < goal &&	979	} while (++buckets < goal &&
980	expired_count < GC_MAX_EVICTS);	980	expired_count < GC_MAX_EVICTS);
981		981
982	if (gc_work->exiting)	982	if (gc_work->exiting)
983	return;	983	return;
984		984
985	ratio = scanned ? expired_count * 100 / scanned : 0;	985	ratio = scanned ? expired_count * 100 / scanned : 0;
986	if (ratio >= 90 \|\| expired_count == GC_MAX_EVICTS)	986	if (ratio >= 90 \|\| expired_count == GC_MAX_EVICTS)
987	next_run = 0;	987	next_run = 0;
988		988
989	gc_work->last_bucket = i;	989	gc_work->last_bucket = i;
990	schedule_delayed_work(&gc_work->dwork, next_run);	990	schedule_delayed_work(&gc_work->dwork, next_run);
991	}	991	}
992		992
993	static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work)	993	static void conntrack_gc_work_init(struct conntrack_gc_work *gc_work)
994	{	994	{
995	INIT_DELAYED_WORK(&gc_work->dwork, gc_worker);	995	INIT_DELAYED_WORK(&gc_work->dwork, gc_worker);
996	gc_work->exiting = false;	996	gc_work->exiting = false;
997	}	997	}
998		998
999	static struct nf_conn *	999	static struct nf_conn *
1000	__nf_conntrack_alloc(struct net *net,	1000	__nf_conntrack_alloc(struct net *net,
1001	const struct nf_conntrack_zone *zone,	1001	const struct nf_conntrack_zone *zone,
1002	const struct nf_conntrack_tuple *orig,	1002	const struct nf_conntrack_tuple *orig,
1003	const struct nf_conntrack_tuple *repl,	1003	const struct nf_conntrack_tuple *repl,
1004	gfp_t gfp, u32 hash)	1004	gfp_t gfp, u32 hash)
1005	{	1005	{
1006	struct nf_conn *ct;	1006	struct nf_conn *ct;
1007		1007
1008	/* We don't want any race condition at early drop stage */	1008	/* We don't want any race condition at early drop stage */
1009	atomic_inc(&net->ct.count);	1009	atomic_inc(&net->ct.count);
1010		1010
1011	if (nf_conntrack_max &&	1011	if (nf_conntrack_max &&
1012	unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {	1012	unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
1013	if (!early_drop(net, hash)) {	1013	if (!early_drop(net, hash)) {
1014	atomic_dec(&net->ct.count);	1014	atomic_dec(&net->ct.count);
1015	net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");	1015	net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
1016	return ERR_PTR(-ENOMEM);	1016	return ERR_PTR(-ENOMEM);
1017	}	1017	}
1018	}	1018	}
1019		1019
1020	/*	1020	/*
1021	* Do not use kmem_cache_zalloc(), as this cache uses	1021	* Do not use kmem_cache_zalloc(), as this cache uses
1022	* SLAB_DESTROY_BY_RCU.	1022	* SLAB_DESTROY_BY_RCU.
1023	*/	1023	*/
1024	ct = kmem_cache_alloc(nf_conntrack_cachep, gfp);	1024	ct = kmem_cache_alloc(nf_conntrack_cachep, gfp);
1025	if (ct == NULL)	1025	if (ct == NULL)
1026	goto out;	1026	goto out;
1027		1027
1028	spin_lock_init(&ct->lock);	1028	spin_lock_init(&ct->lock);
1029	ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;	1029	ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
1030	ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;	1030	ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
1031	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;	1031	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
1032	/* save hash for reusing when confirming */	1032	/* save hash for reusing when confirming */
1033	(unsigned long )(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;	1033	(unsigned long )(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
1034	ct->status = 0;	1034	ct->status = 0;
1035	write_pnet(&ct->ct_net, net);	1035	write_pnet(&ct->ct_net, net);
1036	memset(&ct->__nfct_init_offset[0], 0,	1036	memset(&ct->__nfct_init_offset[0], 0,
1037	offsetof(struct nf_conn, proto) -	1037	offsetof(struct nf_conn, proto) -
1038	offsetof(struct nf_conn, __nfct_init_offset[0]));	1038	offsetof(struct nf_conn, __nfct_init_offset[0]));
1039		1039
1040	nf_ct_zone_add(ct, zone);	1040	nf_ct_zone_add(ct, zone);
1041		1041
1042	/* Because we use RCU lookups, we set ct_general.use to zero before	1042	/* Because we use RCU lookups, we set ct_general.use to zero before
1043	* this is inserted in any list.	1043	* this is inserted in any list.
1044	*/	1044	*/
1045	atomic_set(&ct->ct_general.use, 0);	1045	atomic_set(&ct->ct_general.use, 0);
1046	return ct;	1046	return ct;
1047	out:	1047	out:
1048	atomic_dec(&net->ct.count);	1048	atomic_dec(&net->ct.count);
1049	return ERR_PTR(-ENOMEM);	1049	return ERR_PTR(-ENOMEM);
1050	}	1050	}
1051		1051
1052	struct nf_conn nf_conntrack_alloc(struct net net,	1052	struct nf_conn nf_conntrack_alloc(struct net net,
1053	const struct nf_conntrack_zone *zone,	1053	const struct nf_conntrack_zone *zone,
1054	const struct nf_conntrack_tuple *orig,	1054	const struct nf_conntrack_tuple *orig,
1055	const struct nf_conntrack_tuple *repl,	1055	const struct nf_conntrack_tuple *repl,
1056	gfp_t gfp)	1056	gfp_t gfp)
1057	{	1057	{
1058	return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);	1058	return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
1059	}	1059	}
1060	EXPORT_SYMBOL_GPL(nf_conntrack_alloc);	1060	EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
1061		1061
1062	void nf_conntrack_free(struct nf_conn *ct)	1062	void nf_conntrack_free(struct nf_conn *ct)
1063	{	1063	{
1064	struct net *net = nf_ct_net(ct);	1064	struct net *net = nf_ct_net(ct);
1065		1065
1066	/* A freed object has refcnt == 0, that's	1066	/* A freed object has refcnt == 0, that's
1067	* the golden rule for SLAB_DESTROY_BY_RCU	1067	* the golden rule for SLAB_DESTROY_BY_RCU
1068	*/	1068	*/
1069	NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 0);	1069	NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 0);
1070		1070
1071	nf_ct_ext_destroy(ct);	1071	nf_ct_ext_destroy(ct);
1072	nf_ct_ext_free(ct);	1072	nf_ct_ext_free(ct);
1073	kmem_cache_free(nf_conntrack_cachep, ct);	1073	kmem_cache_free(nf_conntrack_cachep, ct);
1074	smp_mb__before_atomic();	1074	smp_mb__before_atomic();
1075	atomic_dec(&net->ct.count);	1075	atomic_dec(&net->ct.count);
1076	}	1076	}
1077	EXPORT_SYMBOL_GPL(nf_conntrack_free);	1077	EXPORT_SYMBOL_GPL(nf_conntrack_free);
1078		1078
1079		1079
1080	/* Allocate a new conntrack: we return -ENOMEM if classification	1080	/* Allocate a new conntrack: we return -ENOMEM if classification
1081	failed due to stress. Otherwise it really is unclassifiable. */	1081	failed due to stress. Otherwise it really is unclassifiable. */
1082	static struct nf_conntrack_tuple_hash *	1082	static struct nf_conntrack_tuple_hash *
1083	init_conntrack(struct net net, struct nf_conn tmpl,	1083	init_conntrack(struct net net, struct nf_conn tmpl,
1084	const struct nf_conntrack_tuple *tuple,	1084	const struct nf_conntrack_tuple *tuple,
1085	struct nf_conntrack_l3proto *l3proto,	1085	struct nf_conntrack_l3proto *l3proto,
1086	struct nf_conntrack_l4proto *l4proto,	1086	struct nf_conntrack_l4proto *l4proto,
1087	struct sk_buff *skb,	1087	struct sk_buff *skb,
1088	unsigned int dataoff, u32 hash)	1088	unsigned int dataoff, u32 hash)
1089	{	1089	{
1090	struct nf_conn *ct;	1090	struct nf_conn *ct;
1091	struct nf_conn_help *help;	1091	struct nf_conn_help *help;
1092	struct nf_conntrack_tuple repl_tuple;	1092	struct nf_conntrack_tuple repl_tuple;
1093	struct nf_conntrack_ecache *ecache;	1093	struct nf_conntrack_ecache *ecache;
1094	struct nf_conntrack_expect *exp = NULL;	1094	struct nf_conntrack_expect *exp = NULL;
1095	const struct nf_conntrack_zone *zone;	1095	const struct nf_conntrack_zone *zone;
1096	struct nf_conn_timeout *timeout_ext;	1096	struct nf_conn_timeout *timeout_ext;
1097	struct nf_conntrack_zone tmp;	1097	struct nf_conntrack_zone tmp;
1098	unsigned int *timeouts;	1098	unsigned int *timeouts;
1099		1099
1100	if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {	1100	if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
1101	pr_debug("Can't invert tuple.\n");	1101	pr_debug("Can't invert tuple.\n");
1102	return NULL;	1102	return NULL;
1103	}	1103	}
1104		1104
1105	zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);	1105	zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1106	ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,	1106	ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
1107	hash);	1107	hash);
1108	if (IS_ERR(ct))	1108	if (IS_ERR(ct))
1109	return (struct nf_conntrack_tuple_hash *)ct;	1109	return (struct nf_conntrack_tuple_hash *)ct;
1110		1110
1111	if (!nf_ct_add_synproxy(ct, tmpl)) {	1111	if (!nf_ct_add_synproxy(ct, tmpl)) {
1112	nf_conntrack_free(ct);	1112	nf_conntrack_free(ct);
1113	return ERR_PTR(-ENOMEM);	1113	return ERR_PTR(-ENOMEM);
1114	}	1114	}
1115		1115
1116	timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;	1116	timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
1117	if (timeout_ext) {	1117	if (timeout_ext) {
1118	timeouts = nf_ct_timeout_data(timeout_ext);	1118	timeouts = nf_ct_timeout_data(timeout_ext);
1119	if (unlikely(!timeouts))	1119	if (unlikely(!timeouts))
1120	timeouts = l4proto->get_timeouts(net);	1120	timeouts = l4proto->get_timeouts(net);
1121	} else {	1121	} else {
1122	timeouts = l4proto->get_timeouts(net);	1122	timeouts = l4proto->get_timeouts(net);
1123	}	1123	}
1124		1124
1125	if (!l4proto->new(ct, skb, dataoff, timeouts)) {	1125	if (!l4proto->new(ct, skb, dataoff, timeouts)) {
1126	nf_conntrack_free(ct);	1126	nf_conntrack_free(ct);
1127	pr_debug("can't track with proto module\n");	1127	pr_debug("can't track with proto module\n");
1128	return NULL;	1128	return NULL;
1129	}	1129	}
1130		1130
1131	if (timeout_ext)	1131	if (timeout_ext)
1132	nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout),	1132	nf_ct_timeout_ext_add(ct, rcu_dereference(timeout_ext->timeout),
1133	GFP_ATOMIC);	1133	GFP_ATOMIC);
1134		1134
1135	nf_ct_acct_ext_add(ct, GFP_ATOMIC);	1135	nf_ct_acct_ext_add(ct, GFP_ATOMIC);
1136	nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);	1136	nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
1137	nf_ct_labels_ext_add(ct);	1137	nf_ct_labels_ext_add(ct);
1138		1138
1139	ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;	1139	ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
1140	nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,	1140	nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
1141	ecache ? ecache->expmask : 0,	1141	ecache ? ecache->expmask : 0,
1142	GFP_ATOMIC);	1142	GFP_ATOMIC);
1143		1143
1144	local_bh_disable();	1144	local_bh_disable();
1145	if (net->ct.expect_count) {	1145	if (net->ct.expect_count) {
1146	spin_lock(&nf_conntrack_expect_lock);	1146	spin_lock(&nf_conntrack_expect_lock);
1147	exp = nf_ct_find_expectation(net, zone, tuple);	1147	exp = nf_ct_find_expectation(net, zone, tuple);
1148	if (exp) {	1148	if (exp) {
1149	pr_debug("expectation arrives ct=%p exp=%p\n",	1149	pr_debug("expectation arrives ct=%p exp=%p\n",
1150	ct, exp);	1150	ct, exp);
1151	/* Welcome, Mr. Bond. We've been expecting you... */	1151	/* Welcome, Mr. Bond. We've been expecting you... */
1152	__set_bit(IPS_EXPECTED_BIT, &ct->status);	1152	__set_bit(IPS_EXPECTED_BIT, &ct->status);
1153	/* exp->master safe, refcnt bumped in nf_ct_find_expectation */	1153	/* exp->master safe, refcnt bumped in nf_ct_find_expectation */
1154	ct->master = exp->master;	1154	ct->master = exp->master;
1155	if (exp->helper) {	1155	if (exp->helper) {
1156	help = nf_ct_helper_ext_add(ct, exp->helper,	1156	help = nf_ct_helper_ext_add(ct, exp->helper,
1157	GFP_ATOMIC);	1157	GFP_ATOMIC);
1158	if (help)	1158	if (help)
1159	rcu_assign_pointer(help->helper, exp->helper);	1159	rcu_assign_pointer(help->helper, exp->helper);
1160	}	1160	}
1161		1161
1162	#ifdef CONFIG_NF_CONNTRACK_MARK	1162	#ifdef CONFIG_NF_CONNTRACK_MARK
1163	ct->mark = exp->master->mark;	1163	ct->mark = exp->master->mark;
1164	#endif	1164	#endif
1165	#ifdef CONFIG_NF_CONNTRACK_SECMARK	1165	#ifdef CONFIG_NF_CONNTRACK_SECMARK
1166	ct->secmark = exp->master->secmark;	1166	ct->secmark = exp->master->secmark;
1167	#endif	1167	#endif
1168	NF_CT_STAT_INC(net, expect_new);	1168	NF_CT_STAT_INC(net, expect_new);
1169	}	1169	}
1170	spin_unlock(&nf_conntrack_expect_lock);	1170	spin_unlock(&nf_conntrack_expect_lock);
1171	}	1171	}
1172	if (!exp)	1172	if (!exp)
1173	__nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);	1173	__nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
1174		1174
1175	/* Now it is inserted into the unconfirmed list, bump refcount */	1175	/* Now it is inserted into the unconfirmed list, bump refcount */
1176	nf_conntrack_get(&ct->ct_general);	1176	nf_conntrack_get(&ct->ct_general);
1177	nf_ct_add_to_unconfirmed_list(ct);	1177	nf_ct_add_to_unconfirmed_list(ct);
1178		1178
1179	local_bh_enable();	1179	local_bh_enable();
1180		1180
1181	if (exp) {	1181	if (exp) {
1182	if (exp->expectfn)	1182	if (exp->expectfn)
1183	exp->expectfn(ct, exp);	1183	exp->expectfn(ct, exp);
1184	nf_ct_expect_put(exp);	1184	nf_ct_expect_put(exp);
1185	}	1185	}
1186		1186
1187	return &ct->tuplehash[IP_CT_DIR_ORIGINAL];	1187	return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
1188	}	1188	}
1189		1189
1190	/* On success, returns conntrack ptr, sets skb->nfct and ctinfo */	1190	/* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
1191	static inline struct nf_conn *	1191	static inline struct nf_conn *
1192	resolve_normal_ct(struct net net, struct nf_conn tmpl,	1192	resolve_normal_ct(struct net net, struct nf_conn tmpl,
1193	struct sk_buff *skb,	1193	struct sk_buff *skb,
1194	unsigned int dataoff,	1194	unsigned int dataoff,
1195	u_int16_t l3num,	1195	u_int16_t l3num,
1196	u_int8_t protonum,	1196	u_int8_t protonum,
1197	struct nf_conntrack_l3proto *l3proto,	1197	struct nf_conntrack_l3proto *l3proto,
1198	struct nf_conntrack_l4proto *l4proto,	1198	struct nf_conntrack_l4proto *l4proto,
1199	int *set_reply,	1199	int *set_reply,
1200	enum ip_conntrack_info *ctinfo)	1200	enum ip_conntrack_info *ctinfo)
1201	{	1201	{
1202	const struct nf_conntrack_zone *zone;	1202	const struct nf_conntrack_zone *zone;
1203	struct nf_conntrack_tuple tuple;	1203	struct nf_conntrack_tuple tuple;
1204	struct nf_conntrack_tuple_hash *h;	1204	struct nf_conntrack_tuple_hash *h;
1205	struct nf_conntrack_zone tmp;	1205	struct nf_conntrack_zone tmp;
1206	struct nf_conn *ct;	1206	struct nf_conn *ct;
1207	u32 hash;	1207	u32 hash;
1208		1208
1209	if (!nf_ct_get_tuple(skb, skb_network_offset(skb),	1209	if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
1210	dataoff, l3num, protonum, net, &tuple, l3proto,	1210	dataoff, l3num, protonum, net, &tuple, l3proto,
1211	l4proto)) {	1211	l4proto)) {
1212	pr_debug("Can't get tuple\n");	1212	pr_debug("Can't get tuple\n");
1213	return NULL;	1213	return NULL;
1214	}	1214	}
1215		1215
1216	/* look for tuple match */	1216	/* look for tuple match */
1217	zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);	1217	zone = nf_ct_zone_tmpl(tmpl, skb, &tmp);
1218	hash = hash_conntrack_raw(&tuple, net);	1218	hash = hash_conntrack_raw(&tuple, net);
1219	h = __nf_conntrack_find_get(net, zone, &tuple, hash);	1219	h = __nf_conntrack_find_get(net, zone, &tuple, hash);
1220	if (!h) {	1220	if (!h) {
1221	h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,	1221	h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
1222	skb, dataoff, hash);	1222	skb, dataoff, hash);
1223	if (!h)	1223	if (!h)
1224	return NULL;	1224	return NULL;
1225	if (IS_ERR(h))	1225	if (IS_ERR(h))
1226	return (void *)h;	1226	return (void *)h;
1227	}	1227	}
1228	ct = nf_ct_tuplehash_to_ctrack(h);	1228	ct = nf_ct_tuplehash_to_ctrack(h);
1229		1229
1230	/* It exists; we have (non-exclusive) reference. */	1230	/* It exists; we have (non-exclusive) reference. */
1231	if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {	1231	if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
1232	*ctinfo = IP_CT_ESTABLISHED_REPLY;	1232	*ctinfo = IP_CT_ESTABLISHED_REPLY;
1233	/* Please set reply bit if this packet OK */	1233	/* Please set reply bit if this packet OK */
1234	*set_reply = 1;	1234	*set_reply = 1;
1235	} else {	1235	} else {
1236	/* Once we've had two way comms, always ESTABLISHED. */	1236	/* Once we've had two way comms, always ESTABLISHED. */
1237	if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {	1237	if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
1238	pr_debug("normal packet for %p\n", ct);	1238	pr_debug("normal packet for %p\n", ct);
1239	*ctinfo = IP_CT_ESTABLISHED;	1239	*ctinfo = IP_CT_ESTABLISHED;
1240	} else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {	1240	} else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
1241	pr_debug("related packet for %p\n", ct);	1241	pr_debug("related packet for %p\n", ct);
1242	*ctinfo = IP_CT_RELATED;	1242	*ctinfo = IP_CT_RELATED;
1243	} else {	1243	} else {
1244	pr_debug("new packet for %p\n", ct);	1244	pr_debug("new packet for %p\n", ct);
1245	*ctinfo = IP_CT_NEW;	1245	*ctinfo = IP_CT_NEW;
1246	}	1246	}
1247	*set_reply = 0;	1247	*set_reply = 0;
1248	}	1248	}
1249	skb->nfct = &ct->ct_general;	1249	skb->nfct = &ct->ct_general;
1250	skb->nfctinfo = *ctinfo;	1250	skb->nfctinfo = *ctinfo;
1251	return ct;	1251	return ct;
1252	}	1252	}
1253		1253
1254	unsigned int	1254	unsigned int
1255	nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,	1255	nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
1256	struct sk_buff *skb)	1256	struct sk_buff *skb)
1257	{	1257	{
1258	struct nf_conn ct, tmpl = NULL;	1258	struct nf_conn ct, tmpl = NULL;
1259	enum ip_conntrack_info ctinfo;	1259	enum ip_conntrack_info ctinfo;
1260	struct nf_conntrack_l3proto *l3proto;	1260	struct nf_conntrack_l3proto *l3proto;
1261	struct nf_conntrack_l4proto *l4proto;	1261	struct nf_conntrack_l4proto *l4proto;
1262	unsigned int *timeouts;	1262	unsigned int *timeouts;
1263	unsigned int dataoff;	1263	unsigned int dataoff;
1264	u_int8_t protonum;	1264	u_int8_t protonum;
1265	int set_reply = 0;	1265	int set_reply = 0;
1266	int ret;	1266	int ret;
1267		1267
1268	if (skb->nfct) {	1268	if (skb->nfct) {
1269	/* Previously seen (loopback or untracked)? Ignore. */	1269	/* Previously seen (loopback or untracked)? Ignore. */
1270	tmpl = (struct nf_conn *)skb->nfct;	1270	tmpl = (struct nf_conn *)skb->nfct;
1271	if (!nf_ct_is_template(tmpl)) {	1271	if (!nf_ct_is_template(tmpl)) {
1272	NF_CT_STAT_INC_ATOMIC(net, ignore);	1272	NF_CT_STAT_INC_ATOMIC(net, ignore);
1273	return NF_ACCEPT;	1273	return NF_ACCEPT;
1274	}	1274	}
1275	skb->nfct = NULL;	1275	skb->nfct = NULL;
1276	}	1276	}
1277		1277
1278	/* rcu_read_lock()ed by nf_hook_thresh */	1278	/* rcu_read_lock()ed by nf_hook_thresh */
1279	l3proto = __nf_ct_l3proto_find(pf);	1279	l3proto = __nf_ct_l3proto_find(pf);
1280	ret = l3proto->get_l4proto(skb, skb_network_offset(skb),	1280	ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
1281	&dataoff, &protonum);	1281	&dataoff, &protonum);
1282	if (ret <= 0) {	1282	if (ret <= 0) {
1283	pr_debug("not prepared to track yet or error occurred\n");	1283	pr_debug("not prepared to track yet or error occurred\n");
1284	NF_CT_STAT_INC_ATOMIC(net, error);	1284	NF_CT_STAT_INC_ATOMIC(net, error);
1285	NF_CT_STAT_INC_ATOMIC(net, invalid);	1285	NF_CT_STAT_INC_ATOMIC(net, invalid);
1286	ret = -ret;	1286	ret = -ret;
1287	goto out;	1287	goto out;
1288	}	1288	}
1289		1289
1290	l4proto = __nf_ct_l4proto_find(pf, protonum);	1290	l4proto = __nf_ct_l4proto_find(pf, protonum);
1291		1291
1292	/* It may be an special packet, error, unclean...	1292	/* It may be an special packet, error, unclean...
1293	* inverse of the return code tells to the netfilter	1293	* inverse of the return code tells to the netfilter
1294	* core what to do with the packet. */	1294	* core what to do with the packet. */
1295	if (l4proto->error != NULL) {	1295	if (l4proto->error != NULL) {
1296	ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,	1296	ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
1297	pf, hooknum);	1297	pf, hooknum);
1298	if (ret <= 0) {	1298	if (ret <= 0) {
1299	NF_CT_STAT_INC_ATOMIC(net, error);	1299	NF_CT_STAT_INC_ATOMIC(net, error);
1300	NF_CT_STAT_INC_ATOMIC(net, invalid);	1300	NF_CT_STAT_INC_ATOMIC(net, invalid);
1301	ret = -ret;	1301	ret = -ret;
1302	goto out;	1302	goto out;
1303	}	1303	}
1304	/* ICMP[v6] protocol trackers may assign one conntrack. */	1304	/* ICMP[v6] protocol trackers may assign one conntrack. */
1305	if (skb->nfct)	1305	if (skb->nfct)
1306	goto out;	1306	goto out;
1307	}	1307	}
1308		1308	repeat:
1309	ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,	1309	ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
1310	l3proto, l4proto, &set_reply, &ctinfo);	1310	l3proto, l4proto, &set_reply, &ctinfo);
1311	if (!ct) {	1311	if (!ct) {
1312	/* Not valid part of a connection */	1312	/* Not valid part of a connection */
1313	NF_CT_STAT_INC_ATOMIC(net, invalid);	1313	NF_CT_STAT_INC_ATOMIC(net, invalid);
1314	ret = NF_ACCEPT;	1314	ret = NF_ACCEPT;
1315	goto out;	1315	goto out;
1316	}	1316	}
1317		1317
1318	if (IS_ERR(ct)) {	1318	if (IS_ERR(ct)) {
1319	/* Too stressed to deal. */	1319	/* Too stressed to deal. */
1320	NF_CT_STAT_INC_ATOMIC(net, drop);	1320	NF_CT_STAT_INC_ATOMIC(net, drop);
1321	ret = NF_DROP;	1321	ret = NF_DROP;
1322	goto out;	1322	goto out;
1323	}	1323	}
1324		1324
1325	NF_CT_ASSERT(skb->nfct);	1325	NF_CT_ASSERT(skb->nfct);
1326		1326
1327	/* Decide what timeout policy we want to apply to this flow. */	1327	/* Decide what timeout policy we want to apply to this flow. */
1328	timeouts = nf_ct_timeout_lookup(net, ct, l4proto);	1328	timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
1329		1329
1330	ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);	1330	ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
1331	if (ret <= 0) {	1331	if (ret <= 0) {
1332	/* Invalid: inverse of the return code tells	1332	/* Invalid: inverse of the return code tells
1333	* the netfilter core what to do */	1333	* the netfilter core what to do */
1334	pr_debug("nf_conntrack_in: Can't track with proto module\n");	1334	pr_debug("nf_conntrack_in: Can't track with proto module\n");
1335	nf_conntrack_put(skb->nfct);	1335	nf_conntrack_put(skb->nfct);
1336	skb->nfct = NULL;	1336	skb->nfct = NULL;
1337	NF_CT_STAT_INC_ATOMIC(net, invalid);	1337	NF_CT_STAT_INC_ATOMIC(net, invalid);
1338	if (ret == -NF_DROP)	1338	if (ret == -NF_DROP)
1339	NF_CT_STAT_INC_ATOMIC(net, drop);	1339	NF_CT_STAT_INC_ATOMIC(net, drop);
1340	ret = -ret;	1340	ret = -ret;
1341	goto out;	1341	goto out;
1342	}	1342	}
1343		1343
1344	if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))	1344	if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
1345	nf_conntrack_event_cache(IPCT_REPLY, ct);	1345	nf_conntrack_event_cache(IPCT_REPLY, ct);
1346	out:	1346	out:
1347	if (tmpl) {	1347	if (tmpl) {
1348	/* Special case: we have to repeat this hook, assign the	1348	/* Special case: TCP tracker reports an attempt to reopen a
1349	* template again to this packet. We assume that this packet	1349	* closed/aborted connection. We have to go back and create a
1350	* has no conntrack assigned. This is used by nf_ct_tcp. */	1350	* fresh conntrack.
		1351	*/
1351	if (ret == NF_REPEAT)	1352	if (ret == NF_REPEAT)
1352	skb->nfct = (struct nf_conntrack *)tmpl;	1353	goto repeat;
1353	else	1354	else
1354	nf_ct_put(tmpl);	1355	nf_ct_put(tmpl);
1355	}	1356	}
1356		1357
1357	return ret;	1358	return ret;
1358	}	1359	}
1359	EXPORT_SYMBOL_GPL(nf_conntrack_in);	1360	EXPORT_SYMBOL_GPL(nf_conntrack_in);
1360		1361
1361	bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,	1362	bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
1362	const struct nf_conntrack_tuple *orig)	1363	const struct nf_conntrack_tuple *orig)
1363	{	1364	{
1364	bool ret;	1365	bool ret;
1365		1366
1366	rcu_read_lock();	1367	rcu_read_lock();
1367	ret = nf_ct_invert_tuple(inverse, orig,	1368	ret = nf_ct_invert_tuple(inverse, orig,
1368	__nf_ct_l3proto_find(orig->src.l3num),	1369	__nf_ct_l3proto_find(orig->src.l3num),
1369	__nf_ct_l4proto_find(orig->src.l3num,	1370	__nf_ct_l4proto_find(orig->src.l3num,
1370	orig->dst.protonum));	1371	orig->dst.protonum));
1371	rcu_read_unlock();	1372	rcu_read_unlock();
1372	return ret;	1373	return ret;
1373	}	1374	}
1374	EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);	1375	EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
1375		1376
1376	/* Alter reply tuple (maybe alter helper). This is for NAT, and is	1377	/* Alter reply tuple (maybe alter helper). This is for NAT, and is
1377	implicitly racy: see __nf_conntrack_confirm */	1378	implicitly racy: see __nf_conntrack_confirm */
1378	void nf_conntrack_alter_reply(struct nf_conn *ct,	1379	void nf_conntrack_alter_reply(struct nf_conn *ct,
1379	const struct nf_conntrack_tuple *newreply)	1380	const struct nf_conntrack_tuple *newreply)
1380	{	1381	{
1381	struct nf_conn_help *help = nfct_help(ct);	1382	struct nf_conn_help *help = nfct_help(ct);
1382		1383
1383	/* Should be unconfirmed, so not in hash table yet */	1384	/* Should be unconfirmed, so not in hash table yet */
1384	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));	1385	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
1385		1386
1386	pr_debug("Altering reply tuple of %p to ", ct);	1387	pr_debug("Altering reply tuple of %p to ", ct);
1387	nf_ct_dump_tuple(newreply);	1388	nf_ct_dump_tuple(newreply);
1388		1389
1389	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;	1390	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
1390	if (ct->master \|\| (help && !hlist_empty(&help->expectations)))	1391	if (ct->master \|\| (help && !hlist_empty(&help->expectations)))
1391	return;	1392	return;
1392		1393
1393	rcu_read_lock();	1394	rcu_read_lock();
1394	__nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);	1395	__nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
1395	rcu_read_unlock();	1396	rcu_read_unlock();
1396	}	1397	}
1397	EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);	1398	EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
1398		1399
1399	/* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */	1400	/* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
1400	void __nf_ct_refresh_acct(struct nf_conn *ct,	1401	void __nf_ct_refresh_acct(struct nf_conn *ct,
1401	enum ip_conntrack_info ctinfo,	1402	enum ip_conntrack_info ctinfo,
1402	const struct sk_buff *skb,	1403	const struct sk_buff *skb,
1403	unsigned long extra_jiffies,	1404	unsigned long extra_jiffies,
1404	int do_acct)	1405	int do_acct)
1405	{	1406	{
1406	NF_CT_ASSERT(skb);	1407	NF_CT_ASSERT(skb);
1407		1408
1408	/* Only update if this is not a fixed timeout */	1409	/* Only update if this is not a fixed timeout */
1409	if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))	1410	if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
1410	goto acct;	1411	goto acct;
1411		1412
1412	/* If not in hash table, timer will not be active yet */	1413	/* If not in hash table, timer will not be active yet */
1413	if (nf_ct_is_confirmed(ct))	1414	if (nf_ct_is_confirmed(ct))
1414	extra_jiffies += nfct_time_stamp;	1415	extra_jiffies += nfct_time_stamp;
1415		1416
1416	ct->timeout = extra_jiffies;	1417	ct->timeout = extra_jiffies;
1417	acct:	1418	acct:
1418	if (do_acct)	1419	if (do_acct)
1419	nf_ct_acct_update(ct, ctinfo, skb->len);	1420	nf_ct_acct_update(ct, ctinfo, skb->len);
1420	}	1421	}
1421	EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);	1422	EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
1422		1423
1423	bool nf_ct_kill_acct(struct nf_conn *ct,	1424	bool nf_ct_kill_acct(struct nf_conn *ct,
1424	enum ip_conntrack_info ctinfo,	1425	enum ip_conntrack_info ctinfo,
1425	const struct sk_buff *skb)	1426	const struct sk_buff *skb)
1426	{	1427	{
1427	nf_ct_acct_update(ct, ctinfo, skb->len);	1428	nf_ct_acct_update(ct, ctinfo, skb->len);
1428		1429
1429	return nf_ct_delete(ct, 0, 0);	1430	return nf_ct_delete(ct, 0, 0);
1430	}	1431	}
1431	EXPORT_SYMBOL_GPL(nf_ct_kill_acct);	1432	EXPORT_SYMBOL_GPL(nf_ct_kill_acct);
1432		1433
1433	#if IS_ENABLED(CONFIG_NF_CT_NETLINK)	1434	#if IS_ENABLED(CONFIG_NF_CT_NETLINK)
1434		1435
1435	#include <linux/netfilter/nfnetlink.h>	1436	#include <linux/netfilter/nfnetlink.h>
1436	#include <linux/netfilter/nfnetlink_conntrack.h>	1437	#include <linux/netfilter/nfnetlink_conntrack.h>
1437	#include <linux/mutex.h>	1438	#include <linux/mutex.h>
1438		1439
1439	/* Generic function for tcp/udp/sctp/dccp and alike. This needs to be	1440	/* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
1440	* in ip_conntrack_core, since we don't want the protocols to autoload	1441	* in ip_conntrack_core, since we don't want the protocols to autoload
1441	* or depend on ctnetlink */	1442	* or depend on ctnetlink */
1442	int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,	1443	int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
1443	const struct nf_conntrack_tuple *tuple)	1444	const struct nf_conntrack_tuple *tuple)
1444	{	1445	{
1445	if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) \|\|	1446	if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) \|\|
1446	nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))	1447	nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
1447	goto nla_put_failure;	1448	goto nla_put_failure;
1448	return 0;	1449	return 0;
1449		1450
1450	nla_put_failure:	1451	nla_put_failure:
1451	return -1;	1452	return -1;
1452	}	1453	}
1453	EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);	1454	EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
1454		1455
1455	const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {	1456	const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
1456	[CTA_PROTO_SRC_PORT] = { .type = NLA_U16 },	1457	[CTA_PROTO_SRC_PORT] = { .type = NLA_U16 },
1457	[CTA_PROTO_DST_PORT] = { .type = NLA_U16 },	1458	[CTA_PROTO_DST_PORT] = { .type = NLA_U16 },
1458	};	1459	};
1459	EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);	1460	EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
1460		1461
1461	int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],	1462	int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
1462	struct nf_conntrack_tuple *t)	1463	struct nf_conntrack_tuple *t)
1463	{	1464	{
1464	if (!tb[CTA_PROTO_SRC_PORT] \|\| !tb[CTA_PROTO_DST_PORT])	1465	if (!tb[CTA_PROTO_SRC_PORT] \|\| !tb[CTA_PROTO_DST_PORT])
1465	return -EINVAL;	1466	return -EINVAL;
1466		1467
1467	t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);	1468	t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
1468	t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);	1469	t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
1469		1470
1470	return 0;	1471	return 0;
1471	}	1472	}
1472	EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);	1473	EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
1473		1474
1474	int nf_ct_port_nlattr_tuple_size(void)	1475	int nf_ct_port_nlattr_tuple_size(void)
1475	{	1476	{
1476	return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);	1477	return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
1477	}	1478	}
1478	EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);	1479	EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
1479	#endif	1480	#endif
1480		1481
1481	/* Used by ipt_REJECT and ip6t_REJECT. */	1482	/* Used by ipt_REJECT and ip6t_REJECT. */
1482	static void nf_conntrack_attach(struct sk_buff nskb, const struct sk_buff skb)	1483	static void nf_conntrack_attach(struct sk_buff nskb, const struct sk_buff skb)
1483	{	1484	{
1484	struct nf_conn *ct;	1485	struct nf_conn *ct;
1485	enum ip_conntrack_info ctinfo;	1486	enum ip_conntrack_info ctinfo;
1486		1487
1487	/* This ICMP is in reverse direction to the packet which caused it */	1488	/* This ICMP is in reverse direction to the packet which caused it */
1488	ct = nf_ct_get(skb, &ctinfo);	1489	ct = nf_ct_get(skb, &ctinfo);
1489	if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)	1490	if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
1490	ctinfo = IP_CT_RELATED_REPLY;	1491	ctinfo = IP_CT_RELATED_REPLY;
1491	else	1492	else
1492	ctinfo = IP_CT_RELATED;	1493	ctinfo = IP_CT_RELATED;
1493		1494
1494	/* Attach to new skbuff, and increment count */	1495	/* Attach to new skbuff, and increment count */
1495	nskb->nfct = &ct->ct_general;	1496	nskb->nfct = &ct->ct_general;
1496	nskb->nfctinfo = ctinfo;	1497	nskb->nfctinfo = ctinfo;
1497	nf_conntrack_get(nskb->nfct);	1498	nf_conntrack_get(nskb->nfct);
1498	}	1499	}
1499		1500
1500	/* Bring out ya dead! */	1501	/* Bring out ya dead! */
1501	static struct nf_conn *	1502	static struct nf_conn *
1502	get_next_corpse(struct net net, int (iter)(struct nf_conn i, void data),	1503	get_next_corpse(struct net net, int (iter)(struct nf_conn i, void data),
1503	void data, unsigned int bucket)	1504	void data, unsigned int bucket)
1504	{	1505	{
1505	struct nf_conntrack_tuple_hash *h;	1506	struct nf_conntrack_tuple_hash *h;
1506	struct nf_conn *ct;	1507	struct nf_conn *ct;
1507	struct hlist_nulls_node *n;	1508	struct hlist_nulls_node *n;
1508	int cpu;	1509	int cpu;
1509	spinlock_t *lockp;	1510	spinlock_t *lockp;
1510		1511
1511	for (; bucket < nf_conntrack_htable_size; (bucket)++) {	1512	for (; bucket < nf_conntrack_htable_size; (bucket)++) {
1512	lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];	1513	lockp = &nf_conntrack_locks[*bucket % CONNTRACK_LOCKS];
1513	local_bh_disable();	1514	local_bh_disable();
1514	nf_conntrack_lock(lockp);	1515	nf_conntrack_lock(lockp);
1515	if (*bucket < nf_conntrack_htable_size) {	1516	if (*bucket < nf_conntrack_htable_size) {
1516	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnnode) {	1517	hlist_nulls_for_each_entry(h, n, &nf_conntrack_hash[*bucket], hnnode) {
1517	if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)	1518	if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
1518	continue;	1519	continue;
1519	ct = nf_ct_tuplehash_to_ctrack(h);	1520	ct = nf_ct_tuplehash_to_ctrack(h);
1520	if (net_eq(nf_ct_net(ct), net) &&	1521	if (net_eq(nf_ct_net(ct), net) &&
1521	iter(ct, data))	1522	iter(ct, data))
1522	goto found;	1523	goto found;
1523	}	1524	}
1524	}	1525	}
1525	spin_unlock(lockp);	1526	spin_unlock(lockp);
1526	local_bh_enable();	1527	local_bh_enable();
1527	cond_resched();	1528	cond_resched();
1528	}	1529	}
1529		1530
1530	for_each_possible_cpu(cpu) {	1531	for_each_possible_cpu(cpu) {
1531	struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);	1532	struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
1532		1533
1533	spin_lock_bh(&pcpu->lock);	1534	spin_lock_bh(&pcpu->lock);
1534	hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) {	1535	hlist_nulls_for_each_entry(h, n, &pcpu->unconfirmed, hnnode) {
1535	ct = nf_ct_tuplehash_to_ctrack(h);	1536	ct = nf_ct_tuplehash_to_ctrack(h);
1536	if (iter(ct, data))	1537	if (iter(ct, data))
1537	set_bit(IPS_DYING_BIT, &ct->status);	1538	set_bit(IPS_DYING_BIT, &ct->status);
1538	}	1539	}
1539	spin_unlock_bh(&pcpu->lock);	1540	spin_unlock_bh(&pcpu->lock);
1540	cond_resched();	1541	cond_resched();
1541	}	1542	}
1542	return NULL;	1543	return NULL;
1543	found:	1544	found:
1544	atomic_inc(&ct->ct_general.use);	1545	atomic_inc(&ct->ct_general.use);
1545	spin_unlock(lockp);	1546	spin_unlock(lockp);
1546	local_bh_enable();	1547	local_bh_enable();
1547	return ct;	1548	return ct;
1548	}	1549	}
1549		1550
1550	void nf_ct_iterate_cleanup(struct net *net,	1551	void nf_ct_iterate_cleanup(struct net *net,
1551	int (iter)(struct nf_conn i, void *data),	1552	int (iter)(struct nf_conn i, void *data),
1552	void *data, u32 portid, int report)	1553	void *data, u32 portid, int report)
1553	{	1554	{
1554	struct nf_conn *ct;	1555	struct nf_conn *ct;
1555	unsigned int bucket = 0;	1556	unsigned int bucket = 0;
1556		1557
1557	might_sleep();	1558	might_sleep();
1558		1559
1559	if (atomic_read(&net->ct.count) == 0)	1560	if (atomic_read(&net->ct.count) == 0)
1560	return;	1561	return;
1561		1562
1562	while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {	1563	while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
1563	/* Time to push up daises... */	1564	/* Time to push up daises... */
1564		1565
1565	nf_ct_delete(ct, portid, report);	1566	nf_ct_delete(ct, portid, report);
1566	nf_ct_put(ct);	1567	nf_ct_put(ct);
1567	cond_resched();	1568	cond_resched();
1568	}	1569	}
1569	}	1570	}
1570	EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);	1571	EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
1571		1572
1572	static int kill_all(struct nf_conn i, void data)	1573	static int kill_all(struct nf_conn i, void data)
1573	{	1574	{
1574	return 1;	1575	return 1;
1575	}	1576	}
1576		1577
1577	void nf_ct_free_hashtable(void *hash, unsigned int size)	1578	void nf_ct_free_hashtable(void *hash, unsigned int size)
1578	{	1579	{
1579	if (is_vmalloc_addr(hash))	1580	if (is_vmalloc_addr(hash))
1580	vfree(hash);	1581	vfree(hash);
1581	else	1582	else
1582	free_pages((unsigned long)hash,	1583	free_pages((unsigned long)hash,
1583	get_order(sizeof(struct hlist_head) * size));	1584	get_order(sizeof(struct hlist_head) * size));
1584	}	1585	}
1585	EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);	1586	EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
1586		1587
1587	static int untrack_refs(void)	1588	static int untrack_refs(void)
1588	{	1589	{
1589	int cnt = 0, cpu;	1590	int cnt = 0, cpu;
1590		1591
1591	for_each_possible_cpu(cpu) {	1592	for_each_possible_cpu(cpu) {
1592	struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);	1593	struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1593		1594
1594	cnt += atomic_read(&ct->ct_general.use) - 1;	1595	cnt += atomic_read(&ct->ct_general.use) - 1;
1595	}	1596	}
1596	return cnt;	1597	return cnt;
1597	}	1598	}
1598		1599
1599	void nf_conntrack_cleanup_start(void)	1600	void nf_conntrack_cleanup_start(void)
1600	{	1601	{
1601	conntrack_gc_work.exiting = true;	1602	conntrack_gc_work.exiting = true;
1602	RCU_INIT_POINTER(ip_ct_attach, NULL);	1603	RCU_INIT_POINTER(ip_ct_attach, NULL);
1603	}	1604	}
1604		1605
1605	void nf_conntrack_cleanup_end(void)	1606	void nf_conntrack_cleanup_end(void)
1606	{	1607	{
1607	RCU_INIT_POINTER(nf_ct_destroy, NULL);	1608	RCU_INIT_POINTER(nf_ct_destroy, NULL);
1608	while (untrack_refs() > 0)	1609	while (untrack_refs() > 0)
1609	schedule();	1610	schedule();
1610		1611
1611	cancel_delayed_work_sync(&conntrack_gc_work.dwork);	1612	cancel_delayed_work_sync(&conntrack_gc_work.dwork);
1612	nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_htable_size);	1613	nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_htable_size);
1613		1614
1614	nf_conntrack_proto_fini();	1615	nf_conntrack_proto_fini();
1615	nf_conntrack_seqadj_fini();	1616	nf_conntrack_seqadj_fini();
1616	nf_conntrack_labels_fini();	1617	nf_conntrack_labels_fini();
1617	nf_conntrack_helper_fini();	1618	nf_conntrack_helper_fini();
1618	nf_conntrack_timeout_fini();	1619	nf_conntrack_timeout_fini();
1619	nf_conntrack_ecache_fini();	1620	nf_conntrack_ecache_fini();
1620	nf_conntrack_tstamp_fini();	1621	nf_conntrack_tstamp_fini();
1621	nf_conntrack_acct_fini();	1622	nf_conntrack_acct_fini();
1622	nf_conntrack_expect_fini();	1623	nf_conntrack_expect_fini();
1623		1624
1624	kmem_cache_destroy(nf_conntrack_cachep);	1625	kmem_cache_destroy(nf_conntrack_cachep);
1625	}	1626	}
1626		1627
1627	/*	1628	/*
1628	* Mishearing the voices in his head, our hero wonders how he's	1629	* Mishearing the voices in his head, our hero wonders how he's
1629	* supposed to kill the mall.	1630	* supposed to kill the mall.
1630	*/	1631	*/
1631	void nf_conntrack_cleanup_net(struct net *net)	1632	void nf_conntrack_cleanup_net(struct net *net)
1632	{	1633	{
1633	LIST_HEAD(single);	1634	LIST_HEAD(single);
1634		1635
1635	list_add(&net->exit_list, &single);	1636	list_add(&net->exit_list, &single);
1636	nf_conntrack_cleanup_net_list(&single);	1637	nf_conntrack_cleanup_net_list(&single);
1637	}	1638	}
1638		1639
1639	void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)	1640	void nf_conntrack_cleanup_net_list(struct list_head *net_exit_list)
1640	{	1641	{
1641	int busy;	1642	int busy;
1642	struct net *net;	1643	struct net *net;
1643		1644
1644	/*	1645	/*
1645	* This makes sure all current packets have passed through	1646	* This makes sure all current packets have passed through
1646	* netfilter framework. Roll on, two-stage module	1647	* netfilter framework. Roll on, two-stage module
1647	* delete...	1648	* delete...
1648	*/	1649	*/
1649	synchronize_net();	1650	synchronize_net();
1650	i_see_dead_people:	1651	i_see_dead_people:
1651	busy = 0;	1652	busy = 0;
1652	list_for_each_entry(net, net_exit_list, exit_list) {	1653	list_for_each_entry(net, net_exit_list, exit_list) {
1653	nf_ct_iterate_cleanup(net, kill_all, NULL, 0, 0);	1654	nf_ct_iterate_cleanup(net, kill_all, NULL, 0, 0);
1654	if (atomic_read(&net->ct.count) != 0)	1655	if (atomic_read(&net->ct.count) != 0)
1655	busy = 1;	1656	busy = 1;
1656	}	1657	}
1657	if (busy) {	1658	if (busy) {
1658	schedule();	1659	schedule();
1659	goto i_see_dead_people;	1660	goto i_see_dead_people;
1660	}	1661	}
1661		1662
1662	list_for_each_entry(net, net_exit_list, exit_list) {	1663	list_for_each_entry(net, net_exit_list, exit_list) {
1663	nf_conntrack_proto_pernet_fini(net);	1664	nf_conntrack_proto_pernet_fini(net);
1664	nf_conntrack_helper_pernet_fini(net);	1665	nf_conntrack_helper_pernet_fini(net);
1665	nf_conntrack_ecache_pernet_fini(net);	1666	nf_conntrack_ecache_pernet_fini(net);
1666	nf_conntrack_tstamp_pernet_fini(net);	1667	nf_conntrack_tstamp_pernet_fini(net);
1667	nf_conntrack_acct_pernet_fini(net);	1668	nf_conntrack_acct_pernet_fini(net);
1668	nf_conntrack_expect_pernet_fini(net);	1669	nf_conntrack_expect_pernet_fini(net);
1669	free_percpu(net->ct.stat);	1670	free_percpu(net->ct.stat);
1670	free_percpu(net->ct.pcpu_lists);	1671	free_percpu(net->ct.pcpu_lists);
1671	}	1672	}
1672	}	1673	}
1673		1674
1674	void nf_ct_alloc_hashtable(unsigned int sizep, int nulls)	1675	void nf_ct_alloc_hashtable(unsigned int sizep, int nulls)
1675	{	1676	{
1676	struct hlist_nulls_head *hash;	1677	struct hlist_nulls_head *hash;
1677	unsigned int nr_slots, i;	1678	unsigned int nr_slots, i;
1678	size_t sz;	1679	size_t sz;
1679		1680
1680	if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head)))	1681	if (*sizep > (UINT_MAX / sizeof(struct hlist_nulls_head)))
1681	return NULL;	1682	return NULL;
1682		1683
1683	BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));	1684	BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
1684	nr_slots = sizep = roundup(sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));	1685	nr_slots = sizep = roundup(sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
1685		1686
1686	if (nr_slots > (UINT_MAX / sizeof(struct hlist_nulls_head)))	1687	if (nr_slots > (UINT_MAX / sizeof(struct hlist_nulls_head)))
1687	return NULL;	1688	return NULL;
1688		1689
1689	sz = nr_slots * sizeof(struct hlist_nulls_head);	1690	sz = nr_slots * sizeof(struct hlist_nulls_head);
1690	hash = (void *)__get_free_pages(GFP_KERNEL \| __GFP_NOWARN \| __GFP_ZERO,	1691	hash = (void *)__get_free_pages(GFP_KERNEL \| __GFP_NOWARN \| __GFP_ZERO,
1691	get_order(sz));	1692	get_order(sz));
1692	if (!hash)	1693	if (!hash)
1693	hash = vzalloc(sz);	1694	hash = vzalloc(sz);
1694		1695
1695	if (hash && nulls)	1696	if (hash && nulls)
1696	for (i = 0; i < nr_slots; i++)	1697	for (i = 0; i < nr_slots; i++)
1697	INIT_HLIST_NULLS_HEAD(&hash[i], i);	1698	INIT_HLIST_NULLS_HEAD(&hash[i], i);
1698		1699
1699	return hash;	1700	return hash;
1700	}	1701	}
1701	EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);	1702	EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
1702		1703
1703	int nf_conntrack_hash_resize(unsigned int hashsize)	1704	int nf_conntrack_hash_resize(unsigned int hashsize)
1704	{	1705	{
1705	int i, bucket;	1706	int i, bucket;
1706	unsigned int old_size;	1707	unsigned int old_size;
1707	struct hlist_nulls_head hash, old_hash;	1708	struct hlist_nulls_head hash, old_hash;
1708	struct nf_conntrack_tuple_hash *h;	1709	struct nf_conntrack_tuple_hash *h;
1709	struct nf_conn *ct;	1710	struct nf_conn *ct;
1710		1711
1711	if (!hashsize)	1712	if (!hashsize)
1712	return -EINVAL;	1713	return -EINVAL;
1713		1714
1714	hash = nf_ct_alloc_hashtable(&hashsize, 1);	1715	hash = nf_ct_alloc_hashtable(&hashsize, 1);
1715	if (!hash)	1716	if (!hash)
1716	return -ENOMEM;	1717	return -ENOMEM;
1717		1718
1718	old_size = nf_conntrack_htable_size;	1719	old_size = nf_conntrack_htable_size;
1719	if (old_size == hashsize) {	1720	if (old_size == hashsize) {
1720	nf_ct_free_hashtable(hash, hashsize);	1721	nf_ct_free_hashtable(hash, hashsize);
1721	return 0;	1722	return 0;
1722	}	1723	}
1723		1724
1724	local_bh_disable();	1725	local_bh_disable();
1725	nf_conntrack_all_lock();	1726	nf_conntrack_all_lock();
1726	write_seqcount_begin(&nf_conntrack_generation);	1727	write_seqcount_begin(&nf_conntrack_generation);
1727		1728
1728	/* Lookups in the old hash might happen in parallel, which means we	1729	/* Lookups in the old hash might happen in parallel, which means we
1729	* might get false negatives during connection lookup. New connections	1730	* might get false negatives during connection lookup. New connections
1730	* created because of a false negative won't make it into the hash	1731	* created because of a false negative won't make it into the hash
1731	* though since that required taking the locks.	1732	* though since that required taking the locks.
1732	*/	1733	*/
1733		1734
1734	for (i = 0; i < nf_conntrack_htable_size; i++) {	1735	for (i = 0; i < nf_conntrack_htable_size; i++) {
1735	while (!hlist_nulls_empty(&nf_conntrack_hash[i])) {	1736	while (!hlist_nulls_empty(&nf_conntrack_hash[i])) {
1736	h = hlist_nulls_entry(nf_conntrack_hash[i].first,	1737	h = hlist_nulls_entry(nf_conntrack_hash[i].first,
1737	struct nf_conntrack_tuple_hash, hnnode);	1738	struct nf_conntrack_tuple_hash, hnnode);
1738	ct = nf_ct_tuplehash_to_ctrack(h);	1739	ct = nf_ct_tuplehash_to_ctrack(h);
1739	hlist_nulls_del_rcu(&h->hnnode);	1740	hlist_nulls_del_rcu(&h->hnnode);
1740	bucket = __hash_conntrack(nf_ct_net(ct),	1741	bucket = __hash_conntrack(nf_ct_net(ct),
1741	&h->tuple, hashsize);	1742	&h->tuple, hashsize);
1742	hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);	1743	hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
1743	}	1744	}
1744	}	1745	}
1745	old_size = nf_conntrack_htable_size;	1746	old_size = nf_conntrack_htable_size;
1746	old_hash = nf_conntrack_hash;	1747	old_hash = nf_conntrack_hash;
1747		1748
1748	nf_conntrack_hash = hash;	1749	nf_conntrack_hash = hash;
1749	nf_conntrack_htable_size = hashsize;	1750	nf_conntrack_htable_size = hashsize;
1750		1751
1751	write_seqcount_end(&nf_conntrack_generation);	1752	write_seqcount_end(&nf_conntrack_generation);
1752	nf_conntrack_all_unlock();	1753	nf_conntrack_all_unlock();
1753	local_bh_enable();	1754	local_bh_enable();
1754		1755
1755	synchronize_net();	1756	synchronize_net();
1756	nf_ct_free_hashtable(old_hash, old_size);	1757	nf_ct_free_hashtable(old_hash, old_size);
1757	return 0;	1758	return 0;
1758	}	1759	}
1759		1760
1760	int nf_conntrack_set_hashsize(const char val, struct kernel_param kp)	1761	int nf_conntrack_set_hashsize(const char val, struct kernel_param kp)
1761	{	1762	{
1762	unsigned int hashsize;	1763	unsigned int hashsize;
1763	int rc;	1764	int rc;
1764		1765
1765	if (current->nsproxy->net_ns != &init_net)	1766	if (current->nsproxy->net_ns != &init_net)
1766	return -EOPNOTSUPP;	1767	return -EOPNOTSUPP;
1767		1768
1768	/* On boot, we can set this without any fancy locking. */	1769	/* On boot, we can set this without any fancy locking. */
1769	if (!nf_conntrack_htable_size)	1770	if (!nf_conntrack_htable_size)
1770	return param_set_uint(val, kp);	1771	return param_set_uint(val, kp);
1771		1772
1772	rc = kstrtouint(val, 0, &hashsize);	1773	rc = kstrtouint(val, 0, &hashsize);
1773	if (rc)	1774	if (rc)
1774	return rc;	1775	return rc;
1775		1776
1776	return nf_conntrack_hash_resize(hashsize);	1777	return nf_conntrack_hash_resize(hashsize);
1777	}	1778	}
1778	EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);	1779	EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
1779		1780
1780	module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,	1781	module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
1781	&nf_conntrack_htable_size, 0600);	1782	&nf_conntrack_htable_size, 0600);
1782		1783
1783	void nf_ct_untracked_status_or(unsigned long bits)	1784	void nf_ct_untracked_status_or(unsigned long bits)
1784	{	1785	{
1785	int cpu;	1786	int cpu;
1786		1787
1787	for_each_possible_cpu(cpu)	1788	for_each_possible_cpu(cpu)
1788	per_cpu(nf_conntrack_untracked, cpu).status \|= bits;	1789	per_cpu(nf_conntrack_untracked, cpu).status \|= bits;
1789	}	1790	}
1790	EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);	1791	EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
1791		1792
1792	int nf_conntrack_init_start(void)	1793	int nf_conntrack_init_start(void)
1793	{	1794	{
1794	int max_factor = 8;	1795	int max_factor = 8;
1795	int ret = -ENOMEM;	1796	int ret = -ENOMEM;
1796	int i, cpu;	1797	int i, cpu;
1797		1798
1798	seqcount_init(&nf_conntrack_generation);	1799	seqcount_init(&nf_conntrack_generation);
1799		1800
1800	for (i = 0; i < CONNTRACK_LOCKS; i++)	1801	for (i = 0; i < CONNTRACK_LOCKS; i++)
1801	spin_lock_init(&nf_conntrack_locks[i]);	1802	spin_lock_init(&nf_conntrack_locks[i]);
1802		1803
1803	if (!nf_conntrack_htable_size) {	1804	if (!nf_conntrack_htable_size) {
1804	/* Idea from tcp.c: use 1/16384 of memory.	1805	/* Idea from tcp.c: use 1/16384 of memory.
1805	* On i386: 32MB machine has 512 buckets.	1806	* On i386: 32MB machine has 512 buckets.
1806	* >= 1GB machines have 16384 buckets.	1807	* >= 1GB machines have 16384 buckets.
1807	* >= 4GB machines have 65536 buckets.	1808	* >= 4GB machines have 65536 buckets.
1808	*/	1809	*/
1809	nf_conntrack_htable_size	1810	nf_conntrack_htable_size
1810	= (((totalram_pages << PAGE_SHIFT) / 16384)	1811	= (((totalram_pages << PAGE_SHIFT) / 16384)
1811	/ sizeof(struct hlist_head));	1812	/ sizeof(struct hlist_head));
1812	if (totalram_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))	1813	if (totalram_pages > (4 * (1024 * 1024 * 1024 / PAGE_SIZE)))
1813	nf_conntrack_htable_size = 65536;	1814	nf_conntrack_htable_size = 65536;
1814	else if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))	1815	else if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
1815	nf_conntrack_htable_size = 16384;	1816	nf_conntrack_htable_size = 16384;
1816	if (nf_conntrack_htable_size < 32)	1817	if (nf_conntrack_htable_size < 32)
1817	nf_conntrack_htable_size = 32;	1818	nf_conntrack_htable_size = 32;
1818		1819
1819	/* Use a max. factor of four by default to get the same max as	1820	/* Use a max. factor of four by default to get the same max as
1820	* with the old struct list_heads. When a table size is given	1821	* with the old struct list_heads. When a table size is given
1821	* we use the old value of 8 to avoid reducing the max.	1822	* we use the old value of 8 to avoid reducing the max.
1822	* entries. */	1823	* entries. */
1823	max_factor = 4;	1824	max_factor = 4;
1824	}	1825	}
1825		1826
1826	nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1);	1827	nf_conntrack_hash = nf_ct_alloc_hashtable(&nf_conntrack_htable_size, 1);
1827	if (!nf_conntrack_hash)	1828	if (!nf_conntrack_hash)
1828	return -ENOMEM;	1829	return -ENOMEM;
1829		1830
1830	nf_conntrack_max = max_factor * nf_conntrack_htable_size;	1831	nf_conntrack_max = max_factor * nf_conntrack_htable_size;
1831		1832
1832	nf_conntrack_cachep = kmem_cache_create("nf_conntrack",	1833	nf_conntrack_cachep = kmem_cache_create("nf_conntrack",
1833	sizeof(struct nf_conn), 0,	1834	sizeof(struct nf_conn), 0,
1834	SLAB_DESTROY_BY_RCU \| SLAB_HWCACHE_ALIGN, NULL);	1835	SLAB_DESTROY_BY_RCU \| SLAB_HWCACHE_ALIGN, NULL);
1835	if (!nf_conntrack_cachep)	1836	if (!nf_conntrack_cachep)
1836	goto err_cachep;	1837	goto err_cachep;
1837		1838
1838	printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",	1839	printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
1839	NF_CONNTRACK_VERSION, nf_conntrack_htable_size,	1840	NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
1840	nf_conntrack_max);	1841	nf_conntrack_max);
1841		1842
1842	ret = nf_conntrack_expect_init();	1843	ret = nf_conntrack_expect_init();
1843	if (ret < 0)	1844	if (ret < 0)
1844	goto err_expect;	1845	goto err_expect;
1845		1846
1846	ret = nf_conntrack_acct_init();	1847	ret = nf_conntrack_acct_init();
1847	if (ret < 0)	1848	if (ret < 0)
1848	goto err_acct;	1849	goto err_acct;
1849		1850
1850	ret = nf_conntrack_tstamp_init();	1851	ret = nf_conntrack_tstamp_init();
1851	if (ret < 0)	1852	if (ret < 0)
1852	goto err_tstamp;	1853	goto err_tstamp;
1853		1854
1854	ret = nf_conntrack_ecache_init();	1855	ret = nf_conntrack_ecache_init();
1855	if (ret < 0)	1856	if (ret < 0)
1856	goto err_ecache;	1857	goto err_ecache;
1857		1858
1858	ret = nf_conntrack_timeout_init();	1859	ret = nf_conntrack_timeout_init();
1859	if (ret < 0)	1860	if (ret < 0)
1860	goto err_timeout;	1861	goto err_timeout;
1861		1862
1862	ret = nf_conntrack_helper_init();	1863	ret = nf_conntrack_helper_init();
1863	if (ret < 0)	1864	if (ret < 0)
1864	goto err_helper;	1865	goto err_helper;
1865		1866
1866	ret = nf_conntrack_labels_init();	1867	ret = nf_conntrack_labels_init();
1867	if (ret < 0)	1868	if (ret < 0)
1868	goto err_labels;	1869	goto err_labels;
1869		1870
1870	ret = nf_conntrack_seqadj_init();	1871	ret = nf_conntrack_seqadj_init();
1871	if (ret < 0)	1872	if (ret < 0)
1872	goto err_seqadj;	1873	goto err_seqadj;
1873		1874
1874	ret = nf_conntrack_proto_init();	1875	ret = nf_conntrack_proto_init();
1875	if (ret < 0)	1876	if (ret < 0)
1876	goto err_proto;	1877	goto err_proto;
1877		1878
1878	/* Set up fake conntrack: to never be deleted, not in any hashes */	1879	/* Set up fake conntrack: to never be deleted, not in any hashes */
1879	for_each_possible_cpu(cpu) {	1880	for_each_possible_cpu(cpu) {
1880	struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);	1881	struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
1881	write_pnet(&ct->ct_net, &init_net);	1882	write_pnet(&ct->ct_net, &init_net);
1882	atomic_set(&ct->ct_general.use, 1);	1883	atomic_set(&ct->ct_general.use, 1);
1883	}	1884	}
1884	/* - and look it like as a confirmed connection */	1885	/* - and look it like as a confirmed connection */
1885	nf_ct_untracked_status_or(IPS_CONFIRMED \| IPS_UNTRACKED);	1886	nf_ct_untracked_status_or(IPS_CONFIRMED \| IPS_UNTRACKED);
1886		1887
1887	conntrack_gc_work_init(&conntrack_gc_work);	1888	conntrack_gc_work_init(&conntrack_gc_work);
1888	schedule_delayed_work(&conntrack_gc_work.dwork, GC_INTERVAL);	1889	schedule_delayed_work(&conntrack_gc_work.dwork, GC_INTERVAL);
1889		1890
1890	return 0;	1891	return 0;
1891		1892
1892	err_proto:	1893	err_proto:
1893	nf_conntrack_seqadj_fini();	1894	nf_conntrack_seqadj_fini();
1894	err_seqadj:	1895	err_seqadj:
1895	nf_conntrack_labels_fini();	1896	nf_conntrack_labels_fini();
1896	err_labels:	1897	err_labels:
1897	nf_conntrack_helper_fini();	1898	nf_conntrack_helper_fini();
1898	err_helper:	1899	err_helper:
1899	nf_conntrack_timeout_fini();	1900	nf_conntrack_timeout_fini();
1900	err_timeout:	1901	err_timeout:
1901	nf_conntrack_ecache_fini();	1902	nf_conntrack_ecache_fini();
1902	err_ecache:	1903	err_ecache:
1903	nf_conntrack_tstamp_fini();	1904	nf_conntrack_tstamp_fini();
1904	err_tstamp:	1905	err_tstamp:
1905	nf_conntrack_acct_fini();	1906	nf_conntrack_acct_fini();
1906	err_acct:	1907	err_acct:
1907	nf_conntrack_expect_fini();	1908	nf_conntrack_expect_fini();
1908	err_expect:	1909	err_expect:
1909	kmem_cache_destroy(nf_conntrack_cachep);	1910	kmem_cache_destroy(nf_conntrack_cachep);
1910	err_cachep:	1911	err_cachep:
1911	nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_htable_size);	1912	nf_ct_free_hashtable(nf_conntrack_hash, nf_conntrack_htable_size);
1912	return ret;	1913	return ret;
1913	}	1914	}
1914		1915
1915	void nf_conntrack_init_end(void)	1916	void nf_conntrack_init_end(void)
1916	{	1917	{
1917	/* For use by REJECT target */	1918	/* For use by REJECT target */
1918	RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);	1919	RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
1919	RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);	1920	RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
1920	}	1921	}
1921		1922
1922	/*	1923	/*
1923	* We need to use special "null" values, not used in hash table	1924	* We need to use special "null" values, not used in hash table
1924	*/	1925	*/
1925	#define UNCONFIRMED_NULLS_VAL ((1<<30)+0)	1926	#define UNCONFIRMED_NULLS_VAL ((1<<30)+0)
1926	#define DYING_NULLS_VAL ((1<<30)+1)	1927	#define DYING_NULLS_VAL ((1<<30)+1)
1927	#define TEMPLATE_NULLS_VAL ((1<<30)+2)	1928	#define TEMPLATE_NULLS_VAL ((1<<30)+2)
1928		1929
1929	int nf_conntrack_init_net(struct net *net)	1930	int nf_conntrack_init_net(struct net *net)
1930	{	1931	{
1931	int ret = -ENOMEM;	1932	int ret = -ENOMEM;
1932	int cpu;	1933	int cpu;
1933		1934
1934	atomic_set(&net->ct.count, 0);	1935	atomic_set(&net->ct.count, 0);
1935		1936
1936	net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu);	1937	net->ct.pcpu_lists = alloc_percpu(struct ct_pcpu);
1937	if (!net->ct.pcpu_lists)	1938	if (!net->ct.pcpu_lists)
1938	goto err_stat;	1939	goto err_stat;
1939		1940
1940	for_each_possible_cpu(cpu) {	1941	for_each_possible_cpu(cpu) {
1941	struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);	1942	struct ct_pcpu *pcpu = per_cpu_ptr(net->ct.pcpu_lists, cpu);
1942		1943
1943	spin_lock_init(&pcpu->lock);	1944	spin_lock_init(&pcpu->lock);
1944	INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL);	1945	INIT_HLIST_NULLS_HEAD(&pcpu->unconfirmed, UNCONFIRMED_NULLS_VAL);
1945	INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL);	1946	INIT_HLIST_NULLS_HEAD(&pcpu->dying, DYING_NULLS_VAL);
1946	}	1947	}
1947		1948
1948	net->ct.stat = alloc_percpu(struct ip_conntrack_stat);	1949	net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
1949	if (!net->ct.stat)	1950	if (!net->ct.stat)
1950	goto err_pcpu_lists;	1951	goto err_pcpu_lists;
1951		1952
1952	ret = nf_conntrack_expect_pernet_init(net);	1953	ret = nf_conntrack_expect_pernet_init(net);
1953	if (ret < 0)	1954	if (ret < 0)
1954	goto err_expect;	1955	goto err_expect;
1955	ret = nf_conntrack_acct_pernet_init(net);	1956	ret = nf_conntrack_acct_pernet_init(net);
1956	if (ret < 0)	1957	if (ret < 0)
1957	goto err_acct;	1958	goto err_acct;
1958	ret = nf_conntrack_tstamp_pernet_init(net);	1959	ret = nf_conntrack_tstamp_pernet_init(net);
1959	if (ret < 0)	1960	if (ret < 0)
1960	goto err_tstamp;	1961	goto err_tstamp;
1961	ret = nf_conntrack_ecache_pernet_init(net);	1962	ret = nf_conntrack_ecache_pernet_init(net);
1962	if (ret < 0)	1963	if (ret < 0)
1963	goto err_ecache;	1964	goto err_ecache;
1964	ret = nf_conntrack_helper_pernet_init(net);	1965	ret = nf_conntrack_helper_pernet_init(net);
1965	if (ret < 0)	1966	if (ret < 0)
1966	goto err_helper;	1967	goto err_helper;
1967	ret = nf_conntrack_proto_pernet_init(net);	1968	ret = nf_conntrack_proto_pernet_init(net);
1968	if (ret < 0)	1969	if (ret < 0)
1969	goto err_proto;	1970	goto err_proto;
1970	return 0;	1971	return 0;
1971		1972
1972	err_proto:	1973	err_proto:
1973	nf_conntrack_helper_pernet_fini(net);	1974	nf_conntrack_helper_pernet_fini(net);
1974	err_helper:	1975	err_helper:
1975	nf_conntrack_ecache_pernet_fini(net);	1976	nf_conntrack_ecache_pernet_fini(net);
1976	err_ecache:	1977	err_ecache:
1977	nf_conntrack_tstamp_pernet_fini(net);	1978	nf_conntrack_tstamp_pernet_fini(net);
1978	err_tstamp:	1979	err_tstamp:
1979	nf_conntrack_acct_pernet_fini(net);	1980	nf_conntrack_acct_pernet_fini(net);
1980	err_acct:	1981	err_acct:
1981	nf_conntrack_expect_pernet_fini(net);	1982	nf_conntrack_expect_pernet_fini(net);
1982	err_expect:	1983	err_expect:
1983	free_percpu(net->ct.stat);	1984	free_percpu(net->ct.stat);
1984	err_pcpu_lists:	1985	err_pcpu_lists:
1985	free_percpu(net->ct.pcpu_lists);	1986	free_percpu(net->ct.pcpu_lists);
1986	err_stat:	1987	err_stat:
1987	return ret;	1988	return ret;
1988	}	1989	}
1989		1990

net/openvswitch/conntrack.c

Diff comments View file @ 08733a0

 /*
  * Copyright (c) 2015 Nicira, Inc.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of version 2 of the GNU General Public
  * License as published by the Free Software Foundation.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  * General Public License for more details.
  */
 #include <linux/module.h>
 #include <linux/openvswitch.h>
 #include <linux/tcp.h>
 #include <linux/udp.h>
 #include <linux/sctp.h>
 #include <net/ip.h>
 #include <net/netfilter/nf_conntrack_core.h>
 #include <net/netfilter/nf_conntrack_helper.h>
 #include <net/netfilter/nf_conntrack_labels.h>
 #include <net/netfilter/nf_conntrack_seqadj.h>
 #include <net/netfilter/nf_conntrack_zones.h>
 #include <net/netfilter/ipv6/nf_defrag_ipv6.h>
 #ifdef CONFIG_NF_NAT_NEEDED
 #include <linux/netfilter/nf_nat.h>
 #include <net/netfilter/nf_nat_core.h>
 #include <net/netfilter/nf_nat_l3proto.h>
 #endif
 #include "datapath.h"
 #include "conntrack.h"
 #include "flow.h"
 #include "flow_netlink.h"
 struct ovs_ct_len_tbl {
 	int maxlen;
 	int minlen;
 };
 /* Metadata mark for masked write to conntrack mark */
 struct md_mark {
 	u32 value;
 	u32 mask;
 };
 /* Metadata label for masked write to conntrack label. */
 struct md_labels {
 	struct ovs_key_ct_labels value;
 	struct ovs_key_ct_labels mask;
 };
 enum ovs_ct_nat {
 	OVS_CT_NAT = 1 << 0,     /* NAT for committed connections only. */
 	OVS_CT_SRC_NAT = 1 << 1, /* Source NAT for NEW connections. */
 	OVS_CT_DST_NAT = 1 << 2, /* Destination NAT for NEW connections. */
 };
 /* Conntrack action context for execution. */
 struct ovs_conntrack_info {
 	struct nf_conntrack_helper *helper;
 	struct nf_conntrack_zone zone;
 	struct nf_conn *ct;
 	u8 commit : 1;
 	u8 nat : 3;                 /* enum ovs_ct_nat */
 	u16 family;
 	struct md_mark mark;
 	struct md_labels labels;
 #ifdef CONFIG_NF_NAT_NEEDED
 	struct nf_nat_range range;  /* Only present for SRC NAT and DST NAT. */
 #endif
 };
 static void __ovs_ct_free_action(struct ovs_conntrack_info *ct_info);
 static u16 key_to_nfproto(const struct sw_flow_key *key)
 {
 	switch (ntohs(key->eth.type)) {
 	case ETH_P_IP:
 		return NFPROTO_IPV4;
 	case ETH_P_IPV6:
 		return NFPROTO_IPV6;
 	default:
 		return NFPROTO_UNSPEC;
 	}
 }
 /* Map SKB connection state into the values used by flow definition. */
 static u8 ovs_ct_get_state(enum ip_conntrack_info ctinfo)
 {
 	u8 ct_state = OVS_CS_F_TRACKED;
 	switch (ctinfo) {
 	case IP_CT_ESTABLISHED_REPLY:
 	case IP_CT_RELATED_REPLY:
 		ct_state |= OVS_CS_F_REPLY_DIR;
 		break;
 	default:
 		break;
 	}
 	switch (ctinfo) {
 	case IP_CT_ESTABLISHED:
 	case IP_CT_ESTABLISHED_REPLY:
 		ct_state |= OVS_CS_F_ESTABLISHED;
 		break;
 	case IP_CT_RELATED:
 	case IP_CT_RELATED_REPLY:
 		ct_state |= OVS_CS_F_RELATED;
 		break;
 	case IP_CT_NEW:
 		ct_state |= OVS_CS_F_NEW;
 		break;
 	default:
 		break;
 	}
 	return ct_state;
 }
 static u32 ovs_ct_get_mark(const struct nf_conn *ct)
 {
 #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
 	return ct ? ct->mark : 0;
 #else
 	return 0;
 #endif
 }
 static void ovs_ct_get_labels(const struct nf_conn *ct,
 			      struct ovs_key_ct_labels *labels)
 {
 	struct nf_conn_labels *cl = ct ? nf_ct_labels_find(ct) : NULL;
 	if (cl) {
 		size_t len = sizeof(cl->bits);
 		if (len > OVS_CT_LABELS_LEN)
 			len = OVS_CT_LABELS_LEN;
 		else if (len < OVS_CT_LABELS_LEN)
 			memset(labels, 0, OVS_CT_LABELS_LEN);
 		memcpy(labels, cl->bits, len);
 	} else {
 		memset(labels, 0, OVS_CT_LABELS_LEN);
 	}
 }
 static void __ovs_ct_update_key(struct sw_flow_key *key, u8 state,
 				const struct nf_conntrack_zone *zone,
 				const struct nf_conn *ct)
 {
 	key->ct.state = state;
 	key->ct.zone = zone->id;
 	key->ct.mark = ovs_ct_get_mark(ct);
 	ovs_ct_get_labels(ct, &key->ct.labels);
 }
 /* Update 'key' based on skb->nfct.  If 'post_ct' is true, then OVS has
  * previously sent the packet to conntrack via the ct action.  If
  * 'keep_nat_flags' is true, the existing NAT flags retained, else they are
  * initialized from the connection status.
  */
 static void ovs_ct_update_key(const struct sk_buff *skb,
 			      const struct ovs_conntrack_info *info,
 			      struct sw_flow_key *key, bool post_ct,
 			      bool keep_nat_flags)
 {
 	const struct nf_conntrack_zone *zone = &nf_ct_zone_dflt;
 	enum ip_conntrack_info ctinfo;
 	struct nf_conn *ct;
 	u8 state = 0;
 	ct = nf_ct_get(skb, &ctinfo);
 	if (ct) {
 		state = ovs_ct_get_state(ctinfo);
 		/* All unconfirmed entries are NEW connections. */
 		if (!nf_ct_is_confirmed(ct))
 			state |= OVS_CS_F_NEW;
 		/* OVS persists the related flag for the duration of the
 		 * connection.
 		 */
 		if (ct->master)
 			state |= OVS_CS_F_RELATED;
 		if (keep_nat_flags) {
 			state |= key->ct.state & OVS_CS_F_NAT_MASK;
 		} else {
 			if (ct->status & IPS_SRC_NAT)
 				state |= OVS_CS_F_SRC_NAT;
 			if (ct->status & IPS_DST_NAT)
 				state |= OVS_CS_F_DST_NAT;
 		}
 		zone = nf_ct_zone(ct);
 	} else if (post_ct) {
 		state = OVS_CS_F_TRACKED | OVS_CS_F_INVALID;
 		if (info)
 			zone = &info->zone;
 	}
 	__ovs_ct_update_key(key, state, zone, ct);
 }
 /* This is called to initialize CT key fields possibly coming in from the local
  * stack.
  */
 void ovs_ct_fill_key(const struct sk_buff *skb, struct sw_flow_key *key)
 {
 	ovs_ct_update_key(skb, NULL, key, false, false);
 }
 int ovs_ct_put_key(const struct sw_flow_key *key, struct sk_buff *skb)
 {
 	if (nla_put_u32(skb, OVS_KEY_ATTR_CT_STATE, key->ct.state))
 		return -EMSGSIZE;
 	if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
 	    nla_put_u16(skb, OVS_KEY_ATTR_CT_ZONE, key->ct.zone))
 		return -EMSGSIZE;
 	if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) &&
 	    nla_put_u32(skb, OVS_KEY_ATTR_CT_MARK, key->ct.mark))
 		return -EMSGSIZE;
 	if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
 	    nla_put(skb, OVS_KEY_ATTR_CT_LABELS, sizeof(key->ct.labels),
 		    &key->ct.labels))
 		return -EMSGSIZE;
 	return 0;
 }
 static int ovs_ct_set_mark(struct sk_buff *skb, struct sw_flow_key *key,
 			   u32 ct_mark, u32 mask)
 {
 #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
 	enum ip_conntrack_info ctinfo;
 	struct nf_conn *ct;
 	u32 new_mark;
 	/* The connection could be invalid, in which case set_mark is no-op. */
 	ct = nf_ct_get(skb, &ctinfo);
 	if (!ct)
 		return 0;
 	new_mark = ct_mark | (ct->mark & ~(mask));
 	if (ct->mark != new_mark) {
 		ct->mark = new_mark;
 		nf_conntrack_event_cache(IPCT_MARK, ct);
 		key->ct.mark = new_mark;
 	}
 	return 0;
 #else
 	return -ENOTSUPP;
 #endif
 }
 static int ovs_ct_set_labels(struct sk_buff *skb, struct sw_flow_key *key,
 			     const struct ovs_key_ct_labels *labels,
 			     const struct ovs_key_ct_labels *mask)
 {
 	enum ip_conntrack_info ctinfo;
 	struct nf_conn_labels *cl;
 	struct nf_conn *ct;
 	int err;
 	/* The connection could be invalid, in which case set_label is no-op.*/
 	ct = nf_ct_get(skb, &ctinfo);
 	if (!ct)
 		return 0;
 	cl = nf_ct_labels_find(ct);
 	if (!cl) {
 		nf_ct_labels_ext_add(ct);
 		cl = nf_ct_labels_find(ct);
 	}
 	if (!cl || sizeof(cl->bits) < OVS_CT_LABELS_LEN)
 		return -ENOSPC;
 	err = nf_connlabels_replace(ct, (u32 *)labels, (u32 *)mask,
 				    OVS_CT_LABELS_LEN / sizeof(u32));
 	if (err)
 		return err;
 	ovs_ct_get_labels(ct, &key->ct.labels);
 	return 0;
 }
 /* 'skb' should already be pulled to nh_ofs. */
 static int ovs_ct_helper(struct sk_buff *skb, u16 proto)
 {
 	const struct nf_conntrack_helper *helper;
 	const struct nf_conn_help *help;
 	enum ip_conntrack_info ctinfo;
 	unsigned int protoff;
 	struct nf_conn *ct;
 	int err;
 	ct = nf_ct_get(skb, &ctinfo);
 	if (!ct || ctinfo == IP_CT_RELATED_REPLY)
 		return NF_ACCEPT;
 	help = nfct_help(ct);
 	if (!help)
 		return NF_ACCEPT;
 	helper = rcu_dereference(help->helper);
 	if (!helper)
 		return NF_ACCEPT;
 	switch (proto) {
 	case NFPROTO_IPV4:
 		protoff = ip_hdrlen(skb);
 		break;
 	case NFPROTO_IPV6: {
 		u8 nexthdr = ipv6_hdr(skb)->nexthdr;
 		__be16 frag_off;
 		int ofs;
 		ofs = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &nexthdr,
 				       &frag_off);
 		if (ofs < 0 || (frag_off & htons(~0x7)) != 0) {
 			pr_debug("proto header not found\n");
 			return NF_ACCEPT;
 		}
 		protoff = ofs;
 		break;
 	}
 	default:
 		WARN_ONCE(1, "helper invoked on non-IP family!");
 		return NF_DROP;
 	}
 	err = helper->help(skb, protoff, ct, ctinfo);
 	if (err != NF_ACCEPT)
 		return err;
 	/* Adjust seqs after helper.  This is needed due to some helpers (e.g.,
 	 * FTP with NAT) adusting the TCP payload size when mangling IP
 	 * addresses and/or port numbers in the text-based control connection.
 	 */
 	if (test_bit(IPS_SEQ_ADJUST_BIT, &ct->status) &&
 	    !nf_ct_seq_adjust(skb, ct, ctinfo, protoff))
 		return NF_DROP;
 	return NF_ACCEPT;
 }
 /* Returns 0 on success, -EINPROGRESS if 'skb' is stolen, or other nonzero
  * value if 'skb' is freed.
  */
 static int handle_fragments(struct net *net, struct sw_flow_key *key,
 			    u16 zone, struct sk_buff *skb)
 {
 	struct ovs_skb_cb ovs_cb = *OVS_CB(skb);
 	int err;
 	if (key->eth.type == htons(ETH_P_IP)) {
 		enum ip_defrag_users user = IP_DEFRAG_CONNTRACK_IN + zone;
 		memset(IPCB(skb), 0, sizeof(struct inet_skb_parm));
 		err = ip_defrag(net, skb, user);
 		if (err)
 			return err;
 		ovs_cb.mru = IPCB(skb)->frag_max_size;
 #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV6)
 	} else if (key->eth.type == htons(ETH_P_IPV6)) {
 		enum ip6_defrag_users user = IP6_DEFRAG_CONNTRACK_IN + zone;
 		skb_orphan(skb);
 		memset(IP6CB(skb), 0, sizeof(struct inet6_skb_parm));
 		err = nf_ct_frag6_gather(net, skb, user);
 		if (err)
 			return err;
 		key->ip.proto = ipv6_hdr(skb)->nexthdr;
 		ovs_cb.mru = IP6CB(skb)->frag_max_size;
 #endif
 	} else {
 		kfree_skb(skb);
 		return -EPFNOSUPPORT;
 	}
 	key->ip.frag = OVS_FRAG_TYPE_NONE;
 	skb_clear_hash(skb);
 	skb->ignore_df = 1;
 	*OVS_CB(skb) = ovs_cb;
 	return 0;
 }
 static struct nf_conntrack_expect *
 ovs_ct_expect_find(struct net *net, const struct nf_conntrack_zone *zone,
 		   u16 proto, const struct sk_buff *skb)
 {
 	struct nf_conntrack_tuple tuple;
 	if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb), proto, net, &tuple))
 		return NULL;
 	return __nf_ct_expect_find(net, zone, &tuple);
 }
 /* This replicates logic from nf_conntrack_core.c that is not exported. */
 static enum ip_conntrack_info
 ovs_ct_get_info(const struct nf_conntrack_tuple_hash *h)
 {
 	const struct nf_conn *ct = nf_ct_tuplehash_to_ctrack(h);
 	if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY)
 		return IP_CT_ESTABLISHED_REPLY;
 	/* Once we've had two way comms, always ESTABLISHED. */
 	if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status))
 		return IP_CT_ESTABLISHED;
 	if (test_bit(IPS_EXPECTED_BIT, &ct->status))
 		return IP_CT_RELATED;
 	return IP_CT_NEW;
 }
 /* Find an existing connection which this packet belongs to without
  * re-attributing statistics or modifying the connection state.  This allows an
  * skb->nfct lost due to an upcall to be recovered during actions execution.
  *
  * Must be called with rcu_read_lock.
  *
  * On success, populates skb->nfct and skb->nfctinfo, and returns the
  * connection.  Returns NULL if there is no existing entry.
  */
 static struct nf_conn *
 ovs_ct_find_existing(struct net *net, const struct nf_conntrack_zone *zone,
 		     u8 l3num, struct sk_buff *skb)
 {
 	struct nf_conntrack_l3proto *l3proto;
 	struct nf_conntrack_l4proto *l4proto;
 	struct nf_conntrack_tuple tuple;
 	struct nf_conntrack_tuple_hash *h;
 	struct nf_conn *ct;
 	unsigned int dataoff;
 	u8 protonum;
 	l3proto = __nf_ct_l3proto_find(l3num);
 	if (l3proto->get_l4proto(skb, skb_network_offset(skb), &dataoff,
 				 &protonum) <= 0) {
 		pr_debug("ovs_ct_find_existing: Can't get protonum\n");
 		return NULL;
 	}
 	l4proto = __nf_ct_l4proto_find(l3num, protonum);
 	if (!nf_ct_get_tuple(skb, skb_network_offset(skb), dataoff, l3num,
 			     protonum, net, &tuple, l3proto, l4proto)) {
 		pr_debug("ovs_ct_find_existing: Can't get tuple\n");
 		return NULL;
 	}
 	/* look for tuple match */
 	h = nf_conntrack_find_get(net, zone, &tuple);
 	if (!h)
 		return NULL;   /* Not found. */
 	ct = nf_ct_tuplehash_to_ctrack(h);
 	skb->nfct = &ct->ct_general;
 	skb->nfctinfo = ovs_ct_get_info(h);
 	return ct;
 }
 /* Determine whether skb->nfct is equal to the result of conntrack lookup. */
 static bool skb_nfct_cached(struct net *net,
 			    const struct sw_flow_key *key,
 			    const struct ovs_conntrack_info *info,
 			    struct sk_buff *skb)
 {
 	enum ip_conntrack_info ctinfo;
 	struct nf_conn *ct;
 	ct = nf_ct_get(skb, &ctinfo);
 	/* If no ct, check if we have evidence that an existing conntrack entry
 	 * might be found for this skb.  This happens when we lose a skb->nfct
 	 * due to an upcall.  If the connection was not confirmed, it is not
 	 * cached and needs to be run through conntrack again.
 	 */
 	if (!ct && key->ct.state & OVS_CS_F_TRACKED &&
 	    !(key->ct.state & OVS_CS_F_INVALID) &&
 	    key->ct.zone == info->zone.id)
 		ct = ovs_ct_find_existing(net, &info->zone, info->family, skb);
 	if (!ct)
 		return false;
 	if (!net_eq(net, read_pnet(&ct->ct_net)))
 		return false;
 	if (!nf_ct_zone_equal_any(info->ct, nf_ct_zone(ct)))
 		return false;
 	if (info->helper) {
 		struct nf_conn_help *help;
 		help = nf_ct_ext_find(ct, NF_CT_EXT_HELPER);
 		if (help && rcu_access_pointer(help->helper) != info->helper)
 			return false;
 	}
 	return true;
 }
 #ifdef CONFIG_NF_NAT_NEEDED
 /* Modelled after nf_nat_ipv[46]_fn().
  * range is only used for new, uninitialized NAT state.
  * Returns either NF_ACCEPT or NF_DROP.
  */
 static int ovs_ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct,
 			      enum ip_conntrack_info ctinfo,
 			      const struct nf_nat_range *range,
 			      enum nf_nat_manip_type maniptype)
 {
 	int hooknum, nh_off, err = NF_ACCEPT;
 	nh_off = skb_network_offset(skb);
 	skb_pull(skb, nh_off);
 	/* See HOOK2MANIP(). */
 	if (maniptype == NF_NAT_MANIP_SRC)
 		hooknum = NF_INET_LOCAL_IN; /* Source NAT */
 	else
 		hooknum = NF_INET_LOCAL_OUT; /* Destination NAT */
 	switch (ctinfo) {
 	case IP_CT_RELATED:
 	case IP_CT_RELATED_REPLY:
 		if (IS_ENABLED(CONFIG_NF_NAT_IPV4) &&
 		    skb->protocol == htons(ETH_P_IP) &&
 		    ip_hdr(skb)->protocol == IPPROTO_ICMP) {
 			if (!nf_nat_icmp_reply_translation(skb, ct, ctinfo,
 							   hooknum))
 				err = NF_DROP;
 			goto push;
 		} else if (IS_ENABLED(CONFIG_NF_NAT_IPV6) &&
 			   skb->protocol == htons(ETH_P_IPV6)) {
 			__be16 frag_off;
 			u8 nexthdr = ipv6_hdr(skb)->nexthdr;
 			int hdrlen = ipv6_skip_exthdr(skb,
 						      sizeof(struct ipv6hdr),
 						      &nexthdr, &frag_off);
 			if (hdrlen >= 0 && nexthdr == IPPROTO_ICMPV6) {
 				if (!nf_nat_icmpv6_reply_translation(skb, ct,
 								     ctinfo,
 								     hooknum,
 								     hdrlen))
 					err = NF_DROP;
 				goto push;
 			}
 		}
 		/* Non-ICMP, fall thru to initialize if needed. */
 	case IP_CT_NEW:
 		/* Seen it before?  This can happen for loopback, retrans,
 		 * or local packets.
 		 */
 		if (!nf_nat_initialized(ct, maniptype)) {
 			/* Initialize according to the NAT action. */
 			err = (range && range->flags & NF_NAT_RANGE_MAP_IPS)
 				/* Action is set up to establish a new
 				 * mapping.
 				 */
 				? nf_nat_setup_info(ct, range, maniptype)
 				: nf_nat_alloc_null_binding(ct, hooknum);
 			if (err != NF_ACCEPT)
 				goto push;
 		}
 		break;
 	case IP_CT_ESTABLISHED:
 	case IP_CT_ESTABLISHED_REPLY:
 		break;
 	default:
 		err = NF_DROP;
 		goto push;
 	}
 	err = nf_nat_packet(ct, ctinfo, hooknum, skb);
 push:
 	skb_push(skb, nh_off);
 	return err;
 }
 static void ovs_nat_update_key(struct sw_flow_key *key,
 			       const struct sk_buff *skb,
 			       enum nf_nat_manip_type maniptype)
 {
 	if (maniptype == NF_NAT_MANIP_SRC) {
 		__be16 src;
 		key->ct.state |= OVS_CS_F_SRC_NAT;
 		if (key->eth.type == htons(ETH_P_IP))
 			key->ipv4.addr.src = ip_hdr(skb)->saddr;
 		else if (key->eth.type == htons(ETH_P_IPV6))
 			memcpy(&key->ipv6.addr.src, &ipv6_hdr(skb)->saddr,
 			       sizeof(key->ipv6.addr.src));
 		else
 			return;
 		if (key->ip.proto == IPPROTO_UDP)
 			src = udp_hdr(skb)->source;
 		else if (key->ip.proto == IPPROTO_TCP)
 			src = tcp_hdr(skb)->source;
 		else if (key->ip.proto == IPPROTO_SCTP)
 			src = sctp_hdr(skb)->source;
 		else
 			return;
 		key->tp.src = src;
 	} else {
 		__be16 dst;
 		key->ct.state |= OVS_CS_F_DST_NAT;
 		if (key->eth.type == htons(ETH_P_IP))
 			key->ipv4.addr.dst = ip_hdr(skb)->daddr;
 		else if (key->eth.type == htons(ETH_P_IPV6))
 			memcpy(&key->ipv6.addr.dst, &ipv6_hdr(skb)->daddr,
 			       sizeof(key->ipv6.addr.dst));
 		else
 			return;
 		if (key->ip.proto == IPPROTO_UDP)
 			dst = udp_hdr(skb)->dest;
 		else if (key->ip.proto == IPPROTO_TCP)
 			dst = tcp_hdr(skb)->dest;
 		else if (key->ip.proto == IPPROTO_SCTP)
 			dst = sctp_hdr(skb)->dest;
 		else
 			return;
 		key->tp.dst = dst;
 	}
 }
 /* Returns NF_DROP if the packet should be dropped, NF_ACCEPT otherwise. */
 static int ovs_ct_nat(struct net *net, struct sw_flow_key *key,
 		      const struct ovs_conntrack_info *info,
 		      struct sk_buff *skb, struct nf_conn *ct,
 		      enum ip_conntrack_info ctinfo)
 {
 	enum nf_nat_manip_type maniptype;
 	int err;
 	if (nf_ct_is_untracked(ct)) {
 		/* A NAT action may only be performed on tracked packets. */
 		return NF_ACCEPT;
 	}
 	/* Add NAT extension if not confirmed yet. */
 	if (!nf_ct_is_confirmed(ct) && !nf_ct_nat_ext_add(ct))
 		return NF_ACCEPT;   /* Can't NAT. */
 	/* Determine NAT type.
 	 * Check if the NAT type can be deduced from the tracked connection.
 	 * Make sure new expected connections (IP_CT_RELATED) are NATted only
 	 * when committing.
 	 */
 	if (info->nat & OVS_CT_NAT && ctinfo != IP_CT_NEW &&
 	    ct->status & IPS_NAT_MASK &&
 	    (ctinfo != IP_CT_RELATED || info->commit)) {
 		/* NAT an established or related connection like before. */
 		if (CTINFO2DIR(ctinfo) == IP_CT_DIR_REPLY)
 			/* This is the REPLY direction for a connection
 			 * for which NAT was applied in the forward
 			 * direction.  Do the reverse NAT.
 			 */
 			maniptype = ct->status & IPS_SRC_NAT
 				? NF_NAT_MANIP_DST : NF_NAT_MANIP_SRC;
 		else
 			maniptype = ct->status & IPS_SRC_NAT
 				? NF_NAT_MANIP_SRC : NF_NAT_MANIP_DST;
 	} else if (info->nat & OVS_CT_SRC_NAT) {
 		maniptype = NF_NAT_MANIP_SRC;
 	} else if (info->nat & OVS_CT_DST_NAT) {
 		maniptype = NF_NAT_MANIP_DST;
 	} else {
 		return NF_ACCEPT; /* Connection is not NATed. */
 	}
 	err = ovs_ct_nat_execute(skb, ct, ctinfo, &info->range, maniptype);
 	/* Mark NAT done if successful and update the flow key. */
 	if (err == NF_ACCEPT)
 		ovs_nat_update_key(key, skb, maniptype);
 	return err;
 }
 #else /* !CONFIG_NF_NAT_NEEDED */
 static int ovs_ct_nat(struct net *net, struct sw_flow_key *key,
 		      const struct ovs_conntrack_info *info,
 		      struct sk_buff *skb, struct nf_conn *ct,
 		      enum ip_conntrack_info ctinfo)
 {
 	return NF_ACCEPT;
 }
 #endif
 /* Pass 'skb' through conntrack in 'net', using zone configured in 'info', if
  * not done already.  Update key with new CT state after passing the packet
  * through conntrack.
  * Note that if the packet is deemed invalid by conntrack, skb->nfct will be
  * set to NULL and 0 will be returned.
  */
 static int __ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
 			   const struct ovs_conntrack_info *info,
 			   struct sk_buff *skb)
 {
 	/* If we are recirculating packets to match on conntrack fields and
 	 * committing with a separate conntrack action,  then we don't need to
 	 * actually run the packet through conntrack twice unless it's for a
 	 * different zone.
 	 */
 	bool cached = skb_nfct_cached(net, key, info, skb);
 	enum ip_conntrack_info ctinfo;
 	struct nf_conn *ct;
 	if (!cached) {
 		struct nf_conn *tmpl = info->ct;
 		int err;
 		/* Associate skb with specified zone. */
 		if (tmpl) {
 			if (skb->nfct)
 				nf_conntrack_put(skb->nfct);
 			nf_conntrack_get(&tmpl->ct_general);
 			skb->nfct = &tmpl->ct_general;
 			skb->nfctinfo = IP_CT_NEW;
 		}
-		/* Repeat if requested, see nf_iterate(). */
+		err = nf_conntrack_in(net, info->family,
-		do {
+				      NF_INET_PRE_ROUTING, skb);
-			err = nf_conntrack_in(net, info->family,
-					      NF_INET_PRE_ROUTING, skb);
-		} while (err == NF_REPEAT);
 		if (err != NF_ACCEPT)
 			return -ENOENT;
 		/* Clear CT state NAT flags to mark that we have not yet done
 		 * NAT after the nf_conntrack_in() call.  We can actually clear
 		 * the whole state, as it will be re-initialized below.
 		 */
 		key->ct.state = 0;
 		/* Update the key, but keep the NAT flags. */
 		ovs_ct_update_key(skb, info, key, true, true);
 	}
 	ct = nf_ct_get(skb, &ctinfo);
 	if (ct) {
 		/* Packets starting a new connection must be NATted before the
 		 * helper, so that the helper knows about the NAT.  We enforce
 		 * this by delaying both NAT and helper calls for unconfirmed
 		 * connections until the committing CT action.  For later
 		 * packets NAT and Helper may be called in either order.
 		 *
 		 * NAT will be done only if the CT action has NAT, and only
 		 * once per packet (per zone), as guarded by the NAT bits in
 		 * the key->ct.state.
 		 */
 		if (info->nat && !(key->ct.state & OVS_CS_F_NAT_MASK) &&
 		    (nf_ct_is_confirmed(ct) || info->commit) &&
 		    ovs_ct_nat(net, key, info, skb, ct, ctinfo) != NF_ACCEPT) {
 			return -EINVAL;
 		}
 		/* Userspace may decide to perform a ct lookup without a helper
 		 * specified followed by a (recirculate and) commit with one.
 		 * Therefore, for unconfirmed connections which we will commit,
 		 * we need to attach the helper here.
 		 */
 		if (!nf_ct_is_confirmed(ct) && info->commit &&
 		    info->helper && !nfct_help(ct)) {
 			int err = __nf_ct_try_assign_helper(ct, info->ct,
 							    GFP_ATOMIC);
 			if (err)
 				return err;
 		}
 		/* Call the helper only if:
 		 * - nf_conntrack_in() was executed above ("!cached") for a
 		 *   confirmed connection, or
 		 * - When committing an unconfirmed connection.
 		 */
 		if ((nf_ct_is_confirmed(ct) ? !cached : info->commit) &&
 		    ovs_ct_helper(skb, info->family) != NF_ACCEPT) {
 			return -EINVAL;
 		}
 	}
 	return 0;
 }
 /* Lookup connection and read fields into key. */
 static int ovs_ct_lookup(struct net *net, struct sw_flow_key *key,
 			 const struct ovs_conntrack_info *info,
 			 struct sk_buff *skb)
 {
 	struct nf_conntrack_expect *exp;
 	/* If we pass an expected packet through nf_conntrack_in() the
 	 * expectation is typically removed, but the packet could still be
 	 * lost in upcall processing.  To prevent this from happening we
 	 * perform an explicit expectation lookup.  Expected connections are
 	 * always new, and will be passed through conntrack only when they are
 	 * committed, as it is OK to remove the expectation at that time.
 	 */
 	exp = ovs_ct_expect_find(net, &info->zone, info->family, skb);
 	if (exp) {
 		u8 state;
 		/* NOTE: New connections are NATted and Helped only when
 		 * committed, so we are not calling into NAT here.
 		 */
 		state = OVS_CS_F_TRACKED | OVS_CS_F_NEW | OVS_CS_F_RELATED;
 		__ovs_ct_update_key(key, state, &info->zone, exp->master);
 	} else {
 		struct nf_conn *ct;
 		int err;
 		err = __ovs_ct_lookup(net, key, info, skb);
 		if (err)
 			return err;
 		ct = (struct nf_conn *)skb->nfct;
 		if (ct)
 			nf_ct_deliver_cached_events(ct);
 	}
 	return 0;
 }
 static bool labels_nonzero(const struct ovs_key_ct_labels *labels)
 {
 	size_t i;
 	for (i = 0; i < sizeof(*labels); i++)
 		if (labels->ct_labels[i])
 			return true;
 	return false;
 }
 /* Lookup connection and confirm if unconfirmed. */
 static int ovs_ct_commit(struct net *net, struct sw_flow_key *key,
 			 const struct ovs_conntrack_info *info,
 			 struct sk_buff *skb)
 {
 	int err;
 	err = __ovs_ct_lookup(net, key, info, skb);
 	if (err)
 		return err;
 	/* Apply changes before confirming the connection so that the initial
 	 * conntrack NEW netlink event carries the values given in the CT
 	 * action.
 	 */
 	if (info->mark.mask) {
 		err = ovs_ct_set_mark(skb, key, info->mark.value,
 				      info->mark.mask);
 		if (err)
 			return err;
 	}
 	if (labels_nonzero(&info->labels.mask)) {
 		err = ovs_ct_set_labels(skb, key, &info->labels.value,
 					&info->labels.mask);
 		if (err)
 			return err;
 	}
 	/* This will take care of sending queued events even if the connection
 	 * is already confirmed.
 	 */
 	if (nf_conntrack_confirm(skb) != NF_ACCEPT)
 		return -EINVAL;
 	return 0;
 }
 /* Returns 0 on success, -EINPROGRESS if 'skb' is stolen, or other nonzero
  * value if 'skb' is freed.
  */
 int ovs_ct_execute(struct net *net, struct sk_buff *skb,
 		   struct sw_flow_key *key,
 		   const struct ovs_conntrack_info *info)
 {
 	int nh_ofs;
 	int err;
 	/* The conntrack module expects to be working at L3. */
 	nh_ofs = skb_network_offset(skb);
 	skb_pull(skb, nh_ofs);
 	if (key->ip.frag != OVS_FRAG_TYPE_NONE) {
 		err = handle_fragments(net, key, info->zone.id, skb);
 		if (err)
 			return err;
 	}
 	if (info->commit)
 		err = ovs_ct_commit(net, key, info, skb);
 	else
 		err = ovs_ct_lookup(net, key, info, skb);
 	skb_push(skb, nh_ofs);
 	if (err)
 		kfree_skb(skb);
 	return err;
 }
 static int ovs_ct_add_helper(struct ovs_conntrack_info *info, const char *name,
 			     const struct sw_flow_key *key, bool log)
 {
 	struct nf_conntrack_helper *helper;
 	struct nf_conn_help *help;
 	helper = nf_conntrack_helper_try_module_get(name, info->family,
 						    key->ip.proto);
 	if (!helper) {
 		OVS_NLERR(log, "Unknown helper \"%s\"", name);
 		return -EINVAL;
 	}
 	help = nf_ct_helper_ext_add(info->ct, helper, GFP_KERNEL);
 	if (!help) {
 		module_put(helper->me);
 		return -ENOMEM;
 	}
 	rcu_assign_pointer(help->helper, helper);
 	info->helper = helper;
 	return 0;
 }
 #ifdef CONFIG_NF_NAT_NEEDED
 static int parse_nat(const struct nlattr *attr,
 		     struct ovs_conntrack_info *info, bool log)
 {
 	struct nlattr *a;
 	int rem;
 	bool have_ip_max = false;
 	bool have_proto_max = false;
 	bool ip_vers = (info->family == NFPROTO_IPV6);
 	nla_for_each_nested(a, attr, rem) {
 		static const int ovs_nat_attr_lens[OVS_NAT_ATTR_MAX + 1][2] = {
 			[OVS_NAT_ATTR_SRC] = {0, 0},
 			[OVS_NAT_ATTR_DST] = {0, 0},
 			[OVS_NAT_ATTR_IP_MIN] = {sizeof(struct in_addr),
 						 sizeof(struct in6_addr)},
 			[OVS_NAT_ATTR_IP_MAX] = {sizeof(struct in_addr),
 						 sizeof(struct in6_addr)},
 			[OVS_NAT_ATTR_PROTO_MIN] = {sizeof(u16), sizeof(u16)},
 			[OVS_NAT_ATTR_PROTO_MAX] = {sizeof(u16), sizeof(u16)},
 			[OVS_NAT_ATTR_PERSISTENT] = {0, 0},
 			[OVS_NAT_ATTR_PROTO_HASH] = {0, 0},
 			[OVS_NAT_ATTR_PROTO_RANDOM] = {0, 0},
 		};
 		int type = nla_type(a);
 		if (type > OVS_NAT_ATTR_MAX) {
 			OVS_NLERR(log,
 				  "Unknown NAT attribute (type=%d, max=%d).\n",
 				  type, OVS_NAT_ATTR_MAX);
 			return -EINVAL;
 		}
 		if (nla_len(a) != ovs_nat_attr_lens[type][ip_vers]) {
 			OVS_NLERR(log,
 				  "NAT attribute type %d has unexpected length (%d != %d).\n",
 				  type, nla_len(a),
 				  ovs_nat_attr_lens[type][ip_vers]);
 			return -EINVAL;
 		}
 		switch (type) {
 		case OVS_NAT_ATTR_SRC:
 		case OVS_NAT_ATTR_DST:
 			if (info->nat) {
 				OVS_NLERR(log,
 					  "Only one type of NAT may be specified.\n"
 					  );
 				return -ERANGE;
 			}
 			info->nat |= OVS_CT_NAT;
 			info->nat |= ((type == OVS_NAT_ATTR_SRC)
 					? OVS_CT_SRC_NAT : OVS_CT_DST_NAT);
 			break;
 		case OVS_NAT_ATTR_IP_MIN:
 			nla_memcpy(&info->range.min_addr, a,
 				   sizeof(info->range.min_addr));
 			info->range.flags |= NF_NAT_RANGE_MAP_IPS;
 			break;
 		case OVS_NAT_ATTR_IP_MAX:
 			have_ip_max = true;
 			nla_memcpy(&info->range.max_addr, a,
 				   sizeof(info->range.max_addr));
 			info->range.flags |= NF_NAT_RANGE_MAP_IPS;
 			break;
 		case OVS_NAT_ATTR_PROTO_MIN:
 			info->range.min_proto.all = htons(nla_get_u16(a));
 			info->range.flags |= NF_NAT_RANGE_PROTO_SPECIFIED;
 			break;
 		case OVS_NAT_ATTR_PROTO_MAX:
 			have_proto_max = true;
 			info->range.max_proto.all = htons(nla_get_u16(a));
 			info->range.flags |= NF_NAT_RANGE_PROTO_SPECIFIED;
 			break;
 		case OVS_NAT_ATTR_PERSISTENT:
 			info->range.flags |= NF_NAT_RANGE_PERSISTENT;
 			break;
 		case OVS_NAT_ATTR_PROTO_HASH:
 			info->range.flags |= NF_NAT_RANGE_PROTO_RANDOM;
 			break;
 		case OVS_NAT_ATTR_PROTO_RANDOM:
 			info->range.flags |= NF_NAT_RANGE_PROTO_RANDOM_FULLY;
 			break;
 		default:
 			OVS_NLERR(log, "Unknown nat attribute (%d).\n", type);
 			return -EINVAL;
 		}
 	}
 	if (rem > 0) {
 		OVS_NLERR(log, "NAT attribute has %d unknown bytes.\n", rem);
 		return -EINVAL;
 	}
 	if (!info->nat) {
 		/* Do not allow flags if no type is given. */
 		if (info->range.flags) {
 			OVS_NLERR(log,
 				  "NAT flags may be given only when NAT range (SRC or DST) is also specified.\n"
 				  );
 			return -EINVAL;
 		}
 		info->nat = OVS_CT_NAT;   /* NAT existing connections. */
 	} else if (!info->commit) {
 		OVS_NLERR(log,
 			  "NAT attributes may be specified only when CT COMMIT flag is also specified.\n"
 			  );
 		return -EINVAL;
 	}
 	/* Allow missing IP_MAX. */
 	if (info->range.flags & NF_NAT_RANGE_MAP_IPS && !have_ip_max) {
 		memcpy(&info->range.max_addr, &info->range.min_addr,
 		       sizeof(info->range.max_addr));
 	}
 	/* Allow missing PROTO_MAX. */
 	if (info->range.flags & NF_NAT_RANGE_PROTO_SPECIFIED &&
 	    !have_proto_max) {
 		info->range.max_proto.all = info->range.min_proto.all;
 	}
 	return 0;
 }
 #endif
 static const struct ovs_ct_len_tbl ovs_ct_attr_lens[OVS_CT_ATTR_MAX + 1] = {
 	[OVS_CT_ATTR_COMMIT]	= { .minlen = 0, .maxlen = 0 },
 	[OVS_CT_ATTR_ZONE]	= { .minlen = sizeof(u16),
 				    .maxlen = sizeof(u16) },
 	[OVS_CT_ATTR_MARK]	= { .minlen = sizeof(struct md_mark),
 				    .maxlen = sizeof(struct md_mark) },
 	[OVS_CT_ATTR_LABELS]	= { .minlen = sizeof(struct md_labels),
 				    .maxlen = sizeof(struct md_labels) },
 	[OVS_CT_ATTR_HELPER]	= { .minlen = 1,
 				    .maxlen = NF_CT_HELPER_NAME_LEN },
 #ifdef CONFIG_NF_NAT_NEEDED
 	/* NAT length is checked when parsing the nested attributes. */
 	[OVS_CT_ATTR_NAT]	= { .minlen = 0, .maxlen = INT_MAX },
 #endif
 };
 static int parse_ct(const struct nlattr *attr, struct ovs_conntrack_info *info,
 		    const char **helper, bool log)
 {
 	struct nlattr *a;
 	int rem;
 	nla_for_each_nested(a, attr, rem) {
 		int type = nla_type(a);
 		int maxlen = ovs_ct_attr_lens[type].maxlen;
 		int minlen = ovs_ct_attr_lens[type].minlen;
 		if (type > OVS_CT_ATTR_MAX) {
 			OVS_NLERR(log,
 				  "Unknown conntrack attr (type=%d, max=%d)",
 				  type, OVS_CT_ATTR_MAX);
 			return -EINVAL;
 		}
 		if (nla_len(a) < minlen || nla_len(a) > maxlen) {
 			OVS_NLERR(log,
 				  "Conntrack attr type has unexpected length (type=%d, length=%d, expected=%d)",
 				  type, nla_len(a), maxlen);
 			return -EINVAL;
 		}
 		switch (type) {
 		case OVS_CT_ATTR_COMMIT:
 			info->commit = true;
 			break;
 #ifdef CONFIG_NF_CONNTRACK_ZONES
 		case OVS_CT_ATTR_ZONE:
 			info->zone.id = nla_get_u16(a);
 			break;
 #endif
 #ifdef CONFIG_NF_CONNTRACK_MARK
 		case OVS_CT_ATTR_MARK: {
 			struct md_mark *mark = nla_data(a);
 			if (!mark->mask) {
 				OVS_NLERR(log, "ct_mark mask cannot be 0");
 				return -EINVAL;
 			}
 			info->mark = *mark;
 			break;
 		}
 #endif
 #ifdef CONFIG_NF_CONNTRACK_LABELS
 		case OVS_CT_ATTR_LABELS: {
 			struct md_labels *labels = nla_data(a);
 			if (!labels_nonzero(&labels->mask)) {
 				OVS_NLERR(log, "ct_labels mask cannot be 0");
 				return -EINVAL;
 			}
 			info->labels = *labels;
 			break;
 		}
 #endif
 		case OVS_CT_ATTR_HELPER:
 			*helper = nla_data(a);
 			if (!memchr(*helper, '\0', nla_len(a))) {
 				OVS_NLERR(log, "Invalid conntrack helper");
 				return -EINVAL;
 			}
 			break;
 #ifdef CONFIG_NF_NAT_NEEDED
 		case OVS_CT_ATTR_NAT: {
 			int err = parse_nat(a, info, log);
 			if (err)
 				return err;
 			break;
 		}
 #endif
 		default:
 			OVS_NLERR(log, "Unknown conntrack attr (%d)",
 				  type);
 			return -EINVAL;
 		}
 	}
 #ifdef CONFIG_NF_CONNTRACK_MARK
 	if (!info->commit && info->mark.mask) {
 		OVS_NLERR(log,
 			  "Setting conntrack mark requires 'commit' flag.");
 		return -EINVAL;
 	}
 #endif
 #ifdef CONFIG_NF_CONNTRACK_LABELS
 	if (!info->commit && labels_nonzero(&info->labels.mask)) {
 		OVS_NLERR(log,
 			  "Setting conntrack labels requires 'commit' flag.");
 		return -EINVAL;
 	}
 #endif
 	if (rem > 0) {
 		OVS_NLERR(log, "Conntrack attr has %d unknown bytes", rem);
 		return -EINVAL;
 	}
 	return 0;
 }
 bool ovs_ct_verify(struct net *net, enum ovs_key_attr attr)
 {
 	if (attr == OVS_KEY_ATTR_CT_STATE)
 		return true;
 	if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
 	    attr == OVS_KEY_ATTR_CT_ZONE)
 		return true;
 	if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) &&
 	    attr == OVS_KEY_ATTR_CT_MARK)
 		return true;
 	if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
 	    attr == OVS_KEY_ATTR_CT_LABELS) {
 		struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
 		return ovs_net->xt_label;
 	}
 	return false;
 }
 int ovs_ct_copy_action(struct net *net, const struct nlattr *attr,
 		       const struct sw_flow_key *key,
 		       struct sw_flow_actions **sfa,  bool log)
 {
 	struct ovs_conntrack_info ct_info;
 	const char *helper = NULL;
 	u16 family;
 	int err;
 	family = key_to_nfproto(key);
 	if (family == NFPROTO_UNSPEC) {
 		OVS_NLERR(log, "ct family unspecified");
 		return -EINVAL;
 	}
 	memset(&ct_info, 0, sizeof(ct_info));
 	ct_info.family = family;
 	nf_ct_zone_init(&ct_info.zone, NF_CT_DEFAULT_ZONE_ID,
 			NF_CT_DEFAULT_ZONE_DIR, 0);
 	err = parse_ct(attr, &ct_info, &helper, log);
 	if (err)
 		return err;
 	/* Set up template for tracking connections in specific zones. */
 	ct_info.ct = nf_ct_tmpl_alloc(net, &ct_info.zone, GFP_KERNEL);
 	if (!ct_info.ct) {
 		OVS_NLERR(log, "Failed to allocate conntrack template");
 		return -ENOMEM;
 	}
 	__set_bit(IPS_CONFIRMED_BIT, &ct_info.ct->status);
 	nf_conntrack_get(&ct_info.ct->ct_general);
 	if (helper) {
 		err = ovs_ct_add_helper(&ct_info, helper, key, log);
 		if (err)
 			goto err_free_ct;
 	}
 	err = ovs_nla_add_action(sfa, OVS_ACTION_ATTR_CT, &ct_info,
 				 sizeof(ct_info), log);
 	if (err)
 		goto err_free_ct;
 	return 0;
 err_free_ct:
 	__ovs_ct_free_action(&ct_info);
 	return err;
 }
 #ifdef CONFIG_NF_NAT_NEEDED
 static bool ovs_ct_nat_to_attr(const struct ovs_conntrack_info *info,
 			       struct sk_buff *skb)
 {
 	struct nlattr *start;
 	start = nla_nest_start(skb, OVS_CT_ATTR_NAT);
 	if (!start)
 		return false;
 	if (info->nat & OVS_CT_SRC_NAT) {
 		if (nla_put_flag(skb, OVS_NAT_ATTR_SRC))
 			return false;
 	} else if (info->nat & OVS_CT_DST_NAT) {
 		if (nla_put_flag(skb, OVS_NAT_ATTR_DST))
 			return false;
 	} else {
 		goto out;
 	}
 	if (info->range.flags & NF_NAT_RANGE_MAP_IPS) {
 		if (IS_ENABLED(CONFIG_NF_NAT_IPV4) &&
 		    info->family == NFPROTO_IPV4) {
 			if (nla_put_in_addr(skb, OVS_NAT_ATTR_IP_MIN,
 					    info->range.min_addr.ip) ||
 			    (info->range.max_addr.ip
 			     != info->range.min_addr.ip &&
 			     (nla_put_in_addr(skb, OVS_NAT_ATTR_IP_MAX,
 					      info->range.max_addr.ip))))
 				return false;
 		} else if (IS_ENABLED(CONFIG_NF_NAT_IPV6) &&
 			   info->family == NFPROTO_IPV6) {
 			if (nla_put_in6_addr(skb, OVS_NAT_ATTR_IP_MIN,
 					     &info->range.min_addr.in6) ||
 			    (memcmp(&info->range.max_addr.in6,
 				    &info->range.min_addr.in6,
 				    sizeof(info->range.max_addr.in6)) &&
 			     (nla_put_in6_addr(skb, OVS_NAT_ATTR_IP_MAX,
 					       &info->range.max_addr.in6))))
 				return false;
 		} else {
 			return false;
 		}
 	}
 	if (info->range.flags & NF_NAT_RANGE_PROTO_SPECIFIED &&
 	    (nla_put_u16(skb, OVS_NAT_ATTR_PROTO_MIN,
 			 ntohs(info->range.min_proto.all)) ||
 	     (info->range.max_proto.all != info->range.min_proto.all &&
 	      nla_put_u16(skb, OVS_NAT_ATTR_PROTO_MAX,
 			  ntohs(info->range.max_proto.all)))))
 		return false;
 	if (info->range.flags & NF_NAT_RANGE_PERSISTENT &&
 	    nla_put_flag(skb, OVS_NAT_ATTR_PERSISTENT))
 		return false;
 	if (info->range.flags & NF_NAT_RANGE_PROTO_RANDOM &&
 	    nla_put_flag(skb, OVS_NAT_ATTR_PROTO_HASH))
 		return false;
 	if (info->range.flags & NF_NAT_RANGE_PROTO_RANDOM_FULLY &&
 	    nla_put_flag(skb, OVS_NAT_ATTR_PROTO_RANDOM))
 		return false;
 out:
 	nla_nest_end(skb, start);
 	return true;
 }
 #endif
 int ovs_ct_action_to_attr(const struct ovs_conntrack_info *ct_info,
 			  struct sk_buff *skb)
 {
 	struct nlattr *start;
 	start = nla_nest_start(skb, OVS_ACTION_ATTR_CT);
 	if (!start)
 		return -EMSGSIZE;
 	if (ct_info->commit && nla_put_flag(skb, OVS_CT_ATTR_COMMIT))
 		return -EMSGSIZE;
 	if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
 	    nla_put_u16(skb, OVS_CT_ATTR_ZONE, ct_info->zone.id))
 		return -EMSGSIZE;
 	if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) && ct_info->mark.mask &&
 	    nla_put(skb, OVS_CT_ATTR_MARK, sizeof(ct_info->mark),
 		    &ct_info->mark))
 		return -EMSGSIZE;
 	if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
 	    labels_nonzero(&ct_info->labels.mask) &&
 	    nla_put(skb, OVS_CT_ATTR_LABELS, sizeof(ct_info->labels),
 		    &ct_info->labels))
 		return -EMSGSIZE;
 	if (ct_info->helper) {
 		if (nla_put_string(skb, OVS_CT_ATTR_HELPER,
 				   ct_info->helper->name))
 			return -EMSGSIZE;
 	}
 #ifdef CONFIG_NF_NAT_NEEDED
 	if (ct_info->nat && !ovs_ct_nat_to_attr(ct_info, skb))
 		return -EMSGSIZE;
 #endif
 	nla_nest_end(skb, start);
 	return 0;
 }
 void ovs_ct_free_action(const struct nlattr *a)
 {
 	struct ovs_conntrack_info *ct_info = nla_data(a);
 	__ovs_ct_free_action(ct_info);
 }
 static void __ovs_ct_free_action(struct ovs_conntrack_info *ct_info)
 {
 	if (ct_info->helper)
 		module_put(ct_info->helper->me);
 	if (ct_info->ct)
 		nf_ct_tmpl_free(ct_info->ct);
 }
 void ovs_ct_init(struct net *net)
 {
 	unsigned int n_bits = sizeof(struct ovs_key_ct_labels) * BITS_PER_BYTE;
 	struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
 	if (nf_connlabels_get(net, n_bits - 1)) {
 		ovs_net->xt_label = false;
 		OVS_NLERR(true, "Failed to set connlabel length");
 	} else {
 		ovs_net->xt_label = true;
 	}
 }
 void ovs_ct_exit(struct net *net)
 {
 	struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
 	if (ovs_net->xt_label)
 		nf_connlabels_put(net);
 }