Blame view

net/netfilter/nf_conntrack_core.c 42.8 KB
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1
2
3
4
5
  /* Connection state tracking for netfilter.  This is separated from,
     but required by, the NAT layer; it can also be used by an iptables
     extension. */
  
  /* (C) 1999-2001 Paul `Rusty' Russell
dc808fe28   Harald Welte   [NETFILTER] nf_co...
6
   * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
7
8
9
10
11
   * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
   *
   * This program is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License version 2 as
   * published by the Free Software Foundation.
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
12
   */
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
13
14
15
  #include <linux/types.h>
  #include <linux/netfilter.h>
  #include <linux/module.h>
d43c36dc6   Alexey Dobriyan   headers: remove s...
16
  #include <linux/sched.h>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
17
18
19
20
21
22
23
24
25
26
27
28
29
30
  #include <linux/skbuff.h>
  #include <linux/proc_fs.h>
  #include <linux/vmalloc.h>
  #include <linux/stddef.h>
  #include <linux/slab.h>
  #include <linux/random.h>
  #include <linux/jhash.h>
  #include <linux/err.h>
  #include <linux/percpu.h>
  #include <linux/moduleparam.h>
  #include <linux/notifier.h>
  #include <linux/kernel.h>
  #include <linux/netdevice.h>
  #include <linux/socket.h>
d7fe0f241   Al Viro   [PATCH] severing ...
31
  #include <linux/mm.h>
d696c7bda   Patrick McHardy   netfilter: nf_con...
32
  #include <linux/nsproxy.h>
ea781f197   Eric Dumazet   netfilter: nf_con...
33
  #include <linux/rculist_nulls.h>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
34

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
35
36
  #include <net/netfilter/nf_conntrack.h>
  #include <net/netfilter/nf_conntrack_l3proto.h>
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
37
  #include <net/netfilter/nf_conntrack_l4proto.h>
77ab9cff0   Martin Josefsson   [NETFILTER]: nf_c...
38
  #include <net/netfilter/nf_conntrack_expect.h>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
39
40
  #include <net/netfilter/nf_conntrack_helper.h>
  #include <net/netfilter/nf_conntrack_core.h>
ecfab2c9f   Yasuyuki Kozakai   [NETFILTER]: nf_c...
41
  #include <net/netfilter/nf_conntrack_extend.h>
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
42
  #include <net/netfilter/nf_conntrack_acct.h>
a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
43
  #include <net/netfilter/nf_conntrack_ecache.h>
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
44
  #include <net/netfilter/nf_conntrack_zones.h>
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
45
  #include <net/netfilter/nf_conntrack_timestamp.h>
e6a7d3c04   Pablo Neira Ayuso   netfilter: ctnetl...
46
  #include <net/netfilter/nf_nat.h>
e17b666a4   Patrick McHardy   netfilter: nf_con...
47
  #include <net/netfilter/nf_nat_core.h>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
48

dc808fe28   Harald Welte   [NETFILTER] nf_co...
49
  #define NF_CONNTRACK_VERSION	"0.5.0"
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
50

e17b666a4   Patrick McHardy   netfilter: nf_con...
51
52
  int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
  				      enum nf_nat_manip_type manip,
399383246   Patrick McHardy   netfilter: nfnetl...
53
  				      const struct nlattr *attr) __read_mostly;
e6a7d3c04   Pablo Neira Ayuso   netfilter: ctnetl...
54
  EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
55
  DEFINE_SPINLOCK(nf_conntrack_lock);
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
56
  EXPORT_SYMBOL_GPL(nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
57

e2b7606cd   Martin Josefsson   [NETFILTER]: More...
58
  unsigned int nf_conntrack_htable_size __read_mostly;
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
59
  EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
e478075c6   Hagen Paul Pfeifer   netfilter: nf_con...
60
  unsigned int nf_conntrack_max __read_mostly;
a999e6837   Patrick McHardy   [NETFILTER]: nf_c...
61
  EXPORT_SYMBOL_GPL(nf_conntrack_max);
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
62

b3c5163fe   Eric Dumazet   netfilter: nf_con...
63
64
  DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
  EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
65

f682cefa5   Changli Gao   netfilter: fix th...
66
  unsigned int nf_conntrack_hash_rnd __read_mostly;
4d4e61c6c   Patrick McHardy   netfilter: nf_nat...
67
  EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
68

99f07e91b   Changli Gao   netfilter: save t...
69
  static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
70
  {
0794935e2   Patrick McHardy   [NETFILTER]: nf_c...
71
  	unsigned int n;
0794935e2   Patrick McHardy   [NETFILTER]: nf_c...
72
73
74
75
76
77
  
  	/* The direction must be ignored, so we hash everything up to the
  	 * destination ports (which is a multiple of 4) and treat the last
  	 * three bytes manually.
  	 */
  	n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
99f07e91b   Changli Gao   netfilter: save t...
78
79
80
81
82
83
84
85
86
87
88
89
90
91
  	return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^
  		      (((__force __u16)tuple->dst.u.all << 16) |
  		      tuple->dst.protonum));
  }
  
  static u32 __hash_bucket(u32 hash, unsigned int size)
  {
  	return ((u64)hash * size) >> 32;
  }
  
  static u32 hash_bucket(u32 hash, const struct net *net)
  {
  	return __hash_bucket(hash, net->ct.htable_size);
  }
0794935e2   Patrick McHardy   [NETFILTER]: nf_c...
92

99f07e91b   Changli Gao   netfilter: save t...
93
94
95
96
  static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,
  				  u16 zone, unsigned int size)
  {
  	return __hash_bucket(hash_conntrack_raw(tuple, zone), size);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
97
  }
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
98
  static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,
d696c7bda   Patrick McHardy   netfilter: nf_con...
99
  				       const struct nf_conntrack_tuple *tuple)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
100
  {
99f07e91b   Changli Gao   netfilter: save t...
101
  	return __hash_conntrack(tuple, zone, net->ct.htable_size);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
102
  }
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
103
  bool
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
104
105
106
107
108
109
110
  nf_ct_get_tuple(const struct sk_buff *skb,
  		unsigned int nhoff,
  		unsigned int dataoff,
  		u_int16_t l3num,
  		u_int8_t protonum,
  		struct nf_conntrack_tuple *tuple,
  		const struct nf_conntrack_l3proto *l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
111
  		const struct nf_conntrack_l4proto *l4proto)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
112
  {
443a70d50   Philip Craig   netfilter: nf_con...
113
  	memset(tuple, 0, sizeof(*tuple));
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
114
115
116
  
  	tuple->src.l3num = l3num;
  	if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
117
  		return false;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
118
119
120
  
  	tuple->dst.protonum = protonum;
  	tuple->dst.dir = IP_CT_DIR_ORIGINAL;
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
121
  	return l4proto->pkt_to_tuple(skb, dataoff, tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
122
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
123
  EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
124

5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
125
126
  bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
  		       u_int16_t l3num, struct nf_conntrack_tuple *tuple)
e2a3123fb   Yasuyuki Kozakai   [NETFILTER]: nf_c...
127
128
129
130
131
132
133
134
135
136
137
138
139
  {
  	struct nf_conntrack_l3proto *l3proto;
  	struct nf_conntrack_l4proto *l4proto;
  	unsigned int protoff;
  	u_int8_t protonum;
  	int ret;
  
  	rcu_read_lock();
  
  	l3proto = __nf_ct_l3proto_find(l3num);
  	ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
  	if (ret != NF_ACCEPT) {
  		rcu_read_unlock();
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
140
  		return false;
e2a3123fb   Yasuyuki Kozakai   [NETFILTER]: nf_c...
141
142
143
144
145
146
147
148
149
150
151
  	}
  
  	l4proto = __nf_ct_l4proto_find(l3num, protonum);
  
  	ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,
  			      l3proto, l4proto);
  
  	rcu_read_unlock();
  	return ret;
  }
  EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
152
  bool
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
153
154
155
  nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
  		   const struct nf_conntrack_tuple *orig,
  		   const struct nf_conntrack_l3proto *l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
156
  		   const struct nf_conntrack_l4proto *l4proto)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
157
  {
443a70d50   Philip Craig   netfilter: nf_con...
158
  	memset(inverse, 0, sizeof(*inverse));
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
159
160
161
  
  	inverse->src.l3num = orig->src.l3num;
  	if (l3proto->invert_tuple(inverse, orig) == 0)
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
162
  		return false;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
163
164
165
166
  
  	inverse->dst.dir = !orig->dst.dir;
  
  	inverse->dst.protonum = orig->dst.protonum;
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
167
  	return l4proto->invert_tuple(inverse, orig);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
168
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
169
  EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
170

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
171
172
173
  static void
  clean_from_lists(struct nf_conn *ct)
  {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
174
175
  	pr_debug("clean_from_lists(%p)
  ", ct);
ea781f197   Eric Dumazet   netfilter: nf_con...
176
177
  	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
  	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
178
179
  
  	/* Destroy all pending expectations */
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
180
  	nf_ct_remove_expectations(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
181
182
183
184
185
186
  }
  
  static void
  destroy_conntrack(struct nf_conntrack *nfct)
  {
  	struct nf_conn *ct = (struct nf_conn *)nfct;
0d55af879   Alexey Dobriyan   netfilter: netns ...
187
  	struct net *net = nf_ct_net(ct);
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
188
  	struct nf_conntrack_l4proto *l4proto;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
189

0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
190
191
  	pr_debug("destroy_conntrack(%p)
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
192
193
  	NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
  	NF_CT_ASSERT(!timer_pending(&ct->timeout));
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
194
195
196
  	/* To make sure we don't get any weird locking issues here:
  	 * destroy_conntrack() MUST NOT be called with a write lock
  	 * to nf_conntrack_lock!!! -HW */
923f4902f   Patrick McHardy   [NETFILTER]: nf_c...
197
  	rcu_read_lock();
5e8fbe2ac   Patrick McHardy   [NETFILTER]: nf_c...
198
  	l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
199
200
  	if (l4proto && l4proto->destroy)
  		l4proto->destroy(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
201

982d9a9ce   Patrick McHardy   [NETFILTER]: nf_c...
202
  	rcu_read_unlock();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
203

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
204
  	spin_lock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
205
206
207
208
  	/* Expectations will have been removed in clean_from_lists,
  	 * except TFTP can create an expectation on the first packet,
  	 * before connection is in the list, so we need to clean here,
  	 * too. */
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
209
  	nf_ct_remove_expectations(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
210
211
212
  
  	/* We overload first tuple to link into unconfirmed list. */
  	if (!nf_ct_is_confirmed(ct)) {
ea781f197   Eric Dumazet   netfilter: nf_con...
213
214
  		BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
  		hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
215
  	}
0d55af879   Alexey Dobriyan   netfilter: netns ...
216
  	NF_CT_STAT_INC(net, delete);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
217
  	spin_unlock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
218
219
220
  
  	if (ct->master)
  		nf_ct_put(ct->master);
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
221
222
  	pr_debug("destroy_conntrack: returning ct=%p to slab
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
223
224
  	nf_conntrack_free(ct);
  }
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
225
  void nf_ct_delete_from_lists(struct nf_conn *ct)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
226
  {
0d55af879   Alexey Dobriyan   netfilter: netns ...
227
  	struct net *net = nf_ct_net(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
228

9858a3ae1   Pablo Neira Ayuso   netfilter: conntr...
229
  	nf_ct_helper_destroy(ct);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
230
  	spin_lock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
231
232
  	/* Inside lock so preempt is disabled on module removal path.
  	 * Otherwise we can get spurious warnings. */
0d55af879   Alexey Dobriyan   netfilter: netns ...
233
  	NF_CT_STAT_INC(net, delete_list);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
234
  	clean_from_lists(ct);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
235
  	spin_unlock_bh(&nf_conntrack_lock);
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
  }
  EXPORT_SYMBOL_GPL(nf_ct_delete_from_lists);
  
  static void death_by_event(unsigned long ul_conntrack)
  {
  	struct nf_conn *ct = (void *)ul_conntrack;
  	struct net *net = nf_ct_net(ct);
  
  	if (nf_conntrack_event(IPCT_DESTROY, ct) < 0) {
  		/* bad luck, let's retry again */
  		ct->timeout.expires = jiffies +
  			(random32() % net->ct.sysctl_events_retry_timeout);
  		add_timer(&ct->timeout);
  		return;
  	}
  	/* we've got the event delivered, now it's dying */
  	set_bit(IPS_DYING_BIT, &ct->status);
  	spin_lock(&nf_conntrack_lock);
  	hlist_nulls_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
  	spin_unlock(&nf_conntrack_lock);
  	nf_ct_put(ct);
  }
  
  void nf_ct_insert_dying_list(struct nf_conn *ct)
  {
  	struct net *net = nf_ct_net(ct);
  
  	/* add this conntrack to the dying list */
  	spin_lock_bh(&nf_conntrack_lock);
  	hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
  			     &net->ct.dying);
  	spin_unlock_bh(&nf_conntrack_lock);
  	/* set a new timer to retry event delivery */
  	setup_timer(&ct->timeout, death_by_event, (unsigned long)ct);
  	ct->timeout.expires = jiffies +
  		(random32() % net->ct.sysctl_events_retry_timeout);
  	add_timer(&ct->timeout);
  }
  EXPORT_SYMBOL_GPL(nf_ct_insert_dying_list);
  
  static void death_by_timeout(unsigned long ul_conntrack)
  {
  	struct nf_conn *ct = (void *)ul_conntrack;
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
279
280
281
282
283
  	struct nf_conn_tstamp *tstamp;
  
  	tstamp = nf_conn_tstamp_find(ct);
  	if (tstamp && tstamp->stop == 0)
  		tstamp->stop = ktime_to_ns(ktime_get_real());
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
284
285
286
287
288
289
290
291
292
293
  
  	if (!test_bit(IPS_DYING_BIT, &ct->status) &&
  	    unlikely(nf_conntrack_event(IPCT_DESTROY, ct) < 0)) {
  		/* destroy event was not delivered */
  		nf_ct_delete_from_lists(ct);
  		nf_ct_insert_dying_list(ct);
  		return;
  	}
  	set_bit(IPS_DYING_BIT, &ct->status);
  	nf_ct_delete_from_lists(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
294
295
  	nf_ct_put(ct);
  }
ea781f197   Eric Dumazet   netfilter: nf_con...
296
297
298
299
300
301
302
  /*
   * Warning :
   * - Caller must take a reference on returned object
   *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
   * OR
   * - Caller must lock nf_conntrack_lock before calling this function
   */
99f07e91b   Changli Gao   netfilter: save t...
303
304
305
  static struct nf_conntrack_tuple_hash *
  ____nf_conntrack_find(struct net *net, u16 zone,
  		      const struct nf_conntrack_tuple *tuple, u32 hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
306
307
  {
  	struct nf_conntrack_tuple_hash *h;
ea781f197   Eric Dumazet   netfilter: nf_con...
308
  	struct hlist_nulls_node *n;
99f07e91b   Changli Gao   netfilter: save t...
309
  	unsigned int bucket = hash_bucket(hash, net);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
310

4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
311
312
313
314
  	/* Disable BHs the entire time since we normally need to disable them
  	 * at least once for the stats anyway.
  	 */
  	local_bh_disable();
ea781f197   Eric Dumazet   netfilter: nf_con...
315
  begin:
99f07e91b   Changli Gao   netfilter: save t...
316
  	hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
317
318
  		if (nf_ct_tuple_equal(tuple, &h->tuple) &&
  		    nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)) == zone) {
0d55af879   Alexey Dobriyan   netfilter: netns ...
319
  			NF_CT_STAT_INC(net, found);
4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
320
  			local_bh_enable();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
321
322
  			return h;
  		}
0d55af879   Alexey Dobriyan   netfilter: netns ...
323
  		NF_CT_STAT_INC(net, searched);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
324
  	}
ea781f197   Eric Dumazet   netfilter: nf_con...
325
326
327
328
329
  	/*
  	 * if the nulls value we got at the end of this lookup is
  	 * not the expected one, we must restart lookup.
  	 * We probably met an item that was moved to another chain.
  	 */
99f07e91b   Changli Gao   netfilter: save t...
330
  	if (get_nulls_value(n) != bucket) {
af740b2c8   Jesper Dangaard Brouer   netfilter: nf_con...
331
  		NF_CT_STAT_INC(net, search_restart);
ea781f197   Eric Dumazet   netfilter: nf_con...
332
  		goto begin;
af740b2c8   Jesper Dangaard Brouer   netfilter: nf_con...
333
  	}
4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
334
  	local_bh_enable();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
335
336
337
  
  	return NULL;
  }
99f07e91b   Changli Gao   netfilter: save t...
338
339
340
341
342
343
344
345
  
  struct nf_conntrack_tuple_hash *
  __nf_conntrack_find(struct net *net, u16 zone,
  		    const struct nf_conntrack_tuple *tuple)
  {
  	return ____nf_conntrack_find(net, zone, tuple,
  				     hash_conntrack_raw(tuple, zone));
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
346
  EXPORT_SYMBOL_GPL(__nf_conntrack_find);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
347
348
  
  /* Find a connection corresponding to a tuple. */
99f07e91b   Changli Gao   netfilter: save t...
349
350
351
  static struct nf_conntrack_tuple_hash *
  __nf_conntrack_find_get(struct net *net, u16 zone,
  			const struct nf_conntrack_tuple *tuple, u32 hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
352
353
  {
  	struct nf_conntrack_tuple_hash *h;
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
354
  	struct nf_conn *ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
355

76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
356
  	rcu_read_lock();
ea781f197   Eric Dumazet   netfilter: nf_con...
357
  begin:
99f07e91b   Changli Gao   netfilter: save t...
358
  	h = ____nf_conntrack_find(net, zone, tuple, hash);
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
359
360
  	if (h) {
  		ct = nf_ct_tuplehash_to_ctrack(h);
8d8890b77   Patrick McHardy   netfilter: nf_con...
361
362
  		if (unlikely(nf_ct_is_dying(ct) ||
  			     !atomic_inc_not_zero(&ct->ct_general.use)))
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
363
  			h = NULL;
ea781f197   Eric Dumazet   netfilter: nf_con...
364
  		else {
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
365
366
  			if (unlikely(!nf_ct_tuple_equal(tuple, &h->tuple) ||
  				     nf_ct_zone(ct) != zone)) {
ea781f197   Eric Dumazet   netfilter: nf_con...
367
368
369
370
  				nf_ct_put(ct);
  				goto begin;
  			}
  		}
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
371
372
  	}
  	rcu_read_unlock();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
373
374
375
  
  	return h;
  }
99f07e91b   Changli Gao   netfilter: save t...
376
377
378
379
380
381
382
383
  
  struct nf_conntrack_tuple_hash *
  nf_conntrack_find_get(struct net *net, u16 zone,
  		      const struct nf_conntrack_tuple *tuple)
  {
  	return __nf_conntrack_find_get(net, zone, tuple,
  				       hash_conntrack_raw(tuple, zone));
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
384
  EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
385

c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
386
387
  static void __nf_conntrack_hash_insert(struct nf_conn *ct,
  				       unsigned int hash,
601e68e10   YOSHIFUJI Hideaki   [NETFILTER]: Fix ...
388
  				       unsigned int repl_hash)
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
389
  {
400dad39d   Alexey Dobriyan   netfilter: netns ...
390
  	struct net *net = nf_ct_net(ct);
ea781f197   Eric Dumazet   netfilter: nf_con...
391
  	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
400dad39d   Alexey Dobriyan   netfilter: netns ...
392
  			   &net->ct.hash[hash]);
ea781f197   Eric Dumazet   netfilter: nf_con...
393
  	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
400dad39d   Alexey Dobriyan   netfilter: netns ...
394
  			   &net->ct.hash[repl_hash]);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
395
396
397
398
  }
  
  void nf_conntrack_hash_insert(struct nf_conn *ct)
  {
d696c7bda   Patrick McHardy   netfilter: nf_con...
399
  	struct net *net = nf_ct_net(ct);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
400
  	unsigned int hash, repl_hash;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
401
  	u16 zone;
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
402

5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
403
404
405
  	zone = nf_ct_zone(ct);
  	hash = hash_conntrack(net, zone, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
  	repl_hash = hash_conntrack(net, zone, &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
406

c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
407
  	__nf_conntrack_hash_insert(ct, hash, repl_hash);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
408
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
409
  EXPORT_SYMBOL_GPL(nf_conntrack_hash_insert);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
410

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
411
412
  /* Confirm a connection given skb; places it in hash table */
  int
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
413
  __nf_conntrack_confirm(struct sk_buff *skb)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
414
415
  {
  	unsigned int hash, repl_hash;
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
416
  	struct nf_conntrack_tuple_hash *h;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
417
  	struct nf_conn *ct;
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
418
  	struct nf_conn_help *help;
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
419
  	struct nf_conn_tstamp *tstamp;
ea781f197   Eric Dumazet   netfilter: nf_con...
420
  	struct hlist_nulls_node *n;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
421
  	enum ip_conntrack_info ctinfo;
400dad39d   Alexey Dobriyan   netfilter: netns ...
422
  	struct net *net;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
423
  	u16 zone;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
424

3db05fea5   Herbert Xu   [NETFILTER]: Repl...
425
  	ct = nf_ct_get(skb, &ctinfo);
400dad39d   Alexey Dobriyan   netfilter: netns ...
426
  	net = nf_ct_net(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
427
428
429
430
431
432
433
  
  	/* ipt_REJECT uses nf_conntrack_attach to attach related
  	   ICMP/TCP RST packets in other direction.  Actual packet
  	   which created connection will be IP_CT_NEW or for an
  	   expected connection, IP_CT_RELATED. */
  	if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
  		return NF_ACCEPT;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
434
  	zone = nf_ct_zone(ct);
99f07e91b   Changli Gao   netfilter: save t...
435
436
437
438
439
  	/* reuse the hash saved before */
  	hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
  	hash = hash_bucket(hash, net);
  	repl_hash = hash_conntrack(net, zone,
  				   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
440
441
442
443
444
  
  	/* We're not in hash table, and we refuse to set up related
  	   connections for unconfirmed conns.  But packet copies and
  	   REJECT will give spurious warnings here. */
  	/* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
25985edce   Lucas De Marchi   Fix common misspe...
445
  	/* No external references means no one else could have
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
446
447
  	   confirmed us. */
  	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
448
449
  	pr_debug("Confirming conntrack %p
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
450

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
451
  	spin_lock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
452

fc350777c   Joerg Marx   netfilter: nf_con...
453
454
455
456
457
458
459
460
461
  	/* We have to check the DYING flag inside the lock to prevent
  	   a race against nf_ct_get_next_corpse() possibly called from
  	   user context, else we insert an already 'dead' hash, blocking
  	   further use of that particular connection -JM */
  
  	if (unlikely(nf_ct_is_dying(ct))) {
  		spin_unlock_bh(&nf_conntrack_lock);
  		return NF_ACCEPT;
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
462
463
464
  	/* See if there's one in the list already, including reverse:
  	   NAT could have grabbed it without realizing, since we're
  	   not in the hash.  If there is, we lost race. */
ea781f197   Eric Dumazet   netfilter: nf_con...
465
  	hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
466
  		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
467
468
  				      &h->tuple) &&
  		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
469
  			goto out;
ea781f197   Eric Dumazet   netfilter: nf_con...
470
  	hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
471
  		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
472
473
  				      &h->tuple) &&
  		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
474
  			goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
475

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
476
  	/* Remove from unconfirmed list */
ea781f197   Eric Dumazet   netfilter: nf_con...
477
  	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
478

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
479
480
481
482
483
484
  	/* Timer relative to confirmation time, not original
  	   setting time, otherwise we'd get timer wrap in
  	   weird delay cases. */
  	ct->timeout.expires += jiffies;
  	add_timer(&ct->timeout);
  	atomic_inc(&ct->ct_general.use);
45eec3419   Changli Gao   netfilter: nf_con...
485
  	ct->status |= IPS_CONFIRMED;
5c8ec910e   Patrick McHardy   netfilter: nf_con...
486

a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
487
488
489
490
491
492
493
494
  	/* set conntrack timestamp, if enabled. */
  	tstamp = nf_conn_tstamp_find(ct);
  	if (tstamp) {
  		if (skb->tstamp.tv64 == 0)
  			__net_timestamp((struct sk_buff *)skb);
  
  		tstamp->start = ktime_to_ns(skb->tstamp);
  	}
5c8ec910e   Patrick McHardy   netfilter: nf_con...
495
496
497
498
499
500
  	/* Since the lookup is lockless, hash insertion must be done after
  	 * starting the timer and setting the CONFIRMED bit. The RCU barriers
  	 * guarantee that no other CPU can find the conntrack before the above
  	 * stores are visible.
  	 */
  	__nf_conntrack_hash_insert(ct, hash, repl_hash);
0d55af879   Alexey Dobriyan   netfilter: netns ...
501
  	NF_CT_STAT_INC(net, insert);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
502
  	spin_unlock_bh(&nf_conntrack_lock);
5c8ec910e   Patrick McHardy   netfilter: nf_con...
503

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
504
505
  	help = nfct_help(ct);
  	if (help && help->helper)
a71996fcc   Alexey Dobriyan   netfilter: netns ...
506
  		nf_conntrack_event_cache(IPCT_HELPER, ct);
17e6e4eac   Pablo Neira Ayuso   netfilter: conntr...
507

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
508
  	nf_conntrack_event_cache(master_ct(ct) ?
a71996fcc   Alexey Dobriyan   netfilter: netns ...
509
  				 IPCT_RELATED : IPCT_NEW, ct);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
510
  	return NF_ACCEPT;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
511

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
512
  out:
0d55af879   Alexey Dobriyan   netfilter: netns ...
513
  	NF_CT_STAT_INC(net, insert_failed);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
514
  	spin_unlock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
515
516
  	return NF_DROP;
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
517
  EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
518
519
520
521
522
523
524
  
  /* Returns true if a connection correspondings to the tuple (required
     for NAT). */
  int
  nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
  			 const struct nf_conn *ignored_conntrack)
  {
400dad39d   Alexey Dobriyan   netfilter: netns ...
525
  	struct net *net = nf_ct_net(ignored_conntrack);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
526
  	struct nf_conntrack_tuple_hash *h;
ea781f197   Eric Dumazet   netfilter: nf_con...
527
  	struct hlist_nulls_node *n;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
528
529
530
  	struct nf_conn *ct;
  	u16 zone = nf_ct_zone(ignored_conntrack);
  	unsigned int hash = hash_conntrack(net, zone, tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
531

4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
532
533
534
535
  	/* Disable BHs the entire time since we need to disable them at
  	 * least once for the stats anyway.
  	 */
  	rcu_read_lock_bh();
ea781f197   Eric Dumazet   netfilter: nf_con...
536
  	hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
537
538
539
540
  		ct = nf_ct_tuplehash_to_ctrack(h);
  		if (ct != ignored_conntrack &&
  		    nf_ct_tuple_equal(tuple, &h->tuple) &&
  		    nf_ct_zone(ct) == zone) {
0d55af879   Alexey Dobriyan   netfilter: netns ...
541
  			NF_CT_STAT_INC(net, found);
4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
542
  			rcu_read_unlock_bh();
ba419aff2   Patrick McHardy   [NETFILTER]: nf_c...
543
544
  			return 1;
  		}
0d55af879   Alexey Dobriyan   netfilter: netns ...
545
  		NF_CT_STAT_INC(net, searched);
ba419aff2   Patrick McHardy   [NETFILTER]: nf_c...
546
  	}
4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
547
  	rcu_read_unlock_bh();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
548

ba419aff2   Patrick McHardy   [NETFILTER]: nf_c...
549
  	return 0;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
550
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
551
  EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
552

7ae7730fd   Patrick McHardy   [NETFILTER]: nf_c...
553
  #define NF_CT_EVICTION_RANGE	8
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
554
555
  /* There's a small race here where we may free a just-assured
     connection.  Too bad: we're in trouble anyway. */
400dad39d   Alexey Dobriyan   netfilter: netns ...
556
  static noinline int early_drop(struct net *net, unsigned int hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
557
  {
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
558
  	/* Use oldest entry, which is roughly LRU */
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
559
  	struct nf_conntrack_tuple_hash *h;
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
560
  	struct nf_conn *ct = NULL, *tmp;
ea781f197   Eric Dumazet   netfilter: nf_con...
561
  	struct hlist_nulls_node *n;
7ae7730fd   Patrick McHardy   [NETFILTER]: nf_c...
562
  	unsigned int i, cnt = 0;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
563
  	int dropped = 0;
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
564
  	rcu_read_lock();
d696c7bda   Patrick McHardy   netfilter: nf_con...
565
  	for (i = 0; i < net->ct.htable_size; i++) {
ea781f197   Eric Dumazet   netfilter: nf_con...
566
567
  		hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],
  					 hnnode) {
7ae7730fd   Patrick McHardy   [NETFILTER]: nf_c...
568
569
570
571
572
  			tmp = nf_ct_tuplehash_to_ctrack(h);
  			if (!test_bit(IPS_ASSURED_BIT, &tmp->status))
  				ct = tmp;
  			cnt++;
  		}
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
573

5ae27aa2b   Changli Gao   netfilter: nf_con...
574
575
576
577
578
579
580
581
582
  		if (ct != NULL) {
  			if (likely(!nf_ct_is_dying(ct) &&
  				   atomic_inc_not_zero(&ct->ct_general.use)))
  				break;
  			else
  				ct = NULL;
  		}
  
  		if (cnt >= NF_CT_EVICTION_RANGE)
7ae7730fd   Patrick McHardy   [NETFILTER]: nf_c...
583
  			break;
5ae27aa2b   Changli Gao   netfilter: nf_con...
584

d696c7bda   Patrick McHardy   netfilter: nf_con...
585
  		hash = (hash + 1) % net->ct.htable_size;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
586
  	}
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
587
  	rcu_read_unlock();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
588
589
590
591
592
593
594
  
  	if (!ct)
  		return dropped;
  
  	if (del_timer(&ct->timeout)) {
  		death_by_timeout((unsigned long)ct);
  		dropped = 1;
0d55af879   Alexey Dobriyan   netfilter: netns ...
595
  		NF_CT_STAT_INC_ATOMIC(net, early_drop);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
596
597
598
599
  	}
  	nf_ct_put(ct);
  	return dropped;
  }
f682cefa5   Changli Gao   netfilter: fix th...
600
601
602
603
604
605
606
607
608
609
610
611
612
613
  void init_nf_conntrack_hash_rnd(void)
  {
  	unsigned int rand;
  
  	/*
  	 * Why not initialize nf_conntrack_rnd in a "init()" function ?
  	 * Because there isn't enough entropy when system initializing,
  	 * and we initialize it as late as possible.
  	 */
  	do {
  		get_random_bytes(&rand, sizeof(rand));
  	} while (!rand);
  	cmpxchg(&nf_conntrack_hash_rnd, 0, rand);
  }
99f07e91b   Changli Gao   netfilter: save t...
614
615
616
617
618
  static struct nf_conn *
  __nf_conntrack_alloc(struct net *net, u16 zone,
  		     const struct nf_conntrack_tuple *orig,
  		     const struct nf_conntrack_tuple *repl,
  		     gfp_t gfp, u32 hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
619
  {
cd7fcbf1c   Julia Lawall   netfilter 07/09: ...
620
  	struct nf_conn *ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
621

b23909695   Changli Gao   netfilter: nf_con...
622
  	if (unlikely(!nf_conntrack_hash_rnd)) {
f682cefa5   Changli Gao   netfilter: fix th...
623
  		init_nf_conntrack_hash_rnd();
99f07e91b   Changli Gao   netfilter: save t...
624
625
  		/* recompute the hash as nf_conntrack_hash_rnd is initialized */
  		hash = hash_conntrack_raw(orig, zone);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
626
  	}
5251e2d21   Pablo Neira Ayuso   [NETFILTER]: conn...
627
  	/* We don't want any race condition at early drop stage */
49ac8713b   Alexey Dobriyan   netfilter: netns ...
628
  	atomic_inc(&net->ct.count);
5251e2d21   Pablo Neira Ayuso   [NETFILTER]: conn...
629

76eb94604   Patrick McHardy   [NETFILTER]: nf_c...
630
  	if (nf_conntrack_max &&
49ac8713b   Alexey Dobriyan   netfilter: netns ...
631
  	    unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
99f07e91b   Changli Gao   netfilter: save t...
632
  		if (!early_drop(net, hash_bucket(hash, net))) {
49ac8713b   Alexey Dobriyan   netfilter: netns ...
633
  			atomic_dec(&net->ct.count);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
634
635
636
637
638
639
640
641
  			if (net_ratelimit())
  				printk(KERN_WARNING
  				       "nf_conntrack: table full, dropping"
  				       " packet.
  ");
  			return ERR_PTR(-ENOMEM);
  		}
  	}
941297f44   Eric Dumazet   netfilter: nf_con...
642
643
644
645
  	/*
  	 * Do not use kmem_cache_zalloc(), as this cache uses
  	 * SLAB_DESTROY_BY_RCU.
  	 */
5b3501faa   Eric Dumazet   netfilter: nf_con...
646
  	ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
647
  	if (ct == NULL) {
49ac8713b   Alexey Dobriyan   netfilter: netns ...
648
  		atomic_dec(&net->ct.count);
dacd2a1a5   Yasuyuki Kozakai   [NETFILTER]: nf_c...
649
  		return ERR_PTR(-ENOMEM);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
650
  	}
941297f44   Eric Dumazet   netfilter: nf_con...
651
652
653
654
655
  	/*
  	 * Let ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.next
  	 * and ct->tuplehash[IP_CT_DIR_REPLY].hnnode.next unchanged.
  	 */
  	memset(&ct->tuplehash[IP_CT_DIR_MAX], 0,
e5fc9e7a6   Changli Gao   netfilter: nf_con...
656
657
  	       offsetof(struct nf_conn, proto) -
  	       offsetof(struct nf_conn, tuplehash[IP_CT_DIR_MAX]));
440f0d588   Patrick McHardy   netfilter: nf_con...
658
  	spin_lock_init(&ct->lock);
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
659
  	ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
941297f44   Eric Dumazet   netfilter: nf_con...
660
  	ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
661
  	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
99f07e91b   Changli Gao   netfilter: save t...
662
663
  	/* save hash for reusing when confirming */
  	*(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
664
  	/* Don't set timer yet: wait for confirmation */
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
665
  	setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
c2d9ba9bc   Eric Dumazet   net: CONFIG_NET_N...
666
  	write_pnet(&ct->ct_net, net);
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
667
668
669
670
671
672
673
674
675
676
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  	if (zone) {
  		struct nf_conntrack_zone *nf_ct_zone;
  
  		nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);
  		if (!nf_ct_zone)
  			goto out_free;
  		nf_ct_zone->id = zone;
  	}
  #endif
941297f44   Eric Dumazet   netfilter: nf_con...
677
678
679
680
681
  	/*
  	 * changes to lookup keys must be done before setting refcnt to 1
  	 */
  	smp_wmb();
  	atomic_set(&ct->ct_general.use, 1);
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
682
  	return ct;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
683
684
685
686
687
688
  
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  out_free:
  	kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
  	return ERR_PTR(-ENOMEM);
  #endif
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
689
  }
99f07e91b   Changli Gao   netfilter: save t...
690
691
692
693
694
695
696
697
  
  struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,
  				   const struct nf_conntrack_tuple *orig,
  				   const struct nf_conntrack_tuple *repl,
  				   gfp_t gfp)
  {
  	return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
698
  EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
699

c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
700
  void nf_conntrack_free(struct nf_conn *ct)
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
701
  {
1d45209d8   Eric Dumazet   netfilter: nf_con...
702
  	struct net *net = nf_ct_net(ct);
ceeff7541   Patrick McHardy   netfilter: nf_con...
703
  	nf_ct_ext_destroy(ct);
1d45209d8   Eric Dumazet   netfilter: nf_con...
704
  	atomic_dec(&net->ct.count);
ea781f197   Eric Dumazet   netfilter: nf_con...
705
  	nf_ct_ext_free(ct);
5b3501faa   Eric Dumazet   netfilter: nf_con...
706
  	kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
707
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
708
  EXPORT_SYMBOL_GPL(nf_conntrack_free);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
709
710
711
712
  
  /* Allocate a new conntrack: we return -ENOMEM if classification
     failed due to stress.  Otherwise it really is unclassifiable. */
  static struct nf_conntrack_tuple_hash *
b2a15a604   Patrick McHardy   netfilter: nf_con...
713
  init_conntrack(struct net *net, struct nf_conn *tmpl,
5a1fb391d   Alexey Dobriyan   netfilter: netns ...
714
  	       const struct nf_conntrack_tuple *tuple,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
715
  	       struct nf_conntrack_l3proto *l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
716
  	       struct nf_conntrack_l4proto *l4proto,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
717
  	       struct sk_buff *skb,
99f07e91b   Changli Gao   netfilter: save t...
718
  	       unsigned int dataoff, u32 hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
719
  {
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
720
  	struct nf_conn *ct;
3c158f7f5   Patrick McHarrdy   [NETFILTER]: nf_c...
721
  	struct nf_conn_help *help;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
722
  	struct nf_conntrack_tuple repl_tuple;
b2a15a604   Patrick McHardy   netfilter: nf_con...
723
  	struct nf_conntrack_ecache *ecache;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
724
  	struct nf_conntrack_expect *exp;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
725
  	u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
726

605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
727
  	if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
728
729
  		pr_debug("Can't invert tuple.
  ");
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
730
731
  		return NULL;
  	}
99f07e91b   Changli Gao   netfilter: save t...
732
733
  	ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
  				  hash);
0a9ee8134   Joe Perches   netfilter: Remove...
734
  	if (IS_ERR(ct))
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
735
  		return (struct nf_conntrack_tuple_hash *)ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
736

c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
737
738
  	if (!l4proto->new(ct, skb, dataoff)) {
  		nf_conntrack_free(ct);
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
739
740
  		pr_debug("init conntrack: can't track with proto module
  ");
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
741
742
  		return NULL;
  	}
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
743
  	nf_ct_acct_ext_add(ct, GFP_ATOMIC);
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
744
  	nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
b2a15a604   Patrick McHardy   netfilter: nf_con...
745
746
747
748
749
  
  	ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
  	nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
  				 ecache ? ecache->expmask : 0,
  			     GFP_ATOMIC);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
750

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
751
  	spin_lock_bh(&nf_conntrack_lock);
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
752
  	exp = nf_ct_find_expectation(net, zone, tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
753
  	if (exp) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
754
755
  		pr_debug("conntrack: expectation arrives ct=%p exp=%p
  ",
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
756
  			 ct, exp);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
757
  		/* Welcome, Mr. Bond.  We've been expecting you... */
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
758
759
  		__set_bit(IPS_EXPECTED_BIT, &ct->status);
  		ct->master = exp->master;
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
760
  		if (exp->helper) {
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
761
  			help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
762
  			if (help)
cf778b00e   Eric Dumazet   net: reintroduce ...
763
  				rcu_assign_pointer(help->helper, exp->helper);
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
764
  		}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
765
  #ifdef CONFIG_NF_CONNTRACK_MARK
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
766
  		ct->mark = exp->master->mark;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
767
  #endif
7c9728c39   James Morris   [SECMARK]: Add se...
768
  #ifdef CONFIG_NF_CONNTRACK_SECMARK
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
769
  		ct->secmark = exp->master->secmark;
7c9728c39   James Morris   [SECMARK]: Add se...
770
  #endif
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
771
  		nf_conntrack_get(&ct->master->ct_general);
0d55af879   Alexey Dobriyan   netfilter: netns ...
772
  		NF_CT_STAT_INC(net, expect_new);
22e7410b7   Yasuyuki Kozakai   [NETFILTER]: nf_c...
773
  	} else {
b2a15a604   Patrick McHardy   netfilter: nf_con...
774
  		__nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
0d55af879   Alexey Dobriyan   netfilter: netns ...
775
  		NF_CT_STAT_INC(net, new);
22e7410b7   Yasuyuki Kozakai   [NETFILTER]: nf_c...
776
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
777
778
  
  	/* Overload tuple linked list to put us in unconfirmed list. */
ea781f197   Eric Dumazet   netfilter: nf_con...
779
  	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
63c9a2626   Alexey Dobriyan   netfilter: netns ...
780
  		       &net->ct.unconfirmed);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
781

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
782
  	spin_unlock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
783
784
785
  
  	if (exp) {
  		if (exp->expectfn)
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
786
  			exp->expectfn(ct, exp);
6823645d6   Patrick McHardy   [NETFILTER]: nf_c...
787
  		nf_ct_expect_put(exp);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
788
  	}
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
789
  	return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
790
791
792
793
  }
  
  /* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
  static inline struct nf_conn *
b2a15a604   Patrick McHardy   netfilter: nf_con...
794
  resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
a702a65fc   Alexey Dobriyan   netfilter: netns ...
795
  		  struct sk_buff *skb,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
796
797
798
799
  		  unsigned int dataoff,
  		  u_int16_t l3num,
  		  u_int8_t protonum,
  		  struct nf_conntrack_l3proto *l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
800
  		  struct nf_conntrack_l4proto *l4proto,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
801
802
803
804
805
806
  		  int *set_reply,
  		  enum ip_conntrack_info *ctinfo)
  {
  	struct nf_conntrack_tuple tuple;
  	struct nf_conntrack_tuple_hash *h;
  	struct nf_conn *ct;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
807
  	u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
99f07e91b   Changli Gao   netfilter: save t...
808
  	u32 hash;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
809

bbe735e42   Arnaldo Carvalho de Melo   [SK_BUFF]: Introd...
810
  	if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
811
  			     dataoff, l3num, protonum, &tuple, l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
812
  			     l4proto)) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
813
814
  		pr_debug("resolve_normal_ct: Can't get tuple
  ");
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
815
816
817
818
  		return NULL;
  	}
  
  	/* look for tuple match */
99f07e91b   Changli Gao   netfilter: save t...
819
820
  	hash = hash_conntrack_raw(&tuple, zone);
  	h = __nf_conntrack_find_get(net, zone, &tuple, hash);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
821
  	if (!h) {
b2a15a604   Patrick McHardy   netfilter: nf_con...
822
  		h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
99f07e91b   Changli Gao   netfilter: save t...
823
  				   skb, dataoff, hash);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
824
825
826
827
828
829
830
831
832
  		if (!h)
  			return NULL;
  		if (IS_ERR(h))
  			return (void *)h;
  	}
  	ct = nf_ct_tuplehash_to_ctrack(h);
  
  	/* It exists; we have (non-exclusive) reference. */
  	if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
fb0488337   Eric Dumazet   netfilter: add mo...
833
  		*ctinfo = IP_CT_ESTABLISHED_REPLY;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
834
835
836
837
838
  		/* Please set reply bit if this packet OK */
  		*set_reply = 1;
  	} else {
  		/* Once we've had two way comms, always ESTABLISHED. */
  		if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
839
840
  			pr_debug("nf_conntrack_in: normal packet for %p
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
841
842
  			*ctinfo = IP_CT_ESTABLISHED;
  		} else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
843
844
845
  			pr_debug("nf_conntrack_in: related packet for %p
  ",
  				 ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
846
847
  			*ctinfo = IP_CT_RELATED;
  		} else {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
848
849
  			pr_debug("nf_conntrack_in: new packet for %p
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
850
851
852
853
854
855
856
857
858
859
  			*ctinfo = IP_CT_NEW;
  		}
  		*set_reply = 0;
  	}
  	skb->nfct = &ct->ct_general;
  	skb->nfctinfo = *ctinfo;
  	return ct;
  }
  
  unsigned int
a702a65fc   Alexey Dobriyan   netfilter: netns ...
860
861
  nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
  		struct sk_buff *skb)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
862
  {
b2a15a604   Patrick McHardy   netfilter: nf_con...
863
  	struct nf_conn *ct, *tmpl = NULL;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
864
865
  	enum ip_conntrack_info ctinfo;
  	struct nf_conntrack_l3proto *l3proto;
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
866
  	struct nf_conntrack_l4proto *l4proto;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
867
868
869
870
  	unsigned int dataoff;
  	u_int8_t protonum;
  	int set_reply = 0;
  	int ret;
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
871
  	if (skb->nfct) {
b2a15a604   Patrick McHardy   netfilter: nf_con...
872
873
874
875
876
877
878
  		/* Previously seen (loopback or untracked)?  Ignore. */
  		tmpl = (struct nf_conn *)skb->nfct;
  		if (!nf_ct_is_template(tmpl)) {
  			NF_CT_STAT_INC_ATOMIC(net, ignore);
  			return NF_ACCEPT;
  		}
  		skb->nfct = NULL;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
879
  	}
923f4902f   Patrick McHardy   [NETFILTER]: nf_c...
880
  	/* rcu_read_lock()ed by nf_hook_slow */
76108cea0   Jan Engelhardt   netfilter: Use un...
881
  	l3proto = __nf_ct_l3proto_find(pf);
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
882
  	ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
ffc306904   Yasuyuki Kozakai   [NETFILTER]: nf_c...
883
884
  				   &dataoff, &protonum);
  	if (ret <= 0) {
25985edce   Lucas De Marchi   Fix common misspe...
885
886
  		pr_debug("not prepared to track yet or error occurred
  ");
0d55af879   Alexey Dobriyan   netfilter: netns ...
887
888
  		NF_CT_STAT_INC_ATOMIC(net, error);
  		NF_CT_STAT_INC_ATOMIC(net, invalid);
b2a15a604   Patrick McHardy   netfilter: nf_con...
889
890
  		ret = -ret;
  		goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
891
  	}
76108cea0   Jan Engelhardt   netfilter: Use un...
892
  	l4proto = __nf_ct_l4proto_find(pf, protonum);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
893
894
895
896
  
  	/* It may be an special packet, error, unclean...
  	 * inverse of the return code tells to the netfilter
  	 * core what to do with the packet. */
74c51a149   Alexey Dobriyan   netfilter: netns ...
897
  	if (l4proto->error != NULL) {
8fea97ec1   Patrick McHardy   netfilter: nf_con...
898
899
  		ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
  				     pf, hooknum);
74c51a149   Alexey Dobriyan   netfilter: netns ...
900
  		if (ret <= 0) {
0d55af879   Alexey Dobriyan   netfilter: netns ...
901
902
  			NF_CT_STAT_INC_ATOMIC(net, error);
  			NF_CT_STAT_INC_ATOMIC(net, invalid);
b2a15a604   Patrick McHardy   netfilter: nf_con...
903
904
  			ret = -ret;
  			goto out;
74c51a149   Alexey Dobriyan   netfilter: netns ...
905
  		}
88ed01d17   Pablo Neira Ayuso   netfilter: nf_con...
906
907
908
  		/* ICMP[v6] protocol trackers may assign one conntrack. */
  		if (skb->nfct)
  			goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
909
  	}
b2a15a604   Patrick McHardy   netfilter: nf_con...
910
  	ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
a702a65fc   Alexey Dobriyan   netfilter: netns ...
911
  			       l3proto, l4proto, &set_reply, &ctinfo);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
912
913
  	if (!ct) {
  		/* Not valid part of a connection */
0d55af879   Alexey Dobriyan   netfilter: netns ...
914
  		NF_CT_STAT_INC_ATOMIC(net, invalid);
b2a15a604   Patrick McHardy   netfilter: nf_con...
915
916
  		ret = NF_ACCEPT;
  		goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
917
918
919
920
  	}
  
  	if (IS_ERR(ct)) {
  		/* Too stressed to deal. */
0d55af879   Alexey Dobriyan   netfilter: netns ...
921
  		NF_CT_STAT_INC_ATOMIC(net, drop);
b2a15a604   Patrick McHardy   netfilter: nf_con...
922
923
  		ret = NF_DROP;
  		goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
924
  	}
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
925
  	NF_CT_ASSERT(skb->nfct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
926

3db05fea5   Herbert Xu   [NETFILTER]: Repl...
927
  	ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum);
ec8d54096   Christoph Paasch   netfilter: conntr...
928
  	if (ret <= 0) {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
929
930
  		/* Invalid: inverse of the return code tells
  		 * the netfilter core what to do */
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
931
932
  		pr_debug("nf_conntrack_in: Can't track with proto module
  ");
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
933
934
  		nf_conntrack_put(skb->nfct);
  		skb->nfct = NULL;
0d55af879   Alexey Dobriyan   netfilter: netns ...
935
  		NF_CT_STAT_INC_ATOMIC(net, invalid);
7d1e04598   Pablo Neira Ayuso   netfilter: nf_con...
936
937
  		if (ret == -NF_DROP)
  			NF_CT_STAT_INC_ATOMIC(net, drop);
b2a15a604   Patrick McHardy   netfilter: nf_con...
938
939
  		ret = -ret;
  		goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
940
941
942
  	}
  
  	if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
858b31330   Patrick McHardy   netfilter: nf_con...
943
  		nf_conntrack_event_cache(IPCT_REPLY, ct);
b2a15a604   Patrick McHardy   netfilter: nf_con...
944
  out:
c31742864   Pablo Neira Ayuso   netfilter: nf_con...
945
946
947
948
949
950
951
952
953
  	if (tmpl) {
  		/* Special case: we have to repeat this hook, assign the
  		 * template again to this packet. We assume that this packet
  		 * has no conntrack assigned. This is used by nf_ct_tcp. */
  		if (ret == NF_REPEAT)
  			skb->nfct = (struct nf_conntrack *)tmpl;
  		else
  			nf_ct_put(tmpl);
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
954
955
956
  
  	return ret;
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
957
  EXPORT_SYMBOL_GPL(nf_conntrack_in);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
958

5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
959
960
  bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
  			  const struct nf_conntrack_tuple *orig)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
961
  {
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
962
  	bool ret;
923f4902f   Patrick McHardy   [NETFILTER]: nf_c...
963
964
965
966
967
968
969
970
  
  	rcu_read_lock();
  	ret = nf_ct_invert_tuple(inverse, orig,
  				 __nf_ct_l3proto_find(orig->src.l3num),
  				 __nf_ct_l4proto_find(orig->src.l3num,
  						      orig->dst.protonum));
  	rcu_read_unlock();
  	return ret;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
971
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
972
  EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
973

5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
974
975
976
977
978
979
  /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
     implicitly racy: see __nf_conntrack_confirm */
  void nf_conntrack_alter_reply(struct nf_conn *ct,
  			      const struct nf_conntrack_tuple *newreply)
  {
  	struct nf_conn_help *help = nfct_help(ct);
5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
980
981
  	/* Should be unconfirmed, so not in hash table yet */
  	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
982
  	pr_debug("Altering reply tuple of %p to ", ct);
3c9fba656   Jan Engelhardt   [NETFILTER]: nf_c...
983
  	nf_ct_dump_tuple(newreply);
5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
984
985
  
  	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
ef1a5a50b   Patrick McHardy   [NETFILTER]: nf_c...
986
  	if (ct->master || (help && !hlist_empty(&help->expectations)))
c52fbb410   Patrick McHardy   [NETFILTER]: nf_c...
987
  		return;
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
988

c52fbb410   Patrick McHardy   [NETFILTER]: nf_c...
989
  	rcu_read_lock();
b2a15a604   Patrick McHardy   netfilter: nf_con...
990
  	__nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
c52fbb410   Patrick McHardy   [NETFILTER]: nf_c...
991
  	rcu_read_unlock();
5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
992
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
993
  EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
994

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
995
996
997
998
999
1000
1001
  /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
  void __nf_ct_refresh_acct(struct nf_conn *ct,
  			  enum ip_conntrack_info ctinfo,
  			  const struct sk_buff *skb,
  			  unsigned long extra_jiffies,
  			  int do_acct)
  {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1002
1003
  	NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
  	NF_CT_ASSERT(skb);
997ae831a   Eric Leblond   [NETFILTER]: conn...
1004
  	/* Only update if this is not a fixed timeout */
47d950454   Patrick McHardy   [NETFILTER]: nf_c...
1005
1006
  	if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
  		goto acct;
997ae831a   Eric Leblond   [NETFILTER]: conn...
1007

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1008
1009
1010
  	/* If not in hash table, timer will not be active yet */
  	if (!nf_ct_is_confirmed(ct)) {
  		ct->timeout.expires = extra_jiffies;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1011
  	} else {
be00c8e48   Martin Josefsson   [NETFILTER]: nf_c...
1012
1013
1014
1015
1016
  		unsigned long newtime = jiffies + extra_jiffies;
  
  		/* Only update the timeout if the new timeout is at least
  		   HZ jiffies from the old timeout. Need del_timer for race
  		   avoidance (may already be dying). */
65cb9fda3   Patrick McHardy   netfilter: nf_con...
1017
1018
  		if (newtime - ct->timeout.expires >= HZ)
  			mod_timer_pending(&ct->timeout, newtime);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1019
  	}
47d950454   Patrick McHardy   [NETFILTER]: nf_c...
1020
  acct:
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1021
  	if (do_acct) {
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1022
  		struct nf_conn_counter *acct;
3ffd5eeb1   Martin Josefsson   [NETFILTER]: nf_c...
1023

584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1024
1025
  		acct = nf_conn_acct_find(ct);
  		if (acct) {
b3e0bfa71   Eric Dumazet   netfilter: nf_con...
1026
1027
  			atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
  			atomic64_add(skb->len, &acct[CTINFO2DIR(ctinfo)].bytes);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1028
  		}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1029
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1030
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
1031
  EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1032

4c8894980   David S. Miller   netfilter: Let nf...
1033
1034
1035
1036
  bool __nf_ct_kill_acct(struct nf_conn *ct,
  		       enum ip_conntrack_info ctinfo,
  		       const struct sk_buff *skb,
  		       int do_acct)
51091764f   Patrick McHardy   netfilter: nf_con...
1037
  {
718d4ad98   Fabian Hugelshofer   netfilter: nf_con...
1038
  	if (do_acct) {
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1039
  		struct nf_conn_counter *acct;
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1040
1041
  		acct = nf_conn_acct_find(ct);
  		if (acct) {
b3e0bfa71   Eric Dumazet   netfilter: nf_con...
1042
1043
1044
  			atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
  			atomic64_add(skb->len - skb_network_offset(skb),
  				     &acct[CTINFO2DIR(ctinfo)].bytes);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1045
  		}
718d4ad98   Fabian Hugelshofer   netfilter: nf_con...
1046
  	}
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1047

4c8894980   David S. Miller   netfilter: Let nf...
1048
  	if (del_timer(&ct->timeout)) {
51091764f   Patrick McHardy   netfilter: nf_con...
1049
  		ct->timeout.function((unsigned long)ct);
4c8894980   David S. Miller   netfilter: Let nf...
1050
1051
1052
  		return true;
  	}
  	return false;
51091764f   Patrick McHardy   netfilter: nf_con...
1053
  }
718d4ad98   Fabian Hugelshofer   netfilter: nf_con...
1054
  EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);
51091764f   Patrick McHardy   netfilter: nf_con...
1055

5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1056
1057
1058
1059
1060
1061
1062
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {
  	.len	= sizeof(struct nf_conntrack_zone),
  	.align	= __alignof__(struct nf_conntrack_zone),
  	.id	= NF_CT_EXT_ZONE,
  };
  #endif
c0cd11566   Igor Maravić   net:netfilter: us...
1063
  #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1064
1065
1066
  
  #include <linux/netfilter/nfnetlink.h>
  #include <linux/netfilter/nfnetlink_conntrack.h>
57b47a53e   Ingo Molnar   [NET]: sem2mutex ...
1067
  #include <linux/mutex.h>
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1068
1069
1070
  /* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
   * in ip_conntrack_core, since we don't want the protocols to autoload
   * or depend on ctnetlink */
fdf708322   Patrick McHardy   [NETFILTER]: nfne...
1071
  int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1072
1073
  			       const struct nf_conntrack_tuple *tuple)
  {
77236b6e3   Patrick McHardy   [NETFILTER]: ctne...
1074
1075
  	NLA_PUT_BE16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port);
  	NLA_PUT_BE16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1076
  	return 0;
df6fb868d   Patrick McHardy   [NETFILTER]: nfne...
1077
  nla_put_failure:
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1078
1079
  	return -1;
  }
fdf708322   Patrick McHardy   [NETFILTER]: nfne...
1080
  EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1081

f73e924cd   Patrick McHardy   [NETFILTER]: ctne...
1082
1083
1084
  const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
  	[CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
  	[CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1085
  };
f73e924cd   Patrick McHardy   [NETFILTER]: ctne...
1086
  EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1087

fdf708322   Patrick McHardy   [NETFILTER]: nfne...
1088
  int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1089
1090
  			       struct nf_conntrack_tuple *t)
  {
df6fb868d   Patrick McHardy   [NETFILTER]: nfne...
1091
  	if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1092
  		return -EINVAL;
77236b6e3   Patrick McHardy   [NETFILTER]: ctne...
1093
1094
  	t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
  	t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1095
1096
1097
  
  	return 0;
  }
fdf708322   Patrick McHardy   [NETFILTER]: nfne...
1098
  EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
5c0de29d0   Holger Eitzenberger   netfilter: nf_con...
1099
1100
1101
1102
1103
1104
  
  int nf_ct_port_nlattr_tuple_size(void)
  {
  	return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
  }
  EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1105
  #endif
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1106
  /* Used by ipt_REJECT and ip6t_REJECT. */
b334aadc3   Patrick McHardy   [NETFILTER]: nf_c...
1107
  static void nf_conntrack_attach(struct sk_buff *nskb, struct sk_buff *skb)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1108
1109
1110
1111
1112
1113
1114
  {
  	struct nf_conn *ct;
  	enum ip_conntrack_info ctinfo;
  
  	/* This ICMP is in reverse direction to the packet which caused it */
  	ct = nf_ct_get(skb, &ctinfo);
  	if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
fb0488337   Eric Dumazet   netfilter: add mo...
1115
  		ctinfo = IP_CT_RELATED_REPLY;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1116
1117
1118
1119
1120
1121
1122
1123
  	else
  		ctinfo = IP_CT_RELATED;
  
  	/* Attach to new skbuff, and increment count */
  	nskb->nfct = &ct->ct_general;
  	nskb->nfctinfo = ctinfo;
  	nf_conntrack_get(nskb->nfct);
  }
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1124
  /* Bring out ya dead! */
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1125
  static struct nf_conn *
400dad39d   Alexey Dobriyan   netfilter: netns ...
1126
  get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1127
1128
  		void *data, unsigned int *bucket)
  {
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1129
1130
  	struct nf_conntrack_tuple_hash *h;
  	struct nf_conn *ct;
ea781f197   Eric Dumazet   netfilter: nf_con...
1131
  	struct hlist_nulls_node *n;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1132

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1133
  	spin_lock_bh(&nf_conntrack_lock);
d696c7bda   Patrick McHardy   netfilter: nf_con...
1134
  	for (; *bucket < net->ct.htable_size; (*bucket)++) {
ea781f197   Eric Dumazet   netfilter: nf_con...
1135
  		hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1136
1137
1138
1139
  			ct = nf_ct_tuplehash_to_ctrack(h);
  			if (iter(ct, data))
  				goto found;
  		}
601e68e10   YOSHIFUJI Hideaki   [NETFILTER]: Fix ...
1140
  	}
ea781f197   Eric Dumazet   netfilter: nf_con...
1141
  	hlist_nulls_for_each_entry(h, n, &net->ct.unconfirmed, hnnode) {
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1142
1143
  		ct = nf_ct_tuplehash_to_ctrack(h);
  		if (iter(ct, data))
ec68e97de   Patrick McHardy   [NETFILTER]: conn...
1144
  			set_bit(IPS_DYING_BIT, &ct->status);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1145
  	}
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1146
  	spin_unlock_bh(&nf_conntrack_lock);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1147
1148
  	return NULL;
  found:
c073e3fa8   Martin Josefsson   [NETFILTER]: nf_c...
1149
  	atomic_inc(&ct->ct_general.use);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1150
  	spin_unlock_bh(&nf_conntrack_lock);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1151
  	return ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1152
  }
400dad39d   Alexey Dobriyan   netfilter: netns ...
1153
1154
1155
  void nf_ct_iterate_cleanup(struct net *net,
  			   int (*iter)(struct nf_conn *i, void *data),
  			   void *data)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1156
  {
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1157
  	struct nf_conn *ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1158
  	unsigned int bucket = 0;
400dad39d   Alexey Dobriyan   netfilter: netns ...
1159
  	while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1160
1161
1162
1163
1164
1165
1166
1167
  		/* Time to push up daises... */
  		if (del_timer(&ct->timeout))
  			death_by_timeout((unsigned long)ct);
  		/* ... else the timer will get him soon. */
  
  		nf_ct_put(ct);
  	}
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
1168
  EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1169

19abb7b09   Pablo Neira Ayuso   netfilter: ctnetl...
1170
1171
1172
1173
  struct __nf_ct_flush_report {
  	u32 pid;
  	int report;
  };
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1174
  static int kill_report(struct nf_conn *i, void *data)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1175
  {
19abb7b09   Pablo Neira Ayuso   netfilter: ctnetl...
1176
  	struct __nf_ct_flush_report *fr = (struct __nf_ct_flush_report *)data;
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
1177
1178
1179
1180
1181
  	struct nf_conn_tstamp *tstamp;
  
  	tstamp = nf_conn_tstamp_find(i);
  	if (tstamp && tstamp->stop == 0)
  		tstamp->stop = ktime_to_ns(ktime_get_real());
19abb7b09   Pablo Neira Ayuso   netfilter: ctnetl...
1182

dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
1183
1184
1185
1186
1187
1188
1189
  	/* If we fail to deliver the event, death_by_timeout() will retry */
  	if (nf_conntrack_event_report(IPCT_DESTROY, i,
  				      fr->pid, fr->report) < 0)
  		return 1;
  
  	/* Avoid the delivery of the destroy event in death_by_timeout(). */
  	set_bit(IPS_DYING_BIT, &i->status);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1190
1191
  	return 1;
  }
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1192
1193
1194
1195
  static int kill_all(struct nf_conn *i, void *data)
  {
  	return 1;
  }
d862a6622   Patrick McHardy   netfilter: nf_con...
1196
  void nf_ct_free_hashtable(void *hash, unsigned int size)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1197
  {
d862a6622   Patrick McHardy   netfilter: nf_con...
1198
  	if (is_vmalloc_addr(hash))
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1199
1200
  		vfree(hash);
  	else
601e68e10   YOSHIFUJI Hideaki   [NETFILTER]: Fix ...
1201
  		free_pages((unsigned long)hash,
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1202
  			   get_order(sizeof(struct hlist_head) * size));
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1203
  }
ac565e5fc   Patrick McHardy   [NETFILTER]: nf_c...
1204
  EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1205

274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1206
  void nf_conntrack_flush_report(struct net *net, u32 pid, int report)
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1207
  {
19abb7b09   Pablo Neira Ayuso   netfilter: ctnetl...
1208
1209
1210
1211
  	struct __nf_ct_flush_report fr = {
  		.pid 	= pid,
  		.report = report,
  	};
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1212
  	nf_ct_iterate_cleanup(net, kill_report, &fr);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1213
  }
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1214
  EXPORT_SYMBOL_GPL(nf_conntrack_flush_report);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1215

ee254fa44   Alexey Dobriyan   netfilter: nf_con...
1216
  static void nf_ct_release_dying_list(struct net *net)
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
1217
1218
1219
1220
1221
1222
  {
  	struct nf_conntrack_tuple_hash *h;
  	struct nf_conn *ct;
  	struct hlist_nulls_node *n;
  
  	spin_lock_bh(&nf_conntrack_lock);
ee254fa44   Alexey Dobriyan   netfilter: nf_con...
1223
  	hlist_nulls_for_each_entry(h, n, &net->ct.dying, hnnode) {
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
1224
1225
1226
1227
1228
1229
  		ct = nf_ct_tuplehash_to_ctrack(h);
  		/* never fails to remove them, no listeners at this point */
  		nf_ct_kill(ct);
  	}
  	spin_unlock_bh(&nf_conntrack_lock);
  }
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
  static int untrack_refs(void)
  {
  	int cnt = 0, cpu;
  
  	for_each_possible_cpu(cpu) {
  		struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
  
  		cnt += atomic_read(&ct->ct_general.use) - 1;
  	}
  	return cnt;
  }
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1241
  static void nf_conntrack_cleanup_init_net(void)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1242
  {
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1243
  	while (untrack_refs() > 0)
9edd7ca0a   Patrick McHardy   netfilter: nf_con...
1244
  		schedule();
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1245
1246
  	nf_conntrack_helper_fini();
  	nf_conntrack_proto_fini();
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1247
1248
1249
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  	nf_ct_extend_unregister(&nf_ct_zone_extend);
  #endif
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1250
  }
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1251

08f6547d2   Alexey Dobriyan   netfilter: netns ...
1252
1253
  static void nf_conntrack_cleanup_net(struct net *net)
  {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1254
   i_see_dead_people:
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1255
  	nf_ct_iterate_cleanup(net, kill_all, NULL);
ee254fa44   Alexey Dobriyan   netfilter: nf_con...
1256
  	nf_ct_release_dying_list(net);
49ac8713b   Alexey Dobriyan   netfilter: netns ...
1257
  	if (atomic_read(&net->ct.count) != 0) {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1258
1259
1260
  		schedule();
  		goto i_see_dead_people;
  	}
d862a6622   Patrick McHardy   netfilter: nf_con...
1261
  	nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
1262
  	nf_conntrack_ecache_fini(net);
fe8f661f2   Stephen Hemminger   netfilter: nf_con...
1263
  	nf_conntrack_tstamp_fini(net);
d716a4dfb   Alexey Dobriyan   netfilter: netns ...
1264
  	nf_conntrack_acct_fini(net);
9b03f38d0   Alexey Dobriyan   netfilter: netns ...
1265
  	nf_conntrack_expect_fini(net);
5b3501faa   Eric Dumazet   netfilter: nf_con...
1266
1267
  	kmem_cache_destroy(net->ct.nf_conntrack_cachep);
  	kfree(net->ct.slabname);
0d55af879   Alexey Dobriyan   netfilter: netns ...
1268
  	free_percpu(net->ct.stat);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1269
1270
1271
1272
1273
1274
1275
  }
  
  /* Mishearing the voices in his head, our hero wonders how he's
     supposed to kill the mall. */
  void nf_conntrack_cleanup(struct net *net)
  {
  	if (net_eq(net, &init_net))
a9b3cd7f3   Stephen Hemminger   rcu: convert uses...
1276
  		RCU_INIT_POINTER(ip_ct_attach, NULL);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1277
1278
1279
1280
1281
1282
1283
1284
1285
  
  	/* This makes sure all current packets have passed through
  	   netfilter framework.  Roll on, two-stage module
  	   delete... */
  	synchronize_net();
  
  	nf_conntrack_cleanup_net(net);
  
  	if (net_eq(net, &init_net)) {
a9b3cd7f3   Stephen Hemminger   rcu: convert uses...
1286
  		RCU_INIT_POINTER(nf_ct_destroy, NULL);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1287
1288
  		nf_conntrack_cleanup_init_net();
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1289
  }
d862a6622   Patrick McHardy   netfilter: nf_con...
1290
  void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1291
  {
ea781f197   Eric Dumazet   netfilter: nf_con...
1292
1293
1294
  	struct hlist_nulls_head *hash;
  	unsigned int nr_slots, i;
  	size_t sz;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1295

ea781f197   Eric Dumazet   netfilter: nf_con...
1296
1297
1298
1299
1300
  	BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
  	nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
  	sz = nr_slots * sizeof(struct hlist_nulls_head);
  	hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
  					get_order(sz));
601e68e10   YOSHIFUJI Hideaki   [NETFILTER]: Fix ...
1301
  	if (!hash) {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1302
1303
  		printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.
  ");
966567b76   Eric Dumazet   net: two vzalloc(...
1304
  		hash = vzalloc(sz);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1305
  	}
ea781f197   Eric Dumazet   netfilter: nf_con...
1306
1307
1308
  	if (hash && nulls)
  		for (i = 0; i < nr_slots; i++)
  			INIT_HLIST_NULLS_HEAD(&hash[i], i);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1309
1310
1311
  
  	return hash;
  }
ac565e5fc   Patrick McHardy   [NETFILTER]: nf_c...
1312
  EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1313

fae718dda   Patrick McHardy   [NETFILTER]: nf_c...
1314
  int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1315
  {
d862a6622   Patrick McHardy   netfilter: nf_con...
1316
  	int i, bucket;
96eb24d77   Stephen Hemminger   [NETFILTER]: nf_c...
1317
  	unsigned int hashsize, old_size;
ea781f197   Eric Dumazet   netfilter: nf_con...
1318
  	struct hlist_nulls_head *hash, *old_hash;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1319
  	struct nf_conntrack_tuple_hash *h;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1320
  	struct nf_conn *ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1321

d696c7bda   Patrick McHardy   netfilter: nf_con...
1322
1323
  	if (current->nsproxy->net_ns != &init_net)
  		return -EOPNOTSUPP;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1324
1325
1326
  	/* On boot, we can set this without any fancy locking. */
  	if (!nf_conntrack_htable_size)
  		return param_set_uint(val, kp);
96eb24d77   Stephen Hemminger   [NETFILTER]: nf_c...
1327
  	hashsize = simple_strtoul(val, NULL, 0);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1328
1329
  	if (!hashsize)
  		return -EINVAL;
d862a6622   Patrick McHardy   netfilter: nf_con...
1330
  	hash = nf_ct_alloc_hashtable(&hashsize, 1);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1331
1332
  	if (!hash)
  		return -ENOMEM;
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
1333
1334
1335
1336
1337
  	/* Lookups in the old hash might happen in parallel, which means we
  	 * might get false negatives during connection lookup. New connections
  	 * created because of a false negative won't make it into the hash
  	 * though since that required taking the lock.
  	 */
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1338
  	spin_lock_bh(&nf_conntrack_lock);
d696c7bda   Patrick McHardy   netfilter: nf_con...
1339
  	for (i = 0; i < init_net.ct.htable_size; i++) {
ea781f197   Eric Dumazet   netfilter: nf_con...
1340
1341
1342
  		while (!hlist_nulls_empty(&init_net.ct.hash[i])) {
  			h = hlist_nulls_entry(init_net.ct.hash[i].first,
  					struct nf_conntrack_tuple_hash, hnnode);
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1343
  			ct = nf_ct_tuplehash_to_ctrack(h);
ea781f197   Eric Dumazet   netfilter: nf_con...
1344
  			hlist_nulls_del_rcu(&h->hnnode);
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1345
  			bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),
99f07e91b   Changli Gao   netfilter: save t...
1346
  						  hashsize);
ea781f197   Eric Dumazet   netfilter: nf_con...
1347
  			hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1348
1349
  		}
  	}
d696c7bda   Patrick McHardy   netfilter: nf_con...
1350
  	old_size = init_net.ct.htable_size;
400dad39d   Alexey Dobriyan   netfilter: netns ...
1351
  	old_hash = init_net.ct.hash;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1352

d696c7bda   Patrick McHardy   netfilter: nf_con...
1353
  	init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;
400dad39d   Alexey Dobriyan   netfilter: netns ...
1354
  	init_net.ct.hash = hash;
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1355
  	spin_unlock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1356

d862a6622   Patrick McHardy   netfilter: nf_con...
1357
  	nf_ct_free_hashtable(old_hash, old_size);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1358
1359
  	return 0;
  }
fae718dda   Patrick McHardy   [NETFILTER]: nf_c...
1360
  EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1361

fae718dda   Patrick McHardy   [NETFILTER]: nf_c...
1362
  module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1363
  		  &nf_conntrack_htable_size, 0600);
5bfddbd46   Eric Dumazet   netfilter: nf_con...
1364
1365
  void nf_ct_untracked_status_or(unsigned long bits)
  {
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1366
1367
1368
1369
  	int cpu;
  
  	for_each_possible_cpu(cpu)
  		per_cpu(nf_conntrack_untracked, cpu).status |= bits;
5bfddbd46   Eric Dumazet   netfilter: nf_con...
1370
1371
  }
  EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1372
  static int nf_conntrack_init_init_net(void)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1373
  {
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1374
  	int max_factor = 8;
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1375
  	int ret, cpu;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1376
1377
  
  	/* Idea from tcp.c: use 1/16384 of memory.  On i386: 32MB
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1378
  	 * machine has 512 buckets. >= 1GB machines have 16384 buckets. */
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1379
1380
  	if (!nf_conntrack_htable_size) {
  		nf_conntrack_htable_size
4481374ce   Jan Beulich   mm: replace vario...
1381
  			= (((totalram_pages << PAGE_SHIFT) / 16384)
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1382
  			   / sizeof(struct hlist_head));
4481374ce   Jan Beulich   mm: replace vario...
1383
  		if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1384
1385
1386
1387
1388
1389
1390
1391
1392
  			nf_conntrack_htable_size = 16384;
  		if (nf_conntrack_htable_size < 32)
  			nf_conntrack_htable_size = 32;
  
  		/* Use a max. factor of four by default to get the same max as
  		 * with the old struct list_heads. When a table size is given
  		 * we use the old value of 8 to avoid reducing the max.
  		 * entries. */
  		max_factor = 4;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1393
  	}
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1394
  	nf_conntrack_max = max_factor * nf_conntrack_htable_size;
8e5105a0c   Patrick McHardy   [NETFILTER]: nf_c...
1395

654d0fbdc   Stephen Hemminger   netfilter: cleanu...
1396
1397
  	printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)
  ",
8e5105a0c   Patrick McHardy   [NETFILTER]: nf_c...
1398
1399
  	       NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
  	       nf_conntrack_max);
e9c1b084e   Patrick McHardy   [NETFILTER]: nf_c...
1400
1401
  	ret = nf_conntrack_proto_init();
  	if (ret < 0)
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1402
  		goto err_proto;
933a41e7e   Patrick McHardy   [NETFILTER]: nf_c...
1403

ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
1404
1405
  	ret = nf_conntrack_helper_init();
  	if (ret < 0)
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1406
  		goto err_helper;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1407
1408
1409
1410
1411
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  	ret = nf_ct_extend_register(&nf_ct_zone_extend);
  	if (ret < 0)
  		goto err_extend;
  #endif
9edd7ca0a   Patrick McHardy   netfilter: nf_con...
1412
  	/* Set up fake conntrack: to never be deleted, not in any hashes */
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1413
1414
  	for_each_possible_cpu(cpu) {
  		struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1415
1416
1417
  		write_pnet(&ct->ct_net, &init_net);
  		atomic_set(&ct->ct_general.use, 1);
  	}
9edd7ca0a   Patrick McHardy   netfilter: nf_con...
1418
  	/*  - and look it like as a confirmed connection */
5bfddbd46   Eric Dumazet   netfilter: nf_con...
1419
  	nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1420
  	return 0;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1421
1422
1423
1424
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  err_extend:
  	nf_conntrack_helper_fini();
  #endif
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1425
1426
1427
  err_helper:
  	nf_conntrack_proto_fini();
  err_proto:
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1428
1429
  	return ret;
  }
8cc20198c   Eric Dumazet   netfilter: nf_con...
1430
1431
1432
1433
1434
  /*
   * We need to use special "null" values, not used in hash table
   */
  #define UNCONFIRMED_NULLS_VAL	((1<<30)+0)
  #define DYING_NULLS_VAL		((1<<30)+1)
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1435
1436
1437
  static int nf_conntrack_init_net(struct net *net)
  {
  	int ret;
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
1438

08f6547d2   Alexey Dobriyan   netfilter: netns ...
1439
  	atomic_set(&net->ct.count, 0);
8cc20198c   Eric Dumazet   netfilter: nf_con...
1440
1441
  	INIT_HLIST_NULLS_HEAD(&net->ct.unconfirmed, UNCONFIRMED_NULLS_VAL);
  	INIT_HLIST_NULLS_HEAD(&net->ct.dying, DYING_NULLS_VAL);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1442
1443
1444
1445
1446
  	net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
  	if (!net->ct.stat) {
  		ret = -ENOMEM;
  		goto err_stat;
  	}
5b3501faa   Eric Dumazet   netfilter: nf_con...
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
  
  	net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
  	if (!net->ct.slabname) {
  		ret = -ENOMEM;
  		goto err_slabname;
  	}
  
  	net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,
  							sizeof(struct nf_conn), 0,
  							SLAB_DESTROY_BY_RCU, NULL);
  	if (!net->ct.nf_conntrack_cachep) {
  		printk(KERN_ERR "Unable to create nf_conn slab cache
  ");
  		ret = -ENOMEM;
  		goto err_cache;
  	}
d696c7bda   Patrick McHardy   netfilter: nf_con...
1463
1464
  
  	net->ct.htable_size = nf_conntrack_htable_size;
d862a6622   Patrick McHardy   netfilter: nf_con...
1465
  	net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1466
1467
1468
1469
1470
1471
1472
1473
1474
  	if (!net->ct.hash) {
  		ret = -ENOMEM;
  		printk(KERN_ERR "Unable to create nf_conntrack_hash
  ");
  		goto err_hash;
  	}
  	ret = nf_conntrack_expect_init(net);
  	if (ret < 0)
  		goto err_expect;
d716a4dfb   Alexey Dobriyan   netfilter: netns ...
1475
  	ret = nf_conntrack_acct_init(net);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1476
  	if (ret < 0)
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1477
  		goto err_acct;
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
1478
1479
1480
  	ret = nf_conntrack_tstamp_init(net);
  	if (ret < 0)
  		goto err_tstamp;
a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
1481
1482
1483
  	ret = nf_conntrack_ecache_init(net);
  	if (ret < 0)
  		goto err_ecache;
7d3cdc6b5   Yasuyuki Kozakai   [NETFILTER]: nf_c...
1484

08f6547d2   Alexey Dobriyan   netfilter: netns ...
1485
  	return 0;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1486

a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
1487
  err_ecache:
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
1488
1489
  	nf_conntrack_tstamp_fini(net);
  err_tstamp:
a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
1490
  	nf_conntrack_acct_fini(net);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1491
  err_acct:
9b03f38d0   Alexey Dobriyan   netfilter: netns ...
1492
  	nf_conntrack_expect_fini(net);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1493
  err_expect:
d862a6622   Patrick McHardy   netfilter: nf_con...
1494
  	nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
6058fa6bb   Alexey Dobriyan   netfilter: netns ...
1495
  err_hash:
5b3501faa   Eric Dumazet   netfilter: nf_con...
1496
1497
1498
1499
  	kmem_cache_destroy(net->ct.nf_conntrack_cachep);
  err_cache:
  	kfree(net->ct.slabname);
  err_slabname:
0d55af879   Alexey Dobriyan   netfilter: netns ...
1500
1501
  	free_percpu(net->ct.stat);
  err_stat:
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1502
1503
  	return ret;
  }
f9dd09c7f   Jozsef Kadlecsik   netfilter: nf_nat...
1504
1505
1506
1507
  s16 (*nf_ct_nat_offset)(const struct nf_conn *ct,
  			enum ip_conntrack_dir dir,
  			u32 seq);
  EXPORT_SYMBOL_GPL(nf_ct_nat_offset);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
  int nf_conntrack_init(struct net *net)
  {
  	int ret;
  
  	if (net_eq(net, &init_net)) {
  		ret = nf_conntrack_init_init_net();
  		if (ret < 0)
  			goto out_init_net;
  	}
  	ret = nf_conntrack_init_net(net);
  	if (ret < 0)
  		goto out_net;
  
  	if (net_eq(net, &init_net)) {
  		/* For use by REJECT target */
a9b3cd7f3   Stephen Hemminger   rcu: convert uses...
1523
1524
  		RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
  		RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
f9dd09c7f   Jozsef Kadlecsik   netfilter: nf_nat...
1525
1526
  
  		/* Howto get NAT offsets */
a9b3cd7f3   Stephen Hemminger   rcu: convert uses...
1527
  		RCU_INIT_POINTER(nf_ct_nat_offset, NULL);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1528
1529
1530
1531
1532
1533
1534
1535
  	}
  	return 0;
  
  out_net:
  	if (net_eq(net, &init_net))
  		nf_conntrack_cleanup_init_net();
  out_init_net:
  	return ret;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1536
  }