Blame view

net/netfilter/nf_conntrack_core.c 43 KB
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1
2
3
4
5
  /* Connection state tracking for netfilter.  This is separated from,
     but required by, the NAT layer; it can also be used by an iptables
     extension. */
  
  /* (C) 1999-2001 Paul `Rusty' Russell
dc808fe28   Harald Welte   [NETFILTER] nf_co...
6
   * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
7
8
9
10
11
   * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
   *
   * This program is free software; you can redistribute it and/or modify
   * it under the terms of the GNU General Public License version 2 as
   * published by the Free Software Foundation.
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
12
   */
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
13
14
15
  #include <linux/types.h>
  #include <linux/netfilter.h>
  #include <linux/module.h>
d43c36dc6   Alexey Dobriyan   headers: remove s...
16
  #include <linux/sched.h>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
17
18
19
20
21
22
23
24
25
26
27
28
29
30
  #include <linux/skbuff.h>
  #include <linux/proc_fs.h>
  #include <linux/vmalloc.h>
  #include <linux/stddef.h>
  #include <linux/slab.h>
  #include <linux/random.h>
  #include <linux/jhash.h>
  #include <linux/err.h>
  #include <linux/percpu.h>
  #include <linux/moduleparam.h>
  #include <linux/notifier.h>
  #include <linux/kernel.h>
  #include <linux/netdevice.h>
  #include <linux/socket.h>
d7fe0f241   Al Viro   [PATCH] severing ...
31
  #include <linux/mm.h>
d696c7bda   Patrick McHardy   netfilter: nf_con...
32
  #include <linux/nsproxy.h>
ea781f197   Eric Dumazet   netfilter: nf_con...
33
  #include <linux/rculist_nulls.h>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
34

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
35
36
  #include <net/netfilter/nf_conntrack.h>
  #include <net/netfilter/nf_conntrack_l3proto.h>
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
37
  #include <net/netfilter/nf_conntrack_l4proto.h>
77ab9cff0   Martin Josefsson   [NETFILTER]: nf_c...
38
  #include <net/netfilter/nf_conntrack_expect.h>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
39
40
  #include <net/netfilter/nf_conntrack_helper.h>
  #include <net/netfilter/nf_conntrack_core.h>
ecfab2c9f   Yasuyuki Kozakai   [NETFILTER]: nf_c...
41
  #include <net/netfilter/nf_conntrack_extend.h>
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
42
  #include <net/netfilter/nf_conntrack_acct.h>
a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
43
  #include <net/netfilter/nf_conntrack_ecache.h>
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
44
  #include <net/netfilter/nf_conntrack_zones.h>
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
45
  #include <net/netfilter/nf_conntrack_timestamp.h>
e6a7d3c04   Pablo Neira Ayuso   netfilter: ctnetl...
46
  #include <net/netfilter/nf_nat.h>
e17b666a4   Patrick McHardy   netfilter: nf_con...
47
  #include <net/netfilter/nf_nat_core.h>
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
48

dc808fe28   Harald Welte   [NETFILTER] nf_co...
49
  #define NF_CONNTRACK_VERSION	"0.5.0"
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
50

e17b666a4   Patrick McHardy   netfilter: nf_con...
51
52
  int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
  				      enum nf_nat_manip_type manip,
399383246   Patrick McHardy   netfilter: nfnetl...
53
  				      const struct nlattr *attr) __read_mostly;
e6a7d3c04   Pablo Neira Ayuso   netfilter: ctnetl...
54
  EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
55
  DEFINE_SPINLOCK(nf_conntrack_lock);
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
56
  EXPORT_SYMBOL_GPL(nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
57

e2b7606cd   Martin Josefsson   [NETFILTER]: More...
58
  unsigned int nf_conntrack_htable_size __read_mostly;
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
59
  EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
e478075c6   Hagen Paul Pfeifer   netfilter: nf_con...
60
  unsigned int nf_conntrack_max __read_mostly;
a999e6837   Patrick McHardy   [NETFILTER]: nf_c...
61
  EXPORT_SYMBOL_GPL(nf_conntrack_max);
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
62

b3c5163fe   Eric Dumazet   netfilter: nf_con...
63
64
  DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
  EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
65

f682cefa5   Changli Gao   netfilter: fix th...
66
  unsigned int nf_conntrack_hash_rnd __read_mostly;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
67

99f07e91b   Changli Gao   netfilter: save t...
68
  static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
69
  {
0794935e2   Patrick McHardy   [NETFILTER]: nf_c...
70
  	unsigned int n;
0794935e2   Patrick McHardy   [NETFILTER]: nf_c...
71
72
73
74
75
76
  
  	/* The direction must be ignored, so we hash everything up to the
  	 * destination ports (which is a multiple of 4) and treat the last
  	 * three bytes manually.
  	 */
  	n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
99f07e91b   Changli Gao   netfilter: save t...
77
78
79
80
81
82
83
84
85
86
87
88
89
90
  	return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^
  		      (((__force __u16)tuple->dst.u.all << 16) |
  		      tuple->dst.protonum));
  }
  
  static u32 __hash_bucket(u32 hash, unsigned int size)
  {
  	return ((u64)hash * size) >> 32;
  }
  
  static u32 hash_bucket(u32 hash, const struct net *net)
  {
  	return __hash_bucket(hash, net->ct.htable_size);
  }
0794935e2   Patrick McHardy   [NETFILTER]: nf_c...
91

99f07e91b   Changli Gao   netfilter: save t...
92
93
94
95
  static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,
  				  u16 zone, unsigned int size)
  {
  	return __hash_bucket(hash_conntrack_raw(tuple, zone), size);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
96
  }
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
97
  static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,
d696c7bda   Patrick McHardy   netfilter: nf_con...
98
  				       const struct nf_conntrack_tuple *tuple)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
99
  {
99f07e91b   Changli Gao   netfilter: save t...
100
  	return __hash_conntrack(tuple, zone, net->ct.htable_size);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
101
  }
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
102
  bool
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
103
104
105
106
107
108
109
  nf_ct_get_tuple(const struct sk_buff *skb,
  		unsigned int nhoff,
  		unsigned int dataoff,
  		u_int16_t l3num,
  		u_int8_t protonum,
  		struct nf_conntrack_tuple *tuple,
  		const struct nf_conntrack_l3proto *l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
110
  		const struct nf_conntrack_l4proto *l4proto)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
111
  {
443a70d50   Philip Craig   netfilter: nf_con...
112
  	memset(tuple, 0, sizeof(*tuple));
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
113
114
115
  
  	tuple->src.l3num = l3num;
  	if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
116
  		return false;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
117
118
119
  
  	tuple->dst.protonum = protonum;
  	tuple->dst.dir = IP_CT_DIR_ORIGINAL;
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
120
  	return l4proto->pkt_to_tuple(skb, dataoff, tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
121
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
122
  EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
123

5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
124
125
  bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
  		       u_int16_t l3num, struct nf_conntrack_tuple *tuple)
e2a3123fb   Yasuyuki Kozakai   [NETFILTER]: nf_c...
126
127
128
129
130
131
132
133
134
135
136
137
138
  {
  	struct nf_conntrack_l3proto *l3proto;
  	struct nf_conntrack_l4proto *l4proto;
  	unsigned int protoff;
  	u_int8_t protonum;
  	int ret;
  
  	rcu_read_lock();
  
  	l3proto = __nf_ct_l3proto_find(l3num);
  	ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
  	if (ret != NF_ACCEPT) {
  		rcu_read_unlock();
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
139
  		return false;
e2a3123fb   Yasuyuki Kozakai   [NETFILTER]: nf_c...
140
141
142
143
144
145
146
147
148
149
150
  	}
  
  	l4proto = __nf_ct_l4proto_find(l3num, protonum);
  
  	ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,
  			      l3proto, l4proto);
  
  	rcu_read_unlock();
  	return ret;
  }
  EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
151
  bool
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
152
153
154
  nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
  		   const struct nf_conntrack_tuple *orig,
  		   const struct nf_conntrack_l3proto *l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
155
  		   const struct nf_conntrack_l4proto *l4proto)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
156
  {
443a70d50   Philip Craig   netfilter: nf_con...
157
  	memset(inverse, 0, sizeof(*inverse));
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
158
159
160
  
  	inverse->src.l3num = orig->src.l3num;
  	if (l3proto->invert_tuple(inverse, orig) == 0)
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
161
  		return false;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
162
163
164
165
  
  	inverse->dst.dir = !orig->dst.dir;
  
  	inverse->dst.protonum = orig->dst.protonum;
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
166
  	return l4proto->invert_tuple(inverse, orig);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
167
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
168
  EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
169

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
170
171
172
  static void
  clean_from_lists(struct nf_conn *ct)
  {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
173
174
  	pr_debug("clean_from_lists(%p)
  ", ct);
ea781f197   Eric Dumazet   netfilter: nf_con...
175
176
  	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
  	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
177
178
  
  	/* Destroy all pending expectations */
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
179
  	nf_ct_remove_expectations(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
180
181
182
183
184
185
  }
  
  static void
  destroy_conntrack(struct nf_conntrack *nfct)
  {
  	struct nf_conn *ct = (struct nf_conn *)nfct;
0d55af879   Alexey Dobriyan   netfilter: netns ...
186
  	struct net *net = nf_ct_net(ct);
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
187
  	struct nf_conntrack_l4proto *l4proto;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
188

0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
189
190
  	pr_debug("destroy_conntrack(%p)
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
191
192
  	NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
  	NF_CT_ASSERT(!timer_pending(&ct->timeout));
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
193
194
195
  	/* To make sure we don't get any weird locking issues here:
  	 * destroy_conntrack() MUST NOT be called with a write lock
  	 * to nf_conntrack_lock!!! -HW */
923f4902f   Patrick McHardy   [NETFILTER]: nf_c...
196
  	rcu_read_lock();
5e8fbe2ac   Patrick McHardy   [NETFILTER]: nf_c...
197
  	l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
198
199
  	if (l4proto && l4proto->destroy)
  		l4proto->destroy(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
200

982d9a9ce   Patrick McHardy   [NETFILTER]: nf_c...
201
  	rcu_read_unlock();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
202

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
203
  	spin_lock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
204
205
206
207
  	/* Expectations will have been removed in clean_from_lists,
  	 * except TFTP can create an expectation on the first packet,
  	 * before connection is in the list, so we need to clean here,
  	 * too. */
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
208
  	nf_ct_remove_expectations(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
209
210
211
  
  	/* We overload first tuple to link into unconfirmed list. */
  	if (!nf_ct_is_confirmed(ct)) {
ea781f197   Eric Dumazet   netfilter: nf_con...
212
213
  		BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
  		hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
214
  	}
0d55af879   Alexey Dobriyan   netfilter: netns ...
215
  	NF_CT_STAT_INC(net, delete);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
216
  	spin_unlock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
217
218
219
  
  	if (ct->master)
  		nf_ct_put(ct->master);
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
220
221
  	pr_debug("destroy_conntrack: returning ct=%p to slab
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
222
223
  	nf_conntrack_free(ct);
  }
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
224
  void nf_ct_delete_from_lists(struct nf_conn *ct)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
225
  {
0d55af879   Alexey Dobriyan   netfilter: netns ...
226
  	struct net *net = nf_ct_net(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
227

9858a3ae1   Pablo Neira Ayuso   netfilter: conntr...
228
  	nf_ct_helper_destroy(ct);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
229
  	spin_lock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
230
231
  	/* Inside lock so preempt is disabled on module removal path.
  	 * Otherwise we can get spurious warnings. */
0d55af879   Alexey Dobriyan   netfilter: netns ...
232
  	NF_CT_STAT_INC(net, delete_list);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
233
  	clean_from_lists(ct);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
234
  	spin_unlock_bh(&nf_conntrack_lock);
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
  }
  EXPORT_SYMBOL_GPL(nf_ct_delete_from_lists);
  
  static void death_by_event(unsigned long ul_conntrack)
  {
  	struct nf_conn *ct = (void *)ul_conntrack;
  	struct net *net = nf_ct_net(ct);
  
  	if (nf_conntrack_event(IPCT_DESTROY, ct) < 0) {
  		/* bad luck, let's retry again */
  		ct->timeout.expires = jiffies +
  			(random32() % net->ct.sysctl_events_retry_timeout);
  		add_timer(&ct->timeout);
  		return;
  	}
  	/* we've got the event delivered, now it's dying */
  	set_bit(IPS_DYING_BIT, &ct->status);
  	spin_lock(&nf_conntrack_lock);
  	hlist_nulls_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
  	spin_unlock(&nf_conntrack_lock);
  	nf_ct_put(ct);
  }
  
  void nf_ct_insert_dying_list(struct nf_conn *ct)
  {
  	struct net *net = nf_ct_net(ct);
  
  	/* add this conntrack to the dying list */
  	spin_lock_bh(&nf_conntrack_lock);
  	hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
  			     &net->ct.dying);
  	spin_unlock_bh(&nf_conntrack_lock);
  	/* set a new timer to retry event delivery */
  	setup_timer(&ct->timeout, death_by_event, (unsigned long)ct);
  	ct->timeout.expires = jiffies +
  		(random32() % net->ct.sysctl_events_retry_timeout);
  	add_timer(&ct->timeout);
  }
  EXPORT_SYMBOL_GPL(nf_ct_insert_dying_list);
  
  static void death_by_timeout(unsigned long ul_conntrack)
  {
  	struct nf_conn *ct = (void *)ul_conntrack;
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
278
279
280
281
282
  	struct nf_conn_tstamp *tstamp;
  
  	tstamp = nf_conn_tstamp_find(ct);
  	if (tstamp && tstamp->stop == 0)
  		tstamp->stop = ktime_to_ns(ktime_get_real());
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
283
284
285
286
287
288
289
290
291
292
  
  	if (!test_bit(IPS_DYING_BIT, &ct->status) &&
  	    unlikely(nf_conntrack_event(IPCT_DESTROY, ct) < 0)) {
  		/* destroy event was not delivered */
  		nf_ct_delete_from_lists(ct);
  		nf_ct_insert_dying_list(ct);
  		return;
  	}
  	set_bit(IPS_DYING_BIT, &ct->status);
  	nf_ct_delete_from_lists(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
293
294
  	nf_ct_put(ct);
  }
ea781f197   Eric Dumazet   netfilter: nf_con...
295
296
297
298
299
300
301
  /*
   * Warning :
   * - Caller must take a reference on returned object
   *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
   * OR
   * - Caller must lock nf_conntrack_lock before calling this function
   */
99f07e91b   Changli Gao   netfilter: save t...
302
303
304
  static struct nf_conntrack_tuple_hash *
  ____nf_conntrack_find(struct net *net, u16 zone,
  		      const struct nf_conntrack_tuple *tuple, u32 hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
305
306
  {
  	struct nf_conntrack_tuple_hash *h;
ea781f197   Eric Dumazet   netfilter: nf_con...
307
  	struct hlist_nulls_node *n;
99f07e91b   Changli Gao   netfilter: save t...
308
  	unsigned int bucket = hash_bucket(hash, net);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
309

4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
310
311
312
313
  	/* Disable BHs the entire time since we normally need to disable them
  	 * at least once for the stats anyway.
  	 */
  	local_bh_disable();
ea781f197   Eric Dumazet   netfilter: nf_con...
314
  begin:
99f07e91b   Changli Gao   netfilter: save t...
315
  	hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
316
317
  		if (nf_ct_tuple_equal(tuple, &h->tuple) &&
  		    nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)) == zone) {
0d55af879   Alexey Dobriyan   netfilter: netns ...
318
  			NF_CT_STAT_INC(net, found);
4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
319
  			local_bh_enable();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
320
321
  			return h;
  		}
0d55af879   Alexey Dobriyan   netfilter: netns ...
322
  		NF_CT_STAT_INC(net, searched);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
323
  	}
ea781f197   Eric Dumazet   netfilter: nf_con...
324
325
326
327
328
  	/*
  	 * if the nulls value we got at the end of this lookup is
  	 * not the expected one, we must restart lookup.
  	 * We probably met an item that was moved to another chain.
  	 */
99f07e91b   Changli Gao   netfilter: save t...
329
  	if (get_nulls_value(n) != bucket) {
af740b2c8   Jesper Dangaard Brouer   netfilter: nf_con...
330
  		NF_CT_STAT_INC(net, search_restart);
ea781f197   Eric Dumazet   netfilter: nf_con...
331
  		goto begin;
af740b2c8   Jesper Dangaard Brouer   netfilter: nf_con...
332
  	}
4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
333
  	local_bh_enable();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
334
335
336
  
  	return NULL;
  }
99f07e91b   Changli Gao   netfilter: save t...
337
338
339
340
341
342
343
344
  
  struct nf_conntrack_tuple_hash *
  __nf_conntrack_find(struct net *net, u16 zone,
  		    const struct nf_conntrack_tuple *tuple)
  {
  	return ____nf_conntrack_find(net, zone, tuple,
  				     hash_conntrack_raw(tuple, zone));
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
345
  EXPORT_SYMBOL_GPL(__nf_conntrack_find);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
346
347
  
  /* Find a connection corresponding to a tuple. */
99f07e91b   Changli Gao   netfilter: save t...
348
349
350
  static struct nf_conntrack_tuple_hash *
  __nf_conntrack_find_get(struct net *net, u16 zone,
  			const struct nf_conntrack_tuple *tuple, u32 hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
351
352
  {
  	struct nf_conntrack_tuple_hash *h;
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
353
  	struct nf_conn *ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
354

76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
355
  	rcu_read_lock();
ea781f197   Eric Dumazet   netfilter: nf_con...
356
  begin:
99f07e91b   Changli Gao   netfilter: save t...
357
  	h = ____nf_conntrack_find(net, zone, tuple, hash);
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
358
359
  	if (h) {
  		ct = nf_ct_tuplehash_to_ctrack(h);
8d8890b77   Patrick McHardy   netfilter: nf_con...
360
361
  		if (unlikely(nf_ct_is_dying(ct) ||
  			     !atomic_inc_not_zero(&ct->ct_general.use)))
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
362
  			h = NULL;
ea781f197   Eric Dumazet   netfilter: nf_con...
363
  		else {
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
364
365
  			if (unlikely(!nf_ct_tuple_equal(tuple, &h->tuple) ||
  				     nf_ct_zone(ct) != zone)) {
ea781f197   Eric Dumazet   netfilter: nf_con...
366
367
368
369
  				nf_ct_put(ct);
  				goto begin;
  			}
  		}
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
370
371
  	}
  	rcu_read_unlock();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
372
373
374
  
  	return h;
  }
99f07e91b   Changli Gao   netfilter: save t...
375
376
377
378
379
380
381
382
  
  struct nf_conntrack_tuple_hash *
  nf_conntrack_find_get(struct net *net, u16 zone,
  		      const struct nf_conntrack_tuple *tuple)
  {
  	return __nf_conntrack_find_get(net, zone, tuple,
  				       hash_conntrack_raw(tuple, zone));
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
383
  EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
384

c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
385
386
  static void __nf_conntrack_hash_insert(struct nf_conn *ct,
  				       unsigned int hash,
601e68e10   YOSHIFUJI Hideaki   [NETFILTER]: Fix ...
387
  				       unsigned int repl_hash)
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
388
  {
400dad39d   Alexey Dobriyan   netfilter: netns ...
389
  	struct net *net = nf_ct_net(ct);
ea781f197   Eric Dumazet   netfilter: nf_con...
390
  	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
400dad39d   Alexey Dobriyan   netfilter: netns ...
391
  			   &net->ct.hash[hash]);
ea781f197   Eric Dumazet   netfilter: nf_con...
392
  	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
400dad39d   Alexey Dobriyan   netfilter: netns ...
393
  			   &net->ct.hash[repl_hash]);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
394
395
396
397
  }
  
  void nf_conntrack_hash_insert(struct nf_conn *ct)
  {
d696c7bda   Patrick McHardy   netfilter: nf_con...
398
  	struct net *net = nf_ct_net(ct);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
399
  	unsigned int hash, repl_hash;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
400
  	u16 zone;
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
401

5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
402
403
404
  	zone = nf_ct_zone(ct);
  	hash = hash_conntrack(net, zone, &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
  	repl_hash = hash_conntrack(net, zone, &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
405

c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
406
  	__nf_conntrack_hash_insert(ct, hash, repl_hash);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
407
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
408
  EXPORT_SYMBOL_GPL(nf_conntrack_hash_insert);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
409

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
410
411
  /* Confirm a connection given skb; places it in hash table */
  int
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
412
  __nf_conntrack_confirm(struct sk_buff *skb)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
413
414
  {
  	unsigned int hash, repl_hash;
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
415
  	struct nf_conntrack_tuple_hash *h;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
416
  	struct nf_conn *ct;
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
417
  	struct nf_conn_help *help;
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
418
  	struct nf_conn_tstamp *tstamp;
ea781f197   Eric Dumazet   netfilter: nf_con...
419
  	struct hlist_nulls_node *n;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
420
  	enum ip_conntrack_info ctinfo;
400dad39d   Alexey Dobriyan   netfilter: netns ...
421
  	struct net *net;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
422
  	u16 zone;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
423

3db05fea5   Herbert Xu   [NETFILTER]: Repl...
424
  	ct = nf_ct_get(skb, &ctinfo);
400dad39d   Alexey Dobriyan   netfilter: netns ...
425
  	net = nf_ct_net(ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
426
427
428
429
430
431
432
  
  	/* ipt_REJECT uses nf_conntrack_attach to attach related
  	   ICMP/TCP RST packets in other direction.  Actual packet
  	   which created connection will be IP_CT_NEW or for an
  	   expected connection, IP_CT_RELATED. */
  	if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
  		return NF_ACCEPT;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
433
  	zone = nf_ct_zone(ct);
99f07e91b   Changli Gao   netfilter: save t...
434
435
436
437
438
  	/* reuse the hash saved before */
  	hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
  	hash = hash_bucket(hash, net);
  	repl_hash = hash_conntrack(net, zone,
  				   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
439
440
441
442
443
  
  	/* We're not in hash table, and we refuse to set up related
  	   connections for unconfirmed conns.  But packet copies and
  	   REJECT will give spurious warnings here. */
  	/* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
25985edce   Lucas De Marchi   Fix common misspe...
444
  	/* No external references means no one else could have
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
445
446
  	   confirmed us. */
  	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
447
448
  	pr_debug("Confirming conntrack %p
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
449

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
450
  	spin_lock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
451

fc350777c   Joerg Marx   netfilter: nf_con...
452
453
454
455
456
457
458
459
460
  	/* We have to check the DYING flag inside the lock to prevent
  	   a race against nf_ct_get_next_corpse() possibly called from
  	   user context, else we insert an already 'dead' hash, blocking
  	   further use of that particular connection -JM */
  
  	if (unlikely(nf_ct_is_dying(ct))) {
  		spin_unlock_bh(&nf_conntrack_lock);
  		return NF_ACCEPT;
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
461
462
463
  	/* See if there's one in the list already, including reverse:
  	   NAT could have grabbed it without realizing, since we're
  	   not in the hash.  If there is, we lost race. */
ea781f197   Eric Dumazet   netfilter: nf_con...
464
  	hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
465
  		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
466
467
  				      &h->tuple) &&
  		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
468
  			goto out;
ea781f197   Eric Dumazet   netfilter: nf_con...
469
  	hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
470
  		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
471
472
  				      &h->tuple) &&
  		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
473
  			goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
474

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
475
  	/* Remove from unconfirmed list */
ea781f197   Eric Dumazet   netfilter: nf_con...
476
  	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
477

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
478
479
480
481
482
483
  	/* Timer relative to confirmation time, not original
  	   setting time, otherwise we'd get timer wrap in
  	   weird delay cases. */
  	ct->timeout.expires += jiffies;
  	add_timer(&ct->timeout);
  	atomic_inc(&ct->ct_general.use);
45eec3419   Changli Gao   netfilter: nf_con...
484
  	ct->status |= IPS_CONFIRMED;
5c8ec910e   Patrick McHardy   netfilter: nf_con...
485

a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
486
487
488
489
490
491
492
493
  	/* set conntrack timestamp, if enabled. */
  	tstamp = nf_conn_tstamp_find(ct);
  	if (tstamp) {
  		if (skb->tstamp.tv64 == 0)
  			__net_timestamp((struct sk_buff *)skb);
  
  		tstamp->start = ktime_to_ns(skb->tstamp);
  	}
5c8ec910e   Patrick McHardy   netfilter: nf_con...
494
495
496
497
498
499
  	/* Since the lookup is lockless, hash insertion must be done after
  	 * starting the timer and setting the CONFIRMED bit. The RCU barriers
  	 * guarantee that no other CPU can find the conntrack before the above
  	 * stores are visible.
  	 */
  	__nf_conntrack_hash_insert(ct, hash, repl_hash);
0d55af879   Alexey Dobriyan   netfilter: netns ...
500
  	NF_CT_STAT_INC(net, insert);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
501
  	spin_unlock_bh(&nf_conntrack_lock);
5c8ec910e   Patrick McHardy   netfilter: nf_con...
502

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
503
504
  	help = nfct_help(ct);
  	if (help && help->helper)
a71996fcc   Alexey Dobriyan   netfilter: netns ...
505
  		nf_conntrack_event_cache(IPCT_HELPER, ct);
17e6e4eac   Pablo Neira Ayuso   netfilter: conntr...
506

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
507
  	nf_conntrack_event_cache(master_ct(ct) ?
a71996fcc   Alexey Dobriyan   netfilter: netns ...
508
  				 IPCT_RELATED : IPCT_NEW, ct);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
509
  	return NF_ACCEPT;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
510

df0933dcb   Patrick McHardy   [NETFILTER]: kill...
511
  out:
0d55af879   Alexey Dobriyan   netfilter: netns ...
512
  	NF_CT_STAT_INC(net, insert_failed);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
513
  	spin_unlock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
514
515
  	return NF_DROP;
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
516
  EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
517
518
519
520
521
522
523
  
  /* Returns true if a connection correspondings to the tuple (required
     for NAT). */
  int
  nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
  			 const struct nf_conn *ignored_conntrack)
  {
400dad39d   Alexey Dobriyan   netfilter: netns ...
524
  	struct net *net = nf_ct_net(ignored_conntrack);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
525
  	struct nf_conntrack_tuple_hash *h;
ea781f197   Eric Dumazet   netfilter: nf_con...
526
  	struct hlist_nulls_node *n;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
527
528
529
  	struct nf_conn *ct;
  	u16 zone = nf_ct_zone(ignored_conntrack);
  	unsigned int hash = hash_conntrack(net, zone, tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
530

4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
531
532
533
534
  	/* Disable BHs the entire time since we need to disable them at
  	 * least once for the stats anyway.
  	 */
  	rcu_read_lock_bh();
ea781f197   Eric Dumazet   netfilter: nf_con...
535
  	hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
536
537
538
539
  		ct = nf_ct_tuplehash_to_ctrack(h);
  		if (ct != ignored_conntrack &&
  		    nf_ct_tuple_equal(tuple, &h->tuple) &&
  		    nf_ct_zone(ct) == zone) {
0d55af879   Alexey Dobriyan   netfilter: netns ...
540
  			NF_CT_STAT_INC(net, found);
4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
541
  			rcu_read_unlock_bh();
ba419aff2   Patrick McHardy   [NETFILTER]: nf_c...
542
543
  			return 1;
  		}
0d55af879   Alexey Dobriyan   netfilter: netns ...
544
  		NF_CT_STAT_INC(net, searched);
ba419aff2   Patrick McHardy   [NETFILTER]: nf_c...
545
  	}
4e29e9ec7   Patrick McHardy   [NETFILTER]: nf_c...
546
  	rcu_read_unlock_bh();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
547

ba419aff2   Patrick McHardy   [NETFILTER]: nf_c...
548
  	return 0;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
549
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
550
  EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
551

7ae7730fd   Patrick McHardy   [NETFILTER]: nf_c...
552
  #define NF_CT_EVICTION_RANGE	8
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
553
554
  /* There's a small race here where we may free a just-assured
     connection.  Too bad: we're in trouble anyway. */
400dad39d   Alexey Dobriyan   netfilter: netns ...
555
  static noinline int early_drop(struct net *net, unsigned int hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
556
  {
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
557
  	/* Use oldest entry, which is roughly LRU */
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
558
  	struct nf_conntrack_tuple_hash *h;
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
559
  	struct nf_conn *ct = NULL, *tmp;
ea781f197   Eric Dumazet   netfilter: nf_con...
560
  	struct hlist_nulls_node *n;
7ae7730fd   Patrick McHardy   [NETFILTER]: nf_c...
561
  	unsigned int i, cnt = 0;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
562
  	int dropped = 0;
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
563
  	rcu_read_lock();
d696c7bda   Patrick McHardy   netfilter: nf_con...
564
  	for (i = 0; i < net->ct.htable_size; i++) {
ea781f197   Eric Dumazet   netfilter: nf_con...
565
566
  		hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],
  					 hnnode) {
7ae7730fd   Patrick McHardy   [NETFILTER]: nf_c...
567
568
569
570
571
  			tmp = nf_ct_tuplehash_to_ctrack(h);
  			if (!test_bit(IPS_ASSURED_BIT, &tmp->status))
  				ct = tmp;
  			cnt++;
  		}
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
572

5ae27aa2b   Changli Gao   netfilter: nf_con...
573
574
575
576
577
578
579
580
581
  		if (ct != NULL) {
  			if (likely(!nf_ct_is_dying(ct) &&
  				   atomic_inc_not_zero(&ct->ct_general.use)))
  				break;
  			else
  				ct = NULL;
  		}
  
  		if (cnt >= NF_CT_EVICTION_RANGE)
7ae7730fd   Patrick McHardy   [NETFILTER]: nf_c...
582
  			break;
5ae27aa2b   Changli Gao   netfilter: nf_con...
583

d696c7bda   Patrick McHardy   netfilter: nf_con...
584
  		hash = (hash + 1) % net->ct.htable_size;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
585
  	}
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
586
  	rcu_read_unlock();
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
587
588
589
590
591
592
593
  
  	if (!ct)
  		return dropped;
  
  	if (del_timer(&ct->timeout)) {
  		death_by_timeout((unsigned long)ct);
  		dropped = 1;
0d55af879   Alexey Dobriyan   netfilter: netns ...
594
  		NF_CT_STAT_INC_ATOMIC(net, early_drop);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
595
596
597
598
  	}
  	nf_ct_put(ct);
  	return dropped;
  }
f682cefa5   Changli Gao   netfilter: fix th...
599
600
601
602
603
604
605
606
607
608
609
610
611
612
  void init_nf_conntrack_hash_rnd(void)
  {
  	unsigned int rand;
  
  	/*
  	 * Why not initialize nf_conntrack_rnd in a "init()" function ?
  	 * Because there isn't enough entropy when system initializing,
  	 * and we initialize it as late as possible.
  	 */
  	do {
  		get_random_bytes(&rand, sizeof(rand));
  	} while (!rand);
  	cmpxchg(&nf_conntrack_hash_rnd, 0, rand);
  }
99f07e91b   Changli Gao   netfilter: save t...
613
614
615
616
617
  static struct nf_conn *
  __nf_conntrack_alloc(struct net *net, u16 zone,
  		     const struct nf_conntrack_tuple *orig,
  		     const struct nf_conntrack_tuple *repl,
  		     gfp_t gfp, u32 hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
618
  {
cd7fcbf1c   Julia Lawall   netfilter 07/09: ...
619
  	struct nf_conn *ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
620

b23909695   Changli Gao   netfilter: nf_con...
621
  	if (unlikely(!nf_conntrack_hash_rnd)) {
f682cefa5   Changli Gao   netfilter: fix th...
622
  		init_nf_conntrack_hash_rnd();
99f07e91b   Changli Gao   netfilter: save t...
623
624
  		/* recompute the hash as nf_conntrack_hash_rnd is initialized */
  		hash = hash_conntrack_raw(orig, zone);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
625
  	}
5251e2d21   Pablo Neira Ayuso   [NETFILTER]: conn...
626
  	/* We don't want any race condition at early drop stage */
49ac8713b   Alexey Dobriyan   netfilter: netns ...
627
  	atomic_inc(&net->ct.count);
5251e2d21   Pablo Neira Ayuso   [NETFILTER]: conn...
628

76eb94604   Patrick McHardy   [NETFILTER]: nf_c...
629
  	if (nf_conntrack_max &&
49ac8713b   Alexey Dobriyan   netfilter: netns ...
630
  	    unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
99f07e91b   Changli Gao   netfilter: save t...
631
  		if (!early_drop(net, hash_bucket(hash, net))) {
49ac8713b   Alexey Dobriyan   netfilter: netns ...
632
  			atomic_dec(&net->ct.count);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
633
634
635
636
637
638
639
640
  			if (net_ratelimit())
  				printk(KERN_WARNING
  				       "nf_conntrack: table full, dropping"
  				       " packet.
  ");
  			return ERR_PTR(-ENOMEM);
  		}
  	}
941297f44   Eric Dumazet   netfilter: nf_con...
641
642
643
644
  	/*
  	 * Do not use kmem_cache_zalloc(), as this cache uses
  	 * SLAB_DESTROY_BY_RCU.
  	 */
5b3501faa   Eric Dumazet   netfilter: nf_con...
645
  	ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
646
  	if (ct == NULL) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
647
648
  		pr_debug("nf_conntrack_alloc: Can't alloc conntrack.
  ");
49ac8713b   Alexey Dobriyan   netfilter: netns ...
649
  		atomic_dec(&net->ct.count);
dacd2a1a5   Yasuyuki Kozakai   [NETFILTER]: nf_c...
650
  		return ERR_PTR(-ENOMEM);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
651
  	}
941297f44   Eric Dumazet   netfilter: nf_con...
652
653
654
655
656
  	/*
  	 * Let ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.next
  	 * and ct->tuplehash[IP_CT_DIR_REPLY].hnnode.next unchanged.
  	 */
  	memset(&ct->tuplehash[IP_CT_DIR_MAX], 0,
e5fc9e7a6   Changli Gao   netfilter: nf_con...
657
658
  	       offsetof(struct nf_conn, proto) -
  	       offsetof(struct nf_conn, tuplehash[IP_CT_DIR_MAX]));
440f0d588   Patrick McHardy   netfilter: nf_con...
659
  	spin_lock_init(&ct->lock);
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
660
  	ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
941297f44   Eric Dumazet   netfilter: nf_con...
661
  	ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
662
  	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
99f07e91b   Changli Gao   netfilter: save t...
663
664
  	/* save hash for reusing when confirming */
  	*(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
665
  	/* Don't set timer yet: wait for confirmation */
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
666
  	setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
c2d9ba9bc   Eric Dumazet   net: CONFIG_NET_N...
667
  	write_pnet(&ct->ct_net, net);
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
668
669
670
671
672
673
674
675
676
677
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  	if (zone) {
  		struct nf_conntrack_zone *nf_ct_zone;
  
  		nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);
  		if (!nf_ct_zone)
  			goto out_free;
  		nf_ct_zone->id = zone;
  	}
  #endif
941297f44   Eric Dumazet   netfilter: nf_con...
678
679
680
681
682
  	/*
  	 * changes to lookup keys must be done before setting refcnt to 1
  	 */
  	smp_wmb();
  	atomic_set(&ct->ct_general.use, 1);
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
683
  	return ct;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
684
685
686
687
688
689
  
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  out_free:
  	kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
  	return ERR_PTR(-ENOMEM);
  #endif
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
690
  }
99f07e91b   Changli Gao   netfilter: save t...
691
692
693
694
695
696
697
698
  
  struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,
  				   const struct nf_conntrack_tuple *orig,
  				   const struct nf_conntrack_tuple *repl,
  				   gfp_t gfp)
  {
  	return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
699
  EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
700

c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
701
  void nf_conntrack_free(struct nf_conn *ct)
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
702
  {
1d45209d8   Eric Dumazet   netfilter: nf_con...
703
  	struct net *net = nf_ct_net(ct);
ceeff7541   Patrick McHardy   netfilter: nf_con...
704
  	nf_ct_ext_destroy(ct);
1d45209d8   Eric Dumazet   netfilter: nf_con...
705
  	atomic_dec(&net->ct.count);
ea781f197   Eric Dumazet   netfilter: nf_con...
706
  	nf_ct_ext_free(ct);
5b3501faa   Eric Dumazet   netfilter: nf_con...
707
  	kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
708
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
709
  EXPORT_SYMBOL_GPL(nf_conntrack_free);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
710
711
712
713
  
  /* Allocate a new conntrack: we return -ENOMEM if classification
     failed due to stress.  Otherwise it really is unclassifiable. */
  static struct nf_conntrack_tuple_hash *
b2a15a604   Patrick McHardy   netfilter: nf_con...
714
  init_conntrack(struct net *net, struct nf_conn *tmpl,
5a1fb391d   Alexey Dobriyan   netfilter: netns ...
715
  	       const struct nf_conntrack_tuple *tuple,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
716
  	       struct nf_conntrack_l3proto *l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
717
  	       struct nf_conntrack_l4proto *l4proto,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
718
  	       struct sk_buff *skb,
99f07e91b   Changli Gao   netfilter: save t...
719
  	       unsigned int dataoff, u32 hash)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
720
  {
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
721
  	struct nf_conn *ct;
3c158f7f5   Patrick McHarrdy   [NETFILTER]: nf_c...
722
  	struct nf_conn_help *help;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
723
  	struct nf_conntrack_tuple repl_tuple;
b2a15a604   Patrick McHardy   netfilter: nf_con...
724
  	struct nf_conntrack_ecache *ecache;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
725
  	struct nf_conntrack_expect *exp;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
726
  	u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
727

605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
728
  	if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
729
730
  		pr_debug("Can't invert tuple.
  ");
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
731
732
  		return NULL;
  	}
99f07e91b   Changli Gao   netfilter: save t...
733
734
  	ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
  				  hash);
cd7fcbf1c   Julia Lawall   netfilter 07/09: ...
735
  	if (IS_ERR(ct)) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
736
737
  		pr_debug("Can't allocate conntrack.
  ");
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
738
  		return (struct nf_conntrack_tuple_hash *)ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
739
  	}
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
740
741
  	if (!l4proto->new(ct, skb, dataoff)) {
  		nf_conntrack_free(ct);
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
742
743
  		pr_debug("init conntrack: can't track with proto module
  ");
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
744
745
  		return NULL;
  	}
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
746
  	nf_ct_acct_ext_add(ct, GFP_ATOMIC);
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
747
  	nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
b2a15a604   Patrick McHardy   netfilter: nf_con...
748
749
750
751
752
  
  	ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
  	nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
  				 ecache ? ecache->expmask : 0,
  			     GFP_ATOMIC);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
753

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
754
  	spin_lock_bh(&nf_conntrack_lock);
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
755
  	exp = nf_ct_find_expectation(net, zone, tuple);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
756
  	if (exp) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
757
758
  		pr_debug("conntrack: expectation arrives ct=%p exp=%p
  ",
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
759
  			 ct, exp);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
760
  		/* Welcome, Mr. Bond.  We've been expecting you... */
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
761
762
  		__set_bit(IPS_EXPECTED_BIT, &ct->status);
  		ct->master = exp->master;
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
763
  		if (exp->helper) {
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
764
  			help = nf_ct_helper_ext_add(ct, GFP_ATOMIC);
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
765
766
  			if (help)
  				rcu_assign_pointer(help->helper, exp->helper);
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
767
  		}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
768
  #ifdef CONFIG_NF_CONNTRACK_MARK
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
769
  		ct->mark = exp->master->mark;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
770
  #endif
7c9728c39   James Morris   [SECMARK]: Add se...
771
  #ifdef CONFIG_NF_CONNTRACK_SECMARK
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
772
  		ct->secmark = exp->master->secmark;
7c9728c39   James Morris   [SECMARK]: Add se...
773
  #endif
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
774
  		nf_conntrack_get(&ct->master->ct_general);
0d55af879   Alexey Dobriyan   netfilter: netns ...
775
  		NF_CT_STAT_INC(net, expect_new);
22e7410b7   Yasuyuki Kozakai   [NETFILTER]: nf_c...
776
  	} else {
b2a15a604   Patrick McHardy   netfilter: nf_con...
777
  		__nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
0d55af879   Alexey Dobriyan   netfilter: netns ...
778
  		NF_CT_STAT_INC(net, new);
22e7410b7   Yasuyuki Kozakai   [NETFILTER]: nf_c...
779
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
780
781
  
  	/* Overload tuple linked list to put us in unconfirmed list. */
ea781f197   Eric Dumazet   netfilter: nf_con...
782
  	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
63c9a2626   Alexey Dobriyan   netfilter: netns ...
783
  		       &net->ct.unconfirmed);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
784

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
785
  	spin_unlock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
786
787
788
  
  	if (exp) {
  		if (exp->expectfn)
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
789
  			exp->expectfn(ct, exp);
6823645d6   Patrick McHardy   [NETFILTER]: nf_c...
790
  		nf_ct_expect_put(exp);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
791
  	}
c88130bcd   Patrick McHardy   [NETFILTER]: nf_c...
792
  	return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
793
794
795
796
  }
  
  /* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
  static inline struct nf_conn *
b2a15a604   Patrick McHardy   netfilter: nf_con...
797
  resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
a702a65fc   Alexey Dobriyan   netfilter: netns ...
798
  		  struct sk_buff *skb,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
799
800
801
802
  		  unsigned int dataoff,
  		  u_int16_t l3num,
  		  u_int8_t protonum,
  		  struct nf_conntrack_l3proto *l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
803
  		  struct nf_conntrack_l4proto *l4proto,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
804
805
806
807
808
809
  		  int *set_reply,
  		  enum ip_conntrack_info *ctinfo)
  {
  	struct nf_conntrack_tuple tuple;
  	struct nf_conntrack_tuple_hash *h;
  	struct nf_conn *ct;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
810
  	u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
99f07e91b   Changli Gao   netfilter: save t...
811
  	u32 hash;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
812

bbe735e42   Arnaldo Carvalho de Melo   [SK_BUFF]: Introd...
813
  	if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
814
  			     dataoff, l3num, protonum, &tuple, l3proto,
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
815
  			     l4proto)) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
816
817
  		pr_debug("resolve_normal_ct: Can't get tuple
  ");
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
818
819
820
821
  		return NULL;
  	}
  
  	/* look for tuple match */
99f07e91b   Changli Gao   netfilter: save t...
822
823
  	hash = hash_conntrack_raw(&tuple, zone);
  	h = __nf_conntrack_find_get(net, zone, &tuple, hash);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
824
  	if (!h) {
b2a15a604   Patrick McHardy   netfilter: nf_con...
825
  		h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
99f07e91b   Changli Gao   netfilter: save t...
826
  				   skb, dataoff, hash);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
827
828
829
830
831
832
833
834
835
  		if (!h)
  			return NULL;
  		if (IS_ERR(h))
  			return (void *)h;
  	}
  	ct = nf_ct_tuplehash_to_ctrack(h);
  
  	/* It exists; we have (non-exclusive) reference. */
  	if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
fb0488337   Eric Dumazet   netfilter: add mo...
836
  		*ctinfo = IP_CT_ESTABLISHED_REPLY;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
837
838
839
840
841
  		/* Please set reply bit if this packet OK */
  		*set_reply = 1;
  	} else {
  		/* Once we've had two way comms, always ESTABLISHED. */
  		if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
842
843
  			pr_debug("nf_conntrack_in: normal packet for %p
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
844
845
  			*ctinfo = IP_CT_ESTABLISHED;
  		} else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
846
847
848
  			pr_debug("nf_conntrack_in: related packet for %p
  ",
  				 ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
849
850
  			*ctinfo = IP_CT_RELATED;
  		} else {
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
851
852
  			pr_debug("nf_conntrack_in: new packet for %p
  ", ct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
853
854
855
856
857
858
859
860
861
862
  			*ctinfo = IP_CT_NEW;
  		}
  		*set_reply = 0;
  	}
  	skb->nfct = &ct->ct_general;
  	skb->nfctinfo = *ctinfo;
  	return ct;
  }
  
  unsigned int
a702a65fc   Alexey Dobriyan   netfilter: netns ...
863
864
  nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
  		struct sk_buff *skb)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
865
  {
b2a15a604   Patrick McHardy   netfilter: nf_con...
866
  	struct nf_conn *ct, *tmpl = NULL;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
867
868
  	enum ip_conntrack_info ctinfo;
  	struct nf_conntrack_l3proto *l3proto;
605dcad6c   Martin Josefsson   [NETFILTER]: nf_c...
869
  	struct nf_conntrack_l4proto *l4proto;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
870
871
872
873
  	unsigned int dataoff;
  	u_int8_t protonum;
  	int set_reply = 0;
  	int ret;
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
874
  	if (skb->nfct) {
b2a15a604   Patrick McHardy   netfilter: nf_con...
875
876
877
878
879
880
881
  		/* Previously seen (loopback or untracked)?  Ignore. */
  		tmpl = (struct nf_conn *)skb->nfct;
  		if (!nf_ct_is_template(tmpl)) {
  			NF_CT_STAT_INC_ATOMIC(net, ignore);
  			return NF_ACCEPT;
  		}
  		skb->nfct = NULL;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
882
  	}
923f4902f   Patrick McHardy   [NETFILTER]: nf_c...
883
  	/* rcu_read_lock()ed by nf_hook_slow */
76108cea0   Jan Engelhardt   netfilter: Use un...
884
  	l3proto = __nf_ct_l3proto_find(pf);
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
885
  	ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
ffc306904   Yasuyuki Kozakai   [NETFILTER]: nf_c...
886
887
  				   &dataoff, &protonum);
  	if (ret <= 0) {
25985edce   Lucas De Marchi   Fix common misspe...
888
889
  		pr_debug("not prepared to track yet or error occurred
  ");
0d55af879   Alexey Dobriyan   netfilter: netns ...
890
891
  		NF_CT_STAT_INC_ATOMIC(net, error);
  		NF_CT_STAT_INC_ATOMIC(net, invalid);
b2a15a604   Patrick McHardy   netfilter: nf_con...
892
893
  		ret = -ret;
  		goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
894
  	}
76108cea0   Jan Engelhardt   netfilter: Use un...
895
  	l4proto = __nf_ct_l4proto_find(pf, protonum);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
896
897
898
899
  
  	/* It may be an special packet, error, unclean...
  	 * inverse of the return code tells to the netfilter
  	 * core what to do with the packet. */
74c51a149   Alexey Dobriyan   netfilter: netns ...
900
  	if (l4proto->error != NULL) {
8fea97ec1   Patrick McHardy   netfilter: nf_con...
901
902
  		ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
  				     pf, hooknum);
74c51a149   Alexey Dobriyan   netfilter: netns ...
903
  		if (ret <= 0) {
0d55af879   Alexey Dobriyan   netfilter: netns ...
904
905
  			NF_CT_STAT_INC_ATOMIC(net, error);
  			NF_CT_STAT_INC_ATOMIC(net, invalid);
b2a15a604   Patrick McHardy   netfilter: nf_con...
906
907
  			ret = -ret;
  			goto out;
74c51a149   Alexey Dobriyan   netfilter: netns ...
908
  		}
88ed01d17   Pablo Neira Ayuso   netfilter: nf_con...
909
910
911
  		/* ICMP[v6] protocol trackers may assign one conntrack. */
  		if (skb->nfct)
  			goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
912
  	}
b2a15a604   Patrick McHardy   netfilter: nf_con...
913
  	ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
a702a65fc   Alexey Dobriyan   netfilter: netns ...
914
  			       l3proto, l4proto, &set_reply, &ctinfo);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
915
916
  	if (!ct) {
  		/* Not valid part of a connection */
0d55af879   Alexey Dobriyan   netfilter: netns ...
917
  		NF_CT_STAT_INC_ATOMIC(net, invalid);
b2a15a604   Patrick McHardy   netfilter: nf_con...
918
919
  		ret = NF_ACCEPT;
  		goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
920
921
922
923
  	}
  
  	if (IS_ERR(ct)) {
  		/* Too stressed to deal. */
0d55af879   Alexey Dobriyan   netfilter: netns ...
924
  		NF_CT_STAT_INC_ATOMIC(net, drop);
b2a15a604   Patrick McHardy   netfilter: nf_con...
925
926
  		ret = NF_DROP;
  		goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
927
  	}
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
928
  	NF_CT_ASSERT(skb->nfct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
929

3db05fea5   Herbert Xu   [NETFILTER]: Repl...
930
  	ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum);
ec8d54096   Christoph Paasch   netfilter: conntr...
931
  	if (ret <= 0) {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
932
933
  		/* Invalid: inverse of the return code tells
  		 * the netfilter core what to do */
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
934
935
  		pr_debug("nf_conntrack_in: Can't track with proto module
  ");
3db05fea5   Herbert Xu   [NETFILTER]: Repl...
936
937
  		nf_conntrack_put(skb->nfct);
  		skb->nfct = NULL;
0d55af879   Alexey Dobriyan   netfilter: netns ...
938
  		NF_CT_STAT_INC_ATOMIC(net, invalid);
7d1e04598   Pablo Neira Ayuso   netfilter: nf_con...
939
940
  		if (ret == -NF_DROP)
  			NF_CT_STAT_INC_ATOMIC(net, drop);
b2a15a604   Patrick McHardy   netfilter: nf_con...
941
942
  		ret = -ret;
  		goto out;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
943
944
945
  	}
  
  	if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
858b31330   Patrick McHardy   netfilter: nf_con...
946
  		nf_conntrack_event_cache(IPCT_REPLY, ct);
b2a15a604   Patrick McHardy   netfilter: nf_con...
947
  out:
c31742864   Pablo Neira Ayuso   netfilter: nf_con...
948
949
950
951
952
953
954
955
956
  	if (tmpl) {
  		/* Special case: we have to repeat this hook, assign the
  		 * template again to this packet. We assume that this packet
  		 * has no conntrack assigned. This is used by nf_ct_tcp. */
  		if (ret == NF_REPEAT)
  			skb->nfct = (struct nf_conntrack *)tmpl;
  		else
  			nf_ct_put(tmpl);
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
957
958
959
  
  	return ret;
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
960
  EXPORT_SYMBOL_GPL(nf_conntrack_in);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
961

5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
962
963
  bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
  			  const struct nf_conntrack_tuple *orig)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
964
  {
5f2b4c900   Jan Engelhardt   [NETFILTER]: nf_c...
965
  	bool ret;
923f4902f   Patrick McHardy   [NETFILTER]: nf_c...
966
967
968
969
970
971
972
973
  
  	rcu_read_lock();
  	ret = nf_ct_invert_tuple(inverse, orig,
  				 __nf_ct_l3proto_find(orig->src.l3num),
  				 __nf_ct_l4proto_find(orig->src.l3num,
  						      orig->dst.protonum));
  	rcu_read_unlock();
  	return ret;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
974
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
975
  EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
976

5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
977
978
979
980
981
982
  /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
     implicitly racy: see __nf_conntrack_confirm */
  void nf_conntrack_alter_reply(struct nf_conn *ct,
  			      const struct nf_conntrack_tuple *newreply)
  {
  	struct nf_conn_help *help = nfct_help(ct);
5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
983
984
  	/* Should be unconfirmed, so not in hash table yet */
  	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
0d53778e8   Patrick McHardy   [NETFILTER]: Conv...
985
  	pr_debug("Altering reply tuple of %p to ", ct);
3c9fba656   Jan Engelhardt   [NETFILTER]: nf_c...
986
  	nf_ct_dump_tuple(newreply);
5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
987
988
  
  	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
ef1a5a50b   Patrick McHardy   [NETFILTER]: nf_c...
989
  	if (ct->master || (help && !hlist_empty(&help->expectations)))
c52fbb410   Patrick McHardy   [NETFILTER]: nf_c...
990
  		return;
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
991

c52fbb410   Patrick McHardy   [NETFILTER]: nf_c...
992
  	rcu_read_lock();
b2a15a604   Patrick McHardy   netfilter: nf_con...
993
  	__nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
c52fbb410   Patrick McHardy   [NETFILTER]: nf_c...
994
  	rcu_read_unlock();
5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
995
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
996
  EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
5b1158e90   Jozsef Kadlecsik   [NETFILTER]: Add ...
997

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
998
999
1000
1001
1002
1003
1004
  /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
  void __nf_ct_refresh_acct(struct nf_conn *ct,
  			  enum ip_conntrack_info ctinfo,
  			  const struct sk_buff *skb,
  			  unsigned long extra_jiffies,
  			  int do_acct)
  {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1005
1006
  	NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
  	NF_CT_ASSERT(skb);
997ae831a   Eric Leblond   [NETFILTER]: conn...
1007
  	/* Only update if this is not a fixed timeout */
47d950454   Patrick McHardy   [NETFILTER]: nf_c...
1008
1009
  	if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
  		goto acct;
997ae831a   Eric Leblond   [NETFILTER]: conn...
1010

9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1011
1012
1013
  	/* If not in hash table, timer will not be active yet */
  	if (!nf_ct_is_confirmed(ct)) {
  		ct->timeout.expires = extra_jiffies;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1014
  	} else {
be00c8e48   Martin Josefsson   [NETFILTER]: nf_c...
1015
1016
1017
1018
1019
  		unsigned long newtime = jiffies + extra_jiffies;
  
  		/* Only update the timeout if the new timeout is at least
  		   HZ jiffies from the old timeout. Need del_timer for race
  		   avoidance (may already be dying). */
65cb9fda3   Patrick McHardy   netfilter: nf_con...
1020
1021
  		if (newtime - ct->timeout.expires >= HZ)
  			mod_timer_pending(&ct->timeout, newtime);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1022
  	}
47d950454   Patrick McHardy   [NETFILTER]: nf_c...
1023
  acct:
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1024
  	if (do_acct) {
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1025
  		struct nf_conn_counter *acct;
3ffd5eeb1   Martin Josefsson   [NETFILTER]: nf_c...
1026

584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1027
1028
  		acct = nf_conn_acct_find(ct);
  		if (acct) {
65cb9fda3   Patrick McHardy   netfilter: nf_con...
1029
  			spin_lock_bh(&ct->lock);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1030
  			acct[CTINFO2DIR(ctinfo)].packets++;
6661481d5   Changli Gao   netfilter: nf_con...
1031
  			acct[CTINFO2DIR(ctinfo)].bytes += skb->len;
65cb9fda3   Patrick McHardy   netfilter: nf_con...
1032
  			spin_unlock_bh(&ct->lock);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1033
  		}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1034
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1035
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
1036
  EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1037

4c8894980   David S. Miller   netfilter: Let nf...
1038
1039
1040
1041
  bool __nf_ct_kill_acct(struct nf_conn *ct,
  		       enum ip_conntrack_info ctinfo,
  		       const struct sk_buff *skb,
  		       int do_acct)
51091764f   Patrick McHardy   netfilter: nf_con...
1042
  {
718d4ad98   Fabian Hugelshofer   netfilter: nf_con...
1043
  	if (do_acct) {
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1044
  		struct nf_conn_counter *acct;
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1045
1046
  		acct = nf_conn_acct_find(ct);
  		if (acct) {
65cb9fda3   Patrick McHardy   netfilter: nf_con...
1047
  			spin_lock_bh(&ct->lock);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1048
1049
1050
  			acct[CTINFO2DIR(ctinfo)].packets++;
  			acct[CTINFO2DIR(ctinfo)].bytes +=
  				skb->len - skb_network_offset(skb);
65cb9fda3   Patrick McHardy   netfilter: nf_con...
1051
  			spin_unlock_bh(&ct->lock);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1052
  		}
718d4ad98   Fabian Hugelshofer   netfilter: nf_con...
1053
  	}
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1054

4c8894980   David S. Miller   netfilter: Let nf...
1055
  	if (del_timer(&ct->timeout)) {
51091764f   Patrick McHardy   netfilter: nf_con...
1056
  		ct->timeout.function((unsigned long)ct);
4c8894980   David S. Miller   netfilter: Let nf...
1057
1058
1059
  		return true;
  	}
  	return false;
51091764f   Patrick McHardy   netfilter: nf_con...
1060
  }
718d4ad98   Fabian Hugelshofer   netfilter: nf_con...
1061
  EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);
51091764f   Patrick McHardy   netfilter: nf_con...
1062

5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1063
1064
1065
1066
1067
1068
1069
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {
  	.len	= sizeof(struct nf_conntrack_zone),
  	.align	= __alignof__(struct nf_conntrack_zone),
  	.id	= NF_CT_EXT_ZONE,
  };
  #endif
e281db5cd   Patrick McHardy   [NETFILTER]: nf_c...
1070
  #if defined(CONFIG_NF_CT_NETLINK) || defined(CONFIG_NF_CT_NETLINK_MODULE)
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1071
1072
1073
  
  #include <linux/netfilter/nfnetlink.h>
  #include <linux/netfilter/nfnetlink_conntrack.h>
57b47a53e   Ingo Molnar   [NET]: sem2mutex ...
1074
  #include <linux/mutex.h>
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1075
1076
1077
  /* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
   * in ip_conntrack_core, since we don't want the protocols to autoload
   * or depend on ctnetlink */
fdf708322   Patrick McHardy   [NETFILTER]: nfne...
1078
  int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1079
1080
  			       const struct nf_conntrack_tuple *tuple)
  {
77236b6e3   Patrick McHardy   [NETFILTER]: ctne...
1081
1082
  	NLA_PUT_BE16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port);
  	NLA_PUT_BE16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1083
  	return 0;
df6fb868d   Patrick McHardy   [NETFILTER]: nfne...
1084
  nla_put_failure:
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1085
1086
  	return -1;
  }
fdf708322   Patrick McHardy   [NETFILTER]: nfne...
1087
  EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1088

f73e924cd   Patrick McHardy   [NETFILTER]: ctne...
1089
1090
1091
  const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
  	[CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
  	[CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1092
  };
f73e924cd   Patrick McHardy   [NETFILTER]: ctne...
1093
  EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1094

fdf708322   Patrick McHardy   [NETFILTER]: nfne...
1095
  int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1096
1097
  			       struct nf_conntrack_tuple *t)
  {
df6fb868d   Patrick McHardy   [NETFILTER]: nfne...
1098
  	if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1099
  		return -EINVAL;
77236b6e3   Patrick McHardy   [NETFILTER]: ctne...
1100
1101
  	t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
  	t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1102
1103
1104
  
  	return 0;
  }
fdf708322   Patrick McHardy   [NETFILTER]: nfne...
1105
  EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
5c0de29d0   Holger Eitzenberger   netfilter: nf_con...
1106
1107
1108
1109
1110
1111
  
  int nf_ct_port_nlattr_tuple_size(void)
  {
  	return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
  }
  EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1112
  #endif
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1113
  /* Used by ipt_REJECT and ip6t_REJECT. */
b334aadc3   Patrick McHardy   [NETFILTER]: nf_c...
1114
  static void nf_conntrack_attach(struct sk_buff *nskb, struct sk_buff *skb)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1115
1116
1117
1118
1119
1120
1121
  {
  	struct nf_conn *ct;
  	enum ip_conntrack_info ctinfo;
  
  	/* This ICMP is in reverse direction to the packet which caused it */
  	ct = nf_ct_get(skb, &ctinfo);
  	if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
fb0488337   Eric Dumazet   netfilter: add mo...
1122
  		ctinfo = IP_CT_RELATED_REPLY;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1123
1124
1125
1126
1127
1128
1129
1130
  	else
  		ctinfo = IP_CT_RELATED;
  
  	/* Attach to new skbuff, and increment count */
  	nskb->nfct = &ct->ct_general;
  	nskb->nfctinfo = ctinfo;
  	nf_conntrack_get(nskb->nfct);
  }
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1131
  /* Bring out ya dead! */
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1132
  static struct nf_conn *
400dad39d   Alexey Dobriyan   netfilter: netns ...
1133
  get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1134
1135
  		void *data, unsigned int *bucket)
  {
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1136
1137
  	struct nf_conntrack_tuple_hash *h;
  	struct nf_conn *ct;
ea781f197   Eric Dumazet   netfilter: nf_con...
1138
  	struct hlist_nulls_node *n;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1139

f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1140
  	spin_lock_bh(&nf_conntrack_lock);
d696c7bda   Patrick McHardy   netfilter: nf_con...
1141
  	for (; *bucket < net->ct.htable_size; (*bucket)++) {
ea781f197   Eric Dumazet   netfilter: nf_con...
1142
  		hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1143
1144
1145
1146
  			ct = nf_ct_tuplehash_to_ctrack(h);
  			if (iter(ct, data))
  				goto found;
  		}
601e68e10   YOSHIFUJI Hideaki   [NETFILTER]: Fix ...
1147
  	}
ea781f197   Eric Dumazet   netfilter: nf_con...
1148
  	hlist_nulls_for_each_entry(h, n, &net->ct.unconfirmed, hnnode) {
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1149
1150
  		ct = nf_ct_tuplehash_to_ctrack(h);
  		if (iter(ct, data))
ec68e97de   Patrick McHardy   [NETFILTER]: conn...
1151
  			set_bit(IPS_DYING_BIT, &ct->status);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1152
  	}
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1153
  	spin_unlock_bh(&nf_conntrack_lock);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1154
1155
  	return NULL;
  found:
c073e3fa8   Martin Josefsson   [NETFILTER]: nf_c...
1156
  	atomic_inc(&ct->ct_general.use);
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1157
  	spin_unlock_bh(&nf_conntrack_lock);
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1158
  	return ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1159
  }
400dad39d   Alexey Dobriyan   netfilter: netns ...
1160
1161
1162
  void nf_ct_iterate_cleanup(struct net *net,
  			   int (*iter)(struct nf_conn *i, void *data),
  			   void *data)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1163
  {
df0933dcb   Patrick McHardy   [NETFILTER]: kill...
1164
  	struct nf_conn *ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1165
  	unsigned int bucket = 0;
400dad39d   Alexey Dobriyan   netfilter: netns ...
1166
  	while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1167
1168
1169
1170
1171
1172
1173
1174
  		/* Time to push up daises... */
  		if (del_timer(&ct->timeout))
  			death_by_timeout((unsigned long)ct);
  		/* ... else the timer will get him soon. */
  
  		nf_ct_put(ct);
  	}
  }
13b183391   Patrick McHardy   [NETFILTER]: nf_c...
1175
  EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1176

19abb7b09   Pablo Neira Ayuso   netfilter: ctnetl...
1177
1178
1179
1180
  struct __nf_ct_flush_report {
  	u32 pid;
  	int report;
  };
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1181
  static int kill_report(struct nf_conn *i, void *data)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1182
  {
19abb7b09   Pablo Neira Ayuso   netfilter: ctnetl...
1183
  	struct __nf_ct_flush_report *fr = (struct __nf_ct_flush_report *)data;
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
1184
1185
1186
1187
1188
  	struct nf_conn_tstamp *tstamp;
  
  	tstamp = nf_conn_tstamp_find(i);
  	if (tstamp && tstamp->stop == 0)
  		tstamp->stop = ktime_to_ns(ktime_get_real());
19abb7b09   Pablo Neira Ayuso   netfilter: ctnetl...
1189

dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
1190
1191
1192
1193
1194
1195
1196
  	/* If we fail to deliver the event, death_by_timeout() will retry */
  	if (nf_conntrack_event_report(IPCT_DESTROY, i,
  				      fr->pid, fr->report) < 0)
  		return 1;
  
  	/* Avoid the delivery of the destroy event in death_by_timeout(). */
  	set_bit(IPS_DYING_BIT, &i->status);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1197
1198
  	return 1;
  }
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1199
1200
1201
1202
  static int kill_all(struct nf_conn *i, void *data)
  {
  	return 1;
  }
d862a6622   Patrick McHardy   netfilter: nf_con...
1203
  void nf_ct_free_hashtable(void *hash, unsigned int size)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1204
  {
d862a6622   Patrick McHardy   netfilter: nf_con...
1205
  	if (is_vmalloc_addr(hash))
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1206
1207
  		vfree(hash);
  	else
601e68e10   YOSHIFUJI Hideaki   [NETFILTER]: Fix ...
1208
  		free_pages((unsigned long)hash,
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1209
  			   get_order(sizeof(struct hlist_head) * size));
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1210
  }
ac565e5fc   Patrick McHardy   [NETFILTER]: nf_c...
1211
  EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1212

274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1213
  void nf_conntrack_flush_report(struct net *net, u32 pid, int report)
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1214
  {
19abb7b09   Pablo Neira Ayuso   netfilter: ctnetl...
1215
1216
1217
1218
  	struct __nf_ct_flush_report fr = {
  		.pid 	= pid,
  		.report = report,
  	};
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1219
  	nf_ct_iterate_cleanup(net, kill_report, &fr);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1220
  }
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1221
  EXPORT_SYMBOL_GPL(nf_conntrack_flush_report);
c1d10adb4   Pablo Neira Ayuso   [NETFILTER]: Add ...
1222

ee254fa44   Alexey Dobriyan   netfilter: nf_con...
1223
  static void nf_ct_release_dying_list(struct net *net)
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
1224
1225
1226
1227
1228
1229
  {
  	struct nf_conntrack_tuple_hash *h;
  	struct nf_conn *ct;
  	struct hlist_nulls_node *n;
  
  	spin_lock_bh(&nf_conntrack_lock);
ee254fa44   Alexey Dobriyan   netfilter: nf_con...
1230
  	hlist_nulls_for_each_entry(h, n, &net->ct.dying, hnnode) {
dd7669a92   Pablo Neira Ayuso   netfilter: conntr...
1231
1232
1233
1234
1235
1236
  		ct = nf_ct_tuplehash_to_ctrack(h);
  		/* never fails to remove them, no listeners at this point */
  		nf_ct_kill(ct);
  	}
  	spin_unlock_bh(&nf_conntrack_lock);
  }
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
  static int untrack_refs(void)
  {
  	int cnt = 0, cpu;
  
  	for_each_possible_cpu(cpu) {
  		struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
  
  		cnt += atomic_read(&ct->ct_general.use) - 1;
  	}
  	return cnt;
  }
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1248
  static void nf_conntrack_cleanup_init_net(void)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1249
  {
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1250
  	while (untrack_refs() > 0)
9edd7ca0a   Patrick McHardy   netfilter: nf_con...
1251
  		schedule();
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1252
1253
  	nf_conntrack_helper_fini();
  	nf_conntrack_proto_fini();
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1254
1255
1256
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  	nf_ct_extend_unregister(&nf_ct_zone_extend);
  #endif
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1257
  }
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1258

08f6547d2   Alexey Dobriyan   netfilter: netns ...
1259
1260
  static void nf_conntrack_cleanup_net(struct net *net)
  {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1261
   i_see_dead_people:
274d383b9   Pablo Neira Ayuso   netfilter: conntr...
1262
  	nf_ct_iterate_cleanup(net, kill_all, NULL);
ee254fa44   Alexey Dobriyan   netfilter: nf_con...
1263
  	nf_ct_release_dying_list(net);
49ac8713b   Alexey Dobriyan   netfilter: netns ...
1264
  	if (atomic_read(&net->ct.count) != 0) {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1265
1266
1267
  		schedule();
  		goto i_see_dead_people;
  	}
d862a6622   Patrick McHardy   netfilter: nf_con...
1268
  	nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
1269
  	nf_conntrack_ecache_fini(net);
fe8f661f2   Stephen Hemminger   netfilter: nf_con...
1270
  	nf_conntrack_tstamp_fini(net);
d716a4dfb   Alexey Dobriyan   netfilter: netns ...
1271
  	nf_conntrack_acct_fini(net);
9b03f38d0   Alexey Dobriyan   netfilter: netns ...
1272
  	nf_conntrack_expect_fini(net);
5b3501faa   Eric Dumazet   netfilter: nf_con...
1273
1274
  	kmem_cache_destroy(net->ct.nf_conntrack_cachep);
  	kfree(net->ct.slabname);
0d55af879   Alexey Dobriyan   netfilter: netns ...
1275
  	free_percpu(net->ct.stat);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
  }
  
  /* Mishearing the voices in his head, our hero wonders how he's
     supposed to kill the mall. */
  void nf_conntrack_cleanup(struct net *net)
  {
  	if (net_eq(net, &init_net))
  		rcu_assign_pointer(ip_ct_attach, NULL);
  
  	/* This makes sure all current packets have passed through
  	   netfilter framework.  Roll on, two-stage module
  	   delete... */
  	synchronize_net();
  
  	nf_conntrack_cleanup_net(net);
  
  	if (net_eq(net, &init_net)) {
  		rcu_assign_pointer(nf_ct_destroy, NULL);
  		nf_conntrack_cleanup_init_net();
  	}
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1296
  }
d862a6622   Patrick McHardy   netfilter: nf_con...
1297
  void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1298
  {
ea781f197   Eric Dumazet   netfilter: nf_con...
1299
1300
1301
  	struct hlist_nulls_head *hash;
  	unsigned int nr_slots, i;
  	size_t sz;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1302

ea781f197   Eric Dumazet   netfilter: nf_con...
1303
1304
1305
1306
1307
  	BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
  	nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
  	sz = nr_slots * sizeof(struct hlist_nulls_head);
  	hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
  					get_order(sz));
601e68e10   YOSHIFUJI Hideaki   [NETFILTER]: Fix ...
1308
  	if (!hash) {
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1309
1310
  		printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.
  ");
6b1686a71   Eric Dumazet   netfilter: nf_con...
1311
1312
  		hash = __vmalloc(sz, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO,
  				 PAGE_KERNEL);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1313
  	}
ea781f197   Eric Dumazet   netfilter: nf_con...
1314
1315
1316
  	if (hash && nulls)
  		for (i = 0; i < nr_slots; i++)
  			INIT_HLIST_NULLS_HEAD(&hash[i], i);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1317
1318
1319
  
  	return hash;
  }
ac565e5fc   Patrick McHardy   [NETFILTER]: nf_c...
1320
  EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1321

fae718dda   Patrick McHardy   [NETFILTER]: nf_c...
1322
  int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1323
  {
d862a6622   Patrick McHardy   netfilter: nf_con...
1324
  	int i, bucket;
96eb24d77   Stephen Hemminger   [NETFILTER]: nf_c...
1325
  	unsigned int hashsize, old_size;
ea781f197   Eric Dumazet   netfilter: nf_con...
1326
  	struct hlist_nulls_head *hash, *old_hash;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1327
  	struct nf_conntrack_tuple_hash *h;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1328
  	struct nf_conn *ct;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1329

d696c7bda   Patrick McHardy   netfilter: nf_con...
1330
1331
  	if (current->nsproxy->net_ns != &init_net)
  		return -EOPNOTSUPP;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1332
1333
1334
  	/* On boot, we can set this without any fancy locking. */
  	if (!nf_conntrack_htable_size)
  		return param_set_uint(val, kp);
96eb24d77   Stephen Hemminger   [NETFILTER]: nf_c...
1335
  	hashsize = simple_strtoul(val, NULL, 0);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1336
1337
  	if (!hashsize)
  		return -EINVAL;
d862a6622   Patrick McHardy   netfilter: nf_con...
1338
  	hash = nf_ct_alloc_hashtable(&hashsize, 1);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1339
1340
  	if (!hash)
  		return -ENOMEM;
76507f69c   Patrick McHardy   [NETFILTER]: nf_c...
1341
1342
1343
1344
1345
  	/* Lookups in the old hash might happen in parallel, which means we
  	 * might get false negatives during connection lookup. New connections
  	 * created because of a false negative won't make it into the hash
  	 * though since that required taking the lock.
  	 */
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1346
  	spin_lock_bh(&nf_conntrack_lock);
d696c7bda   Patrick McHardy   netfilter: nf_con...
1347
  	for (i = 0; i < init_net.ct.htable_size; i++) {
ea781f197   Eric Dumazet   netfilter: nf_con...
1348
1349
1350
  		while (!hlist_nulls_empty(&init_net.ct.hash[i])) {
  			h = hlist_nulls_entry(init_net.ct.hash[i].first,
  					struct nf_conntrack_tuple_hash, hnnode);
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1351
  			ct = nf_ct_tuplehash_to_ctrack(h);
ea781f197   Eric Dumazet   netfilter: nf_con...
1352
  			hlist_nulls_del_rcu(&h->hnnode);
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1353
  			bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),
99f07e91b   Changli Gao   netfilter: save t...
1354
  						  hashsize);
ea781f197   Eric Dumazet   netfilter: nf_con...
1355
  			hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1356
1357
  		}
  	}
d696c7bda   Patrick McHardy   netfilter: nf_con...
1358
  	old_size = init_net.ct.htable_size;
400dad39d   Alexey Dobriyan   netfilter: netns ...
1359
  	old_hash = init_net.ct.hash;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1360

d696c7bda   Patrick McHardy   netfilter: nf_con...
1361
  	init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;
400dad39d   Alexey Dobriyan   netfilter: netns ...
1362
  	init_net.ct.hash = hash;
f8ba1affa   Patrick McHardy   [NETFILTER]: nf_c...
1363
  	spin_unlock_bh(&nf_conntrack_lock);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1364

d862a6622   Patrick McHardy   netfilter: nf_con...
1365
  	nf_ct_free_hashtable(old_hash, old_size);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1366
1367
  	return 0;
  }
fae718dda   Patrick McHardy   [NETFILTER]: nf_c...
1368
  EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1369

fae718dda   Patrick McHardy   [NETFILTER]: nf_c...
1370
  module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1371
  		  &nf_conntrack_htable_size, 0600);
5bfddbd46   Eric Dumazet   netfilter: nf_con...
1372
1373
  void nf_ct_untracked_status_or(unsigned long bits)
  {
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1374
1375
1376
1377
  	int cpu;
  
  	for_each_possible_cpu(cpu)
  		per_cpu(nf_conntrack_untracked, cpu).status |= bits;
5bfddbd46   Eric Dumazet   netfilter: nf_con...
1378
1379
  }
  EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1380
  static int nf_conntrack_init_init_net(void)
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1381
  {
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1382
  	int max_factor = 8;
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1383
  	int ret, cpu;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1384
1385
  
  	/* Idea from tcp.c: use 1/16384 of memory.  On i386: 32MB
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1386
  	 * machine has 512 buckets. >= 1GB machines have 16384 buckets. */
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1387
1388
  	if (!nf_conntrack_htable_size) {
  		nf_conntrack_htable_size
4481374ce   Jan Beulich   mm: replace vario...
1389
  			= (((totalram_pages << PAGE_SHIFT) / 16384)
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1390
  			   / sizeof(struct hlist_head));
4481374ce   Jan Beulich   mm: replace vario...
1391
  		if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1392
1393
1394
1395
1396
1397
1398
1399
1400
  			nf_conntrack_htable_size = 16384;
  		if (nf_conntrack_htable_size < 32)
  			nf_conntrack_htable_size = 32;
  
  		/* Use a max. factor of four by default to get the same max as
  		 * with the old struct list_heads. When a table size is given
  		 * we use the old value of 8 to avoid reducing the max.
  		 * entries. */
  		max_factor = 4;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1401
  	}
f205c5e0c   Patrick McHardy   [NETFILTER]: nf_c...
1402
  	nf_conntrack_max = max_factor * nf_conntrack_htable_size;
8e5105a0c   Patrick McHardy   [NETFILTER]: nf_c...
1403

654d0fbdc   Stephen Hemminger   netfilter: cleanu...
1404
1405
  	printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)
  ",
8e5105a0c   Patrick McHardy   [NETFILTER]: nf_c...
1406
1407
  	       NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
  	       nf_conntrack_max);
e9c1b084e   Patrick McHardy   [NETFILTER]: nf_c...
1408
1409
  	ret = nf_conntrack_proto_init();
  	if (ret < 0)
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1410
  		goto err_proto;
933a41e7e   Patrick McHardy   [NETFILTER]: nf_c...
1411

ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
1412
1413
  	ret = nf_conntrack_helper_init();
  	if (ret < 0)
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1414
  		goto err_helper;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1415
1416
1417
1418
1419
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  	ret = nf_ct_extend_register(&nf_ct_zone_extend);
  	if (ret < 0)
  		goto err_extend;
  #endif
9edd7ca0a   Patrick McHardy   netfilter: nf_con...
1420
  	/* Set up fake conntrack: to never be deleted, not in any hashes */
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1421
1422
  	for_each_possible_cpu(cpu) {
  		struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
b3c5163fe   Eric Dumazet   netfilter: nf_con...
1423
1424
1425
  		write_pnet(&ct->ct_net, &init_net);
  		atomic_set(&ct->ct_general.use, 1);
  	}
9edd7ca0a   Patrick McHardy   netfilter: nf_con...
1426
  	/*  - and look it like as a confirmed connection */
5bfddbd46   Eric Dumazet   netfilter: nf_con...
1427
  	nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1428
  	return 0;
5d0aa2ccd   Patrick McHardy   netfilter: nf_con...
1429
1430
1431
1432
  #ifdef CONFIG_NF_CONNTRACK_ZONES
  err_extend:
  	nf_conntrack_helper_fini();
  #endif
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1433
1434
1435
  err_helper:
  	nf_conntrack_proto_fini();
  err_proto:
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1436
1437
  	return ret;
  }
8cc20198c   Eric Dumazet   netfilter: nf_con...
1438
1439
1440
1441
1442
  /*
   * We need to use special "null" values, not used in hash table
   */
  #define UNCONFIRMED_NULLS_VAL	((1<<30)+0)
  #define DYING_NULLS_VAL		((1<<30)+1)
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1443
1444
1445
  static int nf_conntrack_init_net(struct net *net)
  {
  	int ret;
ceceae1b1   Yasuyuki Kozakai   [NETFILTER]: nf_c...
1446

08f6547d2   Alexey Dobriyan   netfilter: netns ...
1447
  	atomic_set(&net->ct.count, 0);
8cc20198c   Eric Dumazet   netfilter: nf_con...
1448
1449
  	INIT_HLIST_NULLS_HEAD(&net->ct.unconfirmed, UNCONFIRMED_NULLS_VAL);
  	INIT_HLIST_NULLS_HEAD(&net->ct.dying, DYING_NULLS_VAL);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1450
1451
1452
1453
1454
  	net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
  	if (!net->ct.stat) {
  		ret = -ENOMEM;
  		goto err_stat;
  	}
5b3501faa   Eric Dumazet   netfilter: nf_con...
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
  
  	net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
  	if (!net->ct.slabname) {
  		ret = -ENOMEM;
  		goto err_slabname;
  	}
  
  	net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,
  							sizeof(struct nf_conn), 0,
  							SLAB_DESTROY_BY_RCU, NULL);
  	if (!net->ct.nf_conntrack_cachep) {
  		printk(KERN_ERR "Unable to create nf_conn slab cache
  ");
  		ret = -ENOMEM;
  		goto err_cache;
  	}
d696c7bda   Patrick McHardy   netfilter: nf_con...
1471
1472
  
  	net->ct.htable_size = nf_conntrack_htable_size;
d862a6622   Patrick McHardy   netfilter: nf_con...
1473
  	net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1474
1475
1476
1477
1478
1479
1480
1481
1482
  	if (!net->ct.hash) {
  		ret = -ENOMEM;
  		printk(KERN_ERR "Unable to create nf_conntrack_hash
  ");
  		goto err_hash;
  	}
  	ret = nf_conntrack_expect_init(net);
  	if (ret < 0)
  		goto err_expect;
d716a4dfb   Alexey Dobriyan   netfilter: netns ...
1483
  	ret = nf_conntrack_acct_init(net);
584015727   Krzysztof Piotr Oledzki   netfilter: accoun...
1484
  	if (ret < 0)
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1485
  		goto err_acct;
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
1486
1487
1488
  	ret = nf_conntrack_tstamp_init(net);
  	if (ret < 0)
  		goto err_tstamp;
a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
1489
1490
1491
  	ret = nf_conntrack_ecache_init(net);
  	if (ret < 0)
  		goto err_ecache;
7d3cdc6b5   Yasuyuki Kozakai   [NETFILTER]: nf_c...
1492

08f6547d2   Alexey Dobriyan   netfilter: netns ...
1493
  	return 0;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1494

a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
1495
  err_ecache:
a992ca2a0   Pablo Neira Ayuso   netfilter: nf_con...
1496
1497
  	nf_conntrack_tstamp_fini(net);
  err_tstamp:
a0891aa6a   Pablo Neira Ayuso   netfilter: conntr...
1498
  	nf_conntrack_acct_fini(net);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1499
  err_acct:
9b03f38d0   Alexey Dobriyan   netfilter: netns ...
1500
  	nf_conntrack_expect_fini(net);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1501
  err_expect:
d862a6622   Patrick McHardy   netfilter: nf_con...
1502
  	nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
6058fa6bb   Alexey Dobriyan   netfilter: netns ...
1503
  err_hash:
5b3501faa   Eric Dumazet   netfilter: nf_con...
1504
1505
1506
1507
  	kmem_cache_destroy(net->ct.nf_conntrack_cachep);
  err_cache:
  	kfree(net->ct.slabname);
  err_slabname:
0d55af879   Alexey Dobriyan   netfilter: netns ...
1508
1509
  	free_percpu(net->ct.stat);
  err_stat:
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1510
1511
  	return ret;
  }
f9dd09c7f   Jozsef Kadlecsik   netfilter: nf_nat...
1512
1513
1514
1515
  s16 (*nf_ct_nat_offset)(const struct nf_conn *ct,
  			enum ip_conntrack_dir dir,
  			u32 seq);
  EXPORT_SYMBOL_GPL(nf_ct_nat_offset);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
  int nf_conntrack_init(struct net *net)
  {
  	int ret;
  
  	if (net_eq(net, &init_net)) {
  		ret = nf_conntrack_init_init_net();
  		if (ret < 0)
  			goto out_init_net;
  	}
  	ret = nf_conntrack_init_net(net);
  	if (ret < 0)
  		goto out_net;
  
  	if (net_eq(net, &init_net)) {
  		/* For use by REJECT target */
  		rcu_assign_pointer(ip_ct_attach, nf_conntrack_attach);
  		rcu_assign_pointer(nf_ct_destroy, destroy_conntrack);
f9dd09c7f   Jozsef Kadlecsik   netfilter: nf_nat...
1533
1534
1535
  
  		/* Howto get NAT offsets */
  		rcu_assign_pointer(nf_ct_nat_offset, NULL);
08f6547d2   Alexey Dobriyan   netfilter: netns ...
1536
1537
1538
1539
1540
1541
1542
1543
  	}
  	return 0;
  
  out_net:
  	if (net_eq(net, &init_net))
  		nf_conntrack_cleanup_init_net();
  out_init_net:
  	return ret;
9fb9cbb10   Yasuyuki Kozakai   [NETFILTER]: Add ...
1544
  }