Eric Lee / smarc-fsl-linux-kernel

1

/* Connection state tracking for netfilter. This is separated from,

1

/* Connection state tracking for netfilter. This is separated from,

2

but required by, the NAT layer; it can also be used by an iptables

2

but required by, the NAT layer; it can also be used by an iptables

3

extension. */

3

extension. */

4

5

6

7

8

*

8

*

9

* This program is free software; you can redistribute it and/or modify

9

* This program is free software; you can redistribute it and/or modify

10

* it under the terms of the GNU General Public License version 2 as

10

* it under the terms of the GNU General Public License version 2 as

11

* published by the Free Software Foundation.

11

* published by the Free Software Foundation.

12

*/

12

*/

13

14

#include <linux/types.h>

14

#include <linux/types.h>

15

#include <linux/netfilter.h>

15

#include <linux/netfilter.h>

16

#include <linux/module.h>

16

#include <linux/module.h>

17

#include <linux/sched.h>

17

#include <linux/sched.h>

18

#include <linux/skbuff.h>

18

#include <linux/skbuff.h>

19

#include <linux/proc_fs.h>

19

#include <linux/proc_fs.h>

20

#include <linux/vmalloc.h>

20

#include <linux/vmalloc.h>

21

#include <linux/stddef.h>

21

#include <linux/stddef.h>

22

#include <linux/slab.h>

22

#include <linux/slab.h>

23

#include <linux/random.h>

23

#include <linux/random.h>

24

#include <linux/jhash.h>

24

#include <linux/jhash.h>

25

#include <linux/err.h>

25

#include <linux/err.h>

26

#include <linux/percpu.h>

26

#include <linux/percpu.h>

27

#include <linux/moduleparam.h>

27

#include <linux/moduleparam.h>

28

#include <linux/notifier.h>

28

#include <linux/notifier.h>

29

#include <linux/kernel.h>

29

#include <linux/kernel.h>

30

#include <linux/netdevice.h>

30

#include <linux/netdevice.h>

31

#include <linux/socket.h>

31

#include <linux/socket.h>

32

#include <linux/mm.h>

32

#include <linux/mm.h>

33

#include <linux/nsproxy.h>

33

#include <linux/nsproxy.h>

34

#include <linux/rculist_nulls.h>

34

#include <linux/rculist_nulls.h>

35

36

#include <net/netfilter/nf_conntrack.h>

36

#include <net/netfilter/nf_conntrack.h>

37

#include <net/netfilter/nf_conntrack_l3proto.h>

37

#include <net/netfilter/nf_conntrack_l3proto.h>

38

#include <net/netfilter/nf_conntrack_l4proto.h>

38

#include <net/netfilter/nf_conntrack_l4proto.h>

39

#include <net/netfilter/nf_conntrack_expect.h>

39

#include <net/netfilter/nf_conntrack_expect.h>

40

#include <net/netfilter/nf_conntrack_helper.h>

40

#include <net/netfilter/nf_conntrack_helper.h>

41

#include <net/netfilter/nf_conntrack_core.h>

41

#include <net/netfilter/nf_conntrack_core.h>

42

#include <net/netfilter/nf_conntrack_extend.h>

42

#include <net/netfilter/nf_conntrack_extend.h>

43

#include <net/netfilter/nf_conntrack_acct.h>

43

#include <net/netfilter/nf_conntrack_acct.h>

44

#include <net/netfilter/nf_conntrack_ecache.h>

44

#include <net/netfilter/nf_conntrack_ecache.h>

45

#include <net/netfilter/nf_conntrack_zones.h>

45

#include <net/netfilter/nf_conntrack_zones.h>

46

#include <net/netfilter/nf_conntrack_timestamp.h>

46

#include <net/netfilter/nf_conntrack_timestamp.h>

47

#include <net/netfilter/nf_conntrack_timeout.h>

47

#include <net/netfilter/nf_conntrack_timeout.h>

48

#include <net/netfilter/nf_nat.h>

48

#include <net/netfilter/nf_nat.h>

49

#include <net/netfilter/nf_nat_core.h>

49

#include <net/netfilter/nf_nat_core.h>

50

51

#define NF_CONNTRACK_VERSION "0.5.0"

51

#define NF_CONNTRACK_VERSION "0.5.0"

52

53

int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,

53

int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,

54

enum nf_nat_manip_type manip,

54

enum nf_nat_manip_type manip,

55

const struct nlattr *attr) __read_mostly;

55

const struct nlattr *attr) __read_mostly;

56

EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);

56

EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);

57

58

int (*nf_nat_seq_adjust_hook)(struct sk_buff *skb,

58

int (*nf_nat_seq_adjust_hook)(struct sk_buff *skb,

59

struct nf_conn *ct,

59

struct nf_conn *ct,

60

enum ip_conntrack_info ctinfo,

60

enum ip_conntrack_info ctinfo,

61

unsigned int protoff);

61

unsigned int protoff);

62

EXPORT_SYMBOL_GPL(nf_nat_seq_adjust_hook);

62

EXPORT_SYMBOL_GPL(nf_nat_seq_adjust_hook);

63

64

DEFINE_SPINLOCK(nf_conntrack_lock);

64

DEFINE_SPINLOCK(nf_conntrack_lock);

65

EXPORT_SYMBOL_GPL(nf_conntrack_lock);

65

EXPORT_SYMBOL_GPL(nf_conntrack_lock);

66

67

unsigned int nf_conntrack_htable_size __read_mostly;

67

unsigned int nf_conntrack_htable_size __read_mostly;

68

EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);

68

EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);

69

70

unsigned int nf_conntrack_max __read_mostly;

70

unsigned int nf_conntrack_max __read_mostly;

71

EXPORT_SYMBOL_GPL(nf_conntrack_max);

71

EXPORT_SYMBOL_GPL(nf_conntrack_max);

72

73

DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);

73

DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);

74

EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);

74

EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);

75

76

unsigned int nf_conntrack_hash_rnd __read_mostly;

76

unsigned int nf_conntrack_hash_rnd __read_mostly;

77

EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);

77

EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);

78

79

static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)

79

static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)

80

{

80

{

81

unsigned int n;

81

unsigned int n;

82

83

/* The direction must be ignored, so we hash everything up to the

83

/* The direction must be ignored, so we hash everything up to the

84

* destination ports (which is a multiple of 4) and treat the last

84

* destination ports (which is a multiple of 4) and treat the last

85

* three bytes manually.

85

* three bytes manually.

86

*/

86

*/

87

n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);

87

n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);

88

return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^

88

return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^

89

(((__force __u16)tuple->dst.u.all << 16) |

89

(((__force __u16)tuple->dst.u.all << 16) |

90

tuple->dst.protonum));

90

tuple->dst.protonum));

91

}

91

}

92

93

static u32 __hash_bucket(u32 hash, unsigned int size)

93

static u32 __hash_bucket(u32 hash, unsigned int size)

94

{

94

{

95

return ((u64)hash * size) >> 32;

95

return ((u64)hash * size) >> 32;

96

}

96

}

97

98

static u32 hash_bucket(u32 hash, const struct net *net)

98

static u32 hash_bucket(u32 hash, const struct net *net)

99

{

99

{

100

return __hash_bucket(hash, net->ct.htable_size);

100

return __hash_bucket(hash, net->ct.htable_size);

101

}

101

}

102

103

static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,

103

static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,

104

u16 zone, unsigned int size)

104

u16 zone, unsigned int size)

105

{

105

{

106

return __hash_bucket(hash_conntrack_raw(tuple, zone), size);

106

return __hash_bucket(hash_conntrack_raw(tuple, zone), size);

107

}

107

}

108

109

static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,

109

static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,

110

const struct nf_conntrack_tuple *tuple)

110

const struct nf_conntrack_tuple *tuple)

111

{

111

{

112

return __hash_conntrack(tuple, zone, net->ct.htable_size);

112

return __hash_conntrack(tuple, zone, net->ct.htable_size);

113

}

113

}

114

115

bool

115

bool

116

nf_ct_get_tuple(const struct sk_buff *skb,

116

nf_ct_get_tuple(const struct sk_buff *skb,

117

unsigned int nhoff,

117

unsigned int nhoff,

118

unsigned int dataoff,

118

unsigned int dataoff,

119

u_int16_t l3num,

119

u_int16_t l3num,

120

u_int8_t protonum,

120

u_int8_t protonum,

121

struct nf_conntrack_tuple *tuple,

121

struct nf_conntrack_tuple *tuple,

122

const struct nf_conntrack_l3proto *l3proto,

122

const struct nf_conntrack_l3proto *l3proto,

123

const struct nf_conntrack_l4proto *l4proto)

123

const struct nf_conntrack_l4proto *l4proto)

124

{

124

{

125

memset(tuple, 0, sizeof(*tuple));

125

memset(tuple, 0, sizeof(*tuple));

126

127

tuple->src.l3num = l3num;

127

tuple->src.l3num = l3num;

128

if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)

128

if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)

129

return false;

129

return false;

130

131

tuple->dst.protonum = protonum;

131

tuple->dst.protonum = protonum;

132

tuple->dst.dir = IP_CT_DIR_ORIGINAL;

132

tuple->dst.dir = IP_CT_DIR_ORIGINAL;

133

134

return l4proto->pkt_to_tuple(skb, dataoff, tuple);

134

return l4proto->pkt_to_tuple(skb, dataoff, tuple);

135

}

135

}

136

EXPORT_SYMBOL_GPL(nf_ct_get_tuple);

136

EXPORT_SYMBOL_GPL(nf_ct_get_tuple);

137

138

bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,

138

bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,

139

u_int16_t l3num, struct nf_conntrack_tuple *tuple)

139

u_int16_t l3num, struct nf_conntrack_tuple *tuple)

140

{

140

{

141

struct nf_conntrack_l3proto *l3proto;

141

struct nf_conntrack_l3proto *l3proto;

142

struct nf_conntrack_l4proto *l4proto;

142

struct nf_conntrack_l4proto *l4proto;

143

unsigned int protoff;

143

unsigned int protoff;

144

u_int8_t protonum;

144

u_int8_t protonum;

145

int ret;

145

int ret;

146

147

rcu_read_lock();

147

rcu_read_lock();

148

149

l3proto = __nf_ct_l3proto_find(l3num);

149

l3proto = __nf_ct_l3proto_find(l3num);

150

ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);

150

ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);

151

if (ret != NF_ACCEPT) {

151

if (ret != NF_ACCEPT) {

152

rcu_read_unlock();

152

rcu_read_unlock();

153

return false;

153

return false;

154

}

154

}

155

156

l4proto = __nf_ct_l4proto_find(l3num, protonum);

156

l4proto = __nf_ct_l4proto_find(l3num, protonum);

157

158

ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,

158

ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,

159

l3proto, l4proto);

159

l3proto, l4proto);

160

161

rcu_read_unlock();

161

rcu_read_unlock();

162

return ret;

162

return ret;

163

}

163

}

164

EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);

164

EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);

165

166

bool

166

bool

167

nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,

167

nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,

168

const struct nf_conntrack_tuple *orig,

168

const struct nf_conntrack_tuple *orig,

169

const struct nf_conntrack_l3proto *l3proto,

169

const struct nf_conntrack_l3proto *l3proto,

170

const struct nf_conntrack_l4proto *l4proto)

170

const struct nf_conntrack_l4proto *l4proto)

171

{

171

{

172

memset(inverse, 0, sizeof(*inverse));

172

memset(inverse, 0, sizeof(*inverse));

173

174

inverse->src.l3num = orig->src.l3num;

174

inverse->src.l3num = orig->src.l3num;

175

if (l3proto->invert_tuple(inverse, orig) == 0)

175

if (l3proto->invert_tuple(inverse, orig) == 0)

176

return false;

176

return false;

177

178

inverse->dst.dir = !orig->dst.dir;

178

inverse->dst.dir = !orig->dst.dir;

179

180

inverse->dst.protonum = orig->dst.protonum;

180

inverse->dst.protonum = orig->dst.protonum;

181

return l4proto->invert_tuple(inverse, orig);

181

return l4proto->invert_tuple(inverse, orig);

182

}

182

}

183

EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);

183

EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);

184

185

static void

185

static void

186

clean_from_lists(struct nf_conn *ct)

186

clean_from_lists(struct nf_conn *ct)

187

{

187

{

188

pr_debug("clean_from_lists(%p)\n", ct);

188

pr_debug("clean_from_lists(%p)\n", ct);

189

hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);

189

hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);

190

hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);

190

hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);

191

192

/* Destroy all pending expectations */

192

/* Destroy all pending expectations */

193

nf_ct_remove_expectations(ct);

193

nf_ct_remove_expectations(ct);

194

}

194

}

195

196

static void

196

static void

197

destroy_conntrack(struct nf_conntrack *nfct)

197

destroy_conntrack(struct nf_conntrack *nfct)

198

{

198

{

199

struct nf_conn *ct = (struct nf_conn *)nfct;

199

struct nf_conn *ct = (struct nf_conn *)nfct;

200

struct net *net = nf_ct_net(ct);

200

struct net *net = nf_ct_net(ct);

201

struct nf_conntrack_l4proto *l4proto;

201

struct nf_conntrack_l4proto *l4proto;

202

203

pr_debug("destroy_conntrack(%p)\n", ct);

203

pr_debug("destroy_conntrack(%p)\n", ct);

204

NF_CT_ASSERT(atomic_read(&nfct->use) == 0);

204

NF_CT_ASSERT(atomic_read(&nfct->use) == 0);

205

NF_CT_ASSERT(!timer_pending(&ct->timeout));

205

NF_CT_ASSERT(!timer_pending(&ct->timeout));

206

207

/* To make sure we don't get any weird locking issues here:

207

/* To make sure we don't get any weird locking issues here:

208

* destroy_conntrack() MUST NOT be called with a write lock

208

* destroy_conntrack() MUST NOT be called with a write lock

209

* to nf_conntrack_lock!!! -HW */

209

* to nf_conntrack_lock!!! -HW */

210

rcu_read_lock();

210

rcu_read_lock();

211

l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));

211

l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));

212

if (l4proto && l4proto->destroy)

212

if (l4proto && l4proto->destroy)

213

l4proto->destroy(ct);

213

l4proto->destroy(ct);

214

215

rcu_read_unlock();

215

rcu_read_unlock();

216

217

spin_lock_bh(&nf_conntrack_lock);

217

spin_lock_bh(&nf_conntrack_lock);

218

/* Expectations will have been removed in clean_from_lists,

218

/* Expectations will have been removed in clean_from_lists,

219

* except TFTP can create an expectation on the first packet,

219

* except TFTP can create an expectation on the first packet,

220

* before connection is in the list, so we need to clean here,

220

* before connection is in the list, so we need to clean here,

221

* too. */

221

* too. */

222

nf_ct_remove_expectations(ct);

222

nf_ct_remove_expectations(ct);

223

224

/* We overload first tuple to link into unconfirmed or dying list.*/

224

/* We overload first tuple to link into unconfirmed or dying list.*/

225

BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));

225

BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));

226

hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);

226

hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);

227

228

NF_CT_STAT_INC(net, delete);

228

NF_CT_STAT_INC(net, delete);

229

spin_unlock_bh(&nf_conntrack_lock);

229

spin_unlock_bh(&nf_conntrack_lock);

230

231

if (ct->master)

231

if (ct->master)

232

nf_ct_put(ct->master);

232

nf_ct_put(ct->master);

233

234

pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);

234

pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);

235

nf_conntrack_free(ct);

235

nf_conntrack_free(ct);

236

}

236

}

237

238

void nf_ct_delete_from_lists(struct nf_conn *ct)

238

void nf_ct_delete_from_lists(struct nf_conn *ct)

239

{

239

{

240

struct net *net = nf_ct_net(ct);

240

struct net *net = nf_ct_net(ct);

241

242

nf_ct_helper_destroy(ct);

242

nf_ct_helper_destroy(ct);

243

spin_lock_bh(&nf_conntrack_lock);

243

spin_lock_bh(&nf_conntrack_lock);

244

/* Inside lock so preempt is disabled on module removal path.

244

/* Inside lock so preempt is disabled on module removal path.

245

* Otherwise we can get spurious warnings. */

245

* Otherwise we can get spurious warnings. */

246

NF_CT_STAT_INC(net, delete_list);

246

NF_CT_STAT_INC(net, delete_list);

247

clean_from_lists(ct);

247

clean_from_lists(ct);

248

/* add this conntrack to the dying list */

248

/* add this conntrack to the dying list */

249

hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,

249

hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,

250

&net->ct.dying);

250

&net->ct.dying);

251

spin_unlock_bh(&nf_conntrack_lock);

251

spin_unlock_bh(&nf_conntrack_lock);

252

}

252

}

253

EXPORT_SYMBOL_GPL(nf_ct_delete_from_lists);

253

EXPORT_SYMBOL_GPL(nf_ct_delete_from_lists);

254

255

static void death_by_event(unsigned long ul_conntrack)

255

static void death_by_event(unsigned long ul_conntrack)

256

{

256

{

257

struct nf_conn *ct = (void *)ul_conntrack;

257

struct nf_conn *ct = (void *)ul_conntrack;

258

struct net *net = nf_ct_net(ct);

258

struct net *net = nf_ct_net(ct);

259

struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);

259

struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);

260

261

BUG_ON(ecache == NULL);

261

BUG_ON(ecache == NULL);

262

263

if (nf_conntrack_event(IPCT_DESTROY, ct) < 0) {

263

if (nf_conntrack_event(IPCT_DESTROY, ct) < 0) {

264

/* bad luck, let's retry again */

264

/* bad luck, let's retry again */

265

ecache->timeout.expires = jiffies +

265

ecache->timeout.expires = jiffies +

266

(random32() % net->ct.sysctl_events_retry_timeout);

266

(random32() % net->ct.sysctl_events_retry_timeout);

267

add_timer(&ecache->timeout);

267

add_timer(&ecache->timeout);

268

return;

268

return;

269

}

269

}

270

/* we've got the event delivered, now it's dying */

270

/* we've got the event delivered, now it's dying */

271

set_bit(IPS_DYING_BIT, &ct->status);

271

set_bit(IPS_DYING_BIT, &ct->status);

272

nf_ct_put(ct);

272

nf_ct_put(ct);

273

}

273

}

274

275

void nf_ct_dying_timeout(struct nf_conn *ct)

275

void nf_ct_dying_timeout(struct nf_conn *ct)

276

{

276

{

277

struct net *net = nf_ct_net(ct);

277

struct net *net = nf_ct_net(ct);

278

struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);

278

struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);

279

280

BUG_ON(ecache == NULL);

280

BUG_ON(ecache == NULL);

281

282

/* set a new timer to retry event delivery */

282

/* set a new timer to retry event delivery */

283

setup_timer(&ecache->timeout, death_by_event, (unsigned long)ct);

283

setup_timer(&ecache->timeout, death_by_event, (unsigned long)ct);

284

ecache->timeout.expires = jiffies +

284

ecache->timeout.expires = jiffies +

285

(random32() % net->ct.sysctl_events_retry_timeout);

285

(random32() % net->ct.sysctl_events_retry_timeout);

286

add_timer(&ecache->timeout);

286

add_timer(&ecache->timeout);

287

}

287

}

288

EXPORT_SYMBOL_GPL(nf_ct_dying_timeout);

288

EXPORT_SYMBOL_GPL(nf_ct_dying_timeout);

289

290

static void death_by_timeout(unsigned long ul_conntrack)

290

static void death_by_timeout(unsigned long ul_conntrack)

291

{

291

{

292

struct nf_conn *ct = (void *)ul_conntrack;

292

struct nf_conn *ct = (void *)ul_conntrack;

293

struct nf_conn_tstamp *tstamp;

293

struct nf_conn_tstamp *tstamp;

294

295

tstamp = nf_conn_tstamp_find(ct);

295

tstamp = nf_conn_tstamp_find(ct);

296

if (tstamp && tstamp->stop == 0)

296

if (tstamp && tstamp->stop == 0)

297

tstamp->stop = ktime_to_ns(ktime_get_real());

297

tstamp->stop = ktime_to_ns(ktime_get_real());

298

299

if (!test_bit(IPS_DYING_BIT, &ct->status) &&

299

if (!test_bit(IPS_DYING_BIT, &ct->status) &&

300

unlikely(nf_conntrack_event(IPCT_DESTROY, ct) < 0)) {

300

unlikely(nf_conntrack_event(IPCT_DESTROY, ct) < 0)) {

301

/* destroy event was not delivered */

301

/* destroy event was not delivered */

302

nf_ct_delete_from_lists(ct);

302

nf_ct_delete_from_lists(ct);

303

nf_ct_dying_timeout(ct);

303

nf_ct_dying_timeout(ct);

304

return;

304

return;

305

}

305

}

306

set_bit(IPS_DYING_BIT, &ct->status);

306

set_bit(IPS_DYING_BIT, &ct->status);

307

nf_ct_delete_from_lists(ct);

307

nf_ct_delete_from_lists(ct);

308

nf_ct_put(ct);

308

nf_ct_put(ct);

309

}

309

}

310

311

/*

311

/*

312

* Warning :

312

* Warning :

313

* - Caller must take a reference on returned object

313

* - Caller must take a reference on returned object

314

* and recheck nf_ct_tuple_equal(tuple, &h->tuple)

314

* and recheck nf_ct_tuple_equal(tuple, &h->tuple)

315

* OR

315

* OR

316

* - Caller must lock nf_conntrack_lock before calling this function

316

* - Caller must lock nf_conntrack_lock before calling this function

317

*/

317

*/

318

static struct nf_conntrack_tuple_hash *

318

static struct nf_conntrack_tuple_hash *

319

____nf_conntrack_find(struct net *net, u16 zone,

319

____nf_conntrack_find(struct net *net, u16 zone,

320

const struct nf_conntrack_tuple *tuple, u32 hash)

320

const struct nf_conntrack_tuple *tuple, u32 hash)

321

{

321

{

322

struct nf_conntrack_tuple_hash *h;

322

struct nf_conntrack_tuple_hash *h;

323

struct hlist_nulls_node *n;

323

struct hlist_nulls_node *n;

324

unsigned int bucket = hash_bucket(hash, net);

324

unsigned int bucket = hash_bucket(hash, net);

325

326

/* Disable BHs the entire time since we normally need to disable them

326

/* Disable BHs the entire time since we normally need to disable them

327

* at least once for the stats anyway.

327

* at least once for the stats anyway.

328

*/

328

*/

329

local_bh_disable();

329

local_bh_disable();

330

begin:

330

begin:

331

hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {

331

hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {

332

if (nf_ct_tuple_equal(tuple, &h->tuple) &&

332

if (nf_ct_tuple_equal(tuple, &h->tuple) &&

333

nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)) == zone) {

333

nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)) == zone) {

334

NF_CT_STAT_INC(net, found);

334

NF_CT_STAT_INC(net, found);

335

local_bh_enable();

335

local_bh_enable();

336

return h;

336

return h;

337

}

337

}

338

NF_CT_STAT_INC(net, searched);

338

NF_CT_STAT_INC(net, searched);

339

}

339

}

340

/*

340

/*

341

* if the nulls value we got at the end of this lookup is

341

* if the nulls value we got at the end of this lookup is

342

* not the expected one, we must restart lookup.

342

* not the expected one, we must restart lookup.

343

* We probably met an item that was moved to another chain.

343

* We probably met an item that was moved to another chain.

344

*/

344

*/

345

if (get_nulls_value(n) != bucket) {

345

if (get_nulls_value(n) != bucket) {

346

NF_CT_STAT_INC(net, search_restart);

346

NF_CT_STAT_INC(net, search_restart);

347

goto begin;

347

goto begin;

348

}

348

}

349

local_bh_enable();

349

local_bh_enable();

350

351

return NULL;

351

return NULL;

352

}

352

}

353

354

struct nf_conntrack_tuple_hash *

354

struct nf_conntrack_tuple_hash *

355

__nf_conntrack_find(struct net *net, u16 zone,

355

__nf_conntrack_find(struct net *net, u16 zone,

356

const struct nf_conntrack_tuple *tuple)

356

const struct nf_conntrack_tuple *tuple)

357

{

357

{

358

return ____nf_conntrack_find(net, zone, tuple,

358

return ____nf_conntrack_find(net, zone, tuple,

359

hash_conntrack_raw(tuple, zone));

359

hash_conntrack_raw(tuple, zone));

360

}

360

}

361

EXPORT_SYMBOL_GPL(__nf_conntrack_find);

361

EXPORT_SYMBOL_GPL(__nf_conntrack_find);

362

363

/* Find a connection corresponding to a tuple. */

363

/* Find a connection corresponding to a tuple. */

364

static struct nf_conntrack_tuple_hash *

364

static struct nf_conntrack_tuple_hash *

365

__nf_conntrack_find_get(struct net *net, u16 zone,

365

__nf_conntrack_find_get(struct net *net, u16 zone,

366

const struct nf_conntrack_tuple *tuple, u32 hash)

366

const struct nf_conntrack_tuple *tuple, u32 hash)

367

{

367

{

368

struct nf_conntrack_tuple_hash *h;

368

struct nf_conntrack_tuple_hash *h;

369

struct nf_conn *ct;

369

struct nf_conn *ct;

370

371

rcu_read_lock();

371

rcu_read_lock();

372

begin:

372

begin:

373

h = ____nf_conntrack_find(net, zone, tuple, hash);

373

h = ____nf_conntrack_find(net, zone, tuple, hash);

374

if (h) {

374

if (h) {

375

ct = nf_ct_tuplehash_to_ctrack(h);

375

ct = nf_ct_tuplehash_to_ctrack(h);

376

if (unlikely(nf_ct_is_dying(ct) ||

376

if (unlikely(nf_ct_is_dying(ct) ||

377

!atomic_inc_not_zero(&ct->ct_general.use)))

377

!atomic_inc_not_zero(&ct->ct_general.use)))

378

h = NULL;

378

h = NULL;

379

else {

379

else {

380

if (unlikely(!nf_ct_tuple_equal(tuple, &h->tuple) ||

380

if (unlikely(!nf_ct_tuple_equal(tuple, &h->tuple) ||

381

nf_ct_zone(ct) != zone)) {

381

nf_ct_zone(ct) != zone)) {

382

nf_ct_put(ct);

382

nf_ct_put(ct);

383

goto begin;

383

goto begin;

384

}

384

}

385

}

385

}

386

}

386

}

387

rcu_read_unlock();

387

rcu_read_unlock();

388

389

return h;

389

return h;

390

}

390

}

391

392

struct nf_conntrack_tuple_hash *

392

struct nf_conntrack_tuple_hash *

393

nf_conntrack_find_get(struct net *net, u16 zone,

393

nf_conntrack_find_get(struct net *net, u16 zone,

394

const struct nf_conntrack_tuple *tuple)

394

const struct nf_conntrack_tuple *tuple)

395

{

395

{

396

return __nf_conntrack_find_get(net, zone, tuple,

396

return __nf_conntrack_find_get(net, zone, tuple,

397

hash_conntrack_raw(tuple, zone));

397

hash_conntrack_raw(tuple, zone));

398

}

398

}

399

EXPORT_SYMBOL_GPL(nf_conntrack_find_get);

399

EXPORT_SYMBOL_GPL(nf_conntrack_find_get);

400

401

static void __nf_conntrack_hash_insert(struct nf_conn *ct,

401

static void __nf_conntrack_hash_insert(struct nf_conn *ct,

402

unsigned int hash,

402

unsigned int hash,

403

unsigned int repl_hash)

403

unsigned int repl_hash)

404

{

404

{

405

struct net *net = nf_ct_net(ct);

405

struct net *net = nf_ct_net(ct);

406

407

hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,

407

hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,

408

&net->ct.hash[hash]);

408

&net->ct.hash[hash]);

409

hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,

409

hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,

410

&net->ct.hash[repl_hash]);

410

&net->ct.hash[repl_hash]);

411

}

411

}

412

413

int

413

int

414

nf_conntrack_hash_check_insert(struct nf_conn *ct)

414

nf_conntrack_hash_check_insert(struct nf_conn *ct)

415

{

415

{

416

struct net *net = nf_ct_net(ct);

416

struct net *net = nf_ct_net(ct);

417

unsigned int hash, repl_hash;

417

unsigned int hash, repl_hash;

418

struct nf_conntrack_tuple_hash *h;

418

struct nf_conntrack_tuple_hash *h;

419

struct hlist_nulls_node *n;

419

struct hlist_nulls_node *n;

420

u16 zone;

420

u16 zone;

421

422

zone = nf_ct_zone(ct);

422

zone = nf_ct_zone(ct);

423

hash = hash_conntrack(net, zone,

423

hash = hash_conntrack(net, zone,

424

&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);

424

&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);

425

repl_hash = hash_conntrack(net, zone,

425

repl_hash = hash_conntrack(net, zone,

426

&ct->tuplehash[IP_CT_DIR_REPLY].tuple);

426

&ct->tuplehash[IP_CT_DIR_REPLY].tuple);

427

428

spin_lock_bh(&nf_conntrack_lock);

428

spin_lock_bh(&nf_conntrack_lock);

429

430

/* See if there's one in the list already, including reverse */

430

/* See if there's one in the list already, including reverse */

431

hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)

431

hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)

432

if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,

432

if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,

433

&h->tuple) &&

433

&h->tuple) &&

434

zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))

434

zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))

435

goto out;

435

goto out;

436

hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)

436

hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)

437

if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,

437

if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,

438

&h->tuple) &&

438

&h->tuple) &&

439

zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))

439

zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))

440

goto out;

440

goto out;

441

442

add_timer(&ct->timeout);

442

add_timer(&ct->timeout);

443

nf_conntrack_get(&ct->ct_general);

443

nf_conntrack_get(&ct->ct_general);

444

__nf_conntrack_hash_insert(ct, hash, repl_hash);

444

__nf_conntrack_hash_insert(ct, hash, repl_hash);

445

NF_CT_STAT_INC(net, insert);

445

NF_CT_STAT_INC(net, insert);

446

spin_unlock_bh(&nf_conntrack_lock);

446

spin_unlock_bh(&nf_conntrack_lock);

447

448

return 0;

448

return 0;

449

450

out:

450

out:

451

NF_CT_STAT_INC(net, insert_failed);

451

NF_CT_STAT_INC(net, insert_failed);

452

spin_unlock_bh(&nf_conntrack_lock);

452

spin_unlock_bh(&nf_conntrack_lock);

453

return -EEXIST;

453

return -EEXIST;

454

}

454

}

455

EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);

455

EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);

456

457

/* Confirm a connection given skb; places it in hash table */

457

/* Confirm a connection given skb; places it in hash table */

458

int

458

int

459

__nf_conntrack_confirm(struct sk_buff *skb)

459

__nf_conntrack_confirm(struct sk_buff *skb)

460

{

460

{

461

unsigned int hash, repl_hash;

461

unsigned int hash, repl_hash;

462

struct nf_conntrack_tuple_hash *h;

462

struct nf_conntrack_tuple_hash *h;

463

struct nf_conn *ct;

463

struct nf_conn *ct;

464

struct nf_conn_help *help;

464

struct nf_conn_help *help;

465

struct nf_conn_tstamp *tstamp;

465

struct nf_conn_tstamp *tstamp;

466

struct hlist_nulls_node *n;

466

struct hlist_nulls_node *n;

467

enum ip_conntrack_info ctinfo;

467

enum ip_conntrack_info ctinfo;

468

struct net *net;

468

struct net *net;

469

u16 zone;

469

u16 zone;

470

471

ct = nf_ct_get(skb, &ctinfo);

471

ct = nf_ct_get(skb, &ctinfo);

472

net = nf_ct_net(ct);

472

net = nf_ct_net(ct);

473

474

/* ipt_REJECT uses nf_conntrack_attach to attach related

474

/* ipt_REJECT uses nf_conntrack_attach to attach related

475

ICMP/TCP RST packets in other direction. Actual packet

475

ICMP/TCP RST packets in other direction. Actual packet

476

which created connection will be IP_CT_NEW or for an

476

which created connection will be IP_CT_NEW or for an

477

expected connection, IP_CT_RELATED. */

477

expected connection, IP_CT_RELATED. */

478

if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)

478

if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)

479

return NF_ACCEPT;

479

return NF_ACCEPT;

480

481

zone = nf_ct_zone(ct);

481

zone = nf_ct_zone(ct);

482

/* reuse the hash saved before */

482

/* reuse the hash saved before */

483

hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;

483

hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;

484

hash = hash_bucket(hash, net);

484

hash = hash_bucket(hash, net);

485

repl_hash = hash_conntrack(net, zone,

485

repl_hash = hash_conntrack(net, zone,

486

&ct->tuplehash[IP_CT_DIR_REPLY].tuple);

486

&ct->tuplehash[IP_CT_DIR_REPLY].tuple);

487

488

/* We're not in hash table, and we refuse to set up related

488

/* We're not in hash table, and we refuse to set up related

489

connections for unconfirmed conns. But packet copies and

489

connections for unconfirmed conns. But packet copies and

490

REJECT will give spurious warnings here. */

490

REJECT will give spurious warnings here. */

491

/* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */

491

/* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */

492

493

/* No external references means no one else could have

493

/* No external references means no one else could have

494

confirmed us. */

494

confirmed us. */

495

NF_CT_ASSERT(!nf_ct_is_confirmed(ct));

495

NF_CT_ASSERT(!nf_ct_is_confirmed(ct));

496

pr_debug("Confirming conntrack %p\n", ct);

496

pr_debug("Confirming conntrack %p\n", ct);

497

498

spin_lock_bh(&nf_conntrack_lock);

498

spin_lock_bh(&nf_conntrack_lock);

499

500

/* We have to check the DYING flag inside the lock to prevent

500

/* We have to check the DYING flag inside the lock to prevent

501

a race against nf_ct_get_next_corpse() possibly called from

501

a race against nf_ct_get_next_corpse() possibly called from

502

user context, else we insert an already 'dead' hash, blocking

502

user context, else we insert an already 'dead' hash, blocking

503

further use of that particular connection -JM */

503

further use of that particular connection -JM */

504

505

if (unlikely(nf_ct_is_dying(ct))) {

505

if (unlikely(nf_ct_is_dying(ct))) {

506

spin_unlock_bh(&nf_conntrack_lock);

506

spin_unlock_bh(&nf_conntrack_lock);

507

return NF_ACCEPT;

507

return NF_ACCEPT;

508

}

508

}

509

510

/* See if there's one in the list already, including reverse:

510

/* See if there's one in the list already, including reverse:

511

NAT could have grabbed it without realizing, since we're

511

NAT could have grabbed it without realizing, since we're

512

not in the hash. If there is, we lost race. */

512

not in the hash. If there is, we lost race. */

513

hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)

513

hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)

514

if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,

514

if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,

515

&h->tuple) &&

515

&h->tuple) &&

516

zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))

516

zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))

517

goto out;

517

goto out;

518

hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)

518

hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)

519

if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,

519

if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,

520

&h->tuple) &&

520

&h->tuple) &&

521

zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))

521

zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))

522

goto out;

522

goto out;

523

524

/* Remove from unconfirmed list */

524

/* Remove from unconfirmed list */

525

hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);

525

hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);

526

527

/* Timer relative to confirmation time, not original

527

/* Timer relative to confirmation time, not original

528

setting time, otherwise we'd get timer wrap in

528

setting time, otherwise we'd get timer wrap in

529

weird delay cases. */

529

weird delay cases. */

530

ct->timeout.expires += jiffies;

530

ct->timeout.expires += jiffies;

531

add_timer(&ct->timeout);

531

add_timer(&ct->timeout);

532

atomic_inc(&ct->ct_general.use);

532

atomic_inc(&ct->ct_general.use);

533

ct->status |= IPS_CONFIRMED;

533

ct->status |= IPS_CONFIRMED;

534

535

/* set conntrack timestamp, if enabled. */

535

/* set conntrack timestamp, if enabled. */

536

tstamp = nf_conn_tstamp_find(ct);

536

tstamp = nf_conn_tstamp_find(ct);

537

if (tstamp) {

537

if (tstamp) {

538

if (skb->tstamp.tv64 == 0)

538

if (skb->tstamp.tv64 == 0)

539

__net_timestamp(skb);

539

__net_timestamp(skb);

540

541

tstamp->start = ktime_to_ns(skb->tstamp);

541

tstamp->start = ktime_to_ns(skb->tstamp);

542

}

542

}

543

/* Since the lookup is lockless, hash insertion must be done after

543

/* Since the lookup is lockless, hash insertion must be done after

544

* starting the timer and setting the CONFIRMED bit. The RCU barriers

544

* starting the timer and setting the CONFIRMED bit. The RCU barriers

545

* guarantee that no other CPU can find the conntrack before the above

545

* guarantee that no other CPU can find the conntrack before the above

546

* stores are visible.

546

* stores are visible.

547

*/

547

*/

548

__nf_conntrack_hash_insert(ct, hash, repl_hash);

548

__nf_conntrack_hash_insert(ct, hash, repl_hash);

549

NF_CT_STAT_INC(net, insert);

549

NF_CT_STAT_INC(net, insert);

550

spin_unlock_bh(&nf_conntrack_lock);

550

spin_unlock_bh(&nf_conntrack_lock);

551

552

help = nfct_help(ct);

552

help = nfct_help(ct);

553

if (help && help->helper)

553

if (help && help->helper)

554

nf_conntrack_event_cache(IPCT_HELPER, ct);

554

nf_conntrack_event_cache(IPCT_HELPER, ct);

555

556

nf_conntrack_event_cache(master_ct(ct) ?

556

nf_conntrack_event_cache(master_ct(ct) ?

557

IPCT_RELATED : IPCT_NEW, ct);

557

IPCT_RELATED : IPCT_NEW, ct);

558

return NF_ACCEPT;

558

return NF_ACCEPT;

559

560

out:

560

out:

561

NF_CT_STAT_INC(net, insert_failed);

561

NF_CT_STAT_INC(net, insert_failed);

562

spin_unlock_bh(&nf_conntrack_lock);

562

spin_unlock_bh(&nf_conntrack_lock);

563

return NF_DROP;

563

return NF_DROP;

564

}

564

}

565

EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);

565

EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);

566

567

/* Returns true if a connection correspondings to the tuple (required

567

/* Returns true if a connection correspondings to the tuple (required

568

for NAT). */

568

for NAT). */

569

int

569

int

570

nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,

570

nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,

571

const struct nf_conn *ignored_conntrack)

571

const struct nf_conn *ignored_conntrack)

572

{

572

{

573

struct net *net = nf_ct_net(ignored_conntrack);

573

struct net *net = nf_ct_net(ignored_conntrack);

574

struct nf_conntrack_tuple_hash *h;

574

struct nf_conntrack_tuple_hash *h;

575

struct hlist_nulls_node *n;

575

struct hlist_nulls_node *n;

576

struct nf_conn *ct;

576

struct nf_conn *ct;

577

u16 zone = nf_ct_zone(ignored_conntrack);

577

u16 zone = nf_ct_zone(ignored_conntrack);

578

unsigned int hash = hash_conntrack(net, zone, tuple);

578

unsigned int hash = hash_conntrack(net, zone, tuple);

579

580

/* Disable BHs the entire time since we need to disable them at

580

/* Disable BHs the entire time since we need to disable them at

581

* least once for the stats anyway.

581

* least once for the stats anyway.

582

*/

582

*/

583

rcu_read_lock_bh();

583

rcu_read_lock_bh();

584

hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {

584

hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {

585

ct = nf_ct_tuplehash_to_ctrack(h);

585

ct = nf_ct_tuplehash_to_ctrack(h);

586

if (ct != ignored_conntrack &&

586

if (ct != ignored_conntrack &&

587

nf_ct_tuple_equal(tuple, &h->tuple) &&

587

nf_ct_tuple_equal(tuple, &h->tuple) &&

588

nf_ct_zone(ct) == zone) {

588

nf_ct_zone(ct) == zone) {

589

NF_CT_STAT_INC(net, found);

589

NF_CT_STAT_INC(net, found);

590

rcu_read_unlock_bh();

590

rcu_read_unlock_bh();

591

return 1;

591

return 1;

592

}

592

}

593

NF_CT_STAT_INC(net, searched);

593

NF_CT_STAT_INC(net, searched);

594

}

594

}

595

rcu_read_unlock_bh();

595

rcu_read_unlock_bh();

596

597

return 0;

597

return 0;

598

}

598

}

599

EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);

599

EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);

600

601

#define NF_CT_EVICTION_RANGE 8

601

#define NF_CT_EVICTION_RANGE 8

602

603

/* There's a small race here where we may free a just-assured

603

/* There's a small race here where we may free a just-assured

604

connection. Too bad: we're in trouble anyway. */

604

connection. Too bad: we're in trouble anyway. */

605

static noinline int early_drop(struct net *net, unsigned int hash)

605

static noinline int early_drop(struct net *net, unsigned int hash)

606

{

606

{

607

/* Use oldest entry, which is roughly LRU */

607

/* Use oldest entry, which is roughly LRU */

608

struct nf_conntrack_tuple_hash *h;

608

struct nf_conntrack_tuple_hash *h;

609

struct nf_conn *ct = NULL, *tmp;

609

struct nf_conn *ct = NULL, *tmp;

610

struct hlist_nulls_node *n;

610

struct hlist_nulls_node *n;

611

unsigned int i, cnt = 0;

611

unsigned int i, cnt = 0;

612

int dropped = 0;

612

int dropped = 0;

613

614

rcu_read_lock();

614

rcu_read_lock();

615

for (i = 0; i < net->ct.htable_size; i++) {

615

for (i = 0; i < net->ct.htable_size; i++) {

616

hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],

616

hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],

617

hnnode) {

617

hnnode) {

618

tmp = nf_ct_tuplehash_to_ctrack(h);

618

tmp = nf_ct_tuplehash_to_ctrack(h);

619

if (!test_bit(IPS_ASSURED_BIT, &tmp->status))

619

if (!test_bit(IPS_ASSURED_BIT, &tmp->status))

620

ct = tmp;

620

ct = tmp;

621

cnt++;

621

cnt++;

622

}

622

}

623

624

if (ct != NULL) {

624

if (ct != NULL) {

625

if (likely(!nf_ct_is_dying(ct) &&

625

if (likely(!nf_ct_is_dying(ct) &&

626

atomic_inc_not_zero(&ct->ct_general.use)))

626

atomic_inc_not_zero(&ct->ct_general.use)))

627

break;

627

break;

628

else

628

else

629

ct = NULL;

629

ct = NULL;

630

}

630

}

631

632

if (cnt >= NF_CT_EVICTION_RANGE)

632

if (cnt >= NF_CT_EVICTION_RANGE)

633

break;

633

break;

634

635

hash = (hash + 1) % net->ct.htable_size;

635

hash = (hash + 1) % net->ct.htable_size;

636

}

636

}

637

rcu_read_unlock();

637

rcu_read_unlock();

638

639

if (!ct)

639

if (!ct)

640

return dropped;

640

return dropped;

641

642

if (del_timer(&ct->timeout)) {

642

if (del_timer(&ct->timeout)) {

643

death_by_timeout((unsigned long)ct);

643

death_by_timeout((unsigned long)ct);

644

/* Check if we indeed killed this entry. Reliable event

644

/* Check if we indeed killed this entry. Reliable event

645

delivery may have inserted it into the dying list. */

645

delivery may have inserted it into the dying list. */

646

if (test_bit(IPS_DYING_BIT, &ct->status)) {

646

if (test_bit(IPS_DYING_BIT, &ct->status)) {

647

dropped = 1;

647

dropped = 1;

648

NF_CT_STAT_INC_ATOMIC(net, early_drop);

648

NF_CT_STAT_INC_ATOMIC(net, early_drop);

649

}

649

}

650

}

650

}

651

nf_ct_put(ct);

651

nf_ct_put(ct);

652

return dropped;

652

return dropped;

653

}

653

}

654

655

void init_nf_conntrack_hash_rnd(void)

655

void init_nf_conntrack_hash_rnd(void)

656

{

656

{

657

unsigned int rand;

657

unsigned int rand;

658

659

/*

659

/*

660

* Why not initialize nf_conntrack_rnd in a "init()" function ?

660

* Why not initialize nf_conntrack_rnd in a "init()" function ?

661

* Because there isn't enough entropy when system initializing,

661

* Because there isn't enough entropy when system initializing,

662

* and we initialize it as late as possible.

662

* and we initialize it as late as possible.

663

*/

663

*/

664

do {

664

do {

665

get_random_bytes(&rand, sizeof(rand));

665

get_random_bytes(&rand, sizeof(rand));

666

} while (!rand);

666

} while (!rand);

667

cmpxchg(&nf_conntrack_hash_rnd, 0, rand);

667

cmpxchg(&nf_conntrack_hash_rnd, 0, rand);

668

}

668

}

669

670

static struct nf_conn *

670

static struct nf_conn *

671

__nf_conntrack_alloc(struct net *net, u16 zone,

671

__nf_conntrack_alloc(struct net *net, u16 zone,

672

const struct nf_conntrack_tuple *orig,

672

const struct nf_conntrack_tuple *orig,

673

const struct nf_conntrack_tuple *repl,

673

const struct nf_conntrack_tuple *repl,

674

gfp_t gfp, u32 hash)

674

gfp_t gfp, u32 hash)

675

{

675

{

676

struct nf_conn *ct;

676

struct nf_conn *ct;

677

678

if (unlikely(!nf_conntrack_hash_rnd)) {

678

if (unlikely(!nf_conntrack_hash_rnd)) {

679

init_nf_conntrack_hash_rnd();

679

init_nf_conntrack_hash_rnd();

680

/* recompute the hash as nf_conntrack_hash_rnd is initialized */

680

/* recompute the hash as nf_conntrack_hash_rnd is initialized */

681

hash = hash_conntrack_raw(orig, zone);

681

hash = hash_conntrack_raw(orig, zone);

682

}

682

}

683

684

/* We don't want any race condition at early drop stage */

684

/* We don't want any race condition at early drop stage */

685

atomic_inc(&net->ct.count);

685

atomic_inc(&net->ct.count);

686

687

if (nf_conntrack_max &&

687

if (nf_conntrack_max &&

688

unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {

688

unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {

689

if (!early_drop(net, hash_bucket(hash, net))) {

689

if (!early_drop(net, hash_bucket(hash, net))) {

690

atomic_dec(&net->ct.count);

690

atomic_dec(&net->ct.count);

691

net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");

691

net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");

692

return ERR_PTR(-ENOMEM);

692

return ERR_PTR(-ENOMEM);

693

}

693

}

694

}

694

}

695

696

/*

696

/*

697

* Do not use kmem_cache_zalloc(), as this cache uses

697

* Do not use kmem_cache_zalloc(), as this cache uses

698

* SLAB_DESTROY_BY_RCU.

698

* SLAB_DESTROY_BY_RCU.

699

*/

699

*/

700

ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);

700

ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);

701

if (ct == NULL) {

701

if (ct == NULL) {

702

atomic_dec(&net->ct.count);

702

atomic_dec(&net->ct.count);

703

return ERR_PTR(-ENOMEM);

703

return ERR_PTR(-ENOMEM);

704

}

704

}

705

/*

705

/*

706

* Let ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.next

706

* Let ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.next

707

* and ct->tuplehash[IP_CT_DIR_REPLY].hnnode.next unchanged.

707

* and ct->tuplehash[IP_CT_DIR_REPLY].hnnode.next unchanged.

708

*/

708

*/

709

memset(&ct->tuplehash[IP_CT_DIR_MAX], 0,

709

memset(&ct->tuplehash[IP_CT_DIR_MAX], 0,

710

offsetof(struct nf_conn, proto) -

710

offsetof(struct nf_conn, proto) -

711

offsetof(struct nf_conn, tuplehash[IP_CT_DIR_MAX]));

711

offsetof(struct nf_conn, tuplehash[IP_CT_DIR_MAX]));

712

spin_lock_init(&ct->lock);

712

spin_lock_init(&ct->lock);

713

ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;

713

ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;

714

ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;

714

ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;

715

ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;

715

ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;

716

/* save hash for reusing when confirming */

716

/* save hash for reusing when confirming */

717

*(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;

717

*(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;

718

/* Don't set timer yet: wait for confirmation */

718

/* Don't set timer yet: wait for confirmation */

719

setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);

719

setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);

720

write_pnet(&ct->ct_net, net);

720

write_pnet(&ct->ct_net, net);

721

#ifdef CONFIG_NF_CONNTRACK_ZONES

721

#ifdef CONFIG_NF_CONNTRACK_ZONES

722

if (zone) {

722

if (zone) {

723

struct nf_conntrack_zone *nf_ct_zone;

723

struct nf_conntrack_zone *nf_ct_zone;

724

725

nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);

725

nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);

726

if (!nf_ct_zone)

726

if (!nf_ct_zone)

727

goto out_free;

727

goto out_free;

728

nf_ct_zone->id = zone;

728

nf_ct_zone->id = zone;

729

}

729

}

730

#endif

730

#endif

731

/*

731

/*

732

* changes to lookup keys must be done before setting refcnt to 1

732

* changes to lookup keys must be done before setting refcnt to 1

733

*/

733

*/

734

smp_wmb();

734

smp_wmb();

735

atomic_set(&ct->ct_general.use, 1);

735

atomic_set(&ct->ct_general.use, 1);

736

return ct;

736

return ct;

737

738

#ifdef CONFIG_NF_CONNTRACK_ZONES

738

#ifdef CONFIG_NF_CONNTRACK_ZONES

739

out_free:

739

out_free:

740

atomic_dec(&net->ct.count);

740

atomic_dec(&net->ct.count);

741

kmem_cache_free(net->ct.nf_conntrack_cachep, ct);

741

kmem_cache_free(net->ct.nf_conntrack_cachep, ct);

742

return ERR_PTR(-ENOMEM);

742

return ERR_PTR(-ENOMEM);

743

#endif

743

#endif

744

}

744

}

745

746

struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,

746

struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,

747

const struct nf_conntrack_tuple *orig,

747

const struct nf_conntrack_tuple *orig,

748

const struct nf_conntrack_tuple *repl,

748

const struct nf_conntrack_tuple *repl,

749

gfp_t gfp)

749

gfp_t gfp)

750

{

750

{

751

return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);

751

return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);

752

}

752

}

753

EXPORT_SYMBOL_GPL(nf_conntrack_alloc);

753

EXPORT_SYMBOL_GPL(nf_conntrack_alloc);

754

755

void nf_conntrack_free(struct nf_conn *ct)

755

void nf_conntrack_free(struct nf_conn *ct)

756

{

756

{

757

struct net *net = nf_ct_net(ct);

757

struct net *net = nf_ct_net(ct);

758

759

nf_ct_ext_destroy(ct);

759

nf_ct_ext_destroy(ct);

760

atomic_dec(&net->ct.count);

760

atomic_dec(&net->ct.count);

761

nf_ct_ext_free(ct);

761

nf_ct_ext_free(ct);

762

kmem_cache_free(net->ct.nf_conntrack_cachep, ct);

762

kmem_cache_free(net->ct.nf_conntrack_cachep, ct);

763

}

763

}

764

EXPORT_SYMBOL_GPL(nf_conntrack_free);

764

EXPORT_SYMBOL_GPL(nf_conntrack_free);

765

766

/* Allocate a new conntrack: we return -ENOMEM if classification

766

/* Allocate a new conntrack: we return -ENOMEM if classification

767

failed due to stress. Otherwise it really is unclassifiable. */

767

failed due to stress. Otherwise it really is unclassifiable. */

768

static struct nf_conntrack_tuple_hash *

768

static struct nf_conntrack_tuple_hash *

769

init_conntrack(struct net *net, struct nf_conn *tmpl,

769

init_conntrack(struct net *net, struct nf_conn *tmpl,

770

const struct nf_conntrack_tuple *tuple,

770

const struct nf_conntrack_tuple *tuple,

771

struct nf_conntrack_l3proto *l3proto,

771

struct nf_conntrack_l3proto *l3proto,

772

struct nf_conntrack_l4proto *l4proto,

772

struct nf_conntrack_l4proto *l4proto,

773

struct sk_buff *skb,

773

struct sk_buff *skb,

774

unsigned int dataoff, u32 hash)

774

unsigned int dataoff, u32 hash)

775

{

775

{

776

struct nf_conn *ct;

776

struct nf_conn *ct;

777

struct nf_conn_help *help;

777

struct nf_conn_help *help;

778

struct nf_conntrack_tuple repl_tuple;

778

struct nf_conntrack_tuple repl_tuple;

779

struct nf_conntrack_ecache *ecache;

779

struct nf_conntrack_ecache *ecache;

780

struct nf_conntrack_expect *exp;

780

struct nf_conntrack_expect *exp;

781

u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;

781

u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;

782

struct nf_conn_timeout *timeout_ext;

782

struct nf_conn_timeout *timeout_ext;

783

unsigned int *timeouts;

783

unsigned int *timeouts;

784

785

if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {

785

if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {

786

pr_debug("Can't invert tuple.\n");

786

pr_debug("Can't invert tuple.\n");

787

return NULL;

787

return NULL;

788

}

788

}

789

790

ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,

790

ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,

791

hash);

791

hash);

792

if (IS_ERR(ct))

792

if (IS_ERR(ct))

793

return (struct nf_conntrack_tuple_hash *)ct;

793

return (struct nf_conntrack_tuple_hash *)ct;

794

795

timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;

795

timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;

796

if (timeout_ext)

796

if (timeout_ext)

797

timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);

797

timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);

798

else

798

else

799

timeouts = l4proto->get_timeouts(net);

799

timeouts = l4proto->get_timeouts(net);

800

801

if (!l4proto->new(ct, skb, dataoff, timeouts)) {

801

if (!l4proto->new(ct, skb, dataoff, timeouts)) {

802

nf_conntrack_free(ct);

802

nf_conntrack_free(ct);

803

pr_debug("init conntrack: can't track with proto module\n");

803

pr_debug("init conntrack: can't track with proto module\n");

804

return NULL;

804

return NULL;

805

}

805

}

806

807

if (timeout_ext)

807

if (timeout_ext)

808

nf_ct_timeout_ext_add(ct, timeout_ext->timeout, GFP_ATOMIC);

808

nf_ct_timeout_ext_add(ct, timeout_ext->timeout, GFP_ATOMIC);

809

810

nf_ct_acct_ext_add(ct, GFP_ATOMIC);

810

nf_ct_acct_ext_add(ct, GFP_ATOMIC);

811

nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);

811

nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);

812

813

ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;

813

ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;

814

nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,

814

nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,

815

ecache ? ecache->expmask : 0,

815

ecache ? ecache->expmask : 0,

816

GFP_ATOMIC);

816

GFP_ATOMIC);

817

818

spin_lock_bh(&nf_conntrack_lock);

818

spin_lock_bh(&nf_conntrack_lock);

819

exp = nf_ct_find_expectation(net, zone, tuple);

819

exp = nf_ct_find_expectation(net, zone, tuple);

820

if (exp) {

820

if (exp) {

821

pr_debug("conntrack: expectation arrives ct=%p exp=%p\n",

821

pr_debug("conntrack: expectation arrives ct=%p exp=%p\n",

822

ct, exp);

822

ct, exp);

823

/* Welcome, Mr. Bond. We've been expecting you... */

823

/* Welcome, Mr. Bond. We've been expecting you... */

824

__set_bit(IPS_EXPECTED_BIT, &ct->status);

824

__set_bit(IPS_EXPECTED_BIT, &ct->status);

825

ct->master = exp->master;

825

ct->master = exp->master;

826

if (exp->helper) {

826

if (exp->helper) {

827

help = nf_ct_helper_ext_add(ct, exp->helper,

827

help = nf_ct_helper_ext_add(ct, exp->helper,

828

GFP_ATOMIC);

828

GFP_ATOMIC);

829

if (help)

829

if (help)

830

rcu_assign_pointer(help->helper, exp->helper);

830

rcu_assign_pointer(help->helper, exp->helper);

831

}

831

}

832

833

#ifdef CONFIG_NF_CONNTRACK_MARK

833

#ifdef CONFIG_NF_CONNTRACK_MARK

834

ct->mark = exp->master->mark;

834

ct->mark = exp->master->mark;

835

#endif

835

#endif

836

#ifdef CONFIG_NF_CONNTRACK_SECMARK

836

#ifdef CONFIG_NF_CONNTRACK_SECMARK

837

ct->secmark = exp->master->secmark;

837

ct->secmark = exp->master->secmark;

838

#endif

838

#endif

839

nf_conntrack_get(&ct->master->ct_general);

839

nf_conntrack_get(&ct->master->ct_general);

840

NF_CT_STAT_INC(net, expect_new);

840

NF_CT_STAT_INC(net, expect_new);

841

} else {

841

} else {

842

__nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);

842

__nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);

843

NF_CT_STAT_INC(net, new);

843

NF_CT_STAT_INC(net, new);

844

}

844

}

845

846

/* Overload tuple linked list to put us in unconfirmed list. */

846

/* Overload tuple linked list to put us in unconfirmed list. */

847

hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,

847

hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,

848

&net->ct.unconfirmed);

848

&net->ct.unconfirmed);

849

850

spin_unlock_bh(&nf_conntrack_lock);

850

spin_unlock_bh(&nf_conntrack_lock);

851

852

if (exp) {

852

if (exp) {

853

if (exp->expectfn)

853

if (exp->expectfn)

854

exp->expectfn(ct, exp);

854

exp->expectfn(ct, exp);

855

nf_ct_expect_put(exp);

855

nf_ct_expect_put(exp);

856

}

856

}

857

858

return &ct->tuplehash[IP_CT_DIR_ORIGINAL];

858

return &ct->tuplehash[IP_CT_DIR_ORIGINAL];

859

}

859

}

860

861

/* On success, returns conntrack ptr, sets skb->nfct and ctinfo */

861

/* On success, returns conntrack ptr, sets skb->nfct and ctinfo */

862

static inline struct nf_conn *

862

static inline struct nf_conn *

863

resolve_normal_ct(struct net *net, struct nf_conn *tmpl,

863

resolve_normal_ct(struct net *net, struct nf_conn *tmpl,

864

struct sk_buff *skb,

864

struct sk_buff *skb,

865

unsigned int dataoff,

865

unsigned int dataoff,

866

u_int16_t l3num,

866

u_int16_t l3num,

867

u_int8_t protonum,

867

u_int8_t protonum,

868

struct nf_conntrack_l3proto *l3proto,

868

struct nf_conntrack_l3proto *l3proto,

869

struct nf_conntrack_l4proto *l4proto,

869

struct nf_conntrack_l4proto *l4proto,

870

int *set_reply,

870

int *set_reply,

871

enum ip_conntrack_info *ctinfo)

871

enum ip_conntrack_info *ctinfo)

872

{

872

{

873

struct nf_conntrack_tuple tuple;

873

struct nf_conntrack_tuple tuple;

874

struct nf_conntrack_tuple_hash *h;

874

struct nf_conntrack_tuple_hash *h;

875

struct nf_conn *ct;

875

struct nf_conn *ct;

876

u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;

876

u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;

877

u32 hash;

877

u32 hash;

878

879

if (!nf_ct_get_tuple(skb, skb_network_offset(skb),

879

if (!nf_ct_get_tuple(skb, skb_network_offset(skb),

880

dataoff, l3num, protonum, &tuple, l3proto,

880

dataoff, l3num, protonum, &tuple, l3proto,

881

l4proto)) {

881

l4proto)) {

882

pr_debug("resolve_normal_ct: Can't get tuple\n");

882

pr_debug("resolve_normal_ct: Can't get tuple\n");

883

return NULL;

883

return NULL;

884

}

884

}

885

886

/* look for tuple match */

886

/* look for tuple match */

887

hash = hash_conntrack_raw(&tuple, zone);

887

hash = hash_conntrack_raw(&tuple, zone);

888

h = __nf_conntrack_find_get(net, zone, &tuple, hash);

888

h = __nf_conntrack_find_get(net, zone, &tuple, hash);

889

if (!h) {

889

if (!h) {

890

h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,

890

h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,

891

skb, dataoff, hash);

891

skb, dataoff, hash);

892

if (!h)

892

if (!h)

893

return NULL;

893

return NULL;

894

if (IS_ERR(h))

894

if (IS_ERR(h))

895

return (void *)h;

895

return (void *)h;

896

}

896

}

897

ct = nf_ct_tuplehash_to_ctrack(h);

897

ct = nf_ct_tuplehash_to_ctrack(h);

898

899

/* It exists; we have (non-exclusive) reference. */

899

/* It exists; we have (non-exclusive) reference. */

900

if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {

900

if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {

901

*ctinfo = IP_CT_ESTABLISHED_REPLY;

901

*ctinfo = IP_CT_ESTABLISHED_REPLY;

902

/* Please set reply bit if this packet OK */

902

/* Please set reply bit if this packet OK */

903

*set_reply = 1;

903

*set_reply = 1;

904

} else {

904

} else {

905

/* Once we've had two way comms, always ESTABLISHED. */

905

/* Once we've had two way comms, always ESTABLISHED. */

906

if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {

906

if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {

907

pr_debug("nf_conntrack_in: normal packet for %p\n", ct);

907

pr_debug("nf_conntrack_in: normal packet for %p\n", ct);

908

*ctinfo = IP_CT_ESTABLISHED;

908

*ctinfo = IP_CT_ESTABLISHED;

909

} else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {

909

} else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {

910

pr_debug("nf_conntrack_in: related packet for %p\n",

910

pr_debug("nf_conntrack_in: related packet for %p\n",

911

ct);

911

ct);

912

*ctinfo = IP_CT_RELATED;

912

*ctinfo = IP_CT_RELATED;

913

} else {

913

} else {

914

pr_debug("nf_conntrack_in: new packet for %p\n", ct);

914

pr_debug("nf_conntrack_in: new packet for %p\n", ct);

915

*ctinfo = IP_CT_NEW;

915

*ctinfo = IP_CT_NEW;

916

}

916

}

917

*set_reply = 0;

917

*set_reply = 0;

918

}

918

}

919

skb->nfct = &ct->ct_general;

919

skb->nfct = &ct->ct_general;

920

skb->nfctinfo = *ctinfo;

920

skb->nfctinfo = *ctinfo;

921

return ct;

921

return ct;

922

}

922

}

923

924

unsigned int

924

unsigned int

925

nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,

925

nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,

926

struct sk_buff *skb)

926

struct sk_buff *skb)

927

{

927

{

928

struct nf_conn *ct, *tmpl = NULL;

928

struct nf_conn *ct, *tmpl = NULL;

929

enum ip_conntrack_info ctinfo;

929

enum ip_conntrack_info ctinfo;

930

struct nf_conntrack_l3proto *l3proto;

930

struct nf_conntrack_l3proto *l3proto;

931

struct nf_conntrack_l4proto *l4proto;

931

struct nf_conntrack_l4proto *l4proto;

932

unsigned int *timeouts;

932

unsigned int *timeouts;

933

unsigned int dataoff;

933

unsigned int dataoff;

934

u_int8_t protonum;

934

u_int8_t protonum;

935

int set_reply = 0;

935

int set_reply = 0;

936

int ret;

936

int ret;

937

938

if (skb->nfct) {

938

if (skb->nfct) {

939

/* Previously seen (loopback or untracked)? Ignore. */

939

/* Previously seen (loopback or untracked)? Ignore. */

940

tmpl = (struct nf_conn *)skb->nfct;

940

tmpl = (struct nf_conn *)skb->nfct;

941

if (!nf_ct_is_template(tmpl)) {

941

if (!nf_ct_is_template(tmpl)) {

942

NF_CT_STAT_INC_ATOMIC(net, ignore);

942

NF_CT_STAT_INC_ATOMIC(net, ignore);

943

return NF_ACCEPT;

943

return NF_ACCEPT;

944

}

944

}

945

skb->nfct = NULL;

945

skb->nfct = NULL;

946

}

946

}

947

948

/* rcu_read_lock()ed by nf_hook_slow */

948

/* rcu_read_lock()ed by nf_hook_slow */

949

l3proto = __nf_ct_l3proto_find(pf);

949

l3proto = __nf_ct_l3proto_find(pf);

950

ret = l3proto->get_l4proto(skb, skb_network_offset(skb),

950

ret = l3proto->get_l4proto(skb, skb_network_offset(skb),

951

&dataoff, &protonum);

951

&dataoff, &protonum);

952

if (ret <= 0) {

952

if (ret <= 0) {

953

pr_debug("not prepared to track yet or error occurred\n");

953

pr_debug("not prepared to track yet or error occurred\n");

954

NF_CT_STAT_INC_ATOMIC(net, error);

954

NF_CT_STAT_INC_ATOMIC(net, error);

955

NF_CT_STAT_INC_ATOMIC(net, invalid);

955

NF_CT_STAT_INC_ATOMIC(net, invalid);

956

ret = -ret;

956

ret = -ret;

957

goto out;

957

goto out;

958

}

958

}

959

960

l4proto = __nf_ct_l4proto_find(pf, protonum);

960

l4proto = __nf_ct_l4proto_find(pf, protonum);

961

962

/* It may be an special packet, error, unclean...

962

/* It may be an special packet, error, unclean...

963

* inverse of the return code tells to the netfilter

963

* inverse of the return code tells to the netfilter

964

* core what to do with the packet. */

964

* core what to do with the packet. */

965

if (l4proto->error != NULL) {

965

if (l4proto->error != NULL) {

966

ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,

966

ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,

967

pf, hooknum);

967

pf, hooknum);

968

if (ret <= 0) {

968

if (ret <= 0) {

969

NF_CT_STAT_INC_ATOMIC(net, error);

969

NF_CT_STAT_INC_ATOMIC(net, error);

970

NF_CT_STAT_INC_ATOMIC(net, invalid);

970

NF_CT_STAT_INC_ATOMIC(net, invalid);

971

ret = -ret;

971

ret = -ret;

972

goto out;

972

goto out;

973

}

973

}

974

/* ICMP[v6] protocol trackers may assign one conntrack. */

974

/* ICMP[v6] protocol trackers may assign one conntrack. */

975

if (skb->nfct)

975

if (skb->nfct)

976

goto out;

976

goto out;

977

}

977

}

978

979

ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,

979

ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,

980

l3proto, l4proto, &set_reply, &ctinfo);

980

l3proto, l4proto, &set_reply, &ctinfo);

981

if (!ct) {

981

if (!ct) {

982

/* Not valid part of a connection */

982

/* Not valid part of a connection */

983

NF_CT_STAT_INC_ATOMIC(net, invalid);

983

NF_CT_STAT_INC_ATOMIC(net, invalid);

984

ret = NF_ACCEPT;

984

ret = NF_ACCEPT;

985

goto out;

985

goto out;

986

}

986

}

987

988

if (IS_ERR(ct)) {

988

if (IS_ERR(ct)) {

989

/* Too stressed to deal. */

989

/* Too stressed to deal. */

990

NF_CT_STAT_INC_ATOMIC(net, drop);

990

NF_CT_STAT_INC_ATOMIC(net, drop);

991

ret = NF_DROP;

991

ret = NF_DROP;

992

goto out;

992

goto out;

993

}

993

}

994

995

NF_CT_ASSERT(skb->nfct);

995

NF_CT_ASSERT(skb->nfct);

996

997

/* Decide what timeout policy we want to apply to this flow. */

997

/* Decide what timeout policy we want to apply to this flow. */

998

timeouts = nf_ct_timeout_lookup(net, ct, l4proto);

998

timeouts = nf_ct_timeout_lookup(net, ct, l4proto);

999

1000

ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);

1000

ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);

1001

if (ret <= 0) {

1001

if (ret <= 0) {

1002

/* Invalid: inverse of the return code tells

1002

/* Invalid: inverse of the return code tells

1003

* the netfilter core what to do */

1003

* the netfilter core what to do */

1004

pr_debug("nf_conntrack_in: Can't track with proto module\n");

1004

pr_debug("nf_conntrack_in: Can't track with proto module\n");

1005

nf_conntrack_put(skb->nfct);

1005

nf_conntrack_put(skb->nfct);

1006

skb->nfct = NULL;

1006

skb->nfct = NULL;

1007

NF_CT_STAT_INC_ATOMIC(net, invalid);

1007

NF_CT_STAT_INC_ATOMIC(net, invalid);

1008

if (ret == -NF_DROP)

1008

if (ret == -NF_DROP)

1009

NF_CT_STAT_INC_ATOMIC(net, drop);

1009

NF_CT_STAT_INC_ATOMIC(net, drop);

1010

ret = -ret;

1010

ret = -ret;

1011

goto out;

1011

goto out;

1012

}

1012

}

1013

1014

if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))

1014

if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))

1015

nf_conntrack_event_cache(IPCT_REPLY, ct);

1015

nf_conntrack_event_cache(IPCT_REPLY, ct);

1016

out:

1016

out:

1017

if (tmpl) {

1017

if (tmpl) {

1018

/* Special case: we have to repeat this hook, assign the

1018

/* Special case: we have to repeat this hook, assign the

1019

* template again to this packet. We assume that this packet

1019

* template again to this packet. We assume that this packet

1020

* has no conntrack assigned. This is used by nf_ct_tcp. */

1020

* has no conntrack assigned. This is used by nf_ct_tcp. */

1021

if (ret == NF_REPEAT)

1021

if (ret == NF_REPEAT)

1022

skb->nfct = (struct nf_conntrack *)tmpl;

1022

skb->nfct = (struct nf_conntrack *)tmpl;

1023

else

1023

else

1024

nf_ct_put(tmpl);

1024

nf_ct_put(tmpl);

1025

}

1025

}

1026

1027

return ret;

1027

return ret;

1028

}

1028

}

1029

EXPORT_SYMBOL_GPL(nf_conntrack_in);

1029

EXPORT_SYMBOL_GPL(nf_conntrack_in);

1030

1031

bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,

1031

bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,

1032

const struct nf_conntrack_tuple *orig)

1032

const struct nf_conntrack_tuple *orig)

1033

{

1033

{

1034

bool ret;

1034

bool ret;

1035

1036

rcu_read_lock();

1036

rcu_read_lock();

1037

ret = nf_ct_invert_tuple(inverse, orig,

1037

ret = nf_ct_invert_tuple(inverse, orig,

1038

__nf_ct_l3proto_find(orig->src.l3num),

1038

__nf_ct_l3proto_find(orig->src.l3num),

1039

__nf_ct_l4proto_find(orig->src.l3num,

1039

__nf_ct_l4proto_find(orig->src.l3num,

1040

orig->dst.protonum));

1040

orig->dst.protonum));

1041

rcu_read_unlock();

1041

rcu_read_unlock();

1042

return ret;

1042

return ret;

1043

}

1043

}

1044

EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);

1044

EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);

1045

1046

/* Alter reply tuple (maybe alter helper). This is for NAT, and is

1046

/* Alter reply tuple (maybe alter helper). This is for NAT, and is

1047

implicitly racy: see __nf_conntrack_confirm */

1047

implicitly racy: see __nf_conntrack_confirm */

1048

void nf_conntrack_alter_reply(struct nf_conn *ct,

1048

void nf_conntrack_alter_reply(struct nf_conn *ct,

1049

const struct nf_conntrack_tuple *newreply)

1049

const struct nf_conntrack_tuple *newreply)

1050

{

1050

{

1051

struct nf_conn_help *help = nfct_help(ct);

1051

struct nf_conn_help *help = nfct_help(ct);

1052

1053

/* Should be unconfirmed, so not in hash table yet */

1053

/* Should be unconfirmed, so not in hash table yet */

1054

NF_CT_ASSERT(!nf_ct_is_confirmed(ct));

1054

NF_CT_ASSERT(!nf_ct_is_confirmed(ct));

1055

1056

pr_debug("Altering reply tuple of %p to ", ct);

1056

pr_debug("Altering reply tuple of %p to ", ct);

1057

nf_ct_dump_tuple(newreply);

1057

nf_ct_dump_tuple(newreply);

1058

1059

ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;

1059

ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;

1060

if (ct->master || (help && !hlist_empty(&help->expectations)))

1060

if (ct->master || (help && !hlist_empty(&help->expectations)))

1061

return;

1061

return;

1062

1063

rcu_read_lock();

1063

rcu_read_lock();

1064

__nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);

1064

__nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);

1065

rcu_read_unlock();

1065

rcu_read_unlock();

1066

}

1066

}

1067

EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);

1067

EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);

1068

1069

/* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */

1069

/* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */

1070

void __nf_ct_refresh_acct(struct nf_conn *ct,

1070

void __nf_ct_refresh_acct(struct nf_conn *ct,

1071

enum ip_conntrack_info ctinfo,

1071

enum ip_conntrack_info ctinfo,

1072

const struct sk_buff *skb,

1072

const struct sk_buff *skb,

1073

unsigned long extra_jiffies,

1073

unsigned long extra_jiffies,

1074

int do_acct)

1074

int do_acct)

1075

{

1075

{

1076

NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);

1076

NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);

1077

NF_CT_ASSERT(skb);

1077

NF_CT_ASSERT(skb);

1078

1079

/* Only update if this is not a fixed timeout */

1079

/* Only update if this is not a fixed timeout */

1080

if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))

1080

if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))

1081

goto acct;

1081

goto acct;

1082

1083

/* If not in hash table, timer will not be active yet */

1083

/* If not in hash table, timer will not be active yet */

1084

if (!nf_ct_is_confirmed(ct)) {

1084

if (!nf_ct_is_confirmed(ct)) {

1085

ct->timeout.expires = extra_jiffies;

1085

ct->timeout.expires = extra_jiffies;

1086

} else {

1086

} else {

1087

unsigned long newtime = jiffies + extra_jiffies;

1087

unsigned long newtime = jiffies + extra_jiffies;

1088

1089

/* Only update the timeout if the new timeout is at least

1089

/* Only update the timeout if the new timeout is at least

1090

HZ jiffies from the old timeout. Need del_timer for race

1090

HZ jiffies from the old timeout. Need del_timer for race

1091

avoidance (may already be dying). */

1091

avoidance (may already be dying). */

1092

if (newtime - ct->timeout.expires >= HZ)

1092

if (newtime - ct->timeout.expires >= HZ)

1093

mod_timer_pending(&ct->timeout, newtime);

1093

mod_timer_pending(&ct->timeout, newtime);

1094

}

1094

}

1095

1096

acct:

1096

acct:

1097

if (do_acct) {

1097

if (do_acct) {

1098

struct nf_conn_counter *acct;

1098

struct nf_conn_counter *acct;

1099

1100

acct = nf_conn_acct_find(ct);

1100

acct = nf_conn_acct_find(ct);

1101

if (acct) {

1101

if (acct) {

1102

atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);

1102

atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);

1103

atomic64_add(skb->len, &acct[CTINFO2DIR(ctinfo)].bytes);

1103

atomic64_add(skb->len, &acct[CTINFO2DIR(ctinfo)].bytes);

1104

}

1104

}

1105

}

1105

}

1106

}

1106

}

1107

EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);

1107

EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);

1108

1109

bool __nf_ct_kill_acct(struct nf_conn *ct,

1109

bool __nf_ct_kill_acct(struct nf_conn *ct,

1110

enum ip_conntrack_info ctinfo,

1110

enum ip_conntrack_info ctinfo,

1111

const struct sk_buff *skb,

1111

const struct sk_buff *skb,

1112

int do_acct)

1112

int do_acct)

1113

{

1113

{

1114

if (do_acct) {

1114

if (do_acct) {

1115

struct nf_conn_counter *acct;

1115

struct nf_conn_counter *acct;

1116

1117

acct = nf_conn_acct_find(ct);

1117

acct = nf_conn_acct_find(ct);

1118

if (acct) {

1118

if (acct) {

1119

atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);

1119

atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);

1120

atomic64_add(skb->len - skb_network_offset(skb),

1120

atomic64_add(skb->len - skb_network_offset(skb),

1121

&acct[CTINFO2DIR(ctinfo)].bytes);

1121

&acct[CTINFO2DIR(ctinfo)].bytes);

1122

}

1122

}

1123

}

1123

}

1124

1125

if (del_timer(&ct->timeout)) {

1125

if (del_timer(&ct->timeout)) {

1126

ct->timeout.function((unsigned long)ct);

1126

ct->timeout.function((unsigned long)ct);

1127

return true;

1127

return true;

1128

}

1128

}

1129

return false;

1129

return false;

1130

}

1130

}

1131

EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);

1131

EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);

1132

1133

#ifdef CONFIG_NF_CONNTRACK_ZONES

1133

#ifdef CONFIG_NF_CONNTRACK_ZONES

1134

static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {

1134

static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {

1135

.len = sizeof(struct nf_conntrack_zone),

1135

.len = sizeof(struct nf_conntrack_zone),

1136

.align = __alignof__(struct nf_conntrack_zone),

1136

.align = __alignof__(struct nf_conntrack_zone),

1137

.id = NF_CT_EXT_ZONE,

1137

.id = NF_CT_EXT_ZONE,

1138

};

1138

};

1139

#endif

1139

#endif

1140

1141

#if IS_ENABLED(CONFIG_NF_CT_NETLINK)

1141

#if IS_ENABLED(CONFIG_NF_CT_NETLINK)

1142

1143

#include <linux/netfilter/nfnetlink.h>

1143

#include <linux/netfilter/nfnetlink.h>

1144

#include <linux/netfilter/nfnetlink_conntrack.h>

1144

#include <linux/netfilter/nfnetlink_conntrack.h>

1145

#include <linux/mutex.h>

1145

#include <linux/mutex.h>

1146

1147

/* Generic function for tcp/udp/sctp/dccp and alike. This needs to be

1147

/* Generic function for tcp/udp/sctp/dccp and alike. This needs to be

1148

* in ip_conntrack_core, since we don't want the protocols to autoload

1148

* in ip_conntrack_core, since we don't want the protocols to autoload

1149

* or depend on ctnetlink */

1149

* or depend on ctnetlink */

1150

int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,

1150

int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,

1151

const struct nf_conntrack_tuple *tuple)

1151

const struct nf_conntrack_tuple *tuple)

1152

{

1152

{

1153

if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||

1153

if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||

1154

nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))

1154

nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))

1155

goto nla_put_failure;

1155

goto nla_put_failure;

1156

return 0;

1156

return 0;

1157

1158

nla_put_failure:

1158

nla_put_failure:

1159

return -1;

1159

return -1;

1160

}

1160

}

1161

EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);

1161

EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);

1162

1163

const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {

1163

const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {

1164

[CTA_PROTO_SRC_PORT] = { .type = NLA_U16 },

1164

[CTA_PROTO_SRC_PORT] = { .type = NLA_U16 },

1165

[CTA_PROTO_DST_PORT] = { .type = NLA_U16 },

1165

[CTA_PROTO_DST_PORT] = { .type = NLA_U16 },

1166

};

1166

};

1167

EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);

1167

EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);

1168

1169

int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],

1169

int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],

1170

struct nf_conntrack_tuple *t)

1170

struct nf_conntrack_tuple *t)

1171

{

1171

{

1172

if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])

1172

if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])

1173

return -EINVAL;

1173

return -EINVAL;

1174

1175

t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);

1175

t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);

1176

t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);

1176

t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);

1177

1178

return 0;

1178

return 0;

1179

}

1179

}

1180

EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);

1180

EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);

1181

1182

int nf_ct_port_nlattr_tuple_size(void)

1182

int nf_ct_port_nlattr_tuple_size(void)

1183

{

1183

{

1184

return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);

1184

return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);

1185

}

1185

}

1186

EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);

1186

EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);

1187

#endif

1187

#endif

1188

1189

/* Used by ipt_REJECT and ip6t_REJECT. */

1189

/* Used by ipt_REJECT and ip6t_REJECT. */

1190

static void nf_conntrack_attach(struct sk_buff *nskb, struct sk_buff *skb)

1190

static void nf_conntrack_attach(struct sk_buff *nskb, struct sk_buff *skb)

1191

{

1191

{

1192

struct nf_conn *ct;

1192

struct nf_conn *ct;

1193

enum ip_conntrack_info ctinfo;

1193

enum ip_conntrack_info ctinfo;

1194

1195

/* This ICMP is in reverse direction to the packet which caused it */

1195

/* This ICMP is in reverse direction to the packet which caused it */

1196

ct = nf_ct_get(skb, &ctinfo);

1196

ct = nf_ct_get(skb, &ctinfo);

1197

if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)

1197

if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)

1198

ctinfo = IP_CT_RELATED_REPLY;

1198

ctinfo = IP_CT_RELATED_REPLY;

1199

else

1199

else

1200

ctinfo = IP_CT_RELATED;

1200

ctinfo = IP_CT_RELATED;

1201

1202

/* Attach to new skbuff, and increment count */

1202

/* Attach to new skbuff, and increment count */

1203

nskb->nfct = &ct->ct_general;

1203

nskb->nfct = &ct->ct_general;

1204

nskb->nfctinfo = ctinfo;

1204

nskb->nfctinfo = ctinfo;

1205

nf_conntrack_get(nskb->nfct);

1205

nf_conntrack_get(nskb->nfct);

1206

}

1206

}

1207

1208

/* Bring out ya dead! */

1208

/* Bring out ya dead! */

1209

static struct nf_conn *

1209

static struct nf_conn *

1210

get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),

1210

get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),

1211

void *data, unsigned int *bucket)

1211

void *data, unsigned int *bucket)

1212

{

1212

{

1213

struct nf_conntrack_tuple_hash *h;

1213

struct nf_conntrack_tuple_hash *h;

1214

struct nf_conn *ct;

1214

struct nf_conn *ct;

1215

struct hlist_nulls_node *n;

1215

struct hlist_nulls_node *n;

1216

1217

spin_lock_bh(&nf_conntrack_lock);

1217

spin_lock_bh(&nf_conntrack_lock);

1218

for (; *bucket < net->ct.htable_size; (*bucket)++) {

1218

for (; *bucket < net->ct.htable_size; (*bucket)++) {

1219

hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {

1219

hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {

1220

if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)

1220

if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)

1221

continue;

1221

continue;

1222

ct = nf_ct_tuplehash_to_ctrack(h);

1222

ct = nf_ct_tuplehash_to_ctrack(h);

1223

if (iter(ct, data))

1223

if (iter(ct, data))

1224

goto found;

1224

goto found;

1225

}

1225

}

1226

}

1226

}

1227

hlist_nulls_for_each_entry(h, n, &net->ct.unconfirmed, hnnode) {

1227

hlist_nulls_for_each_entry(h, n, &net->ct.unconfirmed, hnnode) {

1228

ct = nf_ct_tuplehash_to_ctrack(h);

1228

ct = nf_ct_tuplehash_to_ctrack(h);

1229

if (iter(ct, data))

1229

if (iter(ct, data))

1230

set_bit(IPS_DYING_BIT, &ct->status);

1230

set_bit(IPS_DYING_BIT, &ct->status);

1231

}

1231

}

1232

spin_unlock_bh(&nf_conntrack_lock);

1232

spin_unlock_bh(&nf_conntrack_lock);

1233

return NULL;

1233

return NULL;

1234

found:

1234

found:

1235

atomic_inc(&ct->ct_general.use);

1235

atomic_inc(&ct->ct_general.use);

1236

spin_unlock_bh(&nf_conntrack_lock);

1236

spin_unlock_bh(&nf_conntrack_lock);

1237

return ct;

1237

return ct;

1238

}

1238

}

1239

1240

void nf_ct_iterate_cleanup(struct net *net,

1240

void nf_ct_iterate_cleanup(struct net *net,

1241

int (*iter)(struct nf_conn *i, void *data),

1241

int (*iter)(struct nf_conn *i, void *data),

1242

void *data)

1242

void *data)

1243

{

1243

{

1244

struct nf_conn *ct;

1244

struct nf_conn *ct;

1245

unsigned int bucket = 0;

1245

unsigned int bucket = 0;

1246

1247

while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {

1247

while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {

1248

/* Time to push up daises... */

1248

/* Time to push up daises... */

1249

if (del_timer(&ct->timeout))

1249

if (del_timer(&ct->timeout))

1250

death_by_timeout((unsigned long)ct);

1250

death_by_timeout((unsigned long)ct);

1251

/* ... else the timer will get him soon. */

1251

/* ... else the timer will get him soon. */

1252

1253

nf_ct_put(ct);

1253

nf_ct_put(ct);

1254

}

1254

}

1255

}

1255

}

1256

EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);

1256

EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);

1257

1258

struct __nf_ct_flush_report {

1258

struct __nf_ct_flush_report {

1259

u32 pid;

1259

u32 pid;

1260

int report;

1260

int report;

1261

};

1261

};

1262

1263

static int kill_report(struct nf_conn *i, void *data)

1263

static int kill_report(struct nf_conn *i, void *data)

1264

{

1264

{

1265

struct __nf_ct_flush_report *fr = (struct __nf_ct_flush_report *)data;

1265

struct __nf_ct_flush_report *fr = (struct __nf_ct_flush_report *)data;

1266

struct nf_conn_tstamp *tstamp;

1266

struct nf_conn_tstamp *tstamp;

1267

1268

tstamp = nf_conn_tstamp_find(i);

1268

tstamp = nf_conn_tstamp_find(i);

1269

if (tstamp && tstamp->stop == 0)

1269

if (tstamp && tstamp->stop == 0)

1270

tstamp->stop = ktime_to_ns(ktime_get_real());

1270

tstamp->stop = ktime_to_ns(ktime_get_real());

1271

1272

/* If we fail to deliver the event, death_by_timeout() will retry */

1272

/* If we fail to deliver the event, death_by_timeout() will retry */

1273

if (nf_conntrack_event_report(IPCT_DESTROY, i,

1273

if (nf_conntrack_event_report(IPCT_DESTROY, i,

1274

fr->pid, fr->report) < 0)

1274

fr->pid, fr->report) < 0)

1275

return 1;

1275

return 1;

1276

1277

/* Avoid the delivery of the destroy event in death_by_timeout(). */

1277

/* Avoid the delivery of the destroy event in death_by_timeout(). */

1278

set_bit(IPS_DYING_BIT, &i->status);

1278

set_bit(IPS_DYING_BIT, &i->status);

1279

return 1;

1279

return 1;

1280

}

1280

}

1281

1282

static int kill_all(struct nf_conn *i, void *data)

1282

static int kill_all(struct nf_conn *i, void *data)

1283

{

1283

{

1284

return 1;

1284

return 1;

1285

}

1285

}

1286

1287

void nf_ct_free_hashtable(void *hash, unsigned int size)

1287

void nf_ct_free_hashtable(void *hash, unsigned int size)

1288

{

1288

{

1289

if (is_vmalloc_addr(hash))

1289

if (is_vmalloc_addr(hash))

1290

vfree(hash);

1290

vfree(hash);

1291

else

1291

else

1292

free_pages((unsigned long)hash,

1292

free_pages((unsigned long)hash,

1293

get_order(sizeof(struct hlist_head) * size));

1293

get_order(sizeof(struct hlist_head) * size));

1294

}

1294

}

1295

EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);

1295

EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);

1296

1297

void nf_conntrack_flush_report(struct net *net, u32 pid, int report)

1297

void nf_conntrack_flush_report(struct net *net, u32 pid, int report)

1298

{

1298

{

1299

struct __nf_ct_flush_report fr = {

1299

struct __nf_ct_flush_report fr = {

1300

.pid = pid,

1300

.pid = pid,

1301

.report = report,

1301

.report = report,

1302

};

1302

};

1303

nf_ct_iterate_cleanup(net, kill_report, &fr);

1303

nf_ct_iterate_cleanup(net, kill_report, &fr);

1304

}

1304

}

1305

EXPORT_SYMBOL_GPL(nf_conntrack_flush_report);

1305

EXPORT_SYMBOL_GPL(nf_conntrack_flush_report);

1306

1307

static void nf_ct_release_dying_list(struct net *net)

1307

static void nf_ct_release_dying_list(struct net *net)

1308

{

1308

{

1309

struct nf_conntrack_tuple_hash *h;

1309

struct nf_conntrack_tuple_hash *h;

1310

struct nf_conn *ct;

1310

struct nf_conn *ct;

1311

struct hlist_nulls_node *n;

1311

struct hlist_nulls_node *n;

1312

1313

spin_lock_bh(&nf_conntrack_lock);

1313

spin_lock_bh(&nf_conntrack_lock);

1314

hlist_nulls_for_each_entry(h, n, &net->ct.dying, hnnode) {

1314

hlist_nulls_for_each_entry(h, n, &net->ct.dying, hnnode) {

1315

ct = nf_ct_tuplehash_to_ctrack(h);

1315

ct = nf_ct_tuplehash_to_ctrack(h);

1316

/* never fails to remove them, no listeners at this point */

1316

/* never fails to remove them, no listeners at this point */

1317

nf_ct_kill(ct);

1317

nf_ct_kill(ct);

1318

}

1318

}

1319

spin_unlock_bh(&nf_conntrack_lock);

1319

spin_unlock_bh(&nf_conntrack_lock);

1320

}

1320

}

1321

1322

static int untrack_refs(void)

1322

static int untrack_refs(void)

1323

{

1323

{

1324

int cnt = 0, cpu;

1324

int cnt = 0, cpu;

1325

1326

for_each_possible_cpu(cpu) {

1326

for_each_possible_cpu(cpu) {

1327

struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);

1327

struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);

1328

1329

cnt += atomic_read(&ct->ct_general.use) - 1;

1329

cnt += atomic_read(&ct->ct_general.use) - 1;

1330

}

1330

}

1331

return cnt;

1331

return cnt;

1332

}

1332

}

1333

1334

static void nf_conntrack_cleanup_init_net(void)

1334

static void nf_conntrack_cleanup_init_net(void)

1335

{

1335

{

1336

while (untrack_refs() > 0)

1336

while (untrack_refs() > 0)

1337

schedule();

1337

schedule();

1338

1339

#ifdef CONFIG_NF_CONNTRACK_ZONES

1339

#ifdef CONFIG_NF_CONNTRACK_ZONES

1340

nf_ct_extend_unregister(&nf_ct_zone_extend);

1340

nf_ct_extend_unregister(&nf_ct_zone_extend);

1341

#endif

1341

#endif

1342

}

1342

}

1343

1344

static void nf_conntrack_cleanup_net(struct net *net)

1344

static void nf_conntrack_cleanup_net(struct net *net)

1345

{

1345

{

1346

i_see_dead_people:

1346

i_see_dead_people:

1347

nf_ct_iterate_cleanup(net, kill_all, NULL);

1347

nf_ct_iterate_cleanup(net, kill_all, NULL);

1348

nf_ct_release_dying_list(net);

1348

nf_ct_release_dying_list(net);

1349

if (atomic_read(&net->ct.count) != 0) {

1349

if (atomic_read(&net->ct.count) != 0) {

1350

schedule();

1350

schedule();

1351

goto i_see_dead_people;

1351

goto i_see_dead_people;

1352

}

1352

}

1353

1354

nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);

1354

nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);

1355

nf_conntrack_helper_fini(net);

1355

nf_conntrack_helper_fini(net);

1356

nf_conntrack_timeout_fini(net);

1356

nf_conntrack_timeout_fini(net);

1357

nf_conntrack_ecache_fini(net);

1357

nf_conntrack_ecache_fini(net);

1358

nf_conntrack_tstamp_fini(net);

1358

nf_conntrack_tstamp_fini(net);

1359

nf_conntrack_acct_fini(net);

1359

nf_conntrack_acct_fini(net);

1360

nf_conntrack_expect_fini(net);

1360

nf_conntrack_expect_fini(net);

1361

kmem_cache_destroy(net->ct.nf_conntrack_cachep);

1361

kmem_cache_destroy(net->ct.nf_conntrack_cachep);

1362

kfree(net->ct.slabname);

1362

kfree(net->ct.slabname);

1363

free_percpu(net->ct.stat);

1363

free_percpu(net->ct.stat);

1364

}

1364

}

1365

1366

/* Mishearing the voices in his head, our hero wonders how he's

1366

/* Mishearing the voices in his head, our hero wonders how he's

1367

supposed to kill the mall. */

1367

supposed to kill the mall. */

1368

void nf_conntrack_cleanup(struct net *net)

1368

void nf_conntrack_cleanup(struct net *net)

1369

{

1369

{

1370

if (net_eq(net, &init_net))

1370

if (net_eq(net, &init_net))

1371

RCU_INIT_POINTER(ip_ct_attach, NULL);

1371

RCU_INIT_POINTER(ip_ct_attach, NULL);

1372

1373

/* This makes sure all current packets have passed through

1373

/* This makes sure all current packets have passed through

1374

netfilter framework. Roll on, two-stage module

1374

netfilter framework. Roll on, two-stage module

1375

delete... */

1375

delete... */

1376

synchronize_net();

1376

synchronize_net();

1377

nf_conntrack_proto_fini(net);

1377

nf_conntrack_proto_fini(net);

1378

nf_conntrack_cleanup_net(net);

1378

nf_conntrack_cleanup_net(net);

1379

}

1379

1380

if (net_eq(net, &init_net)) {

1381

void nf_conntrack_cleanup_end(void)

1381

RCU_INIT_POINTER(nf_ct_destroy, NULL);

1382

{

1382

nf_conntrack_cleanup_init_net();

1383

RCU_INIT_POINTER(nf_ct_destroy, NULL);

1383

}

1384

nf_conntrack_cleanup_init_net();

1384

}

1385

}

1385

1386

void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)

1387

void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)

1387

{

1388

{

1388

struct hlist_nulls_head *hash;

1389

struct hlist_nulls_head *hash;

1389

unsigned int nr_slots, i;

1390

unsigned int nr_slots, i;

1390

size_t sz;

1391

size_t sz;

1391

1392

BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));

1393

BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));

1393

nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));

1394

nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));

1394

sz = nr_slots * sizeof(struct hlist_nulls_head);

1395

sz = nr_slots * sizeof(struct hlist_nulls_head);

1395

hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,

1396

hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,

1396

get_order(sz));

1397

get_order(sz));

1397

if (!hash) {

1398

if (!hash) {

1398

printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n");

1399

printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n");

1399

hash = vzalloc(sz);

1400

hash = vzalloc(sz);

1400

}

1401

}

1401

1402

if (hash && nulls)

1403

if (hash && nulls)

1403

for (i = 0; i < nr_slots; i++)

1404

for (i = 0; i < nr_slots; i++)

1404

INIT_HLIST_NULLS_HEAD(&hash[i], i);

1405

INIT_HLIST_NULLS_HEAD(&hash[i], i);

1405

1406

return hash;

1407

return hash;

1407

}

1408

}

1408

EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);

1409

EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);

1409

1410

int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)

1411

int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)

1411

{

1412

{

1412

int i, bucket, rc;

1413

int i, bucket, rc;

1413

unsigned int hashsize, old_size;

1414

unsigned int hashsize, old_size;

1414

struct hlist_nulls_head *hash, *old_hash;

1415

struct hlist_nulls_head *hash, *old_hash;

1415

struct nf_conntrack_tuple_hash *h;

1416

struct nf_conntrack_tuple_hash *h;

1416

struct nf_conn *ct;

1417

struct nf_conn *ct;

1417

1418

if (current->nsproxy->net_ns != &init_net)

1419

if (current->nsproxy->net_ns != &init_net)

1419

return -EOPNOTSUPP;

1420

return -EOPNOTSUPP;

1420

1421

/* On boot, we can set this without any fancy locking. */

1422

/* On boot, we can set this without any fancy locking. */

1422

if (!nf_conntrack_htable_size)

1423

if (!nf_conntrack_htable_size)

1423

return param_set_uint(val, kp);

1424

return param_set_uint(val, kp);

1424

1425

rc = kstrtouint(val, 0, &hashsize);

1426

rc = kstrtouint(val, 0, &hashsize);

1426

if (rc)

1427

if (rc)

1427

return rc;

1428

return rc;

1428

if (!hashsize)

1429

if (!hashsize)

1429

return -EINVAL;

1430

return -EINVAL;

1430

1431

hash = nf_ct_alloc_hashtable(&hashsize, 1);

1432

hash = nf_ct_alloc_hashtable(&hashsize, 1);

1432

if (!hash)

1433

if (!hash)

1433

return -ENOMEM;

1434

return -ENOMEM;

1434

1435

/* Lookups in the old hash might happen in parallel, which means we

1436

/* Lookups in the old hash might happen in parallel, which means we

1436

* might get false negatives during connection lookup. New connections

1437

* might get false negatives during connection lookup. New connections

1437

* created because of a false negative won't make it into the hash

1438

* created because of a false negative won't make it into the hash

1438

* though since that required taking the lock.

1439

* though since that required taking the lock.

1439

*/

1440

*/

1440

spin_lock_bh(&nf_conntrack_lock);

1441

spin_lock_bh(&nf_conntrack_lock);

1441

for (i = 0; i < init_net.ct.htable_size; i++) {

1442

for (i = 0; i < init_net.ct.htable_size; i++) {

1442

while (!hlist_nulls_empty(&init_net.ct.hash[i])) {

1443

while (!hlist_nulls_empty(&init_net.ct.hash[i])) {

1443

h = hlist_nulls_entry(init_net.ct.hash[i].first,

1444

h = hlist_nulls_entry(init_net.ct.hash[i].first,

1444

struct nf_conntrack_tuple_hash, hnnode);

1445

struct nf_conntrack_tuple_hash, hnnode);

1445

ct = nf_ct_tuplehash_to_ctrack(h);

1446

ct = nf_ct_tuplehash_to_ctrack(h);

1446

hlist_nulls_del_rcu(&h->hnnode);

1447

hlist_nulls_del_rcu(&h->hnnode);

1447

bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),

1448

bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),

1448

hashsize);

1449

hashsize);

1449

hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);

1450

hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);

1450

}

1451

}

1451

}

1452

}

1452

old_size = init_net.ct.htable_size;

1453

old_size = init_net.ct.htable_size;

1453

old_hash = init_net.ct.hash;

1454

old_hash = init_net.ct.hash;

1454

1455

init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;

1456

init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;

1456

init_net.ct.hash = hash;

1457

init_net.ct.hash = hash;

1457

spin_unlock_bh(&nf_conntrack_lock);

1458

spin_unlock_bh(&nf_conntrack_lock);

1458

1459

nf_ct_free_hashtable(old_hash, old_size);

1460

nf_ct_free_hashtable(old_hash, old_size);

1460

return 0;

1461

return 0;

1461

}

1462

}

1462

EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);

1463

EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);

1463

1464

module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,

1465

module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,

1465

&nf_conntrack_htable_size, 0600);

1466

&nf_conntrack_htable_size, 0600);

1466

1467

void nf_ct_untracked_status_or(unsigned long bits)

1468

void nf_ct_untracked_status_or(unsigned long bits)

1468

{

1469

{

1469

int cpu;

1470

int cpu;

1470

1471

for_each_possible_cpu(cpu)

1472

for_each_possible_cpu(cpu)

1472

per_cpu(nf_conntrack_untracked, cpu).status |= bits;

1473

per_cpu(nf_conntrack_untracked, cpu).status |= bits;

1473

}

1474

}

1474

EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);

1475

EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);

1475

1476

static int nf_conntrack_init_init_net(void)

1477

static int nf_conntrack_init_init_net(void)

1477

{

1478

{

1478

int max_factor = 8;

1479

int max_factor = 8;

1479

int ret, cpu;

1480

int ret, cpu;

1480

1481

/* Idea from tcp.c: use 1/16384 of memory. On i386: 32MB

1482

/* Idea from tcp.c: use 1/16384 of memory. On i386: 32MB

1482

* machine has 512 buckets. >= 1GB machines have 16384 buckets. */

1483

* machine has 512 buckets. >= 1GB machines have 16384 buckets. */

1483

if (!nf_conntrack_htable_size) {

1484

if (!nf_conntrack_htable_size) {

1484

nf_conntrack_htable_size

1485

nf_conntrack_htable_size

1485

= (((totalram_pages << PAGE_SHIFT) / 16384)

1486

= (((totalram_pages << PAGE_SHIFT) / 16384)

1486

/ sizeof(struct hlist_head));

1487

/ sizeof(struct hlist_head));

1487

if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))

1488

if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))

1488

nf_conntrack_htable_size = 16384;

1489

nf_conntrack_htable_size = 16384;

1489

if (nf_conntrack_htable_size < 32)

1490

if (nf_conntrack_htable_size < 32)

1490

nf_conntrack_htable_size = 32;

1491

nf_conntrack_htable_size = 32;

1491

1492

/* Use a max. factor of four by default to get the same max as

1493

/* Use a max. factor of four by default to get the same max as

1493

* with the old struct list_heads. When a table size is given

1494

* with the old struct list_heads. When a table size is given

1494

* we use the old value of 8 to avoid reducing the max.

1495

* we use the old value of 8 to avoid reducing the max.

1495

* entries. */

1496

* entries. */

1496

max_factor = 4;

1497

max_factor = 4;

1497

}

1498

}

1498

nf_conntrack_max = max_factor * nf_conntrack_htable_size;

1499

nf_conntrack_max = max_factor * nf_conntrack_htable_size;

1499

1500

printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",

1501

printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",

1501

NF_CONNTRACK_VERSION, nf_conntrack_htable_size,

1502

NF_CONNTRACK_VERSION, nf_conntrack_htable_size,

1502

nf_conntrack_max);

1503

nf_conntrack_max);

1503

#ifdef CONFIG_NF_CONNTRACK_ZONES

1504

#ifdef CONFIG_NF_CONNTRACK_ZONES

1504

ret = nf_ct_extend_register(&nf_ct_zone_extend);

1505

ret = nf_ct_extend_register(&nf_ct_zone_extend);

1505

if (ret < 0)

1506

if (ret < 0)

1506

goto err_extend;

1507

goto err_extend;

1507

#endif

1508

#endif

1508

/* Set up fake conntrack: to never be deleted, not in any hashes */

1509

/* Set up fake conntrack: to never be deleted, not in any hashes */

1509

for_each_possible_cpu(cpu) {

1510

for_each_possible_cpu(cpu) {

1510

struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);

1511

struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);

1511

write_pnet(&ct->ct_net, &init_net);

1512

write_pnet(&ct->ct_net, &init_net);

1512

atomic_set(&ct->ct_general.use, 1);

1513

atomic_set(&ct->ct_general.use, 1);

1513

}

1514

}

1514

/* - and look it like as a confirmed connection */

1515

/* - and look it like as a confirmed connection */

1515

nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);

1516

nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);

1516

return 0;

1517

return 0;

1517

1518

#ifdef CONFIG_NF_CONNTRACK_ZONES

1519

#ifdef CONFIG_NF_CONNTRACK_ZONES

1519

err_extend:

1520

err_extend:

1520

#endif

1521

#endif

1521

return ret;

1522

return ret;

1522

}

1523

}

1523

1524

/*

1525

/*

1525

* We need to use special "null" values, not used in hash table

1526

* We need to use special "null" values, not used in hash table

1526

*/

1527

*/

1527

#define UNCONFIRMED_NULLS_VAL ((1<<30)+0)

1528

#define UNCONFIRMED_NULLS_VAL ((1<<30)+0)

1528

#define DYING_NULLS_VAL ((1<<30)+1)

1529

#define DYING_NULLS_VAL ((1<<30)+1)

1529

#define TEMPLATE_NULLS_VAL ((1<<30)+2)

1530

#define TEMPLATE_NULLS_VAL ((1<<30)+2)

1530

1531

static int nf_conntrack_init_net(struct net *net)

1532

static int nf_conntrack_init_net(struct net *net)

1532

{

1533

{

1533

int ret;

1534

int ret;

1534

1535

atomic_set(&net->ct.count, 0);

1536

atomic_set(&net->ct.count, 0);

1536

INIT_HLIST_NULLS_HEAD(&net->ct.unconfirmed, UNCONFIRMED_NULLS_VAL);

1537

INIT_HLIST_NULLS_HEAD(&net->ct.unconfirmed, UNCONFIRMED_NULLS_VAL);

1537

INIT_HLIST_NULLS_HEAD(&net->ct.dying, DYING_NULLS_VAL);

1538

INIT_HLIST_NULLS_HEAD(&net->ct.dying, DYING_NULLS_VAL);

1538

INIT_HLIST_NULLS_HEAD(&net->ct.tmpl, TEMPLATE_NULLS_VAL);

1539

INIT_HLIST_NULLS_HEAD(&net->ct.tmpl, TEMPLATE_NULLS_VAL);

1539

net->ct.stat = alloc_percpu(struct ip_conntrack_stat);

1540

net->ct.stat = alloc_percpu(struct ip_conntrack_stat);

1540

if (!net->ct.stat) {

1541

if (!net->ct.stat) {

1541

ret = -ENOMEM;

1542

ret = -ENOMEM;

1542

goto err_stat;

1543

goto err_stat;

1543

}

1544

}

1544

1545

net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);

1546

net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);

1546

if (!net->ct.slabname) {

1547

if (!net->ct.slabname) {

1547

ret = -ENOMEM;

1548

ret = -ENOMEM;

1548

goto err_slabname;

1549

goto err_slabname;

1549

}

1550

}

1550

1551

net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,

1552

net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,

1552

sizeof(struct nf_conn), 0,

1553

sizeof(struct nf_conn), 0,

1553

SLAB_DESTROY_BY_RCU, NULL);

1554

SLAB_DESTROY_BY_RCU, NULL);

1554

if (!net->ct.nf_conntrack_cachep) {

1555

if (!net->ct.nf_conntrack_cachep) {

1555

printk(KERN_ERR "Unable to create nf_conn slab cache\n");

1556

printk(KERN_ERR "Unable to create nf_conn slab cache\n");

1556

ret = -ENOMEM;

1557

ret = -ENOMEM;

1557

goto err_cache;

1558

goto err_cache;

1558

}

1559

}

1559

1560

net->ct.htable_size = nf_conntrack_htable_size;

1561

net->ct.htable_size = nf_conntrack_htable_size;

1561

net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);

1562

net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);

1562

if (!net->ct.hash) {

1563

if (!net->ct.hash) {

1563

ret = -ENOMEM;

1564

ret = -ENOMEM;

1564

printk(KERN_ERR "Unable to create nf_conntrack_hash\n");

1565

printk(KERN_ERR "Unable to create nf_conntrack_hash\n");

1565

goto err_hash;

1566

goto err_hash;

1566

}

1567

}

1567

ret = nf_conntrack_expect_init(net);

1568

ret = nf_conntrack_expect_init(net);

1568

if (ret < 0)

1569

if (ret < 0)

1569

goto err_expect;

1570

goto err_expect;

1570

ret = nf_conntrack_acct_init(net);

1571

ret = nf_conntrack_acct_init(net);

1571

if (ret < 0)

1572

if (ret < 0)

1572

goto err_acct;

1573

goto err_acct;

1573

ret = nf_conntrack_tstamp_init(net);

1574

ret = nf_conntrack_tstamp_init(net);

1574

if (ret < 0)

1575

if (ret < 0)

1575

goto err_tstamp;

1576

goto err_tstamp;

1576

ret = nf_conntrack_ecache_init(net);

1577

ret = nf_conntrack_ecache_init(net);

1577

if (ret < 0)

1578

if (ret < 0)

1578

goto err_ecache;

1579

goto err_ecache;

1579

ret = nf_conntrack_timeout_init(net);

1580

ret = nf_conntrack_timeout_init(net);

1580

if (ret < 0)

1581

if (ret < 0)

1581

goto err_timeout;

1582

goto err_timeout;

1582

ret = nf_conntrack_helper_init(net);

1583

ret = nf_conntrack_helper_init(net);

1583

if (ret < 0)

1584

if (ret < 0)

1584

goto err_helper;

1585

goto err_helper;

1585

return 0;

1586

return 0;

1586

err_helper:

1587

err_helper:

1587

nf_conntrack_timeout_fini(net);

1588

nf_conntrack_timeout_fini(net);

1588

err_timeout:

1589

err_timeout:

1589

nf_conntrack_ecache_fini(net);

1590

nf_conntrack_ecache_fini(net);

1590

err_ecache:

1591

err_ecache:

1591

nf_conntrack_tstamp_fini(net);

1592

nf_conntrack_tstamp_fini(net);

1592

err_tstamp:

1593

err_tstamp:

1593

nf_conntrack_acct_fini(net);

1594

nf_conntrack_acct_fini(net);

1594

err_acct:

1595

err_acct:

1595

nf_conntrack_expect_fini(net);

1596

nf_conntrack_expect_fini(net);

1596

err_expect:

1597

err_expect:

1597

nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);

1598

nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);

1598

err_hash:

1599

err_hash:

1599

kmem_cache_destroy(net->ct.nf_conntrack_cachep);

1600

kmem_cache_destroy(net->ct.nf_conntrack_cachep);

1600

err_cache:

1601

err_cache:

1601

kfree(net->ct.slabname);

1602

kfree(net->ct.slabname);

1602

err_slabname:

1603

err_slabname:

1603

free_percpu(net->ct.stat);

1604

free_percpu(net->ct.stat);

1604

err_stat:

1605

err_stat:

1605

return ret;

1606

return ret;

1606

}

1607

}

1607

1608

s16 (*nf_ct_nat_offset)(const struct nf_conn *ct,

1609

s16 (*nf_ct_nat_offset)(const struct nf_conn *ct,

1609

enum ip_conntrack_dir dir,

1610

enum ip_conntrack_dir dir,

1610

u32 seq);

1611

u32 seq);

1611

EXPORT_SYMBOL_GPL(nf_ct_nat_offset);

1612

EXPORT_SYMBOL_GPL(nf_ct_nat_offset);

1612

1613

int nf_conntrack_init(struct net *net)

1614

int nf_conntrack_init(struct net *net)

1614

{

1615

{

1615

int ret;

1616

int ret;

1616

1617

if (net_eq(net, &init_net)) {

1618

if (net_eq(net, &init_net)) {

1618

ret = nf_conntrack_init_init_net();

1619

ret = nf_conntrack_init_init_net();

1619

if (ret < 0)

1620

if (ret < 0)

1620

goto out_init_net;

1621

goto out_init_net;

1621

}

1622

}

1622

ret = nf_conntrack_proto_init(net);

1623

ret = nf_conntrack_proto_init(net);

1623

if (ret < 0)

1624

if (ret < 0)

1624

goto out_proto;

1625

goto out_proto;

1625

ret = nf_conntrack_init_net(net);

1626

ret = nf_conntrack_init_net(net);

1626

if (ret < 0)

1627

if (ret < 0)

1627

goto out_net;

1628

goto out_net;

1628

1629

if (net_eq(net, &init_net)) {

1630

if (net_eq(net, &init_net)) {

1630

/* For use by REJECT target */

1631

/* For use by REJECT target */

1631

RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);

1632

RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);

1632

RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);

1633

RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);

1633

1634

/* Howto get NAT offsets */

1635

/* Howto get NAT offsets */

1635

RCU_INIT_POINTER(nf_ct_nat_offset, NULL);

1636

RCU_INIT_POINTER(nf_ct_nat_offset, NULL);

1636

}

1637

}

1637

return 0;

1638

return 0;

1638

1639

out_net:

1640

out_net:

1640

nf_conntrack_proto_fini(net);

1641

nf_conntrack_proto_fini(net);

1641

out_proto:

1642

out_proto:

1642

if (net_eq(net, &init_net))

1643

if (net_eq(net, &init_net))

1643

nf_conntrack_cleanup_init_net();

1644

nf_conntrack_cleanup_init_net();

1644

out_init_net:

1645

out_init_net:

1645

return ret;

1646

return ret;

1646

}

1647

}

1647

1648

GITLAB

Eric Lee / smarc-fsl-linux-kernel

netfilter: nf_conntrack: fix BUG_ON while removing nf_conntrack with netns

 /*
  * This header is used to share core functionality between the
  * standalone connection tracking module, and the compatibility layer's use
  * of connection tracking.
  *
  * 16 Dec 2003: Yasuyuki Kozakai @USAGI <yasuyuki.kozakai@toshiba.co.jp>
  *	- generalize L3 protocol dependent part.
  *
  * Derived from include/linux/netfiter_ipv4/ip_conntrack_core.h
  */
 #ifndef _NF_CONNTRACK_CORE_H
 #define _NF_CONNTRACK_CORE_H
 #include <linux/netfilter.h>
 #include <net/netfilter/nf_conntrack_l3proto.h>
 #include <net/netfilter/nf_conntrack_l4proto.h>
 #include <net/netfilter/nf_conntrack_ecache.h>
 /* This header is used to share core functionality between the
    standalone connection tracking module, and the compatibility layer's use
    of connection tracking. */
 extern unsigned int nf_conntrack_in(struct net *net,
 				    u_int8_t pf,
 				    unsigned int hooknum,
 				    struct sk_buff *skb);
 extern int nf_conntrack_init(struct net *net);
 extern void nf_conntrack_cleanup(struct net *net);
 extern int nf_conntrack_proto_init(struct net *net);
 extern void nf_conntrack_proto_fini(struct net *net);
+extern void nf_conntrack_cleanup_end(void);
 extern bool
 nf_ct_get_tuple(const struct sk_buff *skb,
 		unsigned int nhoff,
 		unsigned int dataoff,
 		u_int16_t l3num,
 		u_int8_t protonum,
 		struct nf_conntrack_tuple *tuple,
 		const struct nf_conntrack_l3proto *l3proto,
 		const struct nf_conntrack_l4proto *l4proto);
 extern bool
 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
 		   const struct nf_conntrack_tuple *orig,
 		   const struct nf_conntrack_l3proto *l3proto,
 		   const struct nf_conntrack_l4proto *l4proto);
 /* Find a connection corresponding to a tuple. */
 extern struct nf_conntrack_tuple_hash *
 nf_conntrack_find_get(struct net *net, u16 zone,
 		      const struct nf_conntrack_tuple *tuple);
 extern int __nf_conntrack_confirm(struct sk_buff *skb);
 /* Confirm a connection: returns NF_DROP if packet must be dropped. */
 static inline int nf_conntrack_confirm(struct sk_buff *skb)
 {
 	struct nf_conn *ct = (struct nf_conn *)skb->nfct;
 	int ret = NF_ACCEPT;
 	if (ct && !nf_ct_is_untracked(ct)) {
 		if (!nf_ct_is_confirmed(ct))
 			ret = __nf_conntrack_confirm(skb);
 		if (likely(ret == NF_ACCEPT))
 			nf_ct_deliver_cached_events(ct);
 	}
 	return ret;
 }
 int
 print_tuple(struct seq_file *s, const struct nf_conntrack_tuple *tuple,
             const struct nf_conntrack_l3proto *l3proto,
             const struct nf_conntrack_l4proto *proto);
 extern spinlock_t nf_conntrack_lock ;
 #endif /* _NF_CONNTRACK_CORE_H */

 /* Connection state tracking for netfilter.  This is separated from,
    but required by, the NAT layer; it can also be used by an iptables
    extension. */
 /* (C) 1999-2001 Paul `Rusty' Russell
  * (C) 2002-2006 Netfilter Core Team <coreteam@netfilter.org>
  * (C) 2003,2004 USAGI/WIDE Project <http://www.linux-ipv6.org>
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
  * published by the Free Software Foundation.
  */
 #include <linux/types.h>
 #include <linux/netfilter.h>
 #include <linux/module.h>
 #include <linux/sched.h>
 #include <linux/skbuff.h>
 #include <linux/proc_fs.h>
 #include <linux/vmalloc.h>
 #include <linux/stddef.h>
 #include <linux/slab.h>
 #include <linux/random.h>
 #include <linux/jhash.h>
 #include <linux/err.h>
 #include <linux/percpu.h>
 #include <linux/moduleparam.h>
 #include <linux/notifier.h>
 #include <linux/kernel.h>
 #include <linux/netdevice.h>
 #include <linux/socket.h>
 #include <linux/mm.h>
 #include <linux/nsproxy.h>
 #include <linux/rculist_nulls.h>
 #include <net/netfilter/nf_conntrack.h>
 #include <net/netfilter/nf_conntrack_l3proto.h>
 #include <net/netfilter/nf_conntrack_l4proto.h>
 #include <net/netfilter/nf_conntrack_expect.h>
 #include <net/netfilter/nf_conntrack_helper.h>
 #include <net/netfilter/nf_conntrack_core.h>
 #include <net/netfilter/nf_conntrack_extend.h>
 #include <net/netfilter/nf_conntrack_acct.h>
 #include <net/netfilter/nf_conntrack_ecache.h>
 #include <net/netfilter/nf_conntrack_zones.h>
 #include <net/netfilter/nf_conntrack_timestamp.h>
 #include <net/netfilter/nf_conntrack_timeout.h>
 #include <net/netfilter/nf_nat.h>
 #include <net/netfilter/nf_nat_core.h>
 #define NF_CONNTRACK_VERSION	"0.5.0"
 int (*nfnetlink_parse_nat_setup_hook)(struct nf_conn *ct,
 				      enum nf_nat_manip_type manip,
 				      const struct nlattr *attr) __read_mostly;
 EXPORT_SYMBOL_GPL(nfnetlink_parse_nat_setup_hook);
 int (*nf_nat_seq_adjust_hook)(struct sk_buff *skb,
 			      struct nf_conn *ct,
 			      enum ip_conntrack_info ctinfo,
 			      unsigned int protoff);
 EXPORT_SYMBOL_GPL(nf_nat_seq_adjust_hook);
 DEFINE_SPINLOCK(nf_conntrack_lock);
 EXPORT_SYMBOL_GPL(nf_conntrack_lock);
 unsigned int nf_conntrack_htable_size __read_mostly;
 EXPORT_SYMBOL_GPL(nf_conntrack_htable_size);
 unsigned int nf_conntrack_max __read_mostly;
 EXPORT_SYMBOL_GPL(nf_conntrack_max);
 DEFINE_PER_CPU(struct nf_conn, nf_conntrack_untracked);
 EXPORT_PER_CPU_SYMBOL(nf_conntrack_untracked);
 unsigned int nf_conntrack_hash_rnd __read_mostly;
 EXPORT_SYMBOL_GPL(nf_conntrack_hash_rnd);
 static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, u16 zone)
 {
 	unsigned int n;
 	/* The direction must be ignored, so we hash everything up to the
 	 * destination ports (which is a multiple of 4) and treat the last
 	 * three bytes manually.
 	 */
 	n = (sizeof(tuple->src) + sizeof(tuple->dst.u3)) / sizeof(u32);
 	return jhash2((u32 *)tuple, n, zone ^ nf_conntrack_hash_rnd ^
 		      (((__force __u16)tuple->dst.u.all << 16) |
 		      tuple->dst.protonum));
 }
 static u32 __hash_bucket(u32 hash, unsigned int size)
 {
 	return ((u64)hash * size) >> 32;
 }
 static u32 hash_bucket(u32 hash, const struct net *net)
 {
 	return __hash_bucket(hash, net->ct.htable_size);
 }
 static u_int32_t __hash_conntrack(const struct nf_conntrack_tuple *tuple,
 				  u16 zone, unsigned int size)
 {
 	return __hash_bucket(hash_conntrack_raw(tuple, zone), size);
 }
 static inline u_int32_t hash_conntrack(const struct net *net, u16 zone,
 				       const struct nf_conntrack_tuple *tuple)
 {
 	return __hash_conntrack(tuple, zone, net->ct.htable_size);
 }
 bool
 nf_ct_get_tuple(const struct sk_buff *skb,
 		unsigned int nhoff,
 		unsigned int dataoff,
 		u_int16_t l3num,
 		u_int8_t protonum,
 		struct nf_conntrack_tuple *tuple,
 		const struct nf_conntrack_l3proto *l3proto,
 		const struct nf_conntrack_l4proto *l4proto)
 {
 	memset(tuple, 0, sizeof(*tuple));
 	tuple->src.l3num = l3num;
 	if (l3proto->pkt_to_tuple(skb, nhoff, tuple) == 0)
 		return false;
 	tuple->dst.protonum = protonum;
 	tuple->dst.dir = IP_CT_DIR_ORIGINAL;
 	return l4proto->pkt_to_tuple(skb, dataoff, tuple);
 }
 EXPORT_SYMBOL_GPL(nf_ct_get_tuple);
 bool nf_ct_get_tuplepr(const struct sk_buff *skb, unsigned int nhoff,
 		       u_int16_t l3num, struct nf_conntrack_tuple *tuple)
 {
 	struct nf_conntrack_l3proto *l3proto;
 	struct nf_conntrack_l4proto *l4proto;
 	unsigned int protoff;
 	u_int8_t protonum;
 	int ret;
 	rcu_read_lock();
 	l3proto = __nf_ct_l3proto_find(l3num);
 	ret = l3proto->get_l4proto(skb, nhoff, &protoff, &protonum);
 	if (ret != NF_ACCEPT) {
 		rcu_read_unlock();
 		return false;
 	}
 	l4proto = __nf_ct_l4proto_find(l3num, protonum);
 	ret = nf_ct_get_tuple(skb, nhoff, protoff, l3num, protonum, tuple,
 			      l3proto, l4proto);
 	rcu_read_unlock();
 	return ret;
 }
 EXPORT_SYMBOL_GPL(nf_ct_get_tuplepr);
 bool
 nf_ct_invert_tuple(struct nf_conntrack_tuple *inverse,
 		   const struct nf_conntrack_tuple *orig,
 		   const struct nf_conntrack_l3proto *l3proto,
 		   const struct nf_conntrack_l4proto *l4proto)
 {
 	memset(inverse, 0, sizeof(*inverse));
 	inverse->src.l3num = orig->src.l3num;
 	if (l3proto->invert_tuple(inverse, orig) == 0)
 		return false;
 	inverse->dst.dir = !orig->dst.dir;
 	inverse->dst.protonum = orig->dst.protonum;
 	return l4proto->invert_tuple(inverse, orig);
 }
 EXPORT_SYMBOL_GPL(nf_ct_invert_tuple);
 static void
 clean_from_lists(struct nf_conn *ct)
 {
 	pr_debug("clean_from_lists(%p)\n", ct);
 	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode);
 	/* Destroy all pending expectations */
 	nf_ct_remove_expectations(ct);
 }
 static void
 destroy_conntrack(struct nf_conntrack *nfct)
 {
 	struct nf_conn *ct = (struct nf_conn *)nfct;
 	struct net *net = nf_ct_net(ct);
 	struct nf_conntrack_l4proto *l4proto;
 	pr_debug("destroy_conntrack(%p)\n", ct);
 	NF_CT_ASSERT(atomic_read(&nfct->use) == 0);
 	NF_CT_ASSERT(!timer_pending(&ct->timeout));
 	/* To make sure we don't get any weird locking issues here:
 	 * destroy_conntrack() MUST NOT be called with a write lock
 	 * to nf_conntrack_lock!!! -HW */
 	rcu_read_lock();
 	l4proto = __nf_ct_l4proto_find(nf_ct_l3num(ct), nf_ct_protonum(ct));
 	if (l4proto && l4proto->destroy)
 		l4proto->destroy(ct);
 	rcu_read_unlock();
 	spin_lock_bh(&nf_conntrack_lock);
 	/* Expectations will have been removed in clean_from_lists,
 	 * except TFTP can create an expectation on the first packet,
 	 * before connection is in the list, so we need to clean here,
 	 * too. */
 	nf_ct_remove_expectations(ct);
 	/* We overload first tuple to link into unconfirmed or dying list.*/
 	BUG_ON(hlist_nulls_unhashed(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode));
 	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 	NF_CT_STAT_INC(net, delete);
 	spin_unlock_bh(&nf_conntrack_lock);
 	if (ct->master)
 		nf_ct_put(ct->master);
 	pr_debug("destroy_conntrack: returning ct=%p to slab\n", ct);
 	nf_conntrack_free(ct);
 }
 void nf_ct_delete_from_lists(struct nf_conn *ct)
 {
 	struct net *net = nf_ct_net(ct);
 	nf_ct_helper_destroy(ct);
 	spin_lock_bh(&nf_conntrack_lock);
 	/* Inside lock so preempt is disabled on module removal path.
 	 * Otherwise we can get spurious warnings. */
 	NF_CT_STAT_INC(net, delete_list);
 	clean_from_lists(ct);
 	/* add this conntrack to the dying list */
 	hlist_nulls_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 			     &net->ct.dying);
 	spin_unlock_bh(&nf_conntrack_lock);
 }
 EXPORT_SYMBOL_GPL(nf_ct_delete_from_lists);
 static void death_by_event(unsigned long ul_conntrack)
 {
 	struct nf_conn *ct = (void *)ul_conntrack;
 	struct net *net = nf_ct_net(ct);
 	struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);
 	BUG_ON(ecache == NULL);
 	if (nf_conntrack_event(IPCT_DESTROY, ct) < 0) {
 		/* bad luck, let's retry again */
 		ecache->timeout.expires = jiffies +
 			(random32() % net->ct.sysctl_events_retry_timeout);
 		add_timer(&ecache->timeout);
 		return;
 	}
 	/* we've got the event delivered, now it's dying */
 	set_bit(IPS_DYING_BIT, &ct->status);
 	nf_ct_put(ct);
 }
 void nf_ct_dying_timeout(struct nf_conn *ct)
 {
 	struct net *net = nf_ct_net(ct);
 	struct nf_conntrack_ecache *ecache = nf_ct_ecache_find(ct);
 	BUG_ON(ecache == NULL);
 	/* set a new timer to retry event delivery */
 	setup_timer(&ecache->timeout, death_by_event, (unsigned long)ct);
 	ecache->timeout.expires = jiffies +
 		(random32() % net->ct.sysctl_events_retry_timeout);
 	add_timer(&ecache->timeout);
 }
 EXPORT_SYMBOL_GPL(nf_ct_dying_timeout);
 static void death_by_timeout(unsigned long ul_conntrack)
 {
 	struct nf_conn *ct = (void *)ul_conntrack;
 	struct nf_conn_tstamp *tstamp;
 	tstamp = nf_conn_tstamp_find(ct);
 	if (tstamp && tstamp->stop == 0)
 		tstamp->stop = ktime_to_ns(ktime_get_real());
 	if (!test_bit(IPS_DYING_BIT, &ct->status) &&
 	    unlikely(nf_conntrack_event(IPCT_DESTROY, ct) < 0)) {
 		/* destroy event was not delivered */
 		nf_ct_delete_from_lists(ct);
 		nf_ct_dying_timeout(ct);
 		return;
 	}
 	set_bit(IPS_DYING_BIT, &ct->status);
 	nf_ct_delete_from_lists(ct);
 	nf_ct_put(ct);
 }
 /*
  * Warning :
  * - Caller must take a reference on returned object
  *   and recheck nf_ct_tuple_equal(tuple, &h->tuple)
  * OR
  * - Caller must lock nf_conntrack_lock before calling this function
  */
 static struct nf_conntrack_tuple_hash *
 ____nf_conntrack_find(struct net *net, u16 zone,
 		      const struct nf_conntrack_tuple *tuple, u32 hash)
 {
 	struct nf_conntrack_tuple_hash *h;
 	struct hlist_nulls_node *n;
 	unsigned int bucket = hash_bucket(hash, net);
 	/* Disable BHs the entire time since we normally need to disable them
 	 * at least once for the stats anyway.
 	 */
 	local_bh_disable();
 begin:
 	hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[bucket], hnnode) {
 		if (nf_ct_tuple_equal(tuple, &h->tuple) &&
 		    nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)) == zone) {
 			NF_CT_STAT_INC(net, found);
 			local_bh_enable();
 			return h;
 		}
 		NF_CT_STAT_INC(net, searched);
 	}
 	/*
 	 * if the nulls value we got at the end of this lookup is
 	 * not the expected one, we must restart lookup.
 	 * We probably met an item that was moved to another chain.
 	 */
 	if (get_nulls_value(n) != bucket) {
 		NF_CT_STAT_INC(net, search_restart);
 		goto begin;
 	}
 	local_bh_enable();
 	return NULL;
 }
 struct nf_conntrack_tuple_hash *
 __nf_conntrack_find(struct net *net, u16 zone,
 		    const struct nf_conntrack_tuple *tuple)
 {
 	return ____nf_conntrack_find(net, zone, tuple,
 				     hash_conntrack_raw(tuple, zone));
 }
 EXPORT_SYMBOL_GPL(__nf_conntrack_find);
 /* Find a connection corresponding to a tuple. */
 static struct nf_conntrack_tuple_hash *
 __nf_conntrack_find_get(struct net *net, u16 zone,
 			const struct nf_conntrack_tuple *tuple, u32 hash)
 {
 	struct nf_conntrack_tuple_hash *h;
 	struct nf_conn *ct;
 	rcu_read_lock();
 begin:
 	h = ____nf_conntrack_find(net, zone, tuple, hash);
 	if (h) {
 		ct = nf_ct_tuplehash_to_ctrack(h);
 		if (unlikely(nf_ct_is_dying(ct) ||
 			     !atomic_inc_not_zero(&ct->ct_general.use)))
 			h = NULL;
 		else {
 			if (unlikely(!nf_ct_tuple_equal(tuple, &h->tuple) ||
 				     nf_ct_zone(ct) != zone)) {
 				nf_ct_put(ct);
 				goto begin;
 			}
 		}
 	}
 	rcu_read_unlock();
 	return h;
 }
 struct nf_conntrack_tuple_hash *
 nf_conntrack_find_get(struct net *net, u16 zone,
 		      const struct nf_conntrack_tuple *tuple)
 {
 	return __nf_conntrack_find_get(net, zone, tuple,
 				       hash_conntrack_raw(tuple, zone));
 }
 EXPORT_SYMBOL_GPL(nf_conntrack_find_get);
 static void __nf_conntrack_hash_insert(struct nf_conn *ct,
 				       unsigned int hash,
 				       unsigned int repl_hash)
 {
 	struct net *net = nf_ct_net(ct);
 	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 			   &net->ct.hash[hash]);
 	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode,
 			   &net->ct.hash[repl_hash]);
 }
 int
 nf_conntrack_hash_check_insert(struct nf_conn *ct)
 {
 	struct net *net = nf_ct_net(ct);
 	unsigned int hash, repl_hash;
 	struct nf_conntrack_tuple_hash *h;
 	struct hlist_nulls_node *n;
 	u16 zone;
 	zone = nf_ct_zone(ct);
 	hash = hash_conntrack(net, zone,
 			      &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple);
 	repl_hash = hash_conntrack(net, zone,
 				   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 	spin_lock_bh(&nf_conntrack_lock);
 	/* See if there's one in the list already, including reverse */
 	hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
 		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 				      &h->tuple) &&
 		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 			goto out;
 	hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
 		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 				      &h->tuple) &&
 		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 			goto out;
 	add_timer(&ct->timeout);
 	nf_conntrack_get(&ct->ct_general);
 	__nf_conntrack_hash_insert(ct, hash, repl_hash);
 	NF_CT_STAT_INC(net, insert);
 	spin_unlock_bh(&nf_conntrack_lock);
 	return 0;
 out:
 	NF_CT_STAT_INC(net, insert_failed);
 	spin_unlock_bh(&nf_conntrack_lock);
 	return -EEXIST;
 }
 EXPORT_SYMBOL_GPL(nf_conntrack_hash_check_insert);
 /* Confirm a connection given skb; places it in hash table */
 int
 __nf_conntrack_confirm(struct sk_buff *skb)
 {
 	unsigned int hash, repl_hash;
 	struct nf_conntrack_tuple_hash *h;
 	struct nf_conn *ct;
 	struct nf_conn_help *help;
 	struct nf_conn_tstamp *tstamp;
 	struct hlist_nulls_node *n;
 	enum ip_conntrack_info ctinfo;
 	struct net *net;
 	u16 zone;
 	ct = nf_ct_get(skb, &ctinfo);
 	net = nf_ct_net(ct);
 	/* ipt_REJECT uses nf_conntrack_attach to attach related
 	   ICMP/TCP RST packets in other direction.  Actual packet
 	   which created connection will be IP_CT_NEW or for an
 	   expected connection, IP_CT_RELATED. */
 	if (CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL)
 		return NF_ACCEPT;
 	zone = nf_ct_zone(ct);
 	/* reuse the hash saved before */
 	hash = *(unsigned long *)&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev;
 	hash = hash_bucket(hash, net);
 	repl_hash = hash_conntrack(net, zone,
 				   &ct->tuplehash[IP_CT_DIR_REPLY].tuple);
 	/* We're not in hash table, and we refuse to set up related
 	   connections for unconfirmed conns.  But packet copies and
 	   REJECT will give spurious warnings here. */
 	/* NF_CT_ASSERT(atomic_read(&ct->ct_general.use) == 1); */
 	/* No external references means no one else could have
 	   confirmed us. */
 	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
 	pr_debug("Confirming conntrack %p\n", ct);
 	spin_lock_bh(&nf_conntrack_lock);
 	/* We have to check the DYING flag inside the lock to prevent
 	   a race against nf_ct_get_next_corpse() possibly called from
 	   user context, else we insert an already 'dead' hash, blocking
 	   further use of that particular connection -JM */
 	if (unlikely(nf_ct_is_dying(ct))) {
 		spin_unlock_bh(&nf_conntrack_lock);
 		return NF_ACCEPT;
 	}
 	/* See if there's one in the list already, including reverse:
 	   NAT could have grabbed it without realizing, since we're
 	   not in the hash.  If there is, we lost race. */
 	hlist_nulls_for_each_entry(h, n, &net->ct.hash[hash], hnnode)
 		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple,
 				      &h->tuple) &&
 		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 			goto out;
 	hlist_nulls_for_each_entry(h, n, &net->ct.hash[repl_hash], hnnode)
 		if (nf_ct_tuple_equal(&ct->tuplehash[IP_CT_DIR_REPLY].tuple,
 				      &h->tuple) &&
 		    zone == nf_ct_zone(nf_ct_tuplehash_to_ctrack(h)))
 			goto out;
 	/* Remove from unconfirmed list */
 	hlist_nulls_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode);
 	/* Timer relative to confirmation time, not original
 	   setting time, otherwise we'd get timer wrap in
 	   weird delay cases. */
 	ct->timeout.expires += jiffies;
 	add_timer(&ct->timeout);
 	atomic_inc(&ct->ct_general.use);
 	ct->status |= IPS_CONFIRMED;
 	/* set conntrack timestamp, if enabled. */
 	tstamp = nf_conn_tstamp_find(ct);
 	if (tstamp) {
 		if (skb->tstamp.tv64 == 0)
 			__net_timestamp(skb);
 		tstamp->start = ktime_to_ns(skb->tstamp);
 	}
 	/* Since the lookup is lockless, hash insertion must be done after
 	 * starting the timer and setting the CONFIRMED bit. The RCU barriers
 	 * guarantee that no other CPU can find the conntrack before the above
 	 * stores are visible.
 	 */
 	__nf_conntrack_hash_insert(ct, hash, repl_hash);
 	NF_CT_STAT_INC(net, insert);
 	spin_unlock_bh(&nf_conntrack_lock);
 	help = nfct_help(ct);
 	if (help && help->helper)
 		nf_conntrack_event_cache(IPCT_HELPER, ct);
 	nf_conntrack_event_cache(master_ct(ct) ?
 				 IPCT_RELATED : IPCT_NEW, ct);
 	return NF_ACCEPT;
 out:
 	NF_CT_STAT_INC(net, insert_failed);
 	spin_unlock_bh(&nf_conntrack_lock);
 	return NF_DROP;
 }
 EXPORT_SYMBOL_GPL(__nf_conntrack_confirm);
 /* Returns true if a connection correspondings to the tuple (required
    for NAT). */
 int
 nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
 			 const struct nf_conn *ignored_conntrack)
 {
 	struct net *net = nf_ct_net(ignored_conntrack);
 	struct nf_conntrack_tuple_hash *h;
 	struct hlist_nulls_node *n;
 	struct nf_conn *ct;
 	u16 zone = nf_ct_zone(ignored_conntrack);
 	unsigned int hash = hash_conntrack(net, zone, tuple);
 	/* Disable BHs the entire time since we need to disable them at
 	 * least once for the stats anyway.
 	 */
 	rcu_read_lock_bh();
 	hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash], hnnode) {
 		ct = nf_ct_tuplehash_to_ctrack(h);
 		if (ct != ignored_conntrack &&
 		    nf_ct_tuple_equal(tuple, &h->tuple) &&
 		    nf_ct_zone(ct) == zone) {
 			NF_CT_STAT_INC(net, found);
 			rcu_read_unlock_bh();
 			return 1;
 		}
 		NF_CT_STAT_INC(net, searched);
 	}
 	rcu_read_unlock_bh();
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nf_conntrack_tuple_taken);
 #define NF_CT_EVICTION_RANGE	8
 /* There's a small race here where we may free a just-assured
    connection.  Too bad: we're in trouble anyway. */
 static noinline int early_drop(struct net *net, unsigned int hash)
 {
 	/* Use oldest entry, which is roughly LRU */
 	struct nf_conntrack_tuple_hash *h;
 	struct nf_conn *ct = NULL, *tmp;
 	struct hlist_nulls_node *n;
 	unsigned int i, cnt = 0;
 	int dropped = 0;
 	rcu_read_lock();
 	for (i = 0; i < net->ct.htable_size; i++) {
 		hlist_nulls_for_each_entry_rcu(h, n, &net->ct.hash[hash],
 					 hnnode) {
 			tmp = nf_ct_tuplehash_to_ctrack(h);
 			if (!test_bit(IPS_ASSURED_BIT, &tmp->status))
 				ct = tmp;
 			cnt++;
 		}
 		if (ct != NULL) {
 			if (likely(!nf_ct_is_dying(ct) &&
 				   atomic_inc_not_zero(&ct->ct_general.use)))
 				break;
 			else
 				ct = NULL;
 		}
 		if (cnt >= NF_CT_EVICTION_RANGE)
 			break;
 		hash = (hash + 1) % net->ct.htable_size;
 	}
 	rcu_read_unlock();
 	if (!ct)
 		return dropped;
 	if (del_timer(&ct->timeout)) {
 		death_by_timeout((unsigned long)ct);
 		/* Check if we indeed killed this entry. Reliable event
 		   delivery may have inserted it into the dying list. */
 		if (test_bit(IPS_DYING_BIT, &ct->status)) {
 			dropped = 1;
 			NF_CT_STAT_INC_ATOMIC(net, early_drop);
 		}
 	}
 	nf_ct_put(ct);
 	return dropped;
 }
 void init_nf_conntrack_hash_rnd(void)
 {
 	unsigned int rand;
 	/*
 	 * Why not initialize nf_conntrack_rnd in a "init()" function ?
 	 * Because there isn't enough entropy when system initializing,
 	 * and we initialize it as late as possible.
 	 */
 	do {
 		get_random_bytes(&rand, sizeof(rand));
 	} while (!rand);
 	cmpxchg(&nf_conntrack_hash_rnd, 0, rand);
 }
 static struct nf_conn *
 __nf_conntrack_alloc(struct net *net, u16 zone,
 		     const struct nf_conntrack_tuple *orig,
 		     const struct nf_conntrack_tuple *repl,
 		     gfp_t gfp, u32 hash)
 {
 	struct nf_conn *ct;
 	if (unlikely(!nf_conntrack_hash_rnd)) {
 		init_nf_conntrack_hash_rnd();
 		/* recompute the hash as nf_conntrack_hash_rnd is initialized */
 		hash = hash_conntrack_raw(orig, zone);
 	}
 	/* We don't want any race condition at early drop stage */
 	atomic_inc(&net->ct.count);
 	if (nf_conntrack_max &&
 	    unlikely(atomic_read(&net->ct.count) > nf_conntrack_max)) {
 		if (!early_drop(net, hash_bucket(hash, net))) {
 			atomic_dec(&net->ct.count);
 			net_warn_ratelimited("nf_conntrack: table full, dropping packet\n");
 			return ERR_PTR(-ENOMEM);
 		}
 	}
 	/*
 	 * Do not use kmem_cache_zalloc(), as this cache uses
 	 * SLAB_DESTROY_BY_RCU.
 	 */
 	ct = kmem_cache_alloc(net->ct.nf_conntrack_cachep, gfp);
 	if (ct == NULL) {
 		atomic_dec(&net->ct.count);
 		return ERR_PTR(-ENOMEM);
 	}
 	/*
 	 * Let ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.next
 	 * and ct->tuplehash[IP_CT_DIR_REPLY].hnnode.next unchanged.
 	 */
 	memset(&ct->tuplehash[IP_CT_DIR_MAX], 0,
 	       offsetof(struct nf_conn, proto) -
 	       offsetof(struct nf_conn, tuplehash[IP_CT_DIR_MAX]));
 	spin_lock_init(&ct->lock);
 	ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple = *orig;
 	ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode.pprev = NULL;
 	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *repl;
 	/* save hash for reusing when confirming */
 	*(unsigned long *)(&ct->tuplehash[IP_CT_DIR_REPLY].hnnode.pprev) = hash;
 	/* Don't set timer yet: wait for confirmation */
 	setup_timer(&ct->timeout, death_by_timeout, (unsigned long)ct);
 	write_pnet(&ct->ct_net, net);
 #ifdef CONFIG_NF_CONNTRACK_ZONES
 	if (zone) {
 		struct nf_conntrack_zone *nf_ct_zone;
 		nf_ct_zone = nf_ct_ext_add(ct, NF_CT_EXT_ZONE, GFP_ATOMIC);
 		if (!nf_ct_zone)
 			goto out_free;
 		nf_ct_zone->id = zone;
 	}
 #endif
 	/*
 	 * changes to lookup keys must be done before setting refcnt to 1
 	 */
 	smp_wmb();
 	atomic_set(&ct->ct_general.use, 1);
 	return ct;
 #ifdef CONFIG_NF_CONNTRACK_ZONES
 out_free:
 	atomic_dec(&net->ct.count);
 	kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
 	return ERR_PTR(-ENOMEM);
 #endif
 }
 struct nf_conn *nf_conntrack_alloc(struct net *net, u16 zone,
 				   const struct nf_conntrack_tuple *orig,
 				   const struct nf_conntrack_tuple *repl,
 				   gfp_t gfp)
 {
 	return __nf_conntrack_alloc(net, zone, orig, repl, gfp, 0);
 }
 EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
 void nf_conntrack_free(struct nf_conn *ct)
 {
 	struct net *net = nf_ct_net(ct);
 	nf_ct_ext_destroy(ct);
 	atomic_dec(&net->ct.count);
 	nf_ct_ext_free(ct);
 	kmem_cache_free(net->ct.nf_conntrack_cachep, ct);
 }
 EXPORT_SYMBOL_GPL(nf_conntrack_free);
 /* Allocate a new conntrack: we return -ENOMEM if classification
    failed due to stress.  Otherwise it really is unclassifiable. */
 static struct nf_conntrack_tuple_hash *
 init_conntrack(struct net *net, struct nf_conn *tmpl,
 	       const struct nf_conntrack_tuple *tuple,
 	       struct nf_conntrack_l3proto *l3proto,
 	       struct nf_conntrack_l4proto *l4proto,
 	       struct sk_buff *skb,
 	       unsigned int dataoff, u32 hash)
 {
 	struct nf_conn *ct;
 	struct nf_conn_help *help;
 	struct nf_conntrack_tuple repl_tuple;
 	struct nf_conntrack_ecache *ecache;
 	struct nf_conntrack_expect *exp;
 	u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
 	struct nf_conn_timeout *timeout_ext;
 	unsigned int *timeouts;
 	if (!nf_ct_invert_tuple(&repl_tuple, tuple, l3proto, l4proto)) {
 		pr_debug("Can't invert tuple.\n");
 		return NULL;
 	}
 	ct = __nf_conntrack_alloc(net, zone, tuple, &repl_tuple, GFP_ATOMIC,
 				  hash);
 	if (IS_ERR(ct))
 		return (struct nf_conntrack_tuple_hash *)ct;
 	timeout_ext = tmpl ? nf_ct_timeout_find(tmpl) : NULL;
 	if (timeout_ext)
 		timeouts = NF_CT_TIMEOUT_EXT_DATA(timeout_ext);
 	else
 		timeouts = l4proto->get_timeouts(net);
 	if (!l4proto->new(ct, skb, dataoff, timeouts)) {
 		nf_conntrack_free(ct);
 		pr_debug("init conntrack: can't track with proto module\n");
 		return NULL;
 	}
 	if (timeout_ext)
 		nf_ct_timeout_ext_add(ct, timeout_ext->timeout, GFP_ATOMIC);
 	nf_ct_acct_ext_add(ct, GFP_ATOMIC);
 	nf_ct_tstamp_ext_add(ct, GFP_ATOMIC);
 	ecache = tmpl ? nf_ct_ecache_find(tmpl) : NULL;
 	nf_ct_ecache_ext_add(ct, ecache ? ecache->ctmask : 0,
 				 ecache ? ecache->expmask : 0,
 			     GFP_ATOMIC);
 	spin_lock_bh(&nf_conntrack_lock);
 	exp = nf_ct_find_expectation(net, zone, tuple);
 	if (exp) {
 		pr_debug("conntrack: expectation arrives ct=%p exp=%p\n",
 			 ct, exp);
 		/* Welcome, Mr. Bond.  We've been expecting you... */
 		__set_bit(IPS_EXPECTED_BIT, &ct->status);
 		ct->master = exp->master;
 		if (exp->helper) {
 			help = nf_ct_helper_ext_add(ct, exp->helper,
 						    GFP_ATOMIC);
 			if (help)
 				rcu_assign_pointer(help->helper, exp->helper);
 		}
 #ifdef CONFIG_NF_CONNTRACK_MARK
 		ct->mark = exp->master->mark;
 #endif
 #ifdef CONFIG_NF_CONNTRACK_SECMARK
 		ct->secmark = exp->master->secmark;
 #endif
 		nf_conntrack_get(&ct->master->ct_general);
 		NF_CT_STAT_INC(net, expect_new);
 	} else {
 		__nf_ct_try_assign_helper(ct, tmpl, GFP_ATOMIC);
 		NF_CT_STAT_INC(net, new);
 	}
 	/* Overload tuple linked list to put us in unconfirmed list. */
 	hlist_nulls_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnnode,
 		       &net->ct.unconfirmed);
 	spin_unlock_bh(&nf_conntrack_lock);
 	if (exp) {
 		if (exp->expectfn)
 			exp->expectfn(ct, exp);
 		nf_ct_expect_put(exp);
 	}
 	return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
 }
 /* On success, returns conntrack ptr, sets skb->nfct and ctinfo */
 static inline struct nf_conn *
 resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
 		  struct sk_buff *skb,
 		  unsigned int dataoff,
 		  u_int16_t l3num,
 		  u_int8_t protonum,
 		  struct nf_conntrack_l3proto *l3proto,
 		  struct nf_conntrack_l4proto *l4proto,
 		  int *set_reply,
 		  enum ip_conntrack_info *ctinfo)
 {
 	struct nf_conntrack_tuple tuple;
 	struct nf_conntrack_tuple_hash *h;
 	struct nf_conn *ct;
 	u16 zone = tmpl ? nf_ct_zone(tmpl) : NF_CT_DEFAULT_ZONE;
 	u32 hash;
 	if (!nf_ct_get_tuple(skb, skb_network_offset(skb),
 			     dataoff, l3num, protonum, &tuple, l3proto,
 			     l4proto)) {
 		pr_debug("resolve_normal_ct: Can't get tuple\n");
 		return NULL;
 	}
 	/* look for tuple match */
 	hash = hash_conntrack_raw(&tuple, zone);
 	h = __nf_conntrack_find_get(net, zone, &tuple, hash);
 	if (!h) {
 		h = init_conntrack(net, tmpl, &tuple, l3proto, l4proto,
 				   skb, dataoff, hash);
 		if (!h)
 			return NULL;
 		if (IS_ERR(h))
 			return (void *)h;
 	}
 	ct = nf_ct_tuplehash_to_ctrack(h);
 	/* It exists; we have (non-exclusive) reference. */
 	if (NF_CT_DIRECTION(h) == IP_CT_DIR_REPLY) {
 		*ctinfo = IP_CT_ESTABLISHED_REPLY;
 		/* Please set reply bit if this packet OK */
 		*set_reply = 1;
 	} else {
 		/* Once we've had two way comms, always ESTABLISHED. */
 		if (test_bit(IPS_SEEN_REPLY_BIT, &ct->status)) {
 			pr_debug("nf_conntrack_in: normal packet for %p\n", ct);
 			*ctinfo = IP_CT_ESTABLISHED;
 		} else if (test_bit(IPS_EXPECTED_BIT, &ct->status)) {
 			pr_debug("nf_conntrack_in: related packet for %p\n",
 				 ct);
 			*ctinfo = IP_CT_RELATED;
 		} else {
 			pr_debug("nf_conntrack_in: new packet for %p\n", ct);
 			*ctinfo = IP_CT_NEW;
 		}
 		*set_reply = 0;
 	}
 	skb->nfct = &ct->ct_general;
 	skb->nfctinfo = *ctinfo;
 	return ct;
 }
 unsigned int
 nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
 		struct sk_buff *skb)
 {
 	struct nf_conn *ct, *tmpl = NULL;
 	enum ip_conntrack_info ctinfo;
 	struct nf_conntrack_l3proto *l3proto;
 	struct nf_conntrack_l4proto *l4proto;
 	unsigned int *timeouts;
 	unsigned int dataoff;
 	u_int8_t protonum;
 	int set_reply = 0;
 	int ret;
 	if (skb->nfct) {
 		/* Previously seen (loopback or untracked)?  Ignore. */
 		tmpl = (struct nf_conn *)skb->nfct;
 		if (!nf_ct_is_template(tmpl)) {
 			NF_CT_STAT_INC_ATOMIC(net, ignore);
 			return NF_ACCEPT;
 		}
 		skb->nfct = NULL;
 	}
 	/* rcu_read_lock()ed by nf_hook_slow */
 	l3proto = __nf_ct_l3proto_find(pf);
 	ret = l3proto->get_l4proto(skb, skb_network_offset(skb),
 				   &dataoff, &protonum);
 	if (ret <= 0) {
 		pr_debug("not prepared to track yet or error occurred\n");
 		NF_CT_STAT_INC_ATOMIC(net, error);
 		NF_CT_STAT_INC_ATOMIC(net, invalid);
 		ret = -ret;
 		goto out;
 	}
 	l4proto = __nf_ct_l4proto_find(pf, protonum);
 	/* It may be an special packet, error, unclean...
 	 * inverse of the return code tells to the netfilter
 	 * core what to do with the packet. */
 	if (l4proto->error != NULL) {
 		ret = l4proto->error(net, tmpl, skb, dataoff, &ctinfo,
 				     pf, hooknum);
 		if (ret <= 0) {
 			NF_CT_STAT_INC_ATOMIC(net, error);
 			NF_CT_STAT_INC_ATOMIC(net, invalid);
 			ret = -ret;
 			goto out;
 		}
 		/* ICMP[v6] protocol trackers may assign one conntrack. */
 		if (skb->nfct)
 			goto out;
 	}
 	ct = resolve_normal_ct(net, tmpl, skb, dataoff, pf, protonum,
 			       l3proto, l4proto, &set_reply, &ctinfo);
 	if (!ct) {
 		/* Not valid part of a connection */
 		NF_CT_STAT_INC_ATOMIC(net, invalid);
 		ret = NF_ACCEPT;
 		goto out;
 	}
 	if (IS_ERR(ct)) {
 		/* Too stressed to deal. */
 		NF_CT_STAT_INC_ATOMIC(net, drop);
 		ret = NF_DROP;
 		goto out;
 	}
 	NF_CT_ASSERT(skb->nfct);
 	/* Decide what timeout policy we want to apply to this flow. */
 	timeouts = nf_ct_timeout_lookup(net, ct, l4proto);
 	ret = l4proto->packet(ct, skb, dataoff, ctinfo, pf, hooknum, timeouts);
 	if (ret <= 0) {
 		/* Invalid: inverse of the return code tells
 		 * the netfilter core what to do */
 		pr_debug("nf_conntrack_in: Can't track with proto module\n");
 		nf_conntrack_put(skb->nfct);
 		skb->nfct = NULL;
 		NF_CT_STAT_INC_ATOMIC(net, invalid);
 		if (ret == -NF_DROP)
 			NF_CT_STAT_INC_ATOMIC(net, drop);
 		ret = -ret;
 		goto out;
 	}
 	if (set_reply && !test_and_set_bit(IPS_SEEN_REPLY_BIT, &ct->status))
 		nf_conntrack_event_cache(IPCT_REPLY, ct);
 out:
 	if (tmpl) {
 		/* Special case: we have to repeat this hook, assign the
 		 * template again to this packet. We assume that this packet
 		 * has no conntrack assigned. This is used by nf_ct_tcp. */
 		if (ret == NF_REPEAT)
 			skb->nfct = (struct nf_conntrack *)tmpl;
 		else
 			nf_ct_put(tmpl);
 	}
 	return ret;
 }
 EXPORT_SYMBOL_GPL(nf_conntrack_in);
 bool nf_ct_invert_tuplepr(struct nf_conntrack_tuple *inverse,
 			  const struct nf_conntrack_tuple *orig)
 {
 	bool ret;
 	rcu_read_lock();
 	ret = nf_ct_invert_tuple(inverse, orig,
 				 __nf_ct_l3proto_find(orig->src.l3num),
 				 __nf_ct_l4proto_find(orig->src.l3num,
 						      orig->dst.protonum));
 	rcu_read_unlock();
 	return ret;
 }
 EXPORT_SYMBOL_GPL(nf_ct_invert_tuplepr);
 /* Alter reply tuple (maybe alter helper).  This is for NAT, and is
    implicitly racy: see __nf_conntrack_confirm */
 void nf_conntrack_alter_reply(struct nf_conn *ct,
 			      const struct nf_conntrack_tuple *newreply)
 {
 	struct nf_conn_help *help = nfct_help(ct);
 	/* Should be unconfirmed, so not in hash table yet */
 	NF_CT_ASSERT(!nf_ct_is_confirmed(ct));
 	pr_debug("Altering reply tuple of %p to ", ct);
 	nf_ct_dump_tuple(newreply);
 	ct->tuplehash[IP_CT_DIR_REPLY].tuple = *newreply;
 	if (ct->master || (help && !hlist_empty(&help->expectations)))
 		return;
 	rcu_read_lock();
 	__nf_ct_try_assign_helper(ct, NULL, GFP_ATOMIC);
 	rcu_read_unlock();
 }
 EXPORT_SYMBOL_GPL(nf_conntrack_alter_reply);
 /* Refresh conntrack for this many jiffies and do accounting if do_acct is 1 */
 void __nf_ct_refresh_acct(struct nf_conn *ct,
 			  enum ip_conntrack_info ctinfo,
 			  const struct sk_buff *skb,
 			  unsigned long extra_jiffies,
 			  int do_acct)
 {
 	NF_CT_ASSERT(ct->timeout.data == (unsigned long)ct);
 	NF_CT_ASSERT(skb);
 	/* Only update if this is not a fixed timeout */
 	if (test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
 		goto acct;
 	/* If not in hash table, timer will not be active yet */
 	if (!nf_ct_is_confirmed(ct)) {
 		ct->timeout.expires = extra_jiffies;
 	} else {
 		unsigned long newtime = jiffies + extra_jiffies;
 		/* Only update the timeout if the new timeout is at least
 		   HZ jiffies from the old timeout. Need del_timer for race
 		   avoidance (may already be dying). */
 		if (newtime - ct->timeout.expires >= HZ)
 			mod_timer_pending(&ct->timeout, newtime);
 	}
 acct:
 	if (do_acct) {
 		struct nf_conn_counter *acct;
 		acct = nf_conn_acct_find(ct);
 		if (acct) {
 			atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
 			atomic64_add(skb->len, &acct[CTINFO2DIR(ctinfo)].bytes);
 		}
 	}
 }
 EXPORT_SYMBOL_GPL(__nf_ct_refresh_acct);
 bool __nf_ct_kill_acct(struct nf_conn *ct,
 		       enum ip_conntrack_info ctinfo,
 		       const struct sk_buff *skb,
 		       int do_acct)
 {
 	if (do_acct) {
 		struct nf_conn_counter *acct;
 		acct = nf_conn_acct_find(ct);
 		if (acct) {
 			atomic64_inc(&acct[CTINFO2DIR(ctinfo)].packets);
 			atomic64_add(skb->len - skb_network_offset(skb),
 				     &acct[CTINFO2DIR(ctinfo)].bytes);
 		}
 	}
 	if (del_timer(&ct->timeout)) {
 		ct->timeout.function((unsigned long)ct);
 		return true;
 	}
 	return false;
 }
 EXPORT_SYMBOL_GPL(__nf_ct_kill_acct);
 #ifdef CONFIG_NF_CONNTRACK_ZONES
 static struct nf_ct_ext_type nf_ct_zone_extend __read_mostly = {
 	.len	= sizeof(struct nf_conntrack_zone),
 	.align	= __alignof__(struct nf_conntrack_zone),
 	.id	= NF_CT_EXT_ZONE,
 };
 #endif
 #if IS_ENABLED(CONFIG_NF_CT_NETLINK)
 #include <linux/netfilter/nfnetlink.h>
 #include <linux/netfilter/nfnetlink_conntrack.h>
 #include <linux/mutex.h>
 /* Generic function for tcp/udp/sctp/dccp and alike. This needs to be
  * in ip_conntrack_core, since we don't want the protocols to autoload
  * or depend on ctnetlink */
 int nf_ct_port_tuple_to_nlattr(struct sk_buff *skb,
 			       const struct nf_conntrack_tuple *tuple)
 {
 	if (nla_put_be16(skb, CTA_PROTO_SRC_PORT, tuple->src.u.tcp.port) ||
 	    nla_put_be16(skb, CTA_PROTO_DST_PORT, tuple->dst.u.tcp.port))
 		goto nla_put_failure;
 	return 0;
 nla_put_failure:
 	return -1;
 }
 EXPORT_SYMBOL_GPL(nf_ct_port_tuple_to_nlattr);
 const struct nla_policy nf_ct_port_nla_policy[CTA_PROTO_MAX+1] = {
 	[CTA_PROTO_SRC_PORT]  = { .type = NLA_U16 },
 	[CTA_PROTO_DST_PORT]  = { .type = NLA_U16 },
 };
 EXPORT_SYMBOL_GPL(nf_ct_port_nla_policy);
 int nf_ct_port_nlattr_to_tuple(struct nlattr *tb[],
 			       struct nf_conntrack_tuple *t)
 {
 	if (!tb[CTA_PROTO_SRC_PORT] || !tb[CTA_PROTO_DST_PORT])
 		return -EINVAL;
 	t->src.u.tcp.port = nla_get_be16(tb[CTA_PROTO_SRC_PORT]);
 	t->dst.u.tcp.port = nla_get_be16(tb[CTA_PROTO_DST_PORT]);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_to_tuple);
 int nf_ct_port_nlattr_tuple_size(void)
 {
 	return nla_policy_len(nf_ct_port_nla_policy, CTA_PROTO_MAX + 1);
 }
 EXPORT_SYMBOL_GPL(nf_ct_port_nlattr_tuple_size);
 #endif
 /* Used by ipt_REJECT and ip6t_REJECT. */
 static void nf_conntrack_attach(struct sk_buff *nskb, struct sk_buff *skb)
 {
 	struct nf_conn *ct;
 	enum ip_conntrack_info ctinfo;
 	/* This ICMP is in reverse direction to the packet which caused it */
 	ct = nf_ct_get(skb, &ctinfo);
 	if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL)
 		ctinfo = IP_CT_RELATED_REPLY;
 	else
 		ctinfo = IP_CT_RELATED;
 	/* Attach to new skbuff, and increment count */
 	nskb->nfct = &ct->ct_general;
 	nskb->nfctinfo = ctinfo;
 	nf_conntrack_get(nskb->nfct);
 }
 /* Bring out ya dead! */
 static struct nf_conn *
 get_next_corpse(struct net *net, int (*iter)(struct nf_conn *i, void *data),
 		void *data, unsigned int *bucket)
 {
 	struct nf_conntrack_tuple_hash *h;
 	struct nf_conn *ct;
 	struct hlist_nulls_node *n;
 	spin_lock_bh(&nf_conntrack_lock);
 	for (; *bucket < net->ct.htable_size; (*bucket)++) {
 		hlist_nulls_for_each_entry(h, n, &net->ct.hash[*bucket], hnnode) {
 			if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
 				continue;
 			ct = nf_ct_tuplehash_to_ctrack(h);
 			if (iter(ct, data))
 				goto found;
 		}
 	}
 	hlist_nulls_for_each_entry(h, n, &net->ct.unconfirmed, hnnode) {
 		ct = nf_ct_tuplehash_to_ctrack(h);
 		if (iter(ct, data))
 			set_bit(IPS_DYING_BIT, &ct->status);
 	}
 	spin_unlock_bh(&nf_conntrack_lock);
 	return NULL;
 found:
 	atomic_inc(&ct->ct_general.use);
 	spin_unlock_bh(&nf_conntrack_lock);
 	return ct;
 }
 void nf_ct_iterate_cleanup(struct net *net,
 			   int (*iter)(struct nf_conn *i, void *data),
 			   void *data)
 {
 	struct nf_conn *ct;
 	unsigned int bucket = 0;
 	while ((ct = get_next_corpse(net, iter, data, &bucket)) != NULL) {
 		/* Time to push up daises... */
 		if (del_timer(&ct->timeout))
 			death_by_timeout((unsigned long)ct);
 		/* ... else the timer will get him soon. */
 		nf_ct_put(ct);
 	}
 }
 EXPORT_SYMBOL_GPL(nf_ct_iterate_cleanup);
 struct __nf_ct_flush_report {
 	u32 pid;
 	int report;
 };
 static int kill_report(struct nf_conn *i, void *data)
 {
 	struct __nf_ct_flush_report *fr = (struct __nf_ct_flush_report *)data;
 	struct nf_conn_tstamp *tstamp;
 	tstamp = nf_conn_tstamp_find(i);
 	if (tstamp && tstamp->stop == 0)
 		tstamp->stop = ktime_to_ns(ktime_get_real());
 	/* If we fail to deliver the event, death_by_timeout() will retry */
 	if (nf_conntrack_event_report(IPCT_DESTROY, i,
 				      fr->pid, fr->report) < 0)
 		return 1;
 	/* Avoid the delivery of the destroy event in death_by_timeout(). */
 	set_bit(IPS_DYING_BIT, &i->status);
 	return 1;
 }
 static int kill_all(struct nf_conn *i, void *data)
 {
 	return 1;
 }
 void nf_ct_free_hashtable(void *hash, unsigned int size)
 {
 	if (is_vmalloc_addr(hash))
 		vfree(hash);
 	else
 		free_pages((unsigned long)hash,
 			   get_order(sizeof(struct hlist_head) * size));
 }
 EXPORT_SYMBOL_GPL(nf_ct_free_hashtable);
 void nf_conntrack_flush_report(struct net *net, u32 pid, int report)
 {
 	struct __nf_ct_flush_report fr = {
 		.pid 	= pid,
 		.report = report,
 	};
 	nf_ct_iterate_cleanup(net, kill_report, &fr);
 }
 EXPORT_SYMBOL_GPL(nf_conntrack_flush_report);
 static void nf_ct_release_dying_list(struct net *net)
 {
 	struct nf_conntrack_tuple_hash *h;
 	struct nf_conn *ct;
 	struct hlist_nulls_node *n;
 	spin_lock_bh(&nf_conntrack_lock);
 	hlist_nulls_for_each_entry(h, n, &net->ct.dying, hnnode) {
 		ct = nf_ct_tuplehash_to_ctrack(h);
 		/* never fails to remove them, no listeners at this point */
 		nf_ct_kill(ct);
 	}
 	spin_unlock_bh(&nf_conntrack_lock);
 }
 static int untrack_refs(void)
 {
 	int cnt = 0, cpu;
 	for_each_possible_cpu(cpu) {
 		struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
 		cnt += atomic_read(&ct->ct_general.use) - 1;
 	}
 	return cnt;
 }
 static void nf_conntrack_cleanup_init_net(void)
 {
 	while (untrack_refs() > 0)
 		schedule();
 #ifdef CONFIG_NF_CONNTRACK_ZONES
 	nf_ct_extend_unregister(&nf_ct_zone_extend);
 #endif
 }
 static void nf_conntrack_cleanup_net(struct net *net)
 {
  i_see_dead_people:
 	nf_ct_iterate_cleanup(net, kill_all, NULL);
 	nf_ct_release_dying_list(net);
 	if (atomic_read(&net->ct.count) != 0) {
 		schedule();
 		goto i_see_dead_people;
 	}
 	nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
 	nf_conntrack_helper_fini(net);
 	nf_conntrack_timeout_fini(net);
 	nf_conntrack_ecache_fini(net);
 	nf_conntrack_tstamp_fini(net);
 	nf_conntrack_acct_fini(net);
 	nf_conntrack_expect_fini(net);
 	kmem_cache_destroy(net->ct.nf_conntrack_cachep);
 	kfree(net->ct.slabname);
 	free_percpu(net->ct.stat);
 }
 /* Mishearing the voices in his head, our hero wonders how he's
    supposed to kill the mall. */
 void nf_conntrack_cleanup(struct net *net)
 {
 	if (net_eq(net, &init_net))
 		RCU_INIT_POINTER(ip_ct_attach, NULL);
 	/* This makes sure all current packets have passed through
 	   netfilter framework.  Roll on, two-stage module
 	   delete... */
 	synchronize_net();
 	nf_conntrack_proto_fini(net);
 	nf_conntrack_cleanup_net(net);
+}
-	if (net_eq(net, &init_net)) {
+void nf_conntrack_cleanup_end(void)
-		RCU_INIT_POINTER(nf_ct_destroy, NULL);
+{
-		nf_conntrack_cleanup_init_net();
+	RCU_INIT_POINTER(nf_ct_destroy, NULL);
-	}
+	nf_conntrack_cleanup_init_net();
 }
 void *nf_ct_alloc_hashtable(unsigned int *sizep, int nulls)
 {
 	struct hlist_nulls_head *hash;
 	unsigned int nr_slots, i;
 	size_t sz;
 	BUILD_BUG_ON(sizeof(struct hlist_nulls_head) != sizeof(struct hlist_head));
 	nr_slots = *sizep = roundup(*sizep, PAGE_SIZE / sizeof(struct hlist_nulls_head));
 	sz = nr_slots * sizeof(struct hlist_nulls_head);
 	hash = (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO,
 					get_order(sz));
 	if (!hash) {
 		printk(KERN_WARNING "nf_conntrack: falling back to vmalloc.\n");
 		hash = vzalloc(sz);
 	}
 	if (hash && nulls)
 		for (i = 0; i < nr_slots; i++)
 			INIT_HLIST_NULLS_HEAD(&hash[i], i);
 	return hash;
 }
 EXPORT_SYMBOL_GPL(nf_ct_alloc_hashtable);
 int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
 {
 	int i, bucket, rc;
 	unsigned int hashsize, old_size;
 	struct hlist_nulls_head *hash, *old_hash;
 	struct nf_conntrack_tuple_hash *h;
 	struct nf_conn *ct;
 	if (current->nsproxy->net_ns != &init_net)
 		return -EOPNOTSUPP;
 	/* On boot, we can set this without any fancy locking. */
 	if (!nf_conntrack_htable_size)
 		return param_set_uint(val, kp);
 	rc = kstrtouint(val, 0, &hashsize);
 	if (rc)
 		return rc;
 	if (!hashsize)
 		return -EINVAL;
 	hash = nf_ct_alloc_hashtable(&hashsize, 1);
 	if (!hash)
 		return -ENOMEM;
 	/* Lookups in the old hash might happen in parallel, which means we
 	 * might get false negatives during connection lookup. New connections
 	 * created because of a false negative won't make it into the hash
 	 * though since that required taking the lock.
 	 */
 	spin_lock_bh(&nf_conntrack_lock);
 	for (i = 0; i < init_net.ct.htable_size; i++) {
 		while (!hlist_nulls_empty(&init_net.ct.hash[i])) {
 			h = hlist_nulls_entry(init_net.ct.hash[i].first,
 					struct nf_conntrack_tuple_hash, hnnode);
 			ct = nf_ct_tuplehash_to_ctrack(h);
 			hlist_nulls_del_rcu(&h->hnnode);
 			bucket = __hash_conntrack(&h->tuple, nf_ct_zone(ct),
 						  hashsize);
 			hlist_nulls_add_head_rcu(&h->hnnode, &hash[bucket]);
 		}
 	}
 	old_size = init_net.ct.htable_size;
 	old_hash = init_net.ct.hash;
 	init_net.ct.htable_size = nf_conntrack_htable_size = hashsize;
 	init_net.ct.hash = hash;
 	spin_unlock_bh(&nf_conntrack_lock);
 	nf_ct_free_hashtable(old_hash, old_size);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nf_conntrack_set_hashsize);
 module_param_call(hashsize, nf_conntrack_set_hashsize, param_get_uint,
 		  &nf_conntrack_htable_size, 0600);
 void nf_ct_untracked_status_or(unsigned long bits)
 {
 	int cpu;
 	for_each_possible_cpu(cpu)
 		per_cpu(nf_conntrack_untracked, cpu).status |= bits;
 }
 EXPORT_SYMBOL_GPL(nf_ct_untracked_status_or);
 static int nf_conntrack_init_init_net(void)
 {
 	int max_factor = 8;
 	int ret, cpu;
 	/* Idea from tcp.c: use 1/16384 of memory.  On i386: 32MB
 	 * machine has 512 buckets. >= 1GB machines have 16384 buckets. */
 	if (!nf_conntrack_htable_size) {
 		nf_conntrack_htable_size
 			= (((totalram_pages << PAGE_SHIFT) / 16384)
 			   / sizeof(struct hlist_head));
 		if (totalram_pages > (1024 * 1024 * 1024 / PAGE_SIZE))
 			nf_conntrack_htable_size = 16384;
 		if (nf_conntrack_htable_size < 32)
 			nf_conntrack_htable_size = 32;
 		/* Use a max. factor of four by default to get the same max as
 		 * with the old struct list_heads. When a table size is given
 		 * we use the old value of 8 to avoid reducing the max.
 		 * entries. */
 		max_factor = 4;
 	}
 	nf_conntrack_max = max_factor * nf_conntrack_htable_size;
 	printk(KERN_INFO "nf_conntrack version %s (%u buckets, %d max)\n",
 	       NF_CONNTRACK_VERSION, nf_conntrack_htable_size,
 	       nf_conntrack_max);
 #ifdef CONFIG_NF_CONNTRACK_ZONES
 	ret = nf_ct_extend_register(&nf_ct_zone_extend);
 	if (ret < 0)
 		goto err_extend;
 #endif
 	/* Set up fake conntrack: to never be deleted, not in any hashes */
 	for_each_possible_cpu(cpu) {
 		struct nf_conn *ct = &per_cpu(nf_conntrack_untracked, cpu);
 		write_pnet(&ct->ct_net, &init_net);
 		atomic_set(&ct->ct_general.use, 1);
 	}
 	/*  - and look it like as a confirmed connection */
 	nf_ct_untracked_status_or(IPS_CONFIRMED | IPS_UNTRACKED);
 	return 0;
 #ifdef CONFIG_NF_CONNTRACK_ZONES
 err_extend:
 #endif
 	return ret;
 }
 /*
  * We need to use special "null" values, not used in hash table
  */
 #define UNCONFIRMED_NULLS_VAL	((1<<30)+0)
 #define DYING_NULLS_VAL		((1<<30)+1)
 #define TEMPLATE_NULLS_VAL	((1<<30)+2)
 static int nf_conntrack_init_net(struct net *net)
 {
 	int ret;
 	atomic_set(&net->ct.count, 0);
 	INIT_HLIST_NULLS_HEAD(&net->ct.unconfirmed, UNCONFIRMED_NULLS_VAL);
 	INIT_HLIST_NULLS_HEAD(&net->ct.dying, DYING_NULLS_VAL);
 	INIT_HLIST_NULLS_HEAD(&net->ct.tmpl, TEMPLATE_NULLS_VAL);
 	net->ct.stat = alloc_percpu(struct ip_conntrack_stat);
 	if (!net->ct.stat) {
 		ret = -ENOMEM;
 		goto err_stat;
 	}
 	net->ct.slabname = kasprintf(GFP_KERNEL, "nf_conntrack_%p", net);
 	if (!net->ct.slabname) {
 		ret = -ENOMEM;
 		goto err_slabname;
 	}
 	net->ct.nf_conntrack_cachep = kmem_cache_create(net->ct.slabname,
 							sizeof(struct nf_conn), 0,
 							SLAB_DESTROY_BY_RCU, NULL);
 	if (!net->ct.nf_conntrack_cachep) {
 		printk(KERN_ERR "Unable to create nf_conn slab cache\n");
 		ret = -ENOMEM;
 		goto err_cache;
 	}
 	net->ct.htable_size = nf_conntrack_htable_size;
 	net->ct.hash = nf_ct_alloc_hashtable(&net->ct.htable_size, 1);
 	if (!net->ct.hash) {
 		ret = -ENOMEM;
 		printk(KERN_ERR "Unable to create nf_conntrack_hash\n");
 		goto err_hash;
 	}
 	ret = nf_conntrack_expect_init(net);
 	if (ret < 0)
 		goto err_expect;
 	ret = nf_conntrack_acct_init(net);
 	if (ret < 0)
 		goto err_acct;
 	ret = nf_conntrack_tstamp_init(net);
 	if (ret < 0)
 		goto err_tstamp;
 	ret = nf_conntrack_ecache_init(net);
 	if (ret < 0)
 		goto err_ecache;
 	ret = nf_conntrack_timeout_init(net);
 	if (ret < 0)
 		goto err_timeout;
 	ret = nf_conntrack_helper_init(net);
 	if (ret < 0)
 		goto err_helper;
 	return 0;
 err_helper:
 	nf_conntrack_timeout_fini(net);
 err_timeout:
 	nf_conntrack_ecache_fini(net);
 err_ecache:
 	nf_conntrack_tstamp_fini(net);
 err_tstamp:
 	nf_conntrack_acct_fini(net);
 err_acct:
 	nf_conntrack_expect_fini(net);
 err_expect:
 	nf_ct_free_hashtable(net->ct.hash, net->ct.htable_size);
 err_hash:
 	kmem_cache_destroy(net->ct.nf_conntrack_cachep);
 err_cache:
 	kfree(net->ct.slabname);
 err_slabname:
 	free_percpu(net->ct.stat);
 err_stat:
 	return ret;
 }
 s16 (*nf_ct_nat_offset)(const struct nf_conn *ct,
 			enum ip_conntrack_dir dir,
 			u32 seq);
 EXPORT_SYMBOL_GPL(nf_ct_nat_offset);
 int nf_conntrack_init(struct net *net)
 {
 	int ret;
 	if (net_eq(net, &init_net)) {
 		ret = nf_conntrack_init_init_net();
 		if (ret < 0)
 			goto out_init_net;
 	}
 	ret = nf_conntrack_proto_init(net);
 	if (ret < 0)
 		goto out_proto;
 	ret = nf_conntrack_init_net(net);
 	if (ret < 0)
 		goto out_net;
 	if (net_eq(net, &init_net)) {
 		/* For use by REJECT target */
 		RCU_INIT_POINTER(ip_ct_attach, nf_conntrack_attach);
 		RCU_INIT_POINTER(nf_ct_destroy, destroy_conntrack);
 		/* Howto get NAT offsets */
 		RCU_INIT_POINTER(nf_ct_nat_offset, NULL);
 	}
 	return 0;
 out_net:
 	nf_conntrack_proto_fini(net);
 out_proto:
 	if (net_eq(net, &init_net))
 		nf_conntrack_cleanup_init_net();
 out_init_net:
 	return ret;
 }