Eric Lee / smarc-fsl-linux-kernel

1

/*

1

/*

2

* NET3 Protocol independent device support routines.

2

* NET3 Protocol independent device support routines.

3

*

3

*

4

* This program is free software; you can redistribute it and/or

4

* This program is free software; you can redistribute it and/or

5

* modify it under the terms of the GNU General Public License

5

* modify it under the terms of the GNU General Public License

6

* as published by the Free Software Foundation; either version

6

* as published by the Free Software Foundation; either version

7

* 2 of the License, or (at your option) any later version.

7

* 2 of the License, or (at your option) any later version.

8

*

8

*

9

* Derived from the non IP parts of dev.c 1.0.19

9

* Derived from the non IP parts of dev.c 1.0.19

10

* Authors: Ross Biro

10

* Authors: Ross Biro

11

* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>

11

* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>

12

* Mark Evans, <evansmp@uhura.aston.ac.uk>

12

* Mark Evans, <evansmp@uhura.aston.ac.uk>

13

*

13

*

14

* Additional Authors:

14

* Additional Authors:

15

* Florian la Roche <rzsfl@rz.uni-sb.de>

15

* Florian la Roche <rzsfl@rz.uni-sb.de>

16

* Alan Cox <gw4pts@gw4pts.ampr.org>

16

* Alan Cox <gw4pts@gw4pts.ampr.org>

17

* David Hinds <dahinds@users.sourceforge.net>

17

* David Hinds <dahinds@users.sourceforge.net>

18

* Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>

18

* Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>

19

* Adam Sulmicki <adam@cfar.umd.edu>

19

* Adam Sulmicki <adam@cfar.umd.edu>

20

* Pekka Riikonen <priikone@poesidon.pspt.fi>

20

* Pekka Riikonen <priikone@poesidon.pspt.fi>

21

*

21

*

22

* Changes:

22

* Changes:

23

* D.J. Barrow : Fixed bug where dev->refcnt gets set

23

* D.J. Barrow : Fixed bug where dev->refcnt gets set

24

* to 2 if register_netdev gets called

24

* to 2 if register_netdev gets called

25

* before net_dev_init & also removed a

25

* before net_dev_init & also removed a

26

* few lines of code in the process.

26

* few lines of code in the process.

27

* Alan Cox : device private ioctl copies fields back.

27

* Alan Cox : device private ioctl copies fields back.

28

* Alan Cox : Transmit queue code does relevant

28

* Alan Cox : Transmit queue code does relevant

29

* stunts to keep the queue safe.

29

* stunts to keep the queue safe.

30

* Alan Cox : Fixed double lock.

30

* Alan Cox : Fixed double lock.

31

* Alan Cox : Fixed promisc NULL pointer trap

31

* Alan Cox : Fixed promisc NULL pointer trap

32

* ???????? : Support the full private ioctl range

32

* ???????? : Support the full private ioctl range

33

* Alan Cox : Moved ioctl permission check into

33

* Alan Cox : Moved ioctl permission check into

34

* drivers

34

* drivers

35

* Tim Kordas : SIOCADDMULTI/SIOCDELMULTI

35

* Tim Kordas : SIOCADDMULTI/SIOCDELMULTI

36

* Alan Cox : 100 backlog just doesn't cut it when

36

* Alan Cox : 100 backlog just doesn't cut it when

37

* you start doing multicast video 8)

37

* you start doing multicast video 8)

38

* Alan Cox : Rewrote net_bh and list manager.

38

* Alan Cox : Rewrote net_bh and list manager.

39

* Alan Cox : Fix ETH_P_ALL echoback lengths.

39

* Alan Cox : Fix ETH_P_ALL echoback lengths.

40

* Alan Cox : Took out transmit every packet pass

40

* Alan Cox : Took out transmit every packet pass

41

* Saved a few bytes in the ioctl handler

41

* Saved a few bytes in the ioctl handler

42

* Alan Cox : Network driver sets packet type before

42

* Alan Cox : Network driver sets packet type before

43

* calling netif_rx. Saves a function

43

* calling netif_rx. Saves a function

44

* call a packet.

44

* call a packet.

45

* Alan Cox : Hashed net_bh()

45

* Alan Cox : Hashed net_bh()

46

* Richard Kooijman: Timestamp fixes.

46

* Richard Kooijman: Timestamp fixes.

47

* Alan Cox : Wrong field in SIOCGIFDSTADDR

47

* Alan Cox : Wrong field in SIOCGIFDSTADDR

48

* Alan Cox : Device lock protection.

48

* Alan Cox : Device lock protection.

49

* Alan Cox : Fixed nasty side effect of device close

49

* Alan Cox : Fixed nasty side effect of device close

50

* changes.

50

* changes.

51

* Rudi Cilibrasi : Pass the right thing to

51

* Rudi Cilibrasi : Pass the right thing to

52

* set_mac_address()

52

* set_mac_address()

53

* Dave Miller : 32bit quantity for the device lock to

53

* Dave Miller : 32bit quantity for the device lock to

54

* make it work out on a Sparc.

54

* make it work out on a Sparc.

55

* Bjorn Ekwall : Added KERNELD hack.

55

* Bjorn Ekwall : Added KERNELD hack.

56

* Alan Cox : Cleaned up the backlog initialise.

56

* Alan Cox : Cleaned up the backlog initialise.

57

* Craig Metz : SIOCGIFCONF fix if space for under

57

* Craig Metz : SIOCGIFCONF fix if space for under

58

* 1 device.

58

* 1 device.

59

* Thomas Bogendoerfer : Return ENODEV for dev_open, if there

59

* Thomas Bogendoerfer : Return ENODEV for dev_open, if there

60

* is no device open function.

60

* is no device open function.

61

* Andi Kleen : Fix error reporting for SIOCGIFCONF

61

* Andi Kleen : Fix error reporting for SIOCGIFCONF

62

* Michael Chastain : Fix signed/unsigned for SIOCGIFCONF

62

* Michael Chastain : Fix signed/unsigned for SIOCGIFCONF

63

* Cyrus Durgin : Cleaned for KMOD

63

* Cyrus Durgin : Cleaned for KMOD

64

* Adam Sulmicki : Bug Fix : Network Device Unload

64

* Adam Sulmicki : Bug Fix : Network Device Unload

65

* A network device unload needs to purge

65

* A network device unload needs to purge

66

* the backlog queue.

66

* the backlog queue.

67

* Paul Rusty Russell : SIOCSIFNAME

67

* Paul Rusty Russell : SIOCSIFNAME

68

* Pekka Riikonen : Netdev boot-time settings code

68

* Pekka Riikonen : Netdev boot-time settings code

69

* Andrew Morton : Make unregister_netdevice wait

69

* Andrew Morton : Make unregister_netdevice wait

70

* indefinitely on dev->refcnt

70

* indefinitely on dev->refcnt

71

* J Hadi Salim : - Backlog queue sampling

71

* J Hadi Salim : - Backlog queue sampling

72

* - netif_rx() feedback

72

* - netif_rx() feedback

73

*/

73

*/

74

75

#include <asm/uaccess.h>

75

#include <asm/uaccess.h>

76

#include <asm/system.h>

76

#include <asm/system.h>

77

#include <linux/bitops.h>

77

#include <linux/bitops.h>

78

#include <linux/capability.h>

78

#include <linux/capability.h>

79

#include <linux/cpu.h>

79

#include <linux/cpu.h>

80

#include <linux/types.h>

80

#include <linux/types.h>

81

#include <linux/kernel.h>

81

#include <linux/kernel.h>

82

#include <linux/hash.h>

82

#include <linux/hash.h>

83

#include <linux/slab.h>

83

#include <linux/slab.h>

84

#include <linux/sched.h>

84

#include <linux/sched.h>

85

#include <linux/mutex.h>

85

#include <linux/mutex.h>

86

#include <linux/string.h>

86

#include <linux/string.h>

87

#include <linux/mm.h>

87

#include <linux/mm.h>

88

#include <linux/socket.h>

88

#include <linux/socket.h>

89

#include <linux/sockios.h>

89

#include <linux/sockios.h>

90

#include <linux/errno.h>

90

#include <linux/errno.h>

91

#include <linux/interrupt.h>

91

#include <linux/interrupt.h>

92

#include <linux/if_ether.h>

92

#include <linux/if_ether.h>

93

#include <linux/netdevice.h>

93

#include <linux/netdevice.h>

94

#include <linux/etherdevice.h>

94

#include <linux/etherdevice.h>

95

#include <linux/ethtool.h>

95

#include <linux/ethtool.h>

96

#include <linux/notifier.h>

96

#include <linux/notifier.h>

97

#include <linux/skbuff.h>

97

#include <linux/skbuff.h>

98

#include <net/net_namespace.h>

98

#include <net/net_namespace.h>

99

#include <net/sock.h>

99

#include <net/sock.h>

100

#include <linux/rtnetlink.h>

100

#include <linux/rtnetlink.h>

101

#include <linux/proc_fs.h>

101

#include <linux/proc_fs.h>

102

#include <linux/seq_file.h>

102

#include <linux/seq_file.h>

103

#include <linux/stat.h>

103

#include <linux/stat.h>

104

#include <net/dst.h>

104

#include <net/dst.h>

105

#include <net/pkt_sched.h>

105

#include <net/pkt_sched.h>

106

#include <net/checksum.h>

106

#include <net/checksum.h>

107

#include <net/xfrm.h>

107

#include <net/xfrm.h>

108

#include <linux/highmem.h>

108

#include <linux/highmem.h>

109

#include <linux/init.h>

109

#include <linux/init.h>

110

#include <linux/kmod.h>

110

#include <linux/kmod.h>

111

#include <linux/module.h>

111

#include <linux/module.h>

112

#include <linux/netpoll.h>

112

#include <linux/netpoll.h>

113

#include <linux/rcupdate.h>

113

#include <linux/rcupdate.h>

114

#include <linux/delay.h>

114

#include <linux/delay.h>

115

#include <net/wext.h>

115

#include <net/wext.h>

116

#include <net/iw_handler.h>

116

#include <net/iw_handler.h>

117

#include <asm/current.h>

117

#include <asm/current.h>

118

#include <linux/audit.h>

118

#include <linux/audit.h>

119

#include <linux/dmaengine.h>

119

#include <linux/dmaengine.h>

120

#include <linux/err.h>

120

#include <linux/err.h>

121

#include <linux/ctype.h>

121

#include <linux/ctype.h>

122

#include <linux/if_arp.h>

122

#include <linux/if_arp.h>

123

#include <linux/if_vlan.h>

123

#include <linux/if_vlan.h>

124

#include <linux/ip.h>

124

#include <linux/ip.h>

125

#include <net/ip.h>

125

#include <net/ip.h>

126

#include <linux/ipv6.h>

126

#include <linux/ipv6.h>

127

#include <linux/in.h>

127

#include <linux/in.h>

128

#include <linux/jhash.h>

128

#include <linux/jhash.h>

129

#include <linux/random.h>

129

#include <linux/random.h>

130

#include <trace/events/napi.h>

130

#include <trace/events/napi.h>

131

#include <linux/pci.h>

131

#include <linux/pci.h>

132

#include <linux/inetdevice.h>

132

#include <linux/inetdevice.h>

133

134

#include "net-sysfs.h"

134

#include "net-sysfs.h"

135

136

/* Instead of increasing this, you should create a hash table. */

136

/* Instead of increasing this, you should create a hash table. */

137

#define MAX_GRO_SKBS 8

137

#define MAX_GRO_SKBS 8

138

139

/* This should be increased if a protocol with a bigger head is added. */

139

/* This should be increased if a protocol with a bigger head is added. */

140

#define GRO_MAX_HEAD (MAX_HEADER + 128)

140

#define GRO_MAX_HEAD (MAX_HEADER + 128)

141

142

/*

142

/*

143

* The list of packet types we will receive (as opposed to discard)

143

* The list of packet types we will receive (as opposed to discard)

144

* and the routines to invoke.

144

* and the routines to invoke.

145

*

145

*

146

* Why 16. Because with 16 the only overlap we get on a hash of the

146

* Why 16. Because with 16 the only overlap we get on a hash of the

147

* low nibble of the protocol value is RARP/SNAP/X.25.

147

* low nibble of the protocol value is RARP/SNAP/X.25.

148

*

148

*

149

* NOTE: That is no longer true with the addition of VLAN tags. Not

149

* NOTE: That is no longer true with the addition of VLAN tags. Not

150

* sure which should go first, but I bet it won't make much

150

* sure which should go first, but I bet it won't make much

151

* difference if we are running VLANs. The good news is that

151

* difference if we are running VLANs. The good news is that

152

* this protocol won't be in the list unless compiled in, so

152

* this protocol won't be in the list unless compiled in, so

153

* the average user (w/out VLANs) will not be adversely affected.

153

* the average user (w/out VLANs) will not be adversely affected.

154

* --BLG

154

* --BLG

155

*

155

*

156

* 0800 IP

156

* 0800 IP

157

* 8100 802.1Q VLAN

157

* 8100 802.1Q VLAN

158

* 0001 802.3

158

* 0001 802.3

159

* 0002 AX.25

159

* 0002 AX.25

160

* 0004 802.2

160

* 0004 802.2

161

* 8035 RARP

161

* 8035 RARP

162

* 0005 SNAP

162

* 0005 SNAP

163

* 0805 X.25

163

* 0805 X.25

164

* 0806 ARP

164

* 0806 ARP

165

* 8137 IPX

165

* 8137 IPX

166

* 0009 Localtalk

166

* 0009 Localtalk

167

* 86DD IPv6

167

* 86DD IPv6

168

*/

168

*/

169

170

#define PTYPE_HASH_SIZE (16)

170

#define PTYPE_HASH_SIZE (16)

171

#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)

171

#define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)

172

173

static DEFINE_SPINLOCK(ptype_lock);

173

static DEFINE_SPINLOCK(ptype_lock);

174

static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;

174

static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;

175

static struct list_head ptype_all __read_mostly; /* Taps */

175

static struct list_head ptype_all __read_mostly; /* Taps */

176

177

/*

177

/*

178

* The @dev_base_head list is protected by @dev_base_lock and the rtnl

178

* The @dev_base_head list is protected by @dev_base_lock and the rtnl

179

* semaphore.

179

* semaphore.

180

*

180

*

181

* Pure readers hold dev_base_lock for reading, or rcu_read_lock()

181

* Pure readers hold dev_base_lock for reading, or rcu_read_lock()

182

*

182

*

183

* Writers must hold the rtnl semaphore while they loop through the

183

* Writers must hold the rtnl semaphore while they loop through the

184

* dev_base_head list, and hold dev_base_lock for writing when they do the

184

* dev_base_head list, and hold dev_base_lock for writing when they do the

185

* actual updates. This allows pure readers to access the list even

185

* actual updates. This allows pure readers to access the list even

186

* while a writer is preparing to update it.

186

* while a writer is preparing to update it.

187

*

187

*

188

* To put it another way, dev_base_lock is held for writing only to

188

* To put it another way, dev_base_lock is held for writing only to

189

* protect against pure readers; the rtnl semaphore provides the

189

* protect against pure readers; the rtnl semaphore provides the

190

* protection against other writers.

190

* protection against other writers.

191

*

191

*

192

* See, for example usages, register_netdevice() and

192

* See, for example usages, register_netdevice() and

193

* unregister_netdevice(), which must be called with the rtnl

193

* unregister_netdevice(), which must be called with the rtnl

194

* semaphore held.

194

* semaphore held.

195

*/

195

*/

196

DEFINE_RWLOCK(dev_base_lock);

196

DEFINE_RWLOCK(dev_base_lock);

197

EXPORT_SYMBOL(dev_base_lock);

197

EXPORT_SYMBOL(dev_base_lock);

198

199

static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)

199

static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)

200

{

200

{

201

unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));

201

unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));

202

return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];

202

return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];

203

}

203

}

204

205

static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)

205

static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)

206

{

206

{

207

return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];

207

return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];

208

}

208

}

209

210

static inline void rps_lock(struct softnet_data *sd)

210

static inline void rps_lock(struct softnet_data *sd)

211

{

211

{

212

#ifdef CONFIG_RPS

212

#ifdef CONFIG_RPS

213

spin_lock(&sd->input_pkt_queue.lock);

213

spin_lock(&sd->input_pkt_queue.lock);

214

#endif

214

#endif

215

}

215

}

216

217

static inline void rps_unlock(struct softnet_data *sd)

217

static inline void rps_unlock(struct softnet_data *sd)

218

{

218

{

219

#ifdef CONFIG_RPS

219

#ifdef CONFIG_RPS

220

spin_unlock(&sd->input_pkt_queue.lock);

220

spin_unlock(&sd->input_pkt_queue.lock);

221

#endif

221

#endif

222

}

222

}

223

224

/* Device list insertion */

224

/* Device list insertion */

225

static int list_netdevice(struct net_device *dev)

225

static int list_netdevice(struct net_device *dev)

226

{

226

{

227

struct net *net = dev_net(dev);

227

struct net *net = dev_net(dev);

228

229

ASSERT_RTNL();

229

ASSERT_RTNL();

230

231

write_lock_bh(&dev_base_lock);

231

write_lock_bh(&dev_base_lock);

232

list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);

232

list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);

233

hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));

233

hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));

234

hlist_add_head_rcu(&dev->index_hlist,

234

hlist_add_head_rcu(&dev->index_hlist,

235

dev_index_hash(net, dev->ifindex));

235

dev_index_hash(net, dev->ifindex));

236

write_unlock_bh(&dev_base_lock);

236

write_unlock_bh(&dev_base_lock);

237

return 0;

237

return 0;

238

}

238

}

239

240

/* Device list removal

240

/* Device list removal

241

* caller must respect a RCU grace period before freeing/reusing dev

241

* caller must respect a RCU grace period before freeing/reusing dev

242

*/

242

*/

243

static void unlist_netdevice(struct net_device *dev)

243

static void unlist_netdevice(struct net_device *dev)

244

{

244

{

245

ASSERT_RTNL();

245

ASSERT_RTNL();

246

247

/* Unlink dev from the device chain */

247

/* Unlink dev from the device chain */

248

write_lock_bh(&dev_base_lock);

248

write_lock_bh(&dev_base_lock);

249

list_del_rcu(&dev->dev_list);

249

list_del_rcu(&dev->dev_list);

250

hlist_del_rcu(&dev->name_hlist);

250

hlist_del_rcu(&dev->name_hlist);

251

hlist_del_rcu(&dev->index_hlist);

251

hlist_del_rcu(&dev->index_hlist);

252

write_unlock_bh(&dev_base_lock);

252

write_unlock_bh(&dev_base_lock);

253

}

253

}

254

255

/*

255

/*

256

* Our notifier list

256

* Our notifier list

257

*/

257

*/

258

259

static RAW_NOTIFIER_HEAD(netdev_chain);

259

static RAW_NOTIFIER_HEAD(netdev_chain);

260

261

/*

261

/*

262

* Device drivers call our routines to queue packets here. We empty the

262

* Device drivers call our routines to queue packets here. We empty the

263

* queue in the local softnet handler.

263

* queue in the local softnet handler.

264

*/

264

*/

265

266

DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);

266

DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);

267

EXPORT_PER_CPU_SYMBOL(softnet_data);

267

EXPORT_PER_CPU_SYMBOL(softnet_data);

268

269

#ifdef CONFIG_LOCKDEP

269

#ifdef CONFIG_LOCKDEP

270

/*

270

/*

271

* register_netdevice() inits txq->_xmit_lock and sets lockdep class

271

* register_netdevice() inits txq->_xmit_lock and sets lockdep class

272

* according to dev->type

272

* according to dev->type

273

*/

273

*/

274

static const unsigned short netdev_lock_type[] =

274

static const unsigned short netdev_lock_type[] =

275

{ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,

275

{ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,

276

ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,

276

ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,

277

ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,

277

ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,

278

ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,

278

ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,

279

ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,

279

ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,

280

ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,

280

ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,

281

ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,

281

ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,

282

ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,

282

ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,

283

ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,

283

ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,

284

ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,

284

ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,

285

ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,

285

ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,

286

ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,

286

ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,

287

ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,

287

ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,

288

ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,

288

ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,

289

ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,

289

ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,

290

ARPHRD_VOID, ARPHRD_NONE};

290

ARPHRD_VOID, ARPHRD_NONE};

291

292

static const char *const netdev_lock_name[] =

292

static const char *const netdev_lock_name[] =

293

{"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",

293

{"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",

294

"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",

294

"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",

295

"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",

295

"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",

296

"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",

296

"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",

297

"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",

297

"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",

298

"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",

298

"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",

299

"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",

299

"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",

300

"_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",

300

"_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",

301

"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",

301

"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",

302

"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",

302

"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",

303

"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",

303

"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",

304

"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",

304

"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",

305

"_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",

305

"_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",

306

"_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",

306

"_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",

307

"_xmit_PHONET_PIPE", "_xmit_IEEE802154",

307

"_xmit_PHONET_PIPE", "_xmit_IEEE802154",

308

"_xmit_VOID", "_xmit_NONE"};

308

"_xmit_VOID", "_xmit_NONE"};

309

310

static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];

310

static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];

311

static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];

311

static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];

312

313

static inline unsigned short netdev_lock_pos(unsigned short dev_type)

313

static inline unsigned short netdev_lock_pos(unsigned short dev_type)

314

{

314

{

315

int i;

315

int i;

316

317

for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)

317

for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)

318

if (netdev_lock_type[i] == dev_type)

318

if (netdev_lock_type[i] == dev_type)

319

return i;

319

return i;

320

/* the last key is used by default */

320

/* the last key is used by default */

321

return ARRAY_SIZE(netdev_lock_type) - 1;

321

return ARRAY_SIZE(netdev_lock_type) - 1;

322

}

322

}

323

324

static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,

324

static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,

325

unsigned short dev_type)

325

unsigned short dev_type)

326

{

326

{

327

int i;

327

int i;

328

329

i = netdev_lock_pos(dev_type);

329

i = netdev_lock_pos(dev_type);

330

lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],

330

lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],

331

netdev_lock_name[i]);

331

netdev_lock_name[i]);

332

}

332

}

333

334

static inline void netdev_set_addr_lockdep_class(struct net_device *dev)

334

static inline void netdev_set_addr_lockdep_class(struct net_device *dev)

335

{

335

{

336

int i;

336

int i;

337

338

i = netdev_lock_pos(dev->type);

338

i = netdev_lock_pos(dev->type);

339

lockdep_set_class_and_name(&dev->addr_list_lock,

339

lockdep_set_class_and_name(&dev->addr_list_lock,

340

&netdev_addr_lock_key[i],

340

&netdev_addr_lock_key[i],

341

netdev_lock_name[i]);

341

netdev_lock_name[i]);

342

}

342

}

343

#else

343

#else

344

static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,

344

static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,

345

unsigned short dev_type)

345

unsigned short dev_type)

346

{

346

{

347

}

347

}

348

static inline void netdev_set_addr_lockdep_class(struct net_device *dev)

348

static inline void netdev_set_addr_lockdep_class(struct net_device *dev)

349

{

349

{

350

}

350

}

351

#endif

351

#endif

352

353

/*******************************************************************************

353

/*******************************************************************************

354

355

Protocol management and registration routines

355

Protocol management and registration routines

356

357

*******************************************************************************/

357

*******************************************************************************/

358

359

/*

359

/*

360

* Add a protocol ID to the list. Now that the input handler is

360

* Add a protocol ID to the list. Now that the input handler is

361

* smarter we can dispense with all the messy stuff that used to be

361

* smarter we can dispense with all the messy stuff that used to be

362

* here.

362

* here.

363

*

363

*

364

* BEWARE!!! Protocol handlers, mangling input packets,

364

* BEWARE!!! Protocol handlers, mangling input packets,

365

* MUST BE last in hash buckets and checking protocol handlers

365

* MUST BE last in hash buckets and checking protocol handlers

366

* MUST start from promiscuous ptype_all chain in net_bh.

366

* MUST start from promiscuous ptype_all chain in net_bh.

367

* It is true now, do not change it.

367

* It is true now, do not change it.

368

* Explanation follows: if protocol handler, mangling packet, will

368

* Explanation follows: if protocol handler, mangling packet, will

369

* be the first on list, it is not able to sense, that packet

369

* be the first on list, it is not able to sense, that packet

370

* is cloned and should be copied-on-write, so that it will

370

* is cloned and should be copied-on-write, so that it will

371

* change it and subsequent readers will get broken packet.

371

* change it and subsequent readers will get broken packet.

372

* --ANK (980803)

372

* --ANK (980803)

373

*/

373

*/

374

375

static inline struct list_head *ptype_head(const struct packet_type *pt)

375

static inline struct list_head *ptype_head(const struct packet_type *pt)

376

{

376

{

377

if (pt->type == htons(ETH_P_ALL))

377

if (pt->type == htons(ETH_P_ALL))

378

return &ptype_all;

378

return &ptype_all;

379

else

379

else

380

return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];

380

return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];

381

}

381

}

382

383

/**

383

/**

384

* dev_add_pack - add packet handler

384

* dev_add_pack - add packet handler

385

* @pt: packet type declaration

385

* @pt: packet type declaration

386

*

386

*

387

* Add a protocol handler to the networking stack. The passed &packet_type

387

* Add a protocol handler to the networking stack. The passed &packet_type

388

* is linked into kernel lists and may not be freed until it has been

388

* is linked into kernel lists and may not be freed until it has been

389

* removed from the kernel lists.

389

* removed from the kernel lists.

390

*

390

*

391

* This call does not sleep therefore it can not

391

* This call does not sleep therefore it can not

392

* guarantee all CPU's that are in middle of receiving packets

392

* guarantee all CPU's that are in middle of receiving packets

393

* will see the new packet type (until the next received packet).

393

* will see the new packet type (until the next received packet).

394

*/

394

*/

395

396

void dev_add_pack(struct packet_type *pt)

396

void dev_add_pack(struct packet_type *pt)

397

{

397

{

398

struct list_head *head = ptype_head(pt);

398

struct list_head *head = ptype_head(pt);

399

400

spin_lock(&ptype_lock);

400

spin_lock(&ptype_lock);

401

list_add_rcu(&pt->list, head);

401

list_add_rcu(&pt->list, head);

402

spin_unlock(&ptype_lock);

402

spin_unlock(&ptype_lock);

403

}

403

}

404

EXPORT_SYMBOL(dev_add_pack);

404

EXPORT_SYMBOL(dev_add_pack);

405

406

/**

406

/**

407

* __dev_remove_pack - remove packet handler

407

* __dev_remove_pack - remove packet handler

408

* @pt: packet type declaration

408

* @pt: packet type declaration

409

*

409

*

410

* Remove a protocol handler that was previously added to the kernel

410

* Remove a protocol handler that was previously added to the kernel

411

* protocol handlers by dev_add_pack(). The passed &packet_type is removed

411

* protocol handlers by dev_add_pack(). The passed &packet_type is removed

412

* from the kernel lists and can be freed or reused once this function

412

* from the kernel lists and can be freed or reused once this function

413

* returns.

413

* returns.

414

*

414

*

415

* The packet type might still be in use by receivers

415

* The packet type might still be in use by receivers

416

* and must not be freed until after all the CPU's have gone

416

* and must not be freed until after all the CPU's have gone

417

* through a quiescent state.

417

* through a quiescent state.

418

*/

418

*/

419

void __dev_remove_pack(struct packet_type *pt)

419

void __dev_remove_pack(struct packet_type *pt)

420

{

420

{

421

struct list_head *head = ptype_head(pt);

421

struct list_head *head = ptype_head(pt);

422

struct packet_type *pt1;

422

struct packet_type *pt1;

423

424

spin_lock(&ptype_lock);

424

spin_lock(&ptype_lock);

425

426

list_for_each_entry(pt1, head, list) {

426

list_for_each_entry(pt1, head, list) {

427

if (pt == pt1) {

427

if (pt == pt1) {

428

list_del_rcu(&pt->list);

428

list_del_rcu(&pt->list);

429

goto out;

429

goto out;

430

}

430

}

431

}

431

}

432

433

printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);

433

printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);

434

out:

434

out:

435

spin_unlock(&ptype_lock);

435

spin_unlock(&ptype_lock);

436

}

436

}

437

EXPORT_SYMBOL(__dev_remove_pack);

437

EXPORT_SYMBOL(__dev_remove_pack);

438

439

/**

439

/**

440

* dev_remove_pack - remove packet handler

440

* dev_remove_pack - remove packet handler

441

* @pt: packet type declaration

441

* @pt: packet type declaration

442

*

442

*

443

* Remove a protocol handler that was previously added to the kernel

443

* Remove a protocol handler that was previously added to the kernel

444

* protocol handlers by dev_add_pack(). The passed &packet_type is removed

444

* protocol handlers by dev_add_pack(). The passed &packet_type is removed

445

* from the kernel lists and can be freed or reused once this function

445

* from the kernel lists and can be freed or reused once this function

446

* returns.

446

* returns.

447

*

447

*

448

* This call sleeps to guarantee that no CPU is looking at the packet

448

* This call sleeps to guarantee that no CPU is looking at the packet

449

* type after return.

449

* type after return.

450

*/

450

*/

451

void dev_remove_pack(struct packet_type *pt)

451

void dev_remove_pack(struct packet_type *pt)

452

{

452

{

453

__dev_remove_pack(pt);

453

__dev_remove_pack(pt);

454

455

synchronize_net();

455

synchronize_net();

456

}

456

}

457

EXPORT_SYMBOL(dev_remove_pack);

457

EXPORT_SYMBOL(dev_remove_pack);

458

459

/******************************************************************************

459

/******************************************************************************

460

461

Device Boot-time Settings Routines

461

Device Boot-time Settings Routines

462

463

*******************************************************************************/

463

*******************************************************************************/

464

465

/* Boot time configuration table */

465

/* Boot time configuration table */

466

static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];

466

static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];

467

468

/**

468

/**

469

* netdev_boot_setup_add - add new setup entry

469

* netdev_boot_setup_add - add new setup entry

470

* @name: name of the device

470

* @name: name of the device

471

* @map: configured settings for the device

471

* @map: configured settings for the device

472

*

472

*

473

* Adds new setup entry to the dev_boot_setup list. The function

473

* Adds new setup entry to the dev_boot_setup list. The function

474

* returns 0 on error and 1 on success. This is a generic routine to

474

* returns 0 on error and 1 on success. This is a generic routine to

475

* all netdevices.

475

* all netdevices.

476

*/

476

*/

477

static int netdev_boot_setup_add(char *name, struct ifmap *map)

477

static int netdev_boot_setup_add(char *name, struct ifmap *map)

478

{

478

{

479

struct netdev_boot_setup *s;

479

struct netdev_boot_setup *s;

480

int i;

480

int i;

481

482

s = dev_boot_setup;

482

s = dev_boot_setup;

483

for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {

483

for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {

484

if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {

484

if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {

485

memset(s[i].name, 0, sizeof(s[i].name));

485

memset(s[i].name, 0, sizeof(s[i].name));

486

strlcpy(s[i].name, name, IFNAMSIZ);

486

strlcpy(s[i].name, name, IFNAMSIZ);

487

memcpy(&s[i].map, map, sizeof(s[i].map));

487

memcpy(&s[i].map, map, sizeof(s[i].map));

488

break;

488

break;

489

}

489

}

490

}

490

}

491

492

return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;

492

return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;

493

}

493

}

494

495

/**

495

/**

496

* netdev_boot_setup_check - check boot time settings

496

* netdev_boot_setup_check - check boot time settings

497

* @dev: the netdevice

497

* @dev: the netdevice

498

*

498

*

499

* Check boot time settings for the device.

499

* Check boot time settings for the device.

500

* The found settings are set for the device to be used

500

* The found settings are set for the device to be used

501

* later in the device probing.

501

* later in the device probing.

502

* Returns 0 if no settings found, 1 if they are.

502

* Returns 0 if no settings found, 1 if they are.

503

*/

503

*/

504

int netdev_boot_setup_check(struct net_device *dev)

504

int netdev_boot_setup_check(struct net_device *dev)

505

{

505

{

506

struct netdev_boot_setup *s = dev_boot_setup;

506

struct netdev_boot_setup *s = dev_boot_setup;

507

int i;

507

int i;

508

509

for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {

509

for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {

510

if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&

510

if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&

511

!strcmp(dev->name, s[i].name)) {

511

!strcmp(dev->name, s[i].name)) {

512

dev->irq = s[i].map.irq;

512

dev->irq = s[i].map.irq;

513

dev->base_addr = s[i].map.base_addr;

513

dev->base_addr = s[i].map.base_addr;

514

dev->mem_start = s[i].map.mem_start;

514

dev->mem_start = s[i].map.mem_start;

515

dev->mem_end = s[i].map.mem_end;

515

dev->mem_end = s[i].map.mem_end;

516

return 1;

516

return 1;

517

}

517

}

518

}

518

}

519

return 0;

519

return 0;

520

}

520

}

521

EXPORT_SYMBOL(netdev_boot_setup_check);

521

EXPORT_SYMBOL(netdev_boot_setup_check);

522

523

524

/**

524

/**

525

* netdev_boot_base - get address from boot time settings

525

* netdev_boot_base - get address from boot time settings

526

* @prefix: prefix for network device

526

* @prefix: prefix for network device

527

* @unit: id for network device

527

* @unit: id for network device

528

*

528

*

529

* Check boot time settings for the base address of device.

529

* Check boot time settings for the base address of device.

530

* The found settings are set for the device to be used

530

* The found settings are set for the device to be used

531

* later in the device probing.

531

* later in the device probing.

532

* Returns 0 if no settings found.

532

* Returns 0 if no settings found.

533

*/

533

*/

534

unsigned long netdev_boot_base(const char *prefix, int unit)

534

unsigned long netdev_boot_base(const char *prefix, int unit)

535

{

535

{

536

const struct netdev_boot_setup *s = dev_boot_setup;

536

const struct netdev_boot_setup *s = dev_boot_setup;

537

char name[IFNAMSIZ];

537

char name[IFNAMSIZ];

538

int i;

538

int i;

539

540

sprintf(name, "%s%d", prefix, unit);

540

sprintf(name, "%s%d", prefix, unit);

541

542

/*

542

/*

543

* If device already registered then return base of 1

543

* If device already registered then return base of 1

544

* to indicate not to probe for this interface

544

* to indicate not to probe for this interface

545

*/

545

*/

546

if (__dev_get_by_name(&init_net, name))

546

if (__dev_get_by_name(&init_net, name))

547

return 1;

547

return 1;

548

549

for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)

549

for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)

550

if (!strcmp(name, s[i].name))

550

if (!strcmp(name, s[i].name))

551

return s[i].map.base_addr;

551

return s[i].map.base_addr;

552

return 0;

552

return 0;

553

}

553

}

554

555

/*

555

/*

556

* Saves at boot time configured settings for any netdevice.

556

* Saves at boot time configured settings for any netdevice.

557

*/

557

*/

558

int __init netdev_boot_setup(char *str)

558

int __init netdev_boot_setup(char *str)

559

{

559

{

560

int ints[5];

560

int ints[5];

561

struct ifmap map;

561

struct ifmap map;

562

563

str = get_options(str, ARRAY_SIZE(ints), ints);

563

str = get_options(str, ARRAY_SIZE(ints), ints);

564

if (!str || !*str)

564

if (!str || !*str)

565

return 0;

565

return 0;

566

567

/* Save settings */

567

/* Save settings */

568

memset(&map, 0, sizeof(map));

568

memset(&map, 0, sizeof(map));

569

if (ints[0] > 0)

569

if (ints[0] > 0)

570

map.irq = ints[1];

570

map.irq = ints[1];

571

if (ints[0] > 1)

571

if (ints[0] > 1)

572

map.base_addr = ints[2];

572

map.base_addr = ints[2];

573

if (ints[0] > 2)

573

if (ints[0] > 2)

574

map.mem_start = ints[3];

574

map.mem_start = ints[3];

575

if (ints[0] > 3)

575

if (ints[0] > 3)

576

map.mem_end = ints[4];

576

map.mem_end = ints[4];

577

578

/* Add new entry to the list */

578

/* Add new entry to the list */

579

return netdev_boot_setup_add(str, &map);

579

return netdev_boot_setup_add(str, &map);

580

}

580

}

581

582

__setup("netdev=", netdev_boot_setup);

582

__setup("netdev=", netdev_boot_setup);

583

584

/*******************************************************************************

584

/*******************************************************************************

585

586

Device Interface Subroutines

586

Device Interface Subroutines

587

588

*******************************************************************************/

588

*******************************************************************************/

589

590

/**

590

/**

591

* __dev_get_by_name - find a device by its name

591

* __dev_get_by_name - find a device by its name

592

* @net: the applicable net namespace

592

* @net: the applicable net namespace

593

* @name: name to find

593

* @name: name to find

594

*

594

*

595

* Find an interface by name. Must be called under RTNL semaphore

595

* Find an interface by name. Must be called under RTNL semaphore

596

* or @dev_base_lock. If the name is found a pointer to the device

596

* or @dev_base_lock. If the name is found a pointer to the device

597

* is returned. If the name is not found then %NULL is returned. The

597

* is returned. If the name is not found then %NULL is returned. The

598

* reference counters are not incremented so the caller must be

598

* reference counters are not incremented so the caller must be

599

* careful with locks.

599

* careful with locks.

600

*/

600

*/

601

602

struct net_device *__dev_get_by_name(struct net *net, const char *name)

602

struct net_device *__dev_get_by_name(struct net *net, const char *name)

603

{

603

{

604

struct hlist_node *p;

604

struct hlist_node *p;

605

struct net_device *dev;

605

struct net_device *dev;

606

struct hlist_head *head = dev_name_hash(net, name);

606

struct hlist_head *head = dev_name_hash(net, name);

607

608

hlist_for_each_entry(dev, p, head, name_hlist)

608

hlist_for_each_entry(dev, p, head, name_hlist)

609

if (!strncmp(dev->name, name, IFNAMSIZ))

609

if (!strncmp(dev->name, name, IFNAMSIZ))

610

return dev;

610

return dev;

611

612

return NULL;

612

return NULL;

613

}

613

}

614

EXPORT_SYMBOL(__dev_get_by_name);

614

EXPORT_SYMBOL(__dev_get_by_name);

615

616

/**

616

/**

617

* dev_get_by_name_rcu - find a device by its name

617

* dev_get_by_name_rcu - find a device by its name

618

* @net: the applicable net namespace

618

* @net: the applicable net namespace

619

* @name: name to find

619

* @name: name to find

620

*

620

*

621

* Find an interface by name.

621

* Find an interface by name.

622

* If the name is found a pointer to the device is returned.

622

* If the name is found a pointer to the device is returned.

623

* If the name is not found then %NULL is returned.

623

* If the name is not found then %NULL is returned.

624

* The reference counters are not incremented so the caller must be

624

* The reference counters are not incremented so the caller must be

625

* careful with locks. The caller must hold RCU lock.

625

* careful with locks. The caller must hold RCU lock.

626

*/

626

*/

627

628

struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)

628

struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)

629

{

629

{

630

struct hlist_node *p;

630

struct hlist_node *p;

631

struct net_device *dev;

631

struct net_device *dev;

632

struct hlist_head *head = dev_name_hash(net, name);

632

struct hlist_head *head = dev_name_hash(net, name);

633

634

hlist_for_each_entry_rcu(dev, p, head, name_hlist)

634

hlist_for_each_entry_rcu(dev, p, head, name_hlist)

635

if (!strncmp(dev->name, name, IFNAMSIZ))

635

if (!strncmp(dev->name, name, IFNAMSIZ))

636

return dev;

636

return dev;

637

638

return NULL;

638

return NULL;

639

}

639

}

640

EXPORT_SYMBOL(dev_get_by_name_rcu);

640

EXPORT_SYMBOL(dev_get_by_name_rcu);

641

642

/**

642

/**

643

* dev_get_by_name - find a device by its name

643

* dev_get_by_name - find a device by its name

644

* @net: the applicable net namespace

644

* @net: the applicable net namespace

645

* @name: name to find

645

* @name: name to find

646

*

646

*

647

* Find an interface by name. This can be called from any

647

* Find an interface by name. This can be called from any

648

* context and does its own locking. The returned handle has

648

* context and does its own locking. The returned handle has

649

* the usage count incremented and the caller must use dev_put() to

649

* the usage count incremented and the caller must use dev_put() to

650

* release it when it is no longer needed. %NULL is returned if no

650

* release it when it is no longer needed. %NULL is returned if no

651

* matching device is found.

651

* matching device is found.

652

*/

652

*/

653

654

struct net_device *dev_get_by_name(struct net *net, const char *name)

654

struct net_device *dev_get_by_name(struct net *net, const char *name)

655

{

655

{

656

struct net_device *dev;

656

struct net_device *dev;

657

658

rcu_read_lock();

658

rcu_read_lock();

659

dev = dev_get_by_name_rcu(net, name);

659

dev = dev_get_by_name_rcu(net, name);

660

if (dev)

660

if (dev)

661

dev_hold(dev);

661

dev_hold(dev);

662

rcu_read_unlock();

662

rcu_read_unlock();

663

return dev;

663

return dev;

664

}

664

}

665

EXPORT_SYMBOL(dev_get_by_name);

665

EXPORT_SYMBOL(dev_get_by_name);

666

667

/**

667

/**

668

* __dev_get_by_index - find a device by its ifindex

668

* __dev_get_by_index - find a device by its ifindex

669

* @net: the applicable net namespace

669

* @net: the applicable net namespace

670

* @ifindex: index of device

670

* @ifindex: index of device

671

*

671

*

672

* Search for an interface by index. Returns %NULL if the device

672

* Search for an interface by index. Returns %NULL if the device

673

* is not found or a pointer to the device. The device has not

673

* is not found or a pointer to the device. The device has not

674

* had its reference counter increased so the caller must be careful

674

* had its reference counter increased so the caller must be careful

675

* about locking. The caller must hold either the RTNL semaphore

675

* about locking. The caller must hold either the RTNL semaphore

676

* or @dev_base_lock.

676

* or @dev_base_lock.

677

*/

677

*/

678

679

struct net_device *__dev_get_by_index(struct net *net, int ifindex)

679

struct net_device *__dev_get_by_index(struct net *net, int ifindex)

680

{

680

{

681

struct hlist_node *p;

681

struct hlist_node *p;

682

struct net_device *dev;

682

struct net_device *dev;

683

struct hlist_head *head = dev_index_hash(net, ifindex);

683

struct hlist_head *head = dev_index_hash(net, ifindex);

684

685

hlist_for_each_entry(dev, p, head, index_hlist)

685

hlist_for_each_entry(dev, p, head, index_hlist)

686

if (dev->ifindex == ifindex)

686

if (dev->ifindex == ifindex)

687

return dev;

687

return dev;

688

689

return NULL;

689

return NULL;

690

}

690

}

691

EXPORT_SYMBOL(__dev_get_by_index);

691

EXPORT_SYMBOL(__dev_get_by_index);

692

693

/**

693

/**

694

* dev_get_by_index_rcu - find a device by its ifindex

694

* dev_get_by_index_rcu - find a device by its ifindex

695

* @net: the applicable net namespace

695

* @net: the applicable net namespace

696

* @ifindex: index of device

696

* @ifindex: index of device

697

*

697

*

698

* Search for an interface by index. Returns %NULL if the device

698

* Search for an interface by index. Returns %NULL if the device

699

* is not found or a pointer to the device. The device has not

699

* is not found or a pointer to the device. The device has not

700

* had its reference counter increased so the caller must be careful

700

* had its reference counter increased so the caller must be careful

701

* about locking. The caller must hold RCU lock.

701

* about locking. The caller must hold RCU lock.

702

*/

702

*/

703

704

struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)

704

struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)

705

{

705

{

706

struct hlist_node *p;

706

struct hlist_node *p;

707

struct net_device *dev;

707

struct net_device *dev;

708

struct hlist_head *head = dev_index_hash(net, ifindex);

708

struct hlist_head *head = dev_index_hash(net, ifindex);

709

710

hlist_for_each_entry_rcu(dev, p, head, index_hlist)

710

hlist_for_each_entry_rcu(dev, p, head, index_hlist)

711

if (dev->ifindex == ifindex)

711

if (dev->ifindex == ifindex)

712

return dev;

712

return dev;

713

714

return NULL;

714

return NULL;

715

}

715

}

716

EXPORT_SYMBOL(dev_get_by_index_rcu);

716

EXPORT_SYMBOL(dev_get_by_index_rcu);

717

718

719

/**

719

/**

720

* dev_get_by_index - find a device by its ifindex

720

* dev_get_by_index - find a device by its ifindex

721

* @net: the applicable net namespace

721

* @net: the applicable net namespace

722

* @ifindex: index of device

722

* @ifindex: index of device

723

*

723

*

724

* Search for an interface by index. Returns NULL if the device

724

* Search for an interface by index. Returns NULL if the device

725

* is not found or a pointer to the device. The device returned has

725

* is not found or a pointer to the device. The device returned has

726

* had a reference added and the pointer is safe until the user calls

726

* had a reference added and the pointer is safe until the user calls

727

* dev_put to indicate they have finished with it.

727

* dev_put to indicate they have finished with it.

728

*/

728

*/

729

730

struct net_device *dev_get_by_index(struct net *net, int ifindex)

730

struct net_device *dev_get_by_index(struct net *net, int ifindex)

731

{

731

{

732

struct net_device *dev;

732

struct net_device *dev;

733

734

rcu_read_lock();

734

rcu_read_lock();

735

dev = dev_get_by_index_rcu(net, ifindex);

735

dev = dev_get_by_index_rcu(net, ifindex);

736

if (dev)

736

if (dev)

737

dev_hold(dev);

737

dev_hold(dev);

738

rcu_read_unlock();

738

rcu_read_unlock();

739

return dev;

739

return dev;

740

}

740

}

741

EXPORT_SYMBOL(dev_get_by_index);

741

EXPORT_SYMBOL(dev_get_by_index);

742

743

/**

743

/**

744

* dev_getbyhwaddr - find a device by its hardware address

744

* dev_getbyhwaddr - find a device by its hardware address

745

* @net: the applicable net namespace

745

* @net: the applicable net namespace

746

* @type: media type of device

746

* @type: media type of device

747

* @ha: hardware address

747

* @ha: hardware address

748

*

748

*

749

* Search for an interface by MAC address. Returns NULL if the device

749

* Search for an interface by MAC address. Returns NULL if the device

750

* is not found or a pointer to the device. The caller must hold the

750

* is not found or a pointer to the device. The caller must hold the

751

* rtnl semaphore. The returned device has not had its ref count increased

751

* rtnl semaphore. The returned device has not had its ref count increased

752

* and the caller must therefore be careful about locking

752

* and the caller must therefore be careful about locking

753

*

753

*

754

* BUGS:

754

* BUGS:

755

* If the API was consistent this would be __dev_get_by_hwaddr

755

* If the API was consistent this would be __dev_get_by_hwaddr

756

*/

756

*/

757

758

struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha)

758

struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha)

759

{

759

{

760

struct net_device *dev;

760

struct net_device *dev;

761

762

ASSERT_RTNL();

762

ASSERT_RTNL();

763

764

for_each_netdev(net, dev)

764

for_each_netdev(net, dev)

765

if (dev->type == type &&

765

if (dev->type == type &&

766

!memcmp(dev->dev_addr, ha, dev->addr_len))

766

!memcmp(dev->dev_addr, ha, dev->addr_len))

767

return dev;

767

return dev;

768

769

return NULL;

769

return NULL;

770

}

770

}

771

EXPORT_SYMBOL(dev_getbyhwaddr);

771

EXPORT_SYMBOL(dev_getbyhwaddr);

772

773

struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)

773

struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)

774

{

774

{

775

struct net_device *dev;

775

struct net_device *dev;

776

777

ASSERT_RTNL();

777

ASSERT_RTNL();

778

for_each_netdev(net, dev)

778

for_each_netdev(net, dev)

779

if (dev->type == type)

779

if (dev->type == type)

780

return dev;

780

return dev;

781

782

return NULL;

782

return NULL;

783

}

783

}

784

EXPORT_SYMBOL(__dev_getfirstbyhwtype);

784

EXPORT_SYMBOL(__dev_getfirstbyhwtype);

785

786

struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)

786

struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)

787

{

787

{

788

struct net_device *dev, *ret = NULL;

788

struct net_device *dev, *ret = NULL;

789

790

rcu_read_lock();

790

rcu_read_lock();

791

for_each_netdev_rcu(net, dev)

791

for_each_netdev_rcu(net, dev)

792

if (dev->type == type) {

792

if (dev->type == type) {

793

dev_hold(dev);

793

dev_hold(dev);

794

ret = dev;

794

ret = dev;

795

break;

795

break;

796

}

796

}

797

rcu_read_unlock();

797

rcu_read_unlock();

798

return ret;

798

return ret;

799

}

799

}

800

EXPORT_SYMBOL(dev_getfirstbyhwtype);

800

EXPORT_SYMBOL(dev_getfirstbyhwtype);

801

802

/**

802

/**

803

* dev_get_by_flags_rcu - find any device with given flags

803

* dev_get_by_flags_rcu - find any device with given flags

804

* @net: the applicable net namespace

804

* @net: the applicable net namespace

805

* @if_flags: IFF_* values

805

* @if_flags: IFF_* values

806

* @mask: bitmask of bits in if_flags to check

806

* @mask: bitmask of bits in if_flags to check

807

*

807

*

808

* Search for any interface with the given flags. Returns NULL if a device

808

* Search for any interface with the given flags. Returns NULL if a device

809

* is not found or a pointer to the device. Must be called inside

809

* is not found or a pointer to the device. Must be called inside

810

* rcu_read_lock(), and result refcount is unchanged.

810

* rcu_read_lock(), and result refcount is unchanged.

811

*/

811

*/

812

813

struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,

813

struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,

814

unsigned short mask)

814

unsigned short mask)

815

{

815

{

816

struct net_device *dev, *ret;

816

struct net_device *dev, *ret;

817

818

ret = NULL;

818

ret = NULL;

819

for_each_netdev_rcu(net, dev) {

819

for_each_netdev_rcu(net, dev) {

820

if (((dev->flags ^ if_flags) & mask) == 0) {

820

if (((dev->flags ^ if_flags) & mask) == 0) {

821

ret = dev;

821

ret = dev;

822

break;

822

break;

823

}

823

}

824

}

824

}

825

return ret;

825

return ret;

826

}

826

}

827

EXPORT_SYMBOL(dev_get_by_flags_rcu);

827

EXPORT_SYMBOL(dev_get_by_flags_rcu);

828

829

/**

829

/**

830

* dev_valid_name - check if name is okay for network device

830

* dev_valid_name - check if name is okay for network device

831

* @name: name string

831

* @name: name string

832

*

832

*

833

* Network device names need to be valid file names to

833

* Network device names need to be valid file names to

834

* to allow sysfs to work. We also disallow any kind of

834

* to allow sysfs to work. We also disallow any kind of

835

* whitespace.

835

* whitespace.

836

*/

836

*/

837

int dev_valid_name(const char *name)

837

int dev_valid_name(const char *name)

838

{

838

{

839

if (*name == '\0')

839

if (*name == '\0')

840

return 0;

840

return 0;

841

if (strlen(name) >= IFNAMSIZ)

841

if (strlen(name) >= IFNAMSIZ)

842

return 0;

842

return 0;

843

if (!strcmp(name, ".") || !strcmp(name, ".."))

843

if (!strcmp(name, ".") || !strcmp(name, ".."))

844

return 0;

844

return 0;

845

846

while (*name) {

846

while (*name) {

847

if (*name == '/' || isspace(*name))

847

if (*name == '/' || isspace(*name))

848

return 0;

848

return 0;

849

name++;

849

name++;

850

}

850

}

851

return 1;

851

return 1;

852

}

852

}

853

EXPORT_SYMBOL(dev_valid_name);

853

EXPORT_SYMBOL(dev_valid_name);

854

855

/**

855

/**

856

* __dev_alloc_name - allocate a name for a device

856

* __dev_alloc_name - allocate a name for a device

857

* @net: network namespace to allocate the device name in

857

* @net: network namespace to allocate the device name in

858

* @name: name format string

858

* @name: name format string

859

* @buf: scratch buffer and result name string

859

* @buf: scratch buffer and result name string

860

*

860

*

861

* Passed a format string - eg "lt%d" it will try and find a suitable

861

* Passed a format string - eg "lt%d" it will try and find a suitable

862

* id. It scans list of devices to build up a free map, then chooses

862

* id. It scans list of devices to build up a free map, then chooses

863

* the first empty slot. The caller must hold the dev_base or rtnl lock

863

* the first empty slot. The caller must hold the dev_base or rtnl lock

864

* while allocating the name and adding the device in order to avoid

864

* while allocating the name and adding the device in order to avoid

865

* duplicates.

865

* duplicates.

866

* Limited to bits_per_byte * page size devices (ie 32K on most platforms).

866

* Limited to bits_per_byte * page size devices (ie 32K on most platforms).

867

* Returns the number of the unit assigned or a negative errno code.

867

* Returns the number of the unit assigned or a negative errno code.

868

*/

868

*/

869

870

static int __dev_alloc_name(struct net *net, const char *name, char *buf)

870

static int __dev_alloc_name(struct net *net, const char *name, char *buf)

871

{

871

{

872

int i = 0;

872

int i = 0;

873

const char *p;

873

const char *p;

874

const int max_netdevices = 8*PAGE_SIZE;

874

const int max_netdevices = 8*PAGE_SIZE;

875

unsigned long *inuse;

875

unsigned long *inuse;

876

struct net_device *d;

876

struct net_device *d;

877

878

p = strnchr(name, IFNAMSIZ-1, '%');

878

p = strnchr(name, IFNAMSIZ-1, '%');

879

if (p) {

879

if (p) {

880

/*

880

/*

881

* Verify the string as this thing may have come from

881

* Verify the string as this thing may have come from

882

* the user. There must be either one "%d" and no other "%"

882

* the user. There must be either one "%d" and no other "%"

883

* characters.

883

* characters.

884

*/

884

*/

885

if (p[1] != 'd' || strchr(p + 2, '%'))

885

if (p[1] != 'd' || strchr(p + 2, '%'))

886

return -EINVAL;

886

return -EINVAL;

887

888

/* Use one page as a bit array of possible slots */

888

/* Use one page as a bit array of possible slots */

889

inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);

889

inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);

890

if (!inuse)

890

if (!inuse)

891

return -ENOMEM;

891

return -ENOMEM;

892

893

for_each_netdev(net, d) {

893

for_each_netdev(net, d) {

894

if (!sscanf(d->name, name, &i))

894

if (!sscanf(d->name, name, &i))

895

continue;

895

continue;

896

if (i < 0 || i >= max_netdevices)

896

if (i < 0 || i >= max_netdevices)

897

continue;

897

continue;

898

899

/* avoid cases where sscanf is not exact inverse of printf */

899

/* avoid cases where sscanf is not exact inverse of printf */

900

snprintf(buf, IFNAMSIZ, name, i);

900

snprintf(buf, IFNAMSIZ, name, i);

901

if (!strncmp(buf, d->name, IFNAMSIZ))

901

if (!strncmp(buf, d->name, IFNAMSIZ))

902

set_bit(i, inuse);

902

set_bit(i, inuse);

903

}

903

}

904

905

i = find_first_zero_bit(inuse, max_netdevices);

905

i = find_first_zero_bit(inuse, max_netdevices);

906

free_page((unsigned long) inuse);

906

free_page((unsigned long) inuse);

907

}

907

}

908

909

if (buf != name)

909

if (buf != name)

910

snprintf(buf, IFNAMSIZ, name, i);

910

snprintf(buf, IFNAMSIZ, name, i);

911

if (!__dev_get_by_name(net, buf))

911

if (!__dev_get_by_name(net, buf))

912

return i;

912

return i;

913

914

/* It is possible to run out of possible slots

914

/* It is possible to run out of possible slots

915

* when the name is long and there isn't enough space left

915

* when the name is long and there isn't enough space left

916

* for the digits, or if all bits are used.

916

* for the digits, or if all bits are used.

917

*/

917

*/

918

return -ENFILE;

918

return -ENFILE;

919

}

919

}

920

921

/**

921

/**

922

* dev_alloc_name - allocate a name for a device

922

* dev_alloc_name - allocate a name for a device

923

* @dev: device

923

* @dev: device

924

* @name: name format string

924

* @name: name format string

925

*

925

*

926

* Passed a format string - eg "lt%d" it will try and find a suitable

926

* Passed a format string - eg "lt%d" it will try and find a suitable

927

* id. It scans list of devices to build up a free map, then chooses

927

* id. It scans list of devices to build up a free map, then chooses

928

* the first empty slot. The caller must hold the dev_base or rtnl lock

928

* the first empty slot. The caller must hold the dev_base or rtnl lock

929

* while allocating the name and adding the device in order to avoid

929

* while allocating the name and adding the device in order to avoid

930

* duplicates.

930

* duplicates.

931

* Limited to bits_per_byte * page size devices (ie 32K on most platforms).

931

* Limited to bits_per_byte * page size devices (ie 32K on most platforms).

932

* Returns the number of the unit assigned or a negative errno code.

932

* Returns the number of the unit assigned or a negative errno code.

933

*/

933

*/

934

935

int dev_alloc_name(struct net_device *dev, const char *name)

935

int dev_alloc_name(struct net_device *dev, const char *name)

936

{

936

{

937

char buf[IFNAMSIZ];

937

char buf[IFNAMSIZ];

938

struct net *net;

938

struct net *net;

939

int ret;

939

int ret;

940

941

BUG_ON(!dev_net(dev));

941

BUG_ON(!dev_net(dev));

942

net = dev_net(dev);

942

net = dev_net(dev);

943

ret = __dev_alloc_name(net, name, buf);

943

ret = __dev_alloc_name(net, name, buf);

944

if (ret >= 0)

944

if (ret >= 0)

945

strlcpy(dev->name, buf, IFNAMSIZ);

945

strlcpy(dev->name, buf, IFNAMSIZ);

946

return ret;

946

return ret;

947

}

947

}

948

EXPORT_SYMBOL(dev_alloc_name);

948

EXPORT_SYMBOL(dev_alloc_name);

949

950

static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)

950

static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)

951

{

951

{

952

struct net *net;

952

struct net *net;

953

954

BUG_ON(!dev_net(dev));

954

BUG_ON(!dev_net(dev));

955

net = dev_net(dev);

955

net = dev_net(dev);

956

957

if (!dev_valid_name(name))

957

if (!dev_valid_name(name))

958

return -EINVAL;

958

return -EINVAL;

959

960

if (fmt && strchr(name, '%'))

960

if (fmt && strchr(name, '%'))

961

return dev_alloc_name(dev, name);

961

return dev_alloc_name(dev, name);

962

else if (__dev_get_by_name(net, name))

962

else if (__dev_get_by_name(net, name))

963

return -EEXIST;

963

return -EEXIST;

964

else if (dev->name != name)

964

else if (dev->name != name)

965

strlcpy(dev->name, name, IFNAMSIZ);

965

strlcpy(dev->name, name, IFNAMSIZ);

966

967

return 0;

967

return 0;

968

}

968

}

969

970

/**

970

/**

971

* dev_change_name - change name of a device

971

* dev_change_name - change name of a device

972

* @dev: device

972

* @dev: device

973

* @newname: name (or format string) must be at least IFNAMSIZ

973

* @newname: name (or format string) must be at least IFNAMSIZ

974

*

974

*

975

* Change name of a device, can pass format strings "eth%d".

975

* Change name of a device, can pass format strings "eth%d".

976

* for wildcarding.

976

* for wildcarding.

977

*/

977

*/

978

int dev_change_name(struct net_device *dev, const char *newname)

978

int dev_change_name(struct net_device *dev, const char *newname)

979

{

979

{

980

char oldname[IFNAMSIZ];

980

char oldname[IFNAMSIZ];

981

int err = 0;

981

int err = 0;

982

int ret;

982

int ret;

983

struct net *net;

983

struct net *net;

984

985

ASSERT_RTNL();

985

ASSERT_RTNL();

986

BUG_ON(!dev_net(dev));

986

BUG_ON(!dev_net(dev));

987

988

net = dev_net(dev);

988

net = dev_net(dev);

989

if (dev->flags & IFF_UP)

989

if (dev->flags & IFF_UP)

990

return -EBUSY;

990

return -EBUSY;

991

992

if (strncmp(newname, dev->name, IFNAMSIZ) == 0)

992

if (strncmp(newname, dev->name, IFNAMSIZ) == 0)

993

return 0;

993

return 0;

994

995

memcpy(oldname, dev->name, IFNAMSIZ);

995

memcpy(oldname, dev->name, IFNAMSIZ);

996

997

err = dev_get_valid_name(dev, newname, 1);

997

err = dev_get_valid_name(dev, newname, 1);

998

if (err < 0)

998

if (err < 0)

999

return err;

999

return err;

1000

1001

rollback:

1001

rollback:

1002

ret = device_rename(&dev->dev, dev->name);

1002

ret = device_rename(&dev->dev, dev->name);

1003

if (ret) {

1003

if (ret) {

1004

memcpy(dev->name, oldname, IFNAMSIZ);

1004

memcpy(dev->name, oldname, IFNAMSIZ);

1005

return ret;

1005

return ret;

1006

}

1006

}

1007

1008

write_lock_bh(&dev_base_lock);

1008

write_lock_bh(&dev_base_lock);

1009

hlist_del(&dev->name_hlist);

1009

hlist_del(&dev->name_hlist);

1010

write_unlock_bh(&dev_base_lock);

1010

write_unlock_bh(&dev_base_lock);

1011

1012

synchronize_rcu();

1012

synchronize_rcu();

1013

1014

write_lock_bh(&dev_base_lock);

1014

write_lock_bh(&dev_base_lock);

1015

hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));

1015

hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));

1016

write_unlock_bh(&dev_base_lock);

1016

write_unlock_bh(&dev_base_lock);

1017

1018

ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);

1018

ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);

1019

ret = notifier_to_errno(ret);

1019

ret = notifier_to_errno(ret);

1020

1021

if (ret) {

1021

if (ret) {

1022

/* err >= 0 after dev_alloc_name() or stores the first errno */

1022

/* err >= 0 after dev_alloc_name() or stores the first errno */

1023

if (err >= 0) {

1023

if (err >= 0) {

1024

err = ret;

1024

err = ret;

1025

memcpy(dev->name, oldname, IFNAMSIZ);

1025

memcpy(dev->name, oldname, IFNAMSIZ);

1026

goto rollback;

1026

goto rollback;

1027

} else {

1027

} else {

1028

printk(KERN_ERR

1028

printk(KERN_ERR

1029

"%s: name change rollback failed: %d.\n",

1029

"%s: name change rollback failed: %d.\n",

1030

dev->name, ret);

1030

dev->name, ret);

1031

}

1031

}

1032

}

1032

}

1033

1034

return err;

1034

return err;

1035

}

1035

}

1036

1037

/**

1037

/**

1038

* dev_set_alias - change ifalias of a device

1038

* dev_set_alias - change ifalias of a device

1039

* @dev: device

1039

* @dev: device

1040

* @alias: name up to IFALIASZ

1040

* @alias: name up to IFALIASZ

1041

* @len: limit of bytes to copy from info

1041

* @len: limit of bytes to copy from info

1042

*

1042

*

1043

* Set ifalias for a device,

1043

* Set ifalias for a device,

1044

*/

1044

*/

1045

int dev_set_alias(struct net_device *dev, const char *alias, size_t len)

1045

int dev_set_alias(struct net_device *dev, const char *alias, size_t len)

1046

{

1046

{

1047

ASSERT_RTNL();

1047

ASSERT_RTNL();

1048

1049

if (len >= IFALIASZ)

1049

if (len >= IFALIASZ)

1050

return -EINVAL;

1050

return -EINVAL;

1051

1052

if (!len) {

1052

if (!len) {

1053

if (dev->ifalias) {

1053

if (dev->ifalias) {

1054

kfree(dev->ifalias);

1054

kfree(dev->ifalias);

1055

dev->ifalias = NULL;

1055

dev->ifalias = NULL;

1056

}

1056

}

1057

return 0;

1057

return 0;

1058

}

1058

}

1059

1060

dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);

1060

dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);

1061

if (!dev->ifalias)

1061

if (!dev->ifalias)

1062

return -ENOMEM;

1062

return -ENOMEM;

1063

1064

strlcpy(dev->ifalias, alias, len+1);

1064

strlcpy(dev->ifalias, alias, len+1);

1065

return len;

1065

return len;

1066

}

1066

}

1067

1068

1069

/**

1069

/**

1070

* netdev_features_change - device changes features

1070

* netdev_features_change - device changes features

1071

* @dev: device to cause notification

1071

* @dev: device to cause notification

1072

*

1072

*

1073

* Called to indicate a device has changed features.

1073

* Called to indicate a device has changed features.

1074

*/

1074

*/

1075

void netdev_features_change(struct net_device *dev)

1075

void netdev_features_change(struct net_device *dev)

1076

{

1076

{

1077

call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);

1077

call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);

1078

}

1078

}

1079

EXPORT_SYMBOL(netdev_features_change);

1079

EXPORT_SYMBOL(netdev_features_change);

1080

1081

/**

1081

/**

1082

* netdev_state_change - device changes state

1082

* netdev_state_change - device changes state

1083

* @dev: device to cause notification

1083

* @dev: device to cause notification

1084

*

1084

*

1085

* Called to indicate a device has changed state. This function calls

1085

* Called to indicate a device has changed state. This function calls

1086

* the notifier chains for netdev_chain and sends a NEWLINK message

1086

* the notifier chains for netdev_chain and sends a NEWLINK message

1087

* to the routing socket.

1087

* to the routing socket.

1088

*/

1088

*/

1089

void netdev_state_change(struct net_device *dev)

1089

void netdev_state_change(struct net_device *dev)

1090

{

1090

{

1091

if (dev->flags & IFF_UP) {

1091

if (dev->flags & IFF_UP) {

1092

call_netdevice_notifiers(NETDEV_CHANGE, dev);

1092

call_netdevice_notifiers(NETDEV_CHANGE, dev);

1093

rtmsg_ifinfo(RTM_NEWLINK, dev, 0);

1093

rtmsg_ifinfo(RTM_NEWLINK, dev, 0);

1094

}

1094

}

1095

}

1095

}

1096

EXPORT_SYMBOL(netdev_state_change);

1096

EXPORT_SYMBOL(netdev_state_change);

1097

1098

int netdev_bonding_change(struct net_device *dev, unsigned long event)

1098

int netdev_bonding_change(struct net_device *dev, unsigned long event)

1099

{

1099

{

1100

return call_netdevice_notifiers(event, dev);

1100

return call_netdevice_notifiers(event, dev);

1101

}

1101

}

1102

EXPORT_SYMBOL(netdev_bonding_change);

1102

EXPORT_SYMBOL(netdev_bonding_change);

1103

1104

/**

1104

/**

1105

* dev_load - load a network module

1105

* dev_load - load a network module

1106

* @net: the applicable net namespace

1106

* @net: the applicable net namespace

1107

* @name: name of interface

1107

* @name: name of interface

1108

*

1108

*

1109

* If a network interface is not present and the process has suitable

1109

* If a network interface is not present and the process has suitable

1110

* privileges this function loads the module. If module loading is not

1110

* privileges this function loads the module. If module loading is not

1111

* available in this kernel then it becomes a nop.

1111

* available in this kernel then it becomes a nop.

1112

*/

1112

*/

1113

1114

void dev_load(struct net *net, const char *name)

1114

void dev_load(struct net *net, const char *name)

1115

{

1115

{

1116

struct net_device *dev;

1116

struct net_device *dev;

1117

1118

rcu_read_lock();

1118

rcu_read_lock();

1119

dev = dev_get_by_name_rcu(net, name);

1119

dev = dev_get_by_name_rcu(net, name);

1120

rcu_read_unlock();

1120

rcu_read_unlock();

1121

1122

if (!dev && capable(CAP_NET_ADMIN))

1122

if (!dev && capable(CAP_NET_ADMIN))

1123

request_module("%s", name);

1123

request_module("%s", name);

1124

}

1124

}

1125

EXPORT_SYMBOL(dev_load);

1125

EXPORT_SYMBOL(dev_load);

1126

1127

static int __dev_open(struct net_device *dev)

1127

static int __dev_open(struct net_device *dev)

1128

{

1128

{

1129

const struct net_device_ops *ops = dev->netdev_ops;

1129

const struct net_device_ops *ops = dev->netdev_ops;

1130

int ret;

1130

int ret;

1131

1132

ASSERT_RTNL();

1132

ASSERT_RTNL();

1133

1134

/*

1134

/*

1135

* Is it even present?

1135

* Is it even present?

1136

*/

1136

*/

1137

if (!netif_device_present(dev))

1137

if (!netif_device_present(dev))

1138

return -ENODEV;

1138

return -ENODEV;

1139

1140

ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);

1140

ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);

1141

ret = notifier_to_errno(ret);

1141

ret = notifier_to_errno(ret);

1142

if (ret)

1142

if (ret)

1143

return ret;

1143

return ret;

1144

1145

/*

1145

/*

1146

* Call device private open method

1146

* Call device private open method

1147

*/

1147

*/

1148

set_bit(__LINK_STATE_START, &dev->state);

1148

set_bit(__LINK_STATE_START, &dev->state);

1149

1150

if (ops->ndo_validate_addr)

1150

if (ops->ndo_validate_addr)

1151

ret = ops->ndo_validate_addr(dev);

1151

ret = ops->ndo_validate_addr(dev);

1152

1153

if (!ret && ops->ndo_open)

1153

if (!ret && ops->ndo_open)

1154

ret = ops->ndo_open(dev);

1154

ret = ops->ndo_open(dev);

1155

1156

/*

1156

/*

1157

* If it went open OK then:

1157

* If it went open OK then:

1158

*/

1158

*/

1159

1160

if (ret)

1160

if (ret)

1161

clear_bit(__LINK_STATE_START, &dev->state);

1161

clear_bit(__LINK_STATE_START, &dev->state);

1162

else {

1162

else {

1163

/*

1163

/*

1164

* Set the flags.

1164

* Set the flags.

1165

*/

1165

*/

1166

dev->flags |= IFF_UP;

1166

dev->flags |= IFF_UP;

1167

1168

/*

1168

/*

1169

* Enable NET_DMA

1169

* Enable NET_DMA

1170

*/

1170

*/

1171

net_dmaengine_get();

1171

net_dmaengine_get();

1172

1173

/*

1173

/*

1174

* Initialize multicasting status

1174

* Initialize multicasting status

1175

*/

1175

*/

1176

dev_set_rx_mode(dev);

1176

dev_set_rx_mode(dev);

1177

1178

/*

1178

/*

1179

* Wakeup transmit queue engine

1179

* Wakeup transmit queue engine

1180

*/

1180

*/

1181

dev_activate(dev);

1181

dev_activate(dev);

1182

}

1182

}

1183

1184

return ret;

1184

return ret;

1185

}

1185

}

1186

1187

/**

1187

/**

1188

* dev_open - prepare an interface for use.

1188

* dev_open - prepare an interface for use.

1189

* @dev: device to open

1189

* @dev: device to open

1190

*

1190

*

1191

* Takes a device from down to up state. The device's private open

1191

* Takes a device from down to up state. The device's private open

1192

* function is invoked and then the multicast lists are loaded. Finally

1192

* function is invoked and then the multicast lists are loaded. Finally

1193

* the device is moved into the up state and a %NETDEV_UP message is

1193

* the device is moved into the up state and a %NETDEV_UP message is

1194

* sent to the netdev notifier chain.

1194

* sent to the netdev notifier chain.

1195

*

1195

*

1196

* Calling this function on an active interface is a nop. On a failure

1196

* Calling this function on an active interface is a nop. On a failure

1197

* a negative errno code is returned.

1197

* a negative errno code is returned.

1198

*/

1198

*/

1199

int dev_open(struct net_device *dev)

1199

int dev_open(struct net_device *dev)

1200

{

1200

{

1201

int ret;

1201

int ret;

1202

1203

/*

1203

/*

1204

* Is it already up?

1204

* Is it already up?

1205

*/

1205

*/

1206

if (dev->flags & IFF_UP)

1206

if (dev->flags & IFF_UP)

1207

return 0;

1207

return 0;

1208

1209

/*

1209

/*

1210

* Open device

1210

* Open device

1211

*/

1211

*/

1212

ret = __dev_open(dev);

1212

ret = __dev_open(dev);

1213

if (ret < 0)

1213

if (ret < 0)

1214

return ret;

1214

return ret;

1215

1216

/*

1216

/*

1217

* ... and announce new interface.

1217

* ... and announce new interface.

1218

*/

1218

*/

1219

rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);

1219

rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);

1220

call_netdevice_notifiers(NETDEV_UP, dev);

1220

call_netdevice_notifiers(NETDEV_UP, dev);

1221

1222

return ret;

1222

return ret;

1223

}

1223

}

1224

EXPORT_SYMBOL(dev_open);

1224

EXPORT_SYMBOL(dev_open);

1225

1226

static int __dev_close(struct net_device *dev)

1226

static int __dev_close(struct net_device *dev)

1227

{

1227

{

1228

const struct net_device_ops *ops = dev->netdev_ops;

1228

const struct net_device_ops *ops = dev->netdev_ops;

1229

1230

ASSERT_RTNL();

1230

ASSERT_RTNL();

1231

might_sleep();

1231

might_sleep();

1232

1233

/*

1233

/*

1234

* Tell people we are going down, so that they can

1234

* Tell people we are going down, so that they can

1235

* prepare to death, when device is still operating.

1235

* prepare to death, when device is still operating.

1236

*/

1236

*/

1237

call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);

1237

call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);

1238

1239

clear_bit(__LINK_STATE_START, &dev->state);

1239

clear_bit(__LINK_STATE_START, &dev->state);

1240

1241

/* Synchronize to scheduled poll. We cannot touch poll list,

1241

/* Synchronize to scheduled poll. We cannot touch poll list,

1242

* it can be even on different cpu. So just clear netif_running().

1242

* it can be even on different cpu. So just clear netif_running().

1243

*

1243

*

1244

* dev->stop() will invoke napi_disable() on all of it's

1244

* dev->stop() will invoke napi_disable() on all of it's

1245

* napi_struct instances on this device.

1245

* napi_struct instances on this device.

1246

*/

1246

*/

1247

smp_mb__after_clear_bit(); /* Commit netif_running(). */

1247

smp_mb__after_clear_bit(); /* Commit netif_running(). */

1248

1249

dev_deactivate(dev);

1249

dev_deactivate(dev);

1250

1251

/*

1251

/*

1252

* Call the device specific close. This cannot fail.

1252

* Call the device specific close. This cannot fail.

1253

* Only if device is UP

1253

* Only if device is UP

1254

*

1254

*

1255

* We allow it to be called even after a DETACH hot-plug

1255

* We allow it to be called even after a DETACH hot-plug

1256

* event.

1256

* event.

1257

*/

1257

*/

1258

if (ops->ndo_stop)

1258

if (ops->ndo_stop)

1259

ops->ndo_stop(dev);

1259

ops->ndo_stop(dev);

1260

1261

/*

1261

/*

1262

* Device is now down.

1262

* Device is now down.

1263

*/

1263

*/

1264

1265

dev->flags &= ~IFF_UP;

1265

dev->flags &= ~IFF_UP;

1266

1267

/*

1267

/*

1268

* Shutdown NET_DMA

1268

* Shutdown NET_DMA

1269

*/

1269

*/

1270

net_dmaengine_put();

1270

net_dmaengine_put();

1271

1272

return 0;

1272

return 0;

1273

}

1273

}

1274

1275

/**

1275

/**

1276

* dev_close - shutdown an interface.

1276

* dev_close - shutdown an interface.

1277

* @dev: device to shutdown

1277

* @dev: device to shutdown

1278

*

1278

*

1279

* This function moves an active device into down state. A

1279

* This function moves an active device into down state. A

1280

* %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device

1280

* %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device

1281

* is then deactivated and finally a %NETDEV_DOWN is sent to the notifier

1281

* is then deactivated and finally a %NETDEV_DOWN is sent to the notifier

1282

* chain.

1282

* chain.

1283

*/

1283

*/

1284

int dev_close(struct net_device *dev)

1284

int dev_close(struct net_device *dev)

1285

{

1285

{

1286

if (!(dev->flags & IFF_UP))

1286

if (!(dev->flags & IFF_UP))

1287

return 0;

1287

return 0;

1288

1289

__dev_close(dev);

1289

__dev_close(dev);

1290

1291

/*

1291

/*

1292

* Tell people we are down

1292

* Tell people we are down

1293

*/

1293

*/

1294

rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);

1294

rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);

1295

call_netdevice_notifiers(NETDEV_DOWN, dev);

1295

call_netdevice_notifiers(NETDEV_DOWN, dev);

1296

1297

return 0;

1297

return 0;

1298

}

1298

}

1299

EXPORT_SYMBOL(dev_close);

1299

EXPORT_SYMBOL(dev_close);

1300

1301

1302

/**

1302

/**

1303

* dev_disable_lro - disable Large Receive Offload on a device

1303

* dev_disable_lro - disable Large Receive Offload on a device

1304

* @dev: device

1304

* @dev: device

1305

*

1305

*

1306

* Disable Large Receive Offload (LRO) on a net device. Must be

1306

* Disable Large Receive Offload (LRO) on a net device. Must be

1307

* called under RTNL. This is needed if received packets may be

1307

* called under RTNL. This is needed if received packets may be

1308

* forwarded to another interface.

1308

* forwarded to another interface.

1309

*/

1309

*/

1310

void dev_disable_lro(struct net_device *dev)

1310

void dev_disable_lro(struct net_device *dev)

1311

{

1311

{

1312

if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&

1312

if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&

1313

dev->ethtool_ops->set_flags) {

1313

dev->ethtool_ops->set_flags) {

1314

u32 flags = dev->ethtool_ops->get_flags(dev);

1314

u32 flags = dev->ethtool_ops->get_flags(dev);

1315

if (flags & ETH_FLAG_LRO) {

1315

if (flags & ETH_FLAG_LRO) {

1316

flags &= ~ETH_FLAG_LRO;

1316

flags &= ~ETH_FLAG_LRO;

1317

dev->ethtool_ops->set_flags(dev, flags);

1317

dev->ethtool_ops->set_flags(dev, flags);

1318

}

1318

}

1319

}

1319

}

1320

WARN_ON(dev->features & NETIF_F_LRO);

1320

WARN_ON(dev->features & NETIF_F_LRO);

1321

}

1321

}

1322

EXPORT_SYMBOL(dev_disable_lro);

1322

EXPORT_SYMBOL(dev_disable_lro);

1323

1324

1325

static int dev_boot_phase = 1;

1325

static int dev_boot_phase = 1;

1326

1327

/*

1327

/*

1328

* Device change register/unregister. These are not inline or static

1328

* Device change register/unregister. These are not inline or static

1329

* as we export them to the world.

1329

* as we export them to the world.

1330

*/

1330

*/

1331

1332

/**

1332

/**

1333

* register_netdevice_notifier - register a network notifier block

1333

* register_netdevice_notifier - register a network notifier block

1334

* @nb: notifier

1334

* @nb: notifier

1335

*

1335

*

1336

* Register a notifier to be called when network device events occur.

1336

* Register a notifier to be called when network device events occur.

1337

* The notifier passed is linked into the kernel structures and must

1337

* The notifier passed is linked into the kernel structures and must

1338

* not be reused until it has been unregistered. A negative errno code

1338

* not be reused until it has been unregistered. A negative errno code

1339

* is returned on a failure.

1339

* is returned on a failure.

1340

*

1340

*

1341

* When registered all registration and up events are replayed

1341

* When registered all registration and up events are replayed

1342

* to the new notifier to allow device to have a race free

1342

* to the new notifier to allow device to have a race free

1343

* view of the network device list.

1343

* view of the network device list.

1344

*/

1344

*/

1345

1346

int register_netdevice_notifier(struct notifier_block *nb)

1346

int register_netdevice_notifier(struct notifier_block *nb)

1347

{

1347

{

1348

struct net_device *dev;

1348

struct net_device *dev;

1349

struct net_device *last;

1349

struct net_device *last;

1350

struct net *net;

1350

struct net *net;

1351

int err;

1351

int err;

1352

1353

rtnl_lock();

1353

rtnl_lock();

1354

err = raw_notifier_chain_register(&netdev_chain, nb);

1354

err = raw_notifier_chain_register(&netdev_chain, nb);

1355

if (err)

1355

if (err)

1356

goto unlock;

1356

goto unlock;

1357

if (dev_boot_phase)

1357

if (dev_boot_phase)

1358

goto unlock;

1358

goto unlock;

1359

for_each_net(net) {

1359

for_each_net(net) {

1360

for_each_netdev(net, dev) {

1360

for_each_netdev(net, dev) {

1361

err = nb->notifier_call(nb, NETDEV_REGISTER, dev);

1361

err = nb->notifier_call(nb, NETDEV_REGISTER, dev);

1362

err = notifier_to_errno(err);

1362

err = notifier_to_errno(err);

1363

if (err)

1363

if (err)

1364

goto rollback;

1364

goto rollback;

1365

1366

if (!(dev->flags & IFF_UP))

1366

if (!(dev->flags & IFF_UP))

1367

continue;

1367

continue;

1368

1369

nb->notifier_call(nb, NETDEV_UP, dev);

1369

nb->notifier_call(nb, NETDEV_UP, dev);

1370

}

1370

}

1371

}

1371

}

1372

1373

unlock:

1373

unlock:

1374

rtnl_unlock();

1374

rtnl_unlock();

1375

return err;

1375

return err;

1376

1377

rollback:

1377

rollback:

1378

last = dev;

1378

last = dev;

1379

for_each_net(net) {

1379

for_each_net(net) {

1380

for_each_netdev(net, dev) {

1380

for_each_netdev(net, dev) {

1381

if (dev == last)

1381

if (dev == last)

1382

break;

1382

break;

1383

1384

if (dev->flags & IFF_UP) {

1384

if (dev->flags & IFF_UP) {

1385

nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);

1385

nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);

1386

nb->notifier_call(nb, NETDEV_DOWN, dev);

1386

nb->notifier_call(nb, NETDEV_DOWN, dev);

1387

}

1387

}

1388

nb->notifier_call(nb, NETDEV_UNREGISTER, dev);

1388

nb->notifier_call(nb, NETDEV_UNREGISTER, dev);

1389

nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);

1389

nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);

1390

}

1390

}

1391

}

1391

}

1392

1393

raw_notifier_chain_unregister(&netdev_chain, nb);

1393

raw_notifier_chain_unregister(&netdev_chain, nb);

1394

goto unlock;

1394

goto unlock;

1395

}

1395

}

1396

EXPORT_SYMBOL(register_netdevice_notifier);

1396

EXPORT_SYMBOL(register_netdevice_notifier);

1397

1398

/**

1398

/**

1399

* unregister_netdevice_notifier - unregister a network notifier block

1399

* unregister_netdevice_notifier - unregister a network notifier block

1400

* @nb: notifier

1400

* @nb: notifier

1401

*

1401

*

1402

* Unregister a notifier previously registered by

1402

* Unregister a notifier previously registered by

1403

* register_netdevice_notifier(). The notifier is unlinked into the

1403

* register_netdevice_notifier(). The notifier is unlinked into the

1404

* kernel structures and may then be reused. A negative errno code

1404

* kernel structures and may then be reused. A negative errno code

1405

* is returned on a failure.

1405

* is returned on a failure.

1406

*/

1406

*/

1407

1408

int unregister_netdevice_notifier(struct notifier_block *nb)

1408

int unregister_netdevice_notifier(struct notifier_block *nb)

1409

{

1409

{

1410

int err;

1410

int err;

1411

1412

rtnl_lock();

1412

rtnl_lock();

1413

err = raw_notifier_chain_unregister(&netdev_chain, nb);

1413

err = raw_notifier_chain_unregister(&netdev_chain, nb);

1414

rtnl_unlock();

1414

rtnl_unlock();

1415

return err;

1415

return err;

1416

}

1416

}

1417

EXPORT_SYMBOL(unregister_netdevice_notifier);

1417

EXPORT_SYMBOL(unregister_netdevice_notifier);

1418

1419

/**

1419

/**

1420

* call_netdevice_notifiers - call all network notifier blocks

1420

* call_netdevice_notifiers - call all network notifier blocks

1421

* @val: value passed unmodified to notifier function

1421

* @val: value passed unmodified to notifier function

1422

* @dev: net_device pointer passed unmodified to notifier function

1422

* @dev: net_device pointer passed unmodified to notifier function

1423

*

1423

*

1424

* Call all network notifier blocks. Parameters and return value

1424

* Call all network notifier blocks. Parameters and return value

1425

* are as for raw_notifier_call_chain().

1425

* are as for raw_notifier_call_chain().

1426

*/

1426

*/

1427

1428

int call_netdevice_notifiers(unsigned long val, struct net_device *dev)

1428

int call_netdevice_notifiers(unsigned long val, struct net_device *dev)

1429

{

1429

{

1430

ASSERT_RTNL();

1430

ASSERT_RTNL();

1431

return raw_notifier_call_chain(&netdev_chain, val, dev);

1431

return raw_notifier_call_chain(&netdev_chain, val, dev);

1432

}

1432

}

1433

1434

/* When > 0 there are consumers of rx skb time stamps */

1434

/* When > 0 there are consumers of rx skb time stamps */

1435

static atomic_t netstamp_needed = ATOMIC_INIT(0);

1435

static atomic_t netstamp_needed = ATOMIC_INIT(0);

1436

1437

void net_enable_timestamp(void)

1437

void net_enable_timestamp(void)

1438

{

1438

{

1439

atomic_inc(&netstamp_needed);

1439

atomic_inc(&netstamp_needed);

1440

}

1440

}

1441

EXPORT_SYMBOL(net_enable_timestamp);

1441

EXPORT_SYMBOL(net_enable_timestamp);

1442

1443

void net_disable_timestamp(void)

1443

void net_disable_timestamp(void)

1444

{

1444

{

1445

atomic_dec(&netstamp_needed);

1445

atomic_dec(&netstamp_needed);

1446

}

1446

}

1447

EXPORT_SYMBOL(net_disable_timestamp);

1447

EXPORT_SYMBOL(net_disable_timestamp);

1448

1449

static inline void net_timestamp_set(struct sk_buff *skb)

1449

static inline void net_timestamp_set(struct sk_buff *skb)

1450

{

1450

{

1451

if (atomic_read(&netstamp_needed))

1451

if (atomic_read(&netstamp_needed))

1452

__net_timestamp(skb);

1452

__net_timestamp(skb);

1453

else

1453

else

1454

skb->tstamp.tv64 = 0;

1454

skb->tstamp.tv64 = 0;

1455

}

1455

}

1456

1457

static inline void net_timestamp_check(struct sk_buff *skb)

1457

static inline void net_timestamp_check(struct sk_buff *skb)

1458

{

1458

{

1459

if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))

1459

if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))

1460

__net_timestamp(skb);

1460

__net_timestamp(skb);

1461

}

1461

}

1462

1463

/**

1463

/**

1464

* dev_forward_skb - loopback an skb to another netif

1464

* dev_forward_skb - loopback an skb to another netif

1465

*

1465

*

1466

* @dev: destination network device

1466

* @dev: destination network device

1467

* @skb: buffer to forward

1467

* @skb: buffer to forward

1468

*

1468

*

1469

* return values:

1469

* return values:

1470

* NET_RX_SUCCESS (no congestion)

1470

* NET_RX_SUCCESS (no congestion)

1471

* NET_RX_DROP (packet was dropped, but freed)

1471

* NET_RX_DROP (packet was dropped, but freed)

1472

*

1472

*

1473

* dev_forward_skb can be used for injecting an skb from the

1473

* dev_forward_skb can be used for injecting an skb from the

1474

* start_xmit function of one device into the receive queue

1474

* start_xmit function of one device into the receive queue

1475

* of another device.

1475

* of another device.

1476

*

1476

*

1477

* The receiving device may be in another namespace, so

1477

* The receiving device may be in another namespace, so

1478

* we have to clear all information in the skb that could

1478

* we have to clear all information in the skb that could

1479

* impact namespace isolation.

1479

* impact namespace isolation.

1480

*/

1480

*/

1481

int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)

1481

int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)

1482

{

1482

{

1483

skb_orphan(skb);

1483

skb_orphan(skb);

1484

nf_reset(skb);

1484

nf_reset(skb);

1485

1486

if (unlikely(!(dev->flags & IFF_UP) ||

1486

if (unlikely(!(dev->flags & IFF_UP) ||

1487

(skb->len > (dev->mtu + dev->hard_header_len)))) {

1487

(skb->len > (dev->mtu + dev->hard_header_len)))) {

1488

atomic_long_inc(&dev->rx_dropped);

1488

atomic_long_inc(&dev->rx_dropped);

1489

kfree_skb(skb);

1489

kfree_skb(skb);

1490

return NET_RX_DROP;

1490

return NET_RX_DROP;

1491

}

1491

}

1492

skb_set_dev(skb, dev);

1492

skb_set_dev(skb, dev);

1493

skb->tstamp.tv64 = 0;

1493

skb->tstamp.tv64 = 0;

1494

skb->pkt_type = PACKET_HOST;

1494

skb->pkt_type = PACKET_HOST;

1495

skb->protocol = eth_type_trans(skb, dev);

1495

skb->protocol = eth_type_trans(skb, dev);

1496

return netif_rx(skb);

1496

return netif_rx(skb);

1497

}

1497

}

1498

EXPORT_SYMBOL_GPL(dev_forward_skb);

1498

EXPORT_SYMBOL_GPL(dev_forward_skb);

1499

1500

/*

1500

/*

1501

* Support routine. Sends outgoing frames to any network

1501

* Support routine. Sends outgoing frames to any network

1502

* taps currently in use.

1502

* taps currently in use.

1503

*/

1503

*/

1504

1505

static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)

1505

static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)

1506

{

1506

{

1507

struct packet_type *ptype;

1507

struct packet_type *ptype;

1508

1509

#ifdef CONFIG_NET_CLS_ACT

1509

#ifdef CONFIG_NET_CLS_ACT

1510

if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))

1510

if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))

1511

net_timestamp_set(skb);

1511

net_timestamp_set(skb);

1512

#else

1512

#else

1513

net_timestamp_set(skb);

1513

net_timestamp_set(skb);

1514

#endif

1514

#endif

1515

1516

rcu_read_lock();

1516

rcu_read_lock();

1517

list_for_each_entry_rcu(ptype, &ptype_all, list) {

1517

list_for_each_entry_rcu(ptype, &ptype_all, list) {

1518

/* Never send packets back to the socket

1518

/* Never send packets back to the socket

1519

* they originated from - MvS (miquels@drinkel.ow.org)

1519

* they originated from - MvS (miquels@drinkel.ow.org)

1520

*/

1520

*/

1521

if ((ptype->dev == dev || !ptype->dev) &&

1521

if ((ptype->dev == dev || !ptype->dev) &&

1522

(ptype->af_packet_priv == NULL ||

1522

(ptype->af_packet_priv == NULL ||

1523

(struct sock *)ptype->af_packet_priv != skb->sk)) {

1523

(struct sock *)ptype->af_packet_priv != skb->sk)) {

1524

struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);

1524

struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);

1525

if (!skb2)

1525

if (!skb2)

1526

break;

1526

break;

1527

1528

/* skb->nh should be correctly

1528

/* skb->nh should be correctly

1529

set by sender, so that the second statement is

1529

set by sender, so that the second statement is

1530

just protection against buggy protocols.

1530

just protection against buggy protocols.

1531

*/

1531

*/

1532

skb_reset_mac_header(skb2);

1532

skb_reset_mac_header(skb2);

1533

1534

if (skb_network_header(skb2) < skb2->data ||

1534

if (skb_network_header(skb2) < skb2->data ||

1535

skb2->network_header > skb2->tail) {

1535

skb2->network_header > skb2->tail) {

1536

if (net_ratelimit())

1536

if (net_ratelimit())

1537

printk(KERN_CRIT "protocol %04x is "

1537

printk(KERN_CRIT "protocol %04x is "

1538

"buggy, dev %s\n",

1538

"buggy, dev %s\n",

1539

ntohs(skb2->protocol),

1539

ntohs(skb2->protocol),

1540

dev->name);

1540

dev->name);

1541

skb_reset_network_header(skb2);

1541

skb_reset_network_header(skb2);

1542

}

1542

}

1543

1544

skb2->transport_header = skb2->network_header;

1544

skb2->transport_header = skb2->network_header;

1545

skb2->pkt_type = PACKET_OUTGOING;

1545

skb2->pkt_type = PACKET_OUTGOING;

1546

ptype->func(skb2, skb->dev, ptype, skb->dev);

1546

ptype->func(skb2, skb->dev, ptype, skb->dev);

1547

}

1547

}

1548

}

1548

}

1549

rcu_read_unlock();

1549

rcu_read_unlock();

1550

}

1550

}

1551

1552

/*

1552

/*

1553

* Routine to help set real_num_tx_queues. To avoid skbs mapped to queues

1553

* Routine to help set real_num_tx_queues. To avoid skbs mapped to queues

1554

* greater then real_num_tx_queues stale skbs on the qdisc must be flushed.

1554

* greater then real_num_tx_queues stale skbs on the qdisc must be flushed.

1555

*/

1555

*/

1556

void netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)

1556

void netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)

1557

{

1557

{

1558

unsigned int real_num = dev->real_num_tx_queues;

1558

unsigned int real_num = dev->real_num_tx_queues;

1559

1560

if (unlikely(txq > dev->num_tx_queues))

1560

if (unlikely(txq > dev->num_tx_queues))

1561

;

1561

;

1562

else if (txq > real_num)

1562

else if (txq > real_num)

1563

dev->real_num_tx_queues = txq;

1563

dev->real_num_tx_queues = txq;

1564

else if (txq < real_num) {

1564

else if (txq < real_num) {

1565

dev->real_num_tx_queues = txq;

1565

dev->real_num_tx_queues = txq;

1566

qdisc_reset_all_tx_gt(dev, txq);

1566

qdisc_reset_all_tx_gt(dev, txq);

1567

}

1567

}

1568

}

1568

}

1569

EXPORT_SYMBOL(netif_set_real_num_tx_queues);

1569

EXPORT_SYMBOL(netif_set_real_num_tx_queues);

1570

1571

#ifdef CONFIG_RPS

1571

#ifdef CONFIG_RPS

1572

/**

1572

/**

1573

* netif_set_real_num_rx_queues - set actual number of RX queues used

1573

* netif_set_real_num_rx_queues - set actual number of RX queues used

1574

* @dev: Network device

1574

* @dev: Network device

1575

* @rxq: Actual number of RX queues

1575

* @rxq: Actual number of RX queues

1576

*

1576

*

1577

* This must be called either with the rtnl_lock held or before

1577

* This must be called either with the rtnl_lock held or before

1578

* registration of the net device. Returns 0 on success, or a

1578

* registration of the net device. Returns 0 on success, or a

1579

* negative error code. If called before registration, it always

1579

* negative error code. If called before registration, it always

1580

* succeeds.

1580

* succeeds.

1581

*/

1581

*/

1582

int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)

1582

int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)

1583

{

1583

{

1584

int rc;

1584

int rc;

1585

1586

if (dev->reg_state == NETREG_REGISTERED) {

1586

if (dev->reg_state == NETREG_REGISTERED) {

1587

ASSERT_RTNL();

1587

ASSERT_RTNL();

1588

1589

if (rxq > dev->num_rx_queues)

1589

if (rxq > dev->num_rx_queues)

1590

return -EINVAL;

1590

return -EINVAL;

1591

1592

rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,

1592

rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,

1593

rxq);

1593

rxq);

1594

if (rc)

1594

if (rc)

1595

return rc;

1595

return rc;

1596

}

1596

}

1597

1598

dev->real_num_rx_queues = rxq;

1598

dev->real_num_rx_queues = rxq;

1599

return 0;

1599

return 0;

1600

}

1600

}

1601

EXPORT_SYMBOL(netif_set_real_num_rx_queues);

1601

EXPORT_SYMBOL(netif_set_real_num_rx_queues);

1602

#endif

1602

#endif

1603

1604

static inline void __netif_reschedule(struct Qdisc *q)

1604

static inline void __netif_reschedule(struct Qdisc *q)

1605

{

1605

{

1606

struct softnet_data *sd;

1606

struct softnet_data *sd;

1607

unsigned long flags;

1607

unsigned long flags;

1608

1609

local_irq_save(flags);

1609

local_irq_save(flags);

1610

sd = &__get_cpu_var(softnet_data);

1610

sd = &__get_cpu_var(softnet_data);

1611

q->next_sched = NULL;

1611

q->next_sched = NULL;

1612

*sd->output_queue_tailp = q;

1612

*sd->output_queue_tailp = q;

1613

sd->output_queue_tailp = &q->next_sched;

1613

sd->output_queue_tailp = &q->next_sched;

1614

raise_softirq_irqoff(NET_TX_SOFTIRQ);

1614

raise_softirq_irqoff(NET_TX_SOFTIRQ);

1615

local_irq_restore(flags);

1615

local_irq_restore(flags);

1616

}

1616

}

1617

1618

void __netif_schedule(struct Qdisc *q)

1618

void __netif_schedule(struct Qdisc *q)

1619

{

1619

{

1620

if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))

1620

if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))

1621

__netif_reschedule(q);

1621

__netif_reschedule(q);

1622

}

1622

}

1623

EXPORT_SYMBOL(__netif_schedule);

1623

EXPORT_SYMBOL(__netif_schedule);

1624

1625

void dev_kfree_skb_irq(struct sk_buff *skb)

1625

void dev_kfree_skb_irq(struct sk_buff *skb)

1626

{

1626

{

1627

if (atomic_dec_and_test(&skb->users)) {

1627

if (atomic_dec_and_test(&skb->users)) {

1628

struct softnet_data *sd;

1628

struct softnet_data *sd;

1629

unsigned long flags;

1629

unsigned long flags;

1630

1631

local_irq_save(flags);

1631

local_irq_save(flags);

1632

sd = &__get_cpu_var(softnet_data);

1632

sd = &__get_cpu_var(softnet_data);

1633

skb->next = sd->completion_queue;

1633

skb->next = sd->completion_queue;

1634

sd->completion_queue = skb;

1634

sd->completion_queue = skb;

1635

raise_softirq_irqoff(NET_TX_SOFTIRQ);

1635

raise_softirq_irqoff(NET_TX_SOFTIRQ);

1636

local_irq_restore(flags);

1636

local_irq_restore(flags);

1637

}

1637

}

1638

}

1638

}

1639

EXPORT_SYMBOL(dev_kfree_skb_irq);

1639

EXPORT_SYMBOL(dev_kfree_skb_irq);

1640

1641

void dev_kfree_skb_any(struct sk_buff *skb)

1641

void dev_kfree_skb_any(struct sk_buff *skb)

1642

{

1642

{

1643

if (in_irq() || irqs_disabled())

1643

if (in_irq() || irqs_disabled())

1644

dev_kfree_skb_irq(skb);

1644

dev_kfree_skb_irq(skb);

1645

else

1645

else

1646

dev_kfree_skb(skb);

1646

dev_kfree_skb(skb);

1647

}

1647

}

1648

EXPORT_SYMBOL(dev_kfree_skb_any);

1648

EXPORT_SYMBOL(dev_kfree_skb_any);

1649

1650

1651

/**

1651

/**

1652

* netif_device_detach - mark device as removed

1652

* netif_device_detach - mark device as removed

1653

* @dev: network device

1653

* @dev: network device

1654

*

1654

*

1655

* Mark device as removed from system and therefore no longer available.

1655

* Mark device as removed from system and therefore no longer available.

1656

*/

1656

*/

1657

void netif_device_detach(struct net_device *dev)

1657

void netif_device_detach(struct net_device *dev)

1658

{

1658

{

1659

if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&

1659

if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&

1660

netif_running(dev)) {

1660

netif_running(dev)) {

1661

netif_tx_stop_all_queues(dev);

1661

netif_tx_stop_all_queues(dev);

1662

}

1662

}

1663

}

1663

}

1664

EXPORT_SYMBOL(netif_device_detach);

1664

EXPORT_SYMBOL(netif_device_detach);

1665

1666

/**

1666

/**

1667

* netif_device_attach - mark device as attached

1667

* netif_device_attach - mark device as attached

1668

* @dev: network device

1668

* @dev: network device

1669

*

1669

*

1670

* Mark device as attached from system and restart if needed.

1670

* Mark device as attached from system and restart if needed.

1671

*/

1671

*/

1672

void netif_device_attach(struct net_device *dev)

1672

void netif_device_attach(struct net_device *dev)

1673

{

1673

{

1674

if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&

1674

if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&

1675

netif_running(dev)) {

1675

netif_running(dev)) {

1676

netif_tx_wake_all_queues(dev);

1676

netif_tx_wake_all_queues(dev);

1677

__netdev_watchdog_up(dev);

1677

__netdev_watchdog_up(dev);

1678

}

1678

}

1679

}

1679

}

1680

EXPORT_SYMBOL(netif_device_attach);

1680

EXPORT_SYMBOL(netif_device_attach);

1681

1682

static bool can_checksum_protocol(unsigned long features, __be16 protocol)

1682

static bool can_checksum_protocol(unsigned long features, __be16 protocol)

1683

{

1683

{

1684

return ((features & NETIF_F_GEN_CSUM) ||

1684

return ((features & NETIF_F_GEN_CSUM) ||

1685

((features & NETIF_F_IP_CSUM) &&

1685

((features & NETIF_F_IP_CSUM) &&

1686

protocol == htons(ETH_P_IP)) ||

1686

protocol == htons(ETH_P_IP)) ||

1687

((features & NETIF_F_IPV6_CSUM) &&

1687

((features & NETIF_F_IPV6_CSUM) &&

1688

protocol == htons(ETH_P_IPV6)) ||

1688

protocol == htons(ETH_P_IPV6)) ||

1689

((features & NETIF_F_FCOE_CRC) &&

1689

((features & NETIF_F_FCOE_CRC) &&

1690

protocol == htons(ETH_P_FCOE)));

1690

protocol == htons(ETH_P_FCOE)));

1691

}

1691

}

1692

1693

static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb)

1693

static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb)

1694

{

1694

{

1695

if (can_checksum_protocol(dev->features, skb->protocol))

1695

if (can_checksum_protocol(dev->features, skb->protocol))

1696

return true;

1696

return true;

1697

1698

if (skb->protocol == htons(ETH_P_8021Q)) {

1698

if (skb->protocol == htons(ETH_P_8021Q)) {

1699

struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;

1699

struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;

1700

if (can_checksum_protocol(dev->features & dev->vlan_features,

1700

if (can_checksum_protocol(dev->features & dev->vlan_features,

1701

veh->h_vlan_encapsulated_proto))

1701

veh->h_vlan_encapsulated_proto))

1702

return true;

1702

return true;

1703

}

1703

}

1704

1705

return false;

1705

return false;

1706

}

1706

}

1707

1708

/**

1708

/**

1709

* skb_dev_set -- assign a new device to a buffer

1709

* skb_dev_set -- assign a new device to a buffer

1710

* @skb: buffer for the new device

1710

* @skb: buffer for the new device

1711

* @dev: network device

1711

* @dev: network device

1712

*

1712

*

1713

* If an skb is owned by a device already, we have to reset

1713

* If an skb is owned by a device already, we have to reset

1714

* all data private to the namespace a device belongs to

1714

* all data private to the namespace a device belongs to

1715

* before assigning it a new device.

1715

* before assigning it a new device.

1716

*/

1716

*/

1717

#ifdef CONFIG_NET_NS

1717

#ifdef CONFIG_NET_NS

1718

void skb_set_dev(struct sk_buff *skb, struct net_device *dev)

1718

void skb_set_dev(struct sk_buff *skb, struct net_device *dev)

1719

{

1719

{

1720

skb_dst_drop(skb);

1720

skb_dst_drop(skb);

1721

if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {

1721

if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {

1722

secpath_reset(skb);

1722

secpath_reset(skb);

1723

nf_reset(skb);

1723

nf_reset(skb);

1724

skb_init_secmark(skb);

1724

skb_init_secmark(skb);

1725

skb->mark = 0;

1725

skb->mark = 0;

1726

skb->priority = 0;

1726

skb->priority = 0;

1727

skb->nf_trace = 0;

1727

skb->nf_trace = 0;

1728

skb->ipvs_property = 0;

1728

skb->ipvs_property = 0;

1729

#ifdef CONFIG_NET_SCHED

1729

#ifdef CONFIG_NET_SCHED

1730

skb->tc_index = 0;

1730

skb->tc_index = 0;

1731

#endif

1731

#endif

1732

}

1732

}

1733

skb->dev = dev;

1733

skb->dev = dev;

1734

}

1734

}

1735

EXPORT_SYMBOL(skb_set_dev);

1735

EXPORT_SYMBOL(skb_set_dev);

1736

#endif /* CONFIG_NET_NS */

1736

#endif /* CONFIG_NET_NS */

1737

1738

/*

1738

/*

1739

* Invalidate hardware checksum when packet is to be mangled, and

1739

* Invalidate hardware checksum when packet is to be mangled, and

1740

* complete checksum manually on outgoing path.

1740

* complete checksum manually on outgoing path.

1741

*/

1741

*/

1742

int skb_checksum_help(struct sk_buff *skb)

1742

int skb_checksum_help(struct sk_buff *skb)

1743

{

1743

{

1744

__wsum csum;

1744

__wsum csum;

1745

int ret = 0, offset;

1745

int ret = 0, offset;

1746

1747

if (skb->ip_summed == CHECKSUM_COMPLETE)

1747

if (skb->ip_summed == CHECKSUM_COMPLETE)

1748

goto out_set_summed;

1748

goto out_set_summed;

1749

1750

if (unlikely(skb_shinfo(skb)->gso_size)) {

1750

if (unlikely(skb_shinfo(skb)->gso_size)) {

1751

/* Let GSO fix up the checksum. */

1751

/* Let GSO fix up the checksum. */

1752

goto out_set_summed;

1752

goto out_set_summed;

1753

}

1753

}

1754

1755

offset = skb->csum_start - skb_headroom(skb);

1755

offset = skb->csum_start - skb_headroom(skb);

1756

BUG_ON(offset >= skb_headlen(skb));

1756

BUG_ON(offset >= skb_headlen(skb));

1757

csum = skb_checksum(skb, offset, skb->len - offset, 0);

1757

csum = skb_checksum(skb, offset, skb->len - offset, 0);

1758

1759

offset += skb->csum_offset;

1759

offset += skb->csum_offset;

1760

BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));

1760

BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));

1761

1762

if (skb_cloned(skb) &&

1762

if (skb_cloned(skb) &&

1763

!skb_clone_writable(skb, offset + sizeof(__sum16))) {

1763

!skb_clone_writable(skb, offset + sizeof(__sum16))) {

1764

ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);

1764

ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);

1765

if (ret)

1765

if (ret)

1766

goto out;

1766

goto out;

1767

}

1767

}

1768

1769

*(__sum16 *)(skb->data + offset) = csum_fold(csum);

1769

*(__sum16 *)(skb->data + offset) = csum_fold(csum);

1770

out_set_summed:

1770

out_set_summed:

1771

skb->ip_summed = CHECKSUM_NONE;

1771

skb->ip_summed = CHECKSUM_NONE;

1772

out:

1772

out:

1773

return ret;

1773

return ret;

1774

}

1774

}

1775

EXPORT_SYMBOL(skb_checksum_help);

1775

EXPORT_SYMBOL(skb_checksum_help);

1776

1777

/**

1777

/**

1778

* skb_gso_segment - Perform segmentation on skb.

1778

* skb_gso_segment - Perform segmentation on skb.

1779

* @skb: buffer to segment

1779

* @skb: buffer to segment

1780

* @features: features for the output path (see dev->features)

1780

* @features: features for the output path (see dev->features)

1781

*

1781

*

1782

* This function segments the given skb and returns a list of segments.

1782

* This function segments the given skb and returns a list of segments.

1783

*

1783

*

1784

* It may return NULL if the skb requires no segmentation. This is

1784

* It may return NULL if the skb requires no segmentation. This is

1785

* only possible when GSO is used for verifying header integrity.

1785

* only possible when GSO is used for verifying header integrity.

1786

*/

1786

*/

1787

struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)

1787

struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)

1788

{

1788

{

1789

struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);

1789

struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);

1790

struct packet_type *ptype;

1790

struct packet_type *ptype;

1791

__be16 type = skb->protocol;

1791

__be16 type = skb->protocol;

1792

int err;

1792

int err;

1793

1794

skb_reset_mac_header(skb);

1794

skb_reset_mac_header(skb);

1795

skb->mac_len = skb->network_header - skb->mac_header;

1795

skb->mac_len = skb->network_header - skb->mac_header;

1796

__skb_pull(skb, skb->mac_len);

1796

__skb_pull(skb, skb->mac_len);

1797

1798

if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {

1798

if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {

1799

struct net_device *dev = skb->dev;

1799

struct net_device *dev = skb->dev;

1800

struct ethtool_drvinfo info = {};

1800

struct ethtool_drvinfo info = {};

1801

1802

if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)

1802

if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)

1803

dev->ethtool_ops->get_drvinfo(dev, &info);

1803

dev->ethtool_ops->get_drvinfo(dev, &info);

1804

1805

WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d "

1805

WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d "

1806

"ip_summed=%d",

1806

"ip_summed=%d",

1807

info.driver, dev ? dev->features : 0L,

1807

info.driver, dev ? dev->features : 0L,

1808

skb->sk ? skb->sk->sk_route_caps : 0L,

1808

skb->sk ? skb->sk->sk_route_caps : 0L,

1809

skb->len, skb->data_len, skb->ip_summed);

1809

skb->len, skb->data_len, skb->ip_summed);

1810

1811

if (skb_header_cloned(skb) &&

1811

if (skb_header_cloned(skb) &&

1812

(err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))

1812

(err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))

1813

return ERR_PTR(err);

1813

return ERR_PTR(err);

1814

}

1814

}

1815

1816

rcu_read_lock();

1816

rcu_read_lock();

1817

list_for_each_entry_rcu(ptype,

1817

list_for_each_entry_rcu(ptype,

1818

&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {

1818

&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {

1819

if (ptype->type == type && !ptype->dev && ptype->gso_segment) {

1819

if (ptype->type == type && !ptype->dev && ptype->gso_segment) {

1820

if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {

1820

if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {

1821

err = ptype->gso_send_check(skb);

1821

err = ptype->gso_send_check(skb);

1822

segs = ERR_PTR(err);

1822

segs = ERR_PTR(err);

1823

if (err || skb_gso_ok(skb, features))

1823

if (err || skb_gso_ok(skb, features))

1824

break;

1824

break;

1825

__skb_push(skb, (skb->data -

1825

__skb_push(skb, (skb->data -

1826

skb_network_header(skb)));

1826

skb_network_header(skb)));

1827

}

1827

}

1828

segs = ptype->gso_segment(skb, features);

1828

segs = ptype->gso_segment(skb, features);

1829

break;

1829

break;

1830

}

1830

}

1831

}

1831

}

1832

rcu_read_unlock();

1832

rcu_read_unlock();

1833

1834

__skb_push(skb, skb->data - skb_mac_header(skb));

1834

__skb_push(skb, skb->data - skb_mac_header(skb));

1835

1836

return segs;

1836

return segs;

1837

}

1837

}

1838

EXPORT_SYMBOL(skb_gso_segment);

1838

EXPORT_SYMBOL(skb_gso_segment);

1839

1840

/* Take action when hardware reception checksum errors are detected. */

1840

/* Take action when hardware reception checksum errors are detected. */

1841

#ifdef CONFIG_BUG

1841

#ifdef CONFIG_BUG

1842

void netdev_rx_csum_fault(struct net_device *dev)

1842

void netdev_rx_csum_fault(struct net_device *dev)

1843

{

1843

{

1844

if (net_ratelimit()) {

1844

if (net_ratelimit()) {

1845

printk(KERN_ERR "%s: hw csum failure.\n",

1845

printk(KERN_ERR "%s: hw csum failure.\n",

1846

dev ? dev->name : "<unknown>");

1846

dev ? dev->name : "<unknown>");

1847

dump_stack();

1847

dump_stack();

1848

}

1848

}

1849

}

1849

}

1850

EXPORT_SYMBOL(netdev_rx_csum_fault);

1850

EXPORT_SYMBOL(netdev_rx_csum_fault);

1851

#endif

1851

#endif

1852

1853

/* Actually, we should eliminate this check as soon as we know, that:

1853

/* Actually, we should eliminate this check as soon as we know, that:

1854

* 1. IOMMU is present and allows to map all the memory.

1854

* 1. IOMMU is present and allows to map all the memory.

1855

* 2. No high memory really exists on this machine.

1855

* 2. No high memory really exists on this machine.

1856

*/

1856

*/

1857

1858

static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)

1858

static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)

1859

{

1859

{

1860

#ifdef CONFIG_HIGHMEM

1860

#ifdef CONFIG_HIGHMEM

1861

int i;

1861

int i;

1862

if (!(dev->features & NETIF_F_HIGHDMA)) {

1862

if (!(dev->features & NETIF_F_HIGHDMA)) {

1863

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)

1863

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)

1864

if (PageHighMem(skb_shinfo(skb)->frags[i].page))

1864

if (PageHighMem(skb_shinfo(skb)->frags[i].page))

1865

return 1;

1865

return 1;

1866

}

1866

}

1867

1868

if (PCI_DMA_BUS_IS_PHYS) {

1868

if (PCI_DMA_BUS_IS_PHYS) {

1869

struct device *pdev = dev->dev.parent;

1869

struct device *pdev = dev->dev.parent;

1870

1871

if (!pdev)

1871

if (!pdev)

1872

return 0;

1872

return 0;

1873

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {

1873

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {

1874

dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);

1874

dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);

1875

if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)

1875

if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)

1876

return 1;

1876

return 1;

1877

}

1877

}

1878

}

1878

}

1879

#endif

1879

#endif

1880

return 0;

1880

return 0;

1881

}

1881

}

1882

1883

struct dev_gso_cb {

1883

struct dev_gso_cb {

1884

void (*destructor)(struct sk_buff *skb);

1884

void (*destructor)(struct sk_buff *skb);

1885

};

1885

};

1886

1887

#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)

1887

#define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)

1888

1889

static void dev_gso_skb_destructor(struct sk_buff *skb)

1889

static void dev_gso_skb_destructor(struct sk_buff *skb)

1890

{

1890

{

1891

struct dev_gso_cb *cb;

1891

struct dev_gso_cb *cb;

1892

1893

do {

1893

do {

1894

struct sk_buff *nskb = skb->next;

1894

struct sk_buff *nskb = skb->next;

1895

1896

skb->next = nskb->next;

1896

skb->next = nskb->next;

1897

nskb->next = NULL;

1897

nskb->next = NULL;

1898

kfree_skb(nskb);

1898

kfree_skb(nskb);

1899

} while (skb->next);

1899

} while (skb->next);

1900

1901

cb = DEV_GSO_CB(skb);

1901

cb = DEV_GSO_CB(skb);

1902

if (cb->destructor)

1902

if (cb->destructor)

1903

cb->destructor(skb);

1903

cb->destructor(skb);

1904

}

1904

}

1905

1906

/**

1906

/**

1907

* dev_gso_segment - Perform emulated hardware segmentation on skb.

1907

* dev_gso_segment - Perform emulated hardware segmentation on skb.

1908

* @skb: buffer to segment

1908

* @skb: buffer to segment

1909

*

1909

*

1910

* This function segments the given skb and stores the list of segments

1910

* This function segments the given skb and stores the list of segments

1911

* in skb->next.

1911

* in skb->next.

1912

*/

1912

*/

1913

static int dev_gso_segment(struct sk_buff *skb)

1913

static int dev_gso_segment(struct sk_buff *skb)

1914

{

1914

{

1915

struct net_device *dev = skb->dev;

1915

struct net_device *dev = skb->dev;

1916

struct sk_buff *segs;

1916

struct sk_buff *segs;

1917

int features = dev->features & ~(illegal_highdma(dev, skb) ?

1917

int features = dev->features & ~(illegal_highdma(dev, skb) ?

1918

NETIF_F_SG : 0);

1918

NETIF_F_SG : 0);

1919

1920

segs = skb_gso_segment(skb, features);

1920

segs = skb_gso_segment(skb, features);

1921

1922

/* Verifying header integrity only. */

1922

/* Verifying header integrity only. */

1923

if (!segs)

1923

if (!segs)

1924

return 0;

1924

return 0;

1925

1926

if (IS_ERR(segs))

1926

if (IS_ERR(segs))

1927

return PTR_ERR(segs);

1927

return PTR_ERR(segs);

1928

1929

skb->next = segs;

1929

skb->next = segs;

1930

DEV_GSO_CB(skb)->destructor = skb->destructor;

1930

DEV_GSO_CB(skb)->destructor = skb->destructor;

1931

skb->destructor = dev_gso_skb_destructor;

1931

skb->destructor = dev_gso_skb_destructor;

1932

1933

return 0;

1933

return 0;

1934

}

1934

}

1935

1936

/*

1936

/*

1937

* Try to orphan skb early, right before transmission by the device.

1937

* Try to orphan skb early, right before transmission by the device.

1938

* We cannot orphan skb if tx timestamp is requested or the sk-reference

1938

* We cannot orphan skb if tx timestamp is requested or the sk-reference

1939

* is needed on driver level for other reasons, e.g. see net/can/raw.c

1939

* is needed on driver level for other reasons, e.g. see net/can/raw.c

1940

*/

1940

*/

1941

static inline void skb_orphan_try(struct sk_buff *skb)

1941

static inline void skb_orphan_try(struct sk_buff *skb)

1942

{

1942

{

1943

struct sock *sk = skb->sk;

1943

struct sock *sk = skb->sk;

1944

1945

if (sk && !skb_shinfo(skb)->tx_flags) {

1945

if (sk && !skb_shinfo(skb)->tx_flags) {

1946

/* skb_tx_hash() wont be able to get sk.

1946

/* skb_tx_hash() wont be able to get sk.

1947

* We copy sk_hash into skb->rxhash

1947

* We copy sk_hash into skb->rxhash

1948

*/

1948

*/

1949

if (!skb->rxhash)

1949

if (!skb->rxhash)

1950

skb->rxhash = sk->sk_hash;

1950

skb->rxhash = sk->sk_hash;

1951

skb_orphan(skb);

1951

skb_orphan(skb);

1952

}

1952

}

1953

}

1953

}

1954

1955

/*

1955

/*

1956

* Returns true if either:

1956

* Returns true if either:

1957

* 1. skb has frag_list and the device doesn't support FRAGLIST, or

1957

* 1. skb has frag_list and the device doesn't support FRAGLIST, or

1958

* 2. skb is fragmented and the device does not support SG, or if

1958

* 2. skb is fragmented and the device does not support SG, or if

1959

* at least one of fragments is in highmem and device does not

1959

* at least one of fragments is in highmem and device does not

1960

* support DMA from it.

1960

* support DMA from it.

1961

*/

1961

*/

1962

static inline int skb_needs_linearize(struct sk_buff *skb,

1962

static inline int skb_needs_linearize(struct sk_buff *skb,

1963

struct net_device *dev)

1963

struct net_device *dev)

1964

{

1964

{

1965

return skb_is_nonlinear(skb) &&

1965

return skb_is_nonlinear(skb) &&

1966

((skb_has_frag_list(skb) && !(dev->features & NETIF_F_FRAGLIST)) ||

1966

((skb_has_frag_list(skb) && !(dev->features & NETIF_F_FRAGLIST)) ||

1967

(skb_shinfo(skb)->nr_frags && (!(dev->features & NETIF_F_SG) ||

1967

(skb_shinfo(skb)->nr_frags && (!(dev->features & NETIF_F_SG) ||

1968

illegal_highdma(dev, skb))));

1968

illegal_highdma(dev, skb))));

1969

}

1969

}

1970

1971

int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,

1971

int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,

1972

struct netdev_queue *txq)

1972

struct netdev_queue *txq)

1973

{

1973

{

1974

const struct net_device_ops *ops = dev->netdev_ops;

1974

const struct net_device_ops *ops = dev->netdev_ops;

1975

int rc = NETDEV_TX_OK;

1975

int rc = NETDEV_TX_OK;

1976

1977

if (likely(!skb->next)) {

1977

if (likely(!skb->next)) {

1978

if (!list_empty(&ptype_all))

1978

if (!list_empty(&ptype_all))

1979

dev_queue_xmit_nit(skb, dev);

1979

dev_queue_xmit_nit(skb, dev);

1980

1981

/*

1981

/*

1982

* If device doesnt need skb->dst, release it right now while

1982

* If device doesnt need skb->dst, release it right now while

1983

* its hot in this cpu cache

1983

* its hot in this cpu cache

1984

*/

1984

*/

1985

if (dev->priv_flags & IFF_XMIT_DST_RELEASE)

1985

if (dev->priv_flags & IFF_XMIT_DST_RELEASE)

1986

skb_dst_drop(skb);

1986

skb_dst_drop(skb);

1987

1988

skb_orphan_try(skb);

1988

skb_orphan_try(skb);

1989

1990

if (netif_needs_gso(dev, skb)) {

1990

if (netif_needs_gso(dev, skb)) {

1991

if (unlikely(dev_gso_segment(skb)))

1991

if (unlikely(dev_gso_segment(skb)))

1992

goto out_kfree_skb;

1992

goto out_kfree_skb;

1993

if (skb->next)

1993

if (skb->next)

1994

goto gso;

1994

goto gso;

1995

} else {

1995

} else {

1996

if (skb_needs_linearize(skb, dev) &&

1996

if (skb_needs_linearize(skb, dev) &&

1997

__skb_linearize(skb))

1997

__skb_linearize(skb))

1998

goto out_kfree_skb;

1998

goto out_kfree_skb;

1999

2000

/* If packet is not checksummed and device does not

2000

/* If packet is not checksummed and device does not

2001

* support checksumming for this protocol, complete

2001

* support checksumming for this protocol, complete

2002

* checksumming here.

2002

* checksumming here.

2003

*/

2003

*/

2004

if (skb->ip_summed == CHECKSUM_PARTIAL) {

2004

if (skb->ip_summed == CHECKSUM_PARTIAL) {

2005

skb_set_transport_header(skb, skb->csum_start -

2005

skb_set_transport_header(skb, skb->csum_start -

2006

skb_headroom(skb));

2006

skb_headroom(skb));

2007

if (!dev_can_checksum(dev, skb) &&

2007

if (!dev_can_checksum(dev, skb) &&

2008

skb_checksum_help(skb))

2008

skb_checksum_help(skb))

2009

goto out_kfree_skb;

2009

goto out_kfree_skb;

2010

}

2010

}

2011

}

2011

}

2012

2013

rc = ops->ndo_start_xmit(skb, dev);

2013

rc = ops->ndo_start_xmit(skb, dev);

2014

if (rc == NETDEV_TX_OK)

2014

if (rc == NETDEV_TX_OK)

2015

txq_trans_update(txq);

2015

txq_trans_update(txq);

2016

return rc;

2016

return rc;

2017

}

2017

}

2018

2019

gso:

2019

gso:

2020

do {

2020

do {

2021

struct sk_buff *nskb = skb->next;

2021

struct sk_buff *nskb = skb->next;

2022

2023

skb->next = nskb->next;

2023

skb->next = nskb->next;

2024

nskb->next = NULL;

2024

nskb->next = NULL;

2025

2026

/*

2026

/*

2027

* If device doesnt need nskb->dst, release it right now while

2027

* If device doesnt need nskb->dst, release it right now while

2028

* its hot in this cpu cache

2028

* its hot in this cpu cache

2029

*/

2029

*/

2030

if (dev->priv_flags & IFF_XMIT_DST_RELEASE)

2030

if (dev->priv_flags & IFF_XMIT_DST_RELEASE)

2031

skb_dst_drop(nskb);

2031

skb_dst_drop(nskb);

2032

2033

rc = ops->ndo_start_xmit(nskb, dev);

2033

rc = ops->ndo_start_xmit(nskb, dev);

2034

if (unlikely(rc != NETDEV_TX_OK)) {

2034

if (unlikely(rc != NETDEV_TX_OK)) {

2035

if (rc & ~NETDEV_TX_MASK)

2035

if (rc & ~NETDEV_TX_MASK)

2036

goto out_kfree_gso_skb;

2036

goto out_kfree_gso_skb;

2037

nskb->next = skb->next;

2037

nskb->next = skb->next;

2038

skb->next = nskb;

2038

skb->next = nskb;

2039

return rc;

2039

return rc;

2040

}

2040

}

2041

txq_trans_update(txq);

2041

txq_trans_update(txq);

2042

if (unlikely(netif_tx_queue_stopped(txq) && skb->next))

2042

if (unlikely(netif_tx_queue_stopped(txq) && skb->next))

2043

return NETDEV_TX_BUSY;

2043

return NETDEV_TX_BUSY;

2044

} while (skb->next);

2044

} while (skb->next);

2045

2046

out_kfree_gso_skb:

2046

out_kfree_gso_skb:

2047

if (likely(skb->next == NULL))

2047

if (likely(skb->next == NULL))

2048

skb->destructor = DEV_GSO_CB(skb)->destructor;

2048

skb->destructor = DEV_GSO_CB(skb)->destructor;

2049

out_kfree_skb:

2049

out_kfree_skb:

2050

kfree_skb(skb);

2050

kfree_skb(skb);

2051

return rc;

2051

return rc;

2052

}

2052

}

2053

2054

static u32 hashrnd __read_mostly;

2054

static u32 hashrnd __read_mostly;

2055

2056

u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)

2056

u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)

2057

{

2057

{

2058

u32 hash;

2058

u32 hash;

2059

2060

if (skb_rx_queue_recorded(skb)) {

2060

if (skb_rx_queue_recorded(skb)) {

2061

hash = skb_get_rx_queue(skb);

2061

hash = skb_get_rx_queue(skb);

2062

while (unlikely(hash >= dev->real_num_tx_queues))

2062

while (unlikely(hash >= dev->real_num_tx_queues))

2063

hash -= dev->real_num_tx_queues;

2063

hash -= dev->real_num_tx_queues;

2064

return hash;

2064

return hash;

2065

}

2065

}

2066

2067

if (skb->sk && skb->sk->sk_hash)

2067

if (skb->sk && skb->sk->sk_hash)

2068

hash = skb->sk->sk_hash;

2068

hash = skb->sk->sk_hash;

2069

else

2069

else

2070

hash = (__force u16) skb->protocol ^ skb->rxhash;

2070

hash = (__force u16) skb->protocol ^ skb->rxhash;

2071

hash = jhash_1word(hash, hashrnd);

2071

hash = jhash_1word(hash, hashrnd);

2072

2073

return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);

2073

return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);

2074

}

2074

}

2075

EXPORT_SYMBOL(skb_tx_hash);

2075

EXPORT_SYMBOL(skb_tx_hash);

2076

2077

static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)

2077

static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)

2078

{

2078

{

2079

if (unlikely(queue_index >= dev->real_num_tx_queues)) {

2079

if (unlikely(queue_index >= dev->real_num_tx_queues)) {

2080

if (net_ratelimit()) {

2080

if (net_ratelimit()) {

2081

pr_warning("%s selects TX queue %d, but "

2081

pr_warning("%s selects TX queue %d, but "

2082

"real number of TX queues is %d\n",

2082

"real number of TX queues is %d\n",

2083

dev->name, queue_index, dev->real_num_tx_queues);

2083

dev->name, queue_index, dev->real_num_tx_queues);

2084

}

2084

}

2085

return 0;

2085

return 0;

2086

}

2086

}

2087

return queue_index;

2087

return queue_index;

2088

}

2088

}

2089

2090

static struct netdev_queue *dev_pick_tx(struct net_device *dev,

2090

static struct netdev_queue *dev_pick_tx(struct net_device *dev,

2091

struct sk_buff *skb)

2091

struct sk_buff *skb)

2092

{

2092

{

2093

int queue_index;

2093

int queue_index;

2094

const struct net_device_ops *ops = dev->netdev_ops;

2094

const struct net_device_ops *ops = dev->netdev_ops;

2095

2096

if (ops->ndo_select_queue) {

2096

if (ops->ndo_select_queue) {

2097

queue_index = ops->ndo_select_queue(dev, skb);

2097

queue_index = ops->ndo_select_queue(dev, skb);

2098

queue_index = dev_cap_txqueue(dev, queue_index);

2098

queue_index = dev_cap_txqueue(dev, queue_index);

2099

} else {

2099

} else {

2100

struct sock *sk = skb->sk;

2100

struct sock *sk = skb->sk;

2101

queue_index = sk_tx_queue_get(sk);

2101

queue_index = sk_tx_queue_get(sk);

2102

if (queue_index < 0) {

2102

if (queue_index < 0) {

2103

2104

queue_index = 0;

2104

queue_index = 0;

2105

if (dev->real_num_tx_queues > 1)

2105

if (dev->real_num_tx_queues > 1)

2106

queue_index = skb_tx_hash(dev, skb);

2106

queue_index = skb_tx_hash(dev, skb);

2107

2108

if (sk) {

2108

if (sk) {

2109

struct dst_entry *dst = rcu_dereference_check(sk->sk_dst_cache, 1);

2109

struct dst_entry *dst = rcu_dereference_check(sk->sk_dst_cache, 1);

2110

2111

if (dst && skb_dst(skb) == dst)

2111

if (dst && skb_dst(skb) == dst)

2112

sk_tx_queue_set(sk, queue_index);

2112

sk_tx_queue_set(sk, queue_index);

2113

}

2113

}

2114

}

2114

}

2115

}

2115

}

2116

2117

skb_set_queue_mapping(skb, queue_index);

2117

skb_set_queue_mapping(skb, queue_index);

2118

return netdev_get_tx_queue(dev, queue_index);

2118

return netdev_get_tx_queue(dev, queue_index);

2119

}

2119

}

2120

2121

static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,

2121

static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,

2122

struct net_device *dev,

2122

struct net_device *dev,

2123

struct netdev_queue *txq)

2123

struct netdev_queue *txq)

2124

{

2124

{

2125

spinlock_t *root_lock = qdisc_lock(q);

2125

spinlock_t *root_lock = qdisc_lock(q);

2126

bool contended = qdisc_is_running(q);

2126

bool contended = qdisc_is_running(q);

2127

int rc;

2127

int rc;

2128

2129

/*

2129

/*

2130

* Heuristic to force contended enqueues to serialize on a

2130

* Heuristic to force contended enqueues to serialize on a

2131

* separate lock before trying to get qdisc main lock.

2131

* separate lock before trying to get qdisc main lock.

2132

* This permits __QDISC_STATE_RUNNING owner to get the lock more often

2132

* This permits __QDISC_STATE_RUNNING owner to get the lock more often

2133

* and dequeue packets faster.

2133

* and dequeue packets faster.

2134

*/

2134

*/

2135

if (unlikely(contended))

2135

if (unlikely(contended))

2136

spin_lock(&q->busylock);

2136

spin_lock(&q->busylock);

2137

2138

spin_lock(root_lock);

2138

spin_lock(root_lock);

2139

if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {

2139

if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {

2140

kfree_skb(skb);

2140

kfree_skb(skb);

2141

rc = NET_XMIT_DROP;

2141

rc = NET_XMIT_DROP;

2142

} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&

2142

} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&

2143

qdisc_run_begin(q)) {

2143

qdisc_run_begin(q)) {

2144

/*

2144

/*

2145

* This is a work-conserving queue; there are no old skbs

2145

* This is a work-conserving queue; there are no old skbs

2146

* waiting to be sent out; and the qdisc is not running -

2146

* waiting to be sent out; and the qdisc is not running -

2147

* xmit the skb directly.

2147

* xmit the skb directly.

2148

*/

2148

*/

2149

if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))

2149

if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))

2150

skb_dst_force(skb);

2150

skb_dst_force(skb);

2151

__qdisc_update_bstats(q, skb->len);

2151

__qdisc_update_bstats(q, skb->len);

2152

if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {

2152

if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {

2153

if (unlikely(contended)) {

2153

if (unlikely(contended)) {

2154

spin_unlock(&q->busylock);

2154

spin_unlock(&q->busylock);

2155

contended = false;

2155

contended = false;

2156

}

2156

}

2157

__qdisc_run(q);

2157

__qdisc_run(q);

2158

} else

2158

} else

2159

qdisc_run_end(q);

2159

qdisc_run_end(q);

2160

2161

rc = NET_XMIT_SUCCESS;

2161

rc = NET_XMIT_SUCCESS;

2162

} else {

2162

} else {

2163

skb_dst_force(skb);

2163

skb_dst_force(skb);

2164

rc = qdisc_enqueue_root(skb, q);

2164

rc = qdisc_enqueue_root(skb, q);

2165

if (qdisc_run_begin(q)) {

2165

if (qdisc_run_begin(q)) {

2166

if (unlikely(contended)) {

2166

if (unlikely(contended)) {

2167

spin_unlock(&q->busylock);

2167

spin_unlock(&q->busylock);

2168

contended = false;

2168

contended = false;

2169

}

2169

}

2170

__qdisc_run(q);

2170

__qdisc_run(q);

2171

}

2171

}

2172

}

2172

}

2173

spin_unlock(root_lock);

2173

spin_unlock(root_lock);

2174

if (unlikely(contended))

2174

if (unlikely(contended))

2175

spin_unlock(&q->busylock);

2175

spin_unlock(&q->busylock);

2176

return rc;

2176

return rc;

2177

}

2177

}

2178

2179

static DEFINE_PER_CPU(int, xmit_recursion);

2179

static DEFINE_PER_CPU(int, xmit_recursion);

2180

#define RECURSION_LIMIT 3

2180

#define RECURSION_LIMIT 3

2181

2182

/**

2182

/**

2183

* dev_queue_xmit - transmit a buffer

2183

* dev_queue_xmit - transmit a buffer

2184

* @skb: buffer to transmit

2184

* @skb: buffer to transmit

2185

*

2185

*

2186

* Queue a buffer for transmission to a network device. The caller must

2186

* Queue a buffer for transmission to a network device. The caller must

2187

* have set the device and priority and built the buffer before calling

2187

* have set the device and priority and built the buffer before calling

2188

* this function. The function can be called from an interrupt.

2188

* this function. The function can be called from an interrupt.

2189

*

2189

*

2190

* A negative errno code is returned on a failure. A success does not

2190

* A negative errno code is returned on a failure. A success does not

2191

* guarantee the frame will be transmitted as it may be dropped due

2191

* guarantee the frame will be transmitted as it may be dropped due

2192

* to congestion or traffic shaping.

2192

* to congestion or traffic shaping.

2193

*

2193

*

2194

* -----------------------------------------------------------------------------------

2194

* -----------------------------------------------------------------------------------

2195

* I notice this method can also return errors from the queue disciplines,

2195

* I notice this method can also return errors from the queue disciplines,

2196

* including NET_XMIT_DROP, which is a positive value. So, errors can also

2196

* including NET_XMIT_DROP, which is a positive value. So, errors can also

2197

* be positive.

2197

* be positive.

2198

*

2198

*

2199

* Regardless of the return value, the skb is consumed, so it is currently

2199

* Regardless of the return value, the skb is consumed, so it is currently

2200

* difficult to retry a send to this method. (You can bump the ref count

2200

* difficult to retry a send to this method. (You can bump the ref count

2201

* before sending to hold a reference for retry if you are careful.)

2201

* before sending to hold a reference for retry if you are careful.)

2202

*

2202

*

2203

* When calling this method, interrupts MUST be enabled. This is because

2203

* When calling this method, interrupts MUST be enabled. This is because

2204

* the BH enable code must have IRQs enabled so that it will not deadlock.

2204

* the BH enable code must have IRQs enabled so that it will not deadlock.

2205

* --BLG

2205

* --BLG

2206

*/

2206

*/

2207

int dev_queue_xmit(struct sk_buff *skb)

2207

int dev_queue_xmit(struct sk_buff *skb)

2208

{

2208

{

2209

struct net_device *dev = skb->dev;

2209

struct net_device *dev = skb->dev;

2210

struct netdev_queue *txq;

2210

struct netdev_queue *txq;

2211

struct Qdisc *q;

2211

struct Qdisc *q;

2212

int rc = -ENOMEM;

2212

int rc = -ENOMEM;

2213

2214

/* Disable soft irqs for various locks below. Also

2214

/* Disable soft irqs for various locks below. Also

2215

* stops preemption for RCU.

2215

* stops preemption for RCU.

2216

*/

2216

*/

2217

rcu_read_lock_bh();

2217

rcu_read_lock_bh();

2218

2219

txq = dev_pick_tx(dev, skb);

2219

txq = dev_pick_tx(dev, skb);

2220

q = rcu_dereference_bh(txq->qdisc);

2220

q = rcu_dereference_bh(txq->qdisc);

2221

2222

#ifdef CONFIG_NET_CLS_ACT

2222

#ifdef CONFIG_NET_CLS_ACT

2223

skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);

2223

skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);

2224

#endif

2224

#endif

2225

if (q->enqueue) {

2225

if (q->enqueue) {

2226

rc = __dev_xmit_skb(skb, q, dev, txq);

2226

rc = __dev_xmit_skb(skb, q, dev, txq);

2227

goto out;

2227

goto out;

2228

}

2228

}

2229

2230

/* The device has no queue. Common case for software devices:

2230

/* The device has no queue. Common case for software devices:

2231

loopback, all the sorts of tunnels...

2231

loopback, all the sorts of tunnels...

2232

2233

Really, it is unlikely that netif_tx_lock protection is necessary

2233

Really, it is unlikely that netif_tx_lock protection is necessary

2234

here. (f.e. loopback and IP tunnels are clean ignoring statistics

2234

here. (f.e. loopback and IP tunnels are clean ignoring statistics

2235

counters.)

2235

counters.)

2236

However, it is possible, that they rely on protection

2236

However, it is possible, that they rely on protection

2237

made by us here.

2237

made by us here.

2238

2239

Check this and shot the lock. It is not prone from deadlocks.

2239

Check this and shot the lock. It is not prone from deadlocks.

2240

Either shot noqueue qdisc, it is even simpler 8)

2240

Either shot noqueue qdisc, it is even simpler 8)

2241

*/

2241

*/

2242

if (dev->flags & IFF_UP) {

2242

if (dev->flags & IFF_UP) {

2243

int cpu = smp_processor_id(); /* ok because BHs are off */

2243

int cpu = smp_processor_id(); /* ok because BHs are off */

2244

2245

if (txq->xmit_lock_owner != cpu) {

2245

if (txq->xmit_lock_owner != cpu) {

2246

2247

if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)

2247

if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)

2248

goto recursion_alert;

2248

goto recursion_alert;

2249

2250

HARD_TX_LOCK(dev, txq, cpu);

2250

HARD_TX_LOCK(dev, txq, cpu);

2251

2252

if (!netif_tx_queue_stopped(txq)) {

2252

if (!netif_tx_queue_stopped(txq)) {

2253

__this_cpu_inc(xmit_recursion);

2253

__this_cpu_inc(xmit_recursion);

2254

rc = dev_hard_start_xmit(skb, dev, txq);

2254

rc = dev_hard_start_xmit(skb, dev, txq);

2255

__this_cpu_dec(xmit_recursion);

2255

__this_cpu_dec(xmit_recursion);

2256

if (dev_xmit_complete(rc)) {

2256

if (dev_xmit_complete(rc)) {

2257

HARD_TX_UNLOCK(dev, txq);

2257

HARD_TX_UNLOCK(dev, txq);

2258

goto out;

2258

goto out;

2259

}

2259

}

2260

}

2260

}

2261

HARD_TX_UNLOCK(dev, txq);

2261

HARD_TX_UNLOCK(dev, txq);

2262

if (net_ratelimit())

2262

if (net_ratelimit())

2263

printk(KERN_CRIT "Virtual device %s asks to "

2263

printk(KERN_CRIT "Virtual device %s asks to "

2264

"queue packet!\n", dev->name);

2264

"queue packet!\n", dev->name);

2265

} else {

2265

} else {

2266

/* Recursion is detected! It is possible,

2266

/* Recursion is detected! It is possible,

2267

* unfortunately

2267

* unfortunately

2268

*/

2268

*/

2269

recursion_alert:

2269

recursion_alert:

2270

if (net_ratelimit())

2270

if (net_ratelimit())

2271

printk(KERN_CRIT "Dead loop on virtual device "

2271

printk(KERN_CRIT "Dead loop on virtual device "

2272

"%s, fix it urgently!\n", dev->name);

2272

"%s, fix it urgently!\n", dev->name);

2273

}

2273

}

2274

}

2274

}

2275

2276

rc = -ENETDOWN;

2276

rc = -ENETDOWN;

2277

rcu_read_unlock_bh();

2277

rcu_read_unlock_bh();

2278

2279

kfree_skb(skb);

2279

kfree_skb(skb);

2280

return rc;

2280

return rc;

2281

out:

2281

out:

2282

rcu_read_unlock_bh();

2282

rcu_read_unlock_bh();

2283

return rc;

2283

return rc;

2284

}

2284

}

2285

EXPORT_SYMBOL(dev_queue_xmit);

2285

EXPORT_SYMBOL(dev_queue_xmit);

2286

2287

2288

/*=======================================================================

2288

/*=======================================================================

2289

Receiver routines

2289

Receiver routines

2290

=======================================================================*/

2290

=======================================================================*/

2291

2292

int netdev_max_backlog __read_mostly = 1000;

2292

int netdev_max_backlog __read_mostly = 1000;

2293

int netdev_tstamp_prequeue __read_mostly = 1;

2293

int netdev_tstamp_prequeue __read_mostly = 1;

2294

int netdev_budget __read_mostly = 300;

2294

int netdev_budget __read_mostly = 300;

2295

int weight_p __read_mostly = 64; /* old backlog weight */

2295

int weight_p __read_mostly = 64; /* old backlog weight */

2296

2297

/* Called with irq disabled */

2297

/* Called with irq disabled */

2298

static inline void ____napi_schedule(struct softnet_data *sd,

2298

static inline void ____napi_schedule(struct softnet_data *sd,

2299

struct napi_struct *napi)

2299

struct napi_struct *napi)

2300

{

2300

{

2301

list_add_tail(&napi->poll_list, &sd->poll_list);

2301

list_add_tail(&napi->poll_list, &sd->poll_list);

2302

__raise_softirq_irqoff(NET_RX_SOFTIRQ);

2302

__raise_softirq_irqoff(NET_RX_SOFTIRQ);

2303

}

2303

}

2304

2305

/*

2305

/*

2306

* __skb_get_rxhash: calculate a flow hash based on src/dst addresses

2306

* __skb_get_rxhash: calculate a flow hash based on src/dst addresses

2307

* and src/dst port numbers. Returns a non-zero hash number on success

2307

* and src/dst port numbers. Returns a non-zero hash number on success

2308

* and 0 on failure.

2308

* and 0 on failure.

2309

*/

2309

*/

2310

__u32 __skb_get_rxhash(struct sk_buff *skb)

2310

__u32 __skb_get_rxhash(struct sk_buff *skb)

2311

{

2311

{

2312

int nhoff, hash = 0, poff;

2312

int nhoff, hash = 0, poff;

2313

struct ipv6hdr *ip6;

2313

struct ipv6hdr *ip6;

2314

struct iphdr *ip;

2314

struct iphdr *ip;

2315

u8 ip_proto;

2315

u8 ip_proto;

2316

u32 addr1, addr2, ihl;

2316

u32 addr1, addr2, ihl;

2317

union {

2317

union {

2318

u32 v32;

2318

u32 v32;

2319

u16 v16[2];

2319

u16 v16[2];

2320

} ports;

2320

} ports;

2321

2322

nhoff = skb_network_offset(skb);

2322

nhoff = skb_network_offset(skb);

2323

2324

switch (skb->protocol) {

2324

switch (skb->protocol) {

2325

case __constant_htons(ETH_P_IP):

2325

case __constant_htons(ETH_P_IP):

2326

if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))

2326

if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))

2327

goto done;

2327

goto done;

2328

2329

ip = (struct iphdr *) (skb->data + nhoff);

2329

ip = (struct iphdr *) (skb->data + nhoff);

2330

if (ip->frag_off & htons(IP_MF | IP_OFFSET))

2330

if (ip->frag_off & htons(IP_MF | IP_OFFSET))

2331

ip_proto = 0;

2331

ip_proto = 0;

2332

else

2332

else

2333

ip_proto = ip->protocol;

2333

ip_proto = ip->protocol;

2334

addr1 = (__force u32) ip->saddr;

2334

addr1 = (__force u32) ip->saddr;

2335

addr2 = (__force u32) ip->daddr;

2335

addr2 = (__force u32) ip->daddr;

2336

ihl = ip->ihl;

2336

ihl = ip->ihl;

2337

break;

2337

break;

2338

case __constant_htons(ETH_P_IPV6):

2338

case __constant_htons(ETH_P_IPV6):

2339

if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))

2339

if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))

2340

goto done;

2340

goto done;

2341

2342

ip6 = (struct ipv6hdr *) (skb->data + nhoff);

2342

ip6 = (struct ipv6hdr *) (skb->data + nhoff);

2343

ip_proto = ip6->nexthdr;

2343

ip_proto = ip6->nexthdr;

2344

addr1 = (__force u32) ip6->saddr.s6_addr32[3];

2344

addr1 = (__force u32) ip6->saddr.s6_addr32[3];

2345

addr2 = (__force u32) ip6->daddr.s6_addr32[3];

2345

addr2 = (__force u32) ip6->daddr.s6_addr32[3];

2346

ihl = (40 >> 2);

2346

ihl = (40 >> 2);

2347

break;

2347

break;

2348

default:

2348

default:

2349

goto done;

2349

goto done;

2350

}

2350

}

2351

2352

ports.v32 = 0;

2352

ports.v32 = 0;

2353

poff = proto_ports_offset(ip_proto);

2353

poff = proto_ports_offset(ip_proto);

2354

if (poff >= 0) {

2354

if (poff >= 0) {

2355

nhoff += ihl * 4 + poff;

2355

nhoff += ihl * 4 + poff;

2356

if (pskb_may_pull(skb, nhoff + 4)) {

2356

if (pskb_may_pull(skb, nhoff + 4)) {

2357

ports.v32 = * (__force u32 *) (skb->data + nhoff);

2357

ports.v32 = * (__force u32 *) (skb->data + nhoff);

2358

if (ports.v16[1] < ports.v16[0])

2358

if (ports.v16[1] < ports.v16[0])

2359

swap(ports.v16[0], ports.v16[1]);

2359

swap(ports.v16[0], ports.v16[1]);

2360

}

2360

}

2361

}

2361

}

2362

2363

/* get a consistent hash (same value on both flow directions) */

2363

/* get a consistent hash (same value on both flow directions) */

2364

if (addr2 < addr1)

2364

if (addr2 < addr1)

2365

swap(addr1, addr2);

2365

swap(addr1, addr2);

2366

2367

hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);

2367

hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);

2368

if (!hash)

2368

if (!hash)

2369

hash = 1;

2369

hash = 1;

2370

2371

done:

2371

done:

2372

return hash;

2372

return hash;

2373

}

2373

}

2374

EXPORT_SYMBOL(__skb_get_rxhash);

2374

EXPORT_SYMBOL(__skb_get_rxhash);

2375

2376

#ifdef CONFIG_RPS

2376

#ifdef CONFIG_RPS

2377

2378

/* One global table that all flow-based protocols share. */

2378

/* One global table that all flow-based protocols share. */

2379

struct rps_sock_flow_table *rps_sock_flow_table __read_mostly;

2379

struct rps_sock_flow_table *rps_sock_flow_table __read_mostly;

2380

EXPORT_SYMBOL(rps_sock_flow_table);

2380

EXPORT_SYMBOL(rps_sock_flow_table);

2381

2382

/*

2382

/*

2383

* get_rps_cpu is called from netif_receive_skb and returns the target

2383

* get_rps_cpu is called from netif_receive_skb and returns the target

2384

* CPU from the RPS map of the receiving queue for a given skb.

2384

* CPU from the RPS map of the receiving queue for a given skb.

2385

* rcu_read_lock must be held on entry.

2385

* rcu_read_lock must be held on entry.

2386

*/

2386

*/

2387

static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,

2387

static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,

2388

struct rps_dev_flow **rflowp)

2388

struct rps_dev_flow **rflowp)

2389

{

2389

{

2390

struct netdev_rx_queue *rxqueue;

2390

struct netdev_rx_queue *rxqueue;

2391

struct rps_map *map = NULL;

2391

struct rps_map *map = NULL;

2392

struct rps_dev_flow_table *flow_table;

2392

struct rps_dev_flow_table *flow_table;

2393

struct rps_sock_flow_table *sock_flow_table;

2393

struct rps_sock_flow_table *sock_flow_table;

2394

int cpu = -1;

2394

int cpu = -1;

2395

u16 tcpu;

2395

u16 tcpu;

2396

2397

if (skb_rx_queue_recorded(skb)) {

2397

if (skb_rx_queue_recorded(skb)) {

2398

u16 index = skb_get_rx_queue(skb);

2398

u16 index = skb_get_rx_queue(skb);

2399

if (unlikely(index >= dev->real_num_rx_queues)) {

2399

if (unlikely(index >= dev->real_num_rx_queues)) {

2400

WARN_ONCE(dev->real_num_rx_queues > 1,

2400

WARN_ONCE(dev->real_num_rx_queues > 1,

2401

"%s received packet on queue %u, but number "

2401

"%s received packet on queue %u, but number "

2402

"of RX queues is %u\n",

2402

"of RX queues is %u\n",

2403

dev->name, index, dev->real_num_rx_queues);

2403

dev->name, index, dev->real_num_rx_queues);

2404

goto done;

2404

goto done;

2405

}

2405

}

2406

rxqueue = dev->_rx + index;

2406

rxqueue = dev->_rx + index;

2407

} else

2407

} else

2408

rxqueue = dev->_rx;

2408

rxqueue = dev->_rx;

2409

2410

if (rxqueue->rps_map) {

2410

if (rxqueue->rps_map) {

2411

map = rcu_dereference(rxqueue->rps_map);

2411

map = rcu_dereference(rxqueue->rps_map);

2412

if (map && map->len == 1) {

2412

if (map && map->len == 1) {

2413

tcpu = map->cpus[0];

2413

tcpu = map->cpus[0];

2414

if (cpu_online(tcpu))

2414

if (cpu_online(tcpu))

2415

cpu = tcpu;

2415

cpu = tcpu;

2416

goto done;

2416

goto done;

2417

}

2417

}

2418

} else if (!rxqueue->rps_flow_table) {

2418

} else if (!rxqueue->rps_flow_table) {

2419

goto done;

2419

goto done;

2420

}

2420

}

2421

2422

skb_reset_network_header(skb);

2422

skb_reset_network_header(skb);

2423

if (!skb_get_rxhash(skb))

2423

if (!skb_get_rxhash(skb))

2424

goto done;

2424

goto done;

2425

2426

flow_table = rcu_dereference(rxqueue->rps_flow_table);

2426

flow_table = rcu_dereference(rxqueue->rps_flow_table);

2427

sock_flow_table = rcu_dereference(rps_sock_flow_table);

2427

sock_flow_table = rcu_dereference(rps_sock_flow_table);

2428

if (flow_table && sock_flow_table) {

2428

if (flow_table && sock_flow_table) {

2429

u16 next_cpu;

2429

u16 next_cpu;

2430

struct rps_dev_flow *rflow;

2430

struct rps_dev_flow *rflow;

2431

2432

rflow = &flow_table->flows[skb->rxhash & flow_table->mask];

2432

rflow = &flow_table->flows[skb->rxhash & flow_table->mask];

2433

tcpu = rflow->cpu;

2433

tcpu = rflow->cpu;

2434

2435

next_cpu = sock_flow_table->ents[skb->rxhash &

2435

next_cpu = sock_flow_table->ents[skb->rxhash &

2436

sock_flow_table->mask];

2436

sock_flow_table->mask];

2437

2438

/*

2438

/*

2439

* If the desired CPU (where last recvmsg was done) is

2439

* If the desired CPU (where last recvmsg was done) is

2440

* different from current CPU (one in the rx-queue flow

2440

* different from current CPU (one in the rx-queue flow

2441

* table entry), switch if one of the following holds:

2441

* table entry), switch if one of the following holds:

2442

* - Current CPU is unset (equal to RPS_NO_CPU).

2442

* - Current CPU is unset (equal to RPS_NO_CPU).

2443

* - Current CPU is offline.

2443

* - Current CPU is offline.

2444

* - The current CPU's queue tail has advanced beyond the

2444

* - The current CPU's queue tail has advanced beyond the

2445

* last packet that was enqueued using this table entry.

2445

* last packet that was enqueued using this table entry.

2446

* This guarantees that all previous packets for the flow

2446

* This guarantees that all previous packets for the flow

2447

* have been dequeued, thus preserving in order delivery.

2447

* have been dequeued, thus preserving in order delivery.

2448

*/

2448

*/

2449

if (unlikely(tcpu != next_cpu) &&

2449

if (unlikely(tcpu != next_cpu) &&

2450

(tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||

2450

(tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||

2451

((int)(per_cpu(softnet_data, tcpu).input_queue_head -

2451

((int)(per_cpu(softnet_data, tcpu).input_queue_head -

2452

rflow->last_qtail)) >= 0)) {

2452

rflow->last_qtail)) >= 0)) {

2453

tcpu = rflow->cpu = next_cpu;

2453

tcpu = rflow->cpu = next_cpu;

2454

if (tcpu != RPS_NO_CPU)

2454

if (tcpu != RPS_NO_CPU)

2455

rflow->last_qtail = per_cpu(softnet_data,

2455

rflow->last_qtail = per_cpu(softnet_data,

2456

tcpu).input_queue_head;

2456

tcpu).input_queue_head;

2457

}

2457

}

2458

if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {

2458

if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {

2459

*rflowp = rflow;

2459

*rflowp = rflow;

2460

cpu = tcpu;

2460

cpu = tcpu;

2461

goto done;

2461

goto done;

2462

}

2462

}

2463

}

2463

}

2464

2465

if (map) {

2465

if (map) {

2466

tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];

2466

tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];

2467

2468

if (cpu_online(tcpu)) {

2468

if (cpu_online(tcpu)) {

2469

cpu = tcpu;

2469

cpu = tcpu;

2470

goto done;

2470

goto done;

2471

}

2471

}

2472

}

2472

}

2473

2474

done:

2474

done:

2475

return cpu;

2475

return cpu;

2476

}

2476

}

2477

2478

/* Called from hardirq (IPI) context */

2478

/* Called from hardirq (IPI) context */

2479

static void rps_trigger_softirq(void *data)

2479

static void rps_trigger_softirq(void *data)

2480

{

2480

{

2481

struct softnet_data *sd = data;

2481

struct softnet_data *sd = data;

2482

2483

____napi_schedule(sd, &sd->backlog);

2483

____napi_schedule(sd, &sd->backlog);

2484

sd->received_rps++;

2484

sd->received_rps++;

2485

}

2485

}

2486

2487

#endif /* CONFIG_RPS */

2487

#endif /* CONFIG_RPS */

2488

2489

/*

2489

/*

2490

* Check if this softnet_data structure is another cpu one

2490

* Check if this softnet_data structure is another cpu one

2491

* If yes, queue it to our IPI list and return 1

2491

* If yes, queue it to our IPI list and return 1

2492

* If no, return 0

2492

* If no, return 0

2493

*/

2493

*/

2494

static int rps_ipi_queued(struct softnet_data *sd)

2494

static int rps_ipi_queued(struct softnet_data *sd)

2495

{

2495

{

2496

#ifdef CONFIG_RPS

2496

#ifdef CONFIG_RPS

2497

struct softnet_data *mysd = &__get_cpu_var(softnet_data);

2497

struct softnet_data *mysd = &__get_cpu_var(softnet_data);

2498

2499

if (sd != mysd) {

2499

if (sd != mysd) {

2500

sd->rps_ipi_next = mysd->rps_ipi_list;

2500

sd->rps_ipi_next = mysd->rps_ipi_list;

2501

mysd->rps_ipi_list = sd;

2501

mysd->rps_ipi_list = sd;

2502

2503

__raise_softirq_irqoff(NET_RX_SOFTIRQ);

2503

__raise_softirq_irqoff(NET_RX_SOFTIRQ);

2504

return 1;

2504

return 1;

2505

}

2505

}

2506

#endif /* CONFIG_RPS */

2506

#endif /* CONFIG_RPS */

2507

return 0;

2507

return 0;

2508

}

2508

}

2509

2510

/*

2510

/*

2511

* enqueue_to_backlog is called to queue an skb to a per CPU backlog

2511

* enqueue_to_backlog is called to queue an skb to a per CPU backlog

2512

* queue (may be a remote CPU queue).

2512

* queue (may be a remote CPU queue).

2513

*/

2513

*/

2514

static int enqueue_to_backlog(struct sk_buff *skb, int cpu,

2514

static int enqueue_to_backlog(struct sk_buff *skb, int cpu,

2515

unsigned int *qtail)

2515

unsigned int *qtail)

2516

{

2516

{

2517

struct softnet_data *sd;

2517

struct softnet_data *sd;

2518

unsigned long flags;

2518

unsigned long flags;

2519

2520

sd = &per_cpu(softnet_data, cpu);

2520

sd = &per_cpu(softnet_data, cpu);

2521

2522

local_irq_save(flags);

2522

local_irq_save(flags);

2523

2524

rps_lock(sd);

2524

rps_lock(sd);

2525

if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {

2525

if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {

2526

if (skb_queue_len(&sd->input_pkt_queue)) {

2526

if (skb_queue_len(&sd->input_pkt_queue)) {

2527

enqueue:

2527

enqueue:

2528

__skb_queue_tail(&sd->input_pkt_queue, skb);

2528

__skb_queue_tail(&sd->input_pkt_queue, skb);

2529

input_queue_tail_incr_save(sd, qtail);

2529

input_queue_tail_incr_save(sd, qtail);

2530

rps_unlock(sd);

2530

rps_unlock(sd);

2531

local_irq_restore(flags);

2531

local_irq_restore(flags);

2532

return NET_RX_SUCCESS;

2532

return NET_RX_SUCCESS;

2533

}

2533

}

2534

2535

/* Schedule NAPI for backlog device

2535

/* Schedule NAPI for backlog device

2536

* We can use non atomic operation since we own the queue lock

2536

* We can use non atomic operation since we own the queue lock

2537

*/

2537

*/

2538

if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {

2538

if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {

2539

if (!rps_ipi_queued(sd))

2539

if (!rps_ipi_queued(sd))

2540

____napi_schedule(sd, &sd->backlog);

2540

____napi_schedule(sd, &sd->backlog);

2541

}

2541

}

2542

goto enqueue;

2542

goto enqueue;

2543

}

2543

}

2544

2545

sd->dropped++;

2545

sd->dropped++;

2546

rps_unlock(sd);

2546

rps_unlock(sd);

2547

2548

local_irq_restore(flags);

2548

local_irq_restore(flags);

2549

2550

atomic_long_inc(&skb->dev->rx_dropped);

2550

atomic_long_inc(&skb->dev->rx_dropped);

2551

kfree_skb(skb);

2551

kfree_skb(skb);

2552

return NET_RX_DROP;

2552

return NET_RX_DROP;

2553

}

2553

}

2554

2555

/**

2555

/**

2556

* netif_rx - post buffer to the network code

2556

* netif_rx - post buffer to the network code

2557

* @skb: buffer to post

2557

* @skb: buffer to post

2558

*

2558

*

2559

* This function receives a packet from a device driver and queues it for

2559

* This function receives a packet from a device driver and queues it for

2560

* the upper (protocol) levels to process. It always succeeds. The buffer

2560

* the upper (protocol) levels to process. It always succeeds. The buffer

2561

* may be dropped during processing for congestion control or by the

2561

* may be dropped during processing for congestion control or by the

2562

* protocol layers.

2562

* protocol layers.

2563

*

2563

*

2564

* return values:

2564

* return values:

2565

* NET_RX_SUCCESS (no congestion)

2565

* NET_RX_SUCCESS (no congestion)

2566

* NET_RX_DROP (packet was dropped)

2566

* NET_RX_DROP (packet was dropped)

2567

*

2567

*

2568

*/

2568

*/

2569

2570

int netif_rx(struct sk_buff *skb)

2570

int netif_rx(struct sk_buff *skb)

2571

{

2571

{

2572

int ret;

2572

int ret;

2573

2574

/* if netpoll wants it, pretend we never saw it */

2574

/* if netpoll wants it, pretend we never saw it */

2575

if (netpoll_rx(skb))

2575

if (netpoll_rx(skb))

2576

return NET_RX_DROP;

2576

return NET_RX_DROP;

2577

2578

if (netdev_tstamp_prequeue)

2578

if (netdev_tstamp_prequeue)

2579

net_timestamp_check(skb);

2579

net_timestamp_check(skb);

2580

2581

#ifdef CONFIG_RPS

2581

#ifdef CONFIG_RPS

2582

{

2582

{

2583

struct rps_dev_flow voidflow, *rflow = &voidflow;

2583

struct rps_dev_flow voidflow, *rflow = &voidflow;

2584

int cpu;

2584

int cpu;

2585

2586

preempt_disable();

2586

preempt_disable();

2587

rcu_read_lock();

2587

rcu_read_lock();

2588

2589

cpu = get_rps_cpu(skb->dev, skb, &rflow);

2589

cpu = get_rps_cpu(skb->dev, skb, &rflow);

2590

if (cpu < 0)

2590

if (cpu < 0)

2591

cpu = smp_processor_id();

2591

cpu = smp_processor_id();

2592

2593

ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);

2593

ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);

2594

2595

rcu_read_unlock();

2595

rcu_read_unlock();

2596

preempt_enable();

2596

preempt_enable();

2597

}

2597

}

2598

#else

2598

#else

2599

{

2599

{

2600

unsigned int qtail;

2600

unsigned int qtail;

2601

ret = enqueue_to_backlog(skb, get_cpu(), &qtail);

2601

ret = enqueue_to_backlog(skb, get_cpu(), &qtail);

2602

put_cpu();

2602

put_cpu();

2603

}

2603

}

2604

#endif

2604

#endif

2605

return ret;

2605

return ret;

2606

}

2606

}

2607

EXPORT_SYMBOL(netif_rx);

2607

EXPORT_SYMBOL(netif_rx);

2608

2609

int netif_rx_ni(struct sk_buff *skb)

2609

int netif_rx_ni(struct sk_buff *skb)

2610

{

2610

{

2611

int err;

2611

int err;

2612

2613

preempt_disable();

2613

preempt_disable();

2614

err = netif_rx(skb);

2614

err = netif_rx(skb);

2615

if (local_softirq_pending())

2615

if (local_softirq_pending())

2616

do_softirq();

2616

do_softirq();

2617

preempt_enable();

2617

preempt_enable();

2618

2619

return err;

2619

return err;

2620

}

2620

}

2621

EXPORT_SYMBOL(netif_rx_ni);

2621

EXPORT_SYMBOL(netif_rx_ni);

2622

2623

static void net_tx_action(struct softirq_action *h)

2623

static void net_tx_action(struct softirq_action *h)

2624

{

2624

{

2625

struct softnet_data *sd = &__get_cpu_var(softnet_data);

2625

struct softnet_data *sd = &__get_cpu_var(softnet_data);

2626

2627

if (sd->completion_queue) {

2627

if (sd->completion_queue) {

2628

struct sk_buff *clist;

2628

struct sk_buff *clist;

2629

2630

local_irq_disable();

2630

local_irq_disable();

2631

clist = sd->completion_queue;

2631

clist = sd->completion_queue;

2632

sd->completion_queue = NULL;

2632

sd->completion_queue = NULL;

2633

local_irq_enable();

2633

local_irq_enable();

2634

2635

while (clist) {

2635

while (clist) {

2636

struct sk_buff *skb = clist;

2636

struct sk_buff *skb = clist;

2637

clist = clist->next;

2637

clist = clist->next;

2638

2639

WARN_ON(atomic_read(&skb->users));

2639

WARN_ON(atomic_read(&skb->users));

2640

__kfree_skb(skb);

2640

__kfree_skb(skb);

2641

}

2641

}

2642

}

2642

}

2643

2644

if (sd->output_queue) {

2644

if (sd->output_queue) {

2645

struct Qdisc *head;

2645

struct Qdisc *head;

2646

2647

local_irq_disable();

2647

local_irq_disable();

2648

head = sd->output_queue;

2648

head = sd->output_queue;

2649

sd->output_queue = NULL;

2649

sd->output_queue = NULL;

2650

sd->output_queue_tailp = &sd->output_queue;

2650

sd->output_queue_tailp = &sd->output_queue;

2651

local_irq_enable();

2651

local_irq_enable();

2652

2653

while (head) {

2653

while (head) {

2654

struct Qdisc *q = head;

2654

struct Qdisc *q = head;

2655

spinlock_t *root_lock;

2655

spinlock_t *root_lock;

2656

2657

head = head->next_sched;

2657

head = head->next_sched;

2658

2659

root_lock = qdisc_lock(q);

2659

root_lock = qdisc_lock(q);

2660

if (spin_trylock(root_lock)) {

2660

if (spin_trylock(root_lock)) {

2661

smp_mb__before_clear_bit();

2661

smp_mb__before_clear_bit();

2662

clear_bit(__QDISC_STATE_SCHED,

2662

clear_bit(__QDISC_STATE_SCHED,

2663

&q->state);

2663

&q->state);

2664

qdisc_run(q);

2664

qdisc_run(q);

2665

spin_unlock(root_lock);

2665

spin_unlock(root_lock);

2666

} else {

2666

} else {

2667

if (!test_bit(__QDISC_STATE_DEACTIVATED,

2667

if (!test_bit(__QDISC_STATE_DEACTIVATED,

2668

&q->state)) {

2668

&q->state)) {

2669

__netif_reschedule(q);

2669

__netif_reschedule(q);

2670

} else {

2670

} else {

2671

smp_mb__before_clear_bit();

2671

smp_mb__before_clear_bit();

2672

clear_bit(__QDISC_STATE_SCHED,

2672

clear_bit(__QDISC_STATE_SCHED,

2673

&q->state);

2673

&q->state);

2674

}

2674

}

2675

}

2675

}

2676

}

2676

}

2677

}

2677

}

2678

}

2678

}

2679

2680

static inline int deliver_skb(struct sk_buff *skb,

2680

static inline int deliver_skb(struct sk_buff *skb,

2681

struct packet_type *pt_prev,

2681

struct packet_type *pt_prev,

2682

struct net_device *orig_dev)

2682

struct net_device *orig_dev)

2683

{

2683

{

2684

atomic_inc(&skb->users);

2684

atomic_inc(&skb->users);

2685

return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);

2685

return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);

2686

}

2686

}

2687

2688

#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \

2688

#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \

2689

(defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))

2689

(defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))

2690

/* This hook is defined here for ATM LANE */

2690

/* This hook is defined here for ATM LANE */

2691

int (*br_fdb_test_addr_hook)(struct net_device *dev,

2691

int (*br_fdb_test_addr_hook)(struct net_device *dev,

2692

unsigned char *addr) __read_mostly;

2692

unsigned char *addr) __read_mostly;

2693

EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);

2693

EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);

2694

#endif

2694

#endif

2695

2696

#ifdef CONFIG_NET_CLS_ACT

2696

#ifdef CONFIG_NET_CLS_ACT

2697

/* TODO: Maybe we should just force sch_ingress to be compiled in

2697

/* TODO: Maybe we should just force sch_ingress to be compiled in

2698

* when CONFIG_NET_CLS_ACT is? otherwise some useless instructions

2698

* when CONFIG_NET_CLS_ACT is? otherwise some useless instructions

2699

* a compare and 2 stores extra right now if we dont have it on

2699

* a compare and 2 stores extra right now if we dont have it on

2700

* but have CONFIG_NET_CLS_ACT

2700

* but have CONFIG_NET_CLS_ACT

2701

* NOTE: This doesnt stop any functionality; if you dont have

2701

* NOTE: This doesnt stop any functionality; if you dont have

2702

* the ingress scheduler, you just cant add policies on ingress.

2702

* the ingress scheduler, you just cant add policies on ingress.

2703

*

2703

*

2704

*/

2704

*/

2705

static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)

2705

static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)

2706

{

2706

{

2707

struct net_device *dev = skb->dev;

2707

struct net_device *dev = skb->dev;

2708

u32 ttl = G_TC_RTTL(skb->tc_verd);

2708

u32 ttl = G_TC_RTTL(skb->tc_verd);

2709

int result = TC_ACT_OK;

2709

int result = TC_ACT_OK;

2710

struct Qdisc *q;

2710

struct Qdisc *q;

2711

2712

if (unlikely(MAX_RED_LOOP < ttl++)) {

2712

if (unlikely(MAX_RED_LOOP < ttl++)) {

2713

if (net_ratelimit())

2713

if (net_ratelimit())

2714

pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",

2714

pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",

2715

skb->skb_iif, dev->ifindex);

2715

skb->skb_iif, dev->ifindex);

2716

return TC_ACT_SHOT;

2716

return TC_ACT_SHOT;

2717

}

2717

}

2718

2719

skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);

2719

skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);

2720

skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);

2720

skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);

2721

2722

q = rxq->qdisc;

2722

q = rxq->qdisc;

2723

if (q != &noop_qdisc) {

2723

if (q != &noop_qdisc) {

2724

spin_lock(qdisc_lock(q));

2724

spin_lock(qdisc_lock(q));

2725

if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))

2725

if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))

2726

result = qdisc_enqueue_root(skb, q);

2726

result = qdisc_enqueue_root(skb, q);

2727

spin_unlock(qdisc_lock(q));

2727

spin_unlock(qdisc_lock(q));

2728

}

2728

}

2729

2730

return result;

2730

return result;

2731

}

2731

}

2732

2733

static inline struct sk_buff *handle_ing(struct sk_buff *skb,

2733

static inline struct sk_buff *handle_ing(struct sk_buff *skb,

2734

struct packet_type **pt_prev,

2734

struct packet_type **pt_prev,

2735

int *ret, struct net_device *orig_dev)

2735

int *ret, struct net_device *orig_dev)

2736

{

2736

{

2737

struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);

2737

struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);

2738

2739

if (!rxq || rxq->qdisc == &noop_qdisc)

2739

if (!rxq || rxq->qdisc == &noop_qdisc)

2740

goto out;

2740

goto out;

2741

2742

if (*pt_prev) {

2742

if (*pt_prev) {

2743

*ret = deliver_skb(skb, *pt_prev, orig_dev);

2743

*ret = deliver_skb(skb, *pt_prev, orig_dev);

2744

*pt_prev = NULL;

2744

*pt_prev = NULL;

2745

}

2745

}

2746

2747

switch (ing_filter(skb, rxq)) {

2747

switch (ing_filter(skb, rxq)) {

2748

case TC_ACT_SHOT:

2748

case TC_ACT_SHOT:

2749

case TC_ACT_STOLEN:

2749

case TC_ACT_STOLEN:

2750

kfree_skb(skb);

2750

kfree_skb(skb);

2751

return NULL;

2751

return NULL;

2752

}

2752

}

2753

2754

out:

2754

out:

2755

skb->tc_verd = 0;

2755

skb->tc_verd = 0;

2756

return skb;

2756

return skb;

2757

}

2757

}

2758

#endif

2758

#endif

2759

2760

/*

2760

/*

2761

* netif_nit_deliver - deliver received packets to network taps

2761

* netif_nit_deliver - deliver received packets to network taps

2762

* @skb: buffer

2762

* @skb: buffer

2763

*

2763

*

2764

* This function is used to deliver incoming packets to network

2764

* This function is used to deliver incoming packets to network

2765

* taps. It should be used when the normal netif_receive_skb path

2765

* taps. It should be used when the normal netif_receive_skb path

2766

* is bypassed, for example because of VLAN acceleration.

2766

* is bypassed, for example because of VLAN acceleration.

2767

*/

2767

*/

2768

void netif_nit_deliver(struct sk_buff *skb)

2768

void netif_nit_deliver(struct sk_buff *skb)

2769

{

2769

{

2770

struct packet_type *ptype;

2770

struct packet_type *ptype;

2771

2772

if (list_empty(&ptype_all))

2772

if (list_empty(&ptype_all))

2773

return;

2773

return;

2774

2775

skb_reset_network_header(skb);

2775

skb_reset_network_header(skb);

2776

skb_reset_transport_header(skb);

2776

skb_reset_transport_header(skb);

2777

skb->mac_len = skb->network_header - skb->mac_header;

2777

skb->mac_len = skb->network_header - skb->mac_header;

2778

2779

rcu_read_lock();

2779

rcu_read_lock();

2780

list_for_each_entry_rcu(ptype, &ptype_all, list) {

2780

list_for_each_entry_rcu(ptype, &ptype_all, list) {

2781

if (!ptype->dev || ptype->dev == skb->dev)

2781

if (!ptype->dev || ptype->dev == skb->dev)

2782

deliver_skb(skb, ptype, skb->dev);

2782

deliver_skb(skb, ptype, skb->dev);

2783

}

2783

}

2784

rcu_read_unlock();

2784

rcu_read_unlock();

2785

}

2785

}

2786

2787

/**

2787

/**

2788

* netdev_rx_handler_register - register receive handler

2788

* netdev_rx_handler_register - register receive handler

2789

* @dev: device to register a handler for

2789

* @dev: device to register a handler for

2790

* @rx_handler: receive handler to register

2790

* @rx_handler: receive handler to register

2791

* @rx_handler_data: data pointer that is used by rx handler

2791

* @rx_handler_data: data pointer that is used by rx handler

2792

*

2792

*

2793

* Register a receive hander for a device. This handler will then be

2793

* Register a receive hander for a device. This handler will then be

2794

* called from __netif_receive_skb. A negative errno code is returned

2794

* called from __netif_receive_skb. A negative errno code is returned

2795

* on a failure.

2795

* on a failure.

2796

*

2796

*

2797

* The caller must hold the rtnl_mutex.

2797

* The caller must hold the rtnl_mutex.

2798

*/

2798

*/

2799

int netdev_rx_handler_register(struct net_device *dev,

2799

int netdev_rx_handler_register(struct net_device *dev,

2800

rx_handler_func_t *rx_handler,

2800

rx_handler_func_t *rx_handler,

2801

void *rx_handler_data)

2801

void *rx_handler_data)

2802

{

2802

{

2803

ASSERT_RTNL();

2803

ASSERT_RTNL();

2804

2805

if (dev->rx_handler)

2805

if (dev->rx_handler)

2806

return -EBUSY;

2806

return -EBUSY;

2807

2808

rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);

2808

rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);

2809

rcu_assign_pointer(dev->rx_handler, rx_handler);

2809

rcu_assign_pointer(dev->rx_handler, rx_handler);

2810

2811

return 0;

2811

return 0;

2812

}

2812

}

2813

EXPORT_SYMBOL_GPL(netdev_rx_handler_register);

2813

EXPORT_SYMBOL_GPL(netdev_rx_handler_register);

2814

2815

/**

2815

/**

2816

* netdev_rx_handler_unregister - unregister receive handler

2816

* netdev_rx_handler_unregister - unregister receive handler

2817

* @dev: device to unregister a handler from

2817

* @dev: device to unregister a handler from

2818

*

2818

*

2819

* Unregister a receive hander from a device.

2819

* Unregister a receive hander from a device.

2820

*

2820

*

2821

* The caller must hold the rtnl_mutex.

2821

* The caller must hold the rtnl_mutex.

2822

*/

2822

*/

2823

void netdev_rx_handler_unregister(struct net_device *dev)

2823

void netdev_rx_handler_unregister(struct net_device *dev)

2824

{

2824

{

2825

2826

ASSERT_RTNL();

2826

ASSERT_RTNL();

2827

rcu_assign_pointer(dev->rx_handler, NULL);

2827

rcu_assign_pointer(dev->rx_handler, NULL);

2828

rcu_assign_pointer(dev->rx_handler_data, NULL);

2828

rcu_assign_pointer(dev->rx_handler_data, NULL);

2829

}

2829

}

2830

EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);

2830

EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);

2831

2832

static inline void skb_bond_set_mac_by_master(struct sk_buff *skb,

2832

static inline void skb_bond_set_mac_by_master(struct sk_buff *skb,

2833

struct net_device *master)

2833

struct net_device *master)

2834

{

2834

{

2835

if (skb->pkt_type == PACKET_HOST) {

2835

if (skb->pkt_type == PACKET_HOST) {

2836

u16 *dest = (u16 *) eth_hdr(skb)->h_dest;

2836

u16 *dest = (u16 *) eth_hdr(skb)->h_dest;

2837

2838

memcpy(dest, master->dev_addr, ETH_ALEN);

2838

memcpy(dest, master->dev_addr, ETH_ALEN);

2839

}

2839

}

2840

}

2840

}

2841

2842

/* On bonding slaves other than the currently active slave, suppress

2842

/* On bonding slaves other than the currently active slave, suppress

2843

* duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and

2843

* duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and

2844

* ARP on active-backup slaves with arp_validate enabled.

2844

* ARP on active-backup slaves with arp_validate enabled.

2845

*/

2845

*/

2846

int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master)

2846

int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master)

2847

{

2847

{

2848

struct net_device *dev = skb->dev;

2848

struct net_device *dev = skb->dev;

2849

2850

if (master->priv_flags & IFF_MASTER_ARPMON)

2850

if (master->priv_flags & IFF_MASTER_ARPMON)

2851

dev->last_rx = jiffies;

2851

dev->last_rx = jiffies;

2852

2853

if ((master->priv_flags & IFF_MASTER_ALB) &&

2853

if ((master->priv_flags & IFF_MASTER_ALB) &&

2854

(master->priv_flags & IFF_BRIDGE_PORT)) {

2854

(master->priv_flags & IFF_BRIDGE_PORT)) {

2855

/* Do address unmangle. The local destination address

2855

/* Do address unmangle. The local destination address

2856

* will be always the one master has. Provides the right

2856

* will be always the one master has. Provides the right

2857

* functionality in a bridge.

2857

* functionality in a bridge.

2858

*/

2858

*/

2859

skb_bond_set_mac_by_master(skb, master);

2859

skb_bond_set_mac_by_master(skb, master);

2860

}

2860

}

2861

2862

if (dev->priv_flags & IFF_SLAVE_INACTIVE) {

2862

if (dev->priv_flags & IFF_SLAVE_INACTIVE) {

2863

if ((dev->priv_flags & IFF_SLAVE_NEEDARP) &&

2863

if ((dev->priv_flags & IFF_SLAVE_NEEDARP) &&

2864

skb->protocol == __cpu_to_be16(ETH_P_ARP))

2864

skb->protocol == __cpu_to_be16(ETH_P_ARP))

2865

return 0;

2865

return 0;

2866

2867

if (master->priv_flags & IFF_MASTER_ALB) {

2867

if (master->priv_flags & IFF_MASTER_ALB) {

2868

if (skb->pkt_type != PACKET_BROADCAST &&

2868

if (skb->pkt_type != PACKET_BROADCAST &&

2869

skb->pkt_type != PACKET_MULTICAST)

2869

skb->pkt_type != PACKET_MULTICAST)

2870

return 0;

2870

return 0;

2871

}

2871

}

2872

if (master->priv_flags & IFF_MASTER_8023AD &&

2872

if (master->priv_flags & IFF_MASTER_8023AD &&

2873

skb->protocol == __cpu_to_be16(ETH_P_SLOW))

2873

skb->protocol == __cpu_to_be16(ETH_P_SLOW))

2874

return 0;

2874

return 0;

2875

2876

return 1;

2876

return 1;

2877

}

2877

}

2878

return 0;

2878

return 0;

2879

}

2879

}

2880

EXPORT_SYMBOL(__skb_bond_should_drop);

2880

EXPORT_SYMBOL(__skb_bond_should_drop);

2881

2882

static int __netif_receive_skb(struct sk_buff *skb)

2882

static int __netif_receive_skb(struct sk_buff *skb)

2883

{

2883

{

2884

struct packet_type *ptype, *pt_prev;

2884

struct packet_type *ptype, *pt_prev;

2885

rx_handler_func_t *rx_handler;

2885

rx_handler_func_t *rx_handler;

2886

struct net_device *orig_dev;

2886

struct net_device *orig_dev;

2887

struct net_device *master;

2887

struct net_device *master;

2888

struct net_device *null_or_orig;

2888

struct net_device *null_or_orig;

2889

struct net_device *orig_or_bond;

2889

struct net_device *orig_or_bond;

2890

int ret = NET_RX_DROP;

2890

int ret = NET_RX_DROP;

2891

__be16 type;

2891

__be16 type;

2892

2893

if (!netdev_tstamp_prequeue)

2893

if (!netdev_tstamp_prequeue)

2894

net_timestamp_check(skb);

2894

net_timestamp_check(skb);

2895

2896

if (vlan_tx_tag_present(skb))

2896

if (vlan_tx_tag_present(skb))

2897

vlan_hwaccel_do_receive(skb);

2897

vlan_hwaccel_do_receive(skb);

2898

2899

/* if we've gotten here through NAPI, check netpoll */

2899

/* if we've gotten here through NAPI, check netpoll */

2900

if (netpoll_receive_skb(skb))

2900

if (netpoll_receive_skb(skb))

2901

return NET_RX_DROP;

2901

return NET_RX_DROP;

2902

2903

if (!skb->skb_iif)

2903

if (!skb->skb_iif)

2904

skb->skb_iif = skb->dev->ifindex;

2904

skb->skb_iif = skb->dev->ifindex;

2905

2906

/*

2906

/*

2907

* bonding note: skbs received on inactive slaves should only

2907

* bonding note: skbs received on inactive slaves should only

2908

* be delivered to pkt handlers that are exact matches. Also

2908

* be delivered to pkt handlers that are exact matches. Also

2909

* the deliver_no_wcard flag will be set. If packet handlers

2909

* the deliver_no_wcard flag will be set. If packet handlers

2910

* are sensitive to duplicate packets these skbs will need to

2910

* are sensitive to duplicate packets these skbs will need to

2911

* be dropped at the handler. The vlan accel path may have

2911

* be dropped at the handler. The vlan accel path may have

2912

* already set the deliver_no_wcard flag.

2912

* already set the deliver_no_wcard flag.

2913

*/

2913

*/

2914

null_or_orig = NULL;

2914

null_or_orig = NULL;

2915

orig_dev = skb->dev;

2915

orig_dev = skb->dev;

2916

master = ACCESS_ONCE(orig_dev->master);

2916

master = ACCESS_ONCE(orig_dev->master);

2917

if (skb->deliver_no_wcard)

2917

if (skb->deliver_no_wcard)

2918

null_or_orig = orig_dev;

2918

null_or_orig = orig_dev;

2919

else if (master) {

2919

else if (master) {

2920

if (skb_bond_should_drop(skb, master)) {

2920

if (skb_bond_should_drop(skb, master)) {

2921

skb->deliver_no_wcard = 1;

2921

skb->deliver_no_wcard = 1;

2922

null_or_orig = orig_dev; /* deliver only exact match */

2922

null_or_orig = orig_dev; /* deliver only exact match */

2923

} else

2923

} else

2924

skb->dev = master;

2924

skb->dev = master;

2925

}

2925

}

2926

2927

__this_cpu_inc(softnet_data.processed);

2927

__this_cpu_inc(softnet_data.processed);

2928

skb_reset_network_header(skb);

2928

skb_reset_network_header(skb);

2929

skb_reset_transport_header(skb);

2929

skb_reset_transport_header(skb);

2930

skb->mac_len = skb->network_header - skb->mac_header;

2930

skb->mac_len = skb->network_header - skb->mac_header;

2931

2932

pt_prev = NULL;

2932

pt_prev = NULL;

2933

2934

rcu_read_lock();

2934

rcu_read_lock();

2935

2936

#ifdef CONFIG_NET_CLS_ACT

2936

#ifdef CONFIG_NET_CLS_ACT

2937

if (skb->tc_verd & TC_NCLS) {

2937

if (skb->tc_verd & TC_NCLS) {

2938

skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);

2938

skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);

2939

goto ncls;

2939

goto ncls;

2940

}

2940

}

2941

#endif

2941

#endif

2942

2943

list_for_each_entry_rcu(ptype, &ptype_all, list) {

2943

list_for_each_entry_rcu(ptype, &ptype_all, list) {

2944

if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||

2944

if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||

2945

ptype->dev == orig_dev) {

2945

ptype->dev == orig_dev) {

2946

if (pt_prev)

2946

if (pt_prev)

2947

ret = deliver_skb(skb, pt_prev, orig_dev);

2947

ret = deliver_skb(skb, pt_prev, orig_dev);

2948

pt_prev = ptype;

2948

pt_prev = ptype;

2949

}

2949

}

2950

}

2950

}

2951

2952

#ifdef CONFIG_NET_CLS_ACT

2952

#ifdef CONFIG_NET_CLS_ACT

2953

skb = handle_ing(skb, &pt_prev, &ret, orig_dev);

2953

skb = handle_ing(skb, &pt_prev, &ret, orig_dev);

2954

if (!skb)

2954

if (!skb)

2955

goto out;

2955

goto out;

2956

ncls:

2956

ncls:

2957

#endif

2957

#endif

2958

2959

/* Handle special case of bridge or macvlan */

2959

/* Handle special case of bridge or macvlan */

2960

rx_handler = rcu_dereference(skb->dev->rx_handler);

2960

rx_handler = rcu_dereference(skb->dev->rx_handler);

2961

if (rx_handler) {

2961

if (rx_handler) {

2962

if (pt_prev) {

2962

if (pt_prev) {

2963

ret = deliver_skb(skb, pt_prev, orig_dev);

2963

ret = deliver_skb(skb, pt_prev, orig_dev);

2964

pt_prev = NULL;

2964

pt_prev = NULL;

2965

}

2965

}

2966

skb = rx_handler(skb);

2966

skb = rx_handler(skb);

2967

if (!skb)

2967

if (!skb)

2968

goto out;

2968

goto out;

2969

}

2969

}

2970

2971

/*

2971

/*

2972

* Make sure frames received on VLAN interfaces stacked on

2972

* Make sure frames received on VLAN interfaces stacked on

2973

* bonding interfaces still make their way to any base bonding

2973

* bonding interfaces still make their way to any base bonding

2974

* device that may have registered for a specific ptype. The

2974

* device that may have registered for a specific ptype. The

2975

* handler may have to adjust skb->dev and orig_dev.

2975

* handler may have to adjust skb->dev and orig_dev.

2976

*/

2976

*/

2977

orig_or_bond = orig_dev;

2977

orig_or_bond = orig_dev;

2978

if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) &&

2978

if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) &&

2979

(vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) {

2979

(vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) {

2980

orig_or_bond = vlan_dev_real_dev(skb->dev);

2980

orig_or_bond = vlan_dev_real_dev(skb->dev);

2981

}

2981

}

2982

2983

type = skb->protocol;

2983

type = skb->protocol;

2984

list_for_each_entry_rcu(ptype,

2984

list_for_each_entry_rcu(ptype,

2985

&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {

2985

&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {

2986

if (ptype->type == type && (ptype->dev == null_or_orig ||

2986

if (ptype->type == type && (ptype->dev == null_or_orig ||

2987

ptype->dev == skb->dev || ptype->dev == orig_dev ||

2987

ptype->dev == skb->dev || ptype->dev == orig_dev ||

2988

ptype->dev == orig_or_bond)) {

2988

ptype->dev == orig_or_bond)) {

2989

if (pt_prev)

2989

if (pt_prev)

2990

ret = deliver_skb(skb, pt_prev, orig_dev);

2990

ret = deliver_skb(skb, pt_prev, orig_dev);

2991

pt_prev = ptype;

2991

pt_prev = ptype;

2992

}

2992

}

2993

}

2993

}

2994

2995

if (pt_prev) {

2995

if (pt_prev) {

2996

ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);

2996

ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);

2997

} else {

2997

} else {

2998

atomic_long_inc(&skb->dev->rx_dropped);

2998

atomic_long_inc(&skb->dev->rx_dropped);

2999

kfree_skb(skb);

2999

kfree_skb(skb);

3000

/* Jamal, now you will not able to escape explaining

3000

/* Jamal, now you will not able to escape explaining

3001

* me how you were going to use this. :-)

3001

* me how you were going to use this. :-)

3002

*/

3002

*/

3003

ret = NET_RX_DROP;

3003

ret = NET_RX_DROP;

3004

}

3004

}

3005

3006

out:

3006

out:

3007

rcu_read_unlock();

3007

rcu_read_unlock();

3008

return ret;

3008

return ret;

3009

}

3009

}

3010

3011

/**

3011

/**

3012

* netif_receive_skb - process receive buffer from network

3012

* netif_receive_skb - process receive buffer from network

3013

* @skb: buffer to process

3013

* @skb: buffer to process

3014

*

3014

*

3015

* netif_receive_skb() is the main receive data processing function.

3015

* netif_receive_skb() is the main receive data processing function.

3016

* It always succeeds. The buffer may be dropped during processing

3016

* It always succeeds. The buffer may be dropped during processing

3017

* for congestion control or by the protocol layers.

3017

* for congestion control or by the protocol layers.

3018

*

3018

*

3019

* This function may only be called from softirq context and interrupts

3019

* This function may only be called from softirq context and interrupts

3020

* should be enabled.

3020

* should be enabled.

3021

*

3021

*

3022

* Return values (usually ignored):

3022

* Return values (usually ignored):

3023

* NET_RX_SUCCESS: no congestion

3023

* NET_RX_SUCCESS: no congestion

3024

* NET_RX_DROP: packet was dropped

3024

* NET_RX_DROP: packet was dropped

3025

*/

3025

*/

3026

int netif_receive_skb(struct sk_buff *skb)

3026

int netif_receive_skb(struct sk_buff *skb)

3027

{

3027

{

3028

if (netdev_tstamp_prequeue)

3028

if (netdev_tstamp_prequeue)

3029

net_timestamp_check(skb);

3029

net_timestamp_check(skb);

3030

3031

if (skb_defer_rx_timestamp(skb))

3031

if (skb_defer_rx_timestamp(skb))

3032

return NET_RX_SUCCESS;

3032

return NET_RX_SUCCESS;

3033

3034

#ifdef CONFIG_RPS

3034

#ifdef CONFIG_RPS

3035

{

3035

{

3036

struct rps_dev_flow voidflow, *rflow = &voidflow;

3036

struct rps_dev_flow voidflow, *rflow = &voidflow;

3037

int cpu, ret;

3037

int cpu, ret;

3038

3039

rcu_read_lock();

3039

rcu_read_lock();

3040

3041

cpu = get_rps_cpu(skb->dev, skb, &rflow);

3041

cpu = get_rps_cpu(skb->dev, skb, &rflow);

3042

3043

if (cpu >= 0) {

3043

if (cpu >= 0) {

3044

ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);

3044

ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);

3045

rcu_read_unlock();

3045

rcu_read_unlock();

3046

} else {

3046

} else {

3047

rcu_read_unlock();

3047

rcu_read_unlock();

3048

ret = __netif_receive_skb(skb);

3048

ret = __netif_receive_skb(skb);

3049

}

3049

}

3050

3051

return ret;

3051

return ret;

3052

}

3052

}

3053

#else

3053

#else

3054

return __netif_receive_skb(skb);

3054

return __netif_receive_skb(skb);

3055

#endif

3055

#endif

3056

}

3056

}

3057

EXPORT_SYMBOL(netif_receive_skb);

3057

EXPORT_SYMBOL(netif_receive_skb);

3058

3059

/* Network device is going away, flush any packets still pending

3059

/* Network device is going away, flush any packets still pending

3060

* Called with irqs disabled.

3060

* Called with irqs disabled.

3061

*/

3061

*/

3062

static void flush_backlog(void *arg)

3062

static void flush_backlog(void *arg)

3063

{

3063

{

3064

struct net_device *dev = arg;

3064

struct net_device *dev = arg;

3065

struct softnet_data *sd = &__get_cpu_var(softnet_data);

3065

struct softnet_data *sd = &__get_cpu_var(softnet_data);

3066

struct sk_buff *skb, *tmp;

3066

struct sk_buff *skb, *tmp;

3067

3068

rps_lock(sd);

3068

rps_lock(sd);

3069

skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {

3069

skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {

3070

if (skb->dev == dev) {

3070

if (skb->dev == dev) {

3071

__skb_unlink(skb, &sd->input_pkt_queue);

3071

__skb_unlink(skb, &sd->input_pkt_queue);

3072

kfree_skb(skb);

3072

kfree_skb(skb);

3073

input_queue_head_incr(sd);

3073

input_queue_head_incr(sd);

3074

}

3074

}

3075

}

3075

}

3076

rps_unlock(sd);

3076

rps_unlock(sd);

3077

3078

skb_queue_walk_safe(&sd->process_queue, skb, tmp) {

3078

skb_queue_walk_safe(&sd->process_queue, skb, tmp) {

3079

if (skb->dev == dev) {

3079

if (skb->dev == dev) {

3080

__skb_unlink(skb, &sd->process_queue);

3080

__skb_unlink(skb, &sd->process_queue);

3081

kfree_skb(skb);

3081

kfree_skb(skb);

3082

input_queue_head_incr(sd);

3082

input_queue_head_incr(sd);

3083

}

3083

}

3084

}

3084

}

3085

}

3085

}

3086

3087

static int napi_gro_complete(struct sk_buff *skb)

3087

static int napi_gro_complete(struct sk_buff *skb)

3088

{

3088

{

3089

struct packet_type *ptype;

3089

struct packet_type *ptype;

3090

__be16 type = skb->protocol;

3090

__be16 type = skb->protocol;

3091

struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];

3091

struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];

3092

int err = -ENOENT;

3092

int err = -ENOENT;

3093

3094

if (NAPI_GRO_CB(skb)->count == 1) {

3094

if (NAPI_GRO_CB(skb)->count == 1) {

3095

skb_shinfo(skb)->gso_size = 0;

3095

skb_shinfo(skb)->gso_size = 0;

3096

goto out;

3096

goto out;

3097

}

3097

}

3098

3099

rcu_read_lock();

3099

rcu_read_lock();

3100

list_for_each_entry_rcu(ptype, head, list) {

3100

list_for_each_entry_rcu(ptype, head, list) {

3101

if (ptype->type != type || ptype->dev || !ptype->gro_complete)

3101

if (ptype->type != type || ptype->dev || !ptype->gro_complete)

3102

continue;

3102

continue;

3103

3104

err = ptype->gro_complete(skb);

3104

err = ptype->gro_complete(skb);

3105

break;

3105

break;

3106

}

3106

}

3107

rcu_read_unlock();

3107

rcu_read_unlock();

3108

3109

if (err) {

3109

if (err) {

3110

WARN_ON(&ptype->list == head);

3110

WARN_ON(&ptype->list == head);

3111

kfree_skb(skb);

3111

kfree_skb(skb);

3112

return NET_RX_SUCCESS;

3112

return NET_RX_SUCCESS;

3113

}

3113

}

3114

3115

out:

3115

out:

3116

return netif_receive_skb(skb);

3116

return netif_receive_skb(skb);

3117

}

3117

}

3118

3119

inline void napi_gro_flush(struct napi_struct *napi)

3119

inline void napi_gro_flush(struct napi_struct *napi)

3120

{

3120

{

3121

struct sk_buff *skb, *next;

3121

struct sk_buff *skb, *next;

3122

3123

for (skb = napi->gro_list; skb; skb = next) {

3123

for (skb = napi->gro_list; skb; skb = next) {

3124

next = skb->next;

3124

next = skb->next;

3125

skb->next = NULL;

3125

skb->next = NULL;

3126

napi_gro_complete(skb);

3126

napi_gro_complete(skb);

3127

}

3127

}

3128

3129

napi->gro_count = 0;

3129

napi->gro_count = 0;

3130

napi->gro_list = NULL;

3130

napi->gro_list = NULL;

3131

}

3131

}

3132

EXPORT_SYMBOL(napi_gro_flush);

3132

EXPORT_SYMBOL(napi_gro_flush);

3133

3134

enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)

3134

enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)

3135

{

3135

{

3136

struct sk_buff **pp = NULL;

3136

struct sk_buff **pp = NULL;

3137

struct packet_type *ptype;

3137

struct packet_type *ptype;

3138

__be16 type = skb->protocol;

3138

__be16 type = skb->protocol;

3139

struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];

3139

struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];

3140

int same_flow;

3140

int same_flow;

3141

int mac_len;

3141

int mac_len;

3142

enum gro_result ret;

3142

enum gro_result ret;

3143

3144

if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))

3144

if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))

3145

goto normal;

3145

goto normal;

3146

3147

if (skb_is_gso(skb) || skb_has_frag_list(skb))

3147

if (skb_is_gso(skb) || skb_has_frag_list(skb))

3148

goto normal;

3148

goto normal;

3149

3150

rcu_read_lock();

3150

rcu_read_lock();

3151

list_for_each_entry_rcu(ptype, head, list) {

3151

list_for_each_entry_rcu(ptype, head, list) {

3152

if (ptype->type != type || ptype->dev || !ptype->gro_receive)

3152

if (ptype->type != type || ptype->dev || !ptype->gro_receive)

3153

continue;

3153

continue;

3154

3155

skb_set_network_header(skb, skb_gro_offset(skb));

3155

skb_set_network_header(skb, skb_gro_offset(skb));

3156

mac_len = skb->network_header - skb->mac_header;

3156

mac_len = skb->network_header - skb->mac_header;

3157

skb->mac_len = mac_len;

3157

skb->mac_len = mac_len;

3158

NAPI_GRO_CB(skb)->same_flow = 0;

3158

NAPI_GRO_CB(skb)->same_flow = 0;

3159

NAPI_GRO_CB(skb)->flush = 0;

3159

NAPI_GRO_CB(skb)->flush = 0;

3160

NAPI_GRO_CB(skb)->free = 0;

3160

NAPI_GRO_CB(skb)->free = 0;

3161

3162

pp = ptype->gro_receive(&napi->gro_list, skb);

3162

pp = ptype->gro_receive(&napi->gro_list, skb);

3163

break;

3163

break;

3164

}

3164

}

3165

rcu_read_unlock();

3165

rcu_read_unlock();

3166

3167

if (&ptype->list == head)

3167

if (&ptype->list == head)

3168

goto normal;

3168

goto normal;

3169

3170

same_flow = NAPI_GRO_CB(skb)->same_flow;

3170

same_flow = NAPI_GRO_CB(skb)->same_flow;

3171

ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;

3171

ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;

3172

3173

if (pp) {

3173

if (pp) {

3174

struct sk_buff *nskb = *pp;

3174

struct sk_buff *nskb = *pp;

3175

3176

*pp = nskb->next;

3176

*pp = nskb->next;

3177

nskb->next = NULL;

3177

nskb->next = NULL;

3178

napi_gro_complete(nskb);

3178

napi_gro_complete(nskb);

3179

napi->gro_count--;

3179

napi->gro_count--;

3180

}

3180

}

3181

3182

if (same_flow)

3182

if (same_flow)

3183

goto ok;

3183

goto ok;

3184

3185

if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)

3185

if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)

3186

goto normal;

3186

goto normal;

3187

3188

napi->gro_count++;

3188

napi->gro_count++;

3189

NAPI_GRO_CB(skb)->count = 1;

3189

NAPI_GRO_CB(skb)->count = 1;

3190

skb_shinfo(skb)->gso_size = skb_gro_len(skb);

3190

skb_shinfo(skb)->gso_size = skb_gro_len(skb);

3191

skb->next = napi->gro_list;

3191

skb->next = napi->gro_list;

3192

napi->gro_list = skb;

3192

napi->gro_list = skb;

3193

ret = GRO_HELD;

3193

ret = GRO_HELD;

3194

3195

pull:

3195

pull:

3196

if (skb_headlen(skb) < skb_gro_offset(skb)) {

3196

if (skb_headlen(skb) < skb_gro_offset(skb)) {

3197

int grow = skb_gro_offset(skb) - skb_headlen(skb);

3197

int grow = skb_gro_offset(skb) - skb_headlen(skb);

3198

3199

BUG_ON(skb->end - skb->tail < grow);

3199

BUG_ON(skb->end - skb->tail < grow);

3200

3201

memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);

3201

memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);

3202

3203

skb->tail += grow;

3203

skb->tail += grow;

3204

skb->data_len -= grow;

3204

skb->data_len -= grow;

3205

3206

skb_shinfo(skb)->frags[0].page_offset += grow;

3206

skb_shinfo(skb)->frags[0].page_offset += grow;

3207

skb_shinfo(skb)->frags[0].size -= grow;

3207

skb_shinfo(skb)->frags[0].size -= grow;

3208

3209

if (unlikely(!skb_shinfo(skb)->frags[0].size)) {

3209

if (unlikely(!skb_shinfo(skb)->frags[0].size)) {

3210

put_page(skb_shinfo(skb)->frags[0].page);

3210

put_page(skb_shinfo(skb)->frags[0].page);

3211

memmove(skb_shinfo(skb)->frags,

3211

memmove(skb_shinfo(skb)->frags,

3212

skb_shinfo(skb)->frags + 1,

3212

skb_shinfo(skb)->frags + 1,

3213

--skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));

3213

--skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));

3214

}

3214

}

3215

}

3215

}

3216

3217

ok:

3217

ok:

3218

return ret;

3218

return ret;

3219

3220

normal:

3220

normal:

3221

ret = GRO_NORMAL;

3221

ret = GRO_NORMAL;

3222

goto pull;

3222

goto pull;

3223

}

3223

}

3224

EXPORT_SYMBOL(dev_gro_receive);

3224

EXPORT_SYMBOL(dev_gro_receive);

3225

3226

static inline gro_result_t

3226

static inline gro_result_t

3227

__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)

3227

__napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)

3228

{

3228

{

3229

struct sk_buff *p;

3229

struct sk_buff *p;

3230

3231

for (p = napi->gro_list; p; p = p->next) {

3231

for (p = napi->gro_list; p; p = p->next) {

3232

unsigned long diffs;

3232

unsigned long diffs;

3233

3234

diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;

3234

diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;

3235

diffs |= compare_ether_header(skb_mac_header(p),

3235

diffs |= compare_ether_header(skb_mac_header(p),

3236

skb_gro_mac_header(skb));

3236

skb_gro_mac_header(skb));

3237

NAPI_GRO_CB(p)->same_flow = !diffs;

3237

NAPI_GRO_CB(p)->same_flow = !diffs;

3238

NAPI_GRO_CB(p)->flush = 0;

3238

NAPI_GRO_CB(p)->flush = 0;

3239

}

3239

}

3240

3241

return dev_gro_receive(napi, skb);

3241

return dev_gro_receive(napi, skb);

3242

}

3242

}

3243

3244

gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)

3244

gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)

3245

{

3245

{

3246

switch (ret) {

3246

switch (ret) {

3247

case GRO_NORMAL:

3247

case GRO_NORMAL:

3248

if (netif_receive_skb(skb))

3248

if (netif_receive_skb(skb))

3249

ret = GRO_DROP;

3249

ret = GRO_DROP;

3250

break;

3250

break;

3251

3252

case GRO_DROP:

3252

case GRO_DROP:

3253

case GRO_MERGED_FREE:

3253

case GRO_MERGED_FREE:

3254

kfree_skb(skb);

3254

kfree_skb(skb);

3255

break;

3255

break;

3256

3257

case GRO_HELD:

3257

case GRO_HELD:

3258

case GRO_MERGED:

3258

case GRO_MERGED:

3259

break;

3259

break;

3260

}

3260

}

3261

3262

return ret;

3262

return ret;

3263

}

3263

}

3264

EXPORT_SYMBOL(napi_skb_finish);

3264

EXPORT_SYMBOL(napi_skb_finish);

3265

3266

void skb_gro_reset_offset(struct sk_buff *skb)

3266

void skb_gro_reset_offset(struct sk_buff *skb)

3267

{

3267

{

3268

NAPI_GRO_CB(skb)->data_offset = 0;

3268

NAPI_GRO_CB(skb)->data_offset = 0;

3269

NAPI_GRO_CB(skb)->frag0 = NULL;

3269

NAPI_GRO_CB(skb)->frag0 = NULL;

3270

NAPI_GRO_CB(skb)->frag0_len = 0;

3270

NAPI_GRO_CB(skb)->frag0_len = 0;

3271

3272

if (skb->mac_header == skb->tail &&

3272

if (skb->mac_header == skb->tail &&

3273

!PageHighMem(skb_shinfo(skb)->frags[0].page)) {

3273

!PageHighMem(skb_shinfo(skb)->frags[0].page)) {

3274

NAPI_GRO_CB(skb)->frag0 =

3274

NAPI_GRO_CB(skb)->frag0 =

3275

page_address(skb_shinfo(skb)->frags[0].page) +

3275

page_address(skb_shinfo(skb)->frags[0].page) +

3276

skb_shinfo(skb)->frags[0].page_offset;

3276

skb_shinfo(skb)->frags[0].page_offset;

3277

NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;

3277

NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;

3278

}

3278

}

3279

}

3279

}

3280

EXPORT_SYMBOL(skb_gro_reset_offset);

3280

EXPORT_SYMBOL(skb_gro_reset_offset);

3281

3282

gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)

3282

gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)

3283

{

3283

{

3284

skb_gro_reset_offset(skb);

3284

skb_gro_reset_offset(skb);

3285

3286

return napi_skb_finish(__napi_gro_receive(napi, skb), skb);

3286

return napi_skb_finish(__napi_gro_receive(napi, skb), skb);

3287

}

3287

}

3288

EXPORT_SYMBOL(napi_gro_receive);

3288

EXPORT_SYMBOL(napi_gro_receive);

3289

3290

void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)

3290

void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)

3291

{

3291

{

3292

__skb_pull(skb, skb_headlen(skb));

3292

__skb_pull(skb, skb_headlen(skb));

3293

skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));

3293

skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));

3294

3295

napi->skb = skb;

3295

napi->skb = skb;

3296

}

3296

}

3297

EXPORT_SYMBOL(napi_reuse_skb);

3297

EXPORT_SYMBOL(napi_reuse_skb);

3298

3299

struct sk_buff *napi_get_frags(struct napi_struct *napi)

3299

struct sk_buff *napi_get_frags(struct napi_struct *napi)

3300

{

3300

{

3301

struct sk_buff *skb = napi->skb;

3301

struct sk_buff *skb = napi->skb;

3302

3303

if (!skb) {

3303

if (!skb) {

3304

skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);

3304

skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);

3305

if (skb)

3305

if (skb)

3306

napi->skb = skb;

3306

napi->skb = skb;

3307

}

3307

}

3308

return skb;

3308

return skb;

3309

}

3309

}

3310

EXPORT_SYMBOL(napi_get_frags);

3310

EXPORT_SYMBOL(napi_get_frags);

3311

3312

gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,

3312

gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,

3313

gro_result_t ret)

3313

gro_result_t ret)

3314

{

3314

{

3315

switch (ret) {

3315

switch (ret) {

3316

case GRO_NORMAL:

3316

case GRO_NORMAL:

3317

case GRO_HELD:

3317

case GRO_HELD:

3318

skb->protocol = eth_type_trans(skb, skb->dev);

3318

skb->protocol = eth_type_trans(skb, skb->dev);

3319

3320

if (ret == GRO_HELD)

3320

if (ret == GRO_HELD)

3321

skb_gro_pull(skb, -ETH_HLEN);

3321

skb_gro_pull(skb, -ETH_HLEN);

3322

else if (netif_receive_skb(skb))

3322

else if (netif_receive_skb(skb))

3323

ret = GRO_DROP;

3323

ret = GRO_DROP;

3324

break;

3324

break;

3325

3326

case GRO_DROP:

3326

case GRO_DROP:

3327

case GRO_MERGED_FREE:

3327

case GRO_MERGED_FREE:

3328

napi_reuse_skb(napi, skb);

3328

napi_reuse_skb(napi, skb);

3329

break;

3329

break;

3330

3331

case GRO_MERGED:

3331

case GRO_MERGED:

3332

break;

3332

break;

3333

}

3333

}

3334

3335

return ret;

3335

return ret;

3336

}

3336

}

3337

EXPORT_SYMBOL(napi_frags_finish);

3337

EXPORT_SYMBOL(napi_frags_finish);

3338

3339

struct sk_buff *napi_frags_skb(struct napi_struct *napi)

3339

struct sk_buff *napi_frags_skb(struct napi_struct *napi)

3340

{

3340

{

3341

struct sk_buff *skb = napi->skb;

3341

struct sk_buff *skb = napi->skb;

3342

struct ethhdr *eth;

3342

struct ethhdr *eth;

3343

unsigned int hlen;

3343

unsigned int hlen;

3344

unsigned int off;

3344

unsigned int off;

3345

3346

napi->skb = NULL;

3346

napi->skb = NULL;

3347

3348

skb_reset_mac_header(skb);

3348

skb_reset_mac_header(skb);

3349

skb_gro_reset_offset(skb);

3349

skb_gro_reset_offset(skb);

3350

3351

off = skb_gro_offset(skb);

3351

off = skb_gro_offset(skb);

3352

hlen = off + sizeof(*eth);

3352

hlen = off + sizeof(*eth);

3353

eth = skb_gro_header_fast(skb, off);

3353

eth = skb_gro_header_fast(skb, off);

3354

if (skb_gro_header_hard(skb, hlen)) {

3354

if (skb_gro_header_hard(skb, hlen)) {

3355

eth = skb_gro_header_slow(skb, hlen, off);

3355

eth = skb_gro_header_slow(skb, hlen, off);

3356

if (unlikely(!eth)) {

3356

if (unlikely(!eth)) {

3357

napi_reuse_skb(napi, skb);

3357

napi_reuse_skb(napi, skb);

3358

skb = NULL;

3358

skb = NULL;

3359

goto out;

3359

goto out;

3360

}

3360

}

3361

}

3361

}

3362

3363

skb_gro_pull(skb, sizeof(*eth));

3363

skb_gro_pull(skb, sizeof(*eth));

3364

3365

/*

3365

/*

3366

* This works because the only protocols we care about don't require

3366

* This works because the only protocols we care about don't require

3367

* special handling. We'll fix it up properly at the end.

3367

* special handling. We'll fix it up properly at the end.

3368

*/

3368

*/

3369

skb->protocol = eth->h_proto;

3369

skb->protocol = eth->h_proto;

3370

3371

out:

3371

out:

3372

return skb;

3372

return skb;

3373

}

3373

}

3374

EXPORT_SYMBOL(napi_frags_skb);

3374

EXPORT_SYMBOL(napi_frags_skb);

3375

3376

gro_result_t napi_gro_frags(struct napi_struct *napi)

3376

gro_result_t napi_gro_frags(struct napi_struct *napi)

3377

{

3377

{

3378

struct sk_buff *skb = napi_frags_skb(napi);

3378

struct sk_buff *skb = napi_frags_skb(napi);

3379

3380

if (!skb)

3380

if (!skb)

3381

return GRO_DROP;

3381

return GRO_DROP;

3382

3383

return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));

3383

return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));

3384

}

3384

}

3385

EXPORT_SYMBOL(napi_gro_frags);

3385

EXPORT_SYMBOL(napi_gro_frags);

3386

3387

/*

3387

/*

3388

* net_rps_action sends any pending IPI's for rps.

3388

* net_rps_action sends any pending IPI's for rps.

3389

* Note: called with local irq disabled, but exits with local irq enabled.

3389

* Note: called with local irq disabled, but exits with local irq enabled.

3390

*/

3390

*/

3391

static void net_rps_action_and_irq_enable(struct softnet_data *sd)

3391

static void net_rps_action_and_irq_enable(struct softnet_data *sd)

3392

{

3392

{

3393

#ifdef CONFIG_RPS

3393

#ifdef CONFIG_RPS

3394

struct softnet_data *remsd = sd->rps_ipi_list;

3394

struct softnet_data *remsd = sd->rps_ipi_list;

3395

3396

if (remsd) {

3396

if (remsd) {

3397

sd->rps_ipi_list = NULL;

3397

sd->rps_ipi_list = NULL;

3398

3399

local_irq_enable();

3399

local_irq_enable();

3400

3401

/* Send pending IPI's to kick RPS processing on remote cpus. */

3401

/* Send pending IPI's to kick RPS processing on remote cpus. */

3402

while (remsd) {

3402

while (remsd) {

3403

struct softnet_data *next = remsd->rps_ipi_next;

3403

struct softnet_data *next = remsd->rps_ipi_next;

3404

3405

if (cpu_online(remsd->cpu))

3405

if (cpu_online(remsd->cpu))

3406

__smp_call_function_single(remsd->cpu,

3406

__smp_call_function_single(remsd->cpu,

3407

&remsd->csd, 0);

3407

&remsd->csd, 0);

3408

remsd = next;

3408

remsd = next;

3409

}

3409

}

3410

} else

3410

} else

3411

#endif

3411

#endif

3412

local_irq_enable();

3412

local_irq_enable();

3413

}

3413

}

3414

3415

static int process_backlog(struct napi_struct *napi, int quota)

3415

static int process_backlog(struct napi_struct *napi, int quota)

3416

{

3416

{

3417

int work = 0;

3417

int work = 0;

3418

struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);

3418

struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);

3419

3420

#ifdef CONFIG_RPS

3420

#ifdef CONFIG_RPS

3421

/* Check if we have pending ipi, its better to send them now,

3421

/* Check if we have pending ipi, its better to send them now,

3422

* not waiting net_rx_action() end.

3422

* not waiting net_rx_action() end.

3423

*/

3423

*/

3424

if (sd->rps_ipi_list) {

3424

if (sd->rps_ipi_list) {

3425

local_irq_disable();

3425

local_irq_disable();

3426

net_rps_action_and_irq_enable(sd);

3426

net_rps_action_and_irq_enable(sd);

3427

}

3427

}

3428

#endif

3428

#endif

3429

napi->weight = weight_p;

3429

napi->weight = weight_p;

3430

local_irq_disable();

3430

local_irq_disable();

3431

while (work < quota) {

3431

while (work < quota) {

3432

struct sk_buff *skb;

3432

struct sk_buff *skb;

3433

unsigned int qlen;

3433

unsigned int qlen;

3434

3435

while ((skb = __skb_dequeue(&sd->process_queue))) {

3435

while ((skb = __skb_dequeue(&sd->process_queue))) {

3436

local_irq_enable();

3436

local_irq_enable();

3437

__netif_receive_skb(skb);

3437

__netif_receive_skb(skb);

3438

local_irq_disable();

3438

local_irq_disable();

3439

input_queue_head_incr(sd);

3439

input_queue_head_incr(sd);

3440

if (++work >= quota) {

3440

if (++work >= quota) {

3441

local_irq_enable();

3441

local_irq_enable();

3442

return work;

3442

return work;

3443

}

3443

}

3444

}

3444

}

3445

3446

rps_lock(sd);

3446

rps_lock(sd);

3447

qlen = skb_queue_len(&sd->input_pkt_queue);

3447

qlen = skb_queue_len(&sd->input_pkt_queue);

3448

if (qlen)

3448

if (qlen)

3449

skb_queue_splice_tail_init(&sd->input_pkt_queue,

3449

skb_queue_splice_tail_init(&sd->input_pkt_queue,

3450

&sd->process_queue);

3450

&sd->process_queue);

3451

3452

if (qlen < quota - work) {

3452

if (qlen < quota - work) {

3453

/*

3453

/*

3454

* Inline a custom version of __napi_complete().

3454

* Inline a custom version of __napi_complete().

3455

* only current cpu owns and manipulates this napi,

3455

* only current cpu owns and manipulates this napi,

3456

* and NAPI_STATE_SCHED is the only possible flag set on backlog.

3456

* and NAPI_STATE_SCHED is the only possible flag set on backlog.

3457

* we can use a plain write instead of clear_bit(),

3457

* we can use a plain write instead of clear_bit(),

3458

* and we dont need an smp_mb() memory barrier.

3458

* and we dont need an smp_mb() memory barrier.

3459

*/

3459

*/

3460

list_del(&napi->poll_list);

3460

list_del(&napi->poll_list);

3461

napi->state = 0;

3461

napi->state = 0;

3462

3463

quota = work + qlen;

3463

quota = work + qlen;

3464

}

3464

}

3465

rps_unlock(sd);

3465

rps_unlock(sd);

3466

}

3466

}

3467

local_irq_enable();

3467

local_irq_enable();

3468

3469

return work;

3469

return work;

3470

}

3470

}

3471

3472

/**

3472

/**

3473

* __napi_schedule - schedule for receive

3473

* __napi_schedule - schedule for receive

3474

* @n: entry to schedule

3474

* @n: entry to schedule

3475

*

3475

*

3476

* The entry's receive function will be scheduled to run

3476

* The entry's receive function will be scheduled to run

3477

*/

3477

*/

3478

void __napi_schedule(struct napi_struct *n)

3478

void __napi_schedule(struct napi_struct *n)

3479

{

3479

{

3480

unsigned long flags;

3480

unsigned long flags;

3481

3482

local_irq_save(flags);

3482

local_irq_save(flags);

3483

____napi_schedule(&__get_cpu_var(softnet_data), n);

3483

____napi_schedule(&__get_cpu_var(softnet_data), n);

3484

local_irq_restore(flags);

3484

local_irq_restore(flags);

3485

}

3485

}

3486

EXPORT_SYMBOL(__napi_schedule);

3486

EXPORT_SYMBOL(__napi_schedule);

3487

3488

void __napi_complete(struct napi_struct *n)

3488

void __napi_complete(struct napi_struct *n)

3489

{

3489

{

3490

BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));

3490

BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));

3491

BUG_ON(n->gro_list);

3491

BUG_ON(n->gro_list);

3492

3493

list_del(&n->poll_list);

3493

list_del(&n->poll_list);

3494

smp_mb__before_clear_bit();

3494

smp_mb__before_clear_bit();

3495

clear_bit(NAPI_STATE_SCHED, &n->state);

3495

clear_bit(NAPI_STATE_SCHED, &n->state);

3496

}

3496

}

3497

EXPORT_SYMBOL(__napi_complete);

3497

EXPORT_SYMBOL(__napi_complete);

3498

3499

void napi_complete(struct napi_struct *n)

3499

void napi_complete(struct napi_struct *n)

3500

{

3500

{

3501

unsigned long flags;

3501

unsigned long flags;

3502

3503

/*

3503

/*

3504

* don't let napi dequeue from the cpu poll list

3504

* don't let napi dequeue from the cpu poll list

3505

* just in case its running on a different cpu

3505

* just in case its running on a different cpu

3506

*/

3506

*/

3507

if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))

3507

if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))

3508

return;

3508

return;

3509

3510

napi_gro_flush(n);

3510

napi_gro_flush(n);

3511

local_irq_save(flags);

3511

local_irq_save(flags);

3512

__napi_complete(n);

3512

__napi_complete(n);

3513

local_irq_restore(flags);

3513

local_irq_restore(flags);

3514

}

3514

}

3515

EXPORT_SYMBOL(napi_complete);

3515

EXPORT_SYMBOL(napi_complete);

3516

3517

void netif_napi_add(struct net_device *dev, struct napi_struct *napi,

3517

void netif_napi_add(struct net_device *dev, struct napi_struct *napi,

3518

int (*poll)(struct napi_struct *, int), int weight)

3518

int (*poll)(struct napi_struct *, int), int weight)

3519

{

3519

{

3520

INIT_LIST_HEAD(&napi->poll_list);

3520

INIT_LIST_HEAD(&napi->poll_list);

3521

napi->gro_count = 0;

3521

napi->gro_count = 0;

3522

napi->gro_list = NULL;

3522

napi->gro_list = NULL;

3523

napi->skb = NULL;

3523

napi->skb = NULL;

3524

napi->poll = poll;

3524

napi->poll = poll;

3525

napi->weight = weight;

3525

napi->weight = weight;

3526

list_add(&napi->dev_list, &dev->napi_list);

3526

list_add(&napi->dev_list, &dev->napi_list);

3527

napi->dev = dev;

3527

napi->dev = dev;

3528

#ifdef CONFIG_NETPOLL

3528

#ifdef CONFIG_NETPOLL

3529

spin_lock_init(&napi->poll_lock);

3529

spin_lock_init(&napi->poll_lock);

3530

napi->poll_owner = -1;

3530

napi->poll_owner = -1;

3531

#endif

3531

#endif

3532

set_bit(NAPI_STATE_SCHED, &napi->state);

3532

set_bit(NAPI_STATE_SCHED, &napi->state);

3533

}

3533

}

3534

EXPORT_SYMBOL(netif_napi_add);

3534

EXPORT_SYMBOL(netif_napi_add);

3535

3536

void netif_napi_del(struct napi_struct *napi)

3536

void netif_napi_del(struct napi_struct *napi)

3537

{

3537

{

3538

struct sk_buff *skb, *next;

3538

struct sk_buff *skb, *next;

3539

3540

list_del_init(&napi->dev_list);

3540

list_del_init(&napi->dev_list);

3541

napi_free_frags(napi);

3541

napi_free_frags(napi);

3542

3543

for (skb = napi->gro_list; skb; skb = next) {

3543

for (skb = napi->gro_list; skb; skb = next) {

3544

next = skb->next;

3544

next = skb->next;

3545

skb->next = NULL;

3545

skb->next = NULL;

3546

kfree_skb(skb);

3546

kfree_skb(skb);

3547

}

3547

}

3548

3549

napi->gro_list = NULL;

3549

napi->gro_list = NULL;

3550

napi->gro_count = 0;

3550

napi->gro_count = 0;

3551

}

3551

}

3552

EXPORT_SYMBOL(netif_napi_del);

3552

EXPORT_SYMBOL(netif_napi_del);

3553

3554

static void net_rx_action(struct softirq_action *h)

3554

static void net_rx_action(struct softirq_action *h)

3555

{

3555

{

3556

struct softnet_data *sd = &__get_cpu_var(softnet_data);

3556

struct softnet_data *sd = &__get_cpu_var(softnet_data);

3557

unsigned long time_limit = jiffies + 2;

3557

unsigned long time_limit = jiffies + 2;

3558

int budget = netdev_budget;

3558

int budget = netdev_budget;

3559

void *have;

3559

void *have;

3560

3561

local_irq_disable();

3561

local_irq_disable();

3562

3563

while (!list_empty(&sd->poll_list)) {

3563

while (!list_empty(&sd->poll_list)) {

3564

struct napi_struct *n;

3564

struct napi_struct *n;

3565

int work, weight;

3565

int work, weight;

3566

3567

/* If softirq window is exhuasted then punt.

3567

/* If softirq window is exhuasted then punt.

3568

* Allow this to run for 2 jiffies since which will allow

3568

* Allow this to run for 2 jiffies since which will allow

3569

* an average latency of 1.5/HZ.

3569

* an average latency of 1.5/HZ.

3570

*/

3570

*/

3571

if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))

3571

if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))

3572

goto softnet_break;

3572

goto softnet_break;

3573

3574

local_irq_enable();

3574

local_irq_enable();

3575

3576

/* Even though interrupts have been re-enabled, this

3576

/* Even though interrupts have been re-enabled, this

3577

* access is safe because interrupts can only add new

3577

* access is safe because interrupts can only add new

3578

* entries to the tail of this list, and only ->poll()

3578

* entries to the tail of this list, and only ->poll()

3579

* calls can remove this head entry from the list.

3579

* calls can remove this head entry from the list.

3580

*/

3580

*/

3581

n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);

3581

n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);

3582

3583

have = netpoll_poll_lock(n);

3583

have = netpoll_poll_lock(n);

3584

3585

weight = n->weight;

3585

weight = n->weight;

3586

3587

/* This NAPI_STATE_SCHED test is for avoiding a race

3587

/* This NAPI_STATE_SCHED test is for avoiding a race

3588

* with netpoll's poll_napi(). Only the entity which

3588

* with netpoll's poll_napi(). Only the entity which

3589

* obtains the lock and sees NAPI_STATE_SCHED set will

3589

* obtains the lock and sees NAPI_STATE_SCHED set will

3590

* actually make the ->poll() call. Therefore we avoid

3590

* actually make the ->poll() call. Therefore we avoid

3591

* accidently calling ->poll() when NAPI is not scheduled.

3591

* accidently calling ->poll() when NAPI is not scheduled.

3592

*/

3592

*/

3593

work = 0;

3593

work = 0;

3594

if (test_bit(NAPI_STATE_SCHED, &n->state)) {

3594

if (test_bit(NAPI_STATE_SCHED, &n->state)) {

3595

work = n->poll(n, weight);

3595

work = n->poll(n, weight);

3596

trace_napi_poll(n);

3596

trace_napi_poll(n);

3597

}

3597

}

3598

3599

WARN_ON_ONCE(work > weight);

3599

WARN_ON_ONCE(work > weight);

3600

3601

budget -= work;

3601

budget -= work;

3602

3603

local_irq_disable();

3603

local_irq_disable();

3604

3605

/* Drivers must not modify the NAPI state if they

3605

/* Drivers must not modify the NAPI state if they

3606

* consume the entire weight. In such cases this code

3606

* consume the entire weight. In such cases this code

3607

* still "owns" the NAPI instance and therefore can

3607

* still "owns" the NAPI instance and therefore can

3608

* move the instance around on the list at-will.

3608

* move the instance around on the list at-will.

3609

*/

3609

*/

3610

if (unlikely(work == weight)) {

3610

if (unlikely(work == weight)) {

3611

if (unlikely(napi_disable_pending(n))) {

3611

if (unlikely(napi_disable_pending(n))) {

3612

local_irq_enable();

3612

local_irq_enable();

3613

napi_complete(n);

3613

napi_complete(n);

3614

local_irq_disable();

3614

local_irq_disable();

3615

} else

3615

} else

3616

list_move_tail(&n->poll_list, &sd->poll_list);

3616

list_move_tail(&n->poll_list, &sd->poll_list);

3617

}

3617

}

3618

3619

netpoll_poll_unlock(have);

3619

netpoll_poll_unlock(have);

3620

}

3620

}

3621

out:

3621

out:

3622

net_rps_action_and_irq_enable(sd);

3622

net_rps_action_and_irq_enable(sd);

3623

3624

#ifdef CONFIG_NET_DMA

3624

#ifdef CONFIG_NET_DMA

3625

/*

3625

/*

3626

* There may not be any more sk_buffs coming right now, so push

3626

* There may not be any more sk_buffs coming right now, so push

3627

* any pending DMA copies to hardware

3627

* any pending DMA copies to hardware

3628

*/

3628

*/

3629

dma_issue_pending_all();

3629

dma_issue_pending_all();

3630

#endif

3630

#endif

3631

3632

return;

3632

return;

3633

3634

softnet_break:

3634

softnet_break:

3635

sd->time_squeeze++;

3635

sd->time_squeeze++;

3636

__raise_softirq_irqoff(NET_RX_SOFTIRQ);

3636

__raise_softirq_irqoff(NET_RX_SOFTIRQ);

3637

goto out;

3637

goto out;

3638

}

3638

}

3639

3640

static gifconf_func_t *gifconf_list[NPROTO];

3640

static gifconf_func_t *gifconf_list[NPROTO];

3641

3642

/**

3642

/**

3643

* register_gifconf - register a SIOCGIF handler

3643

* register_gifconf - register a SIOCGIF handler

3644

* @family: Address family

3644

* @family: Address family

3645

* @gifconf: Function handler

3645

* @gifconf: Function handler

3646

*

3646

*

3647

* Register protocol dependent address dumping routines. The handler

3647

* Register protocol dependent address dumping routines. The handler

3648

* that is passed must not be freed or reused until it has been replaced

3648

* that is passed must not be freed or reused until it has been replaced

3649

* by another handler.

3649

* by another handler.

3650

*/

3650

*/

3651

int register_gifconf(unsigned int family, gifconf_func_t *gifconf)

3651

int register_gifconf(unsigned int family, gifconf_func_t *gifconf)

3652

{

3652

{

3653

if (family >= NPROTO)

3653

if (family >= NPROTO)

3654

return -EINVAL;

3654

return -EINVAL;

3655

gifconf_list[family] = gifconf;

3655

gifconf_list[family] = gifconf;

3656

return 0;

3656

return 0;

3657

}

3657

}

3658

EXPORT_SYMBOL(register_gifconf);

3658

EXPORT_SYMBOL(register_gifconf);

3659

3660

3661

/*

3661

/*

3662

* Map an interface index to its name (SIOCGIFNAME)

3662

* Map an interface index to its name (SIOCGIFNAME)

3663

*/

3663

*/

3664

3665

/*

3665

/*

3666

* We need this ioctl for efficient implementation of the

3666

* We need this ioctl for efficient implementation of the

3667

* if_indextoname() function required by the IPv6 API. Without

3667

* if_indextoname() function required by the IPv6 API. Without

3668

* it, we would have to search all the interfaces to find a

3668

* it, we would have to search all the interfaces to find a

3669

* match. --pb

3669

* match. --pb

3670

*/

3670

*/

3671

3672

static int dev_ifname(struct net *net, struct ifreq __user *arg)

3672

static int dev_ifname(struct net *net, struct ifreq __user *arg)

3673

{

3673

{

3674

struct net_device *dev;

3674

struct net_device *dev;

3675

struct ifreq ifr;

3675

struct ifreq ifr;

3676

3677

/*

3677

/*

3678

* Fetch the caller's info block.

3678

* Fetch the caller's info block.

3679

*/

3679

*/

3680

3681

if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))

3681

if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))

3682

return -EFAULT;

3682

return -EFAULT;

3683

3684

rcu_read_lock();

3684

rcu_read_lock();

3685

dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);

3685

dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);

3686

if (!dev) {

3686

if (!dev) {

3687

rcu_read_unlock();

3687

rcu_read_unlock();

3688

return -ENODEV;

3688

return -ENODEV;

3689

}

3689

}

3690

3691

strcpy(ifr.ifr_name, dev->name);

3691

strcpy(ifr.ifr_name, dev->name);

3692

rcu_read_unlock();

3692

rcu_read_unlock();

3693

3694

if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))

3694

if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))

3695

return -EFAULT;

3695

return -EFAULT;

3696

return 0;

3696

return 0;

3697

}

3697

}

3698

3699

/*

3699

/*

3700

* Perform a SIOCGIFCONF call. This structure will change

3700

* Perform a SIOCGIFCONF call. This structure will change

3701

* size eventually, and there is nothing I can do about it.

3701

* size eventually, and there is nothing I can do about it.

3702

* Thus we will need a 'compatibility mode'.

3702

* Thus we will need a 'compatibility mode'.

3703

*/

3703

*/

3704

3705

static int dev_ifconf(struct net *net, char __user *arg)

3705

static int dev_ifconf(struct net *net, char __user *arg)

3706

{

3706

{

3707

struct ifconf ifc;

3707

struct ifconf ifc;

3708

struct net_device *dev;

3708

struct net_device *dev;

3709

char __user *pos;

3709

char __user *pos;

3710

int len;

3710

int len;

3711

int total;

3711

int total;

3712

int i;

3712

int i;

3713

3714

/*

3714

/*

3715

* Fetch the caller's info block.

3715

* Fetch the caller's info block.

3716

*/

3716

*/

3717

3718

if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))

3718

if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))

3719

return -EFAULT;

3719

return -EFAULT;

3720

3721

pos = ifc.ifc_buf;

3721

pos = ifc.ifc_buf;

3722

len = ifc.ifc_len;

3722

len = ifc.ifc_len;

3723

3724

/*

3724

/*

3725

* Loop over the interfaces, and write an info block for each.

3725

* Loop over the interfaces, and write an info block for each.

3726

*/

3726

*/

3727

3728

total = 0;

3728

total = 0;

3729

for_each_netdev(net, dev) {

3729

for_each_netdev(net, dev) {

3730

for (i = 0; i < NPROTO; i++) {

3730

for (i = 0; i < NPROTO; i++) {

3731

if (gifconf_list[i]) {

3731

if (gifconf_list[i]) {

3732

int done;

3732

int done;

3733

if (!pos)

3733

if (!pos)

3734

done = gifconf_list[i](dev, NULL, 0);

3734

done = gifconf_list[i](dev, NULL, 0);

3735

else

3735

else

3736

done = gifconf_list[i](dev, pos + total,

3736

done = gifconf_list[i](dev, pos + total,

3737

len - total);

3737

len - total);

3738

if (done < 0)

3738

if (done < 0)

3739

return -EFAULT;

3739

return -EFAULT;

3740

total += done;

3740

total += done;

3741

}

3741

}

3742

}

3742

}

3743

}

3743

}

3744

3745

/*

3745

/*

3746

* All done. Write the updated control block back to the caller.

3746

* All done. Write the updated control block back to the caller.

3747

*/

3747

*/

3748

ifc.ifc_len = total;

3748

ifc.ifc_len = total;

3749

3750

/*

3750

/*

3751

* Both BSD and Solaris return 0 here, so we do too.

3751

* Both BSD and Solaris return 0 here, so we do too.

3752

*/

3752

*/

3753

return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;

3753

return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;

3754

}

3754

}

3755

3756

#ifdef CONFIG_PROC_FS

3756

#ifdef CONFIG_PROC_FS

3757

/*

3757

/*

3758

* This is invoked by the /proc filesystem handler to display a device

3758

* This is invoked by the /proc filesystem handler to display a device

3759

* in detail.

3759

* in detail.

3760

*/

3760

*/

3761

void *dev_seq_start(struct seq_file *seq, loff_t *pos)

3761

void *dev_seq_start(struct seq_file *seq, loff_t *pos)

3762

__acquires(RCU)

3762

__acquires(RCU)

3763

{

3763

{

3764

struct net *net = seq_file_net(seq);

3764

struct net *net = seq_file_net(seq);

3765

loff_t off;

3765

loff_t off;

3766

struct net_device *dev;

3766

struct net_device *dev;

3767

3768

rcu_read_lock();

3768

rcu_read_lock();

3769

if (!*pos)

3769

if (!*pos)

3770

return SEQ_START_TOKEN;

3770

return SEQ_START_TOKEN;

3771

3772

off = 1;

3772

off = 1;

3773

for_each_netdev_rcu(net, dev)

3773

for_each_netdev_rcu(net, dev)

3774

if (off++ == *pos)

3774

if (off++ == *pos)

3775

return dev;

3775

return dev;

3776

3777

return NULL;

3777

return NULL;

3778

}

3778

}

3779

3780

void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)

3780

void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)

3781

{

3781

{

3782

struct net_device *dev = (v == SEQ_START_TOKEN) ?

3782

struct net_device *dev = (v == SEQ_START_TOKEN) ?

3783

first_net_device(seq_file_net(seq)) :

3783

first_net_device(seq_file_net(seq)) :

3784

next_net_device((struct net_device *)v);

3784

next_net_device((struct net_device *)v);

3785

3786

++*pos;

3786

++*pos;

3787

return rcu_dereference(dev);

3787

return rcu_dereference(dev);

3788

}

3788

}

3789

3790

void dev_seq_stop(struct seq_file *seq, void *v)

3790

void dev_seq_stop(struct seq_file *seq, void *v)

3791

__releases(RCU)

3791

__releases(RCU)

3792

{

3792

{

3793

rcu_read_unlock();

3793

rcu_read_unlock();

3794

}

3794

}

3795

3796

static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)

3796

static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)

3797

{

3797

{

3798

struct rtnl_link_stats64 temp;

3798

struct rtnl_link_stats64 temp;

3799

const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);

3799

const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);

3800

3801

seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "

3801

seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "

3802

"%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",

3802

"%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",

3803

dev->name, stats->rx_bytes, stats->rx_packets,

3803

dev->name, stats->rx_bytes, stats->rx_packets,

3804

stats->rx_errors,

3804

stats->rx_errors,

3805

stats->rx_dropped + stats->rx_missed_errors,

3805

stats->rx_dropped + stats->rx_missed_errors,

3806

stats->rx_fifo_errors,

3806

stats->rx_fifo_errors,

3807

stats->rx_length_errors + stats->rx_over_errors +

3807

stats->rx_length_errors + stats->rx_over_errors +

3808

stats->rx_crc_errors + stats->rx_frame_errors,

3808

stats->rx_crc_errors + stats->rx_frame_errors,

3809

stats->rx_compressed, stats->multicast,

3809

stats->rx_compressed, stats->multicast,

3810

stats->tx_bytes, stats->tx_packets,

3810

stats->tx_bytes, stats->tx_packets,

3811

stats->tx_errors, stats->tx_dropped,

3811

stats->tx_errors, stats->tx_dropped,

3812

stats->tx_fifo_errors, stats->collisions,

3812

stats->tx_fifo_errors, stats->collisions,

3813

stats->tx_carrier_errors +

3813

stats->tx_carrier_errors +

3814

stats->tx_aborted_errors +

3814

stats->tx_aborted_errors +

3815

stats->tx_window_errors +

3815

stats->tx_window_errors +

3816

stats->tx_heartbeat_errors,

3816

stats->tx_heartbeat_errors,

3817

stats->tx_compressed);

3817

stats->tx_compressed);

3818

}

3818

}

3819

3820

/*

3820

/*

3821

* Called from the PROCfs module. This now uses the new arbitrary sized

3821

* Called from the PROCfs module. This now uses the new arbitrary sized

3822

* /proc/net interface to create /proc/net/dev

3822

* /proc/net interface to create /proc/net/dev

3823

*/

3823

*/

3824

static int dev_seq_show(struct seq_file *seq, void *v)

3824

static int dev_seq_show(struct seq_file *seq, void *v)

3825

{

3825

{

3826

if (v == SEQ_START_TOKEN)

3826

if (v == SEQ_START_TOKEN)

3827

seq_puts(seq, "Inter-| Receive "

3827

seq_puts(seq, "Inter-| Receive "

3828

" | Transmit\n"

3828

" | Transmit\n"

3829

" face |bytes packets errs drop fifo frame "

3829

" face |bytes packets errs drop fifo frame "

3830

"compressed multicast|bytes packets errs "

3830

"compressed multicast|bytes packets errs "

3831

"drop fifo colls carrier compressed\n");

3831

"drop fifo colls carrier compressed\n");

3832

else

3832

else

3833

dev_seq_printf_stats(seq, v);

3833

dev_seq_printf_stats(seq, v);

3834

return 0;

3834

return 0;

3835

}

3835

}

3836

3837

static struct softnet_data *softnet_get_online(loff_t *pos)

3837

static struct softnet_data *softnet_get_online(loff_t *pos)

3838

{

3838

{

3839

struct softnet_data *sd = NULL;

3839

struct softnet_data *sd = NULL;

3840

3841

while (*pos < nr_cpu_ids)

3841

while (*pos < nr_cpu_ids)

3842

if (cpu_online(*pos)) {

3842

if (cpu_online(*pos)) {

3843

sd = &per_cpu(softnet_data, *pos);

3843

sd = &per_cpu(softnet_data, *pos);

3844

break;

3844

break;

3845

} else

3845

} else

3846

++*pos;

3846

++*pos;

3847

return sd;

3847

return sd;

3848

}

3848

}

3849

3850

static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)

3850

static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)

3851

{

3851

{

3852

return softnet_get_online(pos);

3852

return softnet_get_online(pos);

3853

}

3853

}

3854

3855

static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)

3855

static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)

3856

{

3856

{

3857

++*pos;

3857

++*pos;

3858

return softnet_get_online(pos);

3858

return softnet_get_online(pos);

3859

}

3859

}

3860

3861

static void softnet_seq_stop(struct seq_file *seq, void *v)

3861

static void softnet_seq_stop(struct seq_file *seq, void *v)

3862

{

3862

{

3863

}

3863

}

3864

3865

static int softnet_seq_show(struct seq_file *seq, void *v)

3865

static int softnet_seq_show(struct seq_file *seq, void *v)

3866

{

3866

{

3867

struct softnet_data *sd = v;

3867

struct softnet_data *sd = v;

3868

3869

seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",

3869

seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",

3870

sd->processed, sd->dropped, sd->time_squeeze, 0,

3870

sd->processed, sd->dropped, sd->time_squeeze, 0,

3871

0, 0, 0, 0, /* was fastroute */

3871

0, 0, 0, 0, /* was fastroute */

3872

sd->cpu_collision, sd->received_rps);

3872

sd->cpu_collision, sd->received_rps);

3873

return 0;

3873

return 0;

3874

}

3874

}

3875

3876

static const struct seq_operations dev_seq_ops = {

3876

static const struct seq_operations dev_seq_ops = {

3877

.start = dev_seq_start,

3877

.start = dev_seq_start,

3878

.next = dev_seq_next,

3878

.next = dev_seq_next,

3879

.stop = dev_seq_stop,

3879

.stop = dev_seq_stop,

3880

.show = dev_seq_show,

3880

.show = dev_seq_show,

3881

};

3881

};

3882

3883

static int dev_seq_open(struct inode *inode, struct file *file)

3883

static int dev_seq_open(struct inode *inode, struct file *file)

3884

{

3884

{

3885

return seq_open_net(inode, file, &dev_seq_ops,

3885

return seq_open_net(inode, file, &dev_seq_ops,

3886

sizeof(struct seq_net_private));

3886

sizeof(struct seq_net_private));

3887

}

3887

}

3888

3889

static const struct file_operations dev_seq_fops = {

3889

static const struct file_operations dev_seq_fops = {

3890

.owner = THIS_MODULE,

3890

.owner = THIS_MODULE,

3891

.open = dev_seq_open,

3891

.open = dev_seq_open,

3892

.read = seq_read,

3892

.read = seq_read,

3893

.llseek = seq_lseek,

3893

.llseek = seq_lseek,

3894

.release = seq_release_net,

3894

.release = seq_release_net,

3895

};

3895

};

3896

3897

static const struct seq_operations softnet_seq_ops = {

3897

static const struct seq_operations softnet_seq_ops = {

3898

.start = softnet_seq_start,

3898

.start = softnet_seq_start,

3899

.next = softnet_seq_next,

3899

.next = softnet_seq_next,

3900

.stop = softnet_seq_stop,

3900

.stop = softnet_seq_stop,

3901

.show = softnet_seq_show,

3901

.show = softnet_seq_show,

3902

};

3902

};

3903

3904

static int softnet_seq_open(struct inode *inode, struct file *file)

3904

static int softnet_seq_open(struct inode *inode, struct file *file)

3905

{

3905

{

3906

return seq_open(file, &softnet_seq_ops);

3906

return seq_open(file, &softnet_seq_ops);

3907

}

3907

}

3908

3909

static const struct file_operations softnet_seq_fops = {

3909

static const struct file_operations softnet_seq_fops = {

3910

.owner = THIS_MODULE,

3910

.owner = THIS_MODULE,

3911

.open = softnet_seq_open,

3911

.open = softnet_seq_open,

3912

.read = seq_read,

3912

.read = seq_read,

3913

.llseek = seq_lseek,

3913

.llseek = seq_lseek,

3914

.release = seq_release,

3914

.release = seq_release,

3915

};

3915

};

3916

3917

static void *ptype_get_idx(loff_t pos)

3917

static void *ptype_get_idx(loff_t pos)

3918

{

3918

{

3919

struct packet_type *pt = NULL;

3919

struct packet_type *pt = NULL;

3920

loff_t i = 0;

3920

loff_t i = 0;

3921

int t;

3921

int t;

3922

3923

list_for_each_entry_rcu(pt, &ptype_all, list) {

3923

list_for_each_entry_rcu(pt, &ptype_all, list) {

3924

if (i == pos)

3924

if (i == pos)

3925

return pt;

3925

return pt;

3926

++i;

3926

++i;

3927

}

3927

}

3928

3929

for (t = 0; t < PTYPE_HASH_SIZE; t++) {

3929

for (t = 0; t < PTYPE_HASH_SIZE; t++) {

3930

list_for_each_entry_rcu(pt, &ptype_base[t], list) {

3930

list_for_each_entry_rcu(pt, &ptype_base[t], list) {

3931

if (i == pos)

3931

if (i == pos)

3932

return pt;

3932

return pt;

3933

++i;

3933

++i;

3934

}

3934

}

3935

}

3935

}

3936

return NULL;

3936

return NULL;

3937

}

3937

}

3938

3939

static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)

3939

static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)

3940

__acquires(RCU)

3940

__acquires(RCU)

3941

{

3941

{

3942

rcu_read_lock();

3942

rcu_read_lock();

3943

return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;

3943

return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;

3944

}

3944

}

3945

3946

static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)

3946

static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)

3947

{

3947

{

3948

struct packet_type *pt;

3948

struct packet_type *pt;

3949

struct list_head *nxt;

3949

struct list_head *nxt;

3950

int hash;

3950

int hash;

3951

3952

++*pos;

3952

++*pos;

3953

if (v == SEQ_START_TOKEN)

3953

if (v == SEQ_START_TOKEN)

3954

return ptype_get_idx(0);

3954

return ptype_get_idx(0);

3955

3956

pt = v;

3956

pt = v;

3957

nxt = pt->list.next;

3957

nxt = pt->list.next;

3958

if (pt->type == htons(ETH_P_ALL)) {

3958

if (pt->type == htons(ETH_P_ALL)) {

3959

if (nxt != &ptype_all)

3959

if (nxt != &ptype_all)

3960

goto found;

3960

goto found;

3961

hash = 0;

3961

hash = 0;

3962

nxt = ptype_base[0].next;

3962

nxt = ptype_base[0].next;

3963

} else

3963

} else

3964

hash = ntohs(pt->type) & PTYPE_HASH_MASK;

3964

hash = ntohs(pt->type) & PTYPE_HASH_MASK;

3965

3966

while (nxt == &ptype_base[hash]) {

3966

while (nxt == &ptype_base[hash]) {

3967

if (++hash >= PTYPE_HASH_SIZE)

3967

if (++hash >= PTYPE_HASH_SIZE)

3968

return NULL;

3968

return NULL;

3969

nxt = ptype_base[hash].next;

3969

nxt = ptype_base[hash].next;

3970

}

3970

}

3971

found:

3971

found:

3972

return list_entry(nxt, struct packet_type, list);

3972

return list_entry(nxt, struct packet_type, list);

3973

}

3973

}

3974

3975

static void ptype_seq_stop(struct seq_file *seq, void *v)

3975

static void ptype_seq_stop(struct seq_file *seq, void *v)

3976

__releases(RCU)

3976

__releases(RCU)

3977

{

3977

{

3978

rcu_read_unlock();

3978

rcu_read_unlock();

3979

}

3979

}

3980

3981

static int ptype_seq_show(struct seq_file *seq, void *v)

3981

static int ptype_seq_show(struct seq_file *seq, void *v)

3982

{

3982

{

3983

struct packet_type *pt = v;

3983

struct packet_type *pt = v;

3984

3985

if (v == SEQ_START_TOKEN)

3985

if (v == SEQ_START_TOKEN)

3986

seq_puts(seq, "Type Device Function\n");

3986

seq_puts(seq, "Type Device Function\n");

3987

else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {

3987

else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {

3988

if (pt->type == htons(ETH_P_ALL))

3988

if (pt->type == htons(ETH_P_ALL))

3989

seq_puts(seq, "ALL ");

3989

seq_puts(seq, "ALL ");

3990

else

3990

else

3991

seq_printf(seq, "%04x", ntohs(pt->type));

3991

seq_printf(seq, "%04x", ntohs(pt->type));

3992

3993

seq_printf(seq, " %-8s %pF\n",

3993

seq_printf(seq, " %-8s %pF\n",

3994

pt->dev ? pt->dev->name : "", pt->func);

3994

pt->dev ? pt->dev->name : "", pt->func);

3995

}

3995

}

3996

3997

return 0;

3997

return 0;

3998

}

3998

}

3999

4000

static const struct seq_operations ptype_seq_ops = {

4000

static const struct seq_operations ptype_seq_ops = {

4001

.start = ptype_seq_start,

4001

.start = ptype_seq_start,

4002

.next = ptype_seq_next,

4002

.next = ptype_seq_next,

4003

.stop = ptype_seq_stop,

4003

.stop = ptype_seq_stop,

4004

.show = ptype_seq_show,

4004

.show = ptype_seq_show,

4005

};

4005

};

4006

4007

static int ptype_seq_open(struct inode *inode, struct file *file)

4007

static int ptype_seq_open(struct inode *inode, struct file *file)

4008

{

4008

{

4009

return seq_open_net(inode, file, &ptype_seq_ops,

4009

return seq_open_net(inode, file, &ptype_seq_ops,

4010

sizeof(struct seq_net_private));

4010

sizeof(struct seq_net_private));

4011

}

4011

}

4012

4013

static const struct file_operations ptype_seq_fops = {

4013

static const struct file_operations ptype_seq_fops = {

4014

.owner = THIS_MODULE,

4014

.owner = THIS_MODULE,

4015

.open = ptype_seq_open,

4015

.open = ptype_seq_open,

4016

.read = seq_read,

4016

.read = seq_read,

4017

.llseek = seq_lseek,

4017

.llseek = seq_lseek,

4018

.release = seq_release_net,

4018

.release = seq_release_net,

4019

};

4019

};

4020

4021

4022

static int __net_init dev_proc_net_init(struct net *net)

4022

static int __net_init dev_proc_net_init(struct net *net)

4023

{

4023

{

4024

int rc = -ENOMEM;

4024

int rc = -ENOMEM;

4025

4026

if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))

4026

if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))

4027

goto out;

4027

goto out;

4028

if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))

4028

if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))

4029

goto out_dev;

4029

goto out_dev;

4030

if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))

4030

if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))

4031

goto out_softnet;

4031

goto out_softnet;

4032

4033

if (wext_proc_init(net))

4033

if (wext_proc_init(net))

4034

goto out_ptype;

4034

goto out_ptype;

4035

rc = 0;

4035

rc = 0;

4036

out:

4036

out:

4037

return rc;

4037

return rc;

4038

out_ptype:

4038

out_ptype:

4039

proc_net_remove(net, "ptype");

4039

proc_net_remove(net, "ptype");

4040

out_softnet:

4040

out_softnet:

4041

proc_net_remove(net, "softnet_stat");

4041

proc_net_remove(net, "softnet_stat");

4042

out_dev:

4042

out_dev:

4043

proc_net_remove(net, "dev");

4043

proc_net_remove(net, "dev");

4044

goto out;

4044

goto out;

4045

}

4045

}

4046

4047

static void __net_exit dev_proc_net_exit(struct net *net)

4047

static void __net_exit dev_proc_net_exit(struct net *net)

4048

{

4048

{

4049

wext_proc_exit(net);

4049

wext_proc_exit(net);

4050

4051

proc_net_remove(net, "ptype");

4051

proc_net_remove(net, "ptype");

4052

proc_net_remove(net, "softnet_stat");

4052

proc_net_remove(net, "softnet_stat");

4053

proc_net_remove(net, "dev");

4053

proc_net_remove(net, "dev");

4054

}

4054

}

4055

4056

static struct pernet_operations __net_initdata dev_proc_ops = {

4056

static struct pernet_operations __net_initdata dev_proc_ops = {

4057

.init = dev_proc_net_init,

4057

.init = dev_proc_net_init,

4058

.exit = dev_proc_net_exit,

4058

.exit = dev_proc_net_exit,

4059

};

4059

};

4060

4061

static int __init dev_proc_init(void)

4061

static int __init dev_proc_init(void)

4062

{

4062

{

4063

return register_pernet_subsys(&dev_proc_ops);

4063

return register_pernet_subsys(&dev_proc_ops);

4064

}

4064

}

4065

#else

4065

#else

4066

#define dev_proc_init() 0

4066

#define dev_proc_init() 0

4067

#endif /* CONFIG_PROC_FS */

4067

#endif /* CONFIG_PROC_FS */

4068

4069

4070

/**

4070

/**

4071

* netdev_set_master - set up master/slave pair

4071

* netdev_set_master - set up master/slave pair

4072

* @slave: slave device

4072

* @slave: slave device

4073

* @master: new master device

4073

* @master: new master device

4074

*

4074

*

4075

* Changes the master device of the slave. Pass %NULL to break the

4075

* Changes the master device of the slave. Pass %NULL to break the

4076

* bonding. The caller must hold the RTNL semaphore. On a failure

4076

* bonding. The caller must hold the RTNL semaphore. On a failure

4077

* a negative errno code is returned. On success the reference counts

4077

* a negative errno code is returned. On success the reference counts

4078

* are adjusted, %RTM_NEWLINK is sent to the routing socket and the

4078

* are adjusted, %RTM_NEWLINK is sent to the routing socket and the

4079

* function returns zero.

4079

* function returns zero.

4080

*/

4080

*/

4081

int netdev_set_master(struct net_device *slave, struct net_device *master)

4081

int netdev_set_master(struct net_device *slave, struct net_device *master)

4082

{

4082

{

4083

struct net_device *old = slave->master;

4083

struct net_device *old = slave->master;

4084

4085

ASSERT_RTNL();

4085

ASSERT_RTNL();

4086

4087

if (master) {

4087

if (master) {

4088

if (old)

4088

if (old)

4089

return -EBUSY;

4089

return -EBUSY;

4090

dev_hold(master);

4090

dev_hold(master);

4091

}

4091

}

4092

4093

slave->master = master;

4093

slave->master = master;

4094

4095

if (old) {

4095

if (old) {

4096

synchronize_net();

4096

synchronize_net();

4097

dev_put(old);

4097

dev_put(old);

4098

}

4098

}

4099

if (master)

4099

if (master)

4100

slave->flags |= IFF_SLAVE;

4100

slave->flags |= IFF_SLAVE;

4101

else

4101

else

4102

slave->flags &= ~IFF_SLAVE;

4102

slave->flags &= ~IFF_SLAVE;

4103

4104

rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);

4104

rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);

4105

return 0;

4105

return 0;

4106

}

4106

}

4107

EXPORT_SYMBOL(netdev_set_master);

4107

EXPORT_SYMBOL(netdev_set_master);

4108

4109

static void dev_change_rx_flags(struct net_device *dev, int flags)

4109

static void dev_change_rx_flags(struct net_device *dev, int flags)

4110

{

4110

{

4111

const struct net_device_ops *ops = dev->netdev_ops;

4111

const struct net_device_ops *ops = dev->netdev_ops;

4112

4113

if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)

4113

if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)

4114

ops->ndo_change_rx_flags(dev, flags);

4114

ops->ndo_change_rx_flags(dev, flags);

4115

}

4115

}

4116

4117

static int __dev_set_promiscuity(struct net_device *dev, int inc)

4117

static int __dev_set_promiscuity(struct net_device *dev, int inc)

4118

{

4118

{

4119

unsigned short old_flags = dev->flags;

4119

unsigned short old_flags = dev->flags;

4120

uid_t uid;

4120

uid_t uid;

4121

gid_t gid;

4121

gid_t gid;

4122

4123

ASSERT_RTNL();

4123

ASSERT_RTNL();

4124

4125

dev->flags |= IFF_PROMISC;

4125

dev->flags |= IFF_PROMISC;

4126

dev->promiscuity += inc;

4126

dev->promiscuity += inc;

4127

if (dev->promiscuity == 0) {

4127

if (dev->promiscuity == 0) {

4128

/*

4128

/*

4129

* Avoid overflow.

4129

* Avoid overflow.

4130

* If inc causes overflow, untouch promisc and return error.

4130

* If inc causes overflow, untouch promisc and return error.

4131

*/

4131

*/

4132

if (inc < 0)

4132

if (inc < 0)

4133

dev->flags &= ~IFF_PROMISC;

4133

dev->flags &= ~IFF_PROMISC;

4134

else {

4134

else {

4135

dev->promiscuity -= inc;

4135

dev->promiscuity -= inc;

4136

printk(KERN_WARNING "%s: promiscuity touches roof, "

4136

printk(KERN_WARNING "%s: promiscuity touches roof, "

4137

"set promiscuity failed, promiscuity feature "

4137

"set promiscuity failed, promiscuity feature "

4138

"of device might be broken.\n", dev->name);

4138

"of device might be broken.\n", dev->name);

4139

return -EOVERFLOW;

4139

return -EOVERFLOW;

4140

}

4140

}

4141

}

4141

}

4142

if (dev->flags != old_flags) {

4142

if (dev->flags != old_flags) {

4143

printk(KERN_INFO "device %s %s promiscuous mode\n",

4143

printk(KERN_INFO "device %s %s promiscuous mode\n",

4144

dev->name, (dev->flags & IFF_PROMISC) ? "entered" :

4144

dev->name, (dev->flags & IFF_PROMISC) ? "entered" :

4145

"left");

4145

"left");

4146

if (audit_enabled) {

4146

if (audit_enabled) {

4147

current_uid_gid(&uid, &gid);

4147

current_uid_gid(&uid, &gid);

4148

audit_log(current->audit_context, GFP_ATOMIC,

4148

audit_log(current->audit_context, GFP_ATOMIC,

4149

AUDIT_ANOM_PROMISCUOUS,

4149

AUDIT_ANOM_PROMISCUOUS,

4150

"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",

4150

"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",

4151

dev->name, (dev->flags & IFF_PROMISC),

4151

dev->name, (dev->flags & IFF_PROMISC),

4152

(old_flags & IFF_PROMISC),

4152

(old_flags & IFF_PROMISC),

4153

audit_get_loginuid(current),

4153

audit_get_loginuid(current),

4154

uid, gid,

4154

uid, gid,

4155

audit_get_sessionid(current));

4155

audit_get_sessionid(current));

4156

}

4156

}

4157

4158

dev_change_rx_flags(dev, IFF_PROMISC);

4158

dev_change_rx_flags(dev, IFF_PROMISC);

4159

}

4159

}

4160

return 0;

4160

return 0;

4161

}

4161

}

4162

4163

/**

4163

/**

4164

* dev_set_promiscuity - update promiscuity count on a device

4164

* dev_set_promiscuity - update promiscuity count on a device

4165

* @dev: device

4165

* @dev: device

4166

* @inc: modifier

4166

* @inc: modifier

4167

*

4167

*

4168

* Add or remove promiscuity from a device. While the count in the device

4168

* Add or remove promiscuity from a device. While the count in the device

4169

* remains above zero the interface remains promiscuous. Once it hits zero

4169

* remains above zero the interface remains promiscuous. Once it hits zero

4170

* the device reverts back to normal filtering operation. A negative inc

4170

* the device reverts back to normal filtering operation. A negative inc

4171

* value is used to drop promiscuity on the device.

4171

* value is used to drop promiscuity on the device.

4172

* Return 0 if successful or a negative errno code on error.

4172

* Return 0 if successful or a negative errno code on error.

4173

*/

4173

*/

4174

int dev_set_promiscuity(struct net_device *dev, int inc)

4174

int dev_set_promiscuity(struct net_device *dev, int inc)

4175

{

4175

{

4176

unsigned short old_flags = dev->flags;

4176

unsigned short old_flags = dev->flags;

4177

int err;

4177

int err;

4178

4179

err = __dev_set_promiscuity(dev, inc);

4179

err = __dev_set_promiscuity(dev, inc);

4180

if (err < 0)

4180

if (err < 0)

4181

return err;

4181

return err;

4182

if (dev->flags != old_flags)

4182

if (dev->flags != old_flags)

4183

dev_set_rx_mode(dev);

4183

dev_set_rx_mode(dev);

4184

return err;

4184

return err;

4185

}

4185

}

4186

EXPORT_SYMBOL(dev_set_promiscuity);

4186

EXPORT_SYMBOL(dev_set_promiscuity);

4187

4188

/**

4188

/**

4189

* dev_set_allmulti - update allmulti count on a device

4189

* dev_set_allmulti - update allmulti count on a device

4190

* @dev: device

4190

* @dev: device

4191

* @inc: modifier

4191

* @inc: modifier

4192

*

4192

*

4193

* Add or remove reception of all multicast frames to a device. While the

4193

* Add or remove reception of all multicast frames to a device. While the

4194

* count in the device remains above zero the interface remains listening

4194

* count in the device remains above zero the interface remains listening

4195

* to all interfaces. Once it hits zero the device reverts back to normal

4195

* to all interfaces. Once it hits zero the device reverts back to normal

4196

* filtering operation. A negative @inc value is used to drop the counter

4196

* filtering operation. A negative @inc value is used to drop the counter

4197

* when releasing a resource needing all multicasts.

4197

* when releasing a resource needing all multicasts.

4198

* Return 0 if successful or a negative errno code on error.

4198

* Return 0 if successful or a negative errno code on error.

4199

*/

4199

*/

4200

4201

int dev_set_allmulti(struct net_device *dev, int inc)

4201

int dev_set_allmulti(struct net_device *dev, int inc)

4202

{

4202

{

4203

unsigned short old_flags = dev->flags;

4203

unsigned short old_flags = dev->flags;

4204

4205

ASSERT_RTNL();

4205

ASSERT_RTNL();

4206

4207

dev->flags |= IFF_ALLMULTI;

4207

dev->flags |= IFF_ALLMULTI;

4208

dev->allmulti += inc;

4208

dev->allmulti += inc;

4209

if (dev->allmulti == 0) {

4209

if (dev->allmulti == 0) {

4210

/*

4210

/*

4211

* Avoid overflow.

4211

* Avoid overflow.

4212

* If inc causes overflow, untouch allmulti and return error.

4212

* If inc causes overflow, untouch allmulti and return error.

4213

*/

4213

*/

4214

if (inc < 0)

4214

if (inc < 0)

4215

dev->flags &= ~IFF_ALLMULTI;

4215

dev->flags &= ~IFF_ALLMULTI;

4216

else {

4216

else {

4217

dev->allmulti -= inc;

4217

dev->allmulti -= inc;

4218

printk(KERN_WARNING "%s: allmulti touches roof, "

4218

printk(KERN_WARNING "%s: allmulti touches roof, "

4219

"set allmulti failed, allmulti feature of "

4219

"set allmulti failed, allmulti feature of "

4220

"device might be broken.\n", dev->name);

4220

"device might be broken.\n", dev->name);

4221

return -EOVERFLOW;

4221

return -EOVERFLOW;

4222

}

4222

}

4223

}

4223

}

4224

if (dev->flags ^ old_flags) {

4224

if (dev->flags ^ old_flags) {

4225

dev_change_rx_flags(dev, IFF_ALLMULTI);

4225

dev_change_rx_flags(dev, IFF_ALLMULTI);

4226

dev_set_rx_mode(dev);

4226

dev_set_rx_mode(dev);

4227

}

4227

}

4228

return 0;

4228

return 0;

4229

}

4229

}

4230

EXPORT_SYMBOL(dev_set_allmulti);

4230

EXPORT_SYMBOL(dev_set_allmulti);

4231

4232

/*

4232

/*

4233

* Upload unicast and multicast address lists to device and

4233

* Upload unicast and multicast address lists to device and

4234

* configure RX filtering. When the device doesn't support unicast

4234

* configure RX filtering. When the device doesn't support unicast

4235

* filtering it is put in promiscuous mode while unicast addresses

4235

* filtering it is put in promiscuous mode while unicast addresses

4236

* are present.

4236

* are present.

4237

*/

4237

*/

4238

void __dev_set_rx_mode(struct net_device *dev)

4238

void __dev_set_rx_mode(struct net_device *dev)

4239

{

4239

{

4240

const struct net_device_ops *ops = dev->netdev_ops;

4240

const struct net_device_ops *ops = dev->netdev_ops;

4241

4242

/* dev_open will call this function so the list will stay sane. */

4242

/* dev_open will call this function so the list will stay sane. */

4243

if (!(dev->flags&IFF_UP))

4243

if (!(dev->flags&IFF_UP))

4244

return;

4244

return;

4245

4246

if (!netif_device_present(dev))

4246

if (!netif_device_present(dev))

4247

return;

4247

return;

4248

4249

if (ops->ndo_set_rx_mode)

4249

if (ops->ndo_set_rx_mode)

4250

ops->ndo_set_rx_mode(dev);

4250

ops->ndo_set_rx_mode(dev);

4251

else {

4251

else {

4252

/* Unicast addresses changes may only happen under the rtnl,

4252

/* Unicast addresses changes may only happen under the rtnl,

4253

* therefore calling __dev_set_promiscuity here is safe.

4253

* therefore calling __dev_set_promiscuity here is safe.

4254

*/

4254

*/

4255

if (!netdev_uc_empty(dev) && !dev->uc_promisc) {

4255

if (!netdev_uc_empty(dev) && !dev->uc_promisc) {

4256

__dev_set_promiscuity(dev, 1);

4256

__dev_set_promiscuity(dev, 1);

4257

dev->uc_promisc = 1;

4257

dev->uc_promisc = 1;

4258

} else if (netdev_uc_empty(dev) && dev->uc_promisc) {

4258

} else if (netdev_uc_empty(dev) && dev->uc_promisc) {

4259

__dev_set_promiscuity(dev, -1);

4259

__dev_set_promiscuity(dev, -1);

4260

dev->uc_promisc = 0;

4260

dev->uc_promisc = 0;

4261

}

4261

}

4262

4263

if (ops->ndo_set_multicast_list)

4263

if (ops->ndo_set_multicast_list)

4264

ops->ndo_set_multicast_list(dev);

4264

ops->ndo_set_multicast_list(dev);

4265

}

4265

}

4266

}

4266

}

4267

4268

void dev_set_rx_mode(struct net_device *dev)

4268

void dev_set_rx_mode(struct net_device *dev)

4269

{

4269

{

4270

netif_addr_lock_bh(dev);

4270

netif_addr_lock_bh(dev);

4271

__dev_set_rx_mode(dev);

4271

__dev_set_rx_mode(dev);

4272

netif_addr_unlock_bh(dev);

4272

netif_addr_unlock_bh(dev);

4273

}

4273

}

4274

4275

/**

4275

/**

4276

* dev_get_flags - get flags reported to userspace

4276

* dev_get_flags - get flags reported to userspace

4277

* @dev: device

4277

* @dev: device

4278

*

4278

*

4279

* Get the combination of flag bits exported through APIs to userspace.

4279

* Get the combination of flag bits exported through APIs to userspace.

4280

*/

4280

*/

4281

unsigned dev_get_flags(const struct net_device *dev)

4281

unsigned dev_get_flags(const struct net_device *dev)

4282

{

4282

{

4283

unsigned flags;

4283

unsigned flags;

4284

4285

flags = (dev->flags & ~(IFF_PROMISC |

4285

flags = (dev->flags & ~(IFF_PROMISC |

4286

IFF_ALLMULTI |

4286

IFF_ALLMULTI |

4287

IFF_RUNNING |

4287

IFF_RUNNING |

4288

IFF_LOWER_UP |

4288

IFF_LOWER_UP |

4289

IFF_DORMANT)) |

4289

IFF_DORMANT)) |

4290

(dev->gflags & (IFF_PROMISC |

4290

(dev->gflags & (IFF_PROMISC |

4291

IFF_ALLMULTI));

4291

IFF_ALLMULTI));

4292

4293

if (netif_running(dev)) {

4293

if (netif_running(dev)) {

4294

if (netif_oper_up(dev))

4294

if (netif_oper_up(dev))

4295

flags |= IFF_RUNNING;

4295

flags |= IFF_RUNNING;

4296

if (netif_carrier_ok(dev))

4296

if (netif_carrier_ok(dev))

4297

flags |= IFF_LOWER_UP;

4297

flags |= IFF_LOWER_UP;

4298

if (netif_dormant(dev))

4298

if (netif_dormant(dev))

4299

flags |= IFF_DORMANT;

4299

flags |= IFF_DORMANT;

4300

}

4300

}

4301

4302

return flags;

4302

return flags;

4303

}

4303

}

4304

EXPORT_SYMBOL(dev_get_flags);

4304

EXPORT_SYMBOL(dev_get_flags);

4305

4306

int __dev_change_flags(struct net_device *dev, unsigned int flags)

4306

int __dev_change_flags(struct net_device *dev, unsigned int flags)

4307

{

4307

{

4308

int old_flags = dev->flags;

4308

int old_flags = dev->flags;

4309

int ret;

4309

int ret;

4310

4311

ASSERT_RTNL();

4311

ASSERT_RTNL();

4312

4313

/*

4313

/*

4314

* Set the flags on our device.

4314

* Set the flags on our device.

4315

*/

4315

*/

4316

4317

dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |

4317

dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |

4318

IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |

4318

IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |

4319

IFF_AUTOMEDIA)) |

4319

IFF_AUTOMEDIA)) |

4320

(dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |

4320

(dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |

4321

IFF_ALLMULTI));

4321

IFF_ALLMULTI));

4322

4323

/*

4323

/*

4324

* Load in the correct multicast list now the flags have changed.

4324

* Load in the correct multicast list now the flags have changed.

4325

*/

4325

*/

4326

4327

if ((old_flags ^ flags) & IFF_MULTICAST)

4327

if ((old_flags ^ flags) & IFF_MULTICAST)

4328

dev_change_rx_flags(dev, IFF_MULTICAST);

4328

dev_change_rx_flags(dev, IFF_MULTICAST);

4329

4330

dev_set_rx_mode(dev);

4330

dev_set_rx_mode(dev);

4331

4332

/*

4332

/*

4333

* Have we downed the interface. We handle IFF_UP ourselves

4333

* Have we downed the interface. We handle IFF_UP ourselves

4334

* according to user attempts to set it, rather than blindly

4334

* according to user attempts to set it, rather than blindly

4335

* setting it.

4335

* setting it.

4336

*/

4336

*/

4337

4338

ret = 0;

4338

ret = 0;

4339

if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */

4339

if ((old_flags ^ flags) & IFF_UP) { /* Bit is different ? */

4340

ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);

4340

ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);

4341

4342

if (!ret)

4342

if (!ret)

4343

dev_set_rx_mode(dev);

4343

dev_set_rx_mode(dev);

4344

}

4344

}

4345

4346

if ((flags ^ dev->gflags) & IFF_PROMISC) {

4346

if ((flags ^ dev->gflags) & IFF_PROMISC) {

4347

int inc = (flags & IFF_PROMISC) ? 1 : -1;

4347

int inc = (flags & IFF_PROMISC) ? 1 : -1;

4348

4349

dev->gflags ^= IFF_PROMISC;

4349

dev->gflags ^= IFF_PROMISC;

4350

dev_set_promiscuity(dev, inc);

4350

dev_set_promiscuity(dev, inc);

4351

}

4351

}

4352

4353

/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI

4353

/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI

4354

is important. Some (broken) drivers set IFF_PROMISC, when

4354

is important. Some (broken) drivers set IFF_PROMISC, when

4355

IFF_ALLMULTI is requested not asking us and not reporting.

4355

IFF_ALLMULTI is requested not asking us and not reporting.

4356

*/

4356

*/

4357

if ((flags ^ dev->gflags) & IFF_ALLMULTI) {

4357

if ((flags ^ dev->gflags) & IFF_ALLMULTI) {

4358

int inc = (flags & IFF_ALLMULTI) ? 1 : -1;

4358

int inc = (flags & IFF_ALLMULTI) ? 1 : -1;

4359

4360

dev->gflags ^= IFF_ALLMULTI;

4360

dev->gflags ^= IFF_ALLMULTI;

4361

dev_set_allmulti(dev, inc);

4361

dev_set_allmulti(dev, inc);

4362

}

4362

}

4363

4364

return ret;

4364

return ret;

4365

}

4365

}

4366

4367

void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)

4367

void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)

4368

{

4368

{

4369

unsigned int changes = dev->flags ^ old_flags;

4369

unsigned int changes = dev->flags ^ old_flags;

4370

4371

if (changes & IFF_UP) {

4371

if (changes & IFF_UP) {

4372

if (dev->flags & IFF_UP)

4372

if (dev->flags & IFF_UP)

4373

call_netdevice_notifiers(NETDEV_UP, dev);

4373

call_netdevice_notifiers(NETDEV_UP, dev);

4374

else

4374

else

4375

call_netdevice_notifiers(NETDEV_DOWN, dev);

4375

call_netdevice_notifiers(NETDEV_DOWN, dev);

4376

}

4376

}

4377

4378

if (dev->flags & IFF_UP &&

4378

if (dev->flags & IFF_UP &&

4379

(changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))

4379

(changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))

4380

call_netdevice_notifiers(NETDEV_CHANGE, dev);

4380

call_netdevice_notifiers(NETDEV_CHANGE, dev);

4381

}

4381

}

4382

4383

/**

4383

/**

4384

* dev_change_flags - change device settings

4384

* dev_change_flags - change device settings

4385

* @dev: device

4385

* @dev: device

4386

* @flags: device state flags

4386

* @flags: device state flags

4387

*

4387

*

4388

* Change settings on device based state flags. The flags are

4388

* Change settings on device based state flags. The flags are

4389

* in the userspace exported format.

4389

* in the userspace exported format.

4390

*/

4390

*/

4391

int dev_change_flags(struct net_device *dev, unsigned flags)

4391

int dev_change_flags(struct net_device *dev, unsigned flags)

4392

{

4392

{

4393

int ret, changes;

4393

int ret, changes;

4394

int old_flags = dev->flags;

4394

int old_flags = dev->flags;

4395

4396

ret = __dev_change_flags(dev, flags);

4396

ret = __dev_change_flags(dev, flags);

4397

if (ret < 0)

4397

if (ret < 0)

4398

return ret;

4398

return ret;

4399

4400

changes = old_flags ^ dev->flags;

4400

changes = old_flags ^ dev->flags;

4401

if (changes)

4401

if (changes)

4402

rtmsg_ifinfo(RTM_NEWLINK, dev, changes);

4402

rtmsg_ifinfo(RTM_NEWLINK, dev, changes);

4403

4404

__dev_notify_flags(dev, old_flags);

4404

__dev_notify_flags(dev, old_flags);

4405

return ret;

4405

return ret;

4406

}

4406

}

4407

EXPORT_SYMBOL(dev_change_flags);

4407

EXPORT_SYMBOL(dev_change_flags);

4408

4409

/**

4409

/**

4410

* dev_set_mtu - Change maximum transfer unit

4410

* dev_set_mtu - Change maximum transfer unit

4411

* @dev: device

4411

* @dev: device

4412

* @new_mtu: new transfer unit

4412

* @new_mtu: new transfer unit

4413

*

4413

*

4414

* Change the maximum transfer size of the network device.

4414

* Change the maximum transfer size of the network device.

4415

*/

4415

*/

4416

int dev_set_mtu(struct net_device *dev, int new_mtu)

4416

int dev_set_mtu(struct net_device *dev, int new_mtu)

4417

{

4417

{

4418

const struct net_device_ops *ops = dev->netdev_ops;

4418

const struct net_device_ops *ops = dev->netdev_ops;

4419

int err;

4419

int err;

4420

4421

if (new_mtu == dev->mtu)

4421

if (new_mtu == dev->mtu)

4422

return 0;

4422

return 0;

4423

4424

/* MTU must be positive. */

4424

/* MTU must be positive. */

4425

if (new_mtu < 0)

4425

if (new_mtu < 0)

4426

return -EINVAL;

4426

return -EINVAL;

4427

4428

if (!netif_device_present(dev))

4428

if (!netif_device_present(dev))

4429

return -ENODEV;

4429

return -ENODEV;

4430

4431

err = 0;

4431

err = 0;

4432

if (ops->ndo_change_mtu)

4432

if (ops->ndo_change_mtu)

4433

err = ops->ndo_change_mtu(dev, new_mtu);

4433

err = ops->ndo_change_mtu(dev, new_mtu);

4434

else

4434

else

4435

dev->mtu = new_mtu;

4435

dev->mtu = new_mtu;

4436

4437

if (!err && dev->flags & IFF_UP)

4437

if (!err && dev->flags & IFF_UP)

4438

call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);

4438

call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);

4439

return err;

4439

return err;

4440

}

4440

}

4441

EXPORT_SYMBOL(dev_set_mtu);

4441

EXPORT_SYMBOL(dev_set_mtu);

4442

4443

/**

4443

/**

4444

* dev_set_mac_address - Change Media Access Control Address

4444

* dev_set_mac_address - Change Media Access Control Address

4445

* @dev: device

4445

* @dev: device

4446

* @sa: new address

4446

* @sa: new address

4447

*

4447

*

4448

* Change the hardware (MAC) address of the device

4448

* Change the hardware (MAC) address of the device

4449

*/

4449

*/

4450

int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)

4450

int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)

4451

{

4451

{

4452

const struct net_device_ops *ops = dev->netdev_ops;

4452

const struct net_device_ops *ops = dev->netdev_ops;

4453

int err;

4453

int err;

4454

4455

if (!ops->ndo_set_mac_address)

4455

if (!ops->ndo_set_mac_address)

4456

return -EOPNOTSUPP;

4456

return -EOPNOTSUPP;

4457

if (sa->sa_family != dev->type)

4457

if (sa->sa_family != dev->type)

4458

return -EINVAL;

4458

return -EINVAL;

4459

if (!netif_device_present(dev))

4459

if (!netif_device_present(dev))

4460

return -ENODEV;

4460

return -ENODEV;

4461

err = ops->ndo_set_mac_address(dev, sa);

4461

err = ops->ndo_set_mac_address(dev, sa);

4462

if (!err)

4462

if (!err)

4463

call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);

4463

call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);

4464

return err;

4464

return err;

4465

}

4465

}

4466

EXPORT_SYMBOL(dev_set_mac_address);

4466

EXPORT_SYMBOL(dev_set_mac_address);

4467

4468

/*

4468

/*

4469

* Perform the SIOCxIFxxx calls, inside rcu_read_lock()

4469

* Perform the SIOCxIFxxx calls, inside rcu_read_lock()

4470

*/

4470

*/

4471

static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)

4471

static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)

4472

{

4472

{

4473

int err;

4473

int err;

4474

struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);

4474

struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);

4475

4476

if (!dev)

4476

if (!dev)

4477

return -ENODEV;

4477

return -ENODEV;

4478

4479

switch (cmd) {

4479

switch (cmd) {

4480

case SIOCGIFFLAGS: /* Get interface flags */

4480

case SIOCGIFFLAGS: /* Get interface flags */

4481

ifr->ifr_flags = (short) dev_get_flags(dev);

4481

ifr->ifr_flags = (short) dev_get_flags(dev);

4482

return 0;

4482

return 0;

4483

4484

case SIOCGIFMETRIC: /* Get the metric on the interface

4484

case SIOCGIFMETRIC: /* Get the metric on the interface

4485

(currently unused) */

4485

(currently unused) */

4486

ifr->ifr_metric = 0;

4486

ifr->ifr_metric = 0;

4487

return 0;

4487

return 0;

4488

4489

case SIOCGIFMTU: /* Get the MTU of a device */

4489

case SIOCGIFMTU: /* Get the MTU of a device */

4490

ifr->ifr_mtu = dev->mtu;

4490

ifr->ifr_mtu = dev->mtu;

4491

return 0;

4491

return 0;

4492

4493

case SIOCGIFHWADDR:

4493

case SIOCGIFHWADDR:

4494

if (!dev->addr_len)

4494

if (!dev->addr_len)

4495

memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);

4495

memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);

4496

else

4496

else

4497

memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,

4497

memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,

4498

min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));

4498

min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));

4499

ifr->ifr_hwaddr.sa_family = dev->type;

4499

ifr->ifr_hwaddr.sa_family = dev->type;

4500

return 0;

4500

return 0;

4501

4502

case SIOCGIFSLAVE:

4502

case SIOCGIFSLAVE:

4503

err = -EINVAL;

4503

err = -EINVAL;

4504

break;

4504

break;

4505

4506

case SIOCGIFMAP:

4506

case SIOCGIFMAP:

4507

ifr->ifr_map.mem_start = dev->mem_start;

4507

ifr->ifr_map.mem_start = dev->mem_start;

4508

ifr->ifr_map.mem_end = dev->mem_end;

4508

ifr->ifr_map.mem_end = dev->mem_end;

4509

ifr->ifr_map.base_addr = dev->base_addr;

4509

ifr->ifr_map.base_addr = dev->base_addr;

4510

ifr->ifr_map.irq = dev->irq;

4510

ifr->ifr_map.irq = dev->irq;

4511

ifr->ifr_map.dma = dev->dma;

4511

ifr->ifr_map.dma = dev->dma;

4512

ifr->ifr_map.port = dev->if_port;

4512

ifr->ifr_map.port = dev->if_port;

4513

return 0;

4513

return 0;

4514

4515

case SIOCGIFINDEX:

4515

case SIOCGIFINDEX:

4516

ifr->ifr_ifindex = dev->ifindex;

4516

ifr->ifr_ifindex = dev->ifindex;

4517

return 0;

4517

return 0;

4518

4519

case SIOCGIFTXQLEN:

4519

case SIOCGIFTXQLEN:

4520

ifr->ifr_qlen = dev->tx_queue_len;

4520

ifr->ifr_qlen = dev->tx_queue_len;

4521

return 0;

4521

return 0;

4522

4523

default:

4523

default:

4524

/* dev_ioctl() should ensure this case

4524

/* dev_ioctl() should ensure this case

4525

* is never reached

4525

* is never reached

4526

*/

4526

*/

4527

WARN_ON(1);

4527

WARN_ON(1);

4528

err = -EINVAL;

4528

err = -EINVAL;

4529

break;

4529

break;

4530

4531

}

4531

}

4532

return err;

4532

return err;

4533

}

4533

}

4534

4535

/*

4535

/*

4536

* Perform the SIOCxIFxxx calls, inside rtnl_lock()

4536

* Perform the SIOCxIFxxx calls, inside rtnl_lock()

4537

*/

4537

*/

4538

static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)

4538

static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)

4539

{

4539

{

4540

int err;

4540

int err;

4541

struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);

4541

struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);

4542

const struct net_device_ops *ops;

4542

const struct net_device_ops *ops;

4543

4544

if (!dev)

4544

if (!dev)

4545

return -ENODEV;

4545

return -ENODEV;

4546

4547

ops = dev->netdev_ops;

4547

ops = dev->netdev_ops;

4548

4549

switch (cmd) {

4549

switch (cmd) {

4550

case SIOCSIFFLAGS: /* Set interface flags */

4550

case SIOCSIFFLAGS: /* Set interface flags */

4551

return dev_change_flags(dev, ifr->ifr_flags);

4551

return dev_change_flags(dev, ifr->ifr_flags);

4552

4553

case SIOCSIFMETRIC: /* Set the metric on the interface

4553

case SIOCSIFMETRIC: /* Set the metric on the interface

4554

(currently unused) */

4554

(currently unused) */

4555

return -EOPNOTSUPP;

4555

return -EOPNOTSUPP;

4556

4557

case SIOCSIFMTU: /* Set the MTU of a device */

4557

case SIOCSIFMTU: /* Set the MTU of a device */

4558

return dev_set_mtu(dev, ifr->ifr_mtu);

4558

return dev_set_mtu(dev, ifr->ifr_mtu);

4559

4560

case SIOCSIFHWADDR:

4560

case SIOCSIFHWADDR:

4561

return dev_set_mac_address(dev, &ifr->ifr_hwaddr);

4561

return dev_set_mac_address(dev, &ifr->ifr_hwaddr);

4562

4563

case SIOCSIFHWBROADCAST:

4563

case SIOCSIFHWBROADCAST:

4564

if (ifr->ifr_hwaddr.sa_family != dev->type)

4564

if (ifr->ifr_hwaddr.sa_family != dev->type)

4565

return -EINVAL;

4565

return -EINVAL;

4566

memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,

4566

memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,

4567

min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));

4567

min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));

4568

call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);

4568

call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);

4569

return 0;

4569

return 0;

4570

4571

case SIOCSIFMAP:

4571

case SIOCSIFMAP:

4572

if (ops->ndo_set_config) {

4572

if (ops->ndo_set_config) {

4573

if (!netif_device_present(dev))

4573

if (!netif_device_present(dev))

4574

return -ENODEV;

4574

return -ENODEV;

4575

return ops->ndo_set_config(dev, &ifr->ifr_map);

4575

return ops->ndo_set_config(dev, &ifr->ifr_map);

4576

}

4576

}

4577

return -EOPNOTSUPP;

4577

return -EOPNOTSUPP;

4578

4579

case SIOCADDMULTI:

4579

case SIOCADDMULTI:

4580

if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||

4580

if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||

4581

ifr->ifr_hwaddr.sa_family != AF_UNSPEC)

4581

ifr->ifr_hwaddr.sa_family != AF_UNSPEC)

4582

return -EINVAL;

4582

return -EINVAL;

4583

if (!netif_device_present(dev))

4583

if (!netif_device_present(dev))

4584

return -ENODEV;

4584

return -ENODEV;

4585

return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);

4585

return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);

4586

4587

case SIOCDELMULTI:

4587

case SIOCDELMULTI:

4588

if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||

4588

if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||

4589

ifr->ifr_hwaddr.sa_family != AF_UNSPEC)

4589

ifr->ifr_hwaddr.sa_family != AF_UNSPEC)

4590

return -EINVAL;

4590

return -EINVAL;

4591

if (!netif_device_present(dev))

4591

if (!netif_device_present(dev))

4592

return -ENODEV;

4592

return -ENODEV;

4593

return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);

4593

return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);

4594

4595

case SIOCSIFTXQLEN:

4595

case SIOCSIFTXQLEN:

4596

if (ifr->ifr_qlen < 0)

4596

if (ifr->ifr_qlen < 0)

4597

return -EINVAL;

4597

return -EINVAL;

4598

dev->tx_queue_len = ifr->ifr_qlen;

4598

dev->tx_queue_len = ifr->ifr_qlen;

4599

return 0;

4599

return 0;

4600

4601

case SIOCSIFNAME:

4601

case SIOCSIFNAME:

4602

ifr->ifr_newname[IFNAMSIZ-1] = '\0';

4602

ifr->ifr_newname[IFNAMSIZ-1] = '\0';

4603

return dev_change_name(dev, ifr->ifr_newname);

4603

return dev_change_name(dev, ifr->ifr_newname);

4604

4605

/*

4605

/*

4606

* Unknown or private ioctl

4606

* Unknown or private ioctl

4607

*/

4607

*/

4608

default:

4608

default:

4609

if ((cmd >= SIOCDEVPRIVATE &&

4609

if ((cmd >= SIOCDEVPRIVATE &&

4610

cmd <= SIOCDEVPRIVATE + 15) ||

4610

cmd <= SIOCDEVPRIVATE + 15) ||

4611

cmd == SIOCBONDENSLAVE ||

4611

cmd == SIOCBONDENSLAVE ||

4612

cmd == SIOCBONDRELEASE ||

4612

cmd == SIOCBONDRELEASE ||

4613

cmd == SIOCBONDSETHWADDR ||

4613

cmd == SIOCBONDSETHWADDR ||

4614

cmd == SIOCBONDSLAVEINFOQUERY ||

4614

cmd == SIOCBONDSLAVEINFOQUERY ||

4615

cmd == SIOCBONDINFOQUERY ||

4615

cmd == SIOCBONDINFOQUERY ||

4616

cmd == SIOCBONDCHANGEACTIVE ||

4616

cmd == SIOCBONDCHANGEACTIVE ||

4617

cmd == SIOCGMIIPHY ||

4617

cmd == SIOCGMIIPHY ||

4618

cmd == SIOCGMIIREG ||

4618

cmd == SIOCGMIIREG ||

4619

cmd == SIOCSMIIREG ||

4619

cmd == SIOCSMIIREG ||

4620

cmd == SIOCBRADDIF ||

4620

cmd == SIOCBRADDIF ||

4621

cmd == SIOCBRDELIF ||

4621

cmd == SIOCBRDELIF ||

4622

cmd == SIOCSHWTSTAMP ||

4622

cmd == SIOCSHWTSTAMP ||

4623

cmd == SIOCWANDEV) {

4623

cmd == SIOCWANDEV) {

4624

err = -EOPNOTSUPP;

4624

err = -EOPNOTSUPP;

4625

if (ops->ndo_do_ioctl) {

4625

if (ops->ndo_do_ioctl) {

4626

if (netif_device_present(dev))

4626

if (netif_device_present(dev))

4627

err = ops->ndo_do_ioctl(dev, ifr, cmd);

4627

err = ops->ndo_do_ioctl(dev, ifr, cmd);

4628

else

4628

else

4629

err = -ENODEV;

4629

err = -ENODEV;

4630

}

4630

}

4631

} else

4631

} else

4632

err = -EINVAL;

4632

err = -EINVAL;

4633

4634

}

4634

}

4635

return err;

4635

return err;

4636

}

4636

}

4637

4638

/*

4638

/*

4639

* This function handles all "interface"-type I/O control requests. The actual

4639

* This function handles all "interface"-type I/O control requests. The actual

4640

* 'doing' part of this is dev_ifsioc above.

4640

* 'doing' part of this is dev_ifsioc above.

4641

*/

4641

*/

4642

4643

/**

4643

/**

4644

* dev_ioctl - network device ioctl

4644

* dev_ioctl - network device ioctl

4645

* @net: the applicable net namespace

4645

* @net: the applicable net namespace

4646

* @cmd: command to issue

4646

* @cmd: command to issue

4647

* @arg: pointer to a struct ifreq in user space

4647

* @arg: pointer to a struct ifreq in user space

4648

*

4648

*

4649

* Issue ioctl functions to devices. This is normally called by the

4649

* Issue ioctl functions to devices. This is normally called by the

4650

* user space syscall interfaces but can sometimes be useful for

4650

* user space syscall interfaces but can sometimes be useful for

4651

* other purposes. The return value is the return from the syscall if

4651

* other purposes. The return value is the return from the syscall if

4652

* positive or a negative errno code on error.

4652

* positive or a negative errno code on error.

4653

*/

4653

*/

4654

4655

int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)

4655

int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)

4656

{

4656

{

4657

struct ifreq ifr;

4657

struct ifreq ifr;

4658

int ret;

4658

int ret;

4659

char *colon;

4659

char *colon;

4660

4661

/* One special case: SIOCGIFCONF takes ifconf argument

4661

/* One special case: SIOCGIFCONF takes ifconf argument

4662

and requires shared lock, because it sleeps writing

4662

and requires shared lock, because it sleeps writing

4663

to user space.

4663

to user space.

4664

*/

4664

*/

4665

4666

if (cmd == SIOCGIFCONF) {

4666

if (cmd == SIOCGIFCONF) {

4667

rtnl_lock();

4667

rtnl_lock();

4668

ret = dev_ifconf(net, (char __user *) arg);

4668

ret = dev_ifconf(net, (char __user *) arg);

4669

rtnl_unlock();

4669

rtnl_unlock();

4670

return ret;

4670

return ret;

4671

}

4671

}

4672

if (cmd == SIOCGIFNAME)

4672

if (cmd == SIOCGIFNAME)

4673

return dev_ifname(net, (struct ifreq __user *)arg);

4673

return dev_ifname(net, (struct ifreq __user *)arg);

4674

4675

if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))

4675

if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))

4676

return -EFAULT;

4676

return -EFAULT;

4677

4678

ifr.ifr_name[IFNAMSIZ-1] = 0;

4678

ifr.ifr_name[IFNAMSIZ-1] = 0;

4679

4680

colon = strchr(ifr.ifr_name, ':');

4680

colon = strchr(ifr.ifr_name, ':');

4681

if (colon)

4681

if (colon)

4682

*colon = 0;

4682

*colon = 0;

4683

4684

/*

4684

/*

4685

* See which interface the caller is talking about.

4685

* See which interface the caller is talking about.

4686

*/

4686

*/

4687

4688

switch (cmd) {

4688

switch (cmd) {

4689

/*

4689

/*

4690

* These ioctl calls:

4690

* These ioctl calls:

4691

* - can be done by all.

4691

* - can be done by all.

4692

* - atomic and do not require locking.

4692

* - atomic and do not require locking.

4693

* - return a value

4693

* - return a value

4694

*/

4694

*/

4695

case SIOCGIFFLAGS:

4695

case SIOCGIFFLAGS:

4696

case SIOCGIFMETRIC:

4696

case SIOCGIFMETRIC:

4697

case SIOCGIFMTU:

4697

case SIOCGIFMTU:

4698

case SIOCGIFHWADDR:

4698

case SIOCGIFHWADDR:

4699

case SIOCGIFSLAVE:

4699

case SIOCGIFSLAVE:

4700

case SIOCGIFMAP:

4700

case SIOCGIFMAP:

4701

case SIOCGIFINDEX:

4701

case SIOCGIFINDEX:

4702

case SIOCGIFTXQLEN:

4702

case SIOCGIFTXQLEN:

4703

dev_load(net, ifr.ifr_name);

4703

dev_load(net, ifr.ifr_name);

4704

rcu_read_lock();

4704

rcu_read_lock();

4705

ret = dev_ifsioc_locked(net, &ifr, cmd);

4705

ret = dev_ifsioc_locked(net, &ifr, cmd);

4706

rcu_read_unlock();

4706

rcu_read_unlock();

4707

if (!ret) {

4707

if (!ret) {

4708

if (colon)

4708

if (colon)

4709

*colon = ':';

4709

*colon = ':';

4710

if (copy_to_user(arg, &ifr,

4710

if (copy_to_user(arg, &ifr,

4711

sizeof(struct ifreq)))

4711

sizeof(struct ifreq)))

4712

ret = -EFAULT;

4712

ret = -EFAULT;

4713

}

4713

}

4714

return ret;

4714

return ret;

4715

4716

case SIOCETHTOOL:

4716

case SIOCETHTOOL:

4717

dev_load(net, ifr.ifr_name);

4717

dev_load(net, ifr.ifr_name);

4718

rtnl_lock();

4718

rtnl_lock();

4719

ret = dev_ethtool(net, &ifr);

4719

ret = dev_ethtool(net, &ifr);

4720

rtnl_unlock();

4720

rtnl_unlock();

4721

if (!ret) {

4721

if (!ret) {

4722

if (colon)

4722

if (colon)

4723

*colon = ':';

4723

*colon = ':';

4724

if (copy_to_user(arg, &ifr,

4724

if (copy_to_user(arg, &ifr,

4725

sizeof(struct ifreq)))

4725

sizeof(struct ifreq)))

4726

ret = -EFAULT;

4726

ret = -EFAULT;

4727

}

4727

}

4728

return ret;

4728

return ret;

4729

4730

/*

4730

/*

4731

* These ioctl calls:

4731

* These ioctl calls:

4732

* - require superuser power.

4732

* - require superuser power.

4733

* - require strict serialization.

4733

* - require strict serialization.

4734

* - return a value

4734

* - return a value

4735

*/

4735

*/

4736

case SIOCGMIIPHY:

4736

case SIOCGMIIPHY:

4737

case SIOCGMIIREG:

4737

case SIOCGMIIREG:

4738

case SIOCSIFNAME:

4738

case SIOCSIFNAME:

4739

if (!capable(CAP_NET_ADMIN))

4739

if (!capable(CAP_NET_ADMIN))

4740

return -EPERM;

4740

return -EPERM;

4741

dev_load(net, ifr.ifr_name);

4741

dev_load(net, ifr.ifr_name);

4742

rtnl_lock();

4742

rtnl_lock();

4743

ret = dev_ifsioc(net, &ifr, cmd);

4743

ret = dev_ifsioc(net, &ifr, cmd);

4744

rtnl_unlock();

4744

rtnl_unlock();

4745

if (!ret) {

4745

if (!ret) {

4746

if (colon)

4746

if (colon)

4747

*colon = ':';

4747

*colon = ':';

4748

if (copy_to_user(arg, &ifr,

4748

if (copy_to_user(arg, &ifr,

4749

sizeof(struct ifreq)))

4749

sizeof(struct ifreq)))

4750

ret = -EFAULT;

4750

ret = -EFAULT;

4751

}

4751

}

4752

return ret;

4752

return ret;

4753

4754

/*

4754

/*

4755

* These ioctl calls:

4755

* These ioctl calls:

4756

* - require superuser power.

4756

* - require superuser power.

4757

* - require strict serialization.

4757

* - require strict serialization.

4758

* - do not return a value

4758

* - do not return a value

4759

*/

4759

*/

4760

case SIOCSIFFLAGS:

4760

case SIOCSIFFLAGS:

4761

case SIOCSIFMETRIC:

4761

case SIOCSIFMETRIC:

4762

case SIOCSIFMTU:

4762

case SIOCSIFMTU:

4763

case SIOCSIFMAP:

4763

case SIOCSIFMAP:

4764

case SIOCSIFHWADDR:

4764

case SIOCSIFHWADDR:

4765

case SIOCSIFSLAVE:

4765

case SIOCSIFSLAVE:

4766

case SIOCADDMULTI:

4766

case SIOCADDMULTI:

4767

case SIOCDELMULTI:

4767

case SIOCDELMULTI:

4768

case SIOCSIFHWBROADCAST:

4768

case SIOCSIFHWBROADCAST:

4769

case SIOCSIFTXQLEN:

4769

case SIOCSIFTXQLEN:

4770

case SIOCSMIIREG:

4770

case SIOCSMIIREG:

4771

case SIOCBONDENSLAVE:

4771

case SIOCBONDENSLAVE:

4772

case SIOCBONDRELEASE:

4772

case SIOCBONDRELEASE:

4773

case SIOCBONDSETHWADDR:

4773

case SIOCBONDSETHWADDR:

4774

case SIOCBONDCHANGEACTIVE:

4774

case SIOCBONDCHANGEACTIVE:

4775

case SIOCBRADDIF:

4775

case SIOCBRADDIF:

4776

case SIOCBRDELIF:

4776

case SIOCBRDELIF:

4777

case SIOCSHWTSTAMP:

4777

case SIOCSHWTSTAMP:

4778

if (!capable(CAP_NET_ADMIN))

4778

if (!capable(CAP_NET_ADMIN))

4779

return -EPERM;

4779

return -EPERM;

4780

/* fall through */

4780

/* fall through */

4781

case SIOCBONDSLAVEINFOQUERY:

4781

case SIOCBONDSLAVEINFOQUERY:

4782

case SIOCBONDINFOQUERY:

4782

case SIOCBONDINFOQUERY:

4783

dev_load(net, ifr.ifr_name);

4783

dev_load(net, ifr.ifr_name);

4784

rtnl_lock();

4784

rtnl_lock();

4785

ret = dev_ifsioc(net, &ifr, cmd);

4785

ret = dev_ifsioc(net, &ifr, cmd);

4786

rtnl_unlock();

4786

rtnl_unlock();

4787

return ret;

4787

return ret;

4788

4789

case SIOCGIFMEM:

4789

case SIOCGIFMEM:

4790

/* Get the per device memory space. We can add this but

4790

/* Get the per device memory space. We can add this but

4791

* currently do not support it */

4791

* currently do not support it */

4792

case SIOCSIFMEM:

4792

case SIOCSIFMEM:

4793

/* Set the per device memory buffer space.

4793

/* Set the per device memory buffer space.

4794

* Not applicable in our case */

4794

* Not applicable in our case */

4795

case SIOCSIFLINK:

4795

case SIOCSIFLINK:

4796

return -EINVAL;

4796

return -EINVAL;

4797

4798

/*

4798

/*

4799

* Unknown or private ioctl.

4799

* Unknown or private ioctl.

4800

*/

4800

*/

4801

default:

4801

default:

4802

if (cmd == SIOCWANDEV ||

4802

if (cmd == SIOCWANDEV ||

4803

(cmd >= SIOCDEVPRIVATE &&

4803

(cmd >= SIOCDEVPRIVATE &&

4804

cmd <= SIOCDEVPRIVATE + 15)) {

4804

cmd <= SIOCDEVPRIVATE + 15)) {

4805

dev_load(net, ifr.ifr_name);

4805

dev_load(net, ifr.ifr_name);

4806

rtnl_lock();

4806

rtnl_lock();

4807

ret = dev_ifsioc(net, &ifr, cmd);

4807

ret = dev_ifsioc(net, &ifr, cmd);

4808

rtnl_unlock();

4808

rtnl_unlock();

4809

if (!ret && copy_to_user(arg, &ifr,

4809

if (!ret && copy_to_user(arg, &ifr,

4810

sizeof(struct ifreq)))

4810

sizeof(struct ifreq)))

4811

ret = -EFAULT;

4811

ret = -EFAULT;

4812

return ret;

4812

return ret;

4813

}

4813

}

4814

/* Take care of Wireless Extensions */

4814

/* Take care of Wireless Extensions */

4815

if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)

4815

if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)

4816

return wext_handle_ioctl(net, &ifr, cmd, arg);

4816

return wext_handle_ioctl(net, &ifr, cmd, arg);

4817

return -EINVAL;

4817

return -EINVAL;

4818

}

4818

}

4819

}

4819

}

4820

4821

4822

/**

4822

/**

4823

* dev_new_index - allocate an ifindex

4823

* dev_new_index - allocate an ifindex

4824

* @net: the applicable net namespace

4824

* @net: the applicable net namespace

4825

*

4825

*

4826

* Returns a suitable unique value for a new device interface

4826

* Returns a suitable unique value for a new device interface

4827

* number. The caller must hold the rtnl semaphore or the

4827

* number. The caller must hold the rtnl semaphore or the

4828

* dev_base_lock to be sure it remains unique.

4828

* dev_base_lock to be sure it remains unique.

4829

*/

4829

*/

4830

static int dev_new_index(struct net *net)

4830

static int dev_new_index(struct net *net)

4831

{

4831

{

4832

static int ifindex;

4832

static int ifindex;

4833

for (;;) {

4833

for (;;) {

4834

if (++ifindex <= 0)

4834

if (++ifindex <= 0)

4835

ifindex = 1;

4835

ifindex = 1;

4836

if (!__dev_get_by_index(net, ifindex))

4836

if (!__dev_get_by_index(net, ifindex))

4837

return ifindex;

4837

return ifindex;

4838

}

4838

}

4839

}

4839

}

4840

4841

/* Delayed registration/unregisteration */

4841

/* Delayed registration/unregisteration */

4842

static LIST_HEAD(net_todo_list);

4842

static LIST_HEAD(net_todo_list);

4843

4844

static void net_set_todo(struct net_device *dev)

4844

static void net_set_todo(struct net_device *dev)

4845

{

4845

{

4846

list_add_tail(&dev->todo_list, &net_todo_list);

4846

list_add_tail(&dev->todo_list, &net_todo_list);

4847

}

4847

}

4848

4849

static void rollback_registered_many(struct list_head *head)

4849

static void rollback_registered_many(struct list_head *head)

4850

{

4850

{

4851

struct net_device *dev, *tmp;

4851

struct net_device *dev, *tmp;

4852

4853

BUG_ON(dev_boot_phase);

4853

BUG_ON(dev_boot_phase);

4854

ASSERT_RTNL();

4854

ASSERT_RTNL();

4855

4856

list_for_each_entry_safe(dev, tmp, head, unreg_list) {

4856

list_for_each_entry_safe(dev, tmp, head, unreg_list) {

4857

/* Some devices call without registering

4857

/* Some devices call without registering

4858

* for initialization unwind. Remove those

4858

* for initialization unwind. Remove those

4859

* devices and proceed with the remaining.

4859

* devices and proceed with the remaining.

4860

*/

4860

*/

4861

if (dev->reg_state == NETREG_UNINITIALIZED) {

4861

if (dev->reg_state == NETREG_UNINITIALIZED) {

4862

pr_debug("unregister_netdevice: device %s/%p never "

4862

pr_debug("unregister_netdevice: device %s/%p never "

4863

"was registered\n", dev->name, dev);

4863

"was registered\n", dev->name, dev);

4864

4865

WARN_ON(1);

4865

WARN_ON(1);

4866

list_del(&dev->unreg_list);

4866

list_del(&dev->unreg_list);

4867

continue;

4867

continue;

4868

}

4868

}

4869

4870

BUG_ON(dev->reg_state != NETREG_REGISTERED);

4870

BUG_ON(dev->reg_state != NETREG_REGISTERED);

4871

4872

/* If device is running, close it first. */

4872

/* If device is running, close it first. */

4873

dev_close(dev);

4873

dev_close(dev);

4874

4875

/* And unlink it from device chain. */

4875

/* And unlink it from device chain. */

4876

unlist_netdevice(dev);

4876

unlist_netdevice(dev);

4877

4878

dev->reg_state = NETREG_UNREGISTERING;

4878

dev->reg_state = NETREG_UNREGISTERING;

4879

}

4879

}

4880

4881

synchronize_net();

4881

synchronize_net();

4882

4883

list_for_each_entry(dev, head, unreg_list) {

4883

list_for_each_entry(dev, head, unreg_list) {

4884

/* Shutdown queueing discipline. */

4884

/* Shutdown queueing discipline. */

4885

dev_shutdown(dev);

4885

dev_shutdown(dev);

4886

4887

4888

/* Notify protocols, that we are about to destroy

4888

/* Notify protocols, that we are about to destroy

4889

this device. They should clean all the things.

4889

this device. They should clean all the things.

4890

*/

4890

*/

4891

call_netdevice_notifiers(NETDEV_UNREGISTER, dev);

4891

call_netdevice_notifiers(NETDEV_UNREGISTER, dev);

4892

4893

if (!dev->rtnl_link_ops ||

4893

if (!dev->rtnl_link_ops ||

4894

dev->rtnl_link_state == RTNL_LINK_INITIALIZED)

4894

dev->rtnl_link_state == RTNL_LINK_INITIALIZED)

4895

rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);

4895

rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);

4896

4897

/*

4897

/*

4898

* Flush the unicast and multicast chains

4898

* Flush the unicast and multicast chains

4899

*/

4899

*/

4900

dev_uc_flush(dev);

4900

dev_uc_flush(dev);

4901

dev_mc_flush(dev);

4901

dev_mc_flush(dev);

4902

4903

if (dev->netdev_ops->ndo_uninit)

4903

if (dev->netdev_ops->ndo_uninit)

4904

dev->netdev_ops->ndo_uninit(dev);

4904

dev->netdev_ops->ndo_uninit(dev);

4905

4906

/* Notifier chain MUST detach us from master device. */

4906

/* Notifier chain MUST detach us from master device. */

4907

WARN_ON(dev->master);

4907

WARN_ON(dev->master);

4908

4909

/* Remove entries from kobject tree */

4909

/* Remove entries from kobject tree */

4910

netdev_unregister_kobject(dev);

4910

netdev_unregister_kobject(dev);

4911

}

4911

}

4912

4913

/* Process any work delayed until the end of the batch */

4913

/* Process any work delayed until the end of the batch */

4914

dev = list_first_entry(head, struct net_device, unreg_list);

4914

dev = list_first_entry(head, struct net_device, unreg_list);

4915

call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);

4915

call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);

4916

4917

rcu_barrier();

4917

rcu_barrier();

4918

4919

list_for_each_entry(dev, head, unreg_list)

4919

list_for_each_entry(dev, head, unreg_list)

4920

dev_put(dev);

4920

dev_put(dev);

4921

}

4921

}

4922

4923

static void rollback_registered(struct net_device *dev)

4923

static void rollback_registered(struct net_device *dev)

4924

{

4924

{

4925

LIST_HEAD(single);

4925

LIST_HEAD(single);

4926

4927

list_add(&dev->unreg_list, &single);

4927

list_add(&dev->unreg_list, &single);

4928

rollback_registered_many(&single);

4928

rollback_registered_many(&single);

4929

}

4929

}

4930

4931

static void __netdev_init_queue_locks_one(struct net_device *dev,

4931

static void __netdev_init_queue_locks_one(struct net_device *dev,

4932

struct netdev_queue *dev_queue,

4932

struct netdev_queue *dev_queue,

4933

void *_unused)

4933

void *_unused)

4934

{

4934

{

4935

spin_lock_init(&dev_queue->_xmit_lock);

4935

spin_lock_init(&dev_queue->_xmit_lock);

4936

netdev_set_xmit_lockdep_class(&dev_queue->_xmit_lock, dev->type);

4936

netdev_set_xmit_lockdep_class(&dev_queue->_xmit_lock, dev->type);

4937

dev_queue->xmit_lock_owner = -1;

4937

dev_queue->xmit_lock_owner = -1;

4938

}

4938

}

4939

4940

static void netdev_init_queue_locks(struct net_device *dev)

4940

static void netdev_init_queue_locks(struct net_device *dev)

4941

{

4941

{

4942

netdev_for_each_tx_queue(dev, __netdev_init_queue_locks_one, NULL);

4942

netdev_for_each_tx_queue(dev, __netdev_init_queue_locks_one, NULL);

4943

}

4943

}

4944

4945

unsigned long netdev_fix_features(unsigned long features, const char *name)

4945

unsigned long netdev_fix_features(unsigned long features, const char *name)

4946

{

4946

{

4947

/* Fix illegal SG+CSUM combinations. */

4947

/* Fix illegal SG+CSUM combinations. */

4948

if ((features & NETIF_F_SG) &&

4948

if ((features & NETIF_F_SG) &&

4949

!(features & NETIF_F_ALL_CSUM)) {

4949

!(features & NETIF_F_ALL_CSUM)) {

4950

if (name)

4950

if (name)

4951

printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "

4951

printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "

4952

"checksum feature.\n", name);

4952

"checksum feature.\n", name);

4953

features &= ~NETIF_F_SG;

4953

features &= ~NETIF_F_SG;

4954

}

4954

}

4955

4956

/* TSO requires that SG is present as well. */

4956

/* TSO requires that SG is present as well. */

4957

if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {

4957

if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {

4958

if (name)

4958

if (name)

4959

printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "

4959

printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "

4960

"SG feature.\n", name);

4960

"SG feature.\n", name);

4961

features &= ~NETIF_F_TSO;

4961

features &= ~NETIF_F_TSO;

4962

}

4962

}

4963

4964

if (features & NETIF_F_UFO) {

4964

if (features & NETIF_F_UFO) {

4965

if (!(features & NETIF_F_GEN_CSUM)) {

4965

if (!(features & NETIF_F_GEN_CSUM)) {

4966

if (name)

4966

if (name)

4967

printk(KERN_ERR "%s: Dropping NETIF_F_UFO "

4967

printk(KERN_ERR "%s: Dropping NETIF_F_UFO "

4968

"since no NETIF_F_HW_CSUM feature.\n",

4968

"since no NETIF_F_HW_CSUM feature.\n",

4969

name);

4969

name);

4970

features &= ~NETIF_F_UFO;

4970

features &= ~NETIF_F_UFO;

4971

}

4971

}

4972

4973

if (!(features & NETIF_F_SG)) {

4973

if (!(features & NETIF_F_SG)) {

4974

if (name)

4974

if (name)

4975

printk(KERN_ERR "%s: Dropping NETIF_F_UFO "

4975

printk(KERN_ERR "%s: Dropping NETIF_F_UFO "

4976

"since no NETIF_F_SG feature.\n", name);

4976

"since no NETIF_F_SG feature.\n", name);

4977

features &= ~NETIF_F_UFO;

4977

features &= ~NETIF_F_UFO;

4978

}

4978

}

4979

}

4979

}

4980

4981

return features;

4981

return features;

4982

}

4982

}

4983

EXPORT_SYMBOL(netdev_fix_features);

4983

EXPORT_SYMBOL(netdev_fix_features);

4984

4985

/**

4985

/**

4986

* netif_stacked_transfer_operstate - transfer operstate

4986

* netif_stacked_transfer_operstate - transfer operstate

4987

* @rootdev: the root or lower level device to transfer state from

4987

* @rootdev: the root or lower level device to transfer state from

4988

* @dev: the device to transfer operstate to

4988

* @dev: the device to transfer operstate to

4989

*

4989

*

4990

* Transfer operational state from root to device. This is normally

4990

* Transfer operational state from root to device. This is normally

4991

* called when a stacking relationship exists between the root

4991

* called when a stacking relationship exists between the root

4992

* device and the device(a leaf device).

4992

* device and the device(a leaf device).

4993

*/

4993

*/

4994

void netif_stacked_transfer_operstate(const struct net_device *rootdev,

4994

void netif_stacked_transfer_operstate(const struct net_device *rootdev,

4995

struct net_device *dev)

4995

struct net_device *dev)

4996

{

4996

{

4997

if (rootdev->operstate == IF_OPER_DORMANT)

4997

if (rootdev->operstate == IF_OPER_DORMANT)

4998

netif_dormant_on(dev);

4998

netif_dormant_on(dev);

4999

else

4999

else

5000

netif_dormant_off(dev);

5000

netif_dormant_off(dev);

5001

5002

if (netif_carrier_ok(rootdev)) {

5002

if (netif_carrier_ok(rootdev)) {

5003

if (!netif_carrier_ok(dev))

5003

if (!netif_carrier_ok(dev))

5004

netif_carrier_on(dev);

5004

netif_carrier_on(dev);

5005

} else {

5005

} else {

5006

if (netif_carrier_ok(dev))

5006

if (netif_carrier_ok(dev))

5007

netif_carrier_off(dev);

5007

netif_carrier_off(dev);

5008

}

5008

}

5009

}

5009

}

5010

EXPORT_SYMBOL(netif_stacked_transfer_operstate);

5010

EXPORT_SYMBOL(netif_stacked_transfer_operstate);

5011

5012

static int netif_alloc_rx_queues(struct net_device *dev)

5012

static int netif_alloc_rx_queues(struct net_device *dev)

5013

{

5013

{

5014

#ifdef CONFIG_RPS

5014

#ifdef CONFIG_RPS

5015

unsigned int i, count = dev->num_rx_queues;

5015

unsigned int i, count = dev->num_rx_queues;

5016

5017

if (count) {

5017

if (count) {

5018

struct netdev_rx_queue *rx;

5018

struct netdev_rx_queue *rx;

5019

5020

rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);

5020

rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);

5021

if (!rx) {

5021

if (!rx) {

5022

pr_err("netdev: Unable to allocate %u rx queues.\n",

5022

pr_err("netdev: Unable to allocate %u rx queues.\n",

5023

count);

5023

count);

5024

return -ENOMEM;

5024

return -ENOMEM;

5025

}

5025

}

5026

dev->_rx = rx;

5026

dev->_rx = rx;

5027

atomic_set(&rx->count, count);

5028

5027

5029

/*

5028

/*

5030

* Set a pointer to first element in the array which holds the

5029

* Set a pointer to first element in the array which holds the

5031

* reference count.

5030

* reference count.

5032

*/

5031

*/

5033

for (i = 0; i < count; i++)

5032

for (i = 0; i < count; i++)

5034

rx[i].first = rx;

5033

rx[i].first = rx;

5035

}

5034

}

5036

#endif

5035

#endif

5037

return 0;

5036

return 0;

5038

}

5037

}

5039

5038

5040

/**

5039

/**

5041

* register_netdevice - register a network device

5040

* register_netdevice - register a network device

5042

* @dev: device to register

5041

* @dev: device to register

5043

*

5042

*

5044

* Take a completed network device structure and add it to the kernel

5043

* Take a completed network device structure and add it to the kernel

5045

* interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier

5044

* interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier

5046

* chain. 0 is returned on success. A negative errno code is returned

5045

* chain. 0 is returned on success. A negative errno code is returned

5047

* on a failure to set up the device, or if the name is a duplicate.

5046

* on a failure to set up the device, or if the name is a duplicate.

5048

*

5047

*

5049

* Callers must hold the rtnl semaphore. You may want

5048

* Callers must hold the rtnl semaphore. You may want

5050

* register_netdev() instead of this.

5049

* register_netdev() instead of this.

5051

*

5050

*

5052

* BUGS:

5051

* BUGS:

5053

* The locking appears insufficient to guarantee two parallel registers

5052

* The locking appears insufficient to guarantee two parallel registers

5054

* will not get the same name.

5053

* will not get the same name.

5055

*/

5054

*/

5056

5055

5057

int register_netdevice(struct net_device *dev)

5056

int register_netdevice(struct net_device *dev)

5058

{

5057

{

5059

int ret;

5058

int ret;

5060

struct net *net = dev_net(dev);

5059

struct net *net = dev_net(dev);

5061

5060

5062

BUG_ON(dev_boot_phase);

5061

BUG_ON(dev_boot_phase);

5063

ASSERT_RTNL();

5062

ASSERT_RTNL();

5064

5063

5065

might_sleep();

5064

might_sleep();

5066

5065

5067

/* When net_device's are persistent, this will be fatal. */

5066

/* When net_device's are persistent, this will be fatal. */

5068

BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);

5067

BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);

5069

BUG_ON(!net);

5068

BUG_ON(!net);

5070

5069

5071

spin_lock_init(&dev->addr_list_lock);

5070

spin_lock_init(&dev->addr_list_lock);

5072

netdev_set_addr_lockdep_class(dev);

5071

netdev_set_addr_lockdep_class(dev);

5073

netdev_init_queue_locks(dev);

5072

netdev_init_queue_locks(dev);

5074

5073

5075

dev->iflink = -1;

5074

dev->iflink = -1;

5076

5075

5077

ret = netif_alloc_rx_queues(dev);

5076

ret = netif_alloc_rx_queues(dev);

5078

if (ret)

5077

if (ret)

5079

goto out;

5078

goto out;

5080

5079

5081

/* Init, if this function is available */

5080

/* Init, if this function is available */

5082

if (dev->netdev_ops->ndo_init) {

5081

if (dev->netdev_ops->ndo_init) {

5083

ret = dev->netdev_ops->ndo_init(dev);

5082

ret = dev->netdev_ops->ndo_init(dev);

5084

if (ret) {

5083

if (ret) {

5085

if (ret > 0)

5084

if (ret > 0)

5086

ret = -EIO;

5085

ret = -EIO;

5087

goto out;

5086

goto out;

5088

}

5087

}

5089

}

5088

}

5090

5089

5091

ret = dev_get_valid_name(dev, dev->name, 0);

5090

ret = dev_get_valid_name(dev, dev->name, 0);

5092

if (ret)

5091

if (ret)

5093

goto err_uninit;

5092

goto err_uninit;

5094

5093

5095

dev->ifindex = dev_new_index(net);

5094

dev->ifindex = dev_new_index(net);

5096

if (dev->iflink == -1)

5095

if (dev->iflink == -1)

5097

dev->iflink = dev->ifindex;

5096

dev->iflink = dev->ifindex;

5098

5097

5099

/* Fix illegal checksum combinations */

5098

/* Fix illegal checksum combinations */

5100

if ((dev->features & NETIF_F_HW_CSUM) &&

5099

if ((dev->features & NETIF_F_HW_CSUM) &&

5101

(dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {

5100

(dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {

5102

printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",

5101

printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",

5103

dev->name);

5102

dev->name);

5104

dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);

5103

dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);

5105

}

5104

}

5106

5105

5107

if ((dev->features & NETIF_F_NO_CSUM) &&

5106

if ((dev->features & NETIF_F_NO_CSUM) &&

5108

(dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {

5107

(dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {

5109

printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",

5108

printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",

5110

dev->name);

5109

dev->name);

5111

dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);

5110

dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);

5112

}

5111

}

5113

5112

5114

dev->features = netdev_fix_features(dev->features, dev->name);

5113

dev->features = netdev_fix_features(dev->features, dev->name);

5115

5114

5116

/* Enable software GSO if SG is supported. */

5115

/* Enable software GSO if SG is supported. */

5117

if (dev->features & NETIF_F_SG)

5116

if (dev->features & NETIF_F_SG)

5118

dev->features |= NETIF_F_GSO;

5117

dev->features |= NETIF_F_GSO;

5119

5118

5120

/* Enable GRO and NETIF_F_HIGHDMA for vlans by default,

5119

/* Enable GRO and NETIF_F_HIGHDMA for vlans by default,

5121

* vlan_dev_init() will do the dev->features check, so these features

5120

* vlan_dev_init() will do the dev->features check, so these features

5122

* are enabled only if supported by underlying device.

5121

* are enabled only if supported by underlying device.

5123

*/

5122

*/

5124

dev->vlan_features |= (NETIF_F_GRO | NETIF_F_HIGHDMA);

5123

dev->vlan_features |= (NETIF_F_GRO | NETIF_F_HIGHDMA);

5125

5124

5126

ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);

5125

ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);

5127

ret = notifier_to_errno(ret);

5126

ret = notifier_to_errno(ret);

5128

if (ret)

5127

if (ret)

5129

goto err_uninit;

5128

goto err_uninit;

5130

5129

5131

ret = netdev_register_kobject(dev);

5130

ret = netdev_register_kobject(dev);

5132

if (ret)

5131

if (ret)

5133

goto err_uninit;

5132

goto err_uninit;

5134

dev->reg_state = NETREG_REGISTERED;

5133

dev->reg_state = NETREG_REGISTERED;

5135

5134

5136

/*

5135

/*

5137

* Default initial state at registry is that the

5136

* Default initial state at registry is that the

5138

* device is present.

5137

* device is present.

5139

*/

5138

*/

5140

5139

5141

set_bit(__LINK_STATE_PRESENT, &dev->state);

5140

set_bit(__LINK_STATE_PRESENT, &dev->state);

5142

5141

5143

dev_init_scheduler(dev);

5142

dev_init_scheduler(dev);

5144

dev_hold(dev);

5143

dev_hold(dev);

5145

list_netdevice(dev);

5144

list_netdevice(dev);

5146

5145

5147

/* Notify protocols, that a new device appeared. */

5146

/* Notify protocols, that a new device appeared. */

5148

ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);

5147

ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);

5149

ret = notifier_to_errno(ret);

5148

ret = notifier_to_errno(ret);

5150

if (ret) {

5149

if (ret) {

5151

rollback_registered(dev);

5150

rollback_registered(dev);

5152

dev->reg_state = NETREG_UNREGISTERED;

5151

dev->reg_state = NETREG_UNREGISTERED;

5153

}

5152

}

5154

/*

5153

/*

5155

* Prevent userspace races by waiting until the network

5154

* Prevent userspace races by waiting until the network

5156

* device is fully setup before sending notifications.

5155

* device is fully setup before sending notifications.

5157

*/

5156

*/

5158

if (!dev->rtnl_link_ops ||

5157

if (!dev->rtnl_link_ops ||

5159

dev->rtnl_link_state == RTNL_LINK_INITIALIZED)

5158

dev->rtnl_link_state == RTNL_LINK_INITIALIZED)

5160

rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);

5159

rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);

5161

5160

5162

out:

5161

out:

5163

return ret;

5162

return ret;

5164

5163

5165

err_uninit:

5164

err_uninit:

5166

if (dev->netdev_ops->ndo_uninit)

5165

if (dev->netdev_ops->ndo_uninit)

5167

dev->netdev_ops->ndo_uninit(dev);

5166

dev->netdev_ops->ndo_uninit(dev);

5168

goto out;

5167

goto out;

5169

}

5168

}

5170

EXPORT_SYMBOL(register_netdevice);

5169

EXPORT_SYMBOL(register_netdevice);

5171

5170

5172

/**

5171

/**

5173

* init_dummy_netdev - init a dummy network device for NAPI

5172

* init_dummy_netdev - init a dummy network device for NAPI

5174

* @dev: device to init

5173

* @dev: device to init

5175

*

5174

*

5176

* This takes a network device structure and initialize the minimum

5175

* This takes a network device structure and initialize the minimum

5177

* amount of fields so it can be used to schedule NAPI polls without

5176

* amount of fields so it can be used to schedule NAPI polls without

5178

* registering a full blown interface. This is to be used by drivers

5177

* registering a full blown interface. This is to be used by drivers

5179

* that need to tie several hardware interfaces to a single NAPI

5178

* that need to tie several hardware interfaces to a single NAPI

5180

* poll scheduler due to HW limitations.

5179

* poll scheduler due to HW limitations.

5181

*/

5180

*/

5182

int init_dummy_netdev(struct net_device *dev)

5181

int init_dummy_netdev(struct net_device *dev)

5183

{

5182

{

5184

/* Clear everything. Note we don't initialize spinlocks

5183

/* Clear everything. Note we don't initialize spinlocks

5185

* are they aren't supposed to be taken by any of the

5184

* are they aren't supposed to be taken by any of the

5186

* NAPI code and this dummy netdev is supposed to be

5185

* NAPI code and this dummy netdev is supposed to be

5187

* only ever used for NAPI polls

5186

* only ever used for NAPI polls

5188

*/

5187

*/

5189

memset(dev, 0, sizeof(struct net_device));

5188

memset(dev, 0, sizeof(struct net_device));

5190

5189

5191

/* make sure we BUG if trying to hit standard

5190

/* make sure we BUG if trying to hit standard

5192

* register/unregister code path

5191

* register/unregister code path

5193

*/

5192

*/

5194

dev->reg_state = NETREG_DUMMY;

5193

dev->reg_state = NETREG_DUMMY;

5195

5194

5196

/* initialize the ref count */

5195

/* initialize the ref count */

5197

atomic_set(&dev->refcnt, 1);

5196

atomic_set(&dev->refcnt, 1);

5198

5197

5199

/* NAPI wants this */

5198

/* NAPI wants this */

5200

INIT_LIST_HEAD(&dev->napi_list);

5199

INIT_LIST_HEAD(&dev->napi_list);

5201

5200

5202

/* a dummy interface is started by default */

5201

/* a dummy interface is started by default */

5203

set_bit(__LINK_STATE_PRESENT, &dev->state);

5202

set_bit(__LINK_STATE_PRESENT, &dev->state);

5204

set_bit(__LINK_STATE_START, &dev->state);

5203

set_bit(__LINK_STATE_START, &dev->state);

5205

5204

5206

return 0;

5205

return 0;

5207

}

5206

}

5208

EXPORT_SYMBOL_GPL(init_dummy_netdev);

5207

EXPORT_SYMBOL_GPL(init_dummy_netdev);

5209

5208

5210

5209

5211

/**

5210

/**

5212

* register_netdev - register a network device

5211

* register_netdev - register a network device

5213

* @dev: device to register

5212

* @dev: device to register

5214

*

5213

*

5215

* Take a completed network device structure and add it to the kernel

5214

* Take a completed network device structure and add it to the kernel

5216

* interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier

5215

* interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier

5217

* chain. 0 is returned on success. A negative errno code is returned

5216

* chain. 0 is returned on success. A negative errno code is returned

5218

* on a failure to set up the device, or if the name is a duplicate.

5217

* on a failure to set up the device, or if the name is a duplicate.

5219

*

5218

*

5220

* This is a wrapper around register_netdevice that takes the rtnl semaphore

5219

* This is a wrapper around register_netdevice that takes the rtnl semaphore

5221

* and expands the device name if you passed a format string to

5220

* and expands the device name if you passed a format string to

5222

* alloc_netdev.

5221

* alloc_netdev.

5223

*/

5222

*/

5224

int register_netdev(struct net_device *dev)

5223

int register_netdev(struct net_device *dev)

5225

{

5224

{

5226

int err;

5225

int err;

5227

5226

5228

rtnl_lock();

5227

rtnl_lock();

5229

5228

5230

/*

5229

/*

5231

* If the name is a format string the caller wants us to do a

5230

* If the name is a format string the caller wants us to do a

5232

* name allocation.

5231

* name allocation.

5233

*/

5232

*/

5234

if (strchr(dev->name, '%')) {

5233

if (strchr(dev->name, '%')) {

5235

err = dev_alloc_name(dev, dev->name);

5234

err = dev_alloc_name(dev, dev->name);

5236

if (err < 0)

5235

if (err < 0)

5237

goto out;

5236

goto out;

5238

}

5237

}

5239

5238

5240

err = register_netdevice(dev);

5239

err = register_netdevice(dev);

5241

out:

5240

out:

5242

rtnl_unlock();

5241

rtnl_unlock();

5243

return err;

5242

return err;

5244

}

5243

}

5245

EXPORT_SYMBOL(register_netdev);

5244

EXPORT_SYMBOL(register_netdev);

5246

5245

5247

/*

5246

/*

5248

* netdev_wait_allrefs - wait until all references are gone.

5247

* netdev_wait_allrefs - wait until all references are gone.

5249

*

5248

*

5250

* This is called when unregistering network devices.

5249

* This is called when unregistering network devices.

5251

*

5250

*

5252

* Any protocol or device that holds a reference should register

5251

* Any protocol or device that holds a reference should register

5253

* for netdevice notification, and cleanup and put back the

5252

* for netdevice notification, and cleanup and put back the

5254

* reference if they receive an UNREGISTER event.

5253

* reference if they receive an UNREGISTER event.

5255

* We can get stuck here if buggy protocols don't correctly

5254

* We can get stuck here if buggy protocols don't correctly

5256

* call dev_put.

5255

* call dev_put.

5257

*/

5256

*/

5258

static void netdev_wait_allrefs(struct net_device *dev)

5257

static void netdev_wait_allrefs(struct net_device *dev)

5259

{

5258

{

5260

unsigned long rebroadcast_time, warning_time;

5259

unsigned long rebroadcast_time, warning_time;

5261

5260

5262

linkwatch_forget_dev(dev);

5261

linkwatch_forget_dev(dev);

5263

5262

5264

rebroadcast_time = warning_time = jiffies;

5263

rebroadcast_time = warning_time = jiffies;

5265

while (atomic_read(&dev->refcnt) != 0) {

5264

while (atomic_read(&dev->refcnt) != 0) {

5266

if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {

5265

if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {

5267

rtnl_lock();

5266

rtnl_lock();

5268

5267

5269

/* Rebroadcast unregister notification */

5268

/* Rebroadcast unregister notification */

5270

call_netdevice_notifiers(NETDEV_UNREGISTER, dev);

5269

call_netdevice_notifiers(NETDEV_UNREGISTER, dev);

5271

/* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users

5270

/* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users

5272

* should have already handle it the first time */

5271

* should have already handle it the first time */

5273

5272

5274

if (test_bit(__LINK_STATE_LINKWATCH_PENDING,

5273

if (test_bit(__LINK_STATE_LINKWATCH_PENDING,

5275

&dev->state)) {

5274

&dev->state)) {

5276

/* We must not have linkwatch events

5275

/* We must not have linkwatch events

5277

* pending on unregister. If this

5276

* pending on unregister. If this

5278

* happens, we simply run the queue

5277

* happens, we simply run the queue

5279

* unscheduled, resulting in a noop

5278

* unscheduled, resulting in a noop

5280

* for this device.

5279

* for this device.

5281

*/

5280

*/

5282

linkwatch_run_queue();

5281

linkwatch_run_queue();

5283

}

5282

}

5284

5283

5285

__rtnl_unlock();

5284

__rtnl_unlock();

5286

5285

5287

rebroadcast_time = jiffies;

5286

rebroadcast_time = jiffies;

5288

}

5287

}

5289

5288

5290

msleep(250);

5289

msleep(250);

5291

5290

5292

if (time_after(jiffies, warning_time + 10 * HZ)) {

5291

if (time_after(jiffies, warning_time + 10 * HZ)) {

5293

printk(KERN_EMERG "unregister_netdevice: "

5292

printk(KERN_EMERG "unregister_netdevice: "

5294

"waiting for %s to become free. Usage "

5293

"waiting for %s to become free. Usage "

5295

"count = %d\n",

5294

"count = %d\n",

5296

dev->name, atomic_read(&dev->refcnt));

5295

dev->name, atomic_read(&dev->refcnt));

5297

warning_time = jiffies;

5296

warning_time = jiffies;

5298

}

5297

}

5299

}

5298

}

5300

}

5299

}

5301

5300

5302

/* The sequence is:

5301

/* The sequence is:

5303

*

5302

*

5304

* rtnl_lock();

5303

* rtnl_lock();

5305

* ...

5304

* ...

5306

* register_netdevice(x1);

5305

* register_netdevice(x1);

5307

* register_netdevice(x2);

5306

* register_netdevice(x2);

5308

* ...

5307

* ...

5309

* unregister_netdevice(y1);

5308

* unregister_netdevice(y1);

5310

* unregister_netdevice(y2);

5309

* unregister_netdevice(y2);

5311

* ...

5310

* ...

5312

* rtnl_unlock();

5311

* rtnl_unlock();

5313

* free_netdev(y1);

5312

* free_netdev(y1);

5314

* free_netdev(y2);

5313

* free_netdev(y2);

5315

*

5314

*

5316

* We are invoked by rtnl_unlock().

5315

* We are invoked by rtnl_unlock().

5317

* This allows us to deal with problems:

5316

* This allows us to deal with problems:

5318

* 1) We can delete sysfs objects which invoke hotplug

5317

* 1) We can delete sysfs objects which invoke hotplug

5319

* without deadlocking with linkwatch via keventd.

5318

* without deadlocking with linkwatch via keventd.

5320

* 2) Since we run with the RTNL semaphore not held, we can sleep

5319

* 2) Since we run with the RTNL semaphore not held, we can sleep

5321

* safely in order to wait for the netdev refcnt to drop to zero.

5320

* safely in order to wait for the netdev refcnt to drop to zero.

5322

*

5321

*

5323

* We must not return until all unregister events added during

5322

* We must not return until all unregister events added during

5324

* the interval the lock was held have been completed.

5323

* the interval the lock was held have been completed.

5325

*/

5324

*/

5326

void netdev_run_todo(void)

5325

void netdev_run_todo(void)

5327

{

5326

{

5328

struct list_head list;

5327

struct list_head list;

5329

5328

5330

/* Snapshot list, allow later requests */

5329

/* Snapshot list, allow later requests */

5331

list_replace_init(&net_todo_list, &list);

5330

list_replace_init(&net_todo_list, &list);

5332

5331

5333

__rtnl_unlock();

5332

__rtnl_unlock();

5334

5333

5335

while (!list_empty(&list)) {

5334

while (!list_empty(&list)) {

5336

struct net_device *dev

5335

struct net_device *dev

5337

= list_first_entry(&list, struct net_device, todo_list);

5336

= list_first_entry(&list, struct net_device, todo_list);

5338

list_del(&dev->todo_list);

5337

list_del(&dev->todo_list);

5339

5338

5340

if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {

5339

if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {

5341

printk(KERN_ERR "network todo '%s' but state %d\n",

5340

printk(KERN_ERR "network todo '%s' but state %d\n",

5342

dev->name, dev->reg_state);

5341

dev->name, dev->reg_state);

5343

dump_stack();

5342

dump_stack();

5344

continue;

5343

continue;

5345

}

5344

}

5346

5345

5347

dev->reg_state = NETREG_UNREGISTERED;

5346

dev->reg_state = NETREG_UNREGISTERED;

5348

5347

5349

on_each_cpu(flush_backlog, dev, 1);

5348

on_each_cpu(flush_backlog, dev, 1);

5350

5349

5351

netdev_wait_allrefs(dev);

5350

netdev_wait_allrefs(dev);

5352

5351

5353

/* paranoia */

5352

/* paranoia */

5354

BUG_ON(atomic_read(&dev->refcnt));

5353

BUG_ON(atomic_read(&dev->refcnt));

5355

WARN_ON(rcu_dereference_raw(dev->ip_ptr));

5354

WARN_ON(rcu_dereference_raw(dev->ip_ptr));

5356

WARN_ON(dev->ip6_ptr);

5355

WARN_ON(dev->ip6_ptr);

5357

WARN_ON(dev->dn_ptr);

5356

WARN_ON(dev->dn_ptr);

5358

5357

5359

if (dev->destructor)

5358

if (dev->destructor)

5360

dev->destructor(dev);

5359

dev->destructor(dev);

5361

5360

5362

/* Free network device */

5361

/* Free network device */

5363

kobject_put(&dev->dev.kobj);

5362

kobject_put(&dev->dev.kobj);

5364

}

5363

}

5365

}

5364

}

5366

5365

5367

/**

5366

/**

5368

* dev_txq_stats_fold - fold tx_queues stats

5367

* dev_txq_stats_fold - fold tx_queues stats

5369

* @dev: device to get statistics from

5368

* @dev: device to get statistics from

5370

* @stats: struct rtnl_link_stats64 to hold results

5369

* @stats: struct rtnl_link_stats64 to hold results

5371

*/

5370

*/

5372

void dev_txq_stats_fold(const struct net_device *dev,

5371

void dev_txq_stats_fold(const struct net_device *dev,

5373

struct rtnl_link_stats64 *stats)

5372

struct rtnl_link_stats64 *stats)

5374

{

5373

{

5375

u64 tx_bytes = 0, tx_packets = 0, tx_dropped = 0;

5374

u64 tx_bytes = 0, tx_packets = 0, tx_dropped = 0;

5376

unsigned int i;

5375

unsigned int i;

5377

struct netdev_queue *txq;

5376

struct netdev_queue *txq;

5378

5377

5379

for (i = 0; i < dev->num_tx_queues; i++) {

5378

for (i = 0; i < dev->num_tx_queues; i++) {

5380

txq = netdev_get_tx_queue(dev, i);

5379

txq = netdev_get_tx_queue(dev, i);

5381

spin_lock_bh(&txq->_xmit_lock);

5380

spin_lock_bh(&txq->_xmit_lock);

5382

tx_bytes += txq->tx_bytes;

5381

tx_bytes += txq->tx_bytes;

5383

tx_packets += txq->tx_packets;

5382

tx_packets += txq->tx_packets;

5384

tx_dropped += txq->tx_dropped;

5383

tx_dropped += txq->tx_dropped;

5385

spin_unlock_bh(&txq->_xmit_lock);

5384

spin_unlock_bh(&txq->_xmit_lock);

5386

}

5385

}

5387

if (tx_bytes || tx_packets || tx_dropped) {

5386

if (tx_bytes || tx_packets || tx_dropped) {

5388

stats->tx_bytes = tx_bytes;

5387

stats->tx_bytes = tx_bytes;

5389

stats->tx_packets = tx_packets;

5388

stats->tx_packets = tx_packets;

5390

stats->tx_dropped = tx_dropped;

5389

stats->tx_dropped = tx_dropped;

5391

}

5390

}

5392

}

5391

}

5393

EXPORT_SYMBOL(dev_txq_stats_fold);

5392

EXPORT_SYMBOL(dev_txq_stats_fold);

5394

5393

5395

/* Convert net_device_stats to rtnl_link_stats64. They have the same

5394

/* Convert net_device_stats to rtnl_link_stats64. They have the same

5396

* fields in the same order, with only the type differing.

5395

* fields in the same order, with only the type differing.

5397

*/

5396

*/

5398

static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,

5397

static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,

5399

const struct net_device_stats *netdev_stats)

5398

const struct net_device_stats *netdev_stats)

5400

{

5399

{

5401

#if BITS_PER_LONG == 64

5400

#if BITS_PER_LONG == 64

5402

BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));

5401

BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));

5403

memcpy(stats64, netdev_stats, sizeof(*stats64));

5402

memcpy(stats64, netdev_stats, sizeof(*stats64));

5404

#else

5403

#else

5405

size_t i, n = sizeof(*stats64) / sizeof(u64);

5404

size_t i, n = sizeof(*stats64) / sizeof(u64);

5406

const unsigned long *src = (const unsigned long *)netdev_stats;

5405

const unsigned long *src = (const unsigned long *)netdev_stats;

5407

u64 *dst = (u64 *)stats64;

5406

u64 *dst = (u64 *)stats64;

5408

5407

5409

BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=

5408

BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=

5410

sizeof(*stats64) / sizeof(u64));

5409

sizeof(*stats64) / sizeof(u64));

5411

for (i = 0; i < n; i++)

5410

for (i = 0; i < n; i++)

5412

dst[i] = src[i];

5411

dst[i] = src[i];

5413

#endif

5412

#endif

5414

}

5413

}

5415

5414

5416

/**

5415

/**

5417

* dev_get_stats - get network device statistics

5416

* dev_get_stats - get network device statistics

5418

* @dev: device to get statistics from

5417

* @dev: device to get statistics from

5419

* @storage: place to store stats

5418

* @storage: place to store stats

5420

*

5419

*

5421

* Get network statistics from device. Return @storage.

5420

* Get network statistics from device. Return @storage.

5422

* The device driver may provide its own method by setting

5421

* The device driver may provide its own method by setting

5423

* dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;

5422

* dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;

5424

* otherwise the internal statistics structure is used.

5423

* otherwise the internal statistics structure is used.

5425

*/

5424

*/

5426

struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,

5425

struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,

5427

struct rtnl_link_stats64 *storage)

5426

struct rtnl_link_stats64 *storage)

5428

{

5427

{

5429

const struct net_device_ops *ops = dev->netdev_ops;

5428

const struct net_device_ops *ops = dev->netdev_ops;

5430

5429

5431

if (ops->ndo_get_stats64) {

5430

if (ops->ndo_get_stats64) {

5432

memset(storage, 0, sizeof(*storage));

5431

memset(storage, 0, sizeof(*storage));

5433

ops->ndo_get_stats64(dev, storage);

5432

ops->ndo_get_stats64(dev, storage);

5434

} else if (ops->ndo_get_stats) {

5433

} else if (ops->ndo_get_stats) {

5435

netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));

5434

netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));

5436

} else {

5435

} else {

5437

netdev_stats_to_stats64(storage, &dev->stats);

5436

netdev_stats_to_stats64(storage, &dev->stats);

5438

dev_txq_stats_fold(dev, storage);

5437

dev_txq_stats_fold(dev, storage);

5439

}

5438

}

5440

storage->rx_dropped += atomic_long_read(&dev->rx_dropped);

5439

storage->rx_dropped += atomic_long_read(&dev->rx_dropped);

5441

return storage;

5440

return storage;

5442

}

5441

}

5443

EXPORT_SYMBOL(dev_get_stats);

5442

EXPORT_SYMBOL(dev_get_stats);

5444

5443

5445

static void netdev_init_one_queue(struct net_device *dev,

5444

static void netdev_init_one_queue(struct net_device *dev,

5446

struct netdev_queue *queue,

5445

struct netdev_queue *queue,

5447

void *_unused)

5446

void *_unused)

5448

{

5447

{

5449

queue->dev = dev;

5448

queue->dev = dev;

5450

}

5449

}

5451

5450

5452

static void netdev_init_queues(struct net_device *dev)

5451

static void netdev_init_queues(struct net_device *dev)

5453

{

5452

{

5454

netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);

5453

netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);

5455

spin_lock_init(&dev->tx_global_lock);

5454

spin_lock_init(&dev->tx_global_lock);

5456

}

5455

}

5457

5456

5458

struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)

5457

struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)

5459

{

5458

{

5460

struct netdev_queue *queue = dev_ingress_queue(dev);

5459

struct netdev_queue *queue = dev_ingress_queue(dev);

5461

5460

5462

#ifdef CONFIG_NET_CLS_ACT

5461

#ifdef CONFIG_NET_CLS_ACT

5463

if (queue)

5462

if (queue)

5464

return queue;

5463

return queue;

5465

queue = kzalloc(sizeof(*queue), GFP_KERNEL);

5464

queue = kzalloc(sizeof(*queue), GFP_KERNEL);

5466

if (!queue)

5465

if (!queue)

5467

return NULL;

5466

return NULL;

5468

netdev_init_one_queue(dev, queue, NULL);

5467

netdev_init_one_queue(dev, queue, NULL);

5469

__netdev_init_queue_locks_one(dev, queue, NULL);

5468

__netdev_init_queue_locks_one(dev, queue, NULL);

5470

queue->qdisc = &noop_qdisc;

5469

queue->qdisc = &noop_qdisc;

5471

queue->qdisc_sleeping = &noop_qdisc;

5470

queue->qdisc_sleeping = &noop_qdisc;

5472

rcu_assign_pointer(dev->ingress_queue, queue);

5471

rcu_assign_pointer(dev->ingress_queue, queue);

5473

#endif

5472

#endif

5474

return queue;

5473

return queue;

5475

}

5474

}

5476

5475

5477

/**

5476

/**

5478

* alloc_netdev_mq - allocate network device

5477

* alloc_netdev_mq - allocate network device

5479

* @sizeof_priv: size of private data to allocate space for

5478

* @sizeof_priv: size of private data to allocate space for

5480

* @name: device name format string

5479

* @name: device name format string

5481

* @setup: callback to initialize device

5480

* @setup: callback to initialize device

5482

* @queue_count: the number of subqueues to allocate

5481

* @queue_count: the number of subqueues to allocate

5483

*

5482

*

5484

* Allocates a struct net_device with private data area for driver use

5483

* Allocates a struct net_device with private data area for driver use

5485

* and performs basic initialization. Also allocates subquue structs

5484

* and performs basic initialization. Also allocates subquue structs

5486

* for each queue on the device at the end of the netdevice.

5485

* for each queue on the device at the end of the netdevice.

5487

*/

5486

*/

5488

struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,

5487

struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,

5489

void (*setup)(struct net_device *), unsigned int queue_count)

5488

void (*setup)(struct net_device *), unsigned int queue_count)

5490

{

5489

{

5491

struct netdev_queue *tx;

5490

struct netdev_queue *tx;

5492

struct net_device *dev;

5491

struct net_device *dev;

5493

size_t alloc_size;

5492

size_t alloc_size;

5494

struct net_device *p;

5493

struct net_device *p;

5495

5494

5496

BUG_ON(strlen(name) >= sizeof(dev->name));

5495

BUG_ON(strlen(name) >= sizeof(dev->name));

5497

5496

5498

alloc_size = sizeof(struct net_device);

5497

alloc_size = sizeof(struct net_device);

5499

if (sizeof_priv) {

5498

if (sizeof_priv) {

5500

/* ensure 32-byte alignment of private area */

5499

/* ensure 32-byte alignment of private area */

5501

alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);

5500

alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);

5502

alloc_size += sizeof_priv;

5501

alloc_size += sizeof_priv;

5503

}

5502

}

5504

/* ensure 32-byte alignment of whole construct */

5503

/* ensure 32-byte alignment of whole construct */

5505

alloc_size += NETDEV_ALIGN - 1;

5504

alloc_size += NETDEV_ALIGN - 1;

5506

5505

5507

p = kzalloc(alloc_size, GFP_KERNEL);

5506

p = kzalloc(alloc_size, GFP_KERNEL);

5508

if (!p) {

5507

if (!p) {

5509

printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");

5508

printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");

5510

return NULL;

5509

return NULL;

5511

}

5510

}

5512

5511

5513

tx = kcalloc(queue_count, sizeof(struct netdev_queue), GFP_KERNEL);

5512

tx = kcalloc(queue_count, sizeof(struct netdev_queue), GFP_KERNEL);

5514

if (!tx) {

5513

if (!tx) {

5515

printk(KERN_ERR "alloc_netdev: Unable to allocate "

5514

printk(KERN_ERR "alloc_netdev: Unable to allocate "

5516

"tx qdiscs.\n");

5515

"tx qdiscs.\n");

5517

goto free_p;

5516

goto free_p;

5518

}

5517

}

5519

5518

5520

5519

5521

dev = PTR_ALIGN(p, NETDEV_ALIGN);

5520

dev = PTR_ALIGN(p, NETDEV_ALIGN);

5522

dev->padded = (char *)dev - (char *)p;

5521

dev->padded = (char *)dev - (char *)p;

5523

5522

5524

if (dev_addr_init(dev))

5523

if (dev_addr_init(dev))

5525

goto free_tx;

5524

goto free_tx;

5526

5525

5527

dev_mc_init(dev);

5526

dev_mc_init(dev);

5528

dev_uc_init(dev);

5527

dev_uc_init(dev);

5529

5528

5530

dev_net_set(dev, &init_net);

5529

dev_net_set(dev, &init_net);

5531

5530

5532

dev->_tx = tx;

5531

dev->_tx = tx;

5533

dev->num_tx_queues = queue_count;

5532

dev->num_tx_queues = queue_count;

5534

dev->real_num_tx_queues = queue_count;

5533

dev->real_num_tx_queues = queue_count;

5535

5534

5536

#ifdef CONFIG_RPS

5535

#ifdef CONFIG_RPS

5537

dev->num_rx_queues = queue_count;

5536

dev->num_rx_queues = queue_count;

5538

dev->real_num_rx_queues = queue_count;

5537

dev->real_num_rx_queues = queue_count;

5539

#endif

5538

#endif

5540

5539

5541

dev->gso_max_size = GSO_MAX_SIZE;

5540

dev->gso_max_size = GSO_MAX_SIZE;

5542

5541

5543

netdev_init_queues(dev);

5542

netdev_init_queues(dev);

5544

5543

5545

INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);

5544

INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);

5546

dev->ethtool_ntuple_list.count = 0;

5545

dev->ethtool_ntuple_list.count = 0;

5547

INIT_LIST_HEAD(&dev->napi_list);

5546

INIT_LIST_HEAD(&dev->napi_list);

5548

INIT_LIST_HEAD(&dev->unreg_list);

5547

INIT_LIST_HEAD(&dev->unreg_list);

5549

INIT_LIST_HEAD(&dev->link_watch_list);

5548

INIT_LIST_HEAD(&dev->link_watch_list);

5550

dev->priv_flags = IFF_XMIT_DST_RELEASE;

5549

dev->priv_flags = IFF_XMIT_DST_RELEASE;

5551

setup(dev);

5550

setup(dev);

5552

strcpy(dev->name, name);

5551

strcpy(dev->name, name);

5553

return dev;

5552

return dev;

5554

5553

5555

free_tx:

5554

free_tx:

5556

kfree(tx);

5555

kfree(tx);

5557

free_p:

5556

free_p:

5558

kfree(p);

5557

kfree(p);

5559

return NULL;

5558

return NULL;

5560

}

5559

}

5561

EXPORT_SYMBOL(alloc_netdev_mq);

5560

EXPORT_SYMBOL(alloc_netdev_mq);

5562

5561

5563

/**

5562

/**

5564

* free_netdev - free network device

5563

* free_netdev - free network device

5565

* @dev: device

5564

* @dev: device

5566

*

5565

*

5567

* This function does the last stage of destroying an allocated device

5566

* This function does the last stage of destroying an allocated device

5568

* interface. The reference to the device object is released.

5567

* interface. The reference to the device object is released.

5569

* If this is the last reference then it will be freed.

5568

* If this is the last reference then it will be freed.

5570

*/

5569

*/

5571

void free_netdev(struct net_device *dev)

5570

void free_netdev(struct net_device *dev)

5572

{

5571

{

5573

struct napi_struct *p, *n;

5572

struct napi_struct *p, *n;

5574

5573

5575

release_net(dev_net(dev));

5574

release_net(dev_net(dev));

5576

5575

5577

kfree(dev->_tx);

5576

kfree(dev->_tx);

5578

5577

5579

kfree(rcu_dereference_raw(dev->ingress_queue));

5578

kfree(rcu_dereference_raw(dev->ingress_queue));

5580

5579

5581

/* Flush device addresses */

5580

/* Flush device addresses */

5582

dev_addr_flush(dev);

5581

dev_addr_flush(dev);

5583

5582

5584

/* Clear ethtool n-tuple list */

5583

/* Clear ethtool n-tuple list */

5585

ethtool_ntuple_flush(dev);

5584

ethtool_ntuple_flush(dev);

5586

5585

5587

list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)

5586

list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)

5588

netif_napi_del(p);

5587

netif_napi_del(p);

5589

5588

5590

/* Compatibility with error handling in drivers */

5589

/* Compatibility with error handling in drivers */

5591

if (dev->reg_state == NETREG_UNINITIALIZED) {

5590

if (dev->reg_state == NETREG_UNINITIALIZED) {

5592

kfree((char *)dev - dev->padded);

5591

kfree((char *)dev - dev->padded);

5593

return;

5592

return;

5594

}

5593

}

5595

5594

5596

BUG_ON(dev->reg_state != NETREG_UNREGISTERED);

5595

BUG_ON(dev->reg_state != NETREG_UNREGISTERED);

5597

dev->reg_state = NETREG_RELEASED;

5596

dev->reg_state = NETREG_RELEASED;

5598

5597

5599

/* will free via device release */

5598

/* will free via device release */

5600

put_device(&dev->dev);

5599

put_device(&dev->dev);

5601

}

5600

}

5602

EXPORT_SYMBOL(free_netdev);

5601

EXPORT_SYMBOL(free_netdev);

5603

5602

5604

/**

5603

/**

5605

* synchronize_net - Synchronize with packet receive processing

5604

* synchronize_net - Synchronize with packet receive processing

5606

*

5605

*

5607

* Wait for packets currently being received to be done.

5606

* Wait for packets currently being received to be done.

5608

* Does not block later packets from starting.

5607

* Does not block later packets from starting.

5609

*/

5608

*/

5610

void synchronize_net(void)

5609

void synchronize_net(void)

5611

{

5610

{

5612

might_sleep();

5611

might_sleep();

5613

synchronize_rcu();

5612

synchronize_rcu();

5614

}

5613

}

5615

EXPORT_SYMBOL(synchronize_net);

5614

EXPORT_SYMBOL(synchronize_net);

5616

5615

5617

/**

5616

/**

5618

* unregister_netdevice_queue - remove device from the kernel

5617

* unregister_netdevice_queue - remove device from the kernel

5619

* @dev: device

5618

* @dev: device

5620

* @head: list

5619

* @head: list

5621

*

5620

*

5622

* This function shuts down a device interface and removes it

5621

* This function shuts down a device interface and removes it

5623

* from the kernel tables.

5622

* from the kernel tables.

5624

* If head not NULL, device is queued to be unregistered later.

5623

* If head not NULL, device is queued to be unregistered later.

5625

*

5624

*

5626

* Callers must hold the rtnl semaphore. You may want

5625

* Callers must hold the rtnl semaphore. You may want

5627

* unregister_netdev() instead of this.

5626

* unregister_netdev() instead of this.

5628

*/

5627

*/

5629

5628

5630

void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)

5629

void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)

5631

{

5630

{

5632

ASSERT_RTNL();

5631

ASSERT_RTNL();

5633

5632

5634

if (head) {

5633

if (head) {

5635

list_move_tail(&dev->unreg_list, head);

5634

list_move_tail(&dev->unreg_list, head);

5636

} else {

5635

} else {

5637

rollback_registered(dev);

5636

rollback_registered(dev);

5638

/* Finish processing unregister after unlock */

5637

/* Finish processing unregister after unlock */

5639

net_set_todo(dev);

5638

net_set_todo(dev);

5640

}

5639

}

5641

}

5640

}

5642

EXPORT_SYMBOL(unregister_netdevice_queue);

5641

EXPORT_SYMBOL(unregister_netdevice_queue);

5643

5642

5644

/**

5643

/**

5645

* unregister_netdevice_many - unregister many devices

5644

* unregister_netdevice_many - unregister many devices

5646

* @head: list of devices

5645

* @head: list of devices

5647

*/

5646

*/

5648

void unregister_netdevice_many(struct list_head *head)

5647

void unregister_netdevice_many(struct list_head *head)

5649

{

5648

{

5650

struct net_device *dev;

5649

struct net_device *dev;

5651

5650

5652

if (!list_empty(head)) {

5651

if (!list_empty(head)) {

5653

rollback_registered_many(head);

5652

rollback_registered_many(head);

5654

list_for_each_entry(dev, head, unreg_list)

5653

list_for_each_entry(dev, head, unreg_list)

5655

net_set_todo(dev);

5654

net_set_todo(dev);

5656

}

5655

}

5657

}

5656

}

5658

EXPORT_SYMBOL(unregister_netdevice_many);

5657

EXPORT_SYMBOL(unregister_netdevice_many);

5659

5658

5660

/**

5659

/**

5661

* unregister_netdev - remove device from the kernel

5660

* unregister_netdev - remove device from the kernel

5662

* @dev: device

5661

* @dev: device

5663

*

5662

*

5664

* This function shuts down a device interface and removes it

5663

* This function shuts down a device interface and removes it

5665

* from the kernel tables.

5664

* from the kernel tables.

5666

*

5665

*

5667

* This is just a wrapper for unregister_netdevice that takes

5666

* This is just a wrapper for unregister_netdevice that takes

5668

* the rtnl semaphore. In general you want to use this and not

5667

* the rtnl semaphore. In general you want to use this and not

5669

* unregister_netdevice.

5668

* unregister_netdevice.

5670

*/

5669

*/

5671

void unregister_netdev(struct net_device *dev)

5670

void unregister_netdev(struct net_device *dev)

5672

{

5671

{

5673

rtnl_lock();

5672

rtnl_lock();

5674

unregister_netdevice(dev);

5673

unregister_netdevice(dev);

5675

rtnl_unlock();

5674

rtnl_unlock();

5676

}

5675

}

5677

EXPORT_SYMBOL(unregister_netdev);

5676

EXPORT_SYMBOL(unregister_netdev);

5678

5677

5679

/**

5678

/**

5680

* dev_change_net_namespace - move device to different nethost namespace

5679

* dev_change_net_namespace - move device to different nethost namespace

5681

* @dev: device

5680

* @dev: device

5682

* @net: network namespace

5681

* @net: network namespace

5683

* @pat: If not NULL name pattern to try if the current device name

5682

* @pat: If not NULL name pattern to try if the current device name

5684

* is already taken in the destination network namespace.

5683

* is already taken in the destination network namespace.

5685

*

5684

*

5686

* This function shuts down a device interface and moves it

5685

* This function shuts down a device interface and moves it

5687

* to a new network namespace. On success 0 is returned, on

5686

* to a new network namespace. On success 0 is returned, on

5688

* a failure a netagive errno code is returned.

5687

* a failure a netagive errno code is returned.

5689

*

5688

*

5690

* Callers must hold the rtnl semaphore.

5689

* Callers must hold the rtnl semaphore.

5691

*/

5690

*/

5692

5691

5693

int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)

5692

int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)

5694

{

5693

{

5695

int err;

5694

int err;

5696

5695

5697

ASSERT_RTNL();

5696

ASSERT_RTNL();

5698

5697

5699

/* Don't allow namespace local devices to be moved. */

5698

/* Don't allow namespace local devices to be moved. */

5700

err = -EINVAL;

5699

err = -EINVAL;

5701

if (dev->features & NETIF_F_NETNS_LOCAL)

5700

if (dev->features & NETIF_F_NETNS_LOCAL)

5702

goto out;

5701

goto out;

5703

5702

5704

/* Ensure the device has been registrered */

5703

/* Ensure the device has been registrered */

5705

err = -EINVAL;

5704

err = -EINVAL;

5706

if (dev->reg_state != NETREG_REGISTERED)

5705

if (dev->reg_state != NETREG_REGISTERED)

5707

goto out;

5706

goto out;

5708

5707

5709

/* Get out if there is nothing todo */

5708

/* Get out if there is nothing todo */

5710

err = 0;

5709

err = 0;

5711

if (net_eq(dev_net(dev), net))

5710

if (net_eq(dev_net(dev), net))

5712

goto out;

5711

goto out;

5713

5712

5714

/* Pick the destination device name, and ensure

5713

/* Pick the destination device name, and ensure

5715

* we can use it in the destination network namespace.

5714

* we can use it in the destination network namespace.

5716

*/

5715

*/

5717

err = -EEXIST;

5716

err = -EEXIST;

5718

if (__dev_get_by_name(net, dev->name)) {

5717

if (__dev_get_by_name(net, dev->name)) {

5719

/* We get here if we can't use the current device name */

5718

/* We get here if we can't use the current device name */

5720

if (!pat)

5719

if (!pat)

5721

goto out;

5720

goto out;

5722

if (dev_get_valid_name(dev, pat, 1))

5721

if (dev_get_valid_name(dev, pat, 1))

5723

goto out;

5722

goto out;

5724

}

5723

}

5725

5724

5726

/*

5725

/*

5727

* And now a mini version of register_netdevice unregister_netdevice.

5726

* And now a mini version of register_netdevice unregister_netdevice.

5728

*/

5727

*/

5729

5728

5730

/* If device is running close it first. */

5729

/* If device is running close it first. */

5731

dev_close(dev);

5730

dev_close(dev);

5732

5731

5733

/* And unlink it from device chain */

5732

/* And unlink it from device chain */

5734

err = -ENODEV;

5733

err = -ENODEV;

5735

unlist_netdevice(dev);

5734

unlist_netdevice(dev);

5736

5735

5737

synchronize_net();

5736

synchronize_net();

5738

5737

5739

/* Shutdown queueing discipline. */

5738

/* Shutdown queueing discipline. */

5740

dev_shutdown(dev);

5739

dev_shutdown(dev);

5741

5740

5742

/* Notify protocols, that we are about to destroy

5741

/* Notify protocols, that we are about to destroy

5743

this device. They should clean all the things.

5742

this device. They should clean all the things.

5744

5743

5745

Note that dev->reg_state stays at NETREG_REGISTERED.

5744

Note that dev->reg_state stays at NETREG_REGISTERED.

5746

This is wanted because this way 8021q and macvlan know

5745

This is wanted because this way 8021q and macvlan know

5747

the device is just moving and can keep their slaves up.

5746

the device is just moving and can keep their slaves up.

5748

*/

5747

*/

5749

call_netdevice_notifiers(NETDEV_UNREGISTER, dev);

5748

call_netdevice_notifiers(NETDEV_UNREGISTER, dev);

5750

call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);

5749

call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);

5751

5750

5752

/*

5751

/*

5753

* Flush the unicast and multicast chains

5752

* Flush the unicast and multicast chains

5754

*/

5753

*/

5755

dev_uc_flush(dev);

5754

dev_uc_flush(dev);

5756

dev_mc_flush(dev);

5755

dev_mc_flush(dev);

5757

5756

5758

/* Actually switch the network namespace */

5757

/* Actually switch the network namespace */

5759

dev_net_set(dev, net);

5758

dev_net_set(dev, net);

5760

5759

5761

/* If there is an ifindex conflict assign a new one */

5760

/* If there is an ifindex conflict assign a new one */

5762

if (__dev_get_by_index(net, dev->ifindex)) {

5761

if (__dev_get_by_index(net, dev->ifindex)) {

5763

int iflink = (dev->iflink == dev->ifindex);

5762

int iflink = (dev->iflink == dev->ifindex);

5764

dev->ifindex = dev_new_index(net);

5763

dev->ifindex = dev_new_index(net);

5765

if (iflink)

5764

if (iflink)

5766

dev->iflink = dev->ifindex;

5765

dev->iflink = dev->ifindex;

5767

}

5766

}

5768

5767

5769

/* Fixup kobjects */

5768

/* Fixup kobjects */

5770

err = device_rename(&dev->dev, dev->name);

5769

err = device_rename(&dev->dev, dev->name);

5771

WARN_ON(err);

5770

WARN_ON(err);

5772

5771

5773

/* Add the device back in the hashes */

5772

/* Add the device back in the hashes */

5774

list_netdevice(dev);

5773

list_netdevice(dev);

5775

5774

5776

/* Notify protocols, that a new device appeared. */

5775

/* Notify protocols, that a new device appeared. */

5777

call_netdevice_notifiers(NETDEV_REGISTER, dev);

5776

call_netdevice_notifiers(NETDEV_REGISTER, dev);

5778

5777

5779

/*

5778

/*

5780

* Prevent userspace races by waiting until the network

5779

* Prevent userspace races by waiting until the network

5781

* device is fully setup before sending notifications.

5780

* device is fully setup before sending notifications.

5782

*/

5781

*/

5783

rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);

5782

rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);

5784

5783

5785

synchronize_net();

5784

synchronize_net();

5786

err = 0;

5785

err = 0;

5787

out:

5786

out:

5788

return err;

5787

return err;

5789

}

5788

}

5790

EXPORT_SYMBOL_GPL(dev_change_net_namespace);

5789

EXPORT_SYMBOL_GPL(dev_change_net_namespace);

5791

5790

5792

static int dev_cpu_callback(struct notifier_block *nfb,

5791

static int dev_cpu_callback(struct notifier_block *nfb,

5793

unsigned long action,

5792

unsigned long action,

5794

void *ocpu)

5793

void *ocpu)

5795

{

5794

{

5796

struct sk_buff **list_skb;

5795

struct sk_buff **list_skb;

5797

struct sk_buff *skb;

5796

struct sk_buff *skb;

5798

unsigned int cpu, oldcpu = (unsigned long)ocpu;

5797

unsigned int cpu, oldcpu = (unsigned long)ocpu;

5799

struct softnet_data *sd, *oldsd;

5798

struct softnet_data *sd, *oldsd;

5800

5799

5801

if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)

5800

if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)

5802

return NOTIFY_OK;

5801

return NOTIFY_OK;

5803

5802

5804

local_irq_disable();

5803

local_irq_disable();

5805

cpu = smp_processor_id();

5804

cpu = smp_processor_id();

5806

sd = &per_cpu(softnet_data, cpu);

5805

sd = &per_cpu(softnet_data, cpu);

5807

oldsd = &per_cpu(softnet_data, oldcpu);

5806

oldsd = &per_cpu(softnet_data, oldcpu);

5808

5807

5809

/* Find end of our completion_queue. */

5808

/* Find end of our completion_queue. */

5810

list_skb = &sd->completion_queue;

5809

list_skb = &sd->completion_queue;

5811

while (*list_skb)

5810

while (*list_skb)

5812

list_skb = &(*list_skb)->next;

5811

list_skb = &(*list_skb)->next;

5813

/* Append completion queue from offline CPU. */

5812

/* Append completion queue from offline CPU. */

5814

*list_skb = oldsd->completion_queue;

5813

*list_skb = oldsd->completion_queue;

5815

oldsd->completion_queue = NULL;

5814

oldsd->completion_queue = NULL;

5816

5815

5817

/* Append output queue from offline CPU. */

5816

/* Append output queue from offline CPU. */

5818

if (oldsd->output_queue) {

5817

if (oldsd->output_queue) {

5819

*sd->output_queue_tailp = oldsd->output_queue;

5818

*sd->output_queue_tailp = oldsd->output_queue;

5820

sd->output_queue_tailp = oldsd->output_queue_tailp;

5819

sd->output_queue_tailp = oldsd->output_queue_tailp;

5821

oldsd->output_queue = NULL;

5820

oldsd->output_queue = NULL;

5822

oldsd->output_queue_tailp = &oldsd->output_queue;

5821

oldsd->output_queue_tailp = &oldsd->output_queue;

5823

}

5822

}

5824

5823

5825

raise_softirq_irqoff(NET_TX_SOFTIRQ);

5824

raise_softirq_irqoff(NET_TX_SOFTIRQ);

5826

local_irq_enable();

5825

local_irq_enable();

5827

5826

5828

/* Process offline CPU's input_pkt_queue */

5827

/* Process offline CPU's input_pkt_queue */

5829

while ((skb = __skb_dequeue(&oldsd->process_queue))) {

5828

while ((skb = __skb_dequeue(&oldsd->process_queue))) {

5830

netif_rx(skb);

5829

netif_rx(skb);

5831

input_queue_head_incr(oldsd);

5830

input_queue_head_incr(oldsd);

5832

}

5831

}

5833

while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {

5832

while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {

5834

netif_rx(skb);

5833

netif_rx(skb);

5835

input_queue_head_incr(oldsd);

5834

input_queue_head_incr(oldsd);

5836

}

5835

}

5837

5836

5838

return NOTIFY_OK;

5837

return NOTIFY_OK;

5839

}

5838

}

5840

5839

5841

5840

5842

/**

5841

/**

5843

* netdev_increment_features - increment feature set by one

5842

* netdev_increment_features - increment feature set by one

5844

* @all: current feature set

5843

* @all: current feature set

5845

* @one: new feature set

5844

* @one: new feature set

5846

* @mask: mask feature set

5845

* @mask: mask feature set

5847

*

5846

*

5848

* Computes a new feature set after adding a device with feature set

5847

* Computes a new feature set after adding a device with feature set

5849

* @one to the master device with current feature set @all. Will not

5848

* @one to the master device with current feature set @all. Will not

5850

* enable anything that is off in @mask. Returns the new feature set.

5849

* enable anything that is off in @mask. Returns the new feature set.

5851

*/

5850

*/

5852

unsigned long netdev_increment_features(unsigned long all, unsigned long one,

5851

unsigned long netdev_increment_features(unsigned long all, unsigned long one,

5853

unsigned long mask)

5852

unsigned long mask)

5854

{

5853

{

5855

/* If device needs checksumming, downgrade to it. */

5854

/* If device needs checksumming, downgrade to it. */

5856

if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))

5855

if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))

5857

all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);

5856

all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);

5858

else if (mask & NETIF_F_ALL_CSUM) {

5857

else if (mask & NETIF_F_ALL_CSUM) {

5859

/* If one device supports v4/v6 checksumming, set for all. */

5858

/* If one device supports v4/v6 checksumming, set for all. */

5860

if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&

5859

if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&

5861

!(all & NETIF_F_GEN_CSUM)) {

5860

!(all & NETIF_F_GEN_CSUM)) {

5862

all &= ~NETIF_F_ALL_CSUM;

5861

all &= ~NETIF_F_ALL_CSUM;

5863

all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);

5862

all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);

5864

}

5863

}

5865

5864

5866

/* If one device supports hw checksumming, set for all. */

5865

/* If one device supports hw checksumming, set for all. */

5867

if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {

5866

if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {

5868

all &= ~NETIF_F_ALL_CSUM;

5867

all &= ~NETIF_F_ALL_CSUM;

5869

all |= NETIF_F_HW_CSUM;

5868

all |= NETIF_F_HW_CSUM;

5870

}

5869

}

5871

}

5870

}

5872

5871

5873

one |= NETIF_F_ALL_CSUM;

5872

one |= NETIF_F_ALL_CSUM;

5874

5873

5875

one |= all & NETIF_F_ONE_FOR_ALL;

5874

one |= all & NETIF_F_ONE_FOR_ALL;

5876

all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO;

5875

all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO;

5877

all |= one & mask & NETIF_F_ONE_FOR_ALL;

5876

all |= one & mask & NETIF_F_ONE_FOR_ALL;

5878

5877

5879

return all;

5878

return all;

5880

}

5879

}

5881

EXPORT_SYMBOL(netdev_increment_features);

5880

EXPORT_SYMBOL(netdev_increment_features);

5882

5881

5883

static struct hlist_head *netdev_create_hash(void)

5882

static struct hlist_head *netdev_create_hash(void)

5884

{

5883

{

5885

int i;

5884

int i;

5886

struct hlist_head *hash;

5885

struct hlist_head *hash;

5887

5886

5888

hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);

5887

hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);

5889

if (hash != NULL)

5888

if (hash != NULL)

5890

for (i = 0; i < NETDEV_HASHENTRIES; i++)

5889

for (i = 0; i < NETDEV_HASHENTRIES; i++)

5891

INIT_HLIST_HEAD(&hash[i]);

5890

INIT_HLIST_HEAD(&hash[i]);

5892

5891

5893

return hash;

5892

return hash;

5894

}

5893

}

5895

5894

5896

/* Initialize per network namespace state */

5895

/* Initialize per network namespace state */

5897

static int __net_init netdev_init(struct net *net)

5896

static int __net_init netdev_init(struct net *net)

5898

{

5897

{

5899

INIT_LIST_HEAD(&net->dev_base_head);

5898

INIT_LIST_HEAD(&net->dev_base_head);

5900

5899

5901

net->dev_name_head = netdev_create_hash();

5900

net->dev_name_head = netdev_create_hash();

5902

if (net->dev_name_head == NULL)

5901

if (net->dev_name_head == NULL)

5903

goto err_name;

5902

goto err_name;

5904

5903

5905

net->dev_index_head = netdev_create_hash();

5904

net->dev_index_head = netdev_create_hash();

5906

if (net->dev_index_head == NULL)

5905

if (net->dev_index_head == NULL)

5907

goto err_idx;

5906

goto err_idx;

5908

5907

5909

return 0;

5908

return 0;

5910

5909

5911

err_idx:

5910

err_idx:

5912

kfree(net->dev_name_head);

5911

kfree(net->dev_name_head);

5913

err_name:

5912

err_name:

5914

return -ENOMEM;

5913

return -ENOMEM;

5915

}

5914

}

5916

5915

5917

/**

5916

/**

5918

* netdev_drivername - network driver for the device

5917

* netdev_drivername - network driver for the device

5919

* @dev: network device

5918

* @dev: network device

5920

* @buffer: buffer for resulting name

5919

* @buffer: buffer for resulting name

5921

* @len: size of buffer

5920

* @len: size of buffer

5922

*

5921

*

5923

* Determine network driver for device.

5922

* Determine network driver for device.

5924

*/

5923

*/

5925

char *netdev_drivername(const struct net_device *dev, char *buffer, int len)

5924

char *netdev_drivername(const struct net_device *dev, char *buffer, int len)

5926

{

5925

{

5927

const struct device_driver *driver;

5926

const struct device_driver *driver;

5928

const struct device *parent;

5927

const struct device *parent;

5929

5928

5930

if (len <= 0 || !buffer)

5929

if (len <= 0 || !buffer)

5931

return buffer;

5930

return buffer;

5932

buffer[0] = 0;

5931

buffer[0] = 0;

5933

5932

5934

parent = dev->dev.parent;

5933

parent = dev->dev.parent;

5935

5934

5936

if (!parent)

5935

if (!parent)

5937

return buffer;

5936

return buffer;

5938

5937

5939

driver = parent->driver;

5938

driver = parent->driver;

5940

if (driver && driver->name)

5939

if (driver && driver->name)

5941

strlcpy(buffer, driver->name, len);

5940

strlcpy(buffer, driver->name, len);

5942

return buffer;

5941

return buffer;

5943

}

5942

}

5944

5943

5945

static int __netdev_printk(const char *level, const struct net_device *dev,

5944

static int __netdev_printk(const char *level, const struct net_device *dev,

5946

struct va_format *vaf)

5945

struct va_format *vaf)

5947

{

5946

{

5948

int r;

5947

int r;

5949

5948

5950

if (dev && dev->dev.parent)

5949

if (dev && dev->dev.parent)

5951

r = dev_printk(level, dev->dev.parent, "%s: %pV",

5950

r = dev_printk(level, dev->dev.parent, "%s: %pV",

5952

netdev_name(dev), vaf);

5951

netdev_name(dev), vaf);

5953

else if (dev)

5952

else if (dev)

5954

r = printk("%s%s: %pV", level, netdev_name(dev), vaf);

5953

r = printk("%s%s: %pV", level, netdev_name(dev), vaf);

5955

else

5954

else

5956

r = printk("%s(NULL net_device): %pV", level, vaf);

5955

r = printk("%s(NULL net_device): %pV", level, vaf);

5957

5956

5958

return r;

5957

return r;

5959

}

5958

}

5960

5959

5961

int netdev_printk(const char *level, const struct net_device *dev,

5960

int netdev_printk(const char *level, const struct net_device *dev,

5962

const char *format, ...)

5961

const char *format, ...)

5963

{

5962

{

5964

struct va_format vaf;

5963

struct va_format vaf;

5965

va_list args;

5964

va_list args;

5966

int r;

5965

int r;

5967

5966

5968

va_start(args, format);

5967

va_start(args, format);

5969

5968

5970

vaf.fmt = format;

5969

vaf.fmt = format;

5971

vaf.va = &args;

5970

vaf.va = &args;

5972

5971

5973

r = __netdev_printk(level, dev, &vaf);

5972

r = __netdev_printk(level, dev, &vaf);

5974

va_end(args);

5973

va_end(args);

5975

5974

5976

return r;

5975

return r;

5977

}

5976

}

5978

EXPORT_SYMBOL(netdev_printk);

5977

EXPORT_SYMBOL(netdev_printk);

5979

5978

5980

#define define_netdev_printk_level(func, level) \

5979

#define define_netdev_printk_level(func, level) \

5981

int func(const struct net_device *dev, const char *fmt, ...) \

5980

int func(const struct net_device *dev, const char *fmt, ...) \

5982

{ \

5981

{ \

5983

int r; \

5982

int r; \

5984

struct va_format vaf; \

5983

struct va_format vaf; \

5985

va_list args; \

5984

va_list args; \

5986

\

5985

\

5987

va_start(args, fmt); \

5986

va_start(args, fmt); \

5988

\

5987

\

5989

vaf.fmt = fmt; \

5988

vaf.fmt = fmt; \

5990

vaf.va = &args; \

5989

vaf.va = &args; \

5991

\

5990

\

5992

r = __netdev_printk(level, dev, &vaf); \

5991

r = __netdev_printk(level, dev, &vaf); \

5993

va_end(args); \

5992

va_end(args); \

5994

\

5993

\

5995

return r; \

5994

return r; \

5996

} \

5995

} \

5997

EXPORT_SYMBOL(func);

5996

EXPORT_SYMBOL(func);

5998

5997

5999

define_netdev_printk_level(netdev_emerg, KERN_EMERG);

5998

define_netdev_printk_level(netdev_emerg, KERN_EMERG);

6000

define_netdev_printk_level(netdev_alert, KERN_ALERT);

5999

define_netdev_printk_level(netdev_alert, KERN_ALERT);

6001

define_netdev_printk_level(netdev_crit, KERN_CRIT);

6000

define_netdev_printk_level(netdev_crit, KERN_CRIT);

6002

define_netdev_printk_level(netdev_err, KERN_ERR);

6001

define_netdev_printk_level(netdev_err, KERN_ERR);

6003

define_netdev_printk_level(netdev_warn, KERN_WARNING);

6002

define_netdev_printk_level(netdev_warn, KERN_WARNING);

6004

define_netdev_printk_level(netdev_notice, KERN_NOTICE);

6003

define_netdev_printk_level(netdev_notice, KERN_NOTICE);

6005

define_netdev_printk_level(netdev_info, KERN_INFO);

6004

define_netdev_printk_level(netdev_info, KERN_INFO);

6006

6005

6007

static void __net_exit netdev_exit(struct net *net)

6006

static void __net_exit netdev_exit(struct net *net)

6008

{

6007

{

6009

kfree(net->dev_name_head);

6008

kfree(net->dev_name_head);

6010

kfree(net->dev_index_head);

6009

kfree(net->dev_index_head);

6011

}

6010

}

6012

6011

6013

static struct pernet_operations __net_initdata netdev_net_ops = {

6012

static struct pernet_operations __net_initdata netdev_net_ops = {

6014

.init = netdev_init,

6013

.init = netdev_init,

6015

.exit = netdev_exit,

6014

.exit = netdev_exit,

6016

};

6015

};

6017

6016

6018

static void __net_exit default_device_exit(struct net *net)

6017

static void __net_exit default_device_exit(struct net *net)

6019

{

6018

{

6020

struct net_device *dev, *aux;

6019

struct net_device *dev, *aux;

6021

/*

6020

/*

6022

* Push all migratable network devices back to the

6021

* Push all migratable network devices back to the

6023

* initial network namespace

6022

* initial network namespace

6024

*/

6023

*/

6025

rtnl_lock();

6024

rtnl_lock();

6026

for_each_netdev_safe(net, dev, aux) {

6025

for_each_netdev_safe(net, dev, aux) {

6027

int err;

6026

int err;

6028

char fb_name[IFNAMSIZ];

6027

char fb_name[IFNAMSIZ];

6029

6028

6030

/* Ignore unmoveable devices (i.e. loopback) */

6029

/* Ignore unmoveable devices (i.e. loopback) */

6031

if (dev->features & NETIF_F_NETNS_LOCAL)

6030

if (dev->features & NETIF_F_NETNS_LOCAL)

6032

continue;

6031

continue;

6033

6032

6034

/* Leave virtual devices for the generic cleanup */

6033

/* Leave virtual devices for the generic cleanup */

6035

if (dev->rtnl_link_ops)

6034

if (dev->rtnl_link_ops)

6036

continue;

6035

continue;

6037

6036

6038

/* Push remaing network devices to init_net */

6037

/* Push remaing network devices to init_net */

6039

snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);

6038

snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);

6040

err = dev_change_net_namespace(dev, &init_net, fb_name);

6039

err = dev_change_net_namespace(dev, &init_net, fb_name);

6041

if (err) {

6040

if (err) {

6042

printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",

6041

printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",

6043

__func__, dev->name, err);

6042

__func__, dev->name, err);

6044

BUG();

6043

BUG();

6045

}

6044

}

6046

}

6045

}

6047

rtnl_unlock();

6046

rtnl_unlock();

6048

}

6047

}

6049

6048

6050

static void __net_exit default_device_exit_batch(struct list_head *net_list)

6049

static void __net_exit default_device_exit_batch(struct list_head *net_list)

6051

{

6050

{

6052

/* At exit all network devices most be removed from a network

6051

/* At exit all network devices most be removed from a network

6053

* namespace. Do this in the reverse order of registeration.

6052

* namespace. Do this in the reverse order of registeration.

6054

* Do this across as many network namespaces as possible to

6053

* Do this across as many network namespaces as possible to

6055

* improve batching efficiency.

6054

* improve batching efficiency.

6056

*/

6055

*/

6057

struct net_device *dev;

6056

struct net_device *dev;

6058

struct net *net;

6057

struct net *net;

6059

LIST_HEAD(dev_kill_list);

6058

LIST_HEAD(dev_kill_list);

6060

6059

6061

rtnl_lock();

6060

rtnl_lock();

6062

list_for_each_entry(net, net_list, exit_list) {

6061

list_for_each_entry(net, net_list, exit_list) {

6063

for_each_netdev_reverse(net, dev) {

6062

for_each_netdev_reverse(net, dev) {

6064

if (dev->rtnl_link_ops)

6063

if (dev->rtnl_link_ops)

6065

dev->rtnl_link_ops->dellink(dev, &dev_kill_list);

6064

dev->rtnl_link_ops->dellink(dev, &dev_kill_list);

6066

else

6065

else

6067

unregister_netdevice_queue(dev, &dev_kill_list);

6066

unregister_netdevice_queue(dev, &dev_kill_list);

6068

}

6067

}

6069

}

6068

}

6070

unregister_netdevice_many(&dev_kill_list);

6069

unregister_netdevice_many(&dev_kill_list);

6071

rtnl_unlock();

6070

rtnl_unlock();

6072

}

6071

}

6073

6072

6074

static struct pernet_operations __net_initdata default_device_ops = {

6073

static struct pernet_operations __net_initdata default_device_ops = {

6075

.exit = default_device_exit,

6074

.exit = default_device_exit,

6076

.exit_batch = default_device_exit_batch,

6075

.exit_batch = default_device_exit_batch,

6077

};

6076

};

6078

6077

6079

/*

6078

/*

6080

* Initialize the DEV module. At boot time this walks the device list and

6079

* Initialize the DEV module. At boot time this walks the device list and

6081

* unhooks any devices that fail to initialise (normally hardware not

6080

* unhooks any devices that fail to initialise (normally hardware not

6082

* present) and leaves us with a valid list of present and active devices.

6081

* present) and leaves us with a valid list of present and active devices.

6083

*

6082

*

6084

*/

6083

*/

6085

6084

6086

/*

6085

/*

6087

* This is called single threaded during boot, so no need

6086

* This is called single threaded during boot, so no need

6088

* to take the rtnl semaphore.

6087

* to take the rtnl semaphore.

6089

*/

6088

*/

6090

static int __init net_dev_init(void)

6089

static int __init net_dev_init(void)

6091

{

6090

{

6092

int i, rc = -ENOMEM;

6091

int i, rc = -ENOMEM;

6093

6092

6094

BUG_ON(!dev_boot_phase);

6093

BUG_ON(!dev_boot_phase);

6095

6094

6096

if (dev_proc_init())

6095

if (dev_proc_init())

6097

goto out;

6096

goto out;

6098

6097

6099

if (netdev_kobject_init())

6098

if (netdev_kobject_init())

6100

goto out;

6099

goto out;

6101

6100

6102

INIT_LIST_HEAD(&ptype_all);

6101

INIT_LIST_HEAD(&ptype_all);

6103

for (i = 0; i < PTYPE_HASH_SIZE; i++)

6102

for (i = 0; i < PTYPE_HASH_SIZE; i++)

6104

INIT_LIST_HEAD(&ptype_base[i]);

6103

INIT_LIST_HEAD(&ptype_base[i]);

6105

6104

6106

if (register_pernet_subsys(&netdev_net_ops))

6105

if (register_pernet_subsys(&netdev_net_ops))

6107

goto out;

6106

goto out;

6108

6107

6109

/*

6108

/*

6110

* Initialise the packet receive queues.

6109

* Initialise the packet receive queues.

6111

*/

6110

*/

6112

6111

6113

for_each_possible_cpu(i) {

6112

for_each_possible_cpu(i) {

6114

struct softnet_data *sd = &per_cpu(softnet_data, i);

6113

struct softnet_data *sd = &per_cpu(softnet_data, i);

6115

6114

6116

memset(sd, 0, sizeof(*sd));

6115

memset(sd, 0, sizeof(*sd));

6117

skb_queue_head_init(&sd->input_pkt_queue);

6116

skb_queue_head_init(&sd->input_pkt_queue);

6118

skb_queue_head_init(&sd->process_queue);

6117

skb_queue_head_init(&sd->process_queue);

6119

sd->completion_queue = NULL;

6118

sd->completion_queue = NULL;

6120

INIT_LIST_HEAD(&sd->poll_list);

6119

INIT_LIST_HEAD(&sd->poll_list);

6121

sd->output_queue = NULL;

6120

sd->output_queue = NULL;

6122

sd->output_queue_tailp = &sd->output_queue;

6121

sd->output_queue_tailp = &sd->output_queue;

6123

#ifdef CONFIG_RPS

6122

#ifdef CONFIG_RPS

6124

sd->csd.func = rps_trigger_softirq;

6123

sd->csd.func = rps_trigger_softirq;

6125

sd->csd.info = sd;

6124

sd->csd.info = sd;

6126

sd->csd.flags = 0;

6125

sd->csd.flags = 0;

6127

sd->cpu = i;

6126

sd->cpu = i;

6128

#endif

6127

#endif

6129

6128

6130

sd->backlog.poll = process_backlog;

6129

sd->backlog.poll = process_backlog;

6131

sd->backlog.weight = weight_p;

6130

sd->backlog.weight = weight_p;

6132

sd->backlog.gro_list = NULL;

6131

sd->backlog.gro_list = NULL;

6133

sd->backlog.gro_count = 0;

6132

sd->backlog.gro_count = 0;

6134

}

6133

}

6135

6134

6136

dev_boot_phase = 0;

6135

dev_boot_phase = 0;

6137

6136

6138

/* The loopback device is special if any other network devices

6137

/* The loopback device is special if any other network devices

6139

* is present in a network namespace the loopback device must

6138

* is present in a network namespace the loopback device must

6140

* be present. Since we now dynamically allocate and free the

6139

* be present. Since we now dynamically allocate and free the

6141

* loopback device ensure this invariant is maintained by

6140

* loopback device ensure this invariant is maintained by

6142

* keeping the loopback device as the first device on the

6141

* keeping the loopback device as the first device on the

6143

* list of network devices. Ensuring the loopback devices

6142

* list of network devices. Ensuring the loopback devices

6144

* is the first device that appears and the last network device

6143

* is the first device that appears and the last network device

6145

* that disappears.

6144

* that disappears.

6146

*/

6145

*/

6147

if (register_pernet_device(&loopback_net_ops))

6146

if (register_pernet_device(&loopback_net_ops))

6148

goto out;

6147

goto out;

6149

6148

6150

if (register_pernet_device(&default_device_ops))

6149

if (register_pernet_device(&default_device_ops))

6151

goto out;

6150

goto out;

6152

6151

6153

open_softirq(NET_TX_SOFTIRQ, net_tx_action);

6152

open_softirq(NET_TX_SOFTIRQ, net_tx_action);

6154

open_softirq(NET_RX_SOFTIRQ, net_rx_action);

6153

open_softirq(NET_RX_SOFTIRQ, net_rx_action);

6155

6154

6156

hotcpu_notifier(dev_cpu_callback, 0);

6155

hotcpu_notifier(dev_cpu_callback, 0);

6157

dst_init();

6156

dst_init();

6158

dev_mcast_init();

6157

dev_mcast_init();

6159

rc = 0;

6158

rc = 0;

6160

out:

6159

out:

6161

return rc;

6160

return rc;

6162

}

6161

}

6163

6162

6164

subsys_initcall(net_dev_init);

6163

subsys_initcall(net_dev_init);

6165

6164

6166

static int __init initialize_hashrnd(void)

6165

static int __init initialize_hashrnd(void)

6167

{

6166

{

6168

get_random_bytes(&hashrnd, sizeof(hashrnd));

6167

get_random_bytes(&hashrnd, sizeof(hashrnd));

6169

return 0;

6168

return 0;

6170

}

6169

}

6171

6170

6172

late_initcall_sync(initialize_hashrnd);

6171

late_initcall_sync(initialize_hashrnd);

6173

6172

6174

6173

GITLAB

Eric Lee / smarc-fsl-linux-kernel

net: Fix rxq ref counting

 /*
  * 	NET3	Protocol independent device support routines.
  *
  *		This program is free software; you can redistribute it and/or
  *		modify it under the terms of the GNU General Public License
  *		as published by the Free Software Foundation; either version
  *		2 of the License, or (at your option) any later version.
  *
  *	Derived from the non IP parts of dev.c 1.0.19
  * 		Authors:	Ross Biro
  *				Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  *				Mark Evans, <evansmp@uhura.aston.ac.uk>
  *
  *	Additional Authors:
  *		Florian la Roche <rzsfl@rz.uni-sb.de>
  *		Alan Cox <gw4pts@gw4pts.ampr.org>
  *		David Hinds <dahinds@users.sourceforge.net>
  *		Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
  *		Adam Sulmicki <adam@cfar.umd.edu>
  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
  *
  *	Changes:
  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
  *              			to 2 if register_netdev gets called
  *              			before net_dev_init & also removed a
  *              			few lines of code in the process.
  *		Alan Cox	:	device private ioctl copies fields back.
  *		Alan Cox	:	Transmit queue code does relevant
  *					stunts to keep the queue safe.
  *		Alan Cox	:	Fixed double lock.
  *		Alan Cox	:	Fixed promisc NULL pointer trap
  *		????????	:	Support the full private ioctl range
  *		Alan Cox	:	Moved ioctl permission check into
  *					drivers
  *		Tim Kordas	:	SIOCADDMULTI/SIOCDELMULTI
  *		Alan Cox	:	100 backlog just doesn't cut it when
  *					you start doing multicast video 8)
  *		Alan Cox	:	Rewrote net_bh and list manager.
  *		Alan Cox	: 	Fix ETH_P_ALL echoback lengths.
  *		Alan Cox	:	Took out transmit every packet pass
  *					Saved a few bytes in the ioctl handler
  *		Alan Cox	:	Network driver sets packet type before
  *					calling netif_rx. Saves a function
  *					call a packet.
  *		Alan Cox	:	Hashed net_bh()
  *		Richard Kooijman:	Timestamp fixes.
  *		Alan Cox	:	Wrong field in SIOCGIFDSTADDR
  *		Alan Cox	:	Device lock protection.
  *		Alan Cox	: 	Fixed nasty side effect of device close
  *					changes.
  *		Rudi Cilibrasi	:	Pass the right thing to
  *					set_mac_address()
  *		Dave Miller	:	32bit quantity for the device lock to
  *					make it work out on a Sparc.
  *		Bjorn Ekwall	:	Added KERNELD hack.
  *		Alan Cox	:	Cleaned up the backlog initialise.
  *		Craig Metz	:	SIOCGIFCONF fix if space for under
  *					1 device.
  *	    Thomas Bogendoerfer :	Return ENODEV for dev_open, if there
  *					is no device open function.
  *		Andi Kleen	:	Fix error reporting for SIOCGIFCONF
  *	    Michael Chastain	:	Fix signed/unsigned for SIOCGIFCONF
  *		Cyrus Durgin	:	Cleaned for KMOD
  *		Adam Sulmicki   :	Bug Fix : Network Device Unload
  *					A network device unload needs to purge
  *					the backlog queue.
  *	Paul Rusty Russell	:	SIOCSIFNAME
  *              Pekka Riikonen  :	Netdev boot-time settings code
  *              Andrew Morton   :       Make unregister_netdevice wait
  *              			indefinitely on dev->refcnt
  * 		J Hadi Salim	:	- Backlog queue sampling
  *				        - netif_rx() feedback
  */
 #include <asm/uaccess.h>
 #include <asm/system.h>
 #include <linux/bitops.h>
 #include <linux/capability.h>
 #include <linux/cpu.h>
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/hash.h>
 #include <linux/slab.h>
 #include <linux/sched.h>
 #include <linux/mutex.h>
 #include <linux/string.h>
 #include <linux/mm.h>
 #include <linux/socket.h>
 #include <linux/sockios.h>
 #include <linux/errno.h>
 #include <linux/interrupt.h>
 #include <linux/if_ether.h>
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
 #include <linux/ethtool.h>
 #include <linux/notifier.h>
 #include <linux/skbuff.h>
 #include <net/net_namespace.h>
 #include <net/sock.h>
 #include <linux/rtnetlink.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/stat.h>
 #include <net/dst.h>
 #include <net/pkt_sched.h>
 #include <net/checksum.h>
 #include <net/xfrm.h>
 #include <linux/highmem.h>
 #include <linux/init.h>
 #include <linux/kmod.h>
 #include <linux/module.h>
 #include <linux/netpoll.h>
 #include <linux/rcupdate.h>
 #include <linux/delay.h>
 #include <net/wext.h>
 #include <net/iw_handler.h>
 #include <asm/current.h>
 #include <linux/audit.h>
 #include <linux/dmaengine.h>
 #include <linux/err.h>
 #include <linux/ctype.h>
 #include <linux/if_arp.h>
 #include <linux/if_vlan.h>
 #include <linux/ip.h>
 #include <net/ip.h>
 #include <linux/ipv6.h>
 #include <linux/in.h>
 #include <linux/jhash.h>
 #include <linux/random.h>
 #include <trace/events/napi.h>
 #include <linux/pci.h>
 #include <linux/inetdevice.h>
 #include "net-sysfs.h"
 /* Instead of increasing this, you should create a hash table. */
 #define MAX_GRO_SKBS 8
 /* This should be increased if a protocol with a bigger head is added. */
 #define GRO_MAX_HEAD (MAX_HEADER + 128)
 /*
  *	The list of packet types we will receive (as opposed to discard)
  *	and the routines to invoke.
  *
  *	Why 16. Because with 16 the only overlap we get on a hash of the
  *	low nibble of the protocol value is RARP/SNAP/X.25.
  *
  *      NOTE:  That is no longer true with the addition of VLAN tags.  Not
  *             sure which should go first, but I bet it won't make much
  *             difference if we are running VLANs.  The good news is that
  *             this protocol won't be in the list unless compiled in, so
  *             the average user (w/out VLANs) will not be adversely affected.
  *             --BLG
  *
  *		0800	IP
  *		8100    802.1Q VLAN
  *		0001	802.3
  *		0002	AX.25
  *		0004	802.2
  *		8035	RARP
  *		0005	SNAP
  *		0805	X.25
  *		0806	ARP
  *		8137	IPX
  *		0009	Localtalk
  *		86DD	IPv6
  */
 #define PTYPE_HASH_SIZE	(16)
 #define PTYPE_HASH_MASK	(PTYPE_HASH_SIZE - 1)
 static DEFINE_SPINLOCK(ptype_lock);
 static struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
 static struct list_head ptype_all __read_mostly;	/* Taps */
 /*
  * The @dev_base_head list is protected by @dev_base_lock and the rtnl
  * semaphore.
  *
  * Pure readers hold dev_base_lock for reading, or rcu_read_lock()
  *
  * Writers must hold the rtnl semaphore while they loop through the
  * dev_base_head list, and hold dev_base_lock for writing when they do the
  * actual updates.  This allows pure readers to access the list even
  * while a writer is preparing to update it.
  *
  * To put it another way, dev_base_lock is held for writing only to
  * protect against pure readers; the rtnl semaphore provides the
  * protection against other writers.
  *
  * See, for example usages, register_netdevice() and
  * unregister_netdevice(), which must be called with the rtnl
  * semaphore held.
  */
 DEFINE_RWLOCK(dev_base_lock);
 EXPORT_SYMBOL(dev_base_lock);
 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
 {
 	unsigned hash = full_name_hash(name, strnlen(name, IFNAMSIZ));
 	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
 }
 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
 {
 	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
 }
 static inline void rps_lock(struct softnet_data *sd)
 {
 #ifdef CONFIG_RPS
 	spin_lock(&sd->input_pkt_queue.lock);
 #endif
 }
 static inline void rps_unlock(struct softnet_data *sd)
 {
 #ifdef CONFIG_RPS
 	spin_unlock(&sd->input_pkt_queue.lock);
 #endif
 }
 /* Device list insertion */
 static int list_netdevice(struct net_device *dev)
 {
 	struct net *net = dev_net(dev);
 	ASSERT_RTNL();
 	write_lock_bh(&dev_base_lock);
 	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
 	hlist_add_head_rcu(&dev->index_hlist,
 			   dev_index_hash(net, dev->ifindex));
 	write_unlock_bh(&dev_base_lock);
 	return 0;
 }
 /* Device list removal
  * caller must respect a RCU grace period before freeing/reusing dev
  */
 static void unlist_netdevice(struct net_device *dev)
 {
 	ASSERT_RTNL();
 	/* Unlink dev from the device chain */
 	write_lock_bh(&dev_base_lock);
 	list_del_rcu(&dev->dev_list);
 	hlist_del_rcu(&dev->name_hlist);
 	hlist_del_rcu(&dev->index_hlist);
 	write_unlock_bh(&dev_base_lock);
 }
 /*
  *	Our notifier list
  */
 static RAW_NOTIFIER_HEAD(netdev_chain);
 /*
  *	Device drivers call our routines to queue packets here. We empty the
  *	queue in the local softnet handler.
  */
 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
 EXPORT_PER_CPU_SYMBOL(softnet_data);
 #ifdef CONFIG_LOCKDEP
 /*
  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
  * according to dev->type
  */
 static const unsigned short netdev_lock_type[] =
 	{ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
 	 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
 	 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
 	 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
 	 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
 	 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
 	 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
 	 ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
 	 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
 	 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
 	 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
 	 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
 	 ARPHRD_FCFABRIC, ARPHRD_IEEE802_TR, ARPHRD_IEEE80211,
 	 ARPHRD_IEEE80211_PRISM, ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET,
 	 ARPHRD_PHONET_PIPE, ARPHRD_IEEE802154,
 	 ARPHRD_VOID, ARPHRD_NONE};
 static const char *const netdev_lock_name[] =
 	{"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
 	 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
 	 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
 	 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
 	 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
 	 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
 	 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
 	 "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
 	 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
 	 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
 	 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
 	 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
 	 "_xmit_FCFABRIC", "_xmit_IEEE802_TR", "_xmit_IEEE80211",
 	 "_xmit_IEEE80211_PRISM", "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET",
 	 "_xmit_PHONET_PIPE", "_xmit_IEEE802154",
 	 "_xmit_VOID", "_xmit_NONE"};
 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
 {
 	int i;
 	for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
 		if (netdev_lock_type[i] == dev_type)
 			return i;
 	/* the last key is used by default */
 	return ARRAY_SIZE(netdev_lock_type) - 1;
 }
 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 						 unsigned short dev_type)
 {
 	int i;
 	i = netdev_lock_pos(dev_type);
 	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
 				   netdev_lock_name[i]);
 }
 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 {
 	int i;
 	i = netdev_lock_pos(dev->type);
 	lockdep_set_class_and_name(&dev->addr_list_lock,
 				   &netdev_addr_lock_key[i],
 				   netdev_lock_name[i]);
 }
 #else
 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
 						 unsigned short dev_type)
 {
 }
 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
 {
 }
 #endif
 /*******************************************************************************
 		Protocol management and registration routines
 *******************************************************************************/
 /*
  *	Add a protocol ID to the list. Now that the input handler is
  *	smarter we can dispense with all the messy stuff that used to be
  *	here.
  *
  *	BEWARE!!! Protocol handlers, mangling input packets,
  *	MUST BE last in hash buckets and checking protocol handlers
  *	MUST start from promiscuous ptype_all chain in net_bh.
  *	It is true now, do not change it.
  *	Explanation follows: if protocol handler, mangling packet, will
  *	be the first on list, it is not able to sense, that packet
  *	is cloned and should be copied-on-write, so that it will
  *	change it and subsequent readers will get broken packet.
  *							--ANK (980803)
  */
 static inline struct list_head *ptype_head(const struct packet_type *pt)
 {
 	if (pt->type == htons(ETH_P_ALL))
 		return &ptype_all;
 	else
 		return &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
 }
 /**
  *	dev_add_pack - add packet handler
  *	@pt: packet type declaration
  *
  *	Add a protocol handler to the networking stack. The passed &packet_type
  *	is linked into kernel lists and may not be freed until it has been
  *	removed from the kernel lists.
  *
  *	This call does not sleep therefore it can not
  *	guarantee all CPU's that are in middle of receiving packets
  *	will see the new packet type (until the next received packet).
  */
 void dev_add_pack(struct packet_type *pt)
 {
 	struct list_head *head = ptype_head(pt);
 	spin_lock(&ptype_lock);
 	list_add_rcu(&pt->list, head);
 	spin_unlock(&ptype_lock);
 }
 EXPORT_SYMBOL(dev_add_pack);
 /**
  *	__dev_remove_pack	 - remove packet handler
  *	@pt: packet type declaration
  *
  *	Remove a protocol handler that was previously added to the kernel
  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
  *	from the kernel lists and can be freed or reused once this function
  *	returns.
  *
  *      The packet type might still be in use by receivers
  *	and must not be freed until after all the CPU's have gone
  *	through a quiescent state.
  */
 void __dev_remove_pack(struct packet_type *pt)
 {
 	struct list_head *head = ptype_head(pt);
 	struct packet_type *pt1;
 	spin_lock(&ptype_lock);
 	list_for_each_entry(pt1, head, list) {
 		if (pt == pt1) {
 			list_del_rcu(&pt->list);
 			goto out;
 		}
 	}
 	printk(KERN_WARNING "dev_remove_pack: %p not found.\n", pt);
 out:
 	spin_unlock(&ptype_lock);
 }
 EXPORT_SYMBOL(__dev_remove_pack);
 /**
  *	dev_remove_pack	 - remove packet handler
  *	@pt: packet type declaration
  *
  *	Remove a protocol handler that was previously added to the kernel
  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
  *	from the kernel lists and can be freed or reused once this function
  *	returns.
  *
  *	This call sleeps to guarantee that no CPU is looking at the packet
  *	type after return.
  */
 void dev_remove_pack(struct packet_type *pt)
 {
 	__dev_remove_pack(pt);
 	synchronize_net();
 }
 EXPORT_SYMBOL(dev_remove_pack);
 /******************************************************************************
 		      Device Boot-time Settings Routines
 *******************************************************************************/
 /* Boot time configuration table */
 static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
 /**
  *	netdev_boot_setup_add	- add new setup entry
  *	@name: name of the device
  *	@map: configured settings for the device
  *
  *	Adds new setup entry to the dev_boot_setup list.  The function
  *	returns 0 on error and 1 on success.  This is a generic routine to
  *	all netdevices.
  */
 static int netdev_boot_setup_add(char *name, struct ifmap *map)
 {
 	struct netdev_boot_setup *s;
 	int i;
 	s = dev_boot_setup;
 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
 		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
 			memset(s[i].name, 0, sizeof(s[i].name));
 			strlcpy(s[i].name, name, IFNAMSIZ);
 			memcpy(&s[i].map, map, sizeof(s[i].map));
 			break;
 		}
 	}
 	return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
 }
 /**
  *	netdev_boot_setup_check	- check boot time settings
  *	@dev: the netdevice
  *
  * 	Check boot time settings for the device.
  *	The found settings are set for the device to be used
  *	later in the device probing.
  *	Returns 0 if no settings found, 1 if they are.
  */
 int netdev_boot_setup_check(struct net_device *dev)
 {
 	struct netdev_boot_setup *s = dev_boot_setup;
 	int i;
 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
 		if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
 		    !strcmp(dev->name, s[i].name)) {
 			dev->irq 	= s[i].map.irq;
 			dev->base_addr 	= s[i].map.base_addr;
 			dev->mem_start 	= s[i].map.mem_start;
 			dev->mem_end 	= s[i].map.mem_end;
 			return 1;
 		}
 	}
 	return 0;
 }
 EXPORT_SYMBOL(netdev_boot_setup_check);
 /**
  *	netdev_boot_base	- get address from boot time settings
  *	@prefix: prefix for network device
  *	@unit: id for network device
  *
  * 	Check boot time settings for the base address of device.
  *	The found settings are set for the device to be used
  *	later in the device probing.
  *	Returns 0 if no settings found.
  */
 unsigned long netdev_boot_base(const char *prefix, int unit)
 {
 	const struct netdev_boot_setup *s = dev_boot_setup;
 	char name[IFNAMSIZ];
 	int i;
 	sprintf(name, "%s%d", prefix, unit);
 	/*
 	 * If device already registered then return base of 1
 	 * to indicate not to probe for this interface
 	 */
 	if (__dev_get_by_name(&init_net, name))
 		return 1;
 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
 		if (!strcmp(name, s[i].name))
 			return s[i].map.base_addr;
 	return 0;
 }
 /*
  * Saves at boot time configured settings for any netdevice.
  */
 int __init netdev_boot_setup(char *str)
 {
 	int ints[5];
 	struct ifmap map;
 	str = get_options(str, ARRAY_SIZE(ints), ints);
 	if (!str || !*str)
 		return 0;
 	/* Save settings */
 	memset(&map, 0, sizeof(map));
 	if (ints[0] > 0)
 		map.irq = ints[1];
 	if (ints[0] > 1)
 		map.base_addr = ints[2];
 	if (ints[0] > 2)
 		map.mem_start = ints[3];
 	if (ints[0] > 3)
 		map.mem_end = ints[4];
 	/* Add new entry to the list */
 	return netdev_boot_setup_add(str, &map);
 }
 __setup("netdev=", netdev_boot_setup);
 /*******************************************************************************
 			    Device Interface Subroutines
 *******************************************************************************/
 /**
  *	__dev_get_by_name	- find a device by its name
  *	@net: the applicable net namespace
  *	@name: name to find
  *
  *	Find an interface by name. Must be called under RTNL semaphore
  *	or @dev_base_lock. If the name is found a pointer to the device
  *	is returned. If the name is not found then %NULL is returned. The
  *	reference counters are not incremented so the caller must be
  *	careful with locks.
  */
 struct net_device *__dev_get_by_name(struct net *net, const char *name)
 {
 	struct hlist_node *p;
 	struct net_device *dev;
 	struct hlist_head *head = dev_name_hash(net, name);
 	hlist_for_each_entry(dev, p, head, name_hlist)
 		if (!strncmp(dev->name, name, IFNAMSIZ))
 			return dev;
 	return NULL;
 }
 EXPORT_SYMBOL(__dev_get_by_name);
 /**
  *	dev_get_by_name_rcu	- find a device by its name
  *	@net: the applicable net namespace
  *	@name: name to find
  *
  *	Find an interface by name.
  *	If the name is found a pointer to the device is returned.
  * 	If the name is not found then %NULL is returned.
  *	The reference counters are not incremented so the caller must be
  *	careful with locks. The caller must hold RCU lock.
  */
 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
 {
 	struct hlist_node *p;
 	struct net_device *dev;
 	struct hlist_head *head = dev_name_hash(net, name);
 	hlist_for_each_entry_rcu(dev, p, head, name_hlist)
 		if (!strncmp(dev->name, name, IFNAMSIZ))
 			return dev;
 	return NULL;
 }
 EXPORT_SYMBOL(dev_get_by_name_rcu);
 /**
  *	dev_get_by_name		- find a device by its name
  *	@net: the applicable net namespace
  *	@name: name to find
  *
  *	Find an interface by name. This can be called from any
  *	context and does its own locking. The returned handle has
  *	the usage count incremented and the caller must use dev_put() to
  *	release it when it is no longer needed. %NULL is returned if no
  *	matching device is found.
  */
 struct net_device *dev_get_by_name(struct net *net, const char *name)
 {
 	struct net_device *dev;
 	rcu_read_lock();
 	dev = dev_get_by_name_rcu(net, name);
 	if (dev)
 		dev_hold(dev);
 	rcu_read_unlock();
 	return dev;
 }
 EXPORT_SYMBOL(dev_get_by_name);
 /**
  *	__dev_get_by_index - find a device by its ifindex
  *	@net: the applicable net namespace
  *	@ifindex: index of device
  *
  *	Search for an interface by index. Returns %NULL if the device
  *	is not found or a pointer to the device. The device has not
  *	had its reference counter increased so the caller must be careful
  *	about locking. The caller must hold either the RTNL semaphore
  *	or @dev_base_lock.
  */
 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
 {
 	struct hlist_node *p;
 	struct net_device *dev;
 	struct hlist_head *head = dev_index_hash(net, ifindex);
 	hlist_for_each_entry(dev, p, head, index_hlist)
 		if (dev->ifindex == ifindex)
 			return dev;
 	return NULL;
 }
 EXPORT_SYMBOL(__dev_get_by_index);
 /**
  *	dev_get_by_index_rcu - find a device by its ifindex
  *	@net: the applicable net namespace
  *	@ifindex: index of device
  *
  *	Search for an interface by index. Returns %NULL if the device
  *	is not found or a pointer to the device. The device has not
  *	had its reference counter increased so the caller must be careful
  *	about locking. The caller must hold RCU lock.
  */
 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
 {
 	struct hlist_node *p;
 	struct net_device *dev;
 	struct hlist_head *head = dev_index_hash(net, ifindex);
 	hlist_for_each_entry_rcu(dev, p, head, index_hlist)
 		if (dev->ifindex == ifindex)
 			return dev;
 	return NULL;
 }
 EXPORT_SYMBOL(dev_get_by_index_rcu);
 /**
  *	dev_get_by_index - find a device by its ifindex
  *	@net: the applicable net namespace
  *	@ifindex: index of device
  *
  *	Search for an interface by index. Returns NULL if the device
  *	is not found or a pointer to the device. The device returned has
  *	had a reference added and the pointer is safe until the user calls
  *	dev_put to indicate they have finished with it.
  */
 struct net_device *dev_get_by_index(struct net *net, int ifindex)
 {
 	struct net_device *dev;
 	rcu_read_lock();
 	dev = dev_get_by_index_rcu(net, ifindex);
 	if (dev)
 		dev_hold(dev);
 	rcu_read_unlock();
 	return dev;
 }
 EXPORT_SYMBOL(dev_get_by_index);
 /**
  *	dev_getbyhwaddr - find a device by its hardware address
  *	@net: the applicable net namespace
  *	@type: media type of device
  *	@ha: hardware address
  *
  *	Search for an interface by MAC address. Returns NULL if the device
  *	is not found or a pointer to the device. The caller must hold the
  *	rtnl semaphore. The returned device has not had its ref count increased
  *	and the caller must therefore be careful about locking
  *
  *	BUGS:
  *	If the API was consistent this would be __dev_get_by_hwaddr
  */
 struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, char *ha)
 {
 	struct net_device *dev;
 	ASSERT_RTNL();
 	for_each_netdev(net, dev)
 		if (dev->type == type &&
 		    !memcmp(dev->dev_addr, ha, dev->addr_len))
 			return dev;
 	return NULL;
 }
 EXPORT_SYMBOL(dev_getbyhwaddr);
 struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type)
 {
 	struct net_device *dev;
 	ASSERT_RTNL();
 	for_each_netdev(net, dev)
 		if (dev->type == type)
 			return dev;
 	return NULL;
 }
 EXPORT_SYMBOL(__dev_getfirstbyhwtype);
 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
 {
 	struct net_device *dev, *ret = NULL;
 	rcu_read_lock();
 	for_each_netdev_rcu(net, dev)
 		if (dev->type == type) {
 			dev_hold(dev);
 			ret = dev;
 			break;
 		}
 	rcu_read_unlock();
 	return ret;
 }
 EXPORT_SYMBOL(dev_getfirstbyhwtype);
 /**
  *	dev_get_by_flags_rcu - find any device with given flags
  *	@net: the applicable net namespace
  *	@if_flags: IFF_* values
  *	@mask: bitmask of bits in if_flags to check
  *
  *	Search for any interface with the given flags. Returns NULL if a device
  *	is not found or a pointer to the device. Must be called inside
  *	rcu_read_lock(), and result refcount is unchanged.
  */
 struct net_device *dev_get_by_flags_rcu(struct net *net, unsigned short if_flags,
 				    unsigned short mask)
 {
 	struct net_device *dev, *ret;
 	ret = NULL;
 	for_each_netdev_rcu(net, dev) {
 		if (((dev->flags ^ if_flags) & mask) == 0) {
 			ret = dev;
 			break;
 		}
 	}
 	return ret;
 }
 EXPORT_SYMBOL(dev_get_by_flags_rcu);
 /**
  *	dev_valid_name - check if name is okay for network device
  *	@name: name string
  *
  *	Network device names need to be valid file names to
  *	to allow sysfs to work.  We also disallow any kind of
  *	whitespace.
  */
 int dev_valid_name(const char *name)
 {
 	if (*name == '\0')
 		return 0;
 	if (strlen(name) >= IFNAMSIZ)
 		return 0;
 	if (!strcmp(name, ".") || !strcmp(name, ".."))
 		return 0;
 	while (*name) {
 		if (*name == '/' || isspace(*name))
 			return 0;
 		name++;
 	}
 	return 1;
 }
 EXPORT_SYMBOL(dev_valid_name);
 /**
  *	__dev_alloc_name - allocate a name for a device
  *	@net: network namespace to allocate the device name in
  *	@name: name format string
  *	@buf:  scratch buffer and result name string
  *
  *	Passed a format string - eg "lt%d" it will try and find a suitable
  *	id. It scans list of devices to build up a free map, then chooses
  *	the first empty slot. The caller must hold the dev_base or rtnl lock
  *	while allocating the name and adding the device in order to avoid
  *	duplicates.
  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
  *	Returns the number of the unit assigned or a negative errno code.
  */
 static int __dev_alloc_name(struct net *net, const char *name, char *buf)
 {
 	int i = 0;
 	const char *p;
 	const int max_netdevices = 8*PAGE_SIZE;
 	unsigned long *inuse;
 	struct net_device *d;
 	p = strnchr(name, IFNAMSIZ-1, '%');
 	if (p) {
 		/*
 		 * Verify the string as this thing may have come from
 		 * the user.  There must be either one "%d" and no other "%"
 		 * characters.
 		 */
 		if (p[1] != 'd' || strchr(p + 2, '%'))
 			return -EINVAL;
 		/* Use one page as a bit array of possible slots */
 		inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC);
 		if (!inuse)
 			return -ENOMEM;
 		for_each_netdev(net, d) {
 			if (!sscanf(d->name, name, &i))
 				continue;
 			if (i < 0 || i >= max_netdevices)
 				continue;
 			/*  avoid cases where sscanf is not exact inverse of printf */
 			snprintf(buf, IFNAMSIZ, name, i);
 			if (!strncmp(buf, d->name, IFNAMSIZ))
 				set_bit(i, inuse);
 		}
 		i = find_first_zero_bit(inuse, max_netdevices);
 		free_page((unsigned long) inuse);
 	}
 	if (buf != name)
 		snprintf(buf, IFNAMSIZ, name, i);
 	if (!__dev_get_by_name(net, buf))
 		return i;
 	/* It is possible to run out of possible slots
 	 * when the name is long and there isn't enough space left
 	 * for the digits, or if all bits are used.
 	 */
 	return -ENFILE;
 }
 /**
  *	dev_alloc_name - allocate a name for a device
  *	@dev: device
  *	@name: name format string
  *
  *	Passed a format string - eg "lt%d" it will try and find a suitable
  *	id. It scans list of devices to build up a free map, then chooses
  *	the first empty slot. The caller must hold the dev_base or rtnl lock
  *	while allocating the name and adding the device in order to avoid
  *	duplicates.
  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
  *	Returns the number of the unit assigned or a negative errno code.
  */
 int dev_alloc_name(struct net_device *dev, const char *name)
 {
 	char buf[IFNAMSIZ];
 	struct net *net;
 	int ret;
 	BUG_ON(!dev_net(dev));
 	net = dev_net(dev);
 	ret = __dev_alloc_name(net, name, buf);
 	if (ret >= 0)
 		strlcpy(dev->name, buf, IFNAMSIZ);
 	return ret;
 }
 EXPORT_SYMBOL(dev_alloc_name);
 static int dev_get_valid_name(struct net_device *dev, const char *name, bool fmt)
 {
 	struct net *net;
 	BUG_ON(!dev_net(dev));
 	net = dev_net(dev);
 	if (!dev_valid_name(name))
 		return -EINVAL;
 	if (fmt && strchr(name, '%'))
 		return dev_alloc_name(dev, name);
 	else if (__dev_get_by_name(net, name))
 		return -EEXIST;
 	else if (dev->name != name)
 		strlcpy(dev->name, name, IFNAMSIZ);
 	return 0;
 }
 /**
  *	dev_change_name - change name of a device
  *	@dev: device
  *	@newname: name (or format string) must be at least IFNAMSIZ
  *
  *	Change name of a device, can pass format strings "eth%d".
  *	for wildcarding.
  */
 int dev_change_name(struct net_device *dev, const char *newname)
 {
 	char oldname[IFNAMSIZ];
 	int err = 0;
 	int ret;
 	struct net *net;
 	ASSERT_RTNL();
 	BUG_ON(!dev_net(dev));
 	net = dev_net(dev);
 	if (dev->flags & IFF_UP)
 		return -EBUSY;
 	if (strncmp(newname, dev->name, IFNAMSIZ) == 0)
 		return 0;
 	memcpy(oldname, dev->name, IFNAMSIZ);
 	err = dev_get_valid_name(dev, newname, 1);
 	if (err < 0)
 		return err;
 rollback:
 	ret = device_rename(&dev->dev, dev->name);
 	if (ret) {
 		memcpy(dev->name, oldname, IFNAMSIZ);
 		return ret;
 	}
 	write_lock_bh(&dev_base_lock);
 	hlist_del(&dev->name_hlist);
 	write_unlock_bh(&dev_base_lock);
 	synchronize_rcu();
 	write_lock_bh(&dev_base_lock);
 	hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name));
 	write_unlock_bh(&dev_base_lock);
 	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
 	ret = notifier_to_errno(ret);
 	if (ret) {
 		/* err >= 0 after dev_alloc_name() or stores the first errno */
 		if (err >= 0) {
 			err = ret;
 			memcpy(dev->name, oldname, IFNAMSIZ);
 			goto rollback;
 		} else {
 			printk(KERN_ERR
 			       "%s: name change rollback failed: %d.\n",
 			       dev->name, ret);
 		}
 	}
 	return err;
 }
 /**
  *	dev_set_alias - change ifalias of a device
  *	@dev: device
  *	@alias: name up to IFALIASZ
  *	@len: limit of bytes to copy from info
  *
  *	Set ifalias for a device,
  */
 int dev_set_alias(struct net_device *dev, const char *alias, size_t len)
 {
 	ASSERT_RTNL();
 	if (len >= IFALIASZ)
 		return -EINVAL;
 	if (!len) {
 		if (dev->ifalias) {
 			kfree(dev->ifalias);
 			dev->ifalias = NULL;
 		}
 		return 0;
 	}
 	dev->ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL);
 	if (!dev->ifalias)
 		return -ENOMEM;
 	strlcpy(dev->ifalias, alias, len+1);
 	return len;
 }
 /**
  *	netdev_features_change - device changes features
  *	@dev: device to cause notification
  *
  *	Called to indicate a device has changed features.
  */
 void netdev_features_change(struct net_device *dev)
 {
 	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
 }
 EXPORT_SYMBOL(netdev_features_change);
 /**
  *	netdev_state_change - device changes state
  *	@dev: device to cause notification
  *
  *	Called to indicate a device has changed state. This function calls
  *	the notifier chains for netdev_chain and sends a NEWLINK message
  *	to the routing socket.
  */
 void netdev_state_change(struct net_device *dev)
 {
 	if (dev->flags & IFF_UP) {
 		call_netdevice_notifiers(NETDEV_CHANGE, dev);
 		rtmsg_ifinfo(RTM_NEWLINK, dev, 0);
 	}
 }
 EXPORT_SYMBOL(netdev_state_change);
 int netdev_bonding_change(struct net_device *dev, unsigned long event)
 {
 	return call_netdevice_notifiers(event, dev);
 }
 EXPORT_SYMBOL(netdev_bonding_change);
 /**
  *	dev_load 	- load a network module
  *	@net: the applicable net namespace
  *	@name: name of interface
  *
  *	If a network interface is not present and the process has suitable
  *	privileges this function loads the module. If module loading is not
  *	available in this kernel then it becomes a nop.
  */
 void dev_load(struct net *net, const char *name)
 {
 	struct net_device *dev;
 	rcu_read_lock();
 	dev = dev_get_by_name_rcu(net, name);
 	rcu_read_unlock();
 	if (!dev && capable(CAP_NET_ADMIN))
 		request_module("%s", name);
 }
 EXPORT_SYMBOL(dev_load);
 static int __dev_open(struct net_device *dev)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	int ret;
 	ASSERT_RTNL();
 	/*
 	 *	Is it even present?
 	 */
 	if (!netif_device_present(dev))
 		return -ENODEV;
 	ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev);
 	ret = notifier_to_errno(ret);
 	if (ret)
 		return ret;
 	/*
 	 *	Call device private open method
 	 */
 	set_bit(__LINK_STATE_START, &dev->state);
 	if (ops->ndo_validate_addr)
 		ret = ops->ndo_validate_addr(dev);
 	if (!ret && ops->ndo_open)
 		ret = ops->ndo_open(dev);
 	/*
 	 *	If it went open OK then:
 	 */
 	if (ret)
 		clear_bit(__LINK_STATE_START, &dev->state);
 	else {
 		/*
 		 *	Set the flags.
 		 */
 		dev->flags |= IFF_UP;
 		/*
 		 *	Enable NET_DMA
 		 */
 		net_dmaengine_get();
 		/*
 		 *	Initialize multicasting status
 		 */
 		dev_set_rx_mode(dev);
 		/*
 		 *	Wakeup transmit queue engine
 		 */
 		dev_activate(dev);
 	}
 	return ret;
 }
 /**
  *	dev_open	- prepare an interface for use.
  *	@dev:	device to open
  *
  *	Takes a device from down to up state. The device's private open
  *	function is invoked and then the multicast lists are loaded. Finally
  *	the device is moved into the up state and a %NETDEV_UP message is
  *	sent to the netdev notifier chain.
  *
  *	Calling this function on an active interface is a nop. On a failure
  *	a negative errno code is returned.
  */
 int dev_open(struct net_device *dev)
 {
 	int ret;
 	/*
 	 *	Is it already up?
 	 */
 	if (dev->flags & IFF_UP)
 		return 0;
 	/*
 	 *	Open device
 	 */
 	ret = __dev_open(dev);
 	if (ret < 0)
 		return ret;
 	/*
 	 *	... and announce new interface.
 	 */
 	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
 	call_netdevice_notifiers(NETDEV_UP, dev);
 	return ret;
 }
 EXPORT_SYMBOL(dev_open);
 static int __dev_close(struct net_device *dev)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	ASSERT_RTNL();
 	might_sleep();
 	/*
 	 *	Tell people we are going down, so that they can
 	 *	prepare to death, when device is still operating.
 	 */
 	call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
 	clear_bit(__LINK_STATE_START, &dev->state);
 	/* Synchronize to scheduled poll. We cannot touch poll list,
 	 * it can be even on different cpu. So just clear netif_running().
 	 *
 	 * dev->stop() will invoke napi_disable() on all of it's
 	 * napi_struct instances on this device.
 	 */
 	smp_mb__after_clear_bit(); /* Commit netif_running(). */
 	dev_deactivate(dev);
 	/*
 	 *	Call the device specific close. This cannot fail.
 	 *	Only if device is UP
 	 *
 	 *	We allow it to be called even after a DETACH hot-plug
 	 *	event.
 	 */
 	if (ops->ndo_stop)
 		ops->ndo_stop(dev);
 	/*
 	 *	Device is now down.
 	 */
 	dev->flags &= ~IFF_UP;
 	/*
 	 *	Shutdown NET_DMA
 	 */
 	net_dmaengine_put();
 	return 0;
 }
 /**
  *	dev_close - shutdown an interface.
  *	@dev: device to shutdown
  *
  *	This function moves an active device into down state. A
  *	%NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device
  *	is then deactivated and finally a %NETDEV_DOWN is sent to the notifier
  *	chain.
  */
 int dev_close(struct net_device *dev)
 {
 	if (!(dev->flags & IFF_UP))
 		return 0;
 	__dev_close(dev);
 	/*
 	 * Tell people we are down
 	 */
 	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING);
 	call_netdevice_notifiers(NETDEV_DOWN, dev);
 	return 0;
 }
 EXPORT_SYMBOL(dev_close);
 /**
  *	dev_disable_lro - disable Large Receive Offload on a device
  *	@dev: device
  *
  *	Disable Large Receive Offload (LRO) on a net device.  Must be
  *	called under RTNL.  This is needed if received packets may be
  *	forwarded to another interface.
  */
 void dev_disable_lro(struct net_device *dev)
 {
 	if (dev->ethtool_ops && dev->ethtool_ops->get_flags &&
 	    dev->ethtool_ops->set_flags) {
 		u32 flags = dev->ethtool_ops->get_flags(dev);
 		if (flags & ETH_FLAG_LRO) {
 			flags &= ~ETH_FLAG_LRO;
 			dev->ethtool_ops->set_flags(dev, flags);
 		}
 	}
 	WARN_ON(dev->features & NETIF_F_LRO);
 }
 EXPORT_SYMBOL(dev_disable_lro);
 static int dev_boot_phase = 1;
 /*
  *	Device change register/unregister. These are not inline or static
  *	as we export them to the world.
  */
 /**
  *	register_netdevice_notifier - register a network notifier block
  *	@nb: notifier
  *
  *	Register a notifier to be called when network device events occur.
  *	The notifier passed is linked into the kernel structures and must
  *	not be reused until it has been unregistered. A negative errno code
  *	is returned on a failure.
  *
  * 	When registered all registration and up events are replayed
  *	to the new notifier to allow device to have a race free
  *	view of the network device list.
  */
 int register_netdevice_notifier(struct notifier_block *nb)
 {
 	struct net_device *dev;
 	struct net_device *last;
 	struct net *net;
 	int err;
 	rtnl_lock();
 	err = raw_notifier_chain_register(&netdev_chain, nb);
 	if (err)
 		goto unlock;
 	if (dev_boot_phase)
 		goto unlock;
 	for_each_net(net) {
 		for_each_netdev(net, dev) {
 			err = nb->notifier_call(nb, NETDEV_REGISTER, dev);
 			err = notifier_to_errno(err);
 			if (err)
 				goto rollback;
 			if (!(dev->flags & IFF_UP))
 				continue;
 			nb->notifier_call(nb, NETDEV_UP, dev);
 		}
 	}
 unlock:
 	rtnl_unlock();
 	return err;
 rollback:
 	last = dev;
 	for_each_net(net) {
 		for_each_netdev(net, dev) {
 			if (dev == last)
 				break;
 			if (dev->flags & IFF_UP) {
 				nb->notifier_call(nb, NETDEV_GOING_DOWN, dev);
 				nb->notifier_call(nb, NETDEV_DOWN, dev);
 			}
 			nb->notifier_call(nb, NETDEV_UNREGISTER, dev);
 			nb->notifier_call(nb, NETDEV_UNREGISTER_BATCH, dev);
 		}
 	}
 	raw_notifier_chain_unregister(&netdev_chain, nb);
 	goto unlock;
 }
 EXPORT_SYMBOL(register_netdevice_notifier);
 /**
  *	unregister_netdevice_notifier - unregister a network notifier block
  *	@nb: notifier
  *
  *	Unregister a notifier previously registered by
  *	register_netdevice_notifier(). The notifier is unlinked into the
  *	kernel structures and may then be reused. A negative errno code
  *	is returned on a failure.
  */
 int unregister_netdevice_notifier(struct notifier_block *nb)
 {
 	int err;
 	rtnl_lock();
 	err = raw_notifier_chain_unregister(&netdev_chain, nb);
 	rtnl_unlock();
 	return err;
 }
 EXPORT_SYMBOL(unregister_netdevice_notifier);
 /**
  *	call_netdevice_notifiers - call all network notifier blocks
  *      @val: value passed unmodified to notifier function
  *      @dev: net_device pointer passed unmodified to notifier function
  *
  *	Call all network notifier blocks.  Parameters and return value
  *	are as for raw_notifier_call_chain().
  */
 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
 {
 	ASSERT_RTNL();
 	return raw_notifier_call_chain(&netdev_chain, val, dev);
 }
 /* When > 0 there are consumers of rx skb time stamps */
 static atomic_t netstamp_needed = ATOMIC_INIT(0);
 void net_enable_timestamp(void)
 {
 	atomic_inc(&netstamp_needed);
 }
 EXPORT_SYMBOL(net_enable_timestamp);
 void net_disable_timestamp(void)
 {
 	atomic_dec(&netstamp_needed);
 }
 EXPORT_SYMBOL(net_disable_timestamp);
 static inline void net_timestamp_set(struct sk_buff *skb)
 {
 	if (atomic_read(&netstamp_needed))
 		__net_timestamp(skb);
 	else
 		skb->tstamp.tv64 = 0;
 }
 static inline void net_timestamp_check(struct sk_buff *skb)
 {
 	if (!skb->tstamp.tv64 && atomic_read(&netstamp_needed))
 		__net_timestamp(skb);
 }
 /**
  * dev_forward_skb - loopback an skb to another netif
  *
  * @dev: destination network device
  * @skb: buffer to forward
  *
  * return values:
  *	NET_RX_SUCCESS	(no congestion)
  *	NET_RX_DROP     (packet was dropped, but freed)
  *
  * dev_forward_skb can be used for injecting an skb from the
  * start_xmit function of one device into the receive queue
  * of another device.
  *
  * The receiving device may be in another namespace, so
  * we have to clear all information in the skb that could
  * impact namespace isolation.
  */
 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
 {
 	skb_orphan(skb);
 	nf_reset(skb);
 	if (unlikely(!(dev->flags & IFF_UP) ||
 		     (skb->len > (dev->mtu + dev->hard_header_len)))) {
 		atomic_long_inc(&dev->rx_dropped);
 		kfree_skb(skb);
 		return NET_RX_DROP;
 	}
 	skb_set_dev(skb, dev);
 	skb->tstamp.tv64 = 0;
 	skb->pkt_type = PACKET_HOST;
 	skb->protocol = eth_type_trans(skb, dev);
 	return netif_rx(skb);
 }
 EXPORT_SYMBOL_GPL(dev_forward_skb);
 /*
  *	Support routine. Sends outgoing frames to any network
  *	taps currently in use.
  */
 static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct packet_type *ptype;
 #ifdef CONFIG_NET_CLS_ACT
 	if (!(skb->tstamp.tv64 && (G_TC_FROM(skb->tc_verd) & AT_INGRESS)))
 		net_timestamp_set(skb);
 #else
 	net_timestamp_set(skb);
 #endif
 	rcu_read_lock();
 	list_for_each_entry_rcu(ptype, &ptype_all, list) {
 		/* Never send packets back to the socket
 		 * they originated from - MvS (miquels@drinkel.ow.org)
 		 */
 		if ((ptype->dev == dev || !ptype->dev) &&
 		    (ptype->af_packet_priv == NULL ||
 		     (struct sock *)ptype->af_packet_priv != skb->sk)) {
 			struct sk_buff *skb2 = skb_clone(skb, GFP_ATOMIC);
 			if (!skb2)
 				break;
 			/* skb->nh should be correctly
 			   set by sender, so that the second statement is
 			   just protection against buggy protocols.
 			 */
 			skb_reset_mac_header(skb2);
 			if (skb_network_header(skb2) < skb2->data ||
 			    skb2->network_header > skb2->tail) {
 				if (net_ratelimit())
 					printk(KERN_CRIT "protocol %04x is "
 					       "buggy, dev %s\n",
 					       ntohs(skb2->protocol),
 					       dev->name);
 				skb_reset_network_header(skb2);
 			}
 			skb2->transport_header = skb2->network_header;
 			skb2->pkt_type = PACKET_OUTGOING;
 			ptype->func(skb2, skb->dev, ptype, skb->dev);
 		}
 	}
 	rcu_read_unlock();
 }
 /*
  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
  * greater then real_num_tx_queues stale skbs on the qdisc must be flushed.
  */
 void netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
 {
 	unsigned int real_num = dev->real_num_tx_queues;
 	if (unlikely(txq > dev->num_tx_queues))
 		;
 	else if (txq > real_num)
 		dev->real_num_tx_queues = txq;
 	else if (txq < real_num) {
 		dev->real_num_tx_queues = txq;
 		qdisc_reset_all_tx_gt(dev, txq);
 	}
 }
 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
 #ifdef CONFIG_RPS
 /**
  *	netif_set_real_num_rx_queues - set actual number of RX queues used
  *	@dev: Network device
  *	@rxq: Actual number of RX queues
  *
  *	This must be called either with the rtnl_lock held or before
  *	registration of the net device.  Returns 0 on success, or a
  *	negative error code.  If called before registration, it always
  *	succeeds.
  */
 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
 {
 	int rc;
 	if (dev->reg_state == NETREG_REGISTERED) {
 		ASSERT_RTNL();
 		if (rxq > dev->num_rx_queues)
 			return -EINVAL;
 		rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
 						  rxq);
 		if (rc)
 			return rc;
 	}
 	dev->real_num_rx_queues = rxq;
 	return 0;
 }
 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
 #endif
 static inline void __netif_reschedule(struct Qdisc *q)
 {
 	struct softnet_data *sd;
 	unsigned long flags;
 	local_irq_save(flags);
 	sd = &__get_cpu_var(softnet_data);
 	q->next_sched = NULL;
 	*sd->output_queue_tailp = q;
 	sd->output_queue_tailp = &q->next_sched;
 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
 	local_irq_restore(flags);
 }
 void __netif_schedule(struct Qdisc *q)
 {
 	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
 		__netif_reschedule(q);
 }
 EXPORT_SYMBOL(__netif_schedule);
 void dev_kfree_skb_irq(struct sk_buff *skb)
 {
 	if (atomic_dec_and_test(&skb->users)) {
 		struct softnet_data *sd;
 		unsigned long flags;
 		local_irq_save(flags);
 		sd = &__get_cpu_var(softnet_data);
 		skb->next = sd->completion_queue;
 		sd->completion_queue = skb;
 		raise_softirq_irqoff(NET_TX_SOFTIRQ);
 		local_irq_restore(flags);
 	}
 }
 EXPORT_SYMBOL(dev_kfree_skb_irq);
 void dev_kfree_skb_any(struct sk_buff *skb)
 {
 	if (in_irq() || irqs_disabled())
 		dev_kfree_skb_irq(skb);
 	else
 		dev_kfree_skb(skb);
 }
 EXPORT_SYMBOL(dev_kfree_skb_any);
 /**
  * netif_device_detach - mark device as removed
  * @dev: network device
  *
  * Mark device as removed from system and therefore no longer available.
  */
 void netif_device_detach(struct net_device *dev)
 {
 	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
 	    netif_running(dev)) {
 		netif_tx_stop_all_queues(dev);
 	}
 }
 EXPORT_SYMBOL(netif_device_detach);
 /**
  * netif_device_attach - mark device as attached
  * @dev: network device
  *
  * Mark device as attached from system and restart if needed.
  */
 void netif_device_attach(struct net_device *dev)
 {
 	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
 	    netif_running(dev)) {
 		netif_tx_wake_all_queues(dev);
 		__netdev_watchdog_up(dev);
 	}
 }
 EXPORT_SYMBOL(netif_device_attach);
 static bool can_checksum_protocol(unsigned long features, __be16 protocol)
 {
 	return ((features & NETIF_F_GEN_CSUM) ||
 		((features & NETIF_F_IP_CSUM) &&
 		 protocol == htons(ETH_P_IP)) ||
 		((features & NETIF_F_IPV6_CSUM) &&
 		 protocol == htons(ETH_P_IPV6)) ||
 		((features & NETIF_F_FCOE_CRC) &&
 		 protocol == htons(ETH_P_FCOE)));
 }
 static bool dev_can_checksum(struct net_device *dev, struct sk_buff *skb)
 {
 	if (can_checksum_protocol(dev->features, skb->protocol))
 		return true;
 	if (skb->protocol == htons(ETH_P_8021Q)) {
 		struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 		if (can_checksum_protocol(dev->features & dev->vlan_features,
 					  veh->h_vlan_encapsulated_proto))
 			return true;
 	}
 	return false;
 }
 /**
  * skb_dev_set -- assign a new device to a buffer
  * @skb: buffer for the new device
  * @dev: network device
  *
  * If an skb is owned by a device already, we have to reset
  * all data private to the namespace a device belongs to
  * before assigning it a new device.
  */
 #ifdef CONFIG_NET_NS
 void skb_set_dev(struct sk_buff *skb, struct net_device *dev)
 {
 	skb_dst_drop(skb);
 	if (skb->dev && !net_eq(dev_net(skb->dev), dev_net(dev))) {
 		secpath_reset(skb);
 		nf_reset(skb);
 		skb_init_secmark(skb);
 		skb->mark = 0;
 		skb->priority = 0;
 		skb->nf_trace = 0;
 		skb->ipvs_property = 0;
 #ifdef CONFIG_NET_SCHED
 		skb->tc_index = 0;
 #endif
 	}
 	skb->dev = dev;
 }
 EXPORT_SYMBOL(skb_set_dev);
 #endif /* CONFIG_NET_NS */
 /*
  * Invalidate hardware checksum when packet is to be mangled, and
  * complete checksum manually on outgoing path.
  */
 int skb_checksum_help(struct sk_buff *skb)
 {
 	__wsum csum;
 	int ret = 0, offset;
 	if (skb->ip_summed == CHECKSUM_COMPLETE)
 		goto out_set_summed;
 	if (unlikely(skb_shinfo(skb)->gso_size)) {
 		/* Let GSO fix up the checksum. */
 		goto out_set_summed;
 	}
 	offset = skb->csum_start - skb_headroom(skb);
 	BUG_ON(offset >= skb_headlen(skb));
 	csum = skb_checksum(skb, offset, skb->len - offset, 0);
 	offset += skb->csum_offset;
 	BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb));
 	if (skb_cloned(skb) &&
 	    !skb_clone_writable(skb, offset + sizeof(__sum16))) {
 		ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
 		if (ret)
 			goto out;
 	}
 	*(__sum16 *)(skb->data + offset) = csum_fold(csum);
 out_set_summed:
 	skb->ip_summed = CHECKSUM_NONE;
 out:
 	return ret;
 }
 EXPORT_SYMBOL(skb_checksum_help);
 /**
  *	skb_gso_segment - Perform segmentation on skb.
  *	@skb: buffer to segment
  *	@features: features for the output path (see dev->features)
  *
  *	This function segments the given skb and returns a list of segments.
  *
  *	It may return NULL if the skb requires no segmentation.  This is
  *	only possible when GSO is used for verifying header integrity.
  */
 struct sk_buff *skb_gso_segment(struct sk_buff *skb, int features)
 {
 	struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT);
 	struct packet_type *ptype;
 	__be16 type = skb->protocol;
 	int err;
 	skb_reset_mac_header(skb);
 	skb->mac_len = skb->network_header - skb->mac_header;
 	__skb_pull(skb, skb->mac_len);
 	if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
 		struct net_device *dev = skb->dev;
 		struct ethtool_drvinfo info = {};
 		if (dev && dev->ethtool_ops && dev->ethtool_ops->get_drvinfo)
 			dev->ethtool_ops->get_drvinfo(dev, &info);
 		WARN(1, "%s: caps=(0x%lx, 0x%lx) len=%d data_len=%d "
 			"ip_summed=%d",
 		     info.driver, dev ? dev->features : 0L,
 		     skb->sk ? skb->sk->sk_route_caps : 0L,
 		     skb->len, skb->data_len, skb->ip_summed);
 		if (skb_header_cloned(skb) &&
 		    (err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC)))
 			return ERR_PTR(err);
 	}
 	rcu_read_lock();
 	list_for_each_entry_rcu(ptype,
 			&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
 		if (ptype->type == type && !ptype->dev && ptype->gso_segment) {
 			if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
 				err = ptype->gso_send_check(skb);
 				segs = ERR_PTR(err);
 				if (err || skb_gso_ok(skb, features))
 					break;
 				__skb_push(skb, (skb->data -
 						 skb_network_header(skb)));
 			}
 			segs = ptype->gso_segment(skb, features);
 			break;
 		}
 	}
 	rcu_read_unlock();
 	__skb_push(skb, skb->data - skb_mac_header(skb));
 	return segs;
 }
 EXPORT_SYMBOL(skb_gso_segment);
 /* Take action when hardware reception checksum errors are detected. */
 #ifdef CONFIG_BUG
 void netdev_rx_csum_fault(struct net_device *dev)
 {
 	if (net_ratelimit()) {
 		printk(KERN_ERR "%s: hw csum failure.\n",
 			dev ? dev->name : "<unknown>");
 		dump_stack();
 	}
 }
 EXPORT_SYMBOL(netdev_rx_csum_fault);
 #endif
 /* Actually, we should eliminate this check as soon as we know, that:
  * 1. IOMMU is present and allows to map all the memory.
  * 2. No high memory really exists on this machine.
  */
 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
 {
 #ifdef CONFIG_HIGHMEM
 	int i;
 	if (!(dev->features & NETIF_F_HIGHDMA)) {
 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++)
 			if (PageHighMem(skb_shinfo(skb)->frags[i].page))
 				return 1;
 	}
 	if (PCI_DMA_BUS_IS_PHYS) {
 		struct device *pdev = dev->dev.parent;
 		if (!pdev)
 			return 0;
 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 			dma_addr_t addr = page_to_phys(skb_shinfo(skb)->frags[i].page);
 			if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask)
 				return 1;
 		}
 	}
 #endif
 	return 0;
 }
 struct dev_gso_cb {
 	void (*destructor)(struct sk_buff *skb);
 };
 #define DEV_GSO_CB(skb) ((struct dev_gso_cb *)(skb)->cb)
 static void dev_gso_skb_destructor(struct sk_buff *skb)
 {
 	struct dev_gso_cb *cb;
 	do {
 		struct sk_buff *nskb = skb->next;
 		skb->next = nskb->next;
 		nskb->next = NULL;
 		kfree_skb(nskb);
 	} while (skb->next);
 	cb = DEV_GSO_CB(skb);
 	if (cb->destructor)
 		cb->destructor(skb);
 }
 /**
  *	dev_gso_segment - Perform emulated hardware segmentation on skb.
  *	@skb: buffer to segment
  *
  *	This function segments the given skb and stores the list of segments
  *	in skb->next.
  */
 static int dev_gso_segment(struct sk_buff *skb)
 {
 	struct net_device *dev = skb->dev;
 	struct sk_buff *segs;
 	int features = dev->features & ~(illegal_highdma(dev, skb) ?
 					 NETIF_F_SG : 0);
 	segs = skb_gso_segment(skb, features);
 	/* Verifying header integrity only. */
 	if (!segs)
 		return 0;
 	if (IS_ERR(segs))
 		return PTR_ERR(segs);
 	skb->next = segs;
 	DEV_GSO_CB(skb)->destructor = skb->destructor;
 	skb->destructor = dev_gso_skb_destructor;
 	return 0;
 }
 /*
  * Try to orphan skb early, right before transmission by the device.
  * We cannot orphan skb if tx timestamp is requested or the sk-reference
  * is needed on driver level for other reasons, e.g. see net/can/raw.c
  */
 static inline void skb_orphan_try(struct sk_buff *skb)
 {
 	struct sock *sk = skb->sk;
 	if (sk && !skb_shinfo(skb)->tx_flags) {
 		/* skb_tx_hash() wont be able to get sk.
 		 * We copy sk_hash into skb->rxhash
 		 */
 		if (!skb->rxhash)
 			skb->rxhash = sk->sk_hash;
 		skb_orphan(skb);
 	}
 }
 /*
  * Returns true if either:
  *	1. skb has frag_list and the device doesn't support FRAGLIST, or
  *	2. skb is fragmented and the device does not support SG, or if
  *	   at least one of fragments is in highmem and device does not
  *	   support DMA from it.
  */
 static inline int skb_needs_linearize(struct sk_buff *skb,
 				      struct net_device *dev)
 {
 	return skb_is_nonlinear(skb) &&
 	       ((skb_has_frag_list(skb) && !(dev->features & NETIF_F_FRAGLIST)) ||
 	        (skb_shinfo(skb)->nr_frags && (!(dev->features & NETIF_F_SG) ||
 					      illegal_highdma(dev, skb))));
 }
 int dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
 			struct netdev_queue *txq)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	int rc = NETDEV_TX_OK;
 	if (likely(!skb->next)) {
 		if (!list_empty(&ptype_all))
 			dev_queue_xmit_nit(skb, dev);
 		/*
 		 * If device doesnt need skb->dst, release it right now while
 		 * its hot in this cpu cache
 		 */
 		if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
 			skb_dst_drop(skb);
 		skb_orphan_try(skb);
 		if (netif_needs_gso(dev, skb)) {
 			if (unlikely(dev_gso_segment(skb)))
 				goto out_kfree_skb;
 			if (skb->next)
 				goto gso;
 		} else {
 			if (skb_needs_linearize(skb, dev) &&
 			    __skb_linearize(skb))
 				goto out_kfree_skb;
 			/* If packet is not checksummed and device does not
 			 * support checksumming for this protocol, complete
 			 * checksumming here.
 			 */
 			if (skb->ip_summed == CHECKSUM_PARTIAL) {
 				skb_set_transport_header(skb, skb->csum_start -
 					      skb_headroom(skb));
 				if (!dev_can_checksum(dev, skb) &&
 				     skb_checksum_help(skb))
 					goto out_kfree_skb;
 			}
 		}
 		rc = ops->ndo_start_xmit(skb, dev);
 		if (rc == NETDEV_TX_OK)
 			txq_trans_update(txq);
 		return rc;
 	}
 gso:
 	do {
 		struct sk_buff *nskb = skb->next;
 		skb->next = nskb->next;
 		nskb->next = NULL;
 		/*
 		 * If device doesnt need nskb->dst, release it right now while
 		 * its hot in this cpu cache
 		 */
 		if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
 			skb_dst_drop(nskb);
 		rc = ops->ndo_start_xmit(nskb, dev);
 		if (unlikely(rc != NETDEV_TX_OK)) {
 			if (rc & ~NETDEV_TX_MASK)
 				goto out_kfree_gso_skb;
 			nskb->next = skb->next;
 			skb->next = nskb;
 			return rc;
 		}
 		txq_trans_update(txq);
 		if (unlikely(netif_tx_queue_stopped(txq) && skb->next))
 			return NETDEV_TX_BUSY;
 	} while (skb->next);
 out_kfree_gso_skb:
 	if (likely(skb->next == NULL))
 		skb->destructor = DEV_GSO_CB(skb)->destructor;
 out_kfree_skb:
 	kfree_skb(skb);
 	return rc;
 }
 static u32 hashrnd __read_mostly;
 u16 skb_tx_hash(const struct net_device *dev, const struct sk_buff *skb)
 {
 	u32 hash;
 	if (skb_rx_queue_recorded(skb)) {
 		hash = skb_get_rx_queue(skb);
 		while (unlikely(hash >= dev->real_num_tx_queues))
 			hash -= dev->real_num_tx_queues;
 		return hash;
 	}
 	if (skb->sk && skb->sk->sk_hash)
 		hash = skb->sk->sk_hash;
 	else
 		hash = (__force u16) skb->protocol ^ skb->rxhash;
 	hash = jhash_1word(hash, hashrnd);
 	return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
 }
 EXPORT_SYMBOL(skb_tx_hash);
 static inline u16 dev_cap_txqueue(struct net_device *dev, u16 queue_index)
 {
 	if (unlikely(queue_index >= dev->real_num_tx_queues)) {
 		if (net_ratelimit()) {
 			pr_warning("%s selects TX queue %d, but "
 				"real number of TX queues is %d\n",
 				dev->name, queue_index, dev->real_num_tx_queues);
 		}
 		return 0;
 	}
 	return queue_index;
 }
 static struct netdev_queue *dev_pick_tx(struct net_device *dev,
 					struct sk_buff *skb)
 {
 	int queue_index;
 	const struct net_device_ops *ops = dev->netdev_ops;
 	if (ops->ndo_select_queue) {
 		queue_index = ops->ndo_select_queue(dev, skb);
 		queue_index = dev_cap_txqueue(dev, queue_index);
 	} else {
 		struct sock *sk = skb->sk;
 		queue_index = sk_tx_queue_get(sk);
 		if (queue_index < 0) {
 			queue_index = 0;
 			if (dev->real_num_tx_queues > 1)
 				queue_index = skb_tx_hash(dev, skb);
 			if (sk) {
 				struct dst_entry *dst = rcu_dereference_check(sk->sk_dst_cache, 1);
 				if (dst && skb_dst(skb) == dst)
 					sk_tx_queue_set(sk, queue_index);
 			}
 		}
 	}
 	skb_set_queue_mapping(skb, queue_index);
 	return netdev_get_tx_queue(dev, queue_index);
 }
 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
 				 struct net_device *dev,
 				 struct netdev_queue *txq)
 {
 	spinlock_t *root_lock = qdisc_lock(q);
 	bool contended = qdisc_is_running(q);
 	int rc;
 	/*
 	 * Heuristic to force contended enqueues to serialize on a
 	 * separate lock before trying to get qdisc main lock.
 	 * This permits __QDISC_STATE_RUNNING owner to get the lock more often
 	 * and dequeue packets faster.
 	 */
 	if (unlikely(contended))
 		spin_lock(&q->busylock);
 	spin_lock(root_lock);
 	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
 		kfree_skb(skb);
 		rc = NET_XMIT_DROP;
 	} else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
 		   qdisc_run_begin(q)) {
 		/*
 		 * This is a work-conserving queue; there are no old skbs
 		 * waiting to be sent out; and the qdisc is not running -
 		 * xmit the skb directly.
 		 */
 		if (!(dev->priv_flags & IFF_XMIT_DST_RELEASE))
 			skb_dst_force(skb);
 		__qdisc_update_bstats(q, skb->len);
 		if (sch_direct_xmit(skb, q, dev, txq, root_lock)) {
 			if (unlikely(contended)) {
 				spin_unlock(&q->busylock);
 				contended = false;
 			}
 			__qdisc_run(q);
 		} else
 			qdisc_run_end(q);
 		rc = NET_XMIT_SUCCESS;
 	} else {
 		skb_dst_force(skb);
 		rc = qdisc_enqueue_root(skb, q);
 		if (qdisc_run_begin(q)) {
 			if (unlikely(contended)) {
 				spin_unlock(&q->busylock);
 				contended = false;
 			}
 			__qdisc_run(q);
 		}
 	}
 	spin_unlock(root_lock);
 	if (unlikely(contended))
 		spin_unlock(&q->busylock);
 	return rc;
 }
 static DEFINE_PER_CPU(int, xmit_recursion);
 #define RECURSION_LIMIT 3
 /**
  *	dev_queue_xmit - transmit a buffer
  *	@skb: buffer to transmit
  *
  *	Queue a buffer for transmission to a network device. The caller must
  *	have set the device and priority and built the buffer before calling
  *	this function. The function can be called from an interrupt.
  *
  *	A negative errno code is returned on a failure. A success does not
  *	guarantee the frame will be transmitted as it may be dropped due
  *	to congestion or traffic shaping.
  *
  * -----------------------------------------------------------------------------------
  *      I notice this method can also return errors from the queue disciplines,
  *      including NET_XMIT_DROP, which is a positive value.  So, errors can also
  *      be positive.
  *
  *      Regardless of the return value, the skb is consumed, so it is currently
  *      difficult to retry a send to this method.  (You can bump the ref count
  *      before sending to hold a reference for retry if you are careful.)
  *
  *      When calling this method, interrupts MUST be enabled.  This is because
  *      the BH enable code must have IRQs enabled so that it will not deadlock.
  *          --BLG
  */
 int dev_queue_xmit(struct sk_buff *skb)
 {
 	struct net_device *dev = skb->dev;
 	struct netdev_queue *txq;
 	struct Qdisc *q;
 	int rc = -ENOMEM;
 	/* Disable soft irqs for various locks below. Also
 	 * stops preemption for RCU.
 	 */
 	rcu_read_lock_bh();
 	txq = dev_pick_tx(dev, skb);
 	q = rcu_dereference_bh(txq->qdisc);
 #ifdef CONFIG_NET_CLS_ACT
 	skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS);
 #endif
 	if (q->enqueue) {
 		rc = __dev_xmit_skb(skb, q, dev, txq);
 		goto out;
 	}
 	/* The device has no queue. Common case for software devices:
 	   loopback, all the sorts of tunnels...
 	   Really, it is unlikely that netif_tx_lock protection is necessary
 	   here.  (f.e. loopback and IP tunnels are clean ignoring statistics
 	   counters.)
 	   However, it is possible, that they rely on protection
 	   made by us here.
 	   Check this and shot the lock. It is not prone from deadlocks.
 	   Either shot noqueue qdisc, it is even simpler 8)
 	 */
 	if (dev->flags & IFF_UP) {
 		int cpu = smp_processor_id(); /* ok because BHs are off */
 		if (txq->xmit_lock_owner != cpu) {
 			if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT)
 				goto recursion_alert;
 			HARD_TX_LOCK(dev, txq, cpu);
 			if (!netif_tx_queue_stopped(txq)) {
 				__this_cpu_inc(xmit_recursion);
 				rc = dev_hard_start_xmit(skb, dev, txq);
 				__this_cpu_dec(xmit_recursion);
 				if (dev_xmit_complete(rc)) {
 					HARD_TX_UNLOCK(dev, txq);
 					goto out;
 				}
 			}
 			HARD_TX_UNLOCK(dev, txq);
 			if (net_ratelimit())
 				printk(KERN_CRIT "Virtual device %s asks to "
 				       "queue packet!\n", dev->name);
 		} else {
 			/* Recursion is detected! It is possible,
 			 * unfortunately
 			 */
 recursion_alert:
 			if (net_ratelimit())
 				printk(KERN_CRIT "Dead loop on virtual device "
 				       "%s, fix it urgently!\n", dev->name);
 		}
 	}
 	rc = -ENETDOWN;
 	rcu_read_unlock_bh();
 	kfree_skb(skb);
 	return rc;
 out:
 	rcu_read_unlock_bh();
 	return rc;
 }
 EXPORT_SYMBOL(dev_queue_xmit);
 /*=======================================================================
 			Receiver routines
   =======================================================================*/
 int netdev_max_backlog __read_mostly = 1000;
 int netdev_tstamp_prequeue __read_mostly = 1;
 int netdev_budget __read_mostly = 300;
 int weight_p __read_mostly = 64;            /* old backlog weight */
 /* Called with irq disabled */
 static inline void ____napi_schedule(struct softnet_data *sd,
 				     struct napi_struct *napi)
 {
 	list_add_tail(&napi->poll_list, &sd->poll_list);
 	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
 }
 /*
  * __skb_get_rxhash: calculate a flow hash based on src/dst addresses
  * and src/dst port numbers. Returns a non-zero hash number on success
  * and 0 on failure.
  */
 __u32 __skb_get_rxhash(struct sk_buff *skb)
 {
 	int nhoff, hash = 0, poff;
 	struct ipv6hdr *ip6;
 	struct iphdr *ip;
 	u8 ip_proto;
 	u32 addr1, addr2, ihl;
 	union {
 		u32 v32;
 		u16 v16[2];
 	} ports;
 	nhoff = skb_network_offset(skb);
 	switch (skb->protocol) {
 	case __constant_htons(ETH_P_IP):
 		if (!pskb_may_pull(skb, sizeof(*ip) + nhoff))
 			goto done;
 		ip = (struct iphdr *) (skb->data + nhoff);
 		if (ip->frag_off & htons(IP_MF | IP_OFFSET))
 			ip_proto = 0;
 		else
 			ip_proto = ip->protocol;
 		addr1 = (__force u32) ip->saddr;
 		addr2 = (__force u32) ip->daddr;
 		ihl = ip->ihl;
 		break;
 	case __constant_htons(ETH_P_IPV6):
 		if (!pskb_may_pull(skb, sizeof(*ip6) + nhoff))
 			goto done;
 		ip6 = (struct ipv6hdr *) (skb->data + nhoff);
 		ip_proto = ip6->nexthdr;
 		addr1 = (__force u32) ip6->saddr.s6_addr32[3];
 		addr2 = (__force u32) ip6->daddr.s6_addr32[3];
 		ihl = (40 >> 2);
 		break;
 	default:
 		goto done;
 	}
 	ports.v32 = 0;
 	poff = proto_ports_offset(ip_proto);
 	if (poff >= 0) {
 		nhoff += ihl * 4 + poff;
 		if (pskb_may_pull(skb, nhoff + 4)) {
 			ports.v32 = * (__force u32 *) (skb->data + nhoff);
 			if (ports.v16[1] < ports.v16[0])
 				swap(ports.v16[0], ports.v16[1]);
 		}
 	}
 	/* get a consistent hash (same value on both flow directions) */
 	if (addr2 < addr1)
 		swap(addr1, addr2);
 	hash = jhash_3words(addr1, addr2, ports.v32, hashrnd);
 	if (!hash)
 		hash = 1;
 done:
 	return hash;
 }
 EXPORT_SYMBOL(__skb_get_rxhash);
 #ifdef CONFIG_RPS
 /* One global table that all flow-based protocols share. */
 struct rps_sock_flow_table *rps_sock_flow_table __read_mostly;
 EXPORT_SYMBOL(rps_sock_flow_table);
 /*
  * get_rps_cpu is called from netif_receive_skb and returns the target
  * CPU from the RPS map of the receiving queue for a given skb.
  * rcu_read_lock must be held on entry.
  */
 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
 		       struct rps_dev_flow **rflowp)
 {
 	struct netdev_rx_queue *rxqueue;
 	struct rps_map *map = NULL;
 	struct rps_dev_flow_table *flow_table;
 	struct rps_sock_flow_table *sock_flow_table;
 	int cpu = -1;
 	u16 tcpu;
 	if (skb_rx_queue_recorded(skb)) {
 		u16 index = skb_get_rx_queue(skb);
 		if (unlikely(index >= dev->real_num_rx_queues)) {
 			WARN_ONCE(dev->real_num_rx_queues > 1,
 				  "%s received packet on queue %u, but number "
 				  "of RX queues is %u\n",
 				  dev->name, index, dev->real_num_rx_queues);
 			goto done;
 		}
 		rxqueue = dev->_rx + index;
 	} else
 		rxqueue = dev->_rx;
 	if (rxqueue->rps_map) {
 		map = rcu_dereference(rxqueue->rps_map);
 		if (map && map->len == 1) {
 			tcpu = map->cpus[0];
 			if (cpu_online(tcpu))
 				cpu = tcpu;
 			goto done;
 		}
 	} else if (!rxqueue->rps_flow_table) {
 		goto done;
 	}
 	skb_reset_network_header(skb);
 	if (!skb_get_rxhash(skb))
 		goto done;
 	flow_table = rcu_dereference(rxqueue->rps_flow_table);
 	sock_flow_table = rcu_dereference(rps_sock_flow_table);
 	if (flow_table && sock_flow_table) {
 		u16 next_cpu;
 		struct rps_dev_flow *rflow;
 		rflow = &flow_table->flows[skb->rxhash & flow_table->mask];
 		tcpu = rflow->cpu;
 		next_cpu = sock_flow_table->ents[skb->rxhash &
 		    sock_flow_table->mask];
 		/*
 		 * If the desired CPU (where last recvmsg was done) is
 		 * different from current CPU (one in the rx-queue flow
 		 * table entry), switch if one of the following holds:
 		 *   - Current CPU is unset (equal to RPS_NO_CPU).
 		 *   - Current CPU is offline.
 		 *   - The current CPU's queue tail has advanced beyond the
 		 *     last packet that was enqueued using this table entry.
 		 *     This guarantees that all previous packets for the flow
 		 *     have been dequeued, thus preserving in order delivery.
 		 */
 		if (unlikely(tcpu != next_cpu) &&
 		    (tcpu == RPS_NO_CPU || !cpu_online(tcpu) ||
 		     ((int)(per_cpu(softnet_data, tcpu).input_queue_head -
 		      rflow->last_qtail)) >= 0)) {
 			tcpu = rflow->cpu = next_cpu;
 			if (tcpu != RPS_NO_CPU)
 				rflow->last_qtail = per_cpu(softnet_data,
 				    tcpu).input_queue_head;
 		}
 		if (tcpu != RPS_NO_CPU && cpu_online(tcpu)) {
 			*rflowp = rflow;
 			cpu = tcpu;
 			goto done;
 		}
 	}
 	if (map) {
 		tcpu = map->cpus[((u64) skb->rxhash * map->len) >> 32];
 		if (cpu_online(tcpu)) {
 			cpu = tcpu;
 			goto done;
 		}
 	}
 done:
 	return cpu;
 }
 /* Called from hardirq (IPI) context */
 static void rps_trigger_softirq(void *data)
 {
 	struct softnet_data *sd = data;
 	____napi_schedule(sd, &sd->backlog);
 	sd->received_rps++;
 }
 #endif /* CONFIG_RPS */
 /*
  * Check if this softnet_data structure is another cpu one
  * If yes, queue it to our IPI list and return 1
  * If no, return 0
  */
 static int rps_ipi_queued(struct softnet_data *sd)
 {
 #ifdef CONFIG_RPS
 	struct softnet_data *mysd = &__get_cpu_var(softnet_data);
 	if (sd != mysd) {
 		sd->rps_ipi_next = mysd->rps_ipi_list;
 		mysd->rps_ipi_list = sd;
 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
 		return 1;
 	}
 #endif /* CONFIG_RPS */
 	return 0;
 }
 /*
  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
  * queue (may be a remote CPU queue).
  */
 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
 			      unsigned int *qtail)
 {
 	struct softnet_data *sd;
 	unsigned long flags;
 	sd = &per_cpu(softnet_data, cpu);
 	local_irq_save(flags);
 	rps_lock(sd);
 	if (skb_queue_len(&sd->input_pkt_queue) <= netdev_max_backlog) {
 		if (skb_queue_len(&sd->input_pkt_queue)) {
 enqueue:
 			__skb_queue_tail(&sd->input_pkt_queue, skb);
 			input_queue_tail_incr_save(sd, qtail);
 			rps_unlock(sd);
 			local_irq_restore(flags);
 			return NET_RX_SUCCESS;
 		}
 		/* Schedule NAPI for backlog device
 		 * We can use non atomic operation since we own the queue lock
 		 */
 		if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
 			if (!rps_ipi_queued(sd))
 				____napi_schedule(sd, &sd->backlog);
 		}
 		goto enqueue;
 	}
 	sd->dropped++;
 	rps_unlock(sd);
 	local_irq_restore(flags);
 	atomic_long_inc(&skb->dev->rx_dropped);
 	kfree_skb(skb);
 	return NET_RX_DROP;
 }
 /**
  *	netif_rx	-	post buffer to the network code
  *	@skb: buffer to post
  *
  *	This function receives a packet from a device driver and queues it for
  *	the upper (protocol) levels to process.  It always succeeds. The buffer
  *	may be dropped during processing for congestion control or by the
  *	protocol layers.
  *
  *	return values:
  *	NET_RX_SUCCESS	(no congestion)
  *	NET_RX_DROP     (packet was dropped)
  *
  */
 int netif_rx(struct sk_buff *skb)
 {
 	int ret;
 	/* if netpoll wants it, pretend we never saw it */
 	if (netpoll_rx(skb))
 		return NET_RX_DROP;
 	if (netdev_tstamp_prequeue)
 		net_timestamp_check(skb);
 #ifdef CONFIG_RPS
 	{
 		struct rps_dev_flow voidflow, *rflow = &voidflow;
 		int cpu;
 		preempt_disable();
 		rcu_read_lock();
 		cpu = get_rps_cpu(skb->dev, skb, &rflow);
 		if (cpu < 0)
 			cpu = smp_processor_id();
 		ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
 		rcu_read_unlock();
 		preempt_enable();
 	}
 #else
 	{
 		unsigned int qtail;
 		ret = enqueue_to_backlog(skb, get_cpu(), &qtail);
 		put_cpu();
 	}
 #endif
 	return ret;
 }
 EXPORT_SYMBOL(netif_rx);
 int netif_rx_ni(struct sk_buff *skb)
 {
 	int err;
 	preempt_disable();
 	err = netif_rx(skb);
 	if (local_softirq_pending())
 		do_softirq();
 	preempt_enable();
 	return err;
 }
 EXPORT_SYMBOL(netif_rx_ni);
 static void net_tx_action(struct softirq_action *h)
 {
 	struct softnet_data *sd = &__get_cpu_var(softnet_data);
 	if (sd->completion_queue) {
 		struct sk_buff *clist;
 		local_irq_disable();
 		clist = sd->completion_queue;
 		sd->completion_queue = NULL;
 		local_irq_enable();
 		while (clist) {
 			struct sk_buff *skb = clist;
 			clist = clist->next;
 			WARN_ON(atomic_read(&skb->users));
 			__kfree_skb(skb);
 		}
 	}
 	if (sd->output_queue) {
 		struct Qdisc *head;
 		local_irq_disable();
 		head = sd->output_queue;
 		sd->output_queue = NULL;
 		sd->output_queue_tailp = &sd->output_queue;
 		local_irq_enable();
 		while (head) {
 			struct Qdisc *q = head;
 			spinlock_t *root_lock;
 			head = head->next_sched;
 			root_lock = qdisc_lock(q);
 			if (spin_trylock(root_lock)) {
 				smp_mb__before_clear_bit();
 				clear_bit(__QDISC_STATE_SCHED,
 					  &q->state);
 				qdisc_run(q);
 				spin_unlock(root_lock);
 			} else {
 				if (!test_bit(__QDISC_STATE_DEACTIVATED,
 					      &q->state)) {
 					__netif_reschedule(q);
 				} else {
 					smp_mb__before_clear_bit();
 					clear_bit(__QDISC_STATE_SCHED,
 						  &q->state);
 				}
 			}
 		}
 	}
 }
 static inline int deliver_skb(struct sk_buff *skb,
 			      struct packet_type *pt_prev,
 			      struct net_device *orig_dev)
 {
 	atomic_inc(&skb->users);
 	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
 }
 #if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \
     (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE))
 /* This hook is defined here for ATM LANE */
 int (*br_fdb_test_addr_hook)(struct net_device *dev,
 			     unsigned char *addr) __read_mostly;
 EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook);
 #endif
 #ifdef CONFIG_NET_CLS_ACT
 /* TODO: Maybe we should just force sch_ingress to be compiled in
  * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions
  * a compare and 2 stores extra right now if we dont have it on
  * but have CONFIG_NET_CLS_ACT
  * NOTE: This doesnt stop any functionality; if you dont have
  * the ingress scheduler, you just cant add policies on ingress.
  *
  */
 static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq)
 {
 	struct net_device *dev = skb->dev;
 	u32 ttl = G_TC_RTTL(skb->tc_verd);
 	int result = TC_ACT_OK;
 	struct Qdisc *q;
 	if (unlikely(MAX_RED_LOOP < ttl++)) {
 		if (net_ratelimit())
 			pr_warning( "Redir loop detected Dropping packet (%d->%d)\n",
 			       skb->skb_iif, dev->ifindex);
 		return TC_ACT_SHOT;
 	}
 	skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl);
 	skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS);
 	q = rxq->qdisc;
 	if (q != &noop_qdisc) {
 		spin_lock(qdisc_lock(q));
 		if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
 			result = qdisc_enqueue_root(skb, q);
 		spin_unlock(qdisc_lock(q));
 	}
 	return result;
 }
 static inline struct sk_buff *handle_ing(struct sk_buff *skb,
 					 struct packet_type **pt_prev,
 					 int *ret, struct net_device *orig_dev)
 {
 	struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue);
 	if (!rxq || rxq->qdisc == &noop_qdisc)
 		goto out;
 	if (*pt_prev) {
 		*ret = deliver_skb(skb, *pt_prev, orig_dev);
 		*pt_prev = NULL;
 	}
 	switch (ing_filter(skb, rxq)) {
 	case TC_ACT_SHOT:
 	case TC_ACT_STOLEN:
 		kfree_skb(skb);
 		return NULL;
 	}
 out:
 	skb->tc_verd = 0;
 	return skb;
 }
 #endif
 /*
  * 	netif_nit_deliver - deliver received packets to network taps
  * 	@skb: buffer
  *
  * 	This function is used to deliver incoming packets to network
  * 	taps. It should be used when the normal netif_receive_skb path
  * 	is bypassed, for example because of VLAN acceleration.
  */
 void netif_nit_deliver(struct sk_buff *skb)
 {
 	struct packet_type *ptype;
 	if (list_empty(&ptype_all))
 		return;
 	skb_reset_network_header(skb);
 	skb_reset_transport_header(skb);
 	skb->mac_len = skb->network_header - skb->mac_header;
 	rcu_read_lock();
 	list_for_each_entry_rcu(ptype, &ptype_all, list) {
 		if (!ptype->dev || ptype->dev == skb->dev)
 			deliver_skb(skb, ptype, skb->dev);
 	}
 	rcu_read_unlock();
 }
 /**
  *	netdev_rx_handler_register - register receive handler
  *	@dev: device to register a handler for
  *	@rx_handler: receive handler to register
  *	@rx_handler_data: data pointer that is used by rx handler
  *
  *	Register a receive hander for a device. This handler will then be
  *	called from __netif_receive_skb. A negative errno code is returned
  *	on a failure.
  *
  *	The caller must hold the rtnl_mutex.
  */
 int netdev_rx_handler_register(struct net_device *dev,
 			       rx_handler_func_t *rx_handler,
 			       void *rx_handler_data)
 {
 	ASSERT_RTNL();
 	if (dev->rx_handler)
 		return -EBUSY;
 	rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
 	rcu_assign_pointer(dev->rx_handler, rx_handler);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
 /**
  *	netdev_rx_handler_unregister - unregister receive handler
  *	@dev: device to unregister a handler from
  *
  *	Unregister a receive hander from a device.
  *
  *	The caller must hold the rtnl_mutex.
  */
 void netdev_rx_handler_unregister(struct net_device *dev)
 {
 	ASSERT_RTNL();
 	rcu_assign_pointer(dev->rx_handler, NULL);
 	rcu_assign_pointer(dev->rx_handler_data, NULL);
 }
 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
 static inline void skb_bond_set_mac_by_master(struct sk_buff *skb,
 					      struct net_device *master)
 {
 	if (skb->pkt_type == PACKET_HOST) {
 		u16 *dest = (u16 *) eth_hdr(skb)->h_dest;
 		memcpy(dest, master->dev_addr, ETH_ALEN);
 	}
 }
 /* On bonding slaves other than the currently active slave, suppress
  * duplicates except for 802.3ad ETH_P_SLOW, alb non-mcast/bcast, and
  * ARP on active-backup slaves with arp_validate enabled.
  */
 int __skb_bond_should_drop(struct sk_buff *skb, struct net_device *master)
 {
 	struct net_device *dev = skb->dev;
 	if (master->priv_flags & IFF_MASTER_ARPMON)
 		dev->last_rx = jiffies;
 	if ((master->priv_flags & IFF_MASTER_ALB) &&
 	    (master->priv_flags & IFF_BRIDGE_PORT)) {
 		/* Do address unmangle. The local destination address
 		 * will be always the one master has. Provides the right
 		 * functionality in a bridge.
 		 */
 		skb_bond_set_mac_by_master(skb, master);
 	}
 	if (dev->priv_flags & IFF_SLAVE_INACTIVE) {
 		if ((dev->priv_flags & IFF_SLAVE_NEEDARP) &&
 		    skb->protocol == __cpu_to_be16(ETH_P_ARP))
 			return 0;
 		if (master->priv_flags & IFF_MASTER_ALB) {
 			if (skb->pkt_type != PACKET_BROADCAST &&
 			    skb->pkt_type != PACKET_MULTICAST)
 				return 0;
 		}
 		if (master->priv_flags & IFF_MASTER_8023AD &&
 		    skb->protocol == __cpu_to_be16(ETH_P_SLOW))
 			return 0;
 		return 1;
 	}
 	return 0;
 }
 EXPORT_SYMBOL(__skb_bond_should_drop);
 static int __netif_receive_skb(struct sk_buff *skb)
 {
 	struct packet_type *ptype, *pt_prev;
 	rx_handler_func_t *rx_handler;
 	struct net_device *orig_dev;
 	struct net_device *master;
 	struct net_device *null_or_orig;
 	struct net_device *orig_or_bond;
 	int ret = NET_RX_DROP;
 	__be16 type;
 	if (!netdev_tstamp_prequeue)
 		net_timestamp_check(skb);
 	if (vlan_tx_tag_present(skb))
 		vlan_hwaccel_do_receive(skb);
 	/* if we've gotten here through NAPI, check netpoll */
 	if (netpoll_receive_skb(skb))
 		return NET_RX_DROP;
 	if (!skb->skb_iif)
 		skb->skb_iif = skb->dev->ifindex;
 	/*
 	 * bonding note: skbs received on inactive slaves should only
 	 * be delivered to pkt handlers that are exact matches.  Also
 	 * the deliver_no_wcard flag will be set.  If packet handlers
 	 * are sensitive to duplicate packets these skbs will need to
 	 * be dropped at the handler.  The vlan accel path may have
 	 * already set the deliver_no_wcard flag.
 	 */
 	null_or_orig = NULL;
 	orig_dev = skb->dev;
 	master = ACCESS_ONCE(orig_dev->master);
 	if (skb->deliver_no_wcard)
 		null_or_orig = orig_dev;
 	else if (master) {
 		if (skb_bond_should_drop(skb, master)) {
 			skb->deliver_no_wcard = 1;
 			null_or_orig = orig_dev; /* deliver only exact match */
 		} else
 			skb->dev = master;
 	}
 	__this_cpu_inc(softnet_data.processed);
 	skb_reset_network_header(skb);
 	skb_reset_transport_header(skb);
 	skb->mac_len = skb->network_header - skb->mac_header;
 	pt_prev = NULL;
 	rcu_read_lock();
 #ifdef CONFIG_NET_CLS_ACT
 	if (skb->tc_verd & TC_NCLS) {
 		skb->tc_verd = CLR_TC_NCLS(skb->tc_verd);
 		goto ncls;
 	}
 #endif
 	list_for_each_entry_rcu(ptype, &ptype_all, list) {
 		if (ptype->dev == null_or_orig || ptype->dev == skb->dev ||
 		    ptype->dev == orig_dev) {
 			if (pt_prev)
 				ret = deliver_skb(skb, pt_prev, orig_dev);
 			pt_prev = ptype;
 		}
 	}
 #ifdef CONFIG_NET_CLS_ACT
 	skb = handle_ing(skb, &pt_prev, &ret, orig_dev);
 	if (!skb)
 		goto out;
 ncls:
 #endif
 	/* Handle special case of bridge or macvlan */
 	rx_handler = rcu_dereference(skb->dev->rx_handler);
 	if (rx_handler) {
 		if (pt_prev) {
 			ret = deliver_skb(skb, pt_prev, orig_dev);
 			pt_prev = NULL;
 		}
 		skb = rx_handler(skb);
 		if (!skb)
 			goto out;
 	}
 	/*
 	 * Make sure frames received on VLAN interfaces stacked on
 	 * bonding interfaces still make their way to any base bonding
 	 * device that may have registered for a specific ptype.  The
 	 * handler may have to adjust skb->dev and orig_dev.
 	 */
 	orig_or_bond = orig_dev;
 	if ((skb->dev->priv_flags & IFF_802_1Q_VLAN) &&
 	    (vlan_dev_real_dev(skb->dev)->priv_flags & IFF_BONDING)) {
 		orig_or_bond = vlan_dev_real_dev(skb->dev);
 	}
 	type = skb->protocol;
 	list_for_each_entry_rcu(ptype,
 			&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
 		if (ptype->type == type && (ptype->dev == null_or_orig ||
 		     ptype->dev == skb->dev || ptype->dev == orig_dev ||
 		     ptype->dev == orig_or_bond)) {
 			if (pt_prev)
 				ret = deliver_skb(skb, pt_prev, orig_dev);
 			pt_prev = ptype;
 		}
 	}
 	if (pt_prev) {
 		ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
 	} else {
 		atomic_long_inc(&skb->dev->rx_dropped);
 		kfree_skb(skb);
 		/* Jamal, now you will not able to escape explaining
 		 * me how you were going to use this. :-)
 		 */
 		ret = NET_RX_DROP;
 	}
 out:
 	rcu_read_unlock();
 	return ret;
 }
 /**
  *	netif_receive_skb - process receive buffer from network
  *	@skb: buffer to process
  *
  *	netif_receive_skb() is the main receive data processing function.
  *	It always succeeds. The buffer may be dropped during processing
  *	for congestion control or by the protocol layers.
  *
  *	This function may only be called from softirq context and interrupts
  *	should be enabled.
  *
  *	Return values (usually ignored):
  *	NET_RX_SUCCESS: no congestion
  *	NET_RX_DROP: packet was dropped
  */
 int netif_receive_skb(struct sk_buff *skb)
 {
 	if (netdev_tstamp_prequeue)
 		net_timestamp_check(skb);
 	if (skb_defer_rx_timestamp(skb))
 		return NET_RX_SUCCESS;
 #ifdef CONFIG_RPS
 	{
 		struct rps_dev_flow voidflow, *rflow = &voidflow;
 		int cpu, ret;
 		rcu_read_lock();
 		cpu = get_rps_cpu(skb->dev, skb, &rflow);
 		if (cpu >= 0) {
 			ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
 			rcu_read_unlock();
 		} else {
 			rcu_read_unlock();
 			ret = __netif_receive_skb(skb);
 		}
 		return ret;
 	}
 #else
 	return __netif_receive_skb(skb);
 #endif
 }
 EXPORT_SYMBOL(netif_receive_skb);
 /* Network device is going away, flush any packets still pending
  * Called with irqs disabled.
  */
 static void flush_backlog(void *arg)
 {
 	struct net_device *dev = arg;
 	struct softnet_data *sd = &__get_cpu_var(softnet_data);
 	struct sk_buff *skb, *tmp;
 	rps_lock(sd);
 	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
 		if (skb->dev == dev) {
 			__skb_unlink(skb, &sd->input_pkt_queue);
 			kfree_skb(skb);
 			input_queue_head_incr(sd);
 		}
 	}
 	rps_unlock(sd);
 	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
 		if (skb->dev == dev) {
 			__skb_unlink(skb, &sd->process_queue);
 			kfree_skb(skb);
 			input_queue_head_incr(sd);
 		}
 	}
 }
 static int napi_gro_complete(struct sk_buff *skb)
 {
 	struct packet_type *ptype;
 	__be16 type = skb->protocol;
 	struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
 	int err = -ENOENT;
 	if (NAPI_GRO_CB(skb)->count == 1) {
 		skb_shinfo(skb)->gso_size = 0;
 		goto out;
 	}
 	rcu_read_lock();
 	list_for_each_entry_rcu(ptype, head, list) {
 		if (ptype->type != type || ptype->dev || !ptype->gro_complete)
 			continue;
 		err = ptype->gro_complete(skb);
 		break;
 	}
 	rcu_read_unlock();
 	if (err) {
 		WARN_ON(&ptype->list == head);
 		kfree_skb(skb);
 		return NET_RX_SUCCESS;
 	}
 out:
 	return netif_receive_skb(skb);
 }
 inline void napi_gro_flush(struct napi_struct *napi)
 {
 	struct sk_buff *skb, *next;
 	for (skb = napi->gro_list; skb; skb = next) {
 		next = skb->next;
 		skb->next = NULL;
 		napi_gro_complete(skb);
 	}
 	napi->gro_count = 0;
 	napi->gro_list = NULL;
 }
 EXPORT_SYMBOL(napi_gro_flush);
 enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 {
 	struct sk_buff **pp = NULL;
 	struct packet_type *ptype;
 	__be16 type = skb->protocol;
 	struct list_head *head = &ptype_base[ntohs(type) & PTYPE_HASH_MASK];
 	int same_flow;
 	int mac_len;
 	enum gro_result ret;
 	if (!(skb->dev->features & NETIF_F_GRO) || netpoll_rx_on(skb))
 		goto normal;
 	if (skb_is_gso(skb) || skb_has_frag_list(skb))
 		goto normal;
 	rcu_read_lock();
 	list_for_each_entry_rcu(ptype, head, list) {
 		if (ptype->type != type || ptype->dev || !ptype->gro_receive)
 			continue;
 		skb_set_network_header(skb, skb_gro_offset(skb));
 		mac_len = skb->network_header - skb->mac_header;
 		skb->mac_len = mac_len;
 		NAPI_GRO_CB(skb)->same_flow = 0;
 		NAPI_GRO_CB(skb)->flush = 0;
 		NAPI_GRO_CB(skb)->free = 0;
 		pp = ptype->gro_receive(&napi->gro_list, skb);
 		break;
 	}
 	rcu_read_unlock();
 	if (&ptype->list == head)
 		goto normal;
 	same_flow = NAPI_GRO_CB(skb)->same_flow;
 	ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED;
 	if (pp) {
 		struct sk_buff *nskb = *pp;
 		*pp = nskb->next;
 		nskb->next = NULL;
 		napi_gro_complete(nskb);
 		napi->gro_count--;
 	}
 	if (same_flow)
 		goto ok;
 	if (NAPI_GRO_CB(skb)->flush || napi->gro_count >= MAX_GRO_SKBS)
 		goto normal;
 	napi->gro_count++;
 	NAPI_GRO_CB(skb)->count = 1;
 	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
 	skb->next = napi->gro_list;
 	napi->gro_list = skb;
 	ret = GRO_HELD;
 pull:
 	if (skb_headlen(skb) < skb_gro_offset(skb)) {
 		int grow = skb_gro_offset(skb) - skb_headlen(skb);
 		BUG_ON(skb->end - skb->tail < grow);
 		memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow);
 		skb->tail += grow;
 		skb->data_len -= grow;
 		skb_shinfo(skb)->frags[0].page_offset += grow;
 		skb_shinfo(skb)->frags[0].size -= grow;
 		if (unlikely(!skb_shinfo(skb)->frags[0].size)) {
 			put_page(skb_shinfo(skb)->frags[0].page);
 			memmove(skb_shinfo(skb)->frags,
 				skb_shinfo(skb)->frags + 1,
 				--skb_shinfo(skb)->nr_frags * sizeof(skb_frag_t));
 		}
 	}
 ok:
 	return ret;
 normal:
 	ret = GRO_NORMAL;
 	goto pull;
 }
 EXPORT_SYMBOL(dev_gro_receive);
 static inline gro_result_t
 __napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 {
 	struct sk_buff *p;
 	for (p = napi->gro_list; p; p = p->next) {
 		unsigned long diffs;
 		diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev;
 		diffs |= compare_ether_header(skb_mac_header(p),
 					      skb_gro_mac_header(skb));
 		NAPI_GRO_CB(p)->same_flow = !diffs;
 		NAPI_GRO_CB(p)->flush = 0;
 	}
 	return dev_gro_receive(napi, skb);
 }
 gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb)
 {
 	switch (ret) {
 	case GRO_NORMAL:
 		if (netif_receive_skb(skb))
 			ret = GRO_DROP;
 		break;
 	case GRO_DROP:
 	case GRO_MERGED_FREE:
 		kfree_skb(skb);
 		break;
 	case GRO_HELD:
 	case GRO_MERGED:
 		break;
 	}
 	return ret;
 }
 EXPORT_SYMBOL(napi_skb_finish);
 void skb_gro_reset_offset(struct sk_buff *skb)
 {
 	NAPI_GRO_CB(skb)->data_offset = 0;
 	NAPI_GRO_CB(skb)->frag0 = NULL;
 	NAPI_GRO_CB(skb)->frag0_len = 0;
 	if (skb->mac_header == skb->tail &&
 	    !PageHighMem(skb_shinfo(skb)->frags[0].page)) {
 		NAPI_GRO_CB(skb)->frag0 =
 			page_address(skb_shinfo(skb)->frags[0].page) +
 			skb_shinfo(skb)->frags[0].page_offset;
 		NAPI_GRO_CB(skb)->frag0_len = skb_shinfo(skb)->frags[0].size;
 	}
 }
 EXPORT_SYMBOL(skb_gro_reset_offset);
 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb)
 {
 	skb_gro_reset_offset(skb);
 	return napi_skb_finish(__napi_gro_receive(napi, skb), skb);
 }
 EXPORT_SYMBOL(napi_gro_receive);
 void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
 {
 	__skb_pull(skb, skb_headlen(skb));
 	skb_reserve(skb, NET_IP_ALIGN - skb_headroom(skb));
 	napi->skb = skb;
 }
 EXPORT_SYMBOL(napi_reuse_skb);
 struct sk_buff *napi_get_frags(struct napi_struct *napi)
 {
 	struct sk_buff *skb = napi->skb;
 	if (!skb) {
 		skb = netdev_alloc_skb_ip_align(napi->dev, GRO_MAX_HEAD);
 		if (skb)
 			napi->skb = skb;
 	}
 	return skb;
 }
 EXPORT_SYMBOL(napi_get_frags);
 gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb,
 			       gro_result_t ret)
 {
 	switch (ret) {
 	case GRO_NORMAL:
 	case GRO_HELD:
 		skb->protocol = eth_type_trans(skb, skb->dev);
 		if (ret == GRO_HELD)
 			skb_gro_pull(skb, -ETH_HLEN);
 		else if (netif_receive_skb(skb))
 			ret = GRO_DROP;
 		break;
 	case GRO_DROP:
 	case GRO_MERGED_FREE:
 		napi_reuse_skb(napi, skb);
 		break;
 	case GRO_MERGED:
 		break;
 	}
 	return ret;
 }
 EXPORT_SYMBOL(napi_frags_finish);
 struct sk_buff *napi_frags_skb(struct napi_struct *napi)
 {
 	struct sk_buff *skb = napi->skb;
 	struct ethhdr *eth;
 	unsigned int hlen;
 	unsigned int off;
 	napi->skb = NULL;
 	skb_reset_mac_header(skb);
 	skb_gro_reset_offset(skb);
 	off = skb_gro_offset(skb);
 	hlen = off + sizeof(*eth);
 	eth = skb_gro_header_fast(skb, off);
 	if (skb_gro_header_hard(skb, hlen)) {
 		eth = skb_gro_header_slow(skb, hlen, off);
 		if (unlikely(!eth)) {
 			napi_reuse_skb(napi, skb);
 			skb = NULL;
 			goto out;
 		}
 	}
 	skb_gro_pull(skb, sizeof(*eth));
 	/*
 	 * This works because the only protocols we care about don't require
 	 * special handling.  We'll fix it up properly at the end.
 	 */
 	skb->protocol = eth->h_proto;
 out:
 	return skb;
 }
 EXPORT_SYMBOL(napi_frags_skb);
 gro_result_t napi_gro_frags(struct napi_struct *napi)
 {
 	struct sk_buff *skb = napi_frags_skb(napi);
 	if (!skb)
 		return GRO_DROP;
 	return napi_frags_finish(napi, skb, __napi_gro_receive(napi, skb));
 }
 EXPORT_SYMBOL(napi_gro_frags);
 /*
  * net_rps_action sends any pending IPI's for rps.
  * Note: called with local irq disabled, but exits with local irq enabled.
  */
 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
 {
 #ifdef CONFIG_RPS
 	struct softnet_data *remsd = sd->rps_ipi_list;
 	if (remsd) {
 		sd->rps_ipi_list = NULL;
 		local_irq_enable();
 		/* Send pending IPI's to kick RPS processing on remote cpus. */
 		while (remsd) {
 			struct softnet_data *next = remsd->rps_ipi_next;
 			if (cpu_online(remsd->cpu))
 				__smp_call_function_single(remsd->cpu,
 							   &remsd->csd, 0);
 			remsd = next;
 		}
 	} else
 #endif
 		local_irq_enable();
 }
 static int process_backlog(struct napi_struct *napi, int quota)
 {
 	int work = 0;
 	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
 #ifdef CONFIG_RPS
 	/* Check if we have pending ipi, its better to send them now,
 	 * not waiting net_rx_action() end.
 	 */
 	if (sd->rps_ipi_list) {
 		local_irq_disable();
 		net_rps_action_and_irq_enable(sd);
 	}
 #endif
 	napi->weight = weight_p;
 	local_irq_disable();
 	while (work < quota) {
 		struct sk_buff *skb;
 		unsigned int qlen;
 		while ((skb = __skb_dequeue(&sd->process_queue))) {
 			local_irq_enable();
 			__netif_receive_skb(skb);
 			local_irq_disable();
 			input_queue_head_incr(sd);
 			if (++work >= quota) {
 				local_irq_enable();
 				return work;
 			}
 		}
 		rps_lock(sd);
 		qlen = skb_queue_len(&sd->input_pkt_queue);
 		if (qlen)
 			skb_queue_splice_tail_init(&sd->input_pkt_queue,
 						   &sd->process_queue);
 		if (qlen < quota - work) {
 			/*
 			 * Inline a custom version of __napi_complete().
 			 * only current cpu owns and manipulates this napi,
 			 * and NAPI_STATE_SCHED is the only possible flag set on backlog.
 			 * we can use a plain write instead of clear_bit(),
 			 * and we dont need an smp_mb() memory barrier.
 			 */
 			list_del(&napi->poll_list);
 			napi->state = 0;
 			quota = work + qlen;
 		}
 		rps_unlock(sd);
 	}
 	local_irq_enable();
 	return work;
 }
 /**
  * __napi_schedule - schedule for receive
  * @n: entry to schedule
  *
  * The entry's receive function will be scheduled to run
  */
 void __napi_schedule(struct napi_struct *n)
 {
 	unsigned long flags;
 	local_irq_save(flags);
 	____napi_schedule(&__get_cpu_var(softnet_data), n);
 	local_irq_restore(flags);
 }
 EXPORT_SYMBOL(__napi_schedule);
 void __napi_complete(struct napi_struct *n)
 {
 	BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state));
 	BUG_ON(n->gro_list);
 	list_del(&n->poll_list);
 	smp_mb__before_clear_bit();
 	clear_bit(NAPI_STATE_SCHED, &n->state);
 }
 EXPORT_SYMBOL(__napi_complete);
 void napi_complete(struct napi_struct *n)
 {
 	unsigned long flags;
 	/*
 	 * don't let napi dequeue from the cpu poll list
 	 * just in case its running on a different cpu
 	 */
 	if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state)))
 		return;
 	napi_gro_flush(n);
 	local_irq_save(flags);
 	__napi_complete(n);
 	local_irq_restore(flags);
 }
 EXPORT_SYMBOL(napi_complete);
 void netif_napi_add(struct net_device *dev, struct napi_struct *napi,
 		    int (*poll)(struct napi_struct *, int), int weight)
 {
 	INIT_LIST_HEAD(&napi->poll_list);
 	napi->gro_count = 0;
 	napi->gro_list = NULL;
 	napi->skb = NULL;
 	napi->poll = poll;
 	napi->weight = weight;
 	list_add(&napi->dev_list, &dev->napi_list);
 	napi->dev = dev;
 #ifdef CONFIG_NETPOLL
 	spin_lock_init(&napi->poll_lock);
 	napi->poll_owner = -1;
 #endif
 	set_bit(NAPI_STATE_SCHED, &napi->state);
 }
 EXPORT_SYMBOL(netif_napi_add);
 void netif_napi_del(struct napi_struct *napi)
 {
 	struct sk_buff *skb, *next;
 	list_del_init(&napi->dev_list);
 	napi_free_frags(napi);
 	for (skb = napi->gro_list; skb; skb = next) {
 		next = skb->next;
 		skb->next = NULL;
 		kfree_skb(skb);
 	}
 	napi->gro_list = NULL;
 	napi->gro_count = 0;
 }
 EXPORT_SYMBOL(netif_napi_del);
 static void net_rx_action(struct softirq_action *h)
 {
 	struct softnet_data *sd = &__get_cpu_var(softnet_data);
 	unsigned long time_limit = jiffies + 2;
 	int budget = netdev_budget;
 	void *have;
 	local_irq_disable();
 	while (!list_empty(&sd->poll_list)) {
 		struct napi_struct *n;
 		int work, weight;
 		/* If softirq window is exhuasted then punt.
 		 * Allow this to run for 2 jiffies since which will allow
 		 * an average latency of 1.5/HZ.
 		 */
 		if (unlikely(budget <= 0 || time_after(jiffies, time_limit)))
 			goto softnet_break;
 		local_irq_enable();
 		/* Even though interrupts have been re-enabled, this
 		 * access is safe because interrupts can only add new
 		 * entries to the tail of this list, and only ->poll()
 		 * calls can remove this head entry from the list.
 		 */
 		n = list_first_entry(&sd->poll_list, struct napi_struct, poll_list);
 		have = netpoll_poll_lock(n);
 		weight = n->weight;
 		/* This NAPI_STATE_SCHED test is for avoiding a race
 		 * with netpoll's poll_napi().  Only the entity which
 		 * obtains the lock and sees NAPI_STATE_SCHED set will
 		 * actually make the ->poll() call.  Therefore we avoid
 		 * accidently calling ->poll() when NAPI is not scheduled.
 		 */
 		work = 0;
 		if (test_bit(NAPI_STATE_SCHED, &n->state)) {
 			work = n->poll(n, weight);
 			trace_napi_poll(n);
 		}
 		WARN_ON_ONCE(work > weight);
 		budget -= work;
 		local_irq_disable();
 		/* Drivers must not modify the NAPI state if they
 		 * consume the entire weight.  In such cases this code
 		 * still "owns" the NAPI instance and therefore can
 		 * move the instance around on the list at-will.
 		 */
 		if (unlikely(work == weight)) {
 			if (unlikely(napi_disable_pending(n))) {
 				local_irq_enable();
 				napi_complete(n);
 				local_irq_disable();
 			} else
 				list_move_tail(&n->poll_list, &sd->poll_list);
 		}
 		netpoll_poll_unlock(have);
 	}
 out:
 	net_rps_action_and_irq_enable(sd);
 #ifdef CONFIG_NET_DMA
 	/*
 	 * There may not be any more sk_buffs coming right now, so push
 	 * any pending DMA copies to hardware
 	 */
 	dma_issue_pending_all();
 #endif
 	return;
 softnet_break:
 	sd->time_squeeze++;
 	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
 	goto out;
 }
 static gifconf_func_t *gifconf_list[NPROTO];
 /**
  *	register_gifconf	-	register a SIOCGIF handler
  *	@family: Address family
  *	@gifconf: Function handler
  *
  *	Register protocol dependent address dumping routines. The handler
  *	that is passed must not be freed or reused until it has been replaced
  *	by another handler.
  */
 int register_gifconf(unsigned int family, gifconf_func_t *gifconf)
 {
 	if (family >= NPROTO)
 		return -EINVAL;
 	gifconf_list[family] = gifconf;
 	return 0;
 }
 EXPORT_SYMBOL(register_gifconf);
 /*
  *	Map an interface index to its name (SIOCGIFNAME)
  */
 /*
  *	We need this ioctl for efficient implementation of the
  *	if_indextoname() function required by the IPv6 API.  Without
  *	it, we would have to search all the interfaces to find a
  *	match.  --pb
  */
 static int dev_ifname(struct net *net, struct ifreq __user *arg)
 {
 	struct net_device *dev;
 	struct ifreq ifr;
 	/*
 	 *	Fetch the caller's info block.
 	 */
 	if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
 		return -EFAULT;
 	rcu_read_lock();
 	dev = dev_get_by_index_rcu(net, ifr.ifr_ifindex);
 	if (!dev) {
 		rcu_read_unlock();
 		return -ENODEV;
 	}
 	strcpy(ifr.ifr_name, dev->name);
 	rcu_read_unlock();
 	if (copy_to_user(arg, &ifr, sizeof(struct ifreq)))
 		return -EFAULT;
 	return 0;
 }
 /*
  *	Perform a SIOCGIFCONF call. This structure will change
  *	size eventually, and there is nothing I can do about it.
  *	Thus we will need a 'compatibility mode'.
  */
 static int dev_ifconf(struct net *net, char __user *arg)
 {
 	struct ifconf ifc;
 	struct net_device *dev;
 	char __user *pos;
 	int len;
 	int total;
 	int i;
 	/*
 	 *	Fetch the caller's info block.
 	 */
 	if (copy_from_user(&ifc, arg, sizeof(struct ifconf)))
 		return -EFAULT;
 	pos = ifc.ifc_buf;
 	len = ifc.ifc_len;
 	/*
 	 *	Loop over the interfaces, and write an info block for each.
 	 */
 	total = 0;
 	for_each_netdev(net, dev) {
 		for (i = 0; i < NPROTO; i++) {
 			if (gifconf_list[i]) {
 				int done;
 				if (!pos)
 					done = gifconf_list[i](dev, NULL, 0);
 				else
 					done = gifconf_list[i](dev, pos + total,
 							       len - total);
 				if (done < 0)
 					return -EFAULT;
 				total += done;
 			}
 		}
 	}
 	/*
 	 *	All done.  Write the updated control block back to the caller.
 	 */
 	ifc.ifc_len = total;
 	/*
 	 * 	Both BSD and Solaris return 0 here, so we do too.
 	 */
 	return copy_to_user(arg, &ifc, sizeof(struct ifconf)) ? -EFAULT : 0;
 }
 #ifdef CONFIG_PROC_FS
 /*
  *	This is invoked by the /proc filesystem handler to display a device
  *	in detail.
  */
 void *dev_seq_start(struct seq_file *seq, loff_t *pos)
 	__acquires(RCU)
 {
 	struct net *net = seq_file_net(seq);
 	loff_t off;
 	struct net_device *dev;
 	rcu_read_lock();
 	if (!*pos)
 		return SEQ_START_TOKEN;
 	off = 1;
 	for_each_netdev_rcu(net, dev)
 		if (off++ == *pos)
 			return dev;
 	return NULL;
 }
 void *dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 {
 	struct net_device *dev = (v == SEQ_START_TOKEN) ?
 				  first_net_device(seq_file_net(seq)) :
 				  next_net_device((struct net_device *)v);
 	++*pos;
 	return rcu_dereference(dev);
 }
 void dev_seq_stop(struct seq_file *seq, void *v)
 	__releases(RCU)
 {
 	rcu_read_unlock();
 }
 static void dev_seq_printf_stats(struct seq_file *seq, struct net_device *dev)
 {
 	struct rtnl_link_stats64 temp;
 	const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
 	seq_printf(seq, "%6s: %7llu %7llu %4llu %4llu %4llu %5llu %10llu %9llu "
 		   "%8llu %7llu %4llu %4llu %4llu %5llu %7llu %10llu\n",
 		   dev->name, stats->rx_bytes, stats->rx_packets,
 		   stats->rx_errors,
 		   stats->rx_dropped + stats->rx_missed_errors,
 		   stats->rx_fifo_errors,
 		   stats->rx_length_errors + stats->rx_over_errors +
 		    stats->rx_crc_errors + stats->rx_frame_errors,
 		   stats->rx_compressed, stats->multicast,
 		   stats->tx_bytes, stats->tx_packets,
 		   stats->tx_errors, stats->tx_dropped,
 		   stats->tx_fifo_errors, stats->collisions,
 		   stats->tx_carrier_errors +
 		    stats->tx_aborted_errors +
 		    stats->tx_window_errors +
 		    stats->tx_heartbeat_errors,
 		   stats->tx_compressed);
 }
 /*
  *	Called from the PROCfs module. This now uses the new arbitrary sized
  *	/proc/net interface to create /proc/net/dev
  */
 static int dev_seq_show(struct seq_file *seq, void *v)
 {
 	if (v == SEQ_START_TOKEN)
 		seq_puts(seq, "Inter-|   Receive                            "
 			      "                    |  Transmit\n"
 			      " face |bytes    packets errs drop fifo frame "
 			      "compressed multicast|bytes    packets errs "
 			      "drop fifo colls carrier compressed\n");
 	else
 		dev_seq_printf_stats(seq, v);
 	return 0;
 }
 static struct softnet_data *softnet_get_online(loff_t *pos)
 {
 	struct softnet_data *sd = NULL;
 	while (*pos < nr_cpu_ids)
 		if (cpu_online(*pos)) {
 			sd = &per_cpu(softnet_data, *pos);
 			break;
 		} else
 			++*pos;
 	return sd;
 }
 static void *softnet_seq_start(struct seq_file *seq, loff_t *pos)
 {
 	return softnet_get_online(pos);
 }
 static void *softnet_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 {
 	++*pos;
 	return softnet_get_online(pos);
 }
 static void softnet_seq_stop(struct seq_file *seq, void *v)
 {
 }
 static int softnet_seq_show(struct seq_file *seq, void *v)
 {
 	struct softnet_data *sd = v;
 	seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x %08x %08x %08x\n",
 		   sd->processed, sd->dropped, sd->time_squeeze, 0,
 		   0, 0, 0, 0, /* was fastroute */
 		   sd->cpu_collision, sd->received_rps);
 	return 0;
 }
 static const struct seq_operations dev_seq_ops = {
 	.start = dev_seq_start,
 	.next  = dev_seq_next,
 	.stop  = dev_seq_stop,
 	.show  = dev_seq_show,
 };
 static int dev_seq_open(struct inode *inode, struct file *file)
 {
 	return seq_open_net(inode, file, &dev_seq_ops,
 			    sizeof(struct seq_net_private));
 }
 static const struct file_operations dev_seq_fops = {
 	.owner	 = THIS_MODULE,
 	.open    = dev_seq_open,
 	.read    = seq_read,
 	.llseek  = seq_lseek,
 	.release = seq_release_net,
 };
 static const struct seq_operations softnet_seq_ops = {
 	.start = softnet_seq_start,
 	.next  = softnet_seq_next,
 	.stop  = softnet_seq_stop,
 	.show  = softnet_seq_show,
 };
 static int softnet_seq_open(struct inode *inode, struct file *file)
 {
 	return seq_open(file, &softnet_seq_ops);
 }
 static const struct file_operations softnet_seq_fops = {
 	.owner	 = THIS_MODULE,
 	.open    = softnet_seq_open,
 	.read    = seq_read,
 	.llseek  = seq_lseek,
 	.release = seq_release,
 };
 static void *ptype_get_idx(loff_t pos)
 {
 	struct packet_type *pt = NULL;
 	loff_t i = 0;
 	int t;
 	list_for_each_entry_rcu(pt, &ptype_all, list) {
 		if (i == pos)
 			return pt;
 		++i;
 	}
 	for (t = 0; t < PTYPE_HASH_SIZE; t++) {
 		list_for_each_entry_rcu(pt, &ptype_base[t], list) {
 			if (i == pos)
 				return pt;
 			++i;
 		}
 	}
 	return NULL;
 }
 static void *ptype_seq_start(struct seq_file *seq, loff_t *pos)
 	__acquires(RCU)
 {
 	rcu_read_lock();
 	return *pos ? ptype_get_idx(*pos - 1) : SEQ_START_TOKEN;
 }
 static void *ptype_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 {
 	struct packet_type *pt;
 	struct list_head *nxt;
 	int hash;
 	++*pos;
 	if (v == SEQ_START_TOKEN)
 		return ptype_get_idx(0);
 	pt = v;
 	nxt = pt->list.next;
 	if (pt->type == htons(ETH_P_ALL)) {
 		if (nxt != &ptype_all)
 			goto found;
 		hash = 0;
 		nxt = ptype_base[0].next;
 	} else
 		hash = ntohs(pt->type) & PTYPE_HASH_MASK;
 	while (nxt == &ptype_base[hash]) {
 		if (++hash >= PTYPE_HASH_SIZE)
 			return NULL;
 		nxt = ptype_base[hash].next;
 	}
 found:
 	return list_entry(nxt, struct packet_type, list);
 }
 static void ptype_seq_stop(struct seq_file *seq, void *v)
 	__releases(RCU)
 {
 	rcu_read_unlock();
 }
 static int ptype_seq_show(struct seq_file *seq, void *v)
 {
 	struct packet_type *pt = v;
 	if (v == SEQ_START_TOKEN)
 		seq_puts(seq, "Type Device      Function\n");
 	else if (pt->dev == NULL || dev_net(pt->dev) == seq_file_net(seq)) {
 		if (pt->type == htons(ETH_P_ALL))
 			seq_puts(seq, "ALL ");
 		else
 			seq_printf(seq, "%04x", ntohs(pt->type));
 		seq_printf(seq, " %-8s %pF\n",
 			   pt->dev ? pt->dev->name : "", pt->func);
 	}
 	return 0;
 }
 static const struct seq_operations ptype_seq_ops = {
 	.start = ptype_seq_start,
 	.next  = ptype_seq_next,
 	.stop  = ptype_seq_stop,
 	.show  = ptype_seq_show,
 };
 static int ptype_seq_open(struct inode *inode, struct file *file)
 {
 	return seq_open_net(inode, file, &ptype_seq_ops,
 			sizeof(struct seq_net_private));
 }
 static const struct file_operations ptype_seq_fops = {
 	.owner	 = THIS_MODULE,
 	.open    = ptype_seq_open,
 	.read    = seq_read,
 	.llseek  = seq_lseek,
 	.release = seq_release_net,
 };
 static int __net_init dev_proc_net_init(struct net *net)
 {
 	int rc = -ENOMEM;
 	if (!proc_net_fops_create(net, "dev", S_IRUGO, &dev_seq_fops))
 		goto out;
 	if (!proc_net_fops_create(net, "softnet_stat", S_IRUGO, &softnet_seq_fops))
 		goto out_dev;
 	if (!proc_net_fops_create(net, "ptype", S_IRUGO, &ptype_seq_fops))
 		goto out_softnet;
 	if (wext_proc_init(net))
 		goto out_ptype;
 	rc = 0;
 out:
 	return rc;
 out_ptype:
 	proc_net_remove(net, "ptype");
 out_softnet:
 	proc_net_remove(net, "softnet_stat");
 out_dev:
 	proc_net_remove(net, "dev");
 	goto out;
 }
 static void __net_exit dev_proc_net_exit(struct net *net)
 {
 	wext_proc_exit(net);
 	proc_net_remove(net, "ptype");
 	proc_net_remove(net, "softnet_stat");
 	proc_net_remove(net, "dev");
 }
 static struct pernet_operations __net_initdata dev_proc_ops = {
 	.init = dev_proc_net_init,
 	.exit = dev_proc_net_exit,
 };
 static int __init dev_proc_init(void)
 {
 	return register_pernet_subsys(&dev_proc_ops);
 }
 #else
 #define dev_proc_init() 0
 #endif	/* CONFIG_PROC_FS */
 /**
  *	netdev_set_master	-	set up master/slave pair
  *	@slave: slave device
  *	@master: new master device
  *
  *	Changes the master device of the slave. Pass %NULL to break the
  *	bonding. The caller must hold the RTNL semaphore. On a failure
  *	a negative errno code is returned. On success the reference counts
  *	are adjusted, %RTM_NEWLINK is sent to the routing socket and the
  *	function returns zero.
  */
 int netdev_set_master(struct net_device *slave, struct net_device *master)
 {
 	struct net_device *old = slave->master;
 	ASSERT_RTNL();
 	if (master) {
 		if (old)
 			return -EBUSY;
 		dev_hold(master);
 	}
 	slave->master = master;
 	if (old) {
 		synchronize_net();
 		dev_put(old);
 	}
 	if (master)
 		slave->flags |= IFF_SLAVE;
 	else
 		slave->flags &= ~IFF_SLAVE;
 	rtmsg_ifinfo(RTM_NEWLINK, slave, IFF_SLAVE);
 	return 0;
 }
 EXPORT_SYMBOL(netdev_set_master);
 static void dev_change_rx_flags(struct net_device *dev, int flags)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	if ((dev->flags & IFF_UP) && ops->ndo_change_rx_flags)
 		ops->ndo_change_rx_flags(dev, flags);
 }
 static int __dev_set_promiscuity(struct net_device *dev, int inc)
 {
 	unsigned short old_flags = dev->flags;
 	uid_t uid;
 	gid_t gid;
 	ASSERT_RTNL();
 	dev->flags |= IFF_PROMISC;
 	dev->promiscuity += inc;
 	if (dev->promiscuity == 0) {
 		/*
 		 * Avoid overflow.
 		 * If inc causes overflow, untouch promisc and return error.
 		 */
 		if (inc < 0)
 			dev->flags &= ~IFF_PROMISC;
 		else {
 			dev->promiscuity -= inc;
 			printk(KERN_WARNING "%s: promiscuity touches roof, "
 				"set promiscuity failed, promiscuity feature "
 				"of device might be broken.\n", dev->name);
 			return -EOVERFLOW;
 		}
 	}
 	if (dev->flags != old_flags) {
 		printk(KERN_INFO "device %s %s promiscuous mode\n",
 		       dev->name, (dev->flags & IFF_PROMISC) ? "entered" :
 							       "left");
 		if (audit_enabled) {
 			current_uid_gid(&uid, &gid);
 			audit_log(current->audit_context, GFP_ATOMIC,
 				AUDIT_ANOM_PROMISCUOUS,
 				"dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
 				dev->name, (dev->flags & IFF_PROMISC),
 				(old_flags & IFF_PROMISC),
 				audit_get_loginuid(current),
 				uid, gid,
 				audit_get_sessionid(current));
 		}
 		dev_change_rx_flags(dev, IFF_PROMISC);
 	}
 	return 0;
 }
 /**
  *	dev_set_promiscuity	- update promiscuity count on a device
  *	@dev: device
  *	@inc: modifier
  *
  *	Add or remove promiscuity from a device. While the count in the device
  *	remains above zero the interface remains promiscuous. Once it hits zero
  *	the device reverts back to normal filtering operation. A negative inc
  *	value is used to drop promiscuity on the device.
  *	Return 0 if successful or a negative errno code on error.
  */
 int dev_set_promiscuity(struct net_device *dev, int inc)
 {
 	unsigned short old_flags = dev->flags;
 	int err;
 	err = __dev_set_promiscuity(dev, inc);
 	if (err < 0)
 		return err;
 	if (dev->flags != old_flags)
 		dev_set_rx_mode(dev);
 	return err;
 }
 EXPORT_SYMBOL(dev_set_promiscuity);
 /**
  *	dev_set_allmulti	- update allmulti count on a device
  *	@dev: device
  *	@inc: modifier
  *
  *	Add or remove reception of all multicast frames to a device. While the
  *	count in the device remains above zero the interface remains listening
  *	to all interfaces. Once it hits zero the device reverts back to normal
  *	filtering operation. A negative @inc value is used to drop the counter
  *	when releasing a resource needing all multicasts.
  *	Return 0 if successful or a negative errno code on error.
  */
 int dev_set_allmulti(struct net_device *dev, int inc)
 {
 	unsigned short old_flags = dev->flags;
 	ASSERT_RTNL();
 	dev->flags |= IFF_ALLMULTI;
 	dev->allmulti += inc;
 	if (dev->allmulti == 0) {
 		/*
 		 * Avoid overflow.
 		 * If inc causes overflow, untouch allmulti and return error.
 		 */
 		if (inc < 0)
 			dev->flags &= ~IFF_ALLMULTI;
 		else {
 			dev->allmulti -= inc;
 			printk(KERN_WARNING "%s: allmulti touches roof, "
 				"set allmulti failed, allmulti feature of "
 				"device might be broken.\n", dev->name);
 			return -EOVERFLOW;
 		}
 	}
 	if (dev->flags ^ old_flags) {
 		dev_change_rx_flags(dev, IFF_ALLMULTI);
 		dev_set_rx_mode(dev);
 	}
 	return 0;
 }
 EXPORT_SYMBOL(dev_set_allmulti);
 /*
  *	Upload unicast and multicast address lists to device and
  *	configure RX filtering. When the device doesn't support unicast
  *	filtering it is put in promiscuous mode while unicast addresses
  *	are present.
  */
 void __dev_set_rx_mode(struct net_device *dev)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	/* dev_open will call this function so the list will stay sane. */
 	if (!(dev->flags&IFF_UP))
 		return;
 	if (!netif_device_present(dev))
 		return;
 	if (ops->ndo_set_rx_mode)
 		ops->ndo_set_rx_mode(dev);
 	else {
 		/* Unicast addresses changes may only happen under the rtnl,
 		 * therefore calling __dev_set_promiscuity here is safe.
 		 */
 		if (!netdev_uc_empty(dev) && !dev->uc_promisc) {
 			__dev_set_promiscuity(dev, 1);
 			dev->uc_promisc = 1;
 		} else if (netdev_uc_empty(dev) && dev->uc_promisc) {
 			__dev_set_promiscuity(dev, -1);
 			dev->uc_promisc = 0;
 		}
 		if (ops->ndo_set_multicast_list)
 			ops->ndo_set_multicast_list(dev);
 	}
 }
 void dev_set_rx_mode(struct net_device *dev)
 {
 	netif_addr_lock_bh(dev);
 	__dev_set_rx_mode(dev);
 	netif_addr_unlock_bh(dev);
 }
 /**
  *	dev_get_flags - get flags reported to userspace
  *	@dev: device
  *
  *	Get the combination of flag bits exported through APIs to userspace.
  */
 unsigned dev_get_flags(const struct net_device *dev)
 {
 	unsigned flags;
 	flags = (dev->flags & ~(IFF_PROMISC |
 				IFF_ALLMULTI |
 				IFF_RUNNING |
 				IFF_LOWER_UP |
 				IFF_DORMANT)) |
 		(dev->gflags & (IFF_PROMISC |
 				IFF_ALLMULTI));
 	if (netif_running(dev)) {
 		if (netif_oper_up(dev))
 			flags |= IFF_RUNNING;
 		if (netif_carrier_ok(dev))
 			flags |= IFF_LOWER_UP;
 		if (netif_dormant(dev))
 			flags |= IFF_DORMANT;
 	}
 	return flags;
 }
 EXPORT_SYMBOL(dev_get_flags);
 int __dev_change_flags(struct net_device *dev, unsigned int flags)
 {
 	int old_flags = dev->flags;
 	int ret;
 	ASSERT_RTNL();
 	/*
 	 *	Set the flags on our device.
 	 */
 	dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
 			       IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
 			       IFF_AUTOMEDIA)) |
 		     (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
 				    IFF_ALLMULTI));
 	/*
 	 *	Load in the correct multicast list now the flags have changed.
 	 */
 	if ((old_flags ^ flags) & IFF_MULTICAST)
 		dev_change_rx_flags(dev, IFF_MULTICAST);
 	dev_set_rx_mode(dev);
 	/*
 	 *	Have we downed the interface. We handle IFF_UP ourselves
 	 *	according to user attempts to set it, rather than blindly
 	 *	setting it.
 	 */
 	ret = 0;
 	if ((old_flags ^ flags) & IFF_UP) {	/* Bit is different  ? */
 		ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev);
 		if (!ret)
 			dev_set_rx_mode(dev);
 	}
 	if ((flags ^ dev->gflags) & IFF_PROMISC) {
 		int inc = (flags & IFF_PROMISC) ? 1 : -1;
 		dev->gflags ^= IFF_PROMISC;
 		dev_set_promiscuity(dev, inc);
 	}
 	/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
 	   is important. Some (broken) drivers set IFF_PROMISC, when
 	   IFF_ALLMULTI is requested not asking us and not reporting.
 	 */
 	if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
 		int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
 		dev->gflags ^= IFF_ALLMULTI;
 		dev_set_allmulti(dev, inc);
 	}
 	return ret;
 }
 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags)
 {
 	unsigned int changes = dev->flags ^ old_flags;
 	if (changes & IFF_UP) {
 		if (dev->flags & IFF_UP)
 			call_netdevice_notifiers(NETDEV_UP, dev);
 		else
 			call_netdevice_notifiers(NETDEV_DOWN, dev);
 	}
 	if (dev->flags & IFF_UP &&
 	    (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE)))
 		call_netdevice_notifiers(NETDEV_CHANGE, dev);
 }
 /**
  *	dev_change_flags - change device settings
  *	@dev: device
  *	@flags: device state flags
  *
  *	Change settings on device based state flags. The flags are
  *	in the userspace exported format.
  */
 int dev_change_flags(struct net_device *dev, unsigned flags)
 {
 	int ret, changes;
 	int old_flags = dev->flags;
 	ret = __dev_change_flags(dev, flags);
 	if (ret < 0)
 		return ret;
 	changes = old_flags ^ dev->flags;
 	if (changes)
 		rtmsg_ifinfo(RTM_NEWLINK, dev, changes);
 	__dev_notify_flags(dev, old_flags);
 	return ret;
 }
 EXPORT_SYMBOL(dev_change_flags);
 /**
  *	dev_set_mtu - Change maximum transfer unit
  *	@dev: device
  *	@new_mtu: new transfer unit
  *
  *	Change the maximum transfer size of the network device.
  */
 int dev_set_mtu(struct net_device *dev, int new_mtu)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	int err;
 	if (new_mtu == dev->mtu)
 		return 0;
 	/*	MTU must be positive.	 */
 	if (new_mtu < 0)
 		return -EINVAL;
 	if (!netif_device_present(dev))
 		return -ENODEV;
 	err = 0;
 	if (ops->ndo_change_mtu)
 		err = ops->ndo_change_mtu(dev, new_mtu);
 	else
 		dev->mtu = new_mtu;
 	if (!err && dev->flags & IFF_UP)
 		call_netdevice_notifiers(NETDEV_CHANGEMTU, dev);
 	return err;
 }
 EXPORT_SYMBOL(dev_set_mtu);
 /**
  *	dev_set_mac_address - Change Media Access Control Address
  *	@dev: device
  *	@sa: new address
  *
  *	Change the hardware (MAC) address of the device
  */
 int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	int err;
 	if (!ops->ndo_set_mac_address)
 		return -EOPNOTSUPP;
 	if (sa->sa_family != dev->type)
 		return -EINVAL;
 	if (!netif_device_present(dev))
 		return -ENODEV;
 	err = ops->ndo_set_mac_address(dev, sa);
 	if (!err)
 		call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
 	return err;
 }
 EXPORT_SYMBOL(dev_set_mac_address);
 /*
  *	Perform the SIOCxIFxxx calls, inside rcu_read_lock()
  */
 static int dev_ifsioc_locked(struct net *net, struct ifreq *ifr, unsigned int cmd)
 {
 	int err;
 	struct net_device *dev = dev_get_by_name_rcu(net, ifr->ifr_name);
 	if (!dev)
 		return -ENODEV;
 	switch (cmd) {
 	case SIOCGIFFLAGS:	/* Get interface flags */
 		ifr->ifr_flags = (short) dev_get_flags(dev);
 		return 0;
 	case SIOCGIFMETRIC:	/* Get the metric on the interface
 				   (currently unused) */
 		ifr->ifr_metric = 0;
 		return 0;
 	case SIOCGIFMTU:	/* Get the MTU of a device */
 		ifr->ifr_mtu = dev->mtu;
 		return 0;
 	case SIOCGIFHWADDR:
 		if (!dev->addr_len)
 			memset(ifr->ifr_hwaddr.sa_data, 0, sizeof ifr->ifr_hwaddr.sa_data);
 		else
 			memcpy(ifr->ifr_hwaddr.sa_data, dev->dev_addr,
 			       min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
 		ifr->ifr_hwaddr.sa_family = dev->type;
 		return 0;
 	case SIOCGIFSLAVE:
 		err = -EINVAL;
 		break;
 	case SIOCGIFMAP:
 		ifr->ifr_map.mem_start = dev->mem_start;
 		ifr->ifr_map.mem_end   = dev->mem_end;
 		ifr->ifr_map.base_addr = dev->base_addr;
 		ifr->ifr_map.irq       = dev->irq;
 		ifr->ifr_map.dma       = dev->dma;
 		ifr->ifr_map.port      = dev->if_port;
 		return 0;
 	case SIOCGIFINDEX:
 		ifr->ifr_ifindex = dev->ifindex;
 		return 0;
 	case SIOCGIFTXQLEN:
 		ifr->ifr_qlen = dev->tx_queue_len;
 		return 0;
 	default:
 		/* dev_ioctl() should ensure this case
 		 * is never reached
 		 */
 		WARN_ON(1);
 		err = -EINVAL;
 		break;
 	}
 	return err;
 }
 /*
  *	Perform the SIOCxIFxxx calls, inside rtnl_lock()
  */
 static int dev_ifsioc(struct net *net, struct ifreq *ifr, unsigned int cmd)
 {
 	int err;
 	struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name);
 	const struct net_device_ops *ops;
 	if (!dev)
 		return -ENODEV;
 	ops = dev->netdev_ops;
 	switch (cmd) {
 	case SIOCSIFFLAGS:	/* Set interface flags */
 		return dev_change_flags(dev, ifr->ifr_flags);
 	case SIOCSIFMETRIC:	/* Set the metric on the interface
 				   (currently unused) */
 		return -EOPNOTSUPP;
 	case SIOCSIFMTU:	/* Set the MTU of a device */
 		return dev_set_mtu(dev, ifr->ifr_mtu);
 	case SIOCSIFHWADDR:
 		return dev_set_mac_address(dev, &ifr->ifr_hwaddr);
 	case SIOCSIFHWBROADCAST:
 		if (ifr->ifr_hwaddr.sa_family != dev->type)
 			return -EINVAL;
 		memcpy(dev->broadcast, ifr->ifr_hwaddr.sa_data,
 		       min(sizeof ifr->ifr_hwaddr.sa_data, (size_t) dev->addr_len));
 		call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
 		return 0;
 	case SIOCSIFMAP:
 		if (ops->ndo_set_config) {
 			if (!netif_device_present(dev))
 				return -ENODEV;
 			return ops->ndo_set_config(dev, &ifr->ifr_map);
 		}
 		return -EOPNOTSUPP;
 	case SIOCADDMULTI:
 		if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
 		    ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
 			return -EINVAL;
 		if (!netif_device_present(dev))
 			return -ENODEV;
 		return dev_mc_add_global(dev, ifr->ifr_hwaddr.sa_data);
 	case SIOCDELMULTI:
 		if ((!ops->ndo_set_multicast_list && !ops->ndo_set_rx_mode) ||
 		    ifr->ifr_hwaddr.sa_family != AF_UNSPEC)
 			return -EINVAL;
 		if (!netif_device_present(dev))
 			return -ENODEV;
 		return dev_mc_del_global(dev, ifr->ifr_hwaddr.sa_data);
 	case SIOCSIFTXQLEN:
 		if (ifr->ifr_qlen < 0)
 			return -EINVAL;
 		dev->tx_queue_len = ifr->ifr_qlen;
 		return 0;
 	case SIOCSIFNAME:
 		ifr->ifr_newname[IFNAMSIZ-1] = '\0';
 		return dev_change_name(dev, ifr->ifr_newname);
 	/*
 	 *	Unknown or private ioctl
 	 */
 	default:
 		if ((cmd >= SIOCDEVPRIVATE &&
 		    cmd <= SIOCDEVPRIVATE + 15) ||
 		    cmd == SIOCBONDENSLAVE ||
 		    cmd == SIOCBONDRELEASE ||
 		    cmd == SIOCBONDSETHWADDR ||
 		    cmd == SIOCBONDSLAVEINFOQUERY ||
 		    cmd == SIOCBONDINFOQUERY ||
 		    cmd == SIOCBONDCHANGEACTIVE ||
 		    cmd == SIOCGMIIPHY ||
 		    cmd == SIOCGMIIREG ||
 		    cmd == SIOCSMIIREG ||
 		    cmd == SIOCBRADDIF ||
 		    cmd == SIOCBRDELIF ||
 		    cmd == SIOCSHWTSTAMP ||
 		    cmd == SIOCWANDEV) {
 			err = -EOPNOTSUPP;
 			if (ops->ndo_do_ioctl) {
 				if (netif_device_present(dev))
 					err = ops->ndo_do_ioctl(dev, ifr, cmd);
 				else
 					err = -ENODEV;
 			}
 		} else
 			err = -EINVAL;
 	}
 	return err;
 }
 /*
  *	This function handles all "interface"-type I/O control requests. The actual
  *	'doing' part of this is dev_ifsioc above.
  */
 /**
  *	dev_ioctl	-	network device ioctl
  *	@net: the applicable net namespace
  *	@cmd: command to issue
  *	@arg: pointer to a struct ifreq in user space
  *
  *	Issue ioctl functions to devices. This is normally called by the
  *	user space syscall interfaces but can sometimes be useful for
  *	other purposes. The return value is the return from the syscall if
  *	positive or a negative errno code on error.
  */
 int dev_ioctl(struct net *net, unsigned int cmd, void __user *arg)
 {
 	struct ifreq ifr;
 	int ret;
 	char *colon;
 	/* One special case: SIOCGIFCONF takes ifconf argument
 	   and requires shared lock, because it sleeps writing
 	   to user space.
 	 */
 	if (cmd == SIOCGIFCONF) {
 		rtnl_lock();
 		ret = dev_ifconf(net, (char __user *) arg);
 		rtnl_unlock();
 		return ret;
 	}
 	if (cmd == SIOCGIFNAME)
 		return dev_ifname(net, (struct ifreq __user *)arg);
 	if (copy_from_user(&ifr, arg, sizeof(struct ifreq)))
 		return -EFAULT;
 	ifr.ifr_name[IFNAMSIZ-1] = 0;
 	colon = strchr(ifr.ifr_name, ':');
 	if (colon)
 		*colon = 0;
 	/*
 	 *	See which interface the caller is talking about.
 	 */
 	switch (cmd) {
 	/*
 	 *	These ioctl calls:
 	 *	- can be done by all.
 	 *	- atomic and do not require locking.
 	 *	- return a value
 	 */
 	case SIOCGIFFLAGS:
 	case SIOCGIFMETRIC:
 	case SIOCGIFMTU:
 	case SIOCGIFHWADDR:
 	case SIOCGIFSLAVE:
 	case SIOCGIFMAP:
 	case SIOCGIFINDEX:
 	case SIOCGIFTXQLEN:
 		dev_load(net, ifr.ifr_name);
 		rcu_read_lock();
 		ret = dev_ifsioc_locked(net, &ifr, cmd);
 		rcu_read_unlock();
 		if (!ret) {
 			if (colon)
 				*colon = ':';
 			if (copy_to_user(arg, &ifr,
 					 sizeof(struct ifreq)))
 				ret = -EFAULT;
 		}
 		return ret;
 	case SIOCETHTOOL:
 		dev_load(net, ifr.ifr_name);
 		rtnl_lock();
 		ret = dev_ethtool(net, &ifr);
 		rtnl_unlock();
 		if (!ret) {
 			if (colon)
 				*colon = ':';
 			if (copy_to_user(arg, &ifr,
 					 sizeof(struct ifreq)))
 				ret = -EFAULT;
 		}
 		return ret;
 	/*
 	 *	These ioctl calls:
 	 *	- require superuser power.
 	 *	- require strict serialization.
 	 *	- return a value
 	 */
 	case SIOCGMIIPHY:
 	case SIOCGMIIREG:
 	case SIOCSIFNAME:
 		if (!capable(CAP_NET_ADMIN))
 			return -EPERM;
 		dev_load(net, ifr.ifr_name);
 		rtnl_lock();
 		ret = dev_ifsioc(net, &ifr, cmd);
 		rtnl_unlock();
 		if (!ret) {
 			if (colon)
 				*colon = ':';
 			if (copy_to_user(arg, &ifr,
 					 sizeof(struct ifreq)))
 				ret = -EFAULT;
 		}
 		return ret;
 	/*
 	 *	These ioctl calls:
 	 *	- require superuser power.
 	 *	- require strict serialization.
 	 *	- do not return a value
 	 */
 	case SIOCSIFFLAGS:
 	case SIOCSIFMETRIC:
 	case SIOCSIFMTU:
 	case SIOCSIFMAP:
 	case SIOCSIFHWADDR:
 	case SIOCSIFSLAVE:
 	case SIOCADDMULTI:
 	case SIOCDELMULTI:
 	case SIOCSIFHWBROADCAST:
 	case SIOCSIFTXQLEN:
 	case SIOCSMIIREG:
 	case SIOCBONDENSLAVE:
 	case SIOCBONDRELEASE:
 	case SIOCBONDSETHWADDR:
 	case SIOCBONDCHANGEACTIVE:
 	case SIOCBRADDIF:
 	case SIOCBRDELIF:
 	case SIOCSHWTSTAMP:
 		if (!capable(CAP_NET_ADMIN))
 			return -EPERM;
 		/* fall through */
 	case SIOCBONDSLAVEINFOQUERY:
 	case SIOCBONDINFOQUERY:
 		dev_load(net, ifr.ifr_name);
 		rtnl_lock();
 		ret = dev_ifsioc(net, &ifr, cmd);
 		rtnl_unlock();
 		return ret;
 	case SIOCGIFMEM:
 		/* Get the per device memory space. We can add this but
 		 * currently do not support it */
 	case SIOCSIFMEM:
 		/* Set the per device memory buffer space.
 		 * Not applicable in our case */
 	case SIOCSIFLINK:
 		return -EINVAL;
 	/*
 	 *	Unknown or private ioctl.
 	 */
 	default:
 		if (cmd == SIOCWANDEV ||
 		    (cmd >= SIOCDEVPRIVATE &&
 		     cmd <= SIOCDEVPRIVATE + 15)) {
 			dev_load(net, ifr.ifr_name);
 			rtnl_lock();
 			ret = dev_ifsioc(net, &ifr, cmd);
 			rtnl_unlock();
 			if (!ret && copy_to_user(arg, &ifr,
 						 sizeof(struct ifreq)))
 				ret = -EFAULT;
 			return ret;
 		}
 		/* Take care of Wireless Extensions */
 		if (cmd >= SIOCIWFIRST && cmd <= SIOCIWLAST)
 			return wext_handle_ioctl(net, &ifr, cmd, arg);
 		return -EINVAL;
 	}
 }
 /**
  *	dev_new_index	-	allocate an ifindex
  *	@net: the applicable net namespace
  *
  *	Returns a suitable unique value for a new device interface
  *	number.  The caller must hold the rtnl semaphore or the
  *	dev_base_lock to be sure it remains unique.
  */
 static int dev_new_index(struct net *net)
 {
 	static int ifindex;
 	for (;;) {
 		if (++ifindex <= 0)
 			ifindex = 1;
 		if (!__dev_get_by_index(net, ifindex))
 			return ifindex;
 	}
 }
 /* Delayed registration/unregisteration */
 static LIST_HEAD(net_todo_list);
 static void net_set_todo(struct net_device *dev)
 {
 	list_add_tail(&dev->todo_list, &net_todo_list);
 }
 static void rollback_registered_many(struct list_head *head)
 {
 	struct net_device *dev, *tmp;
 	BUG_ON(dev_boot_phase);
 	ASSERT_RTNL();
 	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
 		/* Some devices call without registering
 		 * for initialization unwind. Remove those
 		 * devices and proceed with the remaining.
 		 */
 		if (dev->reg_state == NETREG_UNINITIALIZED) {
 			pr_debug("unregister_netdevice: device %s/%p never "
 				 "was registered\n", dev->name, dev);
 			WARN_ON(1);
 			list_del(&dev->unreg_list);
 			continue;
 		}
 		BUG_ON(dev->reg_state != NETREG_REGISTERED);
 		/* If device is running, close it first. */
 		dev_close(dev);
 		/* And unlink it from device chain. */
 		unlist_netdevice(dev);
 		dev->reg_state = NETREG_UNREGISTERING;
 	}
 	synchronize_net();
 	list_for_each_entry(dev, head, unreg_list) {
 		/* Shutdown queueing discipline. */
 		dev_shutdown(dev);
 		/* Notify protocols, that we are about to destroy
 		   this device. They should clean all the things.
 		*/
 		call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
 		if (!dev->rtnl_link_ops ||
 		    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
 			rtmsg_ifinfo(RTM_DELLINK, dev, ~0U);
 		/*
 		 *	Flush the unicast and multicast chains
 		 */
 		dev_uc_flush(dev);
 		dev_mc_flush(dev);
 		if (dev->netdev_ops->ndo_uninit)
 			dev->netdev_ops->ndo_uninit(dev);
 		/* Notifier chain MUST detach us from master device. */
 		WARN_ON(dev->master);
 		/* Remove entries from kobject tree */
 		netdev_unregister_kobject(dev);
 	}
 	/* Process any work delayed until the end of the batch */
 	dev = list_first_entry(head, struct net_device, unreg_list);
 	call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
 	rcu_barrier();
 	list_for_each_entry(dev, head, unreg_list)
 		dev_put(dev);
 }
 static void rollback_registered(struct net_device *dev)
 {
 	LIST_HEAD(single);
 	list_add(&dev->unreg_list, &single);
 	rollback_registered_many(&single);
 }
 static void __netdev_init_queue_locks_one(struct net_device *dev,
 					  struct netdev_queue *dev_queue,
 					  void *_unused)
 {
 	spin_lock_init(&dev_queue->_xmit_lock);
 	netdev_set_xmit_lockdep_class(&dev_queue->_xmit_lock, dev->type);
 	dev_queue->xmit_lock_owner = -1;
 }
 static void netdev_init_queue_locks(struct net_device *dev)
 {
 	netdev_for_each_tx_queue(dev, __netdev_init_queue_locks_one, NULL);
 }
 unsigned long netdev_fix_features(unsigned long features, const char *name)
 {
 	/* Fix illegal SG+CSUM combinations. */
 	if ((features & NETIF_F_SG) &&
 	    !(features & NETIF_F_ALL_CSUM)) {
 		if (name)
 			printk(KERN_NOTICE "%s: Dropping NETIF_F_SG since no "
 			       "checksum feature.\n", name);
 		features &= ~NETIF_F_SG;
 	}
 	/* TSO requires that SG is present as well. */
 	if ((features & NETIF_F_TSO) && !(features & NETIF_F_SG)) {
 		if (name)
 			printk(KERN_NOTICE "%s: Dropping NETIF_F_TSO since no "
 			       "SG feature.\n", name);
 		features &= ~NETIF_F_TSO;
 	}
 	if (features & NETIF_F_UFO) {
 		if (!(features & NETIF_F_GEN_CSUM)) {
 			if (name)
 				printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
 				       "since no NETIF_F_HW_CSUM feature.\n",
 				       name);
 			features &= ~NETIF_F_UFO;
 		}
 		if (!(features & NETIF_F_SG)) {
 			if (name)
 				printk(KERN_ERR "%s: Dropping NETIF_F_UFO "
 				       "since no NETIF_F_SG feature.\n", name);
 			features &= ~NETIF_F_UFO;
 		}
 	}
 	return features;
 }
 EXPORT_SYMBOL(netdev_fix_features);
 /**
  *	netif_stacked_transfer_operstate -	transfer operstate
  *	@rootdev: the root or lower level device to transfer state from
  *	@dev: the device to transfer operstate to
  *
  *	Transfer operational state from root to device. This is normally
  *	called when a stacking relationship exists between the root
  *	device and the device(a leaf device).
  */
 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
 					struct net_device *dev)
 {
 	if (rootdev->operstate == IF_OPER_DORMANT)
 		netif_dormant_on(dev);
 	else
 		netif_dormant_off(dev);
 	if (netif_carrier_ok(rootdev)) {
 		if (!netif_carrier_ok(dev))
 			netif_carrier_on(dev);
 	} else {
 		if (netif_carrier_ok(dev))
 			netif_carrier_off(dev);
 	}
 }
 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
 static int netif_alloc_rx_queues(struct net_device *dev)
 {
 #ifdef CONFIG_RPS
 	unsigned int i, count = dev->num_rx_queues;
 	if (count) {
 		struct netdev_rx_queue *rx;
 		rx = kcalloc(count, sizeof(struct netdev_rx_queue), GFP_KERNEL);
 		if (!rx) {
 			pr_err("netdev: Unable to allocate %u rx queues.\n",
 			       count);
 			return -ENOMEM;
 		}
 		dev->_rx = rx;
-		atomic_set(&rx->count, count);
 		/*
 		 * Set a pointer to first element in the array which holds the
 		 * reference count.
 		 */
 		for (i = 0; i < count; i++)
 			rx[i].first = rx;
 	}
 #endif
 	return 0;
 }
 /**
  *	register_netdevice	- register a network device
  *	@dev: device to register
  *
  *	Take a completed network device structure and add it to the kernel
  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
  *	chain. 0 is returned on success. A negative errno code is returned
  *	on a failure to set up the device, or if the name is a duplicate.
  *
  *	Callers must hold the rtnl semaphore. You may want
  *	register_netdev() instead of this.
  *
  *	BUGS:
  *	The locking appears insufficient to guarantee two parallel registers
  *	will not get the same name.
  */
 int register_netdevice(struct net_device *dev)
 {
 	int ret;
 	struct net *net = dev_net(dev);
 	BUG_ON(dev_boot_phase);
 	ASSERT_RTNL();
 	might_sleep();
 	/* When net_device's are persistent, this will be fatal. */
 	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
 	BUG_ON(!net);
 	spin_lock_init(&dev->addr_list_lock);
 	netdev_set_addr_lockdep_class(dev);
 	netdev_init_queue_locks(dev);
 	dev->iflink = -1;
 	ret = netif_alloc_rx_queues(dev);
 	if (ret)
 		goto out;
 	/* Init, if this function is available */
 	if (dev->netdev_ops->ndo_init) {
 		ret = dev->netdev_ops->ndo_init(dev);
 		if (ret) {
 			if (ret > 0)
 				ret = -EIO;
 			goto out;
 		}
 	}
 	ret = dev_get_valid_name(dev, dev->name, 0);
 	if (ret)
 		goto err_uninit;
 	dev->ifindex = dev_new_index(net);
 	if (dev->iflink == -1)
 		dev->iflink = dev->ifindex;
 	/* Fix illegal checksum combinations */
 	if ((dev->features & NETIF_F_HW_CSUM) &&
 	    (dev->features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
 		printk(KERN_NOTICE "%s: mixed HW and IP checksum settings.\n",
 		       dev->name);
 		dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
 	}
 	if ((dev->features & NETIF_F_NO_CSUM) &&
 	    (dev->features & (NETIF_F_HW_CSUM|NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
 		printk(KERN_NOTICE "%s: mixed no checksumming and other settings.\n",
 		       dev->name);
 		dev->features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM|NETIF_F_HW_CSUM);
 	}
 	dev->features = netdev_fix_features(dev->features, dev->name);
 	/* Enable software GSO if SG is supported. */
 	if (dev->features & NETIF_F_SG)
 		dev->features |= NETIF_F_GSO;
 	/* Enable GRO and NETIF_F_HIGHDMA for vlans by default,
 	 * vlan_dev_init() will do the dev->features check, so these features
 	 * are enabled only if supported by underlying device.
 	 */
 	dev->vlan_features |= (NETIF_F_GRO | NETIF_F_HIGHDMA);
 	ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
 	ret = notifier_to_errno(ret);
 	if (ret)
 		goto err_uninit;
 	ret = netdev_register_kobject(dev);
 	if (ret)
 		goto err_uninit;
 	dev->reg_state = NETREG_REGISTERED;
 	/*
 	 *	Default initial state at registry is that the
 	 *	device is present.
 	 */
 	set_bit(__LINK_STATE_PRESENT, &dev->state);
 	dev_init_scheduler(dev);
 	dev_hold(dev);
 	list_netdevice(dev);
 	/* Notify protocols, that a new device appeared. */
 	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
 	ret = notifier_to_errno(ret);
 	if (ret) {
 		rollback_registered(dev);
 		dev->reg_state = NETREG_UNREGISTERED;
 	}
 	/*
 	 *	Prevent userspace races by waiting until the network
 	 *	device is fully setup before sending notifications.
 	 */
 	if (!dev->rtnl_link_ops ||
 	    dev->rtnl_link_state == RTNL_LINK_INITIALIZED)
 		rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
 out:
 	return ret;
 err_uninit:
 	if (dev->netdev_ops->ndo_uninit)
 		dev->netdev_ops->ndo_uninit(dev);
 	goto out;
 }
 EXPORT_SYMBOL(register_netdevice);
 /**
  *	init_dummy_netdev	- init a dummy network device for NAPI
  *	@dev: device to init
  *
  *	This takes a network device structure and initialize the minimum
  *	amount of fields so it can be used to schedule NAPI polls without
  *	registering a full blown interface. This is to be used by drivers
  *	that need to tie several hardware interfaces to a single NAPI
  *	poll scheduler due to HW limitations.
  */
 int init_dummy_netdev(struct net_device *dev)
 {
 	/* Clear everything. Note we don't initialize spinlocks
 	 * are they aren't supposed to be taken by any of the
 	 * NAPI code and this dummy netdev is supposed to be
 	 * only ever used for NAPI polls
 	 */
 	memset(dev, 0, sizeof(struct net_device));
 	/* make sure we BUG if trying to hit standard
 	 * register/unregister code path
 	 */
 	dev->reg_state = NETREG_DUMMY;
 	/* initialize the ref count */
 	atomic_set(&dev->refcnt, 1);
 	/* NAPI wants this */
 	INIT_LIST_HEAD(&dev->napi_list);
 	/* a dummy interface is started by default */
 	set_bit(__LINK_STATE_PRESENT, &dev->state);
 	set_bit(__LINK_STATE_START, &dev->state);
 	return 0;
 }
 EXPORT_SYMBOL_GPL(init_dummy_netdev);
 /**
  *	register_netdev	- register a network device
  *	@dev: device to register
  *
  *	Take a completed network device structure and add it to the kernel
  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
  *	chain. 0 is returned on success. A negative errno code is returned
  *	on a failure to set up the device, or if the name is a duplicate.
  *
  *	This is a wrapper around register_netdevice that takes the rtnl semaphore
  *	and expands the device name if you passed a format string to
  *	alloc_netdev.
  */
 int register_netdev(struct net_device *dev)
 {
 	int err;
 	rtnl_lock();
 	/*
 	 * If the name is a format string the caller wants us to do a
 	 * name allocation.
 	 */
 	if (strchr(dev->name, '%')) {
 		err = dev_alloc_name(dev, dev->name);
 		if (err < 0)
 			goto out;
 	}
 	err = register_netdevice(dev);
 out:
 	rtnl_unlock();
 	return err;
 }
 EXPORT_SYMBOL(register_netdev);
 /*
  * netdev_wait_allrefs - wait until all references are gone.
  *
  * This is called when unregistering network devices.
  *
  * Any protocol or device that holds a reference should register
  * for netdevice notification, and cleanup and put back the
  * reference if they receive an UNREGISTER event.
  * We can get stuck here if buggy protocols don't correctly
  * call dev_put.
  */
 static void netdev_wait_allrefs(struct net_device *dev)
 {
 	unsigned long rebroadcast_time, warning_time;
 	linkwatch_forget_dev(dev);
 	rebroadcast_time = warning_time = jiffies;
 	while (atomic_read(&dev->refcnt) != 0) {
 		if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
 			rtnl_lock();
 			/* Rebroadcast unregister notification */
 			call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
 			/* don't resend NETDEV_UNREGISTER_BATCH, _BATCH users
 			 * should have already handle it the first time */
 			if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
 				     &dev->state)) {
 				/* We must not have linkwatch events
 				 * pending on unregister. If this
 				 * happens, we simply run the queue
 				 * unscheduled, resulting in a noop
 				 * for this device.
 				 */
 				linkwatch_run_queue();
 			}
 			__rtnl_unlock();
 			rebroadcast_time = jiffies;
 		}
 		msleep(250);
 		if (time_after(jiffies, warning_time + 10 * HZ)) {
 			printk(KERN_EMERG "unregister_netdevice: "
 			       "waiting for %s to become free. Usage "
 			       "count = %d\n",
 			       dev->name, atomic_read(&dev->refcnt));
 			warning_time = jiffies;
 		}
 	}
 }
 /* The sequence is:
  *
  *	rtnl_lock();
  *	...
  *	register_netdevice(x1);
  *	register_netdevice(x2);
  *	...
  *	unregister_netdevice(y1);
  *	unregister_netdevice(y2);
  *      ...
  *	rtnl_unlock();
  *	free_netdev(y1);
  *	free_netdev(y2);
  *
  * We are invoked by rtnl_unlock().
  * This allows us to deal with problems:
  * 1) We can delete sysfs objects which invoke hotplug
  *    without deadlocking with linkwatch via keventd.
  * 2) Since we run with the RTNL semaphore not held, we can sleep
  *    safely in order to wait for the netdev refcnt to drop to zero.
  *
  * We must not return until all unregister events added during
  * the interval the lock was held have been completed.
  */
 void netdev_run_todo(void)
 {
 	struct list_head list;
 	/* Snapshot list, allow later requests */
 	list_replace_init(&net_todo_list, &list);
 	__rtnl_unlock();
 	while (!list_empty(&list)) {
 		struct net_device *dev
 			= list_first_entry(&list, struct net_device, todo_list);
 		list_del(&dev->todo_list);
 		if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
 			printk(KERN_ERR "network todo '%s' but state %d\n",
 			       dev->name, dev->reg_state);
 			dump_stack();
 			continue;
 		}
 		dev->reg_state = NETREG_UNREGISTERED;
 		on_each_cpu(flush_backlog, dev, 1);
 		netdev_wait_allrefs(dev);
 		/* paranoia */
 		BUG_ON(atomic_read(&dev->refcnt));
 		WARN_ON(rcu_dereference_raw(dev->ip_ptr));
 		WARN_ON(dev->ip6_ptr);
 		WARN_ON(dev->dn_ptr);
 		if (dev->destructor)
 			dev->destructor(dev);
 		/* Free network device */
 		kobject_put(&dev->dev.kobj);
 	}
 }
 /**
  *	dev_txq_stats_fold - fold tx_queues stats
  *	@dev: device to get statistics from
  *	@stats: struct rtnl_link_stats64 to hold results
  */
 void dev_txq_stats_fold(const struct net_device *dev,
 			struct rtnl_link_stats64 *stats)
 {
 	u64 tx_bytes = 0, tx_packets = 0, tx_dropped = 0;
 	unsigned int i;
 	struct netdev_queue *txq;
 	for (i = 0; i < dev->num_tx_queues; i++) {
 		txq = netdev_get_tx_queue(dev, i);
 		spin_lock_bh(&txq->_xmit_lock);
 		tx_bytes   += txq->tx_bytes;
 		tx_packets += txq->tx_packets;
 		tx_dropped += txq->tx_dropped;
 		spin_unlock_bh(&txq->_xmit_lock);
 	}
 	if (tx_bytes || tx_packets || tx_dropped) {
 		stats->tx_bytes   = tx_bytes;
 		stats->tx_packets = tx_packets;
 		stats->tx_dropped = tx_dropped;
 	}
 }
 EXPORT_SYMBOL(dev_txq_stats_fold);
 /* Convert net_device_stats to rtnl_link_stats64.  They have the same
  * fields in the same order, with only the type differing.
  */
 static void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
 				    const struct net_device_stats *netdev_stats)
 {
 #if BITS_PER_LONG == 64
         BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats));
         memcpy(stats64, netdev_stats, sizeof(*stats64));
 #else
 	size_t i, n = sizeof(*stats64) / sizeof(u64);
 	const unsigned long *src = (const unsigned long *)netdev_stats;
 	u64 *dst = (u64 *)stats64;
 	BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) !=
 		     sizeof(*stats64) / sizeof(u64));
 	for (i = 0; i < n; i++)
 		dst[i] = src[i];
 #endif
 }
 /**
  *	dev_get_stats	- get network device statistics
  *	@dev: device to get statistics from
  *	@storage: place to store stats
  *
  *	Get network statistics from device. Return @storage.
  *	The device driver may provide its own method by setting
  *	dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
  *	otherwise the internal statistics structure is used.
  */
 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
 					struct rtnl_link_stats64 *storage)
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	if (ops->ndo_get_stats64) {
 		memset(storage, 0, sizeof(*storage));
 		ops->ndo_get_stats64(dev, storage);
 	} else if (ops->ndo_get_stats) {
 		netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
 	} else {
 		netdev_stats_to_stats64(storage, &dev->stats);
 		dev_txq_stats_fold(dev, storage);
 	}
 	storage->rx_dropped += atomic_long_read(&dev->rx_dropped);
 	return storage;
 }
 EXPORT_SYMBOL(dev_get_stats);
 static void netdev_init_one_queue(struct net_device *dev,
 				  struct netdev_queue *queue,
 				  void *_unused)
 {
 	queue->dev = dev;
 }
 static void netdev_init_queues(struct net_device *dev)
 {
 	netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
 	spin_lock_init(&dev->tx_global_lock);
 }
 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
 {
 	struct netdev_queue *queue = dev_ingress_queue(dev);
 #ifdef CONFIG_NET_CLS_ACT
 	if (queue)
 		return queue;
 	queue = kzalloc(sizeof(*queue), GFP_KERNEL);
 	if (!queue)
 		return NULL;
 	netdev_init_one_queue(dev, queue, NULL);
 	__netdev_init_queue_locks_one(dev, queue, NULL);
 	queue->qdisc = &noop_qdisc;
 	queue->qdisc_sleeping = &noop_qdisc;
 	rcu_assign_pointer(dev->ingress_queue, queue);
 #endif
 	return queue;
 }
 /**
  *	alloc_netdev_mq - allocate network device
  *	@sizeof_priv:	size of private data to allocate space for
  *	@name:		device name format string
  *	@setup:		callback to initialize device
  *	@queue_count:	the number of subqueues to allocate
  *
  *	Allocates a struct net_device with private data area for driver use
  *	and performs basic initialization.  Also allocates subquue structs
  *	for each queue on the device at the end of the netdevice.
  */
 struct net_device *alloc_netdev_mq(int sizeof_priv, const char *name,
 		void (*setup)(struct net_device *), unsigned int queue_count)
 {
 	struct netdev_queue *tx;
 	struct net_device *dev;
 	size_t alloc_size;
 	struct net_device *p;
 	BUG_ON(strlen(name) >= sizeof(dev->name));
 	alloc_size = sizeof(struct net_device);
 	if (sizeof_priv) {
 		/* ensure 32-byte alignment of private area */
 		alloc_size = ALIGN(alloc_size, NETDEV_ALIGN);
 		alloc_size += sizeof_priv;
 	}
 	/* ensure 32-byte alignment of whole construct */
 	alloc_size += NETDEV_ALIGN - 1;
 	p = kzalloc(alloc_size, GFP_KERNEL);
 	if (!p) {
 		printk(KERN_ERR "alloc_netdev: Unable to allocate device.\n");
 		return NULL;
 	}
 	tx = kcalloc(queue_count, sizeof(struct netdev_queue), GFP_KERNEL);
 	if (!tx) {
 		printk(KERN_ERR "alloc_netdev: Unable to allocate "
 		       "tx qdiscs.\n");
 		goto free_p;
 	}
 	dev = PTR_ALIGN(p, NETDEV_ALIGN);
 	dev->padded = (char *)dev - (char *)p;
 	if (dev_addr_init(dev))
 		goto free_tx;
 	dev_mc_init(dev);
 	dev_uc_init(dev);
 	dev_net_set(dev, &init_net);
 	dev->_tx = tx;
 	dev->num_tx_queues = queue_count;
 	dev->real_num_tx_queues = queue_count;
 #ifdef CONFIG_RPS
 	dev->num_rx_queues = queue_count;
 	dev->real_num_rx_queues = queue_count;
 #endif
 	dev->gso_max_size = GSO_MAX_SIZE;
 	netdev_init_queues(dev);
 	INIT_LIST_HEAD(&dev->ethtool_ntuple_list.list);
 	dev->ethtool_ntuple_list.count = 0;
 	INIT_LIST_HEAD(&dev->napi_list);
 	INIT_LIST_HEAD(&dev->unreg_list);
 	INIT_LIST_HEAD(&dev->link_watch_list);
 	dev->priv_flags = IFF_XMIT_DST_RELEASE;
 	setup(dev);
 	strcpy(dev->name, name);
 	return dev;
 free_tx:
 	kfree(tx);
 free_p:
 	kfree(p);
 	return NULL;
 }
 EXPORT_SYMBOL(alloc_netdev_mq);
 /**
  *	free_netdev - free network device
  *	@dev: device
  *
  *	This function does the last stage of destroying an allocated device
  * 	interface. The reference to the device object is released.
  *	If this is the last reference then it will be freed.
  */
 void free_netdev(struct net_device *dev)
 {
 	struct napi_struct *p, *n;
 	release_net(dev_net(dev));
 	kfree(dev->_tx);
 	kfree(rcu_dereference_raw(dev->ingress_queue));
 	/* Flush device addresses */
 	dev_addr_flush(dev);
 	/* Clear ethtool n-tuple list */
 	ethtool_ntuple_flush(dev);
 	list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
 		netif_napi_del(p);
 	/*  Compatibility with error handling in drivers */
 	if (dev->reg_state == NETREG_UNINITIALIZED) {
 		kfree((char *)dev - dev->padded);
 		return;
 	}
 	BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
 	dev->reg_state = NETREG_RELEASED;
 	/* will free via device release */
 	put_device(&dev->dev);
 }
 EXPORT_SYMBOL(free_netdev);
 /**
  *	synchronize_net -  Synchronize with packet receive processing
  *
  *	Wait for packets currently being received to be done.
  *	Does not block later packets from starting.
  */
 void synchronize_net(void)
 {
 	might_sleep();
 	synchronize_rcu();
 }
 EXPORT_SYMBOL(synchronize_net);
 /**
  *	unregister_netdevice_queue - remove device from the kernel
  *	@dev: device
  *	@head: list
  *
  *	This function shuts down a device interface and removes it
  *	from the kernel tables.
  *	If head not NULL, device is queued to be unregistered later.
  *
  *	Callers must hold the rtnl semaphore.  You may want
  *	unregister_netdev() instead of this.
  */
 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
 {
 	ASSERT_RTNL();
 	if (head) {
 		list_move_tail(&dev->unreg_list, head);
 	} else {
 		rollback_registered(dev);
 		/* Finish processing unregister after unlock */
 		net_set_todo(dev);
 	}
 }
 EXPORT_SYMBOL(unregister_netdevice_queue);
 /**
  *	unregister_netdevice_many - unregister many devices
  *	@head: list of devices
  */
 void unregister_netdevice_many(struct list_head *head)
 {
 	struct net_device *dev;
 	if (!list_empty(head)) {
 		rollback_registered_many(head);
 		list_for_each_entry(dev, head, unreg_list)
 			net_set_todo(dev);
 	}
 }
 EXPORT_SYMBOL(unregister_netdevice_many);
 /**
  *	unregister_netdev - remove device from the kernel
  *	@dev: device
  *
  *	This function shuts down a device interface and removes it
  *	from the kernel tables.
  *
  *	This is just a wrapper for unregister_netdevice that takes
  *	the rtnl semaphore.  In general you want to use this and not
  *	unregister_netdevice.
  */
 void unregister_netdev(struct net_device *dev)
 {
 	rtnl_lock();
 	unregister_netdevice(dev);
 	rtnl_unlock();
 }
 EXPORT_SYMBOL(unregister_netdev);
 /**
  *	dev_change_net_namespace - move device to different nethost namespace
  *	@dev: device
  *	@net: network namespace
  *	@pat: If not NULL name pattern to try if the current device name
  *	      is already taken in the destination network namespace.
  *
  *	This function shuts down a device interface and moves it
  *	to a new network namespace. On success 0 is returned, on
  *	a failure a netagive errno code is returned.
  *
  *	Callers must hold the rtnl semaphore.
  */
 int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat)
 {
 	int err;
 	ASSERT_RTNL();
 	/* Don't allow namespace local devices to be moved. */
 	err = -EINVAL;
 	if (dev->features & NETIF_F_NETNS_LOCAL)
 		goto out;
 	/* Ensure the device has been registrered */
 	err = -EINVAL;
 	if (dev->reg_state != NETREG_REGISTERED)
 		goto out;
 	/* Get out if there is nothing todo */
 	err = 0;
 	if (net_eq(dev_net(dev), net))
 		goto out;
 	/* Pick the destination device name, and ensure
 	 * we can use it in the destination network namespace.
 	 */
 	err = -EEXIST;
 	if (__dev_get_by_name(net, dev->name)) {
 		/* We get here if we can't use the current device name */
 		if (!pat)
 			goto out;
 		if (dev_get_valid_name(dev, pat, 1))
 			goto out;
 	}
 	/*
 	 * And now a mini version of register_netdevice unregister_netdevice.
 	 */
 	/* If device is running close it first. */
 	dev_close(dev);
 	/* And unlink it from device chain */
 	err = -ENODEV;
 	unlist_netdevice(dev);
 	synchronize_net();
 	/* Shutdown queueing discipline. */
 	dev_shutdown(dev);
 	/* Notify protocols, that we are about to destroy
 	   this device. They should clean all the things.
 	   Note that dev->reg_state stays at NETREG_REGISTERED.
 	   This is wanted because this way 8021q and macvlan know
 	   the device is just moving and can keep their slaves up.
 	*/
 	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
 	call_netdevice_notifiers(NETDEV_UNREGISTER_BATCH, dev);
 	/*
 	 *	Flush the unicast and multicast chains
 	 */
 	dev_uc_flush(dev);
 	dev_mc_flush(dev);
 	/* Actually switch the network namespace */
 	dev_net_set(dev, net);
 	/* If there is an ifindex conflict assign a new one */
 	if (__dev_get_by_index(net, dev->ifindex)) {
 		int iflink = (dev->iflink == dev->ifindex);
 		dev->ifindex = dev_new_index(net);
 		if (iflink)
 			dev->iflink = dev->ifindex;
 	}
 	/* Fixup kobjects */
 	err = device_rename(&dev->dev, dev->name);
 	WARN_ON(err);
 	/* Add the device back in the hashes */
 	list_netdevice(dev);
 	/* Notify protocols, that a new device appeared. */
 	call_netdevice_notifiers(NETDEV_REGISTER, dev);
 	/*
 	 *	Prevent userspace races by waiting until the network
 	 *	device is fully setup before sending notifications.
 	 */
 	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U);
 	synchronize_net();
 	err = 0;
 out:
 	return err;
 }
 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
 static int dev_cpu_callback(struct notifier_block *nfb,
 			    unsigned long action,
 			    void *ocpu)
 {
 	struct sk_buff **list_skb;
 	struct sk_buff *skb;
 	unsigned int cpu, oldcpu = (unsigned long)ocpu;
 	struct softnet_data *sd, *oldsd;
 	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
 		return NOTIFY_OK;
 	local_irq_disable();
 	cpu = smp_processor_id();
 	sd = &per_cpu(softnet_data, cpu);
 	oldsd = &per_cpu(softnet_data, oldcpu);
 	/* Find end of our completion_queue. */
 	list_skb = &sd->completion_queue;
 	while (*list_skb)
 		list_skb = &(*list_skb)->next;
 	/* Append completion queue from offline CPU. */
 	*list_skb = oldsd->completion_queue;
 	oldsd->completion_queue = NULL;
 	/* Append output queue from offline CPU. */
 	if (oldsd->output_queue) {
 		*sd->output_queue_tailp = oldsd->output_queue;
 		sd->output_queue_tailp = oldsd->output_queue_tailp;
 		oldsd->output_queue = NULL;
 		oldsd->output_queue_tailp = &oldsd->output_queue;
 	}
 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
 	local_irq_enable();
 	/* Process offline CPU's input_pkt_queue */
 	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
 		netif_rx(skb);
 		input_queue_head_incr(oldsd);
 	}
 	while ((skb = __skb_dequeue(&oldsd->input_pkt_queue))) {
 		netif_rx(skb);
 		input_queue_head_incr(oldsd);
 	}
 	return NOTIFY_OK;
 }
 /**
  *	netdev_increment_features - increment feature set by one
  *	@all: current feature set
  *	@one: new feature set
  *	@mask: mask feature set
  *
  *	Computes a new feature set after adding a device with feature set
  *	@one to the master device with current feature set @all.  Will not
  *	enable anything that is off in @mask. Returns the new feature set.
  */
 unsigned long netdev_increment_features(unsigned long all, unsigned long one,
 					unsigned long mask)
 {
 	/* If device needs checksumming, downgrade to it. */
 	if (all & NETIF_F_NO_CSUM && !(one & NETIF_F_NO_CSUM))
 		all ^= NETIF_F_NO_CSUM | (one & NETIF_F_ALL_CSUM);
 	else if (mask & NETIF_F_ALL_CSUM) {
 		/* If one device supports v4/v6 checksumming, set for all. */
 		if (one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM) &&
 		    !(all & NETIF_F_GEN_CSUM)) {
 			all &= ~NETIF_F_ALL_CSUM;
 			all |= one & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM);
 		}
 		/* If one device supports hw checksumming, set for all. */
 		if (one & NETIF_F_GEN_CSUM && !(all & NETIF_F_GEN_CSUM)) {
 			all &= ~NETIF_F_ALL_CSUM;
 			all |= NETIF_F_HW_CSUM;
 		}
 	}
 	one |= NETIF_F_ALL_CSUM;
 	one |= all & NETIF_F_ONE_FOR_ALL;
 	all &= one | NETIF_F_LLTX | NETIF_F_GSO | NETIF_F_UFO;
 	all |= one & mask & NETIF_F_ONE_FOR_ALL;
 	return all;
 }
 EXPORT_SYMBOL(netdev_increment_features);
 static struct hlist_head *netdev_create_hash(void)
 {
 	int i;
 	struct hlist_head *hash;
 	hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL);
 	if (hash != NULL)
 		for (i = 0; i < NETDEV_HASHENTRIES; i++)
 			INIT_HLIST_HEAD(&hash[i]);
 	return hash;
 }
 /* Initialize per network namespace state */
 static int __net_init netdev_init(struct net *net)
 {
 	INIT_LIST_HEAD(&net->dev_base_head);
 	net->dev_name_head = netdev_create_hash();
 	if (net->dev_name_head == NULL)
 		goto err_name;
 	net->dev_index_head = netdev_create_hash();
 	if (net->dev_index_head == NULL)
 		goto err_idx;
 	return 0;
 err_idx:
 	kfree(net->dev_name_head);
 err_name:
 	return -ENOMEM;
 }
 /**
  *	netdev_drivername - network driver for the device
  *	@dev: network device
  *	@buffer: buffer for resulting name
  *	@len: size of buffer
  *
  *	Determine network driver for device.
  */
 char *netdev_drivername(const struct net_device *dev, char *buffer, int len)
 {
 	const struct device_driver *driver;
 	const struct device *parent;
 	if (len <= 0 || !buffer)
 		return buffer;
 	buffer[0] = 0;
 	parent = dev->dev.parent;
 	if (!parent)
 		return buffer;
 	driver = parent->driver;
 	if (driver && driver->name)
 		strlcpy(buffer, driver->name, len);
 	return buffer;
 }
 static int __netdev_printk(const char *level, const struct net_device *dev,
 			   struct va_format *vaf)
 {
 	int r;
 	if (dev && dev->dev.parent)
 		r = dev_printk(level, dev->dev.parent, "%s: %pV",
 			       netdev_name(dev), vaf);
 	else if (dev)
 		r = printk("%s%s: %pV", level, netdev_name(dev), vaf);
 	else
 		r = printk("%s(NULL net_device): %pV", level, vaf);
 	return r;
 }
 int netdev_printk(const char *level, const struct net_device *dev,
 		  const char *format, ...)
 {
 	struct va_format vaf;
 	va_list args;
 	int r;
 	va_start(args, format);
 	vaf.fmt = format;
 	vaf.va = &args;
 	r = __netdev_printk(level, dev, &vaf);
 	va_end(args);
 	return r;
 }
 EXPORT_SYMBOL(netdev_printk);
 #define define_netdev_printk_level(func, level)			\
 int func(const struct net_device *dev, const char *fmt, ...)	\
 {								\
 	int r;							\
 	struct va_format vaf;					\
 	va_list args;						\
 								\
 	va_start(args, fmt);					\
 								\
 	vaf.fmt = fmt;						\
 	vaf.va = &args;						\
 								\
 	r = __netdev_printk(level, dev, &vaf);			\
 	va_end(args);						\
 								\
 	return r;						\
 }								\
 EXPORT_SYMBOL(func);
 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
 define_netdev_printk_level(netdev_alert, KERN_ALERT);
 define_netdev_printk_level(netdev_crit, KERN_CRIT);
 define_netdev_printk_level(netdev_err, KERN_ERR);
 define_netdev_printk_level(netdev_warn, KERN_WARNING);
 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
 define_netdev_printk_level(netdev_info, KERN_INFO);
 static void __net_exit netdev_exit(struct net *net)
 {
 	kfree(net->dev_name_head);
 	kfree(net->dev_index_head);
 }
 static struct pernet_operations __net_initdata netdev_net_ops = {
 	.init = netdev_init,
 	.exit = netdev_exit,
 };
 static void __net_exit default_device_exit(struct net *net)
 {
 	struct net_device *dev, *aux;
 	/*
 	 * Push all migratable network devices back to the
 	 * initial network namespace
 	 */
 	rtnl_lock();
 	for_each_netdev_safe(net, dev, aux) {
 		int err;
 		char fb_name[IFNAMSIZ];
 		/* Ignore unmoveable devices (i.e. loopback) */
 		if (dev->features & NETIF_F_NETNS_LOCAL)
 			continue;
 		/* Leave virtual devices for the generic cleanup */
 		if (dev->rtnl_link_ops)
 			continue;
 		/* Push remaing network devices to init_net */
 		snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
 		err = dev_change_net_namespace(dev, &init_net, fb_name);
 		if (err) {
 			printk(KERN_EMERG "%s: failed to move %s to init_net: %d\n",
 				__func__, dev->name, err);
 			BUG();
 		}
 	}
 	rtnl_unlock();
 }
 static void __net_exit default_device_exit_batch(struct list_head *net_list)
 {
 	/* At exit all network devices most be removed from a network
 	 * namespace.  Do this in the reverse order of registeration.
 	 * Do this across as many network namespaces as possible to
 	 * improve batching efficiency.
 	 */
 	struct net_device *dev;
 	struct net *net;
 	LIST_HEAD(dev_kill_list);
 	rtnl_lock();
 	list_for_each_entry(net, net_list, exit_list) {
 		for_each_netdev_reverse(net, dev) {
 			if (dev->rtnl_link_ops)
 				dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
 			else
 				unregister_netdevice_queue(dev, &dev_kill_list);
 		}
 	}
 	unregister_netdevice_many(&dev_kill_list);
 	rtnl_unlock();
 }
 static struct pernet_operations __net_initdata default_device_ops = {
 	.exit = default_device_exit,
 	.exit_batch = default_device_exit_batch,
 };
 /*
  *	Initialize the DEV module. At boot time this walks the device list and
  *	unhooks any devices that fail to initialise (normally hardware not
  *	present) and leaves us with a valid list of present and active devices.
  *
  */
 /*
  *       This is called single threaded during boot, so no need
  *       to take the rtnl semaphore.
  */
 static int __init net_dev_init(void)
 {
 	int i, rc = -ENOMEM;
 	BUG_ON(!dev_boot_phase);
 	if (dev_proc_init())
 		goto out;
 	if (netdev_kobject_init())
 		goto out;
 	INIT_LIST_HEAD(&ptype_all);
 	for (i = 0; i < PTYPE_HASH_SIZE; i++)
 		INIT_LIST_HEAD(&ptype_base[i]);
 	if (register_pernet_subsys(&netdev_net_ops))
 		goto out;
 	/*
 	 *	Initialise the packet receive queues.
 	 */
 	for_each_possible_cpu(i) {
 		struct softnet_data *sd = &per_cpu(softnet_data, i);
 		memset(sd, 0, sizeof(*sd));
 		skb_queue_head_init(&sd->input_pkt_queue);
 		skb_queue_head_init(&sd->process_queue);
 		sd->completion_queue = NULL;
 		INIT_LIST_HEAD(&sd->poll_list);
 		sd->output_queue = NULL;
 		sd->output_queue_tailp = &sd->output_queue;
 #ifdef CONFIG_RPS
 		sd->csd.func = rps_trigger_softirq;
 		sd->csd.info = sd;
 		sd->csd.flags = 0;
 		sd->cpu = i;
 #endif
 		sd->backlog.poll = process_backlog;
 		sd->backlog.weight = weight_p;
 		sd->backlog.gro_list = NULL;
 		sd->backlog.gro_count = 0;
 	}
 	dev_boot_phase = 0;
 	/* The loopback device is special if any other network devices
 	 * is present in a network namespace the loopback device must
 	 * be present. Since we now dynamically allocate and free the
 	 * loopback device ensure this invariant is maintained by
 	 * keeping the loopback device as the first device on the
 	 * list of network devices.  Ensuring the loopback devices
 	 * is the first device that appears and the last network device
 	 * that disappears.
 	 */
 	if (register_pernet_device(&loopback_net_ops))
 		goto out;
 	if (register_pernet_device(&default_device_ops))
 		goto out;
 	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
 	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
 	hotcpu_notifier(dev_cpu_callback, 0);
 	dst_init();
 	dev_mcast_init();
 	rc = 0;
 out:
 	return rc;
 }
 subsys_initcall(net_dev_init);
 static int __init initialize_hashrnd(void)
 {
 	get_random_bytes(&hashrnd, sizeof(hashrnd));
 	return 0;
 }
 late_initcall_sync(initialize_hashrnd);

 /*
  * net-sysfs.c - network device class and attributes
  *
  * Copyright (c) 2003 Stephen Hemminger <shemminger@osdl.org>
  *
  *	This program is free software; you can redistribute it and/or
  *	modify it under the terms of the GNU General Public License
  *	as published by the Free Software Foundation; either version
  *	2 of the License, or (at your option) any later version.
  */
 #include <linux/capability.h>
 #include <linux/kernel.h>
 #include <linux/netdevice.h>
 #include <linux/if_arp.h>
 #include <linux/slab.h>
 #include <linux/nsproxy.h>
 #include <net/sock.h>
 #include <net/net_namespace.h>
 #include <linux/rtnetlink.h>
 #include <linux/wireless.h>
 #include <linux/vmalloc.h>
 #include <net/wext.h>
 #include "net-sysfs.h"
 #ifdef CONFIG_SYSFS
 static const char fmt_hex[] = "%#x\n";
 static const char fmt_long_hex[] = "%#lx\n";
 static const char fmt_dec[] = "%d\n";
 static const char fmt_ulong[] = "%lu\n";
 static const char fmt_u64[] = "%llu\n";
 static inline int dev_isalive(const struct net_device *dev)
 {
 	return dev->reg_state <= NETREG_REGISTERED;
 }
 /* use same locking rules as GIF* ioctl's */
 static ssize_t netdev_show(const struct device *dev,
 			   struct device_attribute *attr, char *buf,
 			   ssize_t (*format)(const struct net_device *, char *))
 {
 	struct net_device *net = to_net_dev(dev);
 	ssize_t ret = -EINVAL;
 	read_lock(&dev_base_lock);
 	if (dev_isalive(net))
 		ret = (*format)(net, buf);
 	read_unlock(&dev_base_lock);
 	return ret;
 }
 /* generate a show function for simple field */
 #define NETDEVICE_SHOW(field, format_string)				\
 static ssize_t format_##field(const struct net_device *net, char *buf)	\
 {									\
 	return sprintf(buf, format_string, net->field);			\
 }									\
 static ssize_t show_##field(struct device *dev,				\
 			    struct device_attribute *attr, char *buf)	\
 {									\
 	return netdev_show(dev, attr, buf, format_##field);		\
 }
 /* use same locking and permission rules as SIF* ioctl's */
 static ssize_t netdev_store(struct device *dev, struct device_attribute *attr,
 			    const char *buf, size_t len,
 			    int (*set)(struct net_device *, unsigned long))
 {
 	struct net_device *net = to_net_dev(dev);
 	char *endp;
 	unsigned long new;
 	int ret = -EINVAL;
 	if (!capable(CAP_NET_ADMIN))
 		return -EPERM;
 	new = simple_strtoul(buf, &endp, 0);
 	if (endp == buf)
 		goto err;
 	if (!rtnl_trylock())
 		return restart_syscall();
 	if (dev_isalive(net)) {
 		if ((ret = (*set)(net, new)) == 0)
 			ret = len;
 	}
 	rtnl_unlock();
  err:
 	return ret;
 }
 NETDEVICE_SHOW(dev_id, fmt_hex);
 NETDEVICE_SHOW(addr_assign_type, fmt_dec);
 NETDEVICE_SHOW(addr_len, fmt_dec);
 NETDEVICE_SHOW(iflink, fmt_dec);
 NETDEVICE_SHOW(ifindex, fmt_dec);
 NETDEVICE_SHOW(features, fmt_long_hex);
 NETDEVICE_SHOW(type, fmt_dec);
 NETDEVICE_SHOW(link_mode, fmt_dec);
 /* use same locking rules as GIFHWADDR ioctl's */
 static ssize_t show_address(struct device *dev, struct device_attribute *attr,
 			    char *buf)
 {
 	struct net_device *net = to_net_dev(dev);
 	ssize_t ret = -EINVAL;
 	read_lock(&dev_base_lock);
 	if (dev_isalive(net))
 		ret = sysfs_format_mac(buf, net->dev_addr, net->addr_len);
 	read_unlock(&dev_base_lock);
 	return ret;
 }
 static ssize_t show_broadcast(struct device *dev,
 			    struct device_attribute *attr, char *buf)
 {
 	struct net_device *net = to_net_dev(dev);
 	if (dev_isalive(net))
 		return sysfs_format_mac(buf, net->broadcast, net->addr_len);
 	return -EINVAL;
 }
 static ssize_t show_carrier(struct device *dev,
 			    struct device_attribute *attr, char *buf)
 {
 	struct net_device *netdev = to_net_dev(dev);
 	if (netif_running(netdev)) {
 		return sprintf(buf, fmt_dec, !!netif_carrier_ok(netdev));
 	}
 	return -EINVAL;
 }
 static ssize_t show_speed(struct device *dev,
 			  struct device_attribute *attr, char *buf)
 {
 	struct net_device *netdev = to_net_dev(dev);
 	int ret = -EINVAL;
 	if (!rtnl_trylock())
 		return restart_syscall();
 	if (netif_running(netdev) &&
 	    netdev->ethtool_ops &&
 	    netdev->ethtool_ops->get_settings) {
 		struct ethtool_cmd cmd = { ETHTOOL_GSET };
 		if (!netdev->ethtool_ops->get_settings(netdev, &cmd))
 			ret = sprintf(buf, fmt_dec, ethtool_cmd_speed(&cmd));
 	}
 	rtnl_unlock();
 	return ret;
 }
 static ssize_t show_duplex(struct device *dev,
 			   struct device_attribute *attr, char *buf)
 {
 	struct net_device *netdev = to_net_dev(dev);
 	int ret = -EINVAL;
 	if (!rtnl_trylock())
 		return restart_syscall();
 	if (netif_running(netdev) &&
 	    netdev->ethtool_ops &&
 	    netdev->ethtool_ops->get_settings) {
 		struct ethtool_cmd cmd = { ETHTOOL_GSET };
 		if (!netdev->ethtool_ops->get_settings(netdev, &cmd))
 			ret = sprintf(buf, "%s\n", cmd.duplex ? "full" : "half");
 	}
 	rtnl_unlock();
 	return ret;
 }
 static ssize_t show_dormant(struct device *dev,
 			    struct device_attribute *attr, char *buf)
 {
 	struct net_device *netdev = to_net_dev(dev);
 	if (netif_running(netdev))
 		return sprintf(buf, fmt_dec, !!netif_dormant(netdev));
 	return -EINVAL;
 }
 static const char *const operstates[] = {
 	"unknown",
 	"notpresent", /* currently unused */
 	"down",
 	"lowerlayerdown",
 	"testing", /* currently unused */
 	"dormant",
 	"up"
 };
 static ssize_t show_operstate(struct device *dev,
 			      struct device_attribute *attr, char *buf)
 {
 	const struct net_device *netdev = to_net_dev(dev);
 	unsigned char operstate;
 	read_lock(&dev_base_lock);
 	operstate = netdev->operstate;
 	if (!netif_running(netdev))
 		operstate = IF_OPER_DOWN;
 	read_unlock(&dev_base_lock);
 	if (operstate >= ARRAY_SIZE(operstates))
 		return -EINVAL; /* should not happen */
 	return sprintf(buf, "%s\n", operstates[operstate]);
 }
 /* read-write attributes */
 NETDEVICE_SHOW(mtu, fmt_dec);
 static int change_mtu(struct net_device *net, unsigned long new_mtu)
 {
 	return dev_set_mtu(net, (int) new_mtu);
 }
 static ssize_t store_mtu(struct device *dev, struct device_attribute *attr,
 			 const char *buf, size_t len)
 {
 	return netdev_store(dev, attr, buf, len, change_mtu);
 }
 NETDEVICE_SHOW(flags, fmt_hex);
 static int change_flags(struct net_device *net, unsigned long new_flags)
 {
 	return dev_change_flags(net, (unsigned) new_flags);
 }
 static ssize_t store_flags(struct device *dev, struct device_attribute *attr,
 			   const char *buf, size_t len)
 {
 	return netdev_store(dev, attr, buf, len, change_flags);
 }
 NETDEVICE_SHOW(tx_queue_len, fmt_ulong);
 static int change_tx_queue_len(struct net_device *net, unsigned long new_len)
 {
 	net->tx_queue_len = new_len;
 	return 0;
 }
 static ssize_t store_tx_queue_len(struct device *dev,
 				  struct device_attribute *attr,
 				  const char *buf, size_t len)
 {
 	return netdev_store(dev, attr, buf, len, change_tx_queue_len);
 }
 static ssize_t store_ifalias(struct device *dev, struct device_attribute *attr,
 			     const char *buf, size_t len)
 {
 	struct net_device *netdev = to_net_dev(dev);
 	size_t count = len;
 	ssize_t ret;
 	if (!capable(CAP_NET_ADMIN))
 		return -EPERM;
 	/* ignore trailing newline */
 	if (len >  0 && buf[len - 1] == '\n')
 		--count;
 	if (!rtnl_trylock())
 		return restart_syscall();
 	ret = dev_set_alias(netdev, buf, count);
 	rtnl_unlock();
 	return ret < 0 ? ret : len;
 }
 static ssize_t show_ifalias(struct device *dev,
 			    struct device_attribute *attr, char *buf)
 {
 	const struct net_device *netdev = to_net_dev(dev);
 	ssize_t ret = 0;
 	if (!rtnl_trylock())
 		return restart_syscall();
 	if (netdev->ifalias)
 		ret = sprintf(buf, "%s\n", netdev->ifalias);
 	rtnl_unlock();
 	return ret;
 }
 static struct device_attribute net_class_attributes[] = {
 	__ATTR(addr_assign_type, S_IRUGO, show_addr_assign_type, NULL),
 	__ATTR(addr_len, S_IRUGO, show_addr_len, NULL),
 	__ATTR(dev_id, S_IRUGO, show_dev_id, NULL),
 	__ATTR(ifalias, S_IRUGO | S_IWUSR, show_ifalias, store_ifalias),
 	__ATTR(iflink, S_IRUGO, show_iflink, NULL),
 	__ATTR(ifindex, S_IRUGO, show_ifindex, NULL),
 	__ATTR(features, S_IRUGO, show_features, NULL),
 	__ATTR(type, S_IRUGO, show_type, NULL),
 	__ATTR(link_mode, S_IRUGO, show_link_mode, NULL),
 	__ATTR(address, S_IRUGO, show_address, NULL),
 	__ATTR(broadcast, S_IRUGO, show_broadcast, NULL),
 	__ATTR(carrier, S_IRUGO, show_carrier, NULL),
 	__ATTR(speed, S_IRUGO, show_speed, NULL),
 	__ATTR(duplex, S_IRUGO, show_duplex, NULL),
 	__ATTR(dormant, S_IRUGO, show_dormant, NULL),
 	__ATTR(operstate, S_IRUGO, show_operstate, NULL),
 	__ATTR(mtu, S_IRUGO | S_IWUSR, show_mtu, store_mtu),
 	__ATTR(flags, S_IRUGO | S_IWUSR, show_flags, store_flags),
 	__ATTR(tx_queue_len, S_IRUGO | S_IWUSR, show_tx_queue_len,
 	       store_tx_queue_len),
 	{}
 };
 /* Show a given an attribute in the statistics group */
 static ssize_t netstat_show(const struct device *d,
 			    struct device_attribute *attr, char *buf,
 			    unsigned long offset)
 {
 	struct net_device *dev = to_net_dev(d);
 	ssize_t ret = -EINVAL;
 	WARN_ON(offset > sizeof(struct rtnl_link_stats64) ||
 			offset % sizeof(u64) != 0);
 	read_lock(&dev_base_lock);
 	if (dev_isalive(dev)) {
 		struct rtnl_link_stats64 temp;
 		const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
 		ret = sprintf(buf, fmt_u64, *(u64 *)(((u8 *) stats) + offset));
 	}
 	read_unlock(&dev_base_lock);
 	return ret;
 }
 /* generate a read-only statistics attribute */
 #define NETSTAT_ENTRY(name)						\
 static ssize_t show_##name(struct device *d,				\
 			   struct device_attribute *attr, char *buf) 	\
 {									\
 	return netstat_show(d, attr, buf,				\
 			    offsetof(struct rtnl_link_stats64, name));	\
 }									\
 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 NETSTAT_ENTRY(rx_packets);
 NETSTAT_ENTRY(tx_packets);
 NETSTAT_ENTRY(rx_bytes);
 NETSTAT_ENTRY(tx_bytes);
 NETSTAT_ENTRY(rx_errors);
 NETSTAT_ENTRY(tx_errors);
 NETSTAT_ENTRY(rx_dropped);
 NETSTAT_ENTRY(tx_dropped);
 NETSTAT_ENTRY(multicast);
 NETSTAT_ENTRY(collisions);
 NETSTAT_ENTRY(rx_length_errors);
 NETSTAT_ENTRY(rx_over_errors);
 NETSTAT_ENTRY(rx_crc_errors);
 NETSTAT_ENTRY(rx_frame_errors);
 NETSTAT_ENTRY(rx_fifo_errors);
 NETSTAT_ENTRY(rx_missed_errors);
 NETSTAT_ENTRY(tx_aborted_errors);
 NETSTAT_ENTRY(tx_carrier_errors);
 NETSTAT_ENTRY(tx_fifo_errors);
 NETSTAT_ENTRY(tx_heartbeat_errors);
 NETSTAT_ENTRY(tx_window_errors);
 NETSTAT_ENTRY(rx_compressed);
 NETSTAT_ENTRY(tx_compressed);
 static struct attribute *netstat_attrs[] = {
 	&dev_attr_rx_packets.attr,
 	&dev_attr_tx_packets.attr,
 	&dev_attr_rx_bytes.attr,
 	&dev_attr_tx_bytes.attr,
 	&dev_attr_rx_errors.attr,
 	&dev_attr_tx_errors.attr,
 	&dev_attr_rx_dropped.attr,
 	&dev_attr_tx_dropped.attr,
 	&dev_attr_multicast.attr,
 	&dev_attr_collisions.attr,
 	&dev_attr_rx_length_errors.attr,
 	&dev_attr_rx_over_errors.attr,
 	&dev_attr_rx_crc_errors.attr,
 	&dev_attr_rx_frame_errors.attr,
 	&dev_attr_rx_fifo_errors.attr,
 	&dev_attr_rx_missed_errors.attr,
 	&dev_attr_tx_aborted_errors.attr,
 	&dev_attr_tx_carrier_errors.attr,
 	&dev_attr_tx_fifo_errors.attr,
 	&dev_attr_tx_heartbeat_errors.attr,
 	&dev_attr_tx_window_errors.attr,
 	&dev_attr_rx_compressed.attr,
 	&dev_attr_tx_compressed.attr,
 	NULL
 };
 static struct attribute_group netstat_group = {
 	.name  = "statistics",
 	.attrs  = netstat_attrs,
 };
 #ifdef CONFIG_WIRELESS_EXT_SYSFS
 /* helper function that does all the locking etc for wireless stats */
 static ssize_t wireless_show(struct device *d, char *buf,
 			     ssize_t (*format)(const struct iw_statistics *,
 					       char *))
 {
 	struct net_device *dev = to_net_dev(d);
 	const struct iw_statistics *iw;
 	ssize_t ret = -EINVAL;
 	if (!rtnl_trylock())
 		return restart_syscall();
 	if (dev_isalive(dev)) {
 		iw = get_wireless_stats(dev);
 		if (iw)
 			ret = (*format)(iw, buf);
 	}
 	rtnl_unlock();
 	return ret;
 }
 /* show function template for wireless fields */
 #define WIRELESS_SHOW(name, field, format_string)			\
 static ssize_t format_iw_##name(const struct iw_statistics *iw, char *buf) \
 {									\
 	return sprintf(buf, format_string, iw->field);			\
 }									\
 static ssize_t show_iw_##name(struct device *d,				\
 			      struct device_attribute *attr, char *buf)	\
 {									\
 	return wireless_show(d, buf, format_iw_##name);			\
 }									\
 static DEVICE_ATTR(name, S_IRUGO, show_iw_##name, NULL)
 WIRELESS_SHOW(status, status, fmt_hex);
 WIRELESS_SHOW(link, qual.qual, fmt_dec);
 WIRELESS_SHOW(level, qual.level, fmt_dec);
 WIRELESS_SHOW(noise, qual.noise, fmt_dec);
 WIRELESS_SHOW(nwid, discard.nwid, fmt_dec);
 WIRELESS_SHOW(crypt, discard.code, fmt_dec);
 WIRELESS_SHOW(fragment, discard.fragment, fmt_dec);
 WIRELESS_SHOW(misc, discard.misc, fmt_dec);
 WIRELESS_SHOW(retries, discard.retries, fmt_dec);
 WIRELESS_SHOW(beacon, miss.beacon, fmt_dec);
 static struct attribute *wireless_attrs[] = {
 	&dev_attr_status.attr,
 	&dev_attr_link.attr,
 	&dev_attr_level.attr,
 	&dev_attr_noise.attr,
 	&dev_attr_nwid.attr,
 	&dev_attr_crypt.attr,
 	&dev_attr_fragment.attr,
 	&dev_attr_retries.attr,
 	&dev_attr_misc.attr,
 	&dev_attr_beacon.attr,
 	NULL
 };
 static struct attribute_group wireless_group = {
 	.name = "wireless",
 	.attrs = wireless_attrs,
 };
 #endif
 #endif /* CONFIG_SYSFS */
 #ifdef CONFIG_RPS
 /*
  * RX queue sysfs structures and functions.
  */
 struct rx_queue_attribute {
 	struct attribute attr;
 	ssize_t (*show)(struct netdev_rx_queue *queue,
 	    struct rx_queue_attribute *attr, char *buf);
 	ssize_t (*store)(struct netdev_rx_queue *queue,
 	    struct rx_queue_attribute *attr, const char *buf, size_t len);
 };
 #define to_rx_queue_attr(_attr) container_of(_attr,		\
     struct rx_queue_attribute, attr)
 #define to_rx_queue(obj) container_of(obj, struct netdev_rx_queue, kobj)
 static ssize_t rx_queue_attr_show(struct kobject *kobj, struct attribute *attr,
 				  char *buf)
 {
 	struct rx_queue_attribute *attribute = to_rx_queue_attr(attr);
 	struct netdev_rx_queue *queue = to_rx_queue(kobj);
 	if (!attribute->show)
 		return -EIO;
 	return attribute->show(queue, attribute, buf);
 }
 static ssize_t rx_queue_attr_store(struct kobject *kobj, struct attribute *attr,
 				   const char *buf, size_t count)
 {
 	struct rx_queue_attribute *attribute = to_rx_queue_attr(attr);
 	struct netdev_rx_queue *queue = to_rx_queue(kobj);
 	if (!attribute->store)
 		return -EIO;
 	return attribute->store(queue, attribute, buf, count);
 }
 static const struct sysfs_ops rx_queue_sysfs_ops = {
 	.show = rx_queue_attr_show,
 	.store = rx_queue_attr_store,
 };
 static ssize_t show_rps_map(struct netdev_rx_queue *queue,
 			    struct rx_queue_attribute *attribute, char *buf)
 {
 	struct rps_map *map;
 	cpumask_var_t mask;
 	size_t len = 0;
 	int i;
 	if (!zalloc_cpumask_var(&mask, GFP_KERNEL))
 		return -ENOMEM;
 	rcu_read_lock();
 	map = rcu_dereference(queue->rps_map);
 	if (map)
 		for (i = 0; i < map->len; i++)
 			cpumask_set_cpu(map->cpus[i], mask);
 	len += cpumask_scnprintf(buf + len, PAGE_SIZE, mask);
 	if (PAGE_SIZE - len < 3) {
 		rcu_read_unlock();
 		free_cpumask_var(mask);
 		return -EINVAL;
 	}
 	rcu_read_unlock();
 	free_cpumask_var(mask);
 	len += sprintf(buf + len, "\n");
 	return len;
 }
 static void rps_map_release(struct rcu_head *rcu)
 {
 	struct rps_map *map = container_of(rcu, struct rps_map, rcu);
 	kfree(map);
 }
 static ssize_t store_rps_map(struct netdev_rx_queue *queue,
 		      struct rx_queue_attribute *attribute,
 		      const char *buf, size_t len)
 {
 	struct rps_map *old_map, *map;
 	cpumask_var_t mask;
 	int err, cpu, i;
 	static DEFINE_SPINLOCK(rps_map_lock);
 	if (!capable(CAP_NET_ADMIN))
 		return -EPERM;
 	if (!alloc_cpumask_var(&mask, GFP_KERNEL))
 		return -ENOMEM;
 	err = bitmap_parse(buf, len, cpumask_bits(mask), nr_cpumask_bits);
 	if (err) {
 		free_cpumask_var(mask);
 		return err;
 	}
 	map = kzalloc(max_t(unsigned,
 	    RPS_MAP_SIZE(cpumask_weight(mask)), L1_CACHE_BYTES),
 	    GFP_KERNEL);
 	if (!map) {
 		free_cpumask_var(mask);
 		return -ENOMEM;
 	}
 	i = 0;
 	for_each_cpu_and(cpu, mask, cpu_online_mask)
 		map->cpus[i++] = cpu;
 	if (i)
 		map->len = i;
 	else {
 		kfree(map);
 		map = NULL;
 	}
 	spin_lock(&rps_map_lock);
 	old_map = queue->rps_map;
 	rcu_assign_pointer(queue->rps_map, map);
 	spin_unlock(&rps_map_lock);
 	if (old_map)
 		call_rcu(&old_map->rcu, rps_map_release);
 	free_cpumask_var(mask);
 	return len;
 }
 static ssize_t show_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue,
 					   struct rx_queue_attribute *attr,
 					   char *buf)
 {
 	struct rps_dev_flow_table *flow_table;
 	unsigned int val = 0;
 	rcu_read_lock();
 	flow_table = rcu_dereference(queue->rps_flow_table);
 	if (flow_table)
 		val = flow_table->mask + 1;
 	rcu_read_unlock();
 	return sprintf(buf, "%u\n", val);
 }
 static void rps_dev_flow_table_release_work(struct work_struct *work)
 {
 	struct rps_dev_flow_table *table = container_of(work,
 	    struct rps_dev_flow_table, free_work);
 	vfree(table);
 }
 static void rps_dev_flow_table_release(struct rcu_head *rcu)
 {
 	struct rps_dev_flow_table *table = container_of(rcu,
 	    struct rps_dev_flow_table, rcu);
 	INIT_WORK(&table->free_work, rps_dev_flow_table_release_work);
 	schedule_work(&table->free_work);
 }
 static ssize_t store_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue,
 				     struct rx_queue_attribute *attr,
 				     const char *buf, size_t len)
 {
 	unsigned int count;
 	char *endp;
 	struct rps_dev_flow_table *table, *old_table;
 	static DEFINE_SPINLOCK(rps_dev_flow_lock);
 	if (!capable(CAP_NET_ADMIN))
 		return -EPERM;
 	count = simple_strtoul(buf, &endp, 0);
 	if (endp == buf)
 		return -EINVAL;
 	if (count) {
 		int i;
 		if (count > 1<<30) {
 			/* Enforce a limit to prevent overflow */
 			return -EINVAL;
 		}
 		count = roundup_pow_of_two(count);
 		table = vmalloc(RPS_DEV_FLOW_TABLE_SIZE(count));
 		if (!table)
 			return -ENOMEM;
 		table->mask = count - 1;
 		for (i = 0; i < count; i++)
 			table->flows[i].cpu = RPS_NO_CPU;
 	} else
 		table = NULL;
 	spin_lock(&rps_dev_flow_lock);
 	old_table = queue->rps_flow_table;
 	rcu_assign_pointer(queue->rps_flow_table, table);
 	spin_unlock(&rps_dev_flow_lock);
 	if (old_table)
 		call_rcu(&old_table->rcu, rps_dev_flow_table_release);
 	return len;
 }
 static struct rx_queue_attribute rps_cpus_attribute =
 	__ATTR(rps_cpus, S_IRUGO | S_IWUSR, show_rps_map, store_rps_map);
 static struct rx_queue_attribute rps_dev_flow_table_cnt_attribute =
 	__ATTR(rps_flow_cnt, S_IRUGO | S_IWUSR,
 	    show_rps_dev_flow_table_cnt, store_rps_dev_flow_table_cnt);
 static struct attribute *rx_queue_default_attrs[] = {
 	&rps_cpus_attribute.attr,
 	&rps_dev_flow_table_cnt_attribute.attr,
 	NULL
 };
 static void rx_queue_release(struct kobject *kobj)
 {
 	struct netdev_rx_queue *queue = to_rx_queue(kobj);
 	struct netdev_rx_queue *first = queue->first;
 	if (queue->rps_map)
 		call_rcu(&queue->rps_map->rcu, rps_map_release);
 	if (queue->rps_flow_table)
 		call_rcu(&queue->rps_flow_table->rcu,
 		    rps_dev_flow_table_release);
 	if (atomic_dec_and_test(&first->count))
 		kfree(first);
 }
 static struct kobj_type rx_queue_ktype = {
 	.sysfs_ops = &rx_queue_sysfs_ops,
 	.release = rx_queue_release,
 	.default_attrs = rx_queue_default_attrs,
 };
 static int rx_queue_add_kobject(struct net_device *net, int index)
 {
 	struct netdev_rx_queue *queue = net->_rx + index;
+	struct netdev_rx_queue *first = queue->first;
 	struct kobject *kobj = &queue->kobj;
 	int error = 0;
 	kobj->kset = net->queues_kset;
 	error = kobject_init_and_add(kobj, &rx_queue_ktype, NULL,
 	    "rx-%u", index);
 	if (error) {
 		kobject_put(kobj);
 		return error;
 	}
 	kobject_uevent(kobj, KOBJ_ADD);
+	atomic_inc(&first->count);
 	return error;
 }
 int
 net_rx_queue_update_kobjects(struct net_device *net, int old_num, int new_num)
 {
 	int i;
 	int error = 0;
 	for (i = old_num; i < new_num; i++) {
 		error = rx_queue_add_kobject(net, i);
 		if (error) {
 			new_num = old_num;
 			break;
 		}
 	}
 	while (--i >= new_num)
 		kobject_put(&net->_rx[i].kobj);
 	return error;
 }
 static int rx_queue_register_kobjects(struct net_device *net)
 {
 	net->queues_kset = kset_create_and_add("queues",
 	    NULL, &net->dev.kobj);
 	if (!net->queues_kset)
 		return -ENOMEM;
 	return net_rx_queue_update_kobjects(net, 0, net->real_num_rx_queues);
 }
 static void rx_queue_remove_kobjects(struct net_device *net)
 {
 	net_rx_queue_update_kobjects(net, net->real_num_rx_queues, 0);
 	kset_unregister(net->queues_kset);
 }
 #endif /* CONFIG_RPS */
 static const void *net_current_ns(void)
 {
 	return current->nsproxy->net_ns;
 }
 static const void *net_initial_ns(void)
 {
 	return &init_net;
 }
 static const void *net_netlink_ns(struct sock *sk)
 {
 	return sock_net(sk);
 }
 struct kobj_ns_type_operations net_ns_type_operations = {
 	.type = KOBJ_NS_TYPE_NET,
 	.current_ns = net_current_ns,
 	.netlink_ns = net_netlink_ns,
 	.initial_ns = net_initial_ns,
 };
 EXPORT_SYMBOL_GPL(net_ns_type_operations);
 static void net_kobj_ns_exit(struct net *net)
 {
 	kobj_ns_exit(KOBJ_NS_TYPE_NET, net);
 }
 static struct pernet_operations kobj_net_ops = {
 	.exit = net_kobj_ns_exit,
 };
 #ifdef CONFIG_HOTPLUG
 static int netdev_uevent(struct device *d, struct kobj_uevent_env *env)
 {
 	struct net_device *dev = to_net_dev(d);
 	int retval;
 	/* pass interface to uevent. */
 	retval = add_uevent_var(env, "INTERFACE=%s", dev->name);
 	if (retval)
 		goto exit;
 	/* pass ifindex to uevent.
 	 * ifindex is useful as it won't change (interface name may change)
 	 * and is what RtNetlink uses natively. */
 	retval = add_uevent_var(env, "IFINDEX=%d", dev->ifindex);
 exit:
 	return retval;
 }
 #endif
 /*
  *	netdev_release -- destroy and free a dead device.
  *	Called when last reference to device kobject is gone.
  */
 static void netdev_release(struct device *d)
 {
 	struct net_device *dev = to_net_dev(d);
 	BUG_ON(dev->reg_state != NETREG_RELEASED);
 	kfree(dev->ifalias);
 	kfree((char *)dev - dev->padded);
 }
 static const void *net_namespace(struct device *d)
 {
 	struct net_device *dev;
 	dev = container_of(d, struct net_device, dev);
 	return dev_net(dev);
 }
 static struct class net_class = {
 	.name = "net",
 	.dev_release = netdev_release,
 #ifdef CONFIG_SYSFS
 	.dev_attrs = net_class_attributes,
 #endif /* CONFIG_SYSFS */
 #ifdef CONFIG_HOTPLUG
 	.dev_uevent = netdev_uevent,
 #endif
 	.ns_type = &net_ns_type_operations,
 	.namespace = net_namespace,
 };
 /* Delete sysfs entries but hold kobject reference until after all
  * netdev references are gone.
  */
 void netdev_unregister_kobject(struct net_device * net)
 {
 	struct device *dev = &(net->dev);
 	kobject_get(&dev->kobj);
 #ifdef CONFIG_RPS
 	rx_queue_remove_kobjects(net);
 #endif
 	device_del(dev);
 }
 /* Create sysfs entries for network device. */
 int netdev_register_kobject(struct net_device *net)
 {
 	struct device *dev = &(net->dev);
 	const struct attribute_group **groups = net->sysfs_groups;
 	int error = 0;
 	device_initialize(dev);
 	dev->class = &net_class;
 	dev->platform_data = net;
 	dev->groups = groups;
 	dev_set_name(dev, "%s", net->name);
 #ifdef CONFIG_SYSFS
 	/* Allow for a device specific group */
 	if (*groups)
 		groups++;
 	*groups++ = &netstat_group;
 #ifdef CONFIG_WIRELESS_EXT_SYSFS
 	if (net->ieee80211_ptr)
 		*groups++ = &wireless_group;
 #ifdef CONFIG_WIRELESS_EXT
 	else if (net->wireless_handlers)
 		*groups++ = &wireless_group;
 #endif
 #endif
 #endif /* CONFIG_SYSFS */
 	error = device_add(dev);
 	if (error)
 		return error;
 #ifdef CONFIG_RPS
 	error = rx_queue_register_kobjects(net);
 	if (error) {
 		device_del(dev);
 		return error;
 	}
 #endif
 	return error;
 }
 int netdev_class_create_file(struct class_attribute *class_attr)
 {
 	return class_create_file(&net_class, class_attr);
 }
 EXPORT_SYMBOL(netdev_class_create_file);
 void netdev_class_remove_file(struct class_attribute *class_attr)
 {
 	class_remove_file(&net_class, class_attr);
 }
 EXPORT_SYMBOL(netdev_class_remove_file);
 int netdev_kobject_init(void)
 {
 	kobj_ns_type_register(&net_ns_type_operations);
 	register_pernet_subsys(&kobj_net_ops);
 	return class_register(&net_class);
 }