Doug / smarc-fsl-linux-kernel

1

/*

1

/*

2

* Linux driver for VMware's vmxnet3 ethernet NIC.

2

* Linux driver for VMware's vmxnet3 ethernet NIC.

3

*

3

*

4

5

*

5

*

6

* This program is free software; you can redistribute it and/or modify it

6

* This program is free software; you can redistribute it and/or modify it

7

* under the terms of the GNU General Public License as published by the

7

* under the terms of the GNU General Public License as published by the

8

* Free Software Foundation; version 2 of the License and no later version.

8

* Free Software Foundation; version 2 of the License and no later version.

9

*

9

*

10

* This program is distributed in the hope that it will be useful, but

10

* This program is distributed in the hope that it will be useful, but

11

* WITHOUT ANY WARRANTY; without even the implied warranty of

11

* WITHOUT ANY WARRANTY; without even the implied warranty of

12

* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or

12

* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or

13

* NON INFRINGEMENT. See the GNU General Public License for more

13

* NON INFRINGEMENT. See the GNU General Public License for more

14

* details.

14

* details.

15

*

15

*

16

* You should have received a copy of the GNU General Public License

16

* You should have received a copy of the GNU General Public License

17

* along with this program; if not, write to the Free Software

17

* along with this program; if not, write to the Free Software

18

* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

18

* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

19

*

19

*

20

* The full GNU General Public License is included in this distribution in

20

* The full GNU General Public License is included in this distribution in

21

* the file called "COPYING".

21

* the file called "COPYING".

22

*

22

*

23

* Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>

23

* Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>

24

*

24

*

25

*/

25

*/

26

27

#include <linux/module.h>

27

#include <linux/module.h>

28

#include <net/ip6_checksum.h>

28

#include <net/ip6_checksum.h>

29

30

#include "vmxnet3_int.h"

30

#include "vmxnet3_int.h"

31

32

char vmxnet3_driver_name[] = "vmxnet3";

32

char vmxnet3_driver_name[] = "vmxnet3";

33

#define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"

33

#define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"

34

35

/*

35

/*

36

* PCI Device ID Table

36

* PCI Device ID Table

37

* Last entry must be all 0s

37

* Last entry must be all 0s

38

*/

38

*/

39

static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {

39

static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {

40

{PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},

40

{PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},

41

{0}

41

{0}

42

};

42

};

43

44

MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);

44

MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);

45

46

static int enable_mq = 1;

46

static int enable_mq = 1;

47

48

static void

48

static void

49

vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);

49

vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);

50

51

/*

51

/*

52

* Enable/Disable the given intr

52

* Enable/Disable the given intr

53

*/

53

*/

54

static void

54

static void

55

vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)

55

vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)

56

{

56

{

57

VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);

57

VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);

58

}

58

}

59

60

61

static void

61

static void

62

vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)

62

vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)

63

{

63

{

64

VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);

64

VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);

65

}

65

}

66

67

68

/*

68

/*

69

* Enable/Disable all intrs used by the device

69

* Enable/Disable all intrs used by the device

70

*/

70

*/

71

static void

71

static void

72

vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)

72

vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)

73

{

73

{

74

int i;

74

int i;

75

76

for (i = 0; i < adapter->intr.num_intrs; i++)

76

for (i = 0; i < adapter->intr.num_intrs; i++)

77

vmxnet3_enable_intr(adapter, i);

77

vmxnet3_enable_intr(adapter, i);

78

adapter->shared->devRead.intrConf.intrCtrl &=

78

adapter->shared->devRead.intrConf.intrCtrl &=

79

cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);

79

cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);

80

}

80

}

81

82

83

static void

83

static void

84

vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)

84

vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)

85

{

85

{

86

int i;

86

int i;

87

88

adapter->shared->devRead.intrConf.intrCtrl |=

88

adapter->shared->devRead.intrConf.intrCtrl |=

89

cpu_to_le32(VMXNET3_IC_DISABLE_ALL);

89

cpu_to_le32(VMXNET3_IC_DISABLE_ALL);

90

for (i = 0; i < adapter->intr.num_intrs; i++)

90

for (i = 0; i < adapter->intr.num_intrs; i++)

91

vmxnet3_disable_intr(adapter, i);

91

vmxnet3_disable_intr(adapter, i);

92

}

92

}

93

94

95

static void

95

static void

96

vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)

96

vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)

97

{

97

{

98

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);

98

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);

99

}

99

}

100

101

102

static bool

102

static bool

103

vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)

103

vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)

104

{

104

{

105

return tq->stopped;

105

return tq->stopped;

106

}

106

}

107

108

109

static void

109

static void

110

vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)

110

vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)

111

{

111

{

112

tq->stopped = false;

112

tq->stopped = false;

113

netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);

113

netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);

114

}

114

}

115

116

117

static void

117

static void

118

vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)

118

vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)

119

{

119

{

120

tq->stopped = false;

120

tq->stopped = false;

121

netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));

121

netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));

122

}

122

}

123

124

125

static void

125

static void

126

vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)

126

vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)

127

{

127

{

128

tq->stopped = true;

128

tq->stopped = true;

129

tq->num_stop++;

129

tq->num_stop++;

130

netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));

130

netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));

131

}

131

}

132

133

134

/*

134

/*

135

* Check the link state. This may start or stop the tx queue.

135

* Check the link state. This may start or stop the tx queue.

136

*/

136

*/

137

static void

137

static void

138

vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)

138

vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)

139

{

139

{

140

u32 ret;

140

u32 ret;

141

int i;

141

int i;

142

unsigned long flags;

142

unsigned long flags;

143

144

spin_lock_irqsave(&adapter->cmd_lock, flags);

144

spin_lock_irqsave(&adapter->cmd_lock, flags);

145

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);

145

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);

146

ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);

146

ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);

147

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

147

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

148

149

adapter->link_speed = ret >> 16;

149

adapter->link_speed = ret >> 16;

150

if (ret & 1) { /* Link is up. */

150

if (ret & 1) { /* Link is up. */

151

netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",

151

netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",

152

adapter->link_speed);

152

adapter->link_speed);

153

netif_carrier_on(adapter->netdev);

153

netif_carrier_on(adapter->netdev);

154

155

if (affectTxQueue) {

155

if (affectTxQueue) {

156

for (i = 0; i < adapter->num_tx_queues; i++)

156

for (i = 0; i < adapter->num_tx_queues; i++)

157

vmxnet3_tq_start(&adapter->tx_queue[i],

157

vmxnet3_tq_start(&adapter->tx_queue[i],

158

adapter);

158

adapter);

159

}

159

}

160

} else {

160

} else {

161

netdev_info(adapter->netdev, "NIC Link is Down\n");

161

netdev_info(adapter->netdev, "NIC Link is Down\n");

162

netif_carrier_off(adapter->netdev);

162

netif_carrier_off(adapter->netdev);

163

164

if (affectTxQueue) {

164

if (affectTxQueue) {

165

for (i = 0; i < adapter->num_tx_queues; i++)

165

for (i = 0; i < adapter->num_tx_queues; i++)

166

vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);

166

vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);

167

}

167

}

168

}

168

}

169

}

169

}

170

171

static void

171

static void

172

vmxnet3_process_events(struct vmxnet3_adapter *adapter)

172

vmxnet3_process_events(struct vmxnet3_adapter *adapter)

173

{

173

{

174

int i;

174

int i;

175

unsigned long flags;

175

unsigned long flags;

176

u32 events = le32_to_cpu(adapter->shared->ecr);

176

u32 events = le32_to_cpu(adapter->shared->ecr);

177

if (!events)

177

if (!events)

178

return;

178

return;

179

180

vmxnet3_ack_events(adapter, events);

180

vmxnet3_ack_events(adapter, events);

181

182

/* Check if link state has changed */

182

/* Check if link state has changed */

183

if (events & VMXNET3_ECR_LINK)

183

if (events & VMXNET3_ECR_LINK)

184

vmxnet3_check_link(adapter, true);

184

vmxnet3_check_link(adapter, true);

185

186

/* Check if there is an error on xmit/recv queues */

186

/* Check if there is an error on xmit/recv queues */

187

if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {

187

if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {

188

spin_lock_irqsave(&adapter->cmd_lock, flags);

188

spin_lock_irqsave(&adapter->cmd_lock, flags);

189

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

189

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

190

VMXNET3_CMD_GET_QUEUE_STATUS);

190

VMXNET3_CMD_GET_QUEUE_STATUS);

191

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

191

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

192

193

for (i = 0; i < adapter->num_tx_queues; i++)

193

for (i = 0; i < adapter->num_tx_queues; i++)

194

if (adapter->tqd_start[i].status.stopped)

194

if (adapter->tqd_start[i].status.stopped)

195

dev_err(&adapter->netdev->dev,

195

dev_err(&adapter->netdev->dev,

196

"%s: tq[%d] error 0x%x\n",

196

"%s: tq[%d] error 0x%x\n",

197

adapter->netdev->name, i, le32_to_cpu(

197

adapter->netdev->name, i, le32_to_cpu(

198

adapter->tqd_start[i].status.error));

198

adapter->tqd_start[i].status.error));

199

for (i = 0; i < adapter->num_rx_queues; i++)

199

for (i = 0; i < adapter->num_rx_queues; i++)

200

if (adapter->rqd_start[i].status.stopped)

200

if (adapter->rqd_start[i].status.stopped)

201

dev_err(&adapter->netdev->dev,

201

dev_err(&adapter->netdev->dev,

202

"%s: rq[%d] error 0x%x\n",

202

"%s: rq[%d] error 0x%x\n",

203

adapter->netdev->name, i,

203

adapter->netdev->name, i,

204

adapter->rqd_start[i].status.error);

204

adapter->rqd_start[i].status.error);

205

206

schedule_work(&adapter->work);

206

schedule_work(&adapter->work);

207

}

207

}

208

}

208

}

209

210

#ifdef __BIG_ENDIAN_BITFIELD

210

#ifdef __BIG_ENDIAN_BITFIELD

211

/*

211

/*

212

* The device expects the bitfields in shared structures to be written in

212

* The device expects the bitfields in shared structures to be written in

213

* little endian. When CPU is big endian, the following routines are used to

213

* little endian. When CPU is big endian, the following routines are used to

214

* correctly read and write into ABI.

214

* correctly read and write into ABI.

215

* The general technique used here is : double word bitfields are defined in

215

* The general technique used here is : double word bitfields are defined in

216

* opposite order for big endian architecture. Then before reading them in

216

* opposite order for big endian architecture. Then before reading them in

217

* driver the complete double word is translated using le32_to_cpu. Similarly

217

* driver the complete double word is translated using le32_to_cpu. Similarly

218

* After the driver writes into bitfields, cpu_to_le32 is used to translate the

218

* After the driver writes into bitfields, cpu_to_le32 is used to translate the

219

* double words into required format.

219

* double words into required format.

220

* In order to avoid touching bits in shared structure more than once, temporary

220

* In order to avoid touching bits in shared structure more than once, temporary

221

* descriptors are used. These are passed as srcDesc to following functions.

221

* descriptors are used. These are passed as srcDesc to following functions.

222

*/

222

*/

223

static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,

223

static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,

224

struct Vmxnet3_RxDesc *dstDesc)

224

struct Vmxnet3_RxDesc *dstDesc)

225

{

225

{

226

u32 *src = (u32 *)srcDesc + 2;

226

u32 *src = (u32 *)srcDesc + 2;

227

u32 *dst = (u32 *)dstDesc + 2;

227

u32 *dst = (u32 *)dstDesc + 2;

228

dstDesc->addr = le64_to_cpu(srcDesc->addr);

228

dstDesc->addr = le64_to_cpu(srcDesc->addr);

229

*dst = le32_to_cpu(*src);

229

*dst = le32_to_cpu(*src);

230

dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);

230

dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);

231

}

231

}

232

233

static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,

233

static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,

234

struct Vmxnet3_TxDesc *dstDesc)

234

struct Vmxnet3_TxDesc *dstDesc)

235

{

235

{

236

int i;

236

int i;

237

u32 *src = (u32 *)(srcDesc + 1);

237

u32 *src = (u32 *)(srcDesc + 1);

238

u32 *dst = (u32 *)(dstDesc + 1);

238

u32 *dst = (u32 *)(dstDesc + 1);

239

240

/* Working backwards so that the gen bit is set at the end. */

240

/* Working backwards so that the gen bit is set at the end. */

241

for (i = 2; i > 0; i--) {

241

for (i = 2; i > 0; i--) {

242

src--;

242

src--;

243

dst--;

243

dst--;

244

*dst = cpu_to_le32(*src);

244

*dst = cpu_to_le32(*src);

245

}

245

}

246

}

246

}

247

248

249

static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,

249

static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,

250

struct Vmxnet3_RxCompDesc *dstDesc)

250

struct Vmxnet3_RxCompDesc *dstDesc)

251

{

251

{

252

int i = 0;

252

int i = 0;

253

u32 *src = (u32 *)srcDesc;

253

u32 *src = (u32 *)srcDesc;

254

u32 *dst = (u32 *)dstDesc;

254

u32 *dst = (u32 *)dstDesc;

255

for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {

255

for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {

256

*dst = le32_to_cpu(*src);

256

*dst = le32_to_cpu(*src);

257

src++;

257

src++;

258

dst++;

258

dst++;

259

}

259

}

260

}

260

}

261

262

263

/* Used to read bitfield values from double words. */

263

/* Used to read bitfield values from double words. */

264

static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)

264

static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)

265

{

265

{

266

u32 temp = le32_to_cpu(*bitfield);

266

u32 temp = le32_to_cpu(*bitfield);

267

u32 mask = ((1 << size) - 1) << pos;

267

u32 mask = ((1 << size) - 1) << pos;

268

temp &= mask;

268

temp &= mask;

269

temp >>= pos;

269

temp >>= pos;

270

return temp;

270

return temp;

271

}

271

}

272

273

274

275

#endif /* __BIG_ENDIAN_BITFIELD */

275

#endif /* __BIG_ENDIAN_BITFIELD */

276

277

#ifdef __BIG_ENDIAN_BITFIELD

277

#ifdef __BIG_ENDIAN_BITFIELD

278

279

# define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \

279

# define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \

280

txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \

280

txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \

281

VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)

281

VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)

282

# define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \

282

# define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \

283

txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \

283

txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \

284

VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)

284

VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)

285

# define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \

285

# define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \

286

VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \

286

VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \

287

VMXNET3_TCD_GEN_SIZE)

287

VMXNET3_TCD_GEN_SIZE)

288

# define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \

288

# define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \

289

VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)

289

VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)

290

# define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \

290

# define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \

291

(dstrcd) = (tmp); \

291

(dstrcd) = (tmp); \

292

vmxnet3_RxCompToCPU((rcd), (tmp)); \

292

vmxnet3_RxCompToCPU((rcd), (tmp)); \

293

} while (0)

293

} while (0)

294

# define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \

294

# define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \

295

(dstrxd) = (tmp); \

295

(dstrxd) = (tmp); \

296

vmxnet3_RxDescToCPU((rxd), (tmp)); \

296

vmxnet3_RxDescToCPU((rxd), (tmp)); \

297

} while (0)

297

} while (0)

298

299

#else

299

#else

300

301

# define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)

301

# define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)

302

# define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)

302

# define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)

303

# define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)

303

# define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)

304

# define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)

304

# define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)

305

# define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)

305

# define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)

306

# define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)

306

# define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)

307

308

#endif /* __BIG_ENDIAN_BITFIELD */

308

#endif /* __BIG_ENDIAN_BITFIELD */

309

310

311

static void

311

static void

312

vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,

312

vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,

313

struct pci_dev *pdev)

313

struct pci_dev *pdev)

314

{

314

{

315

if (tbi->map_type == VMXNET3_MAP_SINGLE)

315

if (tbi->map_type == VMXNET3_MAP_SINGLE)

316

dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,

316

dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,

317

PCI_DMA_TODEVICE);

317

PCI_DMA_TODEVICE);

318

else if (tbi->map_type == VMXNET3_MAP_PAGE)

318

else if (tbi->map_type == VMXNET3_MAP_PAGE)

319

dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,

319

dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,

320

PCI_DMA_TODEVICE);

320

PCI_DMA_TODEVICE);

321

else

321

else

322

BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);

322

BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);

323

324

tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */

324

tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */

325

}

325

}

326

327

328

static int

328

static int

329

vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,

329

vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,

330

struct pci_dev *pdev, struct vmxnet3_adapter *adapter)

330

struct pci_dev *pdev, struct vmxnet3_adapter *adapter)

331

{

331

{

332

struct sk_buff *skb;

332

struct sk_buff *skb;

333

int entries = 0;

333

int entries = 0;

334

335

/* no out of order completion */

335

/* no out of order completion */

336

BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);

336

BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);

337

BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);

337

BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);

338

339

skb = tq->buf_info[eop_idx].skb;

339

skb = tq->buf_info[eop_idx].skb;

340

BUG_ON(skb == NULL);

340

BUG_ON(skb == NULL);

341

tq->buf_info[eop_idx].skb = NULL;

341

tq->buf_info[eop_idx].skb = NULL;

342

343

VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);

343

VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);

344

345

while (tq->tx_ring.next2comp != eop_idx) {

345

while (tq->tx_ring.next2comp != eop_idx) {

346

vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,

346

vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,

347

pdev);

347

pdev);

348

349

/* update next2comp w/o tx_lock. Since we are marking more,

349

/* update next2comp w/o tx_lock. Since we are marking more,

350

* instead of less, tx ring entries avail, the worst case is

350

* instead of less, tx ring entries avail, the worst case is

351

* that the tx routine incorrectly re-queues a pkt due to

351

* that the tx routine incorrectly re-queues a pkt due to

352

* insufficient tx ring entries.

352

* insufficient tx ring entries.

353

*/

353

*/

354

vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);

354

vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);

355

entries++;

355

entries++;

356

}

356

}

357

358

dev_kfree_skb_any(skb);

358

dev_kfree_skb_any(skb);

359

return entries;

359

return entries;

360

}

360

}

361

362

363

static int

363

static int

364

vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,

364

vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,

365

struct vmxnet3_adapter *adapter)

365

struct vmxnet3_adapter *adapter)

366

{

366

{

367

int completed = 0;

367

int completed = 0;

368

union Vmxnet3_GenericDesc *gdesc;

368

union Vmxnet3_GenericDesc *gdesc;

369

370

gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;

370

gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;

371

while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {

371

while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {

372

completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(

372

completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(

373

&gdesc->tcd), tq, adapter->pdev,

373

&gdesc->tcd), tq, adapter->pdev,

374

adapter);

374

adapter);

375

376

vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);

376

vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);

377

gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;

377

gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;

378

}

378

}

379

380

if (completed) {

380

if (completed) {

381

spin_lock(&tq->tx_lock);

381

spin_lock(&tq->tx_lock);

382

if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&

382

if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&

383

vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >

383

vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >

384

VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&

384

VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&

385

netif_carrier_ok(adapter->netdev))) {

385

netif_carrier_ok(adapter->netdev))) {

386

vmxnet3_tq_wake(tq, adapter);

386

vmxnet3_tq_wake(tq, adapter);

387

}

387

}

388

spin_unlock(&tq->tx_lock);

388

spin_unlock(&tq->tx_lock);

389

}

389

}

390

return completed;

390

return completed;

391

}

391

}

392

393

394

static void

394

static void

395

vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,

395

vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,

396

struct vmxnet3_adapter *adapter)

396

struct vmxnet3_adapter *adapter)

397

{

397

{

398

int i;

398

int i;

399

400

while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {

400

while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {

401

struct vmxnet3_tx_buf_info *tbi;

401

struct vmxnet3_tx_buf_info *tbi;

402

403

tbi = tq->buf_info + tq->tx_ring.next2comp;

403

tbi = tq->buf_info + tq->tx_ring.next2comp;

404

405

vmxnet3_unmap_tx_buf(tbi, adapter->pdev);

405

vmxnet3_unmap_tx_buf(tbi, adapter->pdev);

406

if (tbi->skb) {

406

if (tbi->skb) {

407

dev_kfree_skb_any(tbi->skb);

407

dev_kfree_skb_any(tbi->skb);

408

tbi->skb = NULL;

408

tbi->skb = NULL;

409

}

409

}

410

vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);

410

vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);

411

}

411

}

412

413

/* sanity check, verify all buffers are indeed unmapped and freed */

413

/* sanity check, verify all buffers are indeed unmapped and freed */

414

for (i = 0; i < tq->tx_ring.size; i++) {

414

for (i = 0; i < tq->tx_ring.size; i++) {

415

BUG_ON(tq->buf_info[i].skb != NULL ||

415

BUG_ON(tq->buf_info[i].skb != NULL ||

416

tq->buf_info[i].map_type != VMXNET3_MAP_NONE);

416

tq->buf_info[i].map_type != VMXNET3_MAP_NONE);

417

}

417

}

418

419

tq->tx_ring.gen = VMXNET3_INIT_GEN;

419

tq->tx_ring.gen = VMXNET3_INIT_GEN;

420

tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;

420

tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;

421

422

tq->comp_ring.gen = VMXNET3_INIT_GEN;

422

tq->comp_ring.gen = VMXNET3_INIT_GEN;

423

tq->comp_ring.next2proc = 0;

423

tq->comp_ring.next2proc = 0;

424

}

424

}

425

426

427

static void

427

static void

428

vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,

428

vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,

429

struct vmxnet3_adapter *adapter)

429

struct vmxnet3_adapter *adapter)

430

{

430

{

431

if (tq->tx_ring.base) {

431

if (tq->tx_ring.base) {

432

dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *

432

dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *

433

sizeof(struct Vmxnet3_TxDesc),

433

sizeof(struct Vmxnet3_TxDesc),

434

tq->tx_ring.base, tq->tx_ring.basePA);

434

tq->tx_ring.base, tq->tx_ring.basePA);

435

tq->tx_ring.base = NULL;

435

tq->tx_ring.base = NULL;

436

}

436

}

437

if (tq->data_ring.base) {

437

if (tq->data_ring.base) {

438

dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *

438

dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *

439

sizeof(struct Vmxnet3_TxDataDesc),

439

sizeof(struct Vmxnet3_TxDataDesc),

440

tq->data_ring.base, tq->data_ring.basePA);

440

tq->data_ring.base, tq->data_ring.basePA);

441

tq->data_ring.base = NULL;

441

tq->data_ring.base = NULL;

442

}

442

}

443

if (tq->comp_ring.base) {

443

if (tq->comp_ring.base) {

444

dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *

444

dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *

445

sizeof(struct Vmxnet3_TxCompDesc),

445

sizeof(struct Vmxnet3_TxCompDesc),

446

tq->comp_ring.base, tq->comp_ring.basePA);

446

tq->comp_ring.base, tq->comp_ring.basePA);

447

tq->comp_ring.base = NULL;

447

tq->comp_ring.base = NULL;

448

}

448

}

449

if (tq->buf_info) {

449

if (tq->buf_info) {

450

dma_free_coherent(&adapter->pdev->dev,

450

dma_free_coherent(&adapter->pdev->dev,

451

tq->tx_ring.size * sizeof(tq->buf_info[0]),

451

tq->tx_ring.size * sizeof(tq->buf_info[0]),

452

tq->buf_info, tq->buf_info_pa);

452

tq->buf_info, tq->buf_info_pa);

453

tq->buf_info = NULL;

453

tq->buf_info = NULL;

454

}

454

}

455

}

455

}

456

457

458

/* Destroy all tx queues */

458

/* Destroy all tx queues */

459

void

459

void

460

vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)

460

vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)

461

{

461

{

462

int i;

462

int i;

463

464

for (i = 0; i < adapter->num_tx_queues; i++)

464

for (i = 0; i < adapter->num_tx_queues; i++)

465

vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);

465

vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);

466

}

466

}

467

468

469

static void

469

static void

470

vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,

470

vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,

471

struct vmxnet3_adapter *adapter)

471

struct vmxnet3_adapter *adapter)

472

{

472

{

473

int i;

473

int i;

474

475

/* reset the tx ring contents to 0 and reset the tx ring states */

475

/* reset the tx ring contents to 0 and reset the tx ring states */

476

memset(tq->tx_ring.base, 0, tq->tx_ring.size *

476

memset(tq->tx_ring.base, 0, tq->tx_ring.size *

477

sizeof(struct Vmxnet3_TxDesc));

477

sizeof(struct Vmxnet3_TxDesc));

478

tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;

478

tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;

479

tq->tx_ring.gen = VMXNET3_INIT_GEN;

479

tq->tx_ring.gen = VMXNET3_INIT_GEN;

480

481

memset(tq->data_ring.base, 0, tq->data_ring.size *

481

memset(tq->data_ring.base, 0, tq->data_ring.size *

482

sizeof(struct Vmxnet3_TxDataDesc));

482

sizeof(struct Vmxnet3_TxDataDesc));

483

484

/* reset the tx comp ring contents to 0 and reset comp ring states */

484

/* reset the tx comp ring contents to 0 and reset comp ring states */

485

memset(tq->comp_ring.base, 0, tq->comp_ring.size *

485

memset(tq->comp_ring.base, 0, tq->comp_ring.size *

486

sizeof(struct Vmxnet3_TxCompDesc));

486

sizeof(struct Vmxnet3_TxCompDesc));

487

tq->comp_ring.next2proc = 0;

487

tq->comp_ring.next2proc = 0;

488

tq->comp_ring.gen = VMXNET3_INIT_GEN;

488

tq->comp_ring.gen = VMXNET3_INIT_GEN;

489

490

/* reset the bookkeeping data */

490

/* reset the bookkeeping data */

491

memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);

491

memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);

492

for (i = 0; i < tq->tx_ring.size; i++)

492

for (i = 0; i < tq->tx_ring.size; i++)

493

tq->buf_info[i].map_type = VMXNET3_MAP_NONE;

493

tq->buf_info[i].map_type = VMXNET3_MAP_NONE;

494

495

/* stats are not reset */

495

/* stats are not reset */

496

}

496

}

497

498

499

static int

499

static int

500

vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,

500

vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,

501

struct vmxnet3_adapter *adapter)

501

struct vmxnet3_adapter *adapter)

502

{

502

{

503

size_t sz;

503

size_t sz;

504

505

BUG_ON(tq->tx_ring.base || tq->data_ring.base ||

505

BUG_ON(tq->tx_ring.base || tq->data_ring.base ||

506

tq->comp_ring.base || tq->buf_info);

506

tq->comp_ring.base || tq->buf_info);

507

508

tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,

508

tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,

509

tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),

509

tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),

510

&tq->tx_ring.basePA, GFP_KERNEL);

510

&tq->tx_ring.basePA, GFP_KERNEL);

511

if (!tq->tx_ring.base) {

511

if (!tq->tx_ring.base) {

512

netdev_err(adapter->netdev, "failed to allocate tx ring\n");

512

netdev_err(adapter->netdev, "failed to allocate tx ring\n");

513

goto err;

513

goto err;

514

}

514

}

515

516

tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,

516

tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,

517

tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),

517

tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),

518

&tq->data_ring.basePA, GFP_KERNEL);

518

&tq->data_ring.basePA, GFP_KERNEL);

519

if (!tq->data_ring.base) {

519

if (!tq->data_ring.base) {

520

netdev_err(adapter->netdev, "failed to allocate data ring\n");

520

netdev_err(adapter->netdev, "failed to allocate data ring\n");

521

goto err;

521

goto err;

522

}

522

}

523

524

tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,

524

tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,

525

tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),

525

tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),

526

&tq->comp_ring.basePA, GFP_KERNEL);

526

&tq->comp_ring.basePA, GFP_KERNEL);

527

if (!tq->comp_ring.base) {

527

if (!tq->comp_ring.base) {

528

netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");

528

netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");

529

goto err;

529

goto err;

530

}

530

}

531

532

sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);

532

sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);

533

tq->buf_info = dma_zalloc_coherent(&adapter->pdev->dev, sz,

533

tq->buf_info = dma_zalloc_coherent(&adapter->pdev->dev, sz,

534

&tq->buf_info_pa, GFP_KERNEL);

534

&tq->buf_info_pa, GFP_KERNEL);

535

if (!tq->buf_info)

535

if (!tq->buf_info)

536

goto err;

536

goto err;

537

538

return 0;

538

return 0;

539

540

err:

540

err:

541

vmxnet3_tq_destroy(tq, adapter);

541

vmxnet3_tq_destroy(tq, adapter);

542

return -ENOMEM;

542

return -ENOMEM;

543

}

543

}

544

545

static void

545

static void

546

vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)

546

vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)

547

{

547

{

548

int i;

548

int i;

549

550

for (i = 0; i < adapter->num_tx_queues; i++)

550

for (i = 0; i < adapter->num_tx_queues; i++)

551

vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);

551

vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);

552

}

552

}

553

554

/*

554

/*

555

* starting from ring->next2fill, allocate rx buffers for the given ring

555

* starting from ring->next2fill, allocate rx buffers for the given ring

556

* of the rx queue and update the rx desc. stop after @num_to_alloc buffers

556

* of the rx queue and update the rx desc. stop after @num_to_alloc buffers

557

* are allocated or allocation fails

557

* are allocated or allocation fails

558

*/

558

*/

559

560

static int

560

static int

561

vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,

561

vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,

562

int num_to_alloc, struct vmxnet3_adapter *adapter)

562

int num_to_alloc, struct vmxnet3_adapter *adapter)

563

{

563

{

564

int num_allocated = 0;

564

int num_allocated = 0;

565

struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];

565

struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];

566

struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];

566

struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];

567

u32 val;

567

u32 val;

568

569

while (num_allocated <= num_to_alloc) {

569

while (num_allocated <= num_to_alloc) {

570

struct vmxnet3_rx_buf_info *rbi;

570

struct vmxnet3_rx_buf_info *rbi;

571

union Vmxnet3_GenericDesc *gd;

571

union Vmxnet3_GenericDesc *gd;

572

573

rbi = rbi_base + ring->next2fill;

573

rbi = rbi_base + ring->next2fill;

574

gd = ring->base + ring->next2fill;

574

gd = ring->base + ring->next2fill;

575

576

if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {

576

if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {

577

if (rbi->skb == NULL) {

577

if (rbi->skb == NULL) {

578

rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,

578

rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,

579

rbi->len,

579

rbi->len,

580

GFP_KERNEL);

580

GFP_KERNEL);

581

if (unlikely(rbi->skb == NULL)) {

581

if (unlikely(rbi->skb == NULL)) {

582

rq->stats.rx_buf_alloc_failure++;

582

rq->stats.rx_buf_alloc_failure++;

583

break;

583

break;

584

}

584

}

585

586

rbi->dma_addr = dma_map_single(

586

rbi->dma_addr = dma_map_single(

587

&adapter->pdev->dev,

587

&adapter->pdev->dev,

588

rbi->skb->data, rbi->len,

588

rbi->skb->data, rbi->len,

589

PCI_DMA_FROMDEVICE);

589

PCI_DMA_FROMDEVICE);

590

} else {

590

} else {

591

/* rx buffer skipped by the device */

591

/* rx buffer skipped by the device */

592

}

592

}

593

val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;

593

val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;

594

} else {

594

} else {

595

BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||

595

BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||

596

rbi->len != PAGE_SIZE);

596

rbi->len != PAGE_SIZE);

597

598

if (rbi->page == NULL) {

598

if (rbi->page == NULL) {

599

rbi->page = alloc_page(GFP_ATOMIC);

599

rbi->page = alloc_page(GFP_ATOMIC);

600

if (unlikely(rbi->page == NULL)) {

600

if (unlikely(rbi->page == NULL)) {

601

rq->stats.rx_buf_alloc_failure++;

601

rq->stats.rx_buf_alloc_failure++;

602

break;

602

break;

603

}

603

}

604

rbi->dma_addr = dma_map_page(

604

rbi->dma_addr = dma_map_page(

605

&adapter->pdev->dev,

605

&adapter->pdev->dev,

606

rbi->page, 0, PAGE_SIZE,

606

rbi->page, 0, PAGE_SIZE,

607

PCI_DMA_FROMDEVICE);

607

PCI_DMA_FROMDEVICE);

608

} else {

608

} else {

609

/* rx buffers skipped by the device */

609

/* rx buffers skipped by the device */

610

}

610

}

611

val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;

611

val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;

612

}

612

}

613

614

BUG_ON(rbi->dma_addr == 0);

614

BUG_ON(rbi->dma_addr == 0);

615

gd->rxd.addr = cpu_to_le64(rbi->dma_addr);

615

gd->rxd.addr = cpu_to_le64(rbi->dma_addr);

616

gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)

616

gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)

617

| val | rbi->len);

617

| val | rbi->len);

618

619

/* Fill the last buffer but dont mark it ready, or else the

619

/* Fill the last buffer but dont mark it ready, or else the

620

* device will think that the queue is full */

620

* device will think that the queue is full */

621

if (num_allocated == num_to_alloc)

621

if (num_allocated == num_to_alloc)

622

break;

622

break;

623

624

gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);

624

gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);

625

num_allocated++;

625

num_allocated++;

626

vmxnet3_cmd_ring_adv_next2fill(ring);

626

vmxnet3_cmd_ring_adv_next2fill(ring);

627

}

627

}

628

629

netdev_dbg(adapter->netdev,

629

netdev_dbg(adapter->netdev,

630

"alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",

630

"alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",

631

num_allocated, ring->next2fill, ring->next2comp);

631

num_allocated, ring->next2fill, ring->next2comp);

632

633

/* so that the device can distinguish a full ring and an empty ring */

633

/* so that the device can distinguish a full ring and an empty ring */

634

BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);

634

BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);

635

636

return num_allocated;

636

return num_allocated;

637

}

637

}

638

639

640

static void

640

static void

641

vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,

641

vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,

642

struct vmxnet3_rx_buf_info *rbi)

642

struct vmxnet3_rx_buf_info *rbi)

643

{

643

{

644

struct skb_frag_struct *frag = skb_shinfo(skb)->frags +

644

struct skb_frag_struct *frag = skb_shinfo(skb)->frags +

645

skb_shinfo(skb)->nr_frags;

645

skb_shinfo(skb)->nr_frags;

646

647

BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);

647

BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);

648

649

__skb_frag_set_page(frag, rbi->page);

649

__skb_frag_set_page(frag, rbi->page);

650

frag->page_offset = 0;

650

frag->page_offset = 0;

651

skb_frag_size_set(frag, rcd->len);

651

skb_frag_size_set(frag, rcd->len);

652

skb->data_len += rcd->len;

652

skb->data_len += rcd->len;

653

skb->truesize += PAGE_SIZE;

653

skb->truesize += PAGE_SIZE;

654

skb_shinfo(skb)->nr_frags++;

654

skb_shinfo(skb)->nr_frags++;

655

}

655

}

656

657

658

static void

658

static void

659

vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,

659

vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,

660

struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,

660

struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,

661

struct vmxnet3_adapter *adapter)

661

struct vmxnet3_adapter *adapter)

662

{

662

{

663

u32 dw2, len;

663

u32 dw2, len;

664

unsigned long buf_offset;

664

unsigned long buf_offset;

665

int i;

665

int i;

666

union Vmxnet3_GenericDesc *gdesc;

666

union Vmxnet3_GenericDesc *gdesc;

667

struct vmxnet3_tx_buf_info *tbi = NULL;

667

struct vmxnet3_tx_buf_info *tbi = NULL;

668

669

BUG_ON(ctx->copy_size > skb_headlen(skb));

669

BUG_ON(ctx->copy_size > skb_headlen(skb));

670

671

/* use the previous gen bit for the SOP desc */

671

/* use the previous gen bit for the SOP desc */

672

dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;

672

dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;

673

674

ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;

674

ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;

675

gdesc = ctx->sop_txd; /* both loops below can be skipped */

675

gdesc = ctx->sop_txd; /* both loops below can be skipped */

676

677

/* no need to map the buffer if headers are copied */

677

/* no need to map the buffer if headers are copied */

678

if (ctx->copy_size) {

678

if (ctx->copy_size) {

679

ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +

679

ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +

680

tq->tx_ring.next2fill *

680

tq->tx_ring.next2fill *

681

sizeof(struct Vmxnet3_TxDataDesc));

681

sizeof(struct Vmxnet3_TxDataDesc));

682

ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);

682

ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);

683

ctx->sop_txd->dword[3] = 0;

683

ctx->sop_txd->dword[3] = 0;

684

685

tbi = tq->buf_info + tq->tx_ring.next2fill;

685

tbi = tq->buf_info + tq->tx_ring.next2fill;

686

tbi->map_type = VMXNET3_MAP_NONE;

686

tbi->map_type = VMXNET3_MAP_NONE;

687

688

netdev_dbg(adapter->netdev,

688

netdev_dbg(adapter->netdev,

689

"txd[%u]: 0x%Lx 0x%x 0x%x\n",

689

"txd[%u]: 0x%Lx 0x%x 0x%x\n",

690

tq->tx_ring.next2fill,

690

tq->tx_ring.next2fill,

691

le64_to_cpu(ctx->sop_txd->txd.addr),

691

le64_to_cpu(ctx->sop_txd->txd.addr),

692

ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);

692

ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);

693

vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);

693

vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);

694

695

/* use the right gen for non-SOP desc */

695

/* use the right gen for non-SOP desc */

696

dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;

696

dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;

697

}

697

}

698

699

/* linear part can use multiple tx desc if it's big */

699

/* linear part can use multiple tx desc if it's big */

700

len = skb_headlen(skb) - ctx->copy_size;

700

len = skb_headlen(skb) - ctx->copy_size;

701

buf_offset = ctx->copy_size;

701

buf_offset = ctx->copy_size;

702

while (len) {

702

while (len) {

703

u32 buf_size;

703

u32 buf_size;

704

705

if (len < VMXNET3_MAX_TX_BUF_SIZE) {

705

if (len < VMXNET3_MAX_TX_BUF_SIZE) {

706

buf_size = len;

706

buf_size = len;

707

dw2 |= len;

707

dw2 |= len;

708

} else {

708

} else {

709

buf_size = VMXNET3_MAX_TX_BUF_SIZE;

709

buf_size = VMXNET3_MAX_TX_BUF_SIZE;

710

/* spec says that for TxDesc.len, 0 == 2^14 */

710

/* spec says that for TxDesc.len, 0 == 2^14 */

711

}

711

}

712

713

tbi = tq->buf_info + tq->tx_ring.next2fill;

713

tbi = tq->buf_info + tq->tx_ring.next2fill;

714

tbi->map_type = VMXNET3_MAP_SINGLE;

714

tbi->map_type = VMXNET3_MAP_SINGLE;

715

tbi->dma_addr = dma_map_single(&adapter->pdev->dev,

715

tbi->dma_addr = dma_map_single(&adapter->pdev->dev,

716

skb->data + buf_offset, buf_size,

716

skb->data + buf_offset, buf_size,

717

PCI_DMA_TODEVICE);

717

PCI_DMA_TODEVICE);

718

719

tbi->len = buf_size;

719

tbi->len = buf_size;

720

721

gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;

721

gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;

722

BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);

722

BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);

723

724

gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);

724

gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);

725

gdesc->dword[2] = cpu_to_le32(dw2);

725

gdesc->dword[2] = cpu_to_le32(dw2);

726

gdesc->dword[3] = 0;

726

gdesc->dword[3] = 0;

727

728

netdev_dbg(adapter->netdev,

728

netdev_dbg(adapter->netdev,

729

"txd[%u]: 0x%Lx 0x%x 0x%x\n",

729

"txd[%u]: 0x%Lx 0x%x 0x%x\n",

730

tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),

730

tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),

731

le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);

731

le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);

732

vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);

732

vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);

733

dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;

733

dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;

734

735

len -= buf_size;

735

len -= buf_size;

736

buf_offset += buf_size;

736

buf_offset += buf_size;

737

}

737

}

738

739

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {

739

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {

740

const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];

740

const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];

741

u32 buf_size;

741

u32 buf_size;

742

743

buf_offset = 0;

743

buf_offset = 0;

744

len = skb_frag_size(frag);

744

len = skb_frag_size(frag);

745

while (len) {

745

while (len) {

746

tbi = tq->buf_info + tq->tx_ring.next2fill;

746

tbi = tq->buf_info + tq->tx_ring.next2fill;

747

if (len < VMXNET3_MAX_TX_BUF_SIZE) {

747

if (len < VMXNET3_MAX_TX_BUF_SIZE) {

748

buf_size = len;

748

buf_size = len;

749

dw2 |= len;

749

dw2 |= len;

750

} else {

750

} else {

751

buf_size = VMXNET3_MAX_TX_BUF_SIZE;

751

buf_size = VMXNET3_MAX_TX_BUF_SIZE;

752

/* spec says that for TxDesc.len, 0 == 2^14 */

752

/* spec says that for TxDesc.len, 0 == 2^14 */

753

}

753

}

754

tbi->map_type = VMXNET3_MAP_PAGE;

754

tbi->map_type = VMXNET3_MAP_PAGE;

755

tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,

755

tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,

756

buf_offset, buf_size,

756

buf_offset, buf_size,

757

DMA_TO_DEVICE);

757

DMA_TO_DEVICE);

758

759

tbi->len = buf_size;

759

tbi->len = buf_size;

760

761

gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;

761

gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;

762

BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);

762

BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);

763

764

gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);

764

gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);

765

gdesc->dword[2] = cpu_to_le32(dw2);

765

gdesc->dword[2] = cpu_to_le32(dw2);

766

gdesc->dword[3] = 0;

766

gdesc->dword[3] = 0;

767

768

netdev_dbg(adapter->netdev,

768

netdev_dbg(adapter->netdev,

769

"txd[%u]: 0x%llu %u %u\n",

769

"txd[%u]: 0x%llu %u %u\n",

770

tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),

770

tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),

771

le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);

771

le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);

772

vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);

772

vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);

773

dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;

773

dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;

774

775

len -= buf_size;

775

len -= buf_size;

776

buf_offset += buf_size;

776

buf_offset += buf_size;

777

}

777

}

778

}

778

}

779

780

ctx->eop_txd = gdesc;

780

ctx->eop_txd = gdesc;

781

782

/* set the last buf_info for the pkt */

782

/* set the last buf_info for the pkt */

783

tbi->skb = skb;

783

tbi->skb = skb;

784

tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;

784

tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;

785

}

785

}

786

787

788

/* Init all tx queues */

788

/* Init all tx queues */

789

static void

789

static void

790

vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)

790

vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)

791

{

791

{

792

int i;

792

int i;

793

794

for (i = 0; i < adapter->num_tx_queues; i++)

794

for (i = 0; i < adapter->num_tx_queues; i++)

795

vmxnet3_tq_init(&adapter->tx_queue[i], adapter);

795

vmxnet3_tq_init(&adapter->tx_queue[i], adapter);

796

}

796

}

797

798

799

/*

799

/*

800

* parse and copy relevant protocol headers:

800

* parse and copy relevant protocol headers:

801

* For a tso pkt, relevant headers are L2/3/4 including options

801

* For a tso pkt, relevant headers are L2/3/4 including options

802

* For a pkt requesting csum offloading, they are L2/3 and may include L4

802

* For a pkt requesting csum offloading, they are L2/3 and may include L4

803

* if it's a TCP/UDP pkt

803

* if it's a TCP/UDP pkt

804

*

804

*

805

* Returns:

805

* Returns:

806

* -1: error happens during parsing

806

* -1: error happens during parsing

807

* 0: protocol headers parsed, but too big to be copied

807

* 0: protocol headers parsed, but too big to be copied

808

* 1: protocol headers parsed and copied

808

* 1: protocol headers parsed and copied

809

*

809

*

810

* Other effects:

810

* Other effects:

811

* 1. related *ctx fields are updated.

811

* 1. related *ctx fields are updated.

812

* 2. ctx->copy_size is # of bytes copied

812

* 2. ctx->copy_size is # of bytes copied

813

* 3. the portion copied is guaranteed to be in the linear part

813

* 3. the portion copied is guaranteed to be in the linear part

814

*

814

*

815

*/

815

*/

816

static int

816

static int

817

vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,

817

vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,

818

struct vmxnet3_tx_ctx *ctx,

818

struct vmxnet3_tx_ctx *ctx,

819

struct vmxnet3_adapter *adapter)

819

struct vmxnet3_adapter *adapter)

820

{

820

{

821

struct Vmxnet3_TxDataDesc *tdd;

821

struct Vmxnet3_TxDataDesc *tdd;

822

823

if (ctx->mss) { /* TSO */

823

if (ctx->mss) { /* TSO */

824

ctx->eth_ip_hdr_size = skb_transport_offset(skb);

824

ctx->eth_ip_hdr_size = skb_transport_offset(skb);

825

ctx->l4_hdr_size = tcp_hdrlen(skb);

825

ctx->l4_hdr_size = tcp_hdrlen(skb);

826

ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;

826

ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;

827

} else {

827

} else {

828

if (skb->ip_summed == CHECKSUM_PARTIAL) {

828

if (skb->ip_summed == CHECKSUM_PARTIAL) {

829

ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);

829

ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);

830

831

if (ctx->ipv4) {

831

if (ctx->ipv4) {

832

const struct iphdr *iph = ip_hdr(skb);

832

const struct iphdr *iph = ip_hdr(skb);

833

834

if (iph->protocol == IPPROTO_TCP)

834

if (iph->protocol == IPPROTO_TCP)

835

ctx->l4_hdr_size = tcp_hdrlen(skb);

835

ctx->l4_hdr_size = tcp_hdrlen(skb);

836

else if (iph->protocol == IPPROTO_UDP)

836

else if (iph->protocol == IPPROTO_UDP)

837

ctx->l4_hdr_size = sizeof(struct udphdr);

837

ctx->l4_hdr_size = sizeof(struct udphdr);

838

else

838

else

839

ctx->l4_hdr_size = 0;

839

ctx->l4_hdr_size = 0;

840

} else {

840

} else {

841

/* for simplicity, don't copy L4 headers */

841

/* for simplicity, don't copy L4 headers */

842

ctx->l4_hdr_size = 0;

842

ctx->l4_hdr_size = 0;

843

}

843

}

844

ctx->copy_size = min(ctx->eth_ip_hdr_size +

844

ctx->copy_size = min(ctx->eth_ip_hdr_size +

845

ctx->l4_hdr_size, skb->len);

845

ctx->l4_hdr_size, skb->len);

846

} else {

846

} else {

847

ctx->eth_ip_hdr_size = 0;

847

ctx->eth_ip_hdr_size = 0;

848

ctx->l4_hdr_size = 0;

848

ctx->l4_hdr_size = 0;

849

/* copy as much as allowed */

849

/* copy as much as allowed */

850

ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE

850

ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE

851

, skb_headlen(skb));

851

, skb_headlen(skb));

852

}

852

}

853

854

/* make sure headers are accessible directly */

854

/* make sure headers are accessible directly */

855

if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))

855

if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))

856

goto err;

856

goto err;

857

}

857

}

858

859

if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {

859

if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {

860

tq->stats.oversized_hdr++;

860

tq->stats.oversized_hdr++;

861

ctx->copy_size = 0;

861

ctx->copy_size = 0;

862

return 0;

862

return 0;

863

}

863

}

864

865

tdd = tq->data_ring.base + tq->tx_ring.next2fill;

865

tdd = tq->data_ring.base + tq->tx_ring.next2fill;

866

867

memcpy(tdd->data, skb->data, ctx->copy_size);

867

memcpy(tdd->data, skb->data, ctx->copy_size);

868

netdev_dbg(adapter->netdev,

868

netdev_dbg(adapter->netdev,

869

"copy %u bytes to dataRing[%u]\n",

869

"copy %u bytes to dataRing[%u]\n",

870

ctx->copy_size, tq->tx_ring.next2fill);

870

ctx->copy_size, tq->tx_ring.next2fill);

871

return 1;

871

return 1;

872

873

err:

873

err:

874

return -1;

874

return -1;

875

}

875

}

876

877

878

static void

878

static void

879

vmxnet3_prepare_tso(struct sk_buff *skb,

879

vmxnet3_prepare_tso(struct sk_buff *skb,

880

struct vmxnet3_tx_ctx *ctx)

880

struct vmxnet3_tx_ctx *ctx)

881

{

881

{

882

struct tcphdr *tcph = tcp_hdr(skb);

882

struct tcphdr *tcph = tcp_hdr(skb);

883

884

if (ctx->ipv4) {

884

if (ctx->ipv4) {

885

struct iphdr *iph = ip_hdr(skb);

885

struct iphdr *iph = ip_hdr(skb);

886

887

iph->check = 0;

887

iph->check = 0;

888

tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,

888

tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,

889

IPPROTO_TCP, 0);

889

IPPROTO_TCP, 0);

890

} else {

890

} else {

891

struct ipv6hdr *iph = ipv6_hdr(skb);

891

struct ipv6hdr *iph = ipv6_hdr(skb);

892

893

tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,

893

tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,

894

IPPROTO_TCP, 0);

894

IPPROTO_TCP, 0);

895

}

895

}

896

}

896

}

897

898

static int txd_estimate(const struct sk_buff *skb)

898

static int txd_estimate(const struct sk_buff *skb)

899

{

899

{

900

int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;

900

int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;

901

int i;

901

int i;

902

903

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {

903

for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {

904

const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];

904

const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];

905

906

count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));

906

count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));

907

}

907

}

908

return count;

908

return count;

909

}

909

}

910

911

/*

911

/*

912

* Transmits a pkt thru a given tq

912

* Transmits a pkt thru a given tq

913

* Returns:

913

* Returns:

914

* NETDEV_TX_OK: descriptors are setup successfully

914

* NETDEV_TX_OK: descriptors are setup successfully

915

* NETDEV_TX_OK: error occurred, the pkt is dropped

915

* NETDEV_TX_OK: error occurred, the pkt is dropped

916

* NETDEV_TX_BUSY: tx ring is full, queue is stopped

916

* NETDEV_TX_BUSY: tx ring is full, queue is stopped

917

*

917

*

918

* Side-effects:

918

* Side-effects:

919

* 1. tx ring may be changed

919

* 1. tx ring may be changed

920

* 2. tq stats may be updated accordingly

920

* 2. tq stats may be updated accordingly

921

* 3. shared->txNumDeferred may be updated

921

* 3. shared->txNumDeferred may be updated

922

*/

922

*/

923

924

static int

924

static int

925

vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,

925

vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,

926

struct vmxnet3_adapter *adapter, struct net_device *netdev)

926

struct vmxnet3_adapter *adapter, struct net_device *netdev)

927

{

927

{

928

int ret;

928

int ret;

929

u32 count;

929

u32 count;

930

unsigned long flags;

930

unsigned long flags;

931

struct vmxnet3_tx_ctx ctx;

931

struct vmxnet3_tx_ctx ctx;

932

union Vmxnet3_GenericDesc *gdesc;

932

union Vmxnet3_GenericDesc *gdesc;

933

#ifdef __BIG_ENDIAN_BITFIELD

933

#ifdef __BIG_ENDIAN_BITFIELD

934

/* Use temporary descriptor to avoid touching bits multiple times */

934

/* Use temporary descriptor to avoid touching bits multiple times */

935

union Vmxnet3_GenericDesc tempTxDesc;

935

union Vmxnet3_GenericDesc tempTxDesc;

936

#endif

936

#endif

937

938

count = txd_estimate(skb);

938

count = txd_estimate(skb);

939

940

ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));

940

ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));

941

942

ctx.mss = skb_shinfo(skb)->gso_size;

942

ctx.mss = skb_shinfo(skb)->gso_size;

943

if (ctx.mss) {

943

if (ctx.mss) {

944

if (skb_header_cloned(skb)) {

944

if (skb_header_cloned(skb)) {

945

if (unlikely(pskb_expand_head(skb, 0, 0,

945

if (unlikely(pskb_expand_head(skb, 0, 0,

946

GFP_ATOMIC) != 0)) {

946

GFP_ATOMIC) != 0)) {

947

tq->stats.drop_tso++;

947

tq->stats.drop_tso++;

948

goto drop_pkt;

948

goto drop_pkt;

949

}

949

}

950

tq->stats.copy_skb_header++;

950

tq->stats.copy_skb_header++;

951

}

951

}

952

vmxnet3_prepare_tso(skb, &ctx);

952

vmxnet3_prepare_tso(skb, &ctx);

953

} else {

953

} else {

954

if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {

954

if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {

955

956

/* non-tso pkts must not use more than

956

/* non-tso pkts must not use more than

957

* VMXNET3_MAX_TXD_PER_PKT entries

957

* VMXNET3_MAX_TXD_PER_PKT entries

958

*/

958

*/

959

if (skb_linearize(skb) != 0) {

959

if (skb_linearize(skb) != 0) {

960

tq->stats.drop_too_many_frags++;

960

tq->stats.drop_too_many_frags++;

961

goto drop_pkt;

961

goto drop_pkt;

962

}

962

}

963

tq->stats.linearized++;

963

tq->stats.linearized++;

964

965

/* recalculate the # of descriptors to use */

965

/* recalculate the # of descriptors to use */

966

count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;

966

count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;

967

}

967

}

968

}

968

}

969

970

spin_lock_irqsave(&tq->tx_lock, flags);

970

spin_lock_irqsave(&tq->tx_lock, flags);

971

972

if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {

972

if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {

973

tq->stats.tx_ring_full++;

973

tq->stats.tx_ring_full++;

974

netdev_dbg(adapter->netdev,

974

netdev_dbg(adapter->netdev,

975

"tx queue stopped on %s, next2comp %u"

975

"tx queue stopped on %s, next2comp %u"

976

" next2fill %u\n", adapter->netdev->name,

976

" next2fill %u\n", adapter->netdev->name,

977

tq->tx_ring.next2comp, tq->tx_ring.next2fill);

977

tq->tx_ring.next2comp, tq->tx_ring.next2fill);

978

979

vmxnet3_tq_stop(tq, adapter);

979

vmxnet3_tq_stop(tq, adapter);

980

spin_unlock_irqrestore(&tq->tx_lock, flags);

980

spin_unlock_irqrestore(&tq->tx_lock, flags);

981

return NETDEV_TX_BUSY;

981

return NETDEV_TX_BUSY;

982

}

982

}

983

984

985

ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);

985

ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);

986

if (ret >= 0) {

986

if (ret >= 0) {

987

BUG_ON(ret <= 0 && ctx.copy_size != 0);

987

BUG_ON(ret <= 0 && ctx.copy_size != 0);

988

/* hdrs parsed, check against other limits */

988

/* hdrs parsed, check against other limits */

989

if (ctx.mss) {

989

if (ctx.mss) {

990

if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >

990

if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >

991

VMXNET3_MAX_TX_BUF_SIZE)) {

991

VMXNET3_MAX_TX_BUF_SIZE)) {

992

goto hdr_too_big;

992

goto hdr_too_big;

993

}

993

}

994

} else {

994

} else {

995

if (skb->ip_summed == CHECKSUM_PARTIAL) {

995

if (skb->ip_summed == CHECKSUM_PARTIAL) {

996

if (unlikely(ctx.eth_ip_hdr_size +

996

if (unlikely(ctx.eth_ip_hdr_size +

997

skb->csum_offset >

997

skb->csum_offset >

998

VMXNET3_MAX_CSUM_OFFSET)) {

998

VMXNET3_MAX_CSUM_OFFSET)) {

999

goto hdr_too_big;

999

goto hdr_too_big;

1000

}

1000

}

1001

}

1001

}

1002

}

1002

}

1003

} else {

1003

} else {

1004

tq->stats.drop_hdr_inspect_err++;

1004

tq->stats.drop_hdr_inspect_err++;

1005

goto unlock_drop_pkt;

1005

goto unlock_drop_pkt;

1006

}

1006

}

1007

1008

/* fill tx descs related to addr & len */

1008

/* fill tx descs related to addr & len */

1009

vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);

1009

vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);

1010

1011

/* setup the EOP desc */

1011

/* setup the EOP desc */

1012

ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);

1012

ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);

1013

1014

/* setup the SOP desc */

1014

/* setup the SOP desc */

1015

#ifdef __BIG_ENDIAN_BITFIELD

1015

#ifdef __BIG_ENDIAN_BITFIELD

1016

gdesc = &tempTxDesc;

1016

gdesc = &tempTxDesc;

1017

gdesc->dword[2] = ctx.sop_txd->dword[2];

1017

gdesc->dword[2] = ctx.sop_txd->dword[2];

1018

gdesc->dword[3] = ctx.sop_txd->dword[3];

1018

gdesc->dword[3] = ctx.sop_txd->dword[3];

1019

#else

1019

#else

1020

gdesc = ctx.sop_txd;

1020

gdesc = ctx.sop_txd;

1021

#endif

1021

#endif

1022

if (ctx.mss) {

1022

if (ctx.mss) {

1023

gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;

1023

gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;

1024

gdesc->txd.om = VMXNET3_OM_TSO;

1024

gdesc->txd.om = VMXNET3_OM_TSO;

1025

gdesc->txd.msscof = ctx.mss;

1025

gdesc->txd.msscof = ctx.mss;

1026

le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -

1026

le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -

1027

gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);

1027

gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);

1028

} else {

1028

} else {

1029

if (skb->ip_summed == CHECKSUM_PARTIAL) {

1029

if (skb->ip_summed == CHECKSUM_PARTIAL) {

1030

gdesc->txd.hlen = ctx.eth_ip_hdr_size;

1030

gdesc->txd.hlen = ctx.eth_ip_hdr_size;

1031

gdesc->txd.om = VMXNET3_OM_CSUM;

1031

gdesc->txd.om = VMXNET3_OM_CSUM;

1032

gdesc->txd.msscof = ctx.eth_ip_hdr_size +

1032

gdesc->txd.msscof = ctx.eth_ip_hdr_size +

1033

skb->csum_offset;

1033

skb->csum_offset;

1034

} else {

1034

} else {

1035

gdesc->txd.om = 0;

1035

gdesc->txd.om = 0;

1036

gdesc->txd.msscof = 0;

1036

gdesc->txd.msscof = 0;

1037

}

1037

}

1038

le32_add_cpu(&tq->shared->txNumDeferred, 1);

1038

le32_add_cpu(&tq->shared->txNumDeferred, 1);

1039

}

1039

}

1040

1041

if (vlan_tx_tag_present(skb)) {

1041

if (vlan_tx_tag_present(skb)) {

1042

gdesc->txd.ti = 1;

1042

gdesc->txd.ti = 1;

1043

gdesc->txd.tci = vlan_tx_tag_get(skb);

1043

gdesc->txd.tci = vlan_tx_tag_get(skb);

1044

}

1044

}

1045

1046

/* finally flips the GEN bit of the SOP desc. */

1046

/* finally flips the GEN bit of the SOP desc. */

1047

gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^

1047

gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^

1048

VMXNET3_TXD_GEN);

1048

VMXNET3_TXD_GEN);

1049

#ifdef __BIG_ENDIAN_BITFIELD

1049

#ifdef __BIG_ENDIAN_BITFIELD

1050

/* Finished updating in bitfields of Tx Desc, so write them in original

1050

/* Finished updating in bitfields of Tx Desc, so write them in original

1051

* place.

1051

* place.

1052

*/

1052

*/

1053

vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,

1053

vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,

1054

(struct Vmxnet3_TxDesc *)ctx.sop_txd);

1054

(struct Vmxnet3_TxDesc *)ctx.sop_txd);

1055

gdesc = ctx.sop_txd;

1055

gdesc = ctx.sop_txd;

1056

#endif

1056

#endif

1057

netdev_dbg(adapter->netdev,

1057

netdev_dbg(adapter->netdev,

1058

"txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",

1058

"txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",

1059

(u32)(ctx.sop_txd -

1059

(u32)(ctx.sop_txd -

1060

tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),

1060

tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),

1061

le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));

1061

le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));

1062

1063

spin_unlock_irqrestore(&tq->tx_lock, flags);

1063

spin_unlock_irqrestore(&tq->tx_lock, flags);

1064

1065

if (le32_to_cpu(tq->shared->txNumDeferred) >=

1065

if (le32_to_cpu(tq->shared->txNumDeferred) >=

1066

le32_to_cpu(tq->shared->txThreshold)) {

1066

le32_to_cpu(tq->shared->txThreshold)) {

1067

tq->shared->txNumDeferred = 0;

1067

tq->shared->txNumDeferred = 0;

1068

VMXNET3_WRITE_BAR0_REG(adapter,

1068

VMXNET3_WRITE_BAR0_REG(adapter,

1069

VMXNET3_REG_TXPROD + tq->qid * 8,

1069

VMXNET3_REG_TXPROD + tq->qid * 8,

1070

tq->tx_ring.next2fill);

1070

tq->tx_ring.next2fill);

1071

}

1071

}

1072

1073

return NETDEV_TX_OK;

1073

return NETDEV_TX_OK;

1074

1075

hdr_too_big:

1075

hdr_too_big:

1076

tq->stats.drop_oversized_hdr++;

1076

tq->stats.drop_oversized_hdr++;

1077

unlock_drop_pkt:

1077

unlock_drop_pkt:

1078

spin_unlock_irqrestore(&tq->tx_lock, flags);

1078

spin_unlock_irqrestore(&tq->tx_lock, flags);

1079

drop_pkt:

1079

drop_pkt:

1080

tq->stats.drop_total++;

1080

tq->stats.drop_total++;

1081

dev_kfree_skb(skb);

1081

dev_kfree_skb(skb);

1082

return NETDEV_TX_OK;

1082

return NETDEV_TX_OK;

1083

}

1083

}

1084

1085

1086

static netdev_tx_t

1086

static netdev_tx_t

1087

vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)

1087

vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)

1088

{

1088

{

1089

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

1089

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

1090

1091

BUG_ON(skb->queue_mapping > adapter->num_tx_queues);

1091

BUG_ON(skb->queue_mapping > adapter->num_tx_queues);

1092

return vmxnet3_tq_xmit(skb,

1092

return vmxnet3_tq_xmit(skb,

1093

&adapter->tx_queue[skb->queue_mapping],

1093

&adapter->tx_queue[skb->queue_mapping],

1094

adapter, netdev);

1094

adapter, netdev);

1095

}

1095

}

1096

1097

1098

static void

1098

static void

1099

vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,

1099

vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,

1100

struct sk_buff *skb,

1100

struct sk_buff *skb,

1101

union Vmxnet3_GenericDesc *gdesc)

1101

union Vmxnet3_GenericDesc *gdesc)

1102

{

1102

{

1103

if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {

1103

if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {

1104

/* typical case: TCP/UDP over IP and both csums are correct */

1104

/* typical case: TCP/UDP over IP and both csums are correct */

1105

if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==

1105

if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==

1106

VMXNET3_RCD_CSUM_OK) {

1106

VMXNET3_RCD_CSUM_OK) {

1107

skb->ip_summed = CHECKSUM_UNNECESSARY;

1107

skb->ip_summed = CHECKSUM_UNNECESSARY;

1108

BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));

1108

BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));

1109

BUG_ON(!(gdesc->rcd.v4 || gdesc->rcd.v6));

1109

BUG_ON(!(gdesc->rcd.v4 || gdesc->rcd.v6));

1110

BUG_ON(gdesc->rcd.frg);

1110

BUG_ON(gdesc->rcd.frg);

1111

} else {

1111

} else {

1112

if (gdesc->rcd.csum) {

1112

if (gdesc->rcd.csum) {

1113

skb->csum = htons(gdesc->rcd.csum);

1113

skb->csum = htons(gdesc->rcd.csum);

1114

skb->ip_summed = CHECKSUM_PARTIAL;

1114

skb->ip_summed = CHECKSUM_PARTIAL;

1115

} else {

1115

} else {

1116

skb_checksum_none_assert(skb);

1116

skb_checksum_none_assert(skb);

1117

}

1117

}

1118

}

1118

}

1119

} else {

1119

} else {

1120

skb_checksum_none_assert(skb);

1120

skb_checksum_none_assert(skb);

1121

}

1121

}

1122

}

1122

}

1123

1124

1125

static void

1125

static void

1126

vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,

1126

vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,

1127

struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter)

1127

struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter)

1128

{

1128

{

1129

rq->stats.drop_err++;

1129

rq->stats.drop_err++;

1130

if (!rcd->fcs)

1130

if (!rcd->fcs)

1131

rq->stats.drop_fcs++;

1131

rq->stats.drop_fcs++;

1132

1133

rq->stats.drop_total++;

1133

rq->stats.drop_total++;

1134

1135

/*

1135

/*

1136

* We do not unmap and chain the rx buffer to the skb.

1136

* We do not unmap and chain the rx buffer to the skb.

1137

* We basically pretend this buffer is not used and will be recycled

1137

* We basically pretend this buffer is not used and will be recycled

1138

* by vmxnet3_rq_alloc_rx_buf()

1138

* by vmxnet3_rq_alloc_rx_buf()

1139

*/

1139

*/

1140

1141

/*

1141

/*

1142

* ctx->skb may be NULL if this is the first and the only one

1142

* ctx->skb may be NULL if this is the first and the only one

1143

* desc for the pkt

1143

* desc for the pkt

1144

*/

1144

*/

1145

if (ctx->skb)

1145

if (ctx->skb)

1146

dev_kfree_skb_irq(ctx->skb);

1146

dev_kfree_skb_irq(ctx->skb);

1147

1148

ctx->skb = NULL;

1148

ctx->skb = NULL;

1149

}

1149

}

1150

1151

1152

static int

1152

static int

1153

vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,

1153

vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,

1154

struct vmxnet3_adapter *adapter, int quota)

1154

struct vmxnet3_adapter *adapter, int quota)

1155

{

1155

{

1156

static const u32 rxprod_reg[2] = {

1156

static const u32 rxprod_reg[2] = {

1157

VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2

1157

VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2

1158

};

1158

};

1159

u32 num_rxd = 0;

1159

u32 num_rxd = 0;

1160

bool skip_page_frags = false;

1160

bool skip_page_frags = false;

1161

struct Vmxnet3_RxCompDesc *rcd;

1161

struct Vmxnet3_RxCompDesc *rcd;

1162

struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;

1162

struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;

1163

#ifdef __BIG_ENDIAN_BITFIELD

1163

#ifdef __BIG_ENDIAN_BITFIELD

1164

struct Vmxnet3_RxDesc rxCmdDesc;

1164

struct Vmxnet3_RxDesc rxCmdDesc;

1165

struct Vmxnet3_RxCompDesc rxComp;

1165

struct Vmxnet3_RxCompDesc rxComp;

1166

#endif

1166

#endif

1167

vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,

1167

vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,

1168

&rxComp);

1168

&rxComp);

1169

while (rcd->gen == rq->comp_ring.gen) {

1169

while (rcd->gen == rq->comp_ring.gen) {

1170

struct vmxnet3_rx_buf_info *rbi;

1170

struct vmxnet3_rx_buf_info *rbi;

1171

struct sk_buff *skb, *new_skb = NULL;

1171

struct sk_buff *skb, *new_skb = NULL;

1172

struct page *new_page = NULL;

1172

struct page *new_page = NULL;

1173

int num_to_alloc;

1173

int num_to_alloc;

1174

struct Vmxnet3_RxDesc *rxd;

1174

struct Vmxnet3_RxDesc *rxd;

1175

u32 idx, ring_idx;

1175

u32 idx, ring_idx;

1176

struct vmxnet3_cmd_ring *ring = NULL;

1176

struct vmxnet3_cmd_ring *ring = NULL;

1177

if (num_rxd >= quota) {

1177

if (num_rxd >= quota) {

1178

/* we may stop even before we see the EOP desc of

1178

/* we may stop even before we see the EOP desc of

1179

* the current pkt

1179

* the current pkt

1180

*/

1180

*/

1181

break;

1181

break;

1182

}

1182

}

1183

num_rxd++;

1183

num_rxd++;

1184

BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);

1184

BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);

1185

idx = rcd->rxdIdx;

1185

idx = rcd->rxdIdx;

1186

ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;

1186

ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;

1187

ring = rq->rx_ring + ring_idx;

1187

ring = rq->rx_ring + ring_idx;

1188

vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,

1188

vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,

1189

&rxCmdDesc);

1189

&rxCmdDesc);

1190

rbi = rq->buf_info[ring_idx] + idx;

1190

rbi = rq->buf_info[ring_idx] + idx;

1191

1192

BUG_ON(rxd->addr != rbi->dma_addr ||

1192

BUG_ON(rxd->addr != rbi->dma_addr ||

1193

rxd->len != rbi->len);

1193

rxd->len != rbi->len);

1194

1195

if (unlikely(rcd->eop && rcd->err)) {

1195

if (unlikely(rcd->eop && rcd->err)) {

1196

vmxnet3_rx_error(rq, rcd, ctx, adapter);

1196

vmxnet3_rx_error(rq, rcd, ctx, adapter);

1197

goto rcd_done;

1197

goto rcd_done;

1198

}

1198

}

1199

1200

if (rcd->sop) { /* first buf of the pkt */

1200

if (rcd->sop) { /* first buf of the pkt */

1201

BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||

1201

BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||

1202

rcd->rqID != rq->qid);

1202

rcd->rqID != rq->qid);

1203

1204

BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);

1204

BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);

1205

BUG_ON(ctx->skb != NULL || rbi->skb == NULL);

1205

BUG_ON(ctx->skb != NULL || rbi->skb == NULL);

1206

1207

if (unlikely(rcd->len == 0)) {

1207

if (unlikely(rcd->len == 0)) {

1208

/* Pretend the rx buffer is skipped. */

1208

/* Pretend the rx buffer is skipped. */

1209

BUG_ON(!(rcd->sop && rcd->eop));

1209

BUG_ON(!(rcd->sop && rcd->eop));

1210

netdev_dbg(adapter->netdev,

1210

netdev_dbg(adapter->netdev,

1211

"rxRing[%u][%u] 0 length\n",

1211

"rxRing[%u][%u] 0 length\n",

1212

ring_idx, idx);

1212

ring_idx, idx);

1213

goto rcd_done;

1213

goto rcd_done;

1214

}

1214

}

1215

1216

skip_page_frags = false;

1216

skip_page_frags = false;

1217

ctx->skb = rbi->skb;

1217

ctx->skb = rbi->skb;

1218

new_skb = netdev_alloc_skb_ip_align(adapter->netdev,

1218

new_skb = netdev_alloc_skb_ip_align(adapter->netdev,

1219

rbi->len);

1219

rbi->len);

1220

if (new_skb == NULL) {

1220

if (new_skb == NULL) {

1221

/* Skb allocation failed, do not handover this

1221

/* Skb allocation failed, do not handover this

1222

* skb to stack. Reuse it. Drop the existing pkt

1222

* skb to stack. Reuse it. Drop the existing pkt

1223

*/

1223

*/

1224

rq->stats.rx_buf_alloc_failure++;

1224

rq->stats.rx_buf_alloc_failure++;

1225

ctx->skb = NULL;

1225

ctx->skb = NULL;

1226

rq->stats.drop_total++;

1226

rq->stats.drop_total++;

1227

skip_page_frags = true;

1227

skip_page_frags = true;

1228

goto rcd_done;

1228

goto rcd_done;

1229

}

1229

}

1230

1231

dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,

1231

dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,

1232

rbi->len,

1232

rbi->len,

1233

PCI_DMA_FROMDEVICE);

1233

PCI_DMA_FROMDEVICE);

1234

1235

#ifdef VMXNET3_RSS

1235

#ifdef VMXNET3_RSS

1236

if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&

1236

if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&

1237

(adapter->netdev->features & NETIF_F_RXHASH))

1237

(adapter->netdev->features & NETIF_F_RXHASH))

1238

ctx->skb->rxhash = le32_to_cpu(rcd->rssHash);

1238

ctx->skb->rxhash = le32_to_cpu(rcd->rssHash);

1239

#endif

1239

#endif

1240

skb_put(ctx->skb, rcd->len);

1240

skb_put(ctx->skb, rcd->len);

1241

1242

/* Immediate refill */

1242

/* Immediate refill */

1243

rbi->skb = new_skb;

1243

rbi->skb = new_skb;

1244

rbi->dma_addr = dma_map_single(&adapter->pdev->dev,

1244

rbi->dma_addr = dma_map_single(&adapter->pdev->dev,

1245

rbi->skb->data, rbi->len,

1245

rbi->skb->data, rbi->len,

1246

PCI_DMA_FROMDEVICE);

1246

PCI_DMA_FROMDEVICE);

1247

rxd->addr = cpu_to_le64(rbi->dma_addr);

1247

rxd->addr = cpu_to_le64(rbi->dma_addr);

1248

rxd->len = rbi->len;

1248

rxd->len = rbi->len;

1249

1250

} else {

1250

} else {

1251

BUG_ON(ctx->skb == NULL && !skip_page_frags);

1251

BUG_ON(ctx->skb == NULL && !skip_page_frags);

1252

1253

/* non SOP buffer must be type 1 in most cases */

1253

/* non SOP buffer must be type 1 in most cases */

1254

BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);

1254

BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);

1255

BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);

1255

BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);

1256

1257

/* If an sop buffer was dropped, skip all

1257

/* If an sop buffer was dropped, skip all

1258

* following non-sop fragments. They will be reused.

1258

* following non-sop fragments. They will be reused.

1259

*/

1259

*/

1260

if (skip_page_frags)

1260

if (skip_page_frags)

1261

goto rcd_done;

1261

goto rcd_done;

1262

1263

new_page = alloc_page(GFP_ATOMIC);

1263

new_page = alloc_page(GFP_ATOMIC);

1264

if (unlikely(new_page == NULL)) {

1264

if (unlikely(new_page == NULL)) {

1265

/* Replacement page frag could not be allocated.

1265

/* Replacement page frag could not be allocated.

1266

* Reuse this page. Drop the pkt and free the

1266

* Reuse this page. Drop the pkt and free the

1267

* skb which contained this page as a frag. Skip

1267

* skb which contained this page as a frag. Skip

1268

* processing all the following non-sop frags.

1268

* processing all the following non-sop frags.

1269

*/

1269

*/

1270

rq->stats.rx_buf_alloc_failure++;

1270

rq->stats.rx_buf_alloc_failure++;

1271

dev_kfree_skb(ctx->skb);

1271

dev_kfree_skb(ctx->skb);

1272

ctx->skb = NULL;

1272

ctx->skb = NULL;

1273

skip_page_frags = true;

1273

skip_page_frags = true;

1274

goto rcd_done;

1274

goto rcd_done;

1275

}

1275

}

1276

1277

if (rcd->len) {

1277

if (rcd->len) {

1278

dma_unmap_page(&adapter->pdev->dev,

1278

dma_unmap_page(&adapter->pdev->dev,

1279

rbi->dma_addr, rbi->len,

1279

rbi->dma_addr, rbi->len,

1280

PCI_DMA_FROMDEVICE);

1280

PCI_DMA_FROMDEVICE);

1281

1282

vmxnet3_append_frag(ctx->skb, rcd, rbi);

1282

vmxnet3_append_frag(ctx->skb, rcd, rbi);

1283

}

1283

}

1284

1285

/* Immediate refill */

1285

/* Immediate refill */

1286

rbi->page = new_page;

1286

rbi->page = new_page;

1287

rbi->dma_addr = dma_map_page(&adapter->pdev->dev,

1287

rbi->dma_addr = dma_map_page(&adapter->pdev->dev,

1288

rbi->page,

1288

rbi->page,

1289

0, PAGE_SIZE,

1289

0, PAGE_SIZE,

1290

PCI_DMA_FROMDEVICE);

1290

PCI_DMA_FROMDEVICE);

1291

rxd->addr = cpu_to_le64(rbi->dma_addr);

1291

rxd->addr = cpu_to_le64(rbi->dma_addr);

1292

rxd->len = rbi->len;

1292

rxd->len = rbi->len;

1293

}

1293

}

1294

1295

1296

skb = ctx->skb;

1296

skb = ctx->skb;

1297

if (rcd->eop) {

1297

if (rcd->eop) {

1298

skb->len += skb->data_len;

1298

skb->len += skb->data_len;

1299

1300

vmxnet3_rx_csum(adapter, skb,

1300

vmxnet3_rx_csum(adapter, skb,

1301

(union Vmxnet3_GenericDesc *)rcd);

1301

(union Vmxnet3_GenericDesc *)rcd);

1302

skb->protocol = eth_type_trans(skb, adapter->netdev);

1302

skb->protocol = eth_type_trans(skb, adapter->netdev);

1303

1304

if (unlikely(rcd->ts))

1304

if (unlikely(rcd->ts))

1305

__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);

1305

__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);

1306

1307

if (adapter->netdev->features & NETIF_F_LRO)

1307

if (adapter->netdev->features & NETIF_F_LRO)

1308

netif_receive_skb(skb);

1308

netif_receive_skb(skb);

1309

else

1309

else

1310

napi_gro_receive(&rq->napi, skb);

1310

napi_gro_receive(&rq->napi, skb);

1311

1312

ctx->skb = NULL;

1312

ctx->skb = NULL;

1313

}

1313

}

1314

1315

rcd_done:

1315

rcd_done:

1316

/* device may have skipped some rx descs */

1316

/* device may have skipped some rx descs */

1317

ring->next2comp = idx;

1317

ring->next2comp = idx;

1318

num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);

1318

num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);

1319

ring = rq->rx_ring + ring_idx;

1319

ring = rq->rx_ring + ring_idx;

1320

while (num_to_alloc) {

1320

while (num_to_alloc) {

1321

vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,

1321

vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,

1322

&rxCmdDesc);

1322

&rxCmdDesc);

1323

BUG_ON(!rxd->addr);

1323

BUG_ON(!rxd->addr);

1324

1325

/* Recv desc is ready to be used by the device */

1325

/* Recv desc is ready to be used by the device */

1326

rxd->gen = ring->gen;

1326

rxd->gen = ring->gen;

1327

vmxnet3_cmd_ring_adv_next2fill(ring);

1327

vmxnet3_cmd_ring_adv_next2fill(ring);

1328

num_to_alloc--;

1328

num_to_alloc--;

1329

}

1329

}

1330

1331

/* if needed, update the register */

1331

/* if needed, update the register */

1332

if (unlikely(rq->shared->updateRxProd)) {

1332

if (unlikely(rq->shared->updateRxProd)) {

1333

VMXNET3_WRITE_BAR0_REG(adapter,

1333

VMXNET3_WRITE_BAR0_REG(adapter,

1334

rxprod_reg[ring_idx] + rq->qid * 8,

1334

rxprod_reg[ring_idx] + rq->qid * 8,

1335

ring->next2fill);

1335

ring->next2fill);

1336

}

1336

}

1337

1338

vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);

1338

vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);

1339

vmxnet3_getRxComp(rcd,

1339

vmxnet3_getRxComp(rcd,

1340

&rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);

1340

&rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);

1341

}

1341

}

1342

1343

return num_rxd;

1343

return num_rxd;

1344

}

1344

}

1345

1346

1347

static void

1347

static void

1348

vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,

1348

vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,

1349

struct vmxnet3_adapter *adapter)

1349

struct vmxnet3_adapter *adapter)

1350

{

1350

{

1351

u32 i, ring_idx;

1351

u32 i, ring_idx;

1352

struct Vmxnet3_RxDesc *rxd;

1352

struct Vmxnet3_RxDesc *rxd;

1353

1354

for (ring_idx = 0; ring_idx < 2; ring_idx++) {

1354

for (ring_idx = 0; ring_idx < 2; ring_idx++) {

1355

for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {

1355

for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {

1356

#ifdef __BIG_ENDIAN_BITFIELD

1356

#ifdef __BIG_ENDIAN_BITFIELD

1357

struct Vmxnet3_RxDesc rxDesc;

1357

struct Vmxnet3_RxDesc rxDesc;

1358

#endif

1358

#endif

1359

vmxnet3_getRxDesc(rxd,

1359

vmxnet3_getRxDesc(rxd,

1360

&rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);

1360

&rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);

1361

1362

if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&

1362

if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&

1363

rq->buf_info[ring_idx][i].skb) {

1363

rq->buf_info[ring_idx][i].skb) {

1364

dma_unmap_single(&adapter->pdev->dev, rxd->addr,

1364

dma_unmap_single(&adapter->pdev->dev, rxd->addr,

1365

rxd->len, PCI_DMA_FROMDEVICE);

1365

rxd->len, PCI_DMA_FROMDEVICE);

1366

dev_kfree_skb(rq->buf_info[ring_idx][i].skb);

1366

dev_kfree_skb(rq->buf_info[ring_idx][i].skb);

1367

rq->buf_info[ring_idx][i].skb = NULL;

1367

rq->buf_info[ring_idx][i].skb = NULL;

1368

} else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&

1368

} else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&

1369

rq->buf_info[ring_idx][i].page) {

1369

rq->buf_info[ring_idx][i].page) {

1370

dma_unmap_page(&adapter->pdev->dev, rxd->addr,

1370

dma_unmap_page(&adapter->pdev->dev, rxd->addr,

1371

rxd->len, PCI_DMA_FROMDEVICE);

1371

rxd->len, PCI_DMA_FROMDEVICE);

1372

put_page(rq->buf_info[ring_idx][i].page);

1372

put_page(rq->buf_info[ring_idx][i].page);

1373

rq->buf_info[ring_idx][i].page = NULL;

1373

rq->buf_info[ring_idx][i].page = NULL;

1374

}

1374

}

1375

}

1375

}

1376

1377

rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;

1377

rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;

1378

rq->rx_ring[ring_idx].next2fill =

1378

rq->rx_ring[ring_idx].next2fill =

1379

rq->rx_ring[ring_idx].next2comp = 0;

1379

rq->rx_ring[ring_idx].next2comp = 0;

1380

}

1380

}

1381

1382

rq->comp_ring.gen = VMXNET3_INIT_GEN;

1382

rq->comp_ring.gen = VMXNET3_INIT_GEN;

1383

rq->comp_ring.next2proc = 0;

1383

rq->comp_ring.next2proc = 0;

1384

}

1384

}

1385

1386

1387

static void

1387

static void

1388

vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)

1388

vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)

1389

{

1389

{

1390

int i;

1390

int i;

1391

1392

for (i = 0; i < adapter->num_rx_queues; i++)

1392

for (i = 0; i < adapter->num_rx_queues; i++)

1393

vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);

1393

vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);

1394

}

1394

}

1395

1396

1397

static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,

1397

static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,

1398

struct vmxnet3_adapter *adapter)

1398

struct vmxnet3_adapter *adapter)

1399

{

1399

{

1400

int i;

1400

int i;

1401

int j;

1401

int j;

1402

1403

/* all rx buffers must have already been freed */

1403

/* all rx buffers must have already been freed */

1404

for (i = 0; i < 2; i++) {

1404

for (i = 0; i < 2; i++) {

1405

if (rq->buf_info[i]) {

1405

if (rq->buf_info[i]) {

1406

for (j = 0; j < rq->rx_ring[i].size; j++)

1406

for (j = 0; j < rq->rx_ring[i].size; j++)

1407

BUG_ON(rq->buf_info[i][j].page != NULL);

1407

BUG_ON(rq->buf_info[i][j].page != NULL);

1408

}

1408

}

1409

}

1409

}

1410

1411

1412

for (i = 0; i < 2; i++) {

1412

for (i = 0; i < 2; i++) {

1413

if (rq->rx_ring[i].base) {

1413

if (rq->rx_ring[i].base) {

1414

dma_free_coherent(&adapter->pdev->dev,

1414

dma_free_coherent(&adapter->pdev->dev,

1415

rq->rx_ring[i].size

1415

rq->rx_ring[i].size

1416

* sizeof(struct Vmxnet3_RxDesc),

1416

* sizeof(struct Vmxnet3_RxDesc),

1417

rq->rx_ring[i].base,

1417

rq->rx_ring[i].base,

1418

rq->rx_ring[i].basePA);

1418

rq->rx_ring[i].basePA);

1419

rq->rx_ring[i].base = NULL;

1419

rq->rx_ring[i].base = NULL;

1420

}

1420

}

1421

rq->buf_info[i] = NULL;

1421

rq->buf_info[i] = NULL;

1422

}

1422

}

1423

1424

if (rq->comp_ring.base) {

1424

if (rq->comp_ring.base) {

1425

dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size

1425

dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size

1426

* sizeof(struct Vmxnet3_RxCompDesc),

1426

* sizeof(struct Vmxnet3_RxCompDesc),

1427

rq->comp_ring.base, rq->comp_ring.basePA);

1427

rq->comp_ring.base, rq->comp_ring.basePA);

1428

rq->comp_ring.base = NULL;

1428

rq->comp_ring.base = NULL;

1429

}

1429

}

1430

1431

if (rq->buf_info[0]) {

1431

if (rq->buf_info[0]) {

1432

size_t sz = sizeof(struct vmxnet3_rx_buf_info) *

1432

size_t sz = sizeof(struct vmxnet3_rx_buf_info) *

1433

(rq->rx_ring[0].size + rq->rx_ring[1].size);

1433

(rq->rx_ring[0].size + rq->rx_ring[1].size);

1434

dma_free_coherent(&adapter->pdev->dev, sz, rq->buf_info[0],

1434

dma_free_coherent(&adapter->pdev->dev, sz, rq->buf_info[0],

1435

rq->buf_info_pa);

1435

rq->buf_info_pa);

1436

}

1436

}

1437

}

1437

}

1438

1439

1440

static int

1440

static int

1441

vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,

1441

vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,

1442

struct vmxnet3_adapter *adapter)

1442

struct vmxnet3_adapter *adapter)

1443

{

1443

{

1444

int i;

1444

int i;

1445

1446

/* initialize buf_info */

1446

/* initialize buf_info */

1447

for (i = 0; i < rq->rx_ring[0].size; i++) {

1447

for (i = 0; i < rq->rx_ring[0].size; i++) {

1448

1449

/* 1st buf for a pkt is skbuff */

1449

/* 1st buf for a pkt is skbuff */

1450

if (i % adapter->rx_buf_per_pkt == 0) {

1450

if (i % adapter->rx_buf_per_pkt == 0) {

1451

rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;

1451

rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;

1452

rq->buf_info[0][i].len = adapter->skb_buf_size;

1452

rq->buf_info[0][i].len = adapter->skb_buf_size;

1453

} else { /* subsequent bufs for a pkt is frag */

1453

} else { /* subsequent bufs for a pkt is frag */

1454

rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;

1454

rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;

1455

rq->buf_info[0][i].len = PAGE_SIZE;

1455

rq->buf_info[0][i].len = PAGE_SIZE;

1456

}

1456

}

1457

}

1457

}

1458

for (i = 0; i < rq->rx_ring[1].size; i++) {

1458

for (i = 0; i < rq->rx_ring[1].size; i++) {

1459

rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;

1459

rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;

1460

rq->buf_info[1][i].len = PAGE_SIZE;

1460

rq->buf_info[1][i].len = PAGE_SIZE;

1461

}

1461

}

1462

1463

/* reset internal state and allocate buffers for both rings */

1463

/* reset internal state and allocate buffers for both rings */

1464

for (i = 0; i < 2; i++) {

1464

for (i = 0; i < 2; i++) {

1465

rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;

1465

rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;

1466

1467

memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *

1467

memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *

1468

sizeof(struct Vmxnet3_RxDesc));

1468

sizeof(struct Vmxnet3_RxDesc));

1469

rq->rx_ring[i].gen = VMXNET3_INIT_GEN;

1469

rq->rx_ring[i].gen = VMXNET3_INIT_GEN;

1470

}

1470

}

1471

if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,

1471

if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,

1472

adapter) == 0) {

1472

adapter) == 0) {

1473

/* at least has 1 rx buffer for the 1st ring */

1473

/* at least has 1 rx buffer for the 1st ring */

1474

return -ENOMEM;

1474

return -ENOMEM;

1475

}

1475

}

1476

vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);

1476

vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);

1477

1478

/* reset the comp ring */

1478

/* reset the comp ring */

1479

rq->comp_ring.next2proc = 0;

1479

rq->comp_ring.next2proc = 0;

1480

memset(rq->comp_ring.base, 0, rq->comp_ring.size *

1480

memset(rq->comp_ring.base, 0, rq->comp_ring.size *

1481

sizeof(struct Vmxnet3_RxCompDesc));

1481

sizeof(struct Vmxnet3_RxCompDesc));

1482

rq->comp_ring.gen = VMXNET3_INIT_GEN;

1482

rq->comp_ring.gen = VMXNET3_INIT_GEN;

1483

1484

/* reset rxctx */

1484

/* reset rxctx */

1485

rq->rx_ctx.skb = NULL;

1485

rq->rx_ctx.skb = NULL;

1486

1487

/* stats are not reset */

1487

/* stats are not reset */

1488

return 0;

1488

return 0;

1489

}

1489

}

1490

1491

1492

static int

1492

static int

1493

vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)

1493

vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)

1494

{

1494

{

1495

int i, err = 0;

1495

int i, err = 0;

1496

1497

for (i = 0; i < adapter->num_rx_queues; i++) {

1497

for (i = 0; i < adapter->num_rx_queues; i++) {

1498

err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);

1498

err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);

1499

if (unlikely(err)) {

1499

if (unlikely(err)) {

1500

dev_err(&adapter->netdev->dev, "%s: failed to "

1500

dev_err(&adapter->netdev->dev, "%s: failed to "

1501

"initialize rx queue%i\n",

1501

"initialize rx queue%i\n",

1502

adapter->netdev->name, i);

1502

adapter->netdev->name, i);

1503

break;

1503

break;

1504

}

1504

}

1505

}

1505

}

1506

return err;

1506

return err;

1507

1508

}

1508

}

1509

1510

1511

static int

1511

static int

1512

vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)

1512

vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)

1513

{

1513

{

1514

int i;

1514

int i;

1515

size_t sz;

1515

size_t sz;

1516

struct vmxnet3_rx_buf_info *bi;

1516

struct vmxnet3_rx_buf_info *bi;

1517

1518

for (i = 0; i < 2; i++) {

1518

for (i = 0; i < 2; i++) {

1519

1520

sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);

1520

sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);

1521

rq->rx_ring[i].base = dma_alloc_coherent(

1521

rq->rx_ring[i].base = dma_alloc_coherent(

1522

&adapter->pdev->dev, sz,

1522

&adapter->pdev->dev, sz,

1523

&rq->rx_ring[i].basePA,

1523

&rq->rx_ring[i].basePA,

1524

GFP_KERNEL);

1524

GFP_KERNEL);

1525

if (!rq->rx_ring[i].base) {

1525

if (!rq->rx_ring[i].base) {

1526

netdev_err(adapter->netdev,

1526

netdev_err(adapter->netdev,

1527

"failed to allocate rx ring %d\n", i);

1527

"failed to allocate rx ring %d\n", i);

1528

goto err;

1528

goto err;

1529

}

1529

}

1530

}

1530

}

1531

1532

sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);

1532

sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);

1533

rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,

1533

rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,

1534

&rq->comp_ring.basePA,

1534

&rq->comp_ring.basePA,

1535

GFP_KERNEL);

1535

GFP_KERNEL);

1536

if (!rq->comp_ring.base) {

1536

if (!rq->comp_ring.base) {

1537

netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");

1537

netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");

1538

goto err;

1538

goto err;

1539

}

1539

}

1540

1541

sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +

1541

sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +

1542

rq->rx_ring[1].size);

1542

rq->rx_ring[1].size);

1543

bi = dma_zalloc_coherent(&adapter->pdev->dev, sz, &rq->buf_info_pa,

1543

bi = dma_zalloc_coherent(&adapter->pdev->dev, sz, &rq->buf_info_pa,

1544

GFP_KERNEL);

1544

GFP_KERNEL);

1545

if (!bi)

1545

if (!bi)

1546

goto err;

1546

goto err;

1547

1548

rq->buf_info[0] = bi;

1548

rq->buf_info[0] = bi;

1549

rq->buf_info[1] = bi + rq->rx_ring[0].size;

1549

rq->buf_info[1] = bi + rq->rx_ring[0].size;

1550

1551

return 0;

1551

return 0;

1552

1553

err:

1553

err:

1554

vmxnet3_rq_destroy(rq, adapter);

1554

vmxnet3_rq_destroy(rq, adapter);

1555

return -ENOMEM;

1555

return -ENOMEM;

1556

}

1556

}

1557

1558

1559

static int

1559

static int

1560

vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)

1560

vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)

1561

{

1561

{

1562

int i, err = 0;

1562

int i, err = 0;

1563

1564

for (i = 0; i < adapter->num_rx_queues; i++) {

1564

for (i = 0; i < adapter->num_rx_queues; i++) {

1565

err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);

1565

err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);

1566

if (unlikely(err)) {

1566

if (unlikely(err)) {

1567

dev_err(&adapter->netdev->dev,

1567

dev_err(&adapter->netdev->dev,

1568

"%s: failed to create rx queue%i\n",

1568

"%s: failed to create rx queue%i\n",

1569

adapter->netdev->name, i);

1569

adapter->netdev->name, i);

1570

goto err_out;

1570

goto err_out;

1571

}

1571

}

1572

}

1572

}

1573

return err;

1573

return err;

1574

err_out:

1574

err_out:

1575

vmxnet3_rq_destroy_all(adapter);

1575

vmxnet3_rq_destroy_all(adapter);

1576

return err;

1576

return err;

1577

1578

}

1578

}

1579

1580

/* Multiple queue aware polling function for tx and rx */

1580

/* Multiple queue aware polling function for tx and rx */

1581

1582

static int

1582

static int

1583

vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)

1583

vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)

1584

{

1584

{

1585

int rcd_done = 0, i;

1585

int rcd_done = 0, i;

1586

if (unlikely(adapter->shared->ecr))

1586

if (unlikely(adapter->shared->ecr))

1587

vmxnet3_process_events(adapter);

1587

vmxnet3_process_events(adapter);

1588

for (i = 0; i < adapter->num_tx_queues; i++)

1588

for (i = 0; i < adapter->num_tx_queues; i++)

1589

vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);

1589

vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);

1590

1591

for (i = 0; i < adapter->num_rx_queues; i++)

1591

for (i = 0; i < adapter->num_rx_queues; i++)

1592

rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],

1592

rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],

1593

adapter, budget);

1593

adapter, budget);

1594

return rcd_done;

1594

return rcd_done;

1595

}

1595

}

1596

1597

1598

static int

1598

static int

1599

vmxnet3_poll(struct napi_struct *napi, int budget)

1599

vmxnet3_poll(struct napi_struct *napi, int budget)

1600

{

1600

{

1601

struct vmxnet3_rx_queue *rx_queue = container_of(napi,

1601

struct vmxnet3_rx_queue *rx_queue = container_of(napi,

1602

struct vmxnet3_rx_queue, napi);

1602

struct vmxnet3_rx_queue, napi);

1603

int rxd_done;

1603

int rxd_done;

1604

1605

rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);

1605

rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);

1606

1607

if (rxd_done < budget) {

1607

if (rxd_done < budget) {

1608

napi_complete(napi);

1608

napi_complete(napi);

1609

vmxnet3_enable_all_intrs(rx_queue->adapter);

1609

vmxnet3_enable_all_intrs(rx_queue->adapter);

1610

}

1610

}

1611

return rxd_done;

1611

return rxd_done;

1612

}

1612

}

1613

1614

/*

1614

/*

1615

* NAPI polling function for MSI-X mode with multiple Rx queues

1615

* NAPI polling function for MSI-X mode with multiple Rx queues

1616

* Returns the # of the NAPI credit consumed (# of rx descriptors processed)

1616

* Returns the # of the NAPI credit consumed (# of rx descriptors processed)

1617

*/

1617

*/

1618

1619

static int

1619

static int

1620

vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)

1620

vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)

1621

{

1621

{

1622

struct vmxnet3_rx_queue *rq = container_of(napi,

1622

struct vmxnet3_rx_queue *rq = container_of(napi,

1623

struct vmxnet3_rx_queue, napi);

1623

struct vmxnet3_rx_queue, napi);

1624

struct vmxnet3_adapter *adapter = rq->adapter;

1624

struct vmxnet3_adapter *adapter = rq->adapter;

1625

int rxd_done;

1625

int rxd_done;

1626

1627

/* When sharing interrupt with corresponding tx queue, process

1627

/* When sharing interrupt with corresponding tx queue, process

1628

* tx completions in that queue as well

1628

* tx completions in that queue as well

1629

*/

1629

*/

1630

if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {

1630

if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {

1631

struct vmxnet3_tx_queue *tq =

1631

struct vmxnet3_tx_queue *tq =

1632

&adapter->tx_queue[rq - adapter->rx_queue];

1632

&adapter->tx_queue[rq - adapter->rx_queue];

1633

vmxnet3_tq_tx_complete(tq, adapter);

1633

vmxnet3_tq_tx_complete(tq, adapter);

1634

}

1634

}

1635

1636

rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);

1636

rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);

1637

1638

if (rxd_done < budget) {

1638

if (rxd_done < budget) {

1639

napi_complete(napi);

1639

napi_complete(napi);

1640

vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);

1640

vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);

1641

}

1641

}

1642

return rxd_done;

1642

return rxd_done;

1643

}

1643

}

1644

1645

1646

#ifdef CONFIG_PCI_MSI

1646

#ifdef CONFIG_PCI_MSI

1647

1648

/*

1648

/*

1649

* Handle completion interrupts on tx queues

1649

* Handle completion interrupts on tx queues

1650

* Returns whether or not the intr is handled

1650

* Returns whether or not the intr is handled

1651

*/

1651

*/

1652

1653

static irqreturn_t

1653

static irqreturn_t

1654

vmxnet3_msix_tx(int irq, void *data)

1654

vmxnet3_msix_tx(int irq, void *data)

1655

{

1655

{

1656

struct vmxnet3_tx_queue *tq = data;

1656

struct vmxnet3_tx_queue *tq = data;

1657

struct vmxnet3_adapter *adapter = tq->adapter;

1657

struct vmxnet3_adapter *adapter = tq->adapter;

1658

1659

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1659

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1660

vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);

1660

vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);

1661

1662

/* Handle the case where only one irq is allocate for all tx queues */

1662

/* Handle the case where only one irq is allocate for all tx queues */

1663

if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {

1663

if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {

1664

int i;

1664

int i;

1665

for (i = 0; i < adapter->num_tx_queues; i++) {

1665

for (i = 0; i < adapter->num_tx_queues; i++) {

1666

struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];

1666

struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];

1667

vmxnet3_tq_tx_complete(txq, adapter);

1667

vmxnet3_tq_tx_complete(txq, adapter);

1668

}

1668

}

1669

} else {

1669

} else {

1670

vmxnet3_tq_tx_complete(tq, adapter);

1670

vmxnet3_tq_tx_complete(tq, adapter);

1671

}

1671

}

1672

vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);

1672

vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);

1673

1674

return IRQ_HANDLED;

1674

return IRQ_HANDLED;

1675

}

1675

}

1676

1677

1678

/*

1678

/*

1679

* Handle completion interrupts on rx queues. Returns whether or not the

1679

* Handle completion interrupts on rx queues. Returns whether or not the

1680

* intr is handled

1680

* intr is handled

1681

*/

1681

*/

1682

1683

static irqreturn_t

1683

static irqreturn_t

1684

vmxnet3_msix_rx(int irq, void *data)

1684

vmxnet3_msix_rx(int irq, void *data)

1685

{

1685

{

1686

struct vmxnet3_rx_queue *rq = data;

1686

struct vmxnet3_rx_queue *rq = data;

1687

struct vmxnet3_adapter *adapter = rq->adapter;

1687

struct vmxnet3_adapter *adapter = rq->adapter;

1688

1689

/* disable intr if needed */

1689

/* disable intr if needed */

1690

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1690

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1691

vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);

1691

vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);

1692

napi_schedule(&rq->napi);

1692

napi_schedule(&rq->napi);

1693

1694

return IRQ_HANDLED;

1694

return IRQ_HANDLED;

1695

}

1695

}

1696

1697

/*

1697

/*

1698

*----------------------------------------------------------------------------

1698

*----------------------------------------------------------------------------

1699

*

1699

*

1700

* vmxnet3_msix_event --

1700

* vmxnet3_msix_event --

1701

*

1701

*

1702

* vmxnet3 msix event intr handler

1702

* vmxnet3 msix event intr handler

1703

*

1703

*

1704

* Result:

1704

* Result:

1705

* whether or not the intr is handled

1705

* whether or not the intr is handled

1706

*

1706

*

1707

*----------------------------------------------------------------------------

1707

*----------------------------------------------------------------------------

1708

*/

1708

*/

1709

1710

static irqreturn_t

1710

static irqreturn_t

1711

vmxnet3_msix_event(int irq, void *data)

1711

vmxnet3_msix_event(int irq, void *data)

1712

{

1712

{

1713

struct net_device *dev = data;

1713

struct net_device *dev = data;

1714

struct vmxnet3_adapter *adapter = netdev_priv(dev);

1714

struct vmxnet3_adapter *adapter = netdev_priv(dev);

1715

1716

/* disable intr if needed */

1716

/* disable intr if needed */

1717

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1717

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1718

vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);

1718

vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);

1719

1720

if (adapter->shared->ecr)

1720

if (adapter->shared->ecr)

1721

vmxnet3_process_events(adapter);

1721

vmxnet3_process_events(adapter);

1722

1723

vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);

1723

vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);

1724

1725

return IRQ_HANDLED;

1725

return IRQ_HANDLED;

1726

}

1726

}

1727

1728

#endif /* CONFIG_PCI_MSI */

1728

#endif /* CONFIG_PCI_MSI */

1729

1730

1731

/* Interrupt handler for vmxnet3 */

1731

/* Interrupt handler for vmxnet3 */

1732

static irqreturn_t

1732

static irqreturn_t

1733

vmxnet3_intr(int irq, void *dev_id)

1733

vmxnet3_intr(int irq, void *dev_id)

1734

{

1734

{

1735

struct net_device *dev = dev_id;

1735

struct net_device *dev = dev_id;

1736

struct vmxnet3_adapter *adapter = netdev_priv(dev);

1736

struct vmxnet3_adapter *adapter = netdev_priv(dev);

1737

1738

if (adapter->intr.type == VMXNET3_IT_INTX) {

1738

if (adapter->intr.type == VMXNET3_IT_INTX) {

1739

u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);

1739

u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);

1740

if (unlikely(icr == 0))

1740

if (unlikely(icr == 0))

1741

/* not ours */

1741

/* not ours */

1742

return IRQ_NONE;

1742

return IRQ_NONE;

1743

}

1743

}

1744

1745

1746

/* disable intr if needed */

1746

/* disable intr if needed */

1747

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1747

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1748

vmxnet3_disable_all_intrs(adapter);

1748

vmxnet3_disable_all_intrs(adapter);

1749

1750

napi_schedule(&adapter->rx_queue[0].napi);

1750

napi_schedule(&adapter->rx_queue[0].napi);

1751

1752

return IRQ_HANDLED;

1752

return IRQ_HANDLED;

1753

}

1753

}

1754

1755

#ifdef CONFIG_NET_POLL_CONTROLLER

1755

#ifdef CONFIG_NET_POLL_CONTROLLER

1756

1757

/* netpoll callback. */

1757

/* netpoll callback. */

1758

static void

1758

static void

1759

vmxnet3_netpoll(struct net_device *netdev)

1759

vmxnet3_netpoll(struct net_device *netdev)

1760

{

1760

{

1761

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

1761

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

1762

1763

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1763

if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)

1764

vmxnet3_disable_all_intrs(adapter);

1764

vmxnet3_disable_all_intrs(adapter);

1765

1766

vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size);

1766

vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size);

1767

vmxnet3_enable_all_intrs(adapter);

1767

vmxnet3_enable_all_intrs(adapter);

1768

1769

}

1769

}

1770

#endif /* CONFIG_NET_POLL_CONTROLLER */

1770

#endif /* CONFIG_NET_POLL_CONTROLLER */

1771

1772

static int

1772

static int

1773

vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)

1773

vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)

1774

{

1774

{

1775

struct vmxnet3_intr *intr = &adapter->intr;

1775

struct vmxnet3_intr *intr = &adapter->intr;

1776

int err = 0, i;

1776

int err = 0, i;

1777

int vector = 0;

1777

int vector = 0;

1778

1779

#ifdef CONFIG_PCI_MSI

1779

#ifdef CONFIG_PCI_MSI

1780

if (adapter->intr.type == VMXNET3_IT_MSIX) {

1780

if (adapter->intr.type == VMXNET3_IT_MSIX) {

1781

for (i = 0; i < adapter->num_tx_queues; i++) {

1781

for (i = 0; i < adapter->num_tx_queues; i++) {

1782

if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {

1782

if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {

1783

sprintf(adapter->tx_queue[i].name, "%s-tx-%d",

1783

sprintf(adapter->tx_queue[i].name, "%s-tx-%d",

1784

adapter->netdev->name, vector);

1784

adapter->netdev->name, vector);

1785

err = request_irq(

1785

err = request_irq(

1786

intr->msix_entries[vector].vector,

1786

intr->msix_entries[vector].vector,

1787

vmxnet3_msix_tx, 0,

1787

vmxnet3_msix_tx, 0,

1788

adapter->tx_queue[i].name,

1788

adapter->tx_queue[i].name,

1789

&adapter->tx_queue[i]);

1789

&adapter->tx_queue[i]);

1790

} else {

1790

} else {

1791

sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",

1791

sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",

1792

adapter->netdev->name, vector);

1792

adapter->netdev->name, vector);

1793

}

1793

}

1794

if (err) {

1794

if (err) {

1795

dev_err(&adapter->netdev->dev,

1795

dev_err(&adapter->netdev->dev,

1796

"Failed to request irq for MSIX, %s, "

1796

"Failed to request irq for MSIX, %s, "

1797

"error %d\n",

1797

"error %d\n",

1798

adapter->tx_queue[i].name, err);

1798

adapter->tx_queue[i].name, err);

1799

return err;

1799

return err;

1800

}

1800

}

1801

1802

/* Handle the case where only 1 MSIx was allocated for

1802

/* Handle the case where only 1 MSIx was allocated for

1803

* all tx queues */

1803

* all tx queues */

1804

if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {

1804

if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {

1805

for (; i < adapter->num_tx_queues; i++)

1805

for (; i < adapter->num_tx_queues; i++)

1806

adapter->tx_queue[i].comp_ring.intr_idx

1806

adapter->tx_queue[i].comp_ring.intr_idx

1807

= vector;

1807

= vector;

1808

vector++;

1808

vector++;

1809

break;

1809

break;

1810

} else {

1810

} else {

1811

adapter->tx_queue[i].comp_ring.intr_idx

1811

adapter->tx_queue[i].comp_ring.intr_idx

1812

= vector++;

1812

= vector++;

1813

}

1813

}

1814

}

1814

}

1815

if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)

1815

if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)

1816

vector = 0;

1816

vector = 0;

1817

1818

for (i = 0; i < adapter->num_rx_queues; i++) {

1818

for (i = 0; i < adapter->num_rx_queues; i++) {

1819

if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)

1819

if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)

1820

sprintf(adapter->rx_queue[i].name, "%s-rx-%d",

1820

sprintf(adapter->rx_queue[i].name, "%s-rx-%d",

1821

adapter->netdev->name, vector);

1821

adapter->netdev->name, vector);

1822

else

1822

else

1823

sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",

1823

sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",

1824

adapter->netdev->name, vector);

1824

adapter->netdev->name, vector);

1825

err = request_irq(intr->msix_entries[vector].vector,

1825

err = request_irq(intr->msix_entries[vector].vector,

1826

vmxnet3_msix_rx, 0,

1826

vmxnet3_msix_rx, 0,

1827

adapter->rx_queue[i].name,

1827

adapter->rx_queue[i].name,

1828

&(adapter->rx_queue[i]));

1828

&(adapter->rx_queue[i]));

1829

if (err) {

1829

if (err) {

1830

netdev_err(adapter->netdev,

1830

netdev_err(adapter->netdev,

1831

"Failed to request irq for MSIX, "

1831

"Failed to request irq for MSIX, "

1832

"%s, error %d\n",

1832

"%s, error %d\n",

1833

adapter->rx_queue[i].name, err);

1833

adapter->rx_queue[i].name, err);

1834

return err;

1834

return err;

1835

}

1835

}

1836

1837

adapter->rx_queue[i].comp_ring.intr_idx = vector++;

1837

adapter->rx_queue[i].comp_ring.intr_idx = vector++;

1838

}

1838

}

1839

1840

sprintf(intr->event_msi_vector_name, "%s-event-%d",

1840

sprintf(intr->event_msi_vector_name, "%s-event-%d",

1841

adapter->netdev->name, vector);

1841

adapter->netdev->name, vector);

1842

err = request_irq(intr->msix_entries[vector].vector,

1842

err = request_irq(intr->msix_entries[vector].vector,

1843

vmxnet3_msix_event, 0,

1843

vmxnet3_msix_event, 0,

1844

intr->event_msi_vector_name, adapter->netdev);

1844

intr->event_msi_vector_name, adapter->netdev);

1845

intr->event_intr_idx = vector;

1845

intr->event_intr_idx = vector;

1846

1847

} else if (intr->type == VMXNET3_IT_MSI) {

1847

} else if (intr->type == VMXNET3_IT_MSI) {

1848

adapter->num_rx_queues = 1;

1848

adapter->num_rx_queues = 1;

1849

err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,

1849

err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,

1850

adapter->netdev->name, adapter->netdev);

1850

adapter->netdev->name, adapter->netdev);

1851

} else {

1851

} else {

1852

#endif

1852

#endif

1853

adapter->num_rx_queues = 1;

1853

adapter->num_rx_queues = 1;

1854

err = request_irq(adapter->pdev->irq, vmxnet3_intr,

1854

err = request_irq(adapter->pdev->irq, vmxnet3_intr,

1855

IRQF_SHARED, adapter->netdev->name,

1855

IRQF_SHARED, adapter->netdev->name,

1856

adapter->netdev);

1856

adapter->netdev);

1857

#ifdef CONFIG_PCI_MSI

1857

#ifdef CONFIG_PCI_MSI

1858

}

1858

}

1859

#endif

1859

#endif

1860

intr->num_intrs = vector + 1;

1860

intr->num_intrs = vector + 1;

1861

if (err) {

1861

if (err) {

1862

netdev_err(adapter->netdev,

1862

netdev_err(adapter->netdev,

1863

"Failed to request irq (intr type:%d), error %d\n",

1863

"Failed to request irq (intr type:%d), error %d\n",

1864

intr->type, err);

1864

intr->type, err);

1865

} else {

1865

} else {

1866

/* Number of rx queues will not change after this */

1866

/* Number of rx queues will not change after this */

1867

for (i = 0; i < adapter->num_rx_queues; i++) {

1867

for (i = 0; i < adapter->num_rx_queues; i++) {

1868

struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];

1868

struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];

1869

rq->qid = i;

1869

rq->qid = i;

1870

rq->qid2 = i + adapter->num_rx_queues;

1870

rq->qid2 = i + adapter->num_rx_queues;

1871

}

1871

}

1872

1873

1874

1875

/* init our intr settings */

1875

/* init our intr settings */

1876

for (i = 0; i < intr->num_intrs; i++)

1876

for (i = 0; i < intr->num_intrs; i++)

1877

intr->mod_levels[i] = UPT1_IML_ADAPTIVE;

1877

intr->mod_levels[i] = UPT1_IML_ADAPTIVE;

1878

if (adapter->intr.type != VMXNET3_IT_MSIX) {

1878

if (adapter->intr.type != VMXNET3_IT_MSIX) {

1879

adapter->intr.event_intr_idx = 0;

1879

adapter->intr.event_intr_idx = 0;

1880

for (i = 0; i < adapter->num_tx_queues; i++)

1880

for (i = 0; i < adapter->num_tx_queues; i++)

1881

adapter->tx_queue[i].comp_ring.intr_idx = 0;

1881

adapter->tx_queue[i].comp_ring.intr_idx = 0;

1882

adapter->rx_queue[0].comp_ring.intr_idx = 0;

1882

adapter->rx_queue[0].comp_ring.intr_idx = 0;

1883

}

1883

}

1884

1885

netdev_info(adapter->netdev,

1885

netdev_info(adapter->netdev,

1886

"intr type %u, mode %u, %u vectors allocated\n",

1886

"intr type %u, mode %u, %u vectors allocated\n",

1887

intr->type, intr->mask_mode, intr->num_intrs);

1887

intr->type, intr->mask_mode, intr->num_intrs);

1888

}

1888

}

1889

1890

return err;

1890

return err;

1891

}

1891

}

1892

1893

1894

static void

1894

static void

1895

vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)

1895

vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)

1896

{

1896

{

1897

struct vmxnet3_intr *intr = &adapter->intr;

1897

struct vmxnet3_intr *intr = &adapter->intr;

1898

BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);

1898

BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);

1899

1900

switch (intr->type) {

1900

switch (intr->type) {

1901

#ifdef CONFIG_PCI_MSI

1901

#ifdef CONFIG_PCI_MSI

1902

case VMXNET3_IT_MSIX:

1902

case VMXNET3_IT_MSIX:

1903

{

1903

{

1904

int i, vector = 0;

1904

int i, vector = 0;

1905

1906

if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {

1906

if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {

1907

for (i = 0; i < adapter->num_tx_queues; i++) {

1907

for (i = 0; i < adapter->num_tx_queues; i++) {

1908

free_irq(intr->msix_entries[vector++].vector,

1908

free_irq(intr->msix_entries[vector++].vector,

1909

&(adapter->tx_queue[i]));

1909

&(adapter->tx_queue[i]));

1910

if (adapter->share_intr == VMXNET3_INTR_TXSHARE)

1910

if (adapter->share_intr == VMXNET3_INTR_TXSHARE)

1911

break;

1911

break;

1912

}

1912

}

1913

}

1913

}

1914

1915

for (i = 0; i < adapter->num_rx_queues; i++) {

1915

for (i = 0; i < adapter->num_rx_queues; i++) {

1916

free_irq(intr->msix_entries[vector++].vector,

1916

free_irq(intr->msix_entries[vector++].vector,

1917

&(adapter->rx_queue[i]));

1917

&(adapter->rx_queue[i]));

1918

}

1918

}

1919

1920

free_irq(intr->msix_entries[vector].vector,

1920

free_irq(intr->msix_entries[vector].vector,

1921

adapter->netdev);

1921

adapter->netdev);

1922

BUG_ON(vector >= intr->num_intrs);

1922

BUG_ON(vector >= intr->num_intrs);

1923

break;

1923

break;

1924

}

1924

}

1925

#endif

1925

#endif

1926

case VMXNET3_IT_MSI:

1926

case VMXNET3_IT_MSI:

1927

free_irq(adapter->pdev->irq, adapter->netdev);

1927

free_irq(adapter->pdev->irq, adapter->netdev);

1928

break;

1928

break;

1929

case VMXNET3_IT_INTX:

1929

case VMXNET3_IT_INTX:

1930

free_irq(adapter->pdev->irq, adapter->netdev);

1930

free_irq(adapter->pdev->irq, adapter->netdev);

1931

break;

1931

break;

1932

default:

1932

default:

1933

BUG();

1933

BUG();

1934

}

1934

}

1935

}

1935

}

1936

1937

1938

static void

1938

static void

1939

vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)

1939

vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)

1940

{

1940

{

1941

u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;

1941

u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;

1942

u16 vid;

1942

u16 vid;

1943

1944

/* allow untagged pkts */

1944

/* allow untagged pkts */

1945

VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);

1945

VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);

1946

1947

for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)

1947

for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)

1948

VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);

1948

VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);

1949

}

1949

}

1950

1951

1952

static int

1952

static int

1953

vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)

1953

vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)

1954

{

1954

{

1955

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

1955

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

1956

1957

if (!(netdev->flags & IFF_PROMISC)) {

1957

if (!(netdev->flags & IFF_PROMISC)) {

1958

u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;

1958

u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;

1959

unsigned long flags;

1959

unsigned long flags;

1960

1961

VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);

1961

VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);

1962

spin_lock_irqsave(&adapter->cmd_lock, flags);

1962

spin_lock_irqsave(&adapter->cmd_lock, flags);

1963

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

1963

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

1964

VMXNET3_CMD_UPDATE_VLAN_FILTERS);

1964

VMXNET3_CMD_UPDATE_VLAN_FILTERS);

1965

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

1965

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

1966

}

1966

}

1967

1968

set_bit(vid, adapter->active_vlans);

1968

set_bit(vid, adapter->active_vlans);

1969

1970

return 0;

1970

return 0;

1971

}

1971

}

1972

1973

1974

static int

1974

static int

1975

vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)

1975

vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)

1976

{

1976

{

1977

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

1977

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

1978

1979

if (!(netdev->flags & IFF_PROMISC)) {

1979

if (!(netdev->flags & IFF_PROMISC)) {

1980

u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;

1980

u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;

1981

unsigned long flags;

1981

unsigned long flags;

1982

1983

VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);

1983

VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);

1984

spin_lock_irqsave(&adapter->cmd_lock, flags);

1984

spin_lock_irqsave(&adapter->cmd_lock, flags);

1985

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

1985

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

1986

VMXNET3_CMD_UPDATE_VLAN_FILTERS);

1986

VMXNET3_CMD_UPDATE_VLAN_FILTERS);

1987

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

1987

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

1988

}

1988

}

1989

1990

clear_bit(vid, adapter->active_vlans);

1990

clear_bit(vid, adapter->active_vlans);

1991

1992

return 0;

1992

return 0;

1993

}

1993

}

1994

1995

1996

static u8 *

1996

static u8 *

1997

vmxnet3_copy_mc(struct net_device *netdev)

1997

vmxnet3_copy_mc(struct net_device *netdev)

1998

{

1998

{

1999

u8 *buf = NULL;

1999

u8 *buf = NULL;

2000

u32 sz = netdev_mc_count(netdev) * ETH_ALEN;

2000

u32 sz = netdev_mc_count(netdev) * ETH_ALEN;

2001

2002

/* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */

2002

/* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */

2003

if (sz <= 0xffff) {

2003

if (sz <= 0xffff) {

2004

/* We may be called with BH disabled */

2004

/* We may be called with BH disabled */

2005

buf = kmalloc(sz, GFP_ATOMIC);

2005

buf = kmalloc(sz, GFP_ATOMIC);

2006

if (buf) {

2006

if (buf) {

2007

struct netdev_hw_addr *ha;

2007

struct netdev_hw_addr *ha;

2008

int i = 0;

2008

int i = 0;

2009

2010

netdev_for_each_mc_addr(ha, netdev)

2010

netdev_for_each_mc_addr(ha, netdev)

2011

memcpy(buf + i++ * ETH_ALEN, ha->addr,

2011

memcpy(buf + i++ * ETH_ALEN, ha->addr,

2012

ETH_ALEN);

2012

ETH_ALEN);

2013

}

2013

}

2014

}

2014

}

2015

return buf;

2015

return buf;

2016

}

2016

}

2017

2018

2019

static void

2019

static void

2020

vmxnet3_set_mc(struct net_device *netdev)

2020

vmxnet3_set_mc(struct net_device *netdev)

2021

{

2021

{

2022

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2022

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2023

unsigned long flags;

2023

unsigned long flags;

2024

struct Vmxnet3_RxFilterConf *rxConf =

2024

struct Vmxnet3_RxFilterConf *rxConf =

2025

&adapter->shared->devRead.rxFilterConf;

2025

&adapter->shared->devRead.rxFilterConf;

2026

u8 *new_table = NULL;

2026

u8 *new_table = NULL;

2027

dma_addr_t new_table_pa = 0;

2027

dma_addr_t new_table_pa = 0;

2028

u32 new_mode = VMXNET3_RXM_UCAST;

2028

u32 new_mode = VMXNET3_RXM_UCAST;

2029

2030

if (netdev->flags & IFF_PROMISC) {

2030

if (netdev->flags & IFF_PROMISC) {

2031

u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;

2031

u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;

2032

memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));

2032

memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));

2033

2034

new_mode |= VMXNET3_RXM_PROMISC;

2034

new_mode |= VMXNET3_RXM_PROMISC;

2035

} else {

2035

} else {

2036

vmxnet3_restore_vlan(adapter);

2036

vmxnet3_restore_vlan(adapter);

2037

}

2037

}

2038

2039

if (netdev->flags & IFF_BROADCAST)

2039

if (netdev->flags & IFF_BROADCAST)

2040

new_mode |= VMXNET3_RXM_BCAST;

2040

new_mode |= VMXNET3_RXM_BCAST;

2041

2042

if (netdev->flags & IFF_ALLMULTI)

2042

if (netdev->flags & IFF_ALLMULTI)

2043

new_mode |= VMXNET3_RXM_ALL_MULTI;

2043

new_mode |= VMXNET3_RXM_ALL_MULTI;

2044

else

2044

else

2045

if (!netdev_mc_empty(netdev)) {

2045

if (!netdev_mc_empty(netdev)) {

2046

new_table = vmxnet3_copy_mc(netdev);

2046

new_table = vmxnet3_copy_mc(netdev);

2047

if (new_table) {

2047

if (new_table) {

2048

new_mode |= VMXNET3_RXM_MCAST;

2048

new_mode |= VMXNET3_RXM_MCAST;

2049

rxConf->mfTableLen = cpu_to_le16(

2049

rxConf->mfTableLen = cpu_to_le16(

2050

netdev_mc_count(netdev) * ETH_ALEN);

2050

netdev_mc_count(netdev) * ETH_ALEN);

2051

new_table_pa = dma_map_single(

2051

new_table_pa = dma_map_single(

2052

&adapter->pdev->dev,

2052

&adapter->pdev->dev,

2053

new_table,

2053

new_table,

2054

rxConf->mfTableLen,

2054

rxConf->mfTableLen,

2055

PCI_DMA_TODEVICE);

2055

PCI_DMA_TODEVICE);

2056

rxConf->mfTablePA = cpu_to_le64(new_table_pa);

2056

rxConf->mfTablePA = cpu_to_le64(new_table_pa);

2057

} else {

2057

} else {

2058

netdev_info(netdev, "failed to copy mcast list"

2058

netdev_info(netdev, "failed to copy mcast list"

2059

", setting ALL_MULTI\n");

2059

", setting ALL_MULTI\n");

2060

new_mode |= VMXNET3_RXM_ALL_MULTI;

2060

new_mode |= VMXNET3_RXM_ALL_MULTI;

2061

}

2061

}

2062

}

2062

}

2063

2064

2065

if (!(new_mode & VMXNET3_RXM_MCAST)) {

2065

if (!(new_mode & VMXNET3_RXM_MCAST)) {

2066

rxConf->mfTableLen = 0;

2066

rxConf->mfTableLen = 0;

2067

rxConf->mfTablePA = 0;

2067

rxConf->mfTablePA = 0;

2068

}

2068

}

2069

2070

spin_lock_irqsave(&adapter->cmd_lock, flags);

2070

spin_lock_irqsave(&adapter->cmd_lock, flags);

2071

if (new_mode != rxConf->rxMode) {

2071

if (new_mode != rxConf->rxMode) {

2072

rxConf->rxMode = cpu_to_le32(new_mode);

2072

rxConf->rxMode = cpu_to_le32(new_mode);

2073

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2073

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2074

VMXNET3_CMD_UPDATE_RX_MODE);

2074

VMXNET3_CMD_UPDATE_RX_MODE);

2075

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2075

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2076

VMXNET3_CMD_UPDATE_VLAN_FILTERS);

2076

VMXNET3_CMD_UPDATE_VLAN_FILTERS);

2077

}

2077

}

2078

2079

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2079

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2080

VMXNET3_CMD_UPDATE_MAC_FILTERS);

2080

VMXNET3_CMD_UPDATE_MAC_FILTERS);

2081

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2081

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2082

2083

if (new_table) {

2083

if (new_table) {

2084

dma_unmap_single(&adapter->pdev->dev, new_table_pa,

2084

dma_unmap_single(&adapter->pdev->dev, new_table_pa,

2085

rxConf->mfTableLen, PCI_DMA_TODEVICE);

2085

rxConf->mfTableLen, PCI_DMA_TODEVICE);

2086

kfree(new_table);

2086

kfree(new_table);

2087

}

2087

}

2088

}

2088

}

2089

2090

void

2090

void

2091

vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)

2091

vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)

2092

{

2092

{

2093

int i;

2093

int i;

2094

2095

for (i = 0; i < adapter->num_rx_queues; i++)

2095

for (i = 0; i < adapter->num_rx_queues; i++)

2096

vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);

2096

vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);

2097

}

2097

}

2098

2099

2100

/*

2100

/*

2101

* Set up driver_shared based on settings in adapter.

2101

* Set up driver_shared based on settings in adapter.

2102

*/

2102

*/

2103

2104

static void

2104

static void

2105

vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)

2105

vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)

2106

{

2106

{

2107

struct Vmxnet3_DriverShared *shared = adapter->shared;

2107

struct Vmxnet3_DriverShared *shared = adapter->shared;

2108

struct Vmxnet3_DSDevRead *devRead = &shared->devRead;

2108

struct Vmxnet3_DSDevRead *devRead = &shared->devRead;

2109

struct Vmxnet3_TxQueueConf *tqc;

2109

struct Vmxnet3_TxQueueConf *tqc;

2110

struct Vmxnet3_RxQueueConf *rqc;

2110

struct Vmxnet3_RxQueueConf *rqc;

2111

int i;

2111

int i;

2112

2113

memset(shared, 0, sizeof(*shared));

2113

memset(shared, 0, sizeof(*shared));

2114

2115

/* driver settings */

2115

/* driver settings */

2116

shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);

2116

shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);

2117

devRead->misc.driverInfo.version = cpu_to_le32(

2117

devRead->misc.driverInfo.version = cpu_to_le32(

2118

VMXNET3_DRIVER_VERSION_NUM);

2118

VMXNET3_DRIVER_VERSION_NUM);

2119

devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?

2119

devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?

2120

VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);

2120

VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);

2121

devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;

2121

devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;

2122

*((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(

2122

*((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(

2123

*((u32 *)&devRead->misc.driverInfo.gos));

2123

*((u32 *)&devRead->misc.driverInfo.gos));

2124

devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);

2124

devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);

2125

devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);

2125

devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);

2126

2127

devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);

2127

devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);

2128

devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));

2128

devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));

2129

2130

/* set up feature flags */

2130

/* set up feature flags */

2131

if (adapter->netdev->features & NETIF_F_RXCSUM)

2131

if (adapter->netdev->features & NETIF_F_RXCSUM)

2132

devRead->misc.uptFeatures |= UPT1_F_RXCSUM;

2132

devRead->misc.uptFeatures |= UPT1_F_RXCSUM;

2133

2134

if (adapter->netdev->features & NETIF_F_LRO) {

2134

if (adapter->netdev->features & NETIF_F_LRO) {

2135

devRead->misc.uptFeatures |= UPT1_F_LRO;

2135

devRead->misc.uptFeatures |= UPT1_F_LRO;

2136

devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);

2136

devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);

2137

}

2137

}

2138

if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)

2138

if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)

2139

devRead->misc.uptFeatures |= UPT1_F_RXVLAN;

2139

devRead->misc.uptFeatures |= UPT1_F_RXVLAN;

2140

2141

devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);

2141

devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);

2142

devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);

2142

devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);

2143

devRead->misc.queueDescLen = cpu_to_le32(

2143

devRead->misc.queueDescLen = cpu_to_le32(

2144

adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +

2144

adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +

2145

adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));

2145

adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));

2146

2147

/* tx queue settings */

2147

/* tx queue settings */

2148

devRead->misc.numTxQueues = adapter->num_tx_queues;

2148

devRead->misc.numTxQueues = adapter->num_tx_queues;

2149

for (i = 0; i < adapter->num_tx_queues; i++) {

2149

for (i = 0; i < adapter->num_tx_queues; i++) {

2150

struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];

2150

struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];

2151

BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);

2151

BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);

2152

tqc = &adapter->tqd_start[i].conf;

2152

tqc = &adapter->tqd_start[i].conf;

2153

tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);

2153

tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA);

2154

tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);

2154

tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);

2155

tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);

2155

tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);

2156

tqc->ddPA = cpu_to_le64(tq->buf_info_pa);

2156

tqc->ddPA = cpu_to_le64(tq->buf_info_pa);

2157

tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);

2157

tqc->txRingSize = cpu_to_le32(tq->tx_ring.size);

2158

tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);

2158

tqc->dataRingSize = cpu_to_le32(tq->data_ring.size);

2159

tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);

2159

tqc->compRingSize = cpu_to_le32(tq->comp_ring.size);

2160

tqc->ddLen = cpu_to_le32(

2160

tqc->ddLen = cpu_to_le32(

2161

sizeof(struct vmxnet3_tx_buf_info) *

2161

sizeof(struct vmxnet3_tx_buf_info) *

2162

tqc->txRingSize);

2162

tqc->txRingSize);

2163

tqc->intrIdx = tq->comp_ring.intr_idx;

2163

tqc->intrIdx = tq->comp_ring.intr_idx;

2164

}

2164

}

2165

2166

/* rx queue settings */

2166

/* rx queue settings */

2167

devRead->misc.numRxQueues = adapter->num_rx_queues;

2167

devRead->misc.numRxQueues = adapter->num_rx_queues;

2168

for (i = 0; i < adapter->num_rx_queues; i++) {

2168

for (i = 0; i < adapter->num_rx_queues; i++) {

2169

struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];

2169

struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];

2170

rqc = &adapter->rqd_start[i].conf;

2170

rqc = &adapter->rqd_start[i].conf;

2171

rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);

2171

rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);

2172

rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);

2172

rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);

2173

rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);

2173

rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA);

2174

rqc->ddPA = cpu_to_le64(rq->buf_info_pa);

2174

rqc->ddPA = cpu_to_le64(rq->buf_info_pa);

2175

rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);

2175

rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size);

2176

rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);

2176

rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size);

2177

rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);

2177

rqc->compRingSize = cpu_to_le32(rq->comp_ring.size);

2178

rqc->ddLen = cpu_to_le32(

2178

rqc->ddLen = cpu_to_le32(

2179

sizeof(struct vmxnet3_rx_buf_info) *

2179

sizeof(struct vmxnet3_rx_buf_info) *

2180

(rqc->rxRingSize[0] +

2180

(rqc->rxRingSize[0] +

2181

rqc->rxRingSize[1]));

2181

rqc->rxRingSize[1]));

2182

rqc->intrIdx = rq->comp_ring.intr_idx;

2182

rqc->intrIdx = rq->comp_ring.intr_idx;

2183

}

2183

}

2184

2185

#ifdef VMXNET3_RSS

2185

#ifdef VMXNET3_RSS

2186

memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));

2186

memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));

2187

2188

if (adapter->rss) {

2188

if (adapter->rss) {

2189

struct UPT1_RSSConf *rssConf = adapter->rss_conf;

2189

struct UPT1_RSSConf *rssConf = adapter->rss_conf;

2190

static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {

2190

static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {

2191

0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,

2191

0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,

2192

0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,

2192

0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,

2193

0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,

2193

0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,

2194

0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,

2194

0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,

2195

0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,

2195

0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,

2196

};

2196

};

2197

2198

devRead->misc.uptFeatures |= UPT1_F_RSS;

2198

devRead->misc.uptFeatures |= UPT1_F_RSS;

2199

devRead->misc.numRxQueues = adapter->num_rx_queues;

2199

devRead->misc.numRxQueues = adapter->num_rx_queues;

2200

rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |

2200

rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |

2201

UPT1_RSS_HASH_TYPE_IPV4 |

2201

UPT1_RSS_HASH_TYPE_IPV4 |

2202

UPT1_RSS_HASH_TYPE_TCP_IPV6 |

2202

UPT1_RSS_HASH_TYPE_TCP_IPV6 |

2203

UPT1_RSS_HASH_TYPE_IPV6;

2203

UPT1_RSS_HASH_TYPE_IPV6;

2204

rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;

2204

rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;

2205

rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;

2205

rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;

2206

rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;

2206

rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;

2207

memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));

2207

memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));

2208

2209

for (i = 0; i < rssConf->indTableSize; i++)

2209

for (i = 0; i < rssConf->indTableSize; i++)

2210

rssConf->indTable[i] = ethtool_rxfh_indir_default(

2210

rssConf->indTable[i] = ethtool_rxfh_indir_default(

2211

i, adapter->num_rx_queues);

2211

i, adapter->num_rx_queues);

2212

2213

devRead->rssConfDesc.confVer = 1;

2213

devRead->rssConfDesc.confVer = 1;

2214

devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));

2214

devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));

2215

devRead->rssConfDesc.confPA =

2215

devRead->rssConfDesc.confPA =

2216

cpu_to_le64(adapter->rss_conf_pa);

2216

cpu_to_le64(adapter->rss_conf_pa);

2217

}

2217

}

2218

2219

#endif /* VMXNET3_RSS */

2219

#endif /* VMXNET3_RSS */

2220

2221

/* intr settings */

2221

/* intr settings */

2222

devRead->intrConf.autoMask = adapter->intr.mask_mode ==

2222

devRead->intrConf.autoMask = adapter->intr.mask_mode ==

2223

VMXNET3_IMM_AUTO;

2223

VMXNET3_IMM_AUTO;

2224

devRead->intrConf.numIntrs = adapter->intr.num_intrs;

2224

devRead->intrConf.numIntrs = adapter->intr.num_intrs;

2225

for (i = 0; i < adapter->intr.num_intrs; i++)

2225

for (i = 0; i < adapter->intr.num_intrs; i++)

2226

devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];

2226

devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];

2227

2228

devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;

2228

devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;

2229

devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);

2229

devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);

2230

2231

/* rx filter settings */

2231

/* rx filter settings */

2232

devRead->rxFilterConf.rxMode = 0;

2232

devRead->rxFilterConf.rxMode = 0;

2233

vmxnet3_restore_vlan(adapter);

2233

vmxnet3_restore_vlan(adapter);

2234

vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);

2234

vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);

2235

2236

/* the rest are already zeroed */

2236

/* the rest are already zeroed */

2237

}

2237

}

2238

2239

2240

int

2240

int

2241

vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)

2241

vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)

2242

{

2242

{

2243

int err, i;

2243

int err, i;

2244

u32 ret;

2244

u32 ret;

2245

unsigned long flags;

2245

unsigned long flags;

2246

2247

netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"

2247

netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"

2248

" ring sizes %u %u %u\n", adapter->netdev->name,

2248

" ring sizes %u %u %u\n", adapter->netdev->name,

2249

adapter->skb_buf_size, adapter->rx_buf_per_pkt,

2249

adapter->skb_buf_size, adapter->rx_buf_per_pkt,

2250

adapter->tx_queue[0].tx_ring.size,

2250

adapter->tx_queue[0].tx_ring.size,

2251

adapter->rx_queue[0].rx_ring[0].size,

2251

adapter->rx_queue[0].rx_ring[0].size,

2252

adapter->rx_queue[0].rx_ring[1].size);

2252

adapter->rx_queue[0].rx_ring[1].size);

2253

2254

vmxnet3_tq_init_all(adapter);

2254

vmxnet3_tq_init_all(adapter);

2255

err = vmxnet3_rq_init_all(adapter);

2255

err = vmxnet3_rq_init_all(adapter);

2256

if (err) {

2256

if (err) {

2257

netdev_err(adapter->netdev,

2257

netdev_err(adapter->netdev,

2258

"Failed to init rx queue error %d\n", err);

2258

"Failed to init rx queue error %d\n", err);

2259

goto rq_err;

2259

goto rq_err;

2260

}

2260

}

2261

2262

err = vmxnet3_request_irqs(adapter);

2262

err = vmxnet3_request_irqs(adapter);

2263

if (err) {

2263

if (err) {

2264

netdev_err(adapter->netdev,

2264

netdev_err(adapter->netdev,

2265

"Failed to setup irq for error %d\n", err);

2265

"Failed to setup irq for error %d\n", err);

2266

goto irq_err;

2266

goto irq_err;

2267

}

2267

}

2268

2269

vmxnet3_setup_driver_shared(adapter);

2269

vmxnet3_setup_driver_shared(adapter);

2270

2271

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(

2271

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(

2272

adapter->shared_pa));

2272

adapter->shared_pa));

2273

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(

2273

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(

2274

adapter->shared_pa));

2274

adapter->shared_pa));

2275

spin_lock_irqsave(&adapter->cmd_lock, flags);

2275

spin_lock_irqsave(&adapter->cmd_lock, flags);

2276

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2276

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2277

VMXNET3_CMD_ACTIVATE_DEV);

2277

VMXNET3_CMD_ACTIVATE_DEV);

2278

ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);

2278

ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);

2279

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2279

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2280

2281

if (ret != 0) {

2281

if (ret != 0) {

2282

netdev_err(adapter->netdev,

2282

netdev_err(adapter->netdev,

2283

"Failed to activate dev: error %u\n", ret);

2283

"Failed to activate dev: error %u\n", ret);

2284

err = -EINVAL;

2284

err = -EINVAL;

2285

goto activate_err;

2285

goto activate_err;

2286

}

2286

}

2287

2288

for (i = 0; i < adapter->num_rx_queues; i++) {

2288

for (i = 0; i < adapter->num_rx_queues; i++) {

2289

VMXNET3_WRITE_BAR0_REG(adapter,

2289

VMXNET3_WRITE_BAR0_REG(adapter,

2290

VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,

2290

VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,

2291

adapter->rx_queue[i].rx_ring[0].next2fill);

2291

adapter->rx_queue[i].rx_ring[0].next2fill);

2292

VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +

2292

VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +

2293

(i * VMXNET3_REG_ALIGN)),

2293

(i * VMXNET3_REG_ALIGN)),

2294

adapter->rx_queue[i].rx_ring[1].next2fill);

2294

adapter->rx_queue[i].rx_ring[1].next2fill);

2295

}

2295

}

2296

2297

/* Apply the rx filter settins last. */

2297

/* Apply the rx filter settins last. */

2298

vmxnet3_set_mc(adapter->netdev);

2298

vmxnet3_set_mc(adapter->netdev);

2299

2300

/*

2300

/*

2301

* Check link state when first activating device. It will start the

2301

* Check link state when first activating device. It will start the

2302

* tx queue if the link is up.

2302

* tx queue if the link is up.

2303

*/

2303

*/

2304

vmxnet3_check_link(adapter, true);

2304

vmxnet3_check_link(adapter, true);

2305

for (i = 0; i < adapter->num_rx_queues; i++)

2305

for (i = 0; i < adapter->num_rx_queues; i++)

2306

napi_enable(&adapter->rx_queue[i].napi);

2306

napi_enable(&adapter->rx_queue[i].napi);

2307

vmxnet3_enable_all_intrs(adapter);

2307

vmxnet3_enable_all_intrs(adapter);

2308

clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);

2308

clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);

2309

return 0;

2309

return 0;

2310

2311

activate_err:

2311

activate_err:

2312

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);

2312

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);

2313

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);

2313

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);

2314

vmxnet3_free_irqs(adapter);

2314

vmxnet3_free_irqs(adapter);

2315

irq_err:

2315

irq_err:

2316

rq_err:

2316

rq_err:

2317

/* free up buffers we allocated */

2317

/* free up buffers we allocated */

2318

vmxnet3_rq_cleanup_all(adapter);

2318

vmxnet3_rq_cleanup_all(adapter);

2319

return err;

2319

return err;

2320

}

2320

}

2321

2322

2323

void

2323

void

2324

vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)

2324

vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)

2325

{

2325

{

2326

unsigned long flags;

2326

unsigned long flags;

2327

spin_lock_irqsave(&adapter->cmd_lock, flags);

2327

spin_lock_irqsave(&adapter->cmd_lock, flags);

2328

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);

2328

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);

2329

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2329

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2330

}

2330

}

2331

2332

2333

int

2333

int

2334

vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)

2334

vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)

2335

{

2335

{

2336

int i;

2336

int i;

2337

unsigned long flags;

2337

unsigned long flags;

2338

if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))

2338

if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))

2339

return 0;

2339

return 0;

2340

2341

2342

spin_lock_irqsave(&adapter->cmd_lock, flags);

2342

spin_lock_irqsave(&adapter->cmd_lock, flags);

2343

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2343

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2344

VMXNET3_CMD_QUIESCE_DEV);

2344

VMXNET3_CMD_QUIESCE_DEV);

2345

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2345

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2346

vmxnet3_disable_all_intrs(adapter);

2346

vmxnet3_disable_all_intrs(adapter);

2347

2348

for (i = 0; i < adapter->num_rx_queues; i++)

2348

for (i = 0; i < adapter->num_rx_queues; i++)

2349

napi_disable(&adapter->rx_queue[i].napi);

2349

napi_disable(&adapter->rx_queue[i].napi);

2350

netif_tx_disable(adapter->netdev);

2350

netif_tx_disable(adapter->netdev);

2351

adapter->link_speed = 0;

2351

adapter->link_speed = 0;

2352

netif_carrier_off(adapter->netdev);

2352

netif_carrier_off(adapter->netdev);

2353

2354

vmxnet3_tq_cleanup_all(adapter);

2354

vmxnet3_tq_cleanup_all(adapter);

2355

vmxnet3_rq_cleanup_all(adapter);

2355

vmxnet3_rq_cleanup_all(adapter);

2356

vmxnet3_free_irqs(adapter);

2356

vmxnet3_free_irqs(adapter);

2357

return 0;

2357

return 0;

2358

}

2358

}

2359

2360

2361

static void

2361

static void

2362

vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)

2362

vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)

2363

{

2363

{

2364

u32 tmp;

2364

u32 tmp;

2365

2366

tmp = *(u32 *)mac;

2366

tmp = *(u32 *)mac;

2367

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);

2367

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);

2368

2369

tmp = (mac[5] << 8) | mac[4];

2369

tmp = (mac[5] << 8) | mac[4];

2370

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);

2370

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);

2371

}

2371

}

2372

2373

2374

static int

2374

static int

2375

vmxnet3_set_mac_addr(struct net_device *netdev, void *p)

2375

vmxnet3_set_mac_addr(struct net_device *netdev, void *p)

2376

{

2376

{

2377

struct sockaddr *addr = p;

2377

struct sockaddr *addr = p;

2378

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2378

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2379

2380

memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);

2380

memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);

2381

vmxnet3_write_mac_addr(adapter, addr->sa_data);

2381

vmxnet3_write_mac_addr(adapter, addr->sa_data);

2382

2383

return 0;

2383

return 0;

2384

}

2384

}

2385

2386

2387

/* ==================== initialization and cleanup routines ============ */

2387

/* ==================== initialization and cleanup routines ============ */

2388

2389

static int

2389

static int

2390

vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)

2390

vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)

2391

{

2391

{

2392

int err;

2392

int err;

2393

unsigned long mmio_start, mmio_len;

2393

unsigned long mmio_start, mmio_len;

2394

struct pci_dev *pdev = adapter->pdev;

2394

struct pci_dev *pdev = adapter->pdev;

2395

2396

err = pci_enable_device(pdev);

2396

err = pci_enable_device(pdev);

2397

if (err) {

2397

if (err) {

2398

dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);

2398

dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);

2399

return err;

2399

return err;

2400

}

2400

}

2401

2402

if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {

2402

if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {

2403

if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {

2403

if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {

2404

dev_err(&pdev->dev,

2404

dev_err(&pdev->dev,

2405

"pci_set_consistent_dma_mask failed\n");

2405

"pci_set_consistent_dma_mask failed\n");

2406

err = -EIO;

2406

err = -EIO;

2407

goto err_set_mask;

2407

goto err_set_mask;

2408

}

2408

}

2409

*dma64 = true;

2409

*dma64 = true;

2410

} else {

2410

} else {

2411

if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {

2411

if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {

2412

dev_err(&pdev->dev,

2412

dev_err(&pdev->dev,

2413

"pci_set_dma_mask failed\n");

2413

"pci_set_dma_mask failed\n");

2414

err = -EIO;

2414

err = -EIO;

2415

goto err_set_mask;

2415

goto err_set_mask;

2416

}

2416

}

2417

*dma64 = false;

2417

*dma64 = false;

2418

}

2418

}

2419

2420

err = pci_request_selected_regions(pdev, (1 << 2) - 1,

2420

err = pci_request_selected_regions(pdev, (1 << 2) - 1,

2421

vmxnet3_driver_name);

2421

vmxnet3_driver_name);

2422

if (err) {

2422

if (err) {

2423

dev_err(&pdev->dev,

2423

dev_err(&pdev->dev,

2424

"Failed to request region for adapter: error %d\n", err);

2424

"Failed to request region for adapter: error %d\n", err);

2425

goto err_set_mask;

2425

goto err_set_mask;

2426

}

2426

}

2427

2428

pci_set_master(pdev);

2428

pci_set_master(pdev);

2429

2430

mmio_start = pci_resource_start(pdev, 0);

2430

mmio_start = pci_resource_start(pdev, 0);

2431

mmio_len = pci_resource_len(pdev, 0);

2431

mmio_len = pci_resource_len(pdev, 0);

2432

adapter->hw_addr0 = ioremap(mmio_start, mmio_len);

2432

adapter->hw_addr0 = ioremap(mmio_start, mmio_len);

2433

if (!adapter->hw_addr0) {

2433

if (!adapter->hw_addr0) {

2434

dev_err(&pdev->dev, "Failed to map bar0\n");

2434

dev_err(&pdev->dev, "Failed to map bar0\n");

2435

err = -EIO;

2435

err = -EIO;

2436

goto err_ioremap;

2436

goto err_ioremap;

2437

}

2437

}

2438

2439

mmio_start = pci_resource_start(pdev, 1);

2439

mmio_start = pci_resource_start(pdev, 1);

2440

mmio_len = pci_resource_len(pdev, 1);

2440

mmio_len = pci_resource_len(pdev, 1);

2441

adapter->hw_addr1 = ioremap(mmio_start, mmio_len);

2441

adapter->hw_addr1 = ioremap(mmio_start, mmio_len);

2442

if (!adapter->hw_addr1) {

2442

if (!adapter->hw_addr1) {

2443

dev_err(&pdev->dev, "Failed to map bar1\n");

2443

dev_err(&pdev->dev, "Failed to map bar1\n");

2444

err = -EIO;

2444

err = -EIO;

2445

goto err_bar1;

2445

goto err_bar1;

2446

}

2446

}

2447

return 0;

2447

return 0;

2448

2449

err_bar1:

2449

err_bar1:

2450

iounmap(adapter->hw_addr0);

2450

iounmap(adapter->hw_addr0);

2451

err_ioremap:

2451

err_ioremap:

2452

pci_release_selected_regions(pdev, (1 << 2) - 1);

2452

pci_release_selected_regions(pdev, (1 << 2) - 1);

2453

err_set_mask:

2453

err_set_mask:

2454

pci_disable_device(pdev);

2454

pci_disable_device(pdev);

2455

return err;

2455

return err;

2456

}

2456

}

2457

2458

2459

static void

2459

static void

2460

vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)

2460

vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)

2461

{

2461

{

2462

BUG_ON(!adapter->pdev);

2462

BUG_ON(!adapter->pdev);

2463

2464

iounmap(adapter->hw_addr0);

2464

iounmap(adapter->hw_addr0);

2465

iounmap(adapter->hw_addr1);

2465

iounmap(adapter->hw_addr1);

2466

pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);

2466

pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);

2467

pci_disable_device(adapter->pdev);

2467

pci_disable_device(adapter->pdev);

2468

}

2468

}

2469

2470

2471

static void

2471

static void

2472

vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)

2472

vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)

2473

{

2473

{

2474

size_t sz, i, ring0_size, ring1_size, comp_size;

2474

size_t sz, i, ring0_size, ring1_size, comp_size;

2475

struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];

2475

struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0];

2476

2477

2478

if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -

2478

if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -

2479

VMXNET3_MAX_ETH_HDR_SIZE) {

2479

VMXNET3_MAX_ETH_HDR_SIZE) {

2480

adapter->skb_buf_size = adapter->netdev->mtu +

2480

adapter->skb_buf_size = adapter->netdev->mtu +

2481

VMXNET3_MAX_ETH_HDR_SIZE;

2481

VMXNET3_MAX_ETH_HDR_SIZE;

2482

if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)

2482

if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)

2483

adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;

2483

adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;

2484

2485

adapter->rx_buf_per_pkt = 1;

2485

adapter->rx_buf_per_pkt = 1;

2486

} else {

2486

} else {

2487

adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;

2487

adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;

2488

sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +

2488

sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +

2489

VMXNET3_MAX_ETH_HDR_SIZE;

2489

VMXNET3_MAX_ETH_HDR_SIZE;

2490

adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;

2490

adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;

2491

}

2491

}

2492

2493

/*

2493

/*

2494

* for simplicity, force the ring0 size to be a multiple of

2494

* for simplicity, force the ring0 size to be a multiple of

2495

* rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN

2495

* rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN

2496

*/

2496

*/

2497

sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;

2497

sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;

2498

ring0_size = adapter->rx_queue[0].rx_ring[0].size;

2498

ring0_size = adapter->rx_queue[0].rx_ring[0].size;

2499

ring0_size = (ring0_size + sz - 1) / sz * sz;

2499

ring0_size = (ring0_size + sz - 1) / sz * sz;

2500

ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /

2500

ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /

2501

sz * sz);

2501

sz * sz);

2502

ring1_size = adapter->rx_queue[0].rx_ring[1].size;

2502

ring1_size = adapter->rx_queue[0].rx_ring[1].size;

2503

comp_size = ring0_size + ring1_size;

2503

comp_size = ring0_size + ring1_size;

2504

2505

for (i = 0; i < adapter->num_rx_queues; i++) {

2505

for (i = 0; i < adapter->num_rx_queues; i++) {

2506

rq = &adapter->rx_queue[i];

2506

rq = &adapter->rx_queue[i];

2507

rq->rx_ring[0].size = ring0_size;

2507

rq->rx_ring[0].size = ring0_size;

2508

rq->rx_ring[1].size = ring1_size;

2508

rq->rx_ring[1].size = ring1_size;

2509

rq->comp_ring.size = comp_size;

2509

rq->comp_ring.size = comp_size;

2510

}

2510

}

2511

}

2511

}

2512

2513

2514

int

2514

int

2515

vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,

2515

vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,

2516

u32 rx_ring_size, u32 rx_ring2_size)

2516

u32 rx_ring_size, u32 rx_ring2_size)

2517

{

2517

{

2518

int err = 0, i;

2518

int err = 0, i;

2519

2520

for (i = 0; i < adapter->num_tx_queues; i++) {

2520

for (i = 0; i < adapter->num_tx_queues; i++) {

2521

struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];

2521

struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i];

2522

tq->tx_ring.size = tx_ring_size;

2522

tq->tx_ring.size = tx_ring_size;

2523

tq->data_ring.size = tx_ring_size;

2523

tq->data_ring.size = tx_ring_size;

2524

tq->comp_ring.size = tx_ring_size;

2524

tq->comp_ring.size = tx_ring_size;

2525

tq->shared = &adapter->tqd_start[i].ctrl;

2525

tq->shared = &adapter->tqd_start[i].ctrl;

2526

tq->stopped = true;

2526

tq->stopped = true;

2527

tq->adapter = adapter;

2527

tq->adapter = adapter;

2528

tq->qid = i;

2528

tq->qid = i;

2529

err = vmxnet3_tq_create(tq, adapter);

2529

err = vmxnet3_tq_create(tq, adapter);

2530

/*

2530

/*

2531

* Too late to change num_tx_queues. We cannot do away with

2531

* Too late to change num_tx_queues. We cannot do away with

2532

* lesser number of queues than what we asked for

2532

* lesser number of queues than what we asked for

2533

*/

2533

*/

2534

if (err)

2534

if (err)

2535

goto queue_err;

2535

goto queue_err;

2536

}

2536

}

2537

2538

adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;

2538

adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;

2539

adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;

2539

adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;

2540

vmxnet3_adjust_rx_ring_size(adapter);

2540

vmxnet3_adjust_rx_ring_size(adapter);

2541

for (i = 0; i < adapter->num_rx_queues; i++) {

2541

for (i = 0; i < adapter->num_rx_queues; i++) {

2542

struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];

2542

struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];

2543

/* qid and qid2 for rx queues will be assigned later when num

2543

/* qid and qid2 for rx queues will be assigned later when num

2544

* of rx queues is finalized after allocating intrs */

2544

* of rx queues is finalized after allocating intrs */

2545

rq->shared = &adapter->rqd_start[i].ctrl;

2545

rq->shared = &adapter->rqd_start[i].ctrl;

2546

rq->adapter = adapter;

2546

rq->adapter = adapter;

2547

err = vmxnet3_rq_create(rq, adapter);

2547

err = vmxnet3_rq_create(rq, adapter);

2548

if (err) {

2548

if (err) {

2549

if (i == 0) {

2549

if (i == 0) {

2550

netdev_err(adapter->netdev,

2550

netdev_err(adapter->netdev,

2551

"Could not allocate any rx queues. "

2551

"Could not allocate any rx queues. "

2552

"Aborting.\n");

2552

"Aborting.\n");

2553

goto queue_err;

2553

goto queue_err;

2554

} else {

2554

} else {

2555

netdev_info(adapter->netdev,

2555

netdev_info(adapter->netdev,

2556

"Number of rx queues changed "

2556

"Number of rx queues changed "

2557

"to : %d.\n", i);

2557

"to : %d.\n", i);

2558

adapter->num_rx_queues = i;

2558

adapter->num_rx_queues = i;

2559

err = 0;

2559

err = 0;

2560

break;

2560

break;

2561

}

2561

}

2562

}

2562

}

2563

}

2563

}

2564

return err;

2564

return err;

2565

queue_err:

2565

queue_err:

2566

vmxnet3_tq_destroy_all(adapter);

2566

vmxnet3_tq_destroy_all(adapter);

2567

return err;

2567

return err;

2568

}

2568

}

2569

2570

static int

2570

static int

2571

vmxnet3_open(struct net_device *netdev)

2571

vmxnet3_open(struct net_device *netdev)

2572

{

2572

{

2573

struct vmxnet3_adapter *adapter;

2573

struct vmxnet3_adapter *adapter;

2574

int err, i;

2574

int err, i;

2575

2576

adapter = netdev_priv(netdev);

2576

adapter = netdev_priv(netdev);

2577

2578

for (i = 0; i < adapter->num_tx_queues; i++)

2578

for (i = 0; i < adapter->num_tx_queues; i++)

2579

spin_lock_init(&adapter->tx_queue[i].tx_lock);

2579

spin_lock_init(&adapter->tx_queue[i].tx_lock);

2580

2581

err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,

2581

err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,

2582

VMXNET3_DEF_RX_RING_SIZE,

2582

VMXNET3_DEF_RX_RING_SIZE,

2583

VMXNET3_DEF_RX_RING_SIZE);

2583

VMXNET3_DEF_RX_RING_SIZE);

2584

if (err)

2584

if (err)

2585

goto queue_err;

2585

goto queue_err;

2586

2587

err = vmxnet3_activate_dev(adapter);

2587

err = vmxnet3_activate_dev(adapter);

2588

if (err)

2588

if (err)

2589

goto activate_err;

2589

goto activate_err;

2590

2591

return 0;

2591

return 0;

2592

2593

activate_err:

2593

activate_err:

2594

vmxnet3_rq_destroy_all(adapter);

2594

vmxnet3_rq_destroy_all(adapter);

2595

vmxnet3_tq_destroy_all(adapter);

2595

vmxnet3_tq_destroy_all(adapter);

2596

queue_err:

2596

queue_err:

2597

return err;

2597

return err;

2598

}

2598

}

2599

2600

2601

static int

2601

static int

2602

vmxnet3_close(struct net_device *netdev)

2602

vmxnet3_close(struct net_device *netdev)

2603

{

2603

{

2604

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2604

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2605

2606

/*

2606

/*

2607

* Reset_work may be in the middle of resetting the device, wait for its

2607

* Reset_work may be in the middle of resetting the device, wait for its

2608

* completion.

2608

* completion.

2609

*/

2609

*/

2610

while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))

2610

while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))

2611

msleep(1);

2611

msleep(1);

2612

2613

vmxnet3_quiesce_dev(adapter);

2613

vmxnet3_quiesce_dev(adapter);

2614

2615

vmxnet3_rq_destroy_all(adapter);

2615

vmxnet3_rq_destroy_all(adapter);

2616

vmxnet3_tq_destroy_all(adapter);

2616

vmxnet3_tq_destroy_all(adapter);

2617

2618

clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);

2618

clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);

2619

2620

2621

return 0;

2621

return 0;

2622

}

2622

}

2623

2624

2625

void

2625

void

2626

vmxnet3_force_close(struct vmxnet3_adapter *adapter)

2626

vmxnet3_force_close(struct vmxnet3_adapter *adapter)

2627

{

2627

{

2628

int i;

2628

int i;

2629

2630

/*

2630

/*

2631

* we must clear VMXNET3_STATE_BIT_RESETTING, otherwise

2631

* we must clear VMXNET3_STATE_BIT_RESETTING, otherwise

2632

* vmxnet3_close() will deadlock.

2632

* vmxnet3_close() will deadlock.

2633

*/

2633

*/

2634

BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));

2634

BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));

2635

2636

/* we need to enable NAPI, otherwise dev_close will deadlock */

2636

/* we need to enable NAPI, otherwise dev_close will deadlock */

2637

for (i = 0; i < adapter->num_rx_queues; i++)

2637

for (i = 0; i < adapter->num_rx_queues; i++)

2638

napi_enable(&adapter->rx_queue[i].napi);

2638

napi_enable(&adapter->rx_queue[i].napi);

2639

dev_close(adapter->netdev);

2639

dev_close(adapter->netdev);

2640

}

2640

}

2641

2642

2643

static int

2643

static int

2644

vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)

2644

vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)

2645

{

2645

{

2646

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2646

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2647

int err = 0;

2647

int err = 0;

2648

2649

if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)

2649

if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)

2650

return -EINVAL;

2650

return -EINVAL;

2651

2652

netdev->mtu = new_mtu;

2652

netdev->mtu = new_mtu;

2653

2654

/*

2654

/*

2655

* Reset_work may be in the middle of resetting the device, wait for its

2655

* Reset_work may be in the middle of resetting the device, wait for its

2656

* completion.

2656

* completion.

2657

*/

2657

*/

2658

while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))

2658

while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))

2659

msleep(1);

2659

msleep(1);

2660

2661

if (netif_running(netdev)) {

2661

if (netif_running(netdev)) {

2662

vmxnet3_quiesce_dev(adapter);

2662

vmxnet3_quiesce_dev(adapter);

2663

vmxnet3_reset_dev(adapter);

2663

vmxnet3_reset_dev(adapter);

2664

2665

/* we need to re-create the rx queue based on the new mtu */

2665

/* we need to re-create the rx queue based on the new mtu */

2666

vmxnet3_rq_destroy_all(adapter);

2666

vmxnet3_rq_destroy_all(adapter);

2667

vmxnet3_adjust_rx_ring_size(adapter);

2667

vmxnet3_adjust_rx_ring_size(adapter);

2668

err = vmxnet3_rq_create_all(adapter);

2668

err = vmxnet3_rq_create_all(adapter);

2669

if (err) {

2669

if (err) {

2670

netdev_err(netdev,

2670

netdev_err(netdev,

2671

"failed to re-create rx queues, "

2671

"failed to re-create rx queues, "

2672

" error %d. Closing it.\n", err);

2672

" error %d. Closing it.\n", err);

2673

goto out;

2673

goto out;

2674

}

2674

}

2675

2676

err = vmxnet3_activate_dev(adapter);

2676

err = vmxnet3_activate_dev(adapter);

2677

if (err) {

2677

if (err) {

2678

netdev_err(netdev,

2678

netdev_err(netdev,

2679

"failed to re-activate, error %d. "

2679

"failed to re-activate, error %d. "

2680

"Closing it\n", err);

2680

"Closing it\n", err);

2681

goto out;

2681

goto out;

2682

}

2682

}

2683

}

2683

}

2684

2685

out:

2685

out:

2686

clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);

2686

clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);

2687

if (err)

2687

if (err)

2688

vmxnet3_force_close(adapter);

2688

vmxnet3_force_close(adapter);

2689

2690

return err;

2690

return err;

2691

}

2691

}

2692

2693

2694

static void

2694

static void

2695

vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)

2695

vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)

2696

{

2696

{

2697

struct net_device *netdev = adapter->netdev;

2697

struct net_device *netdev = adapter->netdev;

2698

2699

netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |

2699

netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |

2700

NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |

2700

NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |

2701

NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |

2701

NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |

2702

NETIF_F_LRO;

2702

NETIF_F_LRO;

2703

if (dma64)

2703

if (dma64)

2704

netdev->hw_features |= NETIF_F_HIGHDMA;

2704

netdev->hw_features |= NETIF_F_HIGHDMA;

2705

netdev->vlan_features = netdev->hw_features &

2705

netdev->vlan_features = netdev->hw_features &

2706

~(NETIF_F_HW_VLAN_CTAG_TX |

2706

~(NETIF_F_HW_VLAN_CTAG_TX |

2707

NETIF_F_HW_VLAN_CTAG_RX);

2707

NETIF_F_HW_VLAN_CTAG_RX);

2708

netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;

2708

netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;

2709

}

2709

}

2710

2711

2712

static void

2712

static void

2713

vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)

2713

vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)

2714

{

2714

{

2715

u32 tmp;

2715

u32 tmp;

2716

2717

tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);

2717

tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);

2718

*(u32 *)mac = tmp;

2718

*(u32 *)mac = tmp;

2719

2720

tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);

2720

tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);

2721

mac[4] = tmp & 0xff;

2721

mac[4] = tmp & 0xff;

2722

mac[5] = (tmp >> 8) & 0xff;

2722

mac[5] = (tmp >> 8) & 0xff;

2723

}

2723

}

2724

2725

#ifdef CONFIG_PCI_MSI

2725

#ifdef CONFIG_PCI_MSI

2726

2727

/*

2727

/*

2728

* Enable MSIx vectors.

2728

* Enable MSIx vectors.

2729

* Returns :

2729

* Returns :

2730

* 0 on successful enabling of required vectors,

2730

* 0 on successful enabling of required vectors,

2731

* VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required

2731

* VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required

2732

* could be enabled.

2732

* could be enabled.

2733

* number of vectors which can be enabled otherwise (this number is smaller

2733

* number of vectors which can be enabled otherwise (this number is smaller

2734

* than VMXNET3_LINUX_MIN_MSIX_VECT)

2734

* than VMXNET3_LINUX_MIN_MSIX_VECT)

2735

*/

2735

*/

2736

2737

static int

2737

static int

2738

vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,

2738

vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,

2739

int vectors)

2739

int vectors)

2740

{

2740

{

2741

int err = 0, vector_threshold;

2741

int err = 0, vector_threshold;

2742

vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;

2742

vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;

2743

2744

while (vectors >= vector_threshold) {

2744

while (vectors >= vector_threshold) {

2745

err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,

2745

err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,

2746

vectors);

2746

vectors);

2747

if (!err) {

2747

if (!err) {

2748

adapter->intr.num_intrs = vectors;

2748

adapter->intr.num_intrs = vectors;

2749

return 0;

2749

return 0;

2750

} else if (err < 0) {

2750

} else if (err < 0) {

2751

dev_err(&adapter->netdev->dev,

2751

dev_err(&adapter->netdev->dev,

2752

"Failed to enable MSI-X, error: %d\n", err);

2752

"Failed to enable MSI-X, error: %d\n", err);

2753

vectors = 0;

2753

vectors = 0;

2754

} else if (err < vector_threshold) {

2754

} else if (err < vector_threshold) {

2755

break;

2755

break;

2756

} else {

2756

} else {

2757

/* If fails to enable required number of MSI-x vectors

2757

/* If fails to enable required number of MSI-x vectors

2758

* try enabling minimum number of vectors required.

2758

* try enabling minimum number of vectors required.

2759

*/

2759

*/

2760

dev_err(&adapter->netdev->dev,

2760

dev_err(&adapter->netdev->dev,

2761

"Failed to enable %d MSI-X, trying %d instead\n",

2761

"Failed to enable %d MSI-X, trying %d instead\n",

2762

vectors, vector_threshold);

2762

vectors, vector_threshold);

2763

vectors = vector_threshold;

2763

vectors = vector_threshold;

2764

}

2764

}

2765

}

2765

}

2766

2767

dev_info(&adapter->pdev->dev,

2767

dev_info(&adapter->pdev->dev,

2768

"Number of MSI-X interrupts which can be allocated "

2768

"Number of MSI-X interrupts which can be allocated "

2769

"is lower than min threshold required.\n");

2769

"is lower than min threshold required.\n");

2770

return err;

2770

return err;

2771

}

2771

}

2772

2773

2774

#endif /* CONFIG_PCI_MSI */

2774

#endif /* CONFIG_PCI_MSI */

2775

2776

static void

2776

static void

2777

vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)

2777

vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)

2778

{

2778

{

2779

u32 cfg;

2779

u32 cfg;

2780

unsigned long flags;

2780

unsigned long flags;

2781

2782

/* intr settings */

2782

/* intr settings */

2783

spin_lock_irqsave(&adapter->cmd_lock, flags);

2783

spin_lock_irqsave(&adapter->cmd_lock, flags);

2784

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2784

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

2785

VMXNET3_CMD_GET_CONF_INTR);

2785

VMXNET3_CMD_GET_CONF_INTR);

2786

cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);

2786

cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);

2787

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2787

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

2788

adapter->intr.type = cfg & 0x3;

2788

adapter->intr.type = cfg & 0x3;

2789

adapter->intr.mask_mode = (cfg >> 2) & 0x3;

2789

adapter->intr.mask_mode = (cfg >> 2) & 0x3;

2790

2791

if (adapter->intr.type == VMXNET3_IT_AUTO) {

2791

if (adapter->intr.type == VMXNET3_IT_AUTO) {

2792

adapter->intr.type = VMXNET3_IT_MSIX;

2792

adapter->intr.type = VMXNET3_IT_MSIX;

2793

}

2793

}

2794

2795

#ifdef CONFIG_PCI_MSI

2795

#ifdef CONFIG_PCI_MSI

2796

if (adapter->intr.type == VMXNET3_IT_MSIX) {

2796

if (adapter->intr.type == VMXNET3_IT_MSIX) {

2797

int vector, err = 0;

2797

int vector, err = 0;

2798

2799

adapter->intr.num_intrs = (adapter->share_intr ==

2799

adapter->intr.num_intrs = (adapter->share_intr ==

2800

VMXNET3_INTR_TXSHARE) ? 1 :

2800

VMXNET3_INTR_TXSHARE) ? 1 :

2801

adapter->num_tx_queues;

2801

adapter->num_tx_queues;

2802

adapter->intr.num_intrs += (adapter->share_intr ==

2802

adapter->intr.num_intrs += (adapter->share_intr ==

2803

VMXNET3_INTR_BUDDYSHARE) ? 0 :

2803

VMXNET3_INTR_BUDDYSHARE) ? 0 :

2804

adapter->num_rx_queues;

2804

adapter->num_rx_queues;

2805

adapter->intr.num_intrs += 1; /* for link event */

2805

adapter->intr.num_intrs += 1; /* for link event */

2806

2807

adapter->intr.num_intrs = (adapter->intr.num_intrs >

2807

adapter->intr.num_intrs = (adapter->intr.num_intrs >

2808

VMXNET3_LINUX_MIN_MSIX_VECT

2808

VMXNET3_LINUX_MIN_MSIX_VECT

2809

? adapter->intr.num_intrs :

2809

? adapter->intr.num_intrs :

2810

VMXNET3_LINUX_MIN_MSIX_VECT);

2810

VMXNET3_LINUX_MIN_MSIX_VECT);

2811

2812

for (vector = 0; vector < adapter->intr.num_intrs; vector++)

2812

for (vector = 0; vector < adapter->intr.num_intrs; vector++)

2813

adapter->intr.msix_entries[vector].entry = vector;

2813

adapter->intr.msix_entries[vector].entry = vector;

2814

2815

err = vmxnet3_acquire_msix_vectors(adapter,

2815

err = vmxnet3_acquire_msix_vectors(adapter,

2816

adapter->intr.num_intrs);

2816

adapter->intr.num_intrs);

2817

/* If we cannot allocate one MSIx vector per queue

2817

/* If we cannot allocate one MSIx vector per queue

2818

* then limit the number of rx queues to 1

2818

* then limit the number of rx queues to 1

2819

*/

2819

*/

2820

if (err == VMXNET3_LINUX_MIN_MSIX_VECT) {

2820

if (err == VMXNET3_LINUX_MIN_MSIX_VECT) {

2821

if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE

2821

if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE

2822

|| adapter->num_rx_queues != 1) {

2822

|| adapter->num_rx_queues != 1) {

2823

adapter->share_intr = VMXNET3_INTR_TXSHARE;

2823

adapter->share_intr = VMXNET3_INTR_TXSHARE;

2824

netdev_err(adapter->netdev,

2824

netdev_err(adapter->netdev,

2825

"Number of rx queues : 1\n");

2825

"Number of rx queues : 1\n");

2826

adapter->num_rx_queues = 1;

2826

adapter->num_rx_queues = 1;

2827

adapter->intr.num_intrs =

2827

adapter->intr.num_intrs =

2828

VMXNET3_LINUX_MIN_MSIX_VECT;

2828

VMXNET3_LINUX_MIN_MSIX_VECT;

2829

}

2829

}

2830

return;

2830

return;

2831

}

2831

}

2832

if (!err)

2832

if (!err)

2833

return;

2833

return;

2834

2835

/* If we cannot allocate MSIx vectors use only one rx queue */

2835

/* If we cannot allocate MSIx vectors use only one rx queue */

2836

dev_info(&adapter->pdev->dev,

2836

dev_info(&adapter->pdev->dev,

2837

"Failed to enable MSI-X, error %d. "

2837

"Failed to enable MSI-X, error %d. "

2838

"Limiting #rx queues to 1, try MSI.\n", err);

2838

"Limiting #rx queues to 1, try MSI.\n", err);

2839

2840

adapter->intr.type = VMXNET3_IT_MSI;

2840

adapter->intr.type = VMXNET3_IT_MSI;

2841

}

2841

}

2842

2843

if (adapter->intr.type == VMXNET3_IT_MSI) {

2843

if (adapter->intr.type == VMXNET3_IT_MSI) {

2844

int err;

2844

int err;

2845

err = pci_enable_msi(adapter->pdev);

2845

err = pci_enable_msi(adapter->pdev);

2846

if (!err) {

2846

if (!err) {

2847

adapter->num_rx_queues = 1;

2847

adapter->num_rx_queues = 1;

2848

adapter->intr.num_intrs = 1;

2848

adapter->intr.num_intrs = 1;

2849

return;

2849

return;

2850

}

2850

}

2851

}

2851

}

2852

#endif /* CONFIG_PCI_MSI */

2852

#endif /* CONFIG_PCI_MSI */

2853

2854

adapter->num_rx_queues = 1;

2854

adapter->num_rx_queues = 1;

2855

dev_info(&adapter->netdev->dev,

2855

dev_info(&adapter->netdev->dev,

2856

"Using INTx interrupt, #Rx queues: 1.\n");

2856

"Using INTx interrupt, #Rx queues: 1.\n");

2857

adapter->intr.type = VMXNET3_IT_INTX;

2857

adapter->intr.type = VMXNET3_IT_INTX;

2858

2859

/* INT-X related setting */

2859

/* INT-X related setting */

2860

adapter->intr.num_intrs = 1;

2860

adapter->intr.num_intrs = 1;

2861

}

2861

}

2862

2863

2864

static void

2864

static void

2865

vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)

2865

vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)

2866

{

2866

{

2867

if (adapter->intr.type == VMXNET3_IT_MSIX)

2867

if (adapter->intr.type == VMXNET3_IT_MSIX)

2868

pci_disable_msix(adapter->pdev);

2868

pci_disable_msix(adapter->pdev);

2869

else if (adapter->intr.type == VMXNET3_IT_MSI)

2869

else if (adapter->intr.type == VMXNET3_IT_MSI)

2870

pci_disable_msi(adapter->pdev);

2870

pci_disable_msi(adapter->pdev);

2871

else

2871

else

2872

BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);

2872

BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);

2873

}

2873

}

2874

2875

2876

static void

2876

static void

2877

vmxnet3_tx_timeout(struct net_device *netdev)

2877

vmxnet3_tx_timeout(struct net_device *netdev)

2878

{

2878

{

2879

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2879

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

2880

adapter->tx_timeout_count++;

2880

adapter->tx_timeout_count++;

2881

2882

netdev_err(adapter->netdev, "tx hang\n");

2882

netdev_err(adapter->netdev, "tx hang\n");

2883

schedule_work(&adapter->work);

2883

schedule_work(&adapter->work);

2884

netif_wake_queue(adapter->netdev);

2884

netif_wake_queue(adapter->netdev);

2885

}

2885

}

2886

2887

2888

static void

2888

static void

2889

vmxnet3_reset_work(struct work_struct *data)

2889

vmxnet3_reset_work(struct work_struct *data)

2890

{

2890

{

2891

struct vmxnet3_adapter *adapter;

2891

struct vmxnet3_adapter *adapter;

2892

2893

adapter = container_of(data, struct vmxnet3_adapter, work);

2893

adapter = container_of(data, struct vmxnet3_adapter, work);

2894

2895

/* if another thread is resetting the device, no need to proceed */

2895

/* if another thread is resetting the device, no need to proceed */

2896

if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))

2896

if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))

2897

return;

2897

return;

2898

2899

/* if the device is closed, we must leave it alone */

2899

/* if the device is closed, we must leave it alone */

2900

rtnl_lock();

2900

rtnl_lock();

2901

if (netif_running(adapter->netdev)) {

2901

if (netif_running(adapter->netdev)) {

2902

netdev_notice(adapter->netdev, "resetting\n");

2902

netdev_notice(adapter->netdev, "resetting\n");

2903

vmxnet3_quiesce_dev(adapter);

2903

vmxnet3_quiesce_dev(adapter);

2904

vmxnet3_reset_dev(adapter);

2904

vmxnet3_reset_dev(adapter);

2905

vmxnet3_activate_dev(adapter);

2905

vmxnet3_activate_dev(adapter);

2906

} else {

2906

} else {

2907

netdev_info(adapter->netdev, "already closed\n");

2907

netdev_info(adapter->netdev, "already closed\n");

2908

}

2908

}

2909

rtnl_unlock();

2909

rtnl_unlock();

2910

2911

clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);

2911

clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);

2912

}

2912

}

2913

2914

2915

static int

2915

static int

2916

vmxnet3_probe_device(struct pci_dev *pdev,

2916

vmxnet3_probe_device(struct pci_dev *pdev,

2917

const struct pci_device_id *id)

2917

const struct pci_device_id *id)

2918

{

2918

{

2919

static const struct net_device_ops vmxnet3_netdev_ops = {

2919

static const struct net_device_ops vmxnet3_netdev_ops = {

2920

.ndo_open = vmxnet3_open,

2920

.ndo_open = vmxnet3_open,

2921

.ndo_stop = vmxnet3_close,

2921

.ndo_stop = vmxnet3_close,

2922

.ndo_start_xmit = vmxnet3_xmit_frame,

2922

.ndo_start_xmit = vmxnet3_xmit_frame,

2923

.ndo_set_mac_address = vmxnet3_set_mac_addr,

2923

.ndo_set_mac_address = vmxnet3_set_mac_addr,

2924

.ndo_change_mtu = vmxnet3_change_mtu,

2924

.ndo_change_mtu = vmxnet3_change_mtu,

2925

.ndo_set_features = vmxnet3_set_features,

2925

.ndo_set_features = vmxnet3_set_features,

2926

.ndo_get_stats64 = vmxnet3_get_stats64,

2926

.ndo_get_stats64 = vmxnet3_get_stats64,

2927

.ndo_tx_timeout = vmxnet3_tx_timeout,

2927

.ndo_tx_timeout = vmxnet3_tx_timeout,

2928

.ndo_set_rx_mode = vmxnet3_set_mc,

2928

.ndo_set_rx_mode = vmxnet3_set_mc,

2929

.ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,

2929

.ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,

2930

.ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,

2930

.ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,

2931

#ifdef CONFIG_NET_POLL_CONTROLLER

2931

#ifdef CONFIG_NET_POLL_CONTROLLER

2932

.ndo_poll_controller = vmxnet3_netpoll,

2932

.ndo_poll_controller = vmxnet3_netpoll,

2933

#endif

2933

#endif

2934

};

2934

};

2935

int err;

2935

int err;

2936

bool dma64 = false; /* stupid gcc */

2936

bool dma64 = false; /* stupid gcc */

2937

u32 ver;

2937

u32 ver;

2938

struct net_device *netdev;

2938

struct net_device *netdev;

2939

struct vmxnet3_adapter *adapter;

2939

struct vmxnet3_adapter *adapter;

2940

u8 mac[ETH_ALEN];

2940

u8 mac[ETH_ALEN];

2941

int size;

2941

int size;

2942

int num_tx_queues;

2942

int num_tx_queues;

2943

int num_rx_queues;

2943

int num_rx_queues;

2944

2945

if (!pci_msi_enabled())

2945

if (!pci_msi_enabled())

2946

enable_mq = 0;

2946

enable_mq = 0;

2947

2948

#ifdef VMXNET3_RSS

2948

#ifdef VMXNET3_RSS

2949

if (enable_mq)

2949

if (enable_mq)

2950

num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,

2950

num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,

2951

(int)num_online_cpus());

2951

(int)num_online_cpus());

2952

else

2952

else

2953

#endif

2953

#endif

2954

num_rx_queues = 1;

2954

num_rx_queues = 1;

2955

num_rx_queues = rounddown_pow_of_two(num_rx_queues);

2955

num_rx_queues = rounddown_pow_of_two(num_rx_queues);

2956

2957

if (enable_mq)

2957

if (enable_mq)

2958

num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,

2958

num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,

2959

(int)num_online_cpus());

2959

(int)num_online_cpus());

2960

else

2960

else

2961

num_tx_queues = 1;

2961

num_tx_queues = 1;

2962

2963

num_tx_queues = rounddown_pow_of_two(num_tx_queues);

2963

num_tx_queues = rounddown_pow_of_two(num_tx_queues);

2964

netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),

2964

netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),

2965

max(num_tx_queues, num_rx_queues));

2965

max(num_tx_queues, num_rx_queues));

2966

dev_info(&pdev->dev,

2966

dev_info(&pdev->dev,

2967

"# of Tx queues : %d, # of Rx queues : %d\n",

2967

"# of Tx queues : %d, # of Rx queues : %d\n",

2968

num_tx_queues, num_rx_queues);

2968

num_tx_queues, num_rx_queues);

2969

2970

if (!netdev)

2970

if (!netdev)

2971

return -ENOMEM;

2971

return -ENOMEM;

2972

2973

pci_set_drvdata(pdev, netdev);

2973

pci_set_drvdata(pdev, netdev);

2974

adapter = netdev_priv(netdev);

2974

adapter = netdev_priv(netdev);

2975

adapter->netdev = netdev;

2975

adapter->netdev = netdev;

2976

adapter->pdev = pdev;

2976

adapter->pdev = pdev;

2977

2978

spin_lock_init(&adapter->cmd_lock);

2978

spin_lock_init(&adapter->cmd_lock);

2979

adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,

2979

adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,

2980

sizeof(struct vmxnet3_adapter),

2980

sizeof(struct vmxnet3_adapter),

2981

PCI_DMA_TODEVICE);

2981

PCI_DMA_TODEVICE);

2982

adapter->shared = dma_alloc_coherent(

2982

adapter->shared = dma_alloc_coherent(

2983

&adapter->pdev->dev,

2983

&adapter->pdev->dev,

2984

sizeof(struct Vmxnet3_DriverShared),

2984

sizeof(struct Vmxnet3_DriverShared),

2985

&adapter->shared_pa, GFP_KERNEL);

2985

&adapter->shared_pa, GFP_KERNEL);

2986

if (!adapter->shared) {

2986

if (!adapter->shared) {

2987

dev_err(&pdev->dev, "Failed to allocate memory\n");

2987

dev_err(&pdev->dev, "Failed to allocate memory\n");

2988

err = -ENOMEM;

2988

err = -ENOMEM;

2989

goto err_alloc_shared;

2989

goto err_alloc_shared;

2990

}

2990

}

2991

2992

adapter->num_rx_queues = num_rx_queues;

2992

adapter->num_rx_queues = num_rx_queues;

2993

adapter->num_tx_queues = num_tx_queues;

2993

adapter->num_tx_queues = num_tx_queues;

2994

adapter->rx_buf_per_pkt = 1;

2994

adapter->rx_buf_per_pkt = 1;

2995

2996

size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;

2996

size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;

2997

size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;

2997

size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;

2998

adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,

2998

adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,

2999

&adapter->queue_desc_pa,

2999

&adapter->queue_desc_pa,

3000

GFP_KERNEL);

3000

GFP_KERNEL);

3001

3002

if (!adapter->tqd_start) {

3002

if (!adapter->tqd_start) {

3003

dev_err(&pdev->dev, "Failed to allocate memory\n");

3003

dev_err(&pdev->dev, "Failed to allocate memory\n");

3004

err = -ENOMEM;

3004

err = -ENOMEM;

3005

goto err_alloc_queue_desc;

3005

goto err_alloc_queue_desc;

3006

}

3006

}

3007

adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +

3007

adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +

3008

adapter->num_tx_queues);

3008

adapter->num_tx_queues);

3009

3010

adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,

3010

adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,

3011

sizeof(struct Vmxnet3_PMConf),

3011

sizeof(struct Vmxnet3_PMConf),

3012

&adapter->pm_conf_pa,

3012

&adapter->pm_conf_pa,

3013

GFP_KERNEL);

3013

GFP_KERNEL);

3014

if (adapter->pm_conf == NULL) {

3014

if (adapter->pm_conf == NULL) {

3015

err = -ENOMEM;

3015

err = -ENOMEM;

3016

goto err_alloc_pm;

3016

goto err_alloc_pm;

3017

}

3017

}

3018

3019

#ifdef VMXNET3_RSS

3019

#ifdef VMXNET3_RSS

3020

3021

adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,

3021

adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,

3022

sizeof(struct UPT1_RSSConf),

3022

sizeof(struct UPT1_RSSConf),

3023

&adapter->rss_conf_pa,

3023

&adapter->rss_conf_pa,

3024

GFP_KERNEL);

3024

GFP_KERNEL);

3025

if (adapter->rss_conf == NULL) {

3025

if (adapter->rss_conf == NULL) {

3026

err = -ENOMEM;

3026

err = -ENOMEM;

3027

goto err_alloc_rss;

3027

goto err_alloc_rss;

3028

}

3028

}

3029

#endif /* VMXNET3_RSS */

3029

#endif /* VMXNET3_RSS */

3030

3031

err = vmxnet3_alloc_pci_resources(adapter, &dma64);

3031

err = vmxnet3_alloc_pci_resources(adapter, &dma64);

3032

if (err < 0)

3032

if (err < 0)

3033

goto err_alloc_pci;

3033

goto err_alloc_pci;

3034

3035

ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);

3035

ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);

3036

if (ver & 1) {

3036

if (ver & 1) {

3037

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);

3037

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);

3038

} else {

3038

} else {

3039

dev_err(&pdev->dev,

3039

dev_err(&pdev->dev,

3040

"Incompatible h/w version (0x%x) for adapter\n", ver);

3040

"Incompatible h/w version (0x%x) for adapter\n", ver);

3041

err = -EBUSY;

3041

err = -EBUSY;

3042

goto err_ver;

3042

goto err_ver;

3043

}

3043

}

3044

3045

ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);

3045

ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);

3046

if (ver & 1) {

3046

if (ver & 1) {

3047

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);

3047

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);

3048

} else {

3048

} else {

3049

dev_err(&pdev->dev,

3049

dev_err(&pdev->dev,

3050

"Incompatible upt version (0x%x) for adapter\n", ver);

3050

"Incompatible upt version (0x%x) for adapter\n", ver);

3051

err = -EBUSY;

3051

err = -EBUSY;

3052

goto err_ver;

3052

goto err_ver;

3053

}

3053

}

3054

3055

SET_NETDEV_DEV(netdev, &pdev->dev);

3055

SET_NETDEV_DEV(netdev, &pdev->dev);

3056

vmxnet3_declare_features(adapter, dma64);

3056

vmxnet3_declare_features(adapter, dma64);

3057

3058

if (adapter->num_tx_queues == adapter->num_rx_queues)

3058

if (adapter->num_tx_queues == adapter->num_rx_queues)

3059

adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;

3059

adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;

3060

else

3060

else

3061

adapter->share_intr = VMXNET3_INTR_DONTSHARE;

3061

adapter->share_intr = VMXNET3_INTR_DONTSHARE;

3062

3063

vmxnet3_alloc_intr_resources(adapter);

3063

vmxnet3_alloc_intr_resources(adapter);

3064

3065

#ifdef VMXNET3_RSS

3065

#ifdef VMXNET3_RSS

3066

if (adapter->num_rx_queues > 1 &&

3066

if (adapter->num_rx_queues > 1 &&

3067

adapter->intr.type == VMXNET3_IT_MSIX) {

3067

adapter->intr.type == VMXNET3_IT_MSIX) {

3068

adapter->rss = true;

3068

adapter->rss = true;

3069

netdev->hw_features |= NETIF_F_RXHASH;

3069

netdev->hw_features |= NETIF_F_RXHASH;

3070

netdev->features |= NETIF_F_RXHASH;

3070

netdev->features |= NETIF_F_RXHASH;

3071

dev_dbg(&pdev->dev, "RSS is enabled.\n");

3071

dev_dbg(&pdev->dev, "RSS is enabled.\n");

3072

} else {

3072

} else {

3073

adapter->rss = false;

3073

adapter->rss = false;

3074

}

3074

}

3075

#endif

3075

#endif

3076

3077

vmxnet3_read_mac_addr(adapter, mac);

3077

vmxnet3_read_mac_addr(adapter, mac);

3078

memcpy(netdev->dev_addr, mac, netdev->addr_len);

3078

memcpy(netdev->dev_addr, mac, netdev->addr_len);

3079

3080

netdev->netdev_ops = &vmxnet3_netdev_ops;

3080

netdev->netdev_ops = &vmxnet3_netdev_ops;

3081

vmxnet3_set_ethtool_ops(netdev);

3081

vmxnet3_set_ethtool_ops(netdev);

3082

netdev->watchdog_timeo = 5 * HZ;

3082

netdev->watchdog_timeo = 5 * HZ;

3083

3084

INIT_WORK(&adapter->work, vmxnet3_reset_work);

3084

INIT_WORK(&adapter->work, vmxnet3_reset_work);

3085

set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);

3085

set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);

3086

3087

if (adapter->intr.type == VMXNET3_IT_MSIX) {

3087

if (adapter->intr.type == VMXNET3_IT_MSIX) {

3088

int i;

3088

int i;

3089

for (i = 0; i < adapter->num_rx_queues; i++) {

3089

for (i = 0; i < adapter->num_rx_queues; i++) {

3090

netif_napi_add(adapter->netdev,

3090

netif_napi_add(adapter->netdev,

3091

&adapter->rx_queue[i].napi,

3091

&adapter->rx_queue[i].napi,

3092

vmxnet3_poll_rx_only, 64);

3092

vmxnet3_poll_rx_only, 64);

3093

}

3093

}

3094

} else {

3094

} else {

3095

netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,

3095

netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,

3096

vmxnet3_poll, 64);

3096

vmxnet3_poll, 64);

3097

}

3097

}

3098

3099

netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);

3099

netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);

3100

netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);

3100

netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);

3101

3102

netif_carrier_off(netdev);

3102

netif_carrier_off(netdev);

3103

err = register_netdev(netdev);

3103

err = register_netdev(netdev);

3104

3105

if (err) {

3105

if (err) {

3106

dev_err(&pdev->dev, "Failed to register adapter\n");

3106

dev_err(&pdev->dev, "Failed to register adapter\n");

3107

goto err_register;

3107

goto err_register;

3108

}

3108

}

3109

3110

vmxnet3_check_link(adapter, false);

3110

vmxnet3_check_link(adapter, false);

3111

return 0;

3111

return 0;

3112

3113

err_register:

3113

err_register:

3114

vmxnet3_free_intr_resources(adapter);

3114

vmxnet3_free_intr_resources(adapter);

3115

err_ver:

3115

err_ver:

3116

vmxnet3_free_pci_resources(adapter);

3116

vmxnet3_free_pci_resources(adapter);

3117

err_alloc_pci:

3117

err_alloc_pci:

3118

#ifdef VMXNET3_RSS

3118

#ifdef VMXNET3_RSS

3119

dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),

3119

dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),

3120

adapter->rss_conf, adapter->rss_conf_pa);

3120

adapter->rss_conf, adapter->rss_conf_pa);

3121

err_alloc_rss:

3121

err_alloc_rss:

3122

#endif

3122

#endif

3123

dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),

3123

dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),

3124

adapter->pm_conf, adapter->pm_conf_pa);

3124

adapter->pm_conf, adapter->pm_conf_pa);

3125

err_alloc_pm:

3125

err_alloc_pm:

3126

dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,

3126

dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,

3127

adapter->queue_desc_pa);

3127

adapter->queue_desc_pa);

3128

err_alloc_queue_desc:

3128

err_alloc_queue_desc:

3129

dma_free_coherent(&adapter->pdev->dev,

3129

dma_free_coherent(&adapter->pdev->dev,

3130

sizeof(struct Vmxnet3_DriverShared),

3130

sizeof(struct Vmxnet3_DriverShared),

3131

adapter->shared, adapter->shared_pa);

3131

adapter->shared, adapter->shared_pa);

3132

err_alloc_shared:

3132

err_alloc_shared:

3133

dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,

3133

dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,

3134

sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);

3134

sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);

3135

pci_set_drvdata(pdev, NULL);

3136

free_netdev(netdev);

3135

free_netdev(netdev);

3137

return err;

3136

return err;

3138

}

3137

}

3139

3138

3140

3139

3141

static void

3140

static void

3142

vmxnet3_remove_device(struct pci_dev *pdev)

3141

vmxnet3_remove_device(struct pci_dev *pdev)

3143

{

3142

{

3144

struct net_device *netdev = pci_get_drvdata(pdev);

3143

struct net_device *netdev = pci_get_drvdata(pdev);

3145

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

3144

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

3146

int size = 0;

3145

int size = 0;

3147

int num_rx_queues;

3146

int num_rx_queues;

3148

3147

3149

#ifdef VMXNET3_RSS

3148

#ifdef VMXNET3_RSS

3150

if (enable_mq)

3149

if (enable_mq)

3151

num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,

3150

num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,

3152

(int)num_online_cpus());

3151

(int)num_online_cpus());

3153

else

3152

else

3154

#endif

3153

#endif

3155

num_rx_queues = 1;

3154

num_rx_queues = 1;

3156

num_rx_queues = rounddown_pow_of_two(num_rx_queues);

3155

num_rx_queues = rounddown_pow_of_two(num_rx_queues);

3157

3156

3158

cancel_work_sync(&adapter->work);

3157

cancel_work_sync(&adapter->work);

3159

3158

3160

unregister_netdev(netdev);

3159

unregister_netdev(netdev);

3161

3160

3162

vmxnet3_free_intr_resources(adapter);

3161

vmxnet3_free_intr_resources(adapter);

3163

vmxnet3_free_pci_resources(adapter);

3162

vmxnet3_free_pci_resources(adapter);

3164

#ifdef VMXNET3_RSS

3163

#ifdef VMXNET3_RSS

3165

dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),

3164

dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),

3166

adapter->rss_conf, adapter->rss_conf_pa);

3165

adapter->rss_conf, adapter->rss_conf_pa);

3167

#endif

3166

#endif

3168

dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),

3167

dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),

3169

adapter->pm_conf, adapter->pm_conf_pa);

3168

adapter->pm_conf, adapter->pm_conf_pa);

3170

3169

3171

size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;

3170

size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;

3172

size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;

3171

size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;

3173

dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,

3172

dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,

3174

adapter->queue_desc_pa);

3173

adapter->queue_desc_pa);

3175

dma_free_coherent(&adapter->pdev->dev,

3174

dma_free_coherent(&adapter->pdev->dev,

3176

sizeof(struct Vmxnet3_DriverShared),

3175

sizeof(struct Vmxnet3_DriverShared),

3177

adapter->shared, adapter->shared_pa);

3176

adapter->shared, adapter->shared_pa);

3178

dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,

3177

dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,

3179

sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);

3178

sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);

3180

free_netdev(netdev);

3179

free_netdev(netdev);

3181

}

3180

}

3182

3181

3183

3182

3184

#ifdef CONFIG_PM

3183

#ifdef CONFIG_PM

3185

3184

3186

static int

3185

static int

3187

vmxnet3_suspend(struct device *device)

3186

vmxnet3_suspend(struct device *device)

3188

{

3187

{

3189

struct pci_dev *pdev = to_pci_dev(device);

3188

struct pci_dev *pdev = to_pci_dev(device);

3190

struct net_device *netdev = pci_get_drvdata(pdev);

3189

struct net_device *netdev = pci_get_drvdata(pdev);

3191

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

3190

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

3192

struct Vmxnet3_PMConf *pmConf;

3191

struct Vmxnet3_PMConf *pmConf;

3193

struct ethhdr *ehdr;

3192

struct ethhdr *ehdr;

3194

struct arphdr *ahdr;

3193

struct arphdr *ahdr;

3195

u8 *arpreq;

3194

u8 *arpreq;

3196

struct in_device *in_dev;

3195

struct in_device *in_dev;

3197

struct in_ifaddr *ifa;

3196

struct in_ifaddr *ifa;

3198

unsigned long flags;

3197

unsigned long flags;

3199

int i = 0;

3198

int i = 0;

3200

3199

3201

if (!netif_running(netdev))

3200

if (!netif_running(netdev))

3202

return 0;

3201

return 0;

3203

3202

3204

for (i = 0; i < adapter->num_rx_queues; i++)

3203

for (i = 0; i < adapter->num_rx_queues; i++)

3205

napi_disable(&adapter->rx_queue[i].napi);

3204

napi_disable(&adapter->rx_queue[i].napi);

3206

3205

3207

vmxnet3_disable_all_intrs(adapter);

3206

vmxnet3_disable_all_intrs(adapter);

3208

vmxnet3_free_irqs(adapter);

3207

vmxnet3_free_irqs(adapter);

3209

vmxnet3_free_intr_resources(adapter);

3208

vmxnet3_free_intr_resources(adapter);

3210

3209

3211

netif_device_detach(netdev);

3210

netif_device_detach(netdev);

3212

netif_tx_stop_all_queues(netdev);

3211

netif_tx_stop_all_queues(netdev);

3213

3212

3214

/* Create wake-up filters. */

3213

/* Create wake-up filters. */

3215

pmConf = adapter->pm_conf;

3214

pmConf = adapter->pm_conf;

3216

memset(pmConf, 0, sizeof(*pmConf));

3215

memset(pmConf, 0, sizeof(*pmConf));

3217

3216

3218

if (adapter->wol & WAKE_UCAST) {

3217

if (adapter->wol & WAKE_UCAST) {

3219

pmConf->filters[i].patternSize = ETH_ALEN;

3218

pmConf->filters[i].patternSize = ETH_ALEN;

3220

pmConf->filters[i].maskSize = 1;

3219

pmConf->filters[i].maskSize = 1;

3221

memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);

3220

memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);

3222

pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */

3221

pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */

3223

3222

3224

pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;

3223

pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;

3225

i++;

3224

i++;

3226

}

3225

}

3227

3226

3228

if (adapter->wol & WAKE_ARP) {

3227

if (adapter->wol & WAKE_ARP) {

3229

in_dev = in_dev_get(netdev);

3228

in_dev = in_dev_get(netdev);

3230

if (!in_dev)

3229

if (!in_dev)

3231

goto skip_arp;

3230

goto skip_arp;

3232

3231

3233

ifa = (struct in_ifaddr *)in_dev->ifa_list;

3232

ifa = (struct in_ifaddr *)in_dev->ifa_list;

3234

if (!ifa)

3233

if (!ifa)

3235

goto skip_arp;

3234

goto skip_arp;

3236

3235

3237

pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/

3236

pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/

3238

sizeof(struct arphdr) + /* ARP header */

3237

sizeof(struct arphdr) + /* ARP header */

3239

2 * ETH_ALEN + /* 2 Ethernet addresses*/

3238

2 * ETH_ALEN + /* 2 Ethernet addresses*/

3240

2 * sizeof(u32); /*2 IPv4 addresses */

3239

2 * sizeof(u32); /*2 IPv4 addresses */

3241

pmConf->filters[i].maskSize =

3240

pmConf->filters[i].maskSize =

3242

(pmConf->filters[i].patternSize - 1) / 8 + 1;

3241

(pmConf->filters[i].patternSize - 1) / 8 + 1;

3243

3242

3244

/* ETH_P_ARP in Ethernet header. */

3243

/* ETH_P_ARP in Ethernet header. */

3245

ehdr = (struct ethhdr *)pmConf->filters[i].pattern;

3244

ehdr = (struct ethhdr *)pmConf->filters[i].pattern;

3246

ehdr->h_proto = htons(ETH_P_ARP);

3245

ehdr->h_proto = htons(ETH_P_ARP);

3247

3246

3248

/* ARPOP_REQUEST in ARP header. */

3247

/* ARPOP_REQUEST in ARP header. */

3249

ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];

3248

ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];

3250

ahdr->ar_op = htons(ARPOP_REQUEST);

3249

ahdr->ar_op = htons(ARPOP_REQUEST);

3251

arpreq = (u8 *)(ahdr + 1);

3250

arpreq = (u8 *)(ahdr + 1);

3252

3251

3253

/* The Unicast IPv4 address in 'tip' field. */

3252

/* The Unicast IPv4 address in 'tip' field. */

3254

arpreq += 2 * ETH_ALEN + sizeof(u32);

3253

arpreq += 2 * ETH_ALEN + sizeof(u32);

3255

*(u32 *)arpreq = ifa->ifa_address;

3254

*(u32 *)arpreq = ifa->ifa_address;

3256

3255

3257

/* The mask for the relevant bits. */

3256

/* The mask for the relevant bits. */

3258

pmConf->filters[i].mask[0] = 0x00;

3257

pmConf->filters[i].mask[0] = 0x00;

3259

pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */

3258

pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */

3260

pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */

3259

pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */

3261

pmConf->filters[i].mask[3] = 0x00;

3260

pmConf->filters[i].mask[3] = 0x00;

3262

pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */

3261

pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */

3263

pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */

3262

pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */

3264

in_dev_put(in_dev);

3263

in_dev_put(in_dev);

3265

3264

3266

pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;

3265

pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;

3267

i++;

3266

i++;

3268

}

3267

}

3269

3268

3270

skip_arp:

3269

skip_arp:

3271

if (adapter->wol & WAKE_MAGIC)

3270

if (adapter->wol & WAKE_MAGIC)

3272

pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;

3271

pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;

3273

3272

3274

pmConf->numFilters = i;

3273

pmConf->numFilters = i;

3275

3274

3276

adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);

3275

adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);

3277

adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(

3276

adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(

3278

*pmConf));

3277

*pmConf));

3279

adapter->shared->devRead.pmConfDesc.confPA =

3278

adapter->shared->devRead.pmConfDesc.confPA =

3280

cpu_to_le64(adapter->pm_conf_pa);

3279

cpu_to_le64(adapter->pm_conf_pa);

3281

3280

3282

spin_lock_irqsave(&adapter->cmd_lock, flags);

3281

spin_lock_irqsave(&adapter->cmd_lock, flags);

3283

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

3282

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

3284

VMXNET3_CMD_UPDATE_PMCFG);

3283

VMXNET3_CMD_UPDATE_PMCFG);

3285

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

3284

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

3286

3285

3287

pci_save_state(pdev);

3286

pci_save_state(pdev);

3288

pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),

3287

pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),

3289

adapter->wol);

3288

adapter->wol);

3290

pci_disable_device(pdev);

3289

pci_disable_device(pdev);

3291

pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));

3290

pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));

3292

3291

3293

return 0;

3292

return 0;

3294

}

3293

}

3295

3294

3296

3295

3297

static int

3296

static int

3298

vmxnet3_resume(struct device *device)

3297

vmxnet3_resume(struct device *device)

3299

{

3298

{

3300

int err, i = 0;

3299

int err, i = 0;

3301

unsigned long flags;

3300

unsigned long flags;

3302

struct pci_dev *pdev = to_pci_dev(device);

3301

struct pci_dev *pdev = to_pci_dev(device);

3303

struct net_device *netdev = pci_get_drvdata(pdev);

3302

struct net_device *netdev = pci_get_drvdata(pdev);

3304

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

3303

struct vmxnet3_adapter *adapter = netdev_priv(netdev);

3305

struct Vmxnet3_PMConf *pmConf;

3304

struct Vmxnet3_PMConf *pmConf;

3306

3305

3307

if (!netif_running(netdev))

3306

if (!netif_running(netdev))

3308

return 0;

3307

return 0;

3309

3308

3310

/* Destroy wake-up filters. */

3309

/* Destroy wake-up filters. */

3311

pmConf = adapter->pm_conf;

3310

pmConf = adapter->pm_conf;

3312

memset(pmConf, 0, sizeof(*pmConf));

3311

memset(pmConf, 0, sizeof(*pmConf));

3313

3312

3314

adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);

3313

adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);

3315

adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(

3314

adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(

3316

*pmConf));

3315

*pmConf));

3317

adapter->shared->devRead.pmConfDesc.confPA =

3316

adapter->shared->devRead.pmConfDesc.confPA =

3318

cpu_to_le64(adapter->pm_conf_pa);

3317

cpu_to_le64(adapter->pm_conf_pa);

3319

3318

3320

netif_device_attach(netdev);

3319

netif_device_attach(netdev);

3321

pci_set_power_state(pdev, PCI_D0);

3320

pci_set_power_state(pdev, PCI_D0);

3322

pci_restore_state(pdev);

3321

pci_restore_state(pdev);

3323

err = pci_enable_device_mem(pdev);

3322

err = pci_enable_device_mem(pdev);

3324

if (err != 0)

3323

if (err != 0)

3325

return err;

3324

return err;

3326

3325

3327

pci_enable_wake(pdev, PCI_D0, 0);

3326

pci_enable_wake(pdev, PCI_D0, 0);

3328

3327

3329

spin_lock_irqsave(&adapter->cmd_lock, flags);

3328

spin_lock_irqsave(&adapter->cmd_lock, flags);

3330

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

3329

VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,

3331

VMXNET3_CMD_UPDATE_PMCFG);

3330

VMXNET3_CMD_UPDATE_PMCFG);

3332

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

3331

spin_unlock_irqrestore(&adapter->cmd_lock, flags);

3333

vmxnet3_alloc_intr_resources(adapter);

3332

vmxnet3_alloc_intr_resources(adapter);

3334

vmxnet3_request_irqs(adapter);

3333

vmxnet3_request_irqs(adapter);

3335

for (i = 0; i < adapter->num_rx_queues; i++)

3334

for (i = 0; i < adapter->num_rx_queues; i++)

3336

napi_enable(&adapter->rx_queue[i].napi);

3335

napi_enable(&adapter->rx_queue[i].napi);

3337

vmxnet3_enable_all_intrs(adapter);

3336

vmxnet3_enable_all_intrs(adapter);

3338

3337

3339

return 0;

3338

return 0;

3340

}

3339

}

3341

3340

3342

static const struct dev_pm_ops vmxnet3_pm_ops = {

3341

static const struct dev_pm_ops vmxnet3_pm_ops = {

3343

.suspend = vmxnet3_suspend,

3342

.suspend = vmxnet3_suspend,

3344

.resume = vmxnet3_resume,

3343

.resume = vmxnet3_resume,

3345

};

3344

};

3346

#endif

3345

#endif

3347

3346

3348

static struct pci_driver vmxnet3_driver = {

3347

static struct pci_driver vmxnet3_driver = {

3349

.name = vmxnet3_driver_name,

3348

.name = vmxnet3_driver_name,

3350

.id_table = vmxnet3_pciid_table,

3349

.id_table = vmxnet3_pciid_table,

3351

.probe = vmxnet3_probe_device,

3350

.probe = vmxnet3_probe_device,

3352

.remove = vmxnet3_remove_device,

3351

.remove = vmxnet3_remove_device,

3353

#ifdef CONFIG_PM

3352

#ifdef CONFIG_PM

3354

.driver.pm = &vmxnet3_pm_ops,

3353

.driver.pm = &vmxnet3_pm_ops,

3355

#endif

3354

#endif

3356

};

3355

};

3357

3356

3358

3357

3359

static int __init

3358

static int __init

3360

vmxnet3_init_module(void)

3359

vmxnet3_init_module(void)

3361

{

3360

{

3362

pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,

3361

pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,

3363

VMXNET3_DRIVER_VERSION_REPORT);

3362

VMXNET3_DRIVER_VERSION_REPORT);

3364

return pci_register_driver(&vmxnet3_driver);

3363

return pci_register_driver(&vmxnet3_driver);

3365

}

3364

}

3366

3365

3367

module_init(vmxnet3_init_module);

3366

module_init(vmxnet3_init_module);

3368

3367

3369

3368

3370

static void

3369

static void

3371

vmxnet3_exit_module(void)

3370

vmxnet3_exit_module(void)

3372

{

3371

{

3373

pci_unregister_driver(&vmxnet3_driver);

3372

pci_unregister_driver(&vmxnet3_driver);

3374

}

3373

}

3375

3374

3376

module_exit(vmxnet3_exit_module);

3375

module_exit(vmxnet3_exit_module);

3377

3376

3378

MODULE_AUTHOR("VMware, Inc.");

3377

MODULE_AUTHOR("VMware, Inc.");

3379

MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);

3378

MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);

3380

MODULE_LICENSE("GPL v2");

3379

MODULE_LICENSE("GPL v2");

3381

MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);

3380

MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);

3382

3381

GITLAB

net: vmxnet3: remove unnecessary pci_set_drvdata()

 /*
  * Linux driver for VMware's vmxnet3 ethernet NIC.
  *
  * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License as published by the
  * Free Software Foundation; version 2 of the License and no later version.
  *
  * This program is distributed in the hope that it will be useful, but
  * WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
  * NON INFRINGEMENT. See the GNU General Public License for more
  * details.
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  *
  * The full GNU General Public License is included in this distribution in
  * the file called "COPYING".
  *
  * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com>
  *
  */
 #include <linux/module.h>
 #include <net/ip6_checksum.h>
 #include "vmxnet3_int.h"
 char vmxnet3_driver_name[] = "vmxnet3";
 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
 /*
  * PCI Device ID Table
  * Last entry must be all 0s
  */
 static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = {
 	{PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
 	{0}
 };
 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
 static int enable_mq = 1;
 static void
 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac);
 /*
  *    Enable/Disable the given intr
  */
 static void
 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
 {
 	VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
 }
 static void
 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
 {
 	VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
 }
 /*
  *    Enable/Disable all intrs used by the device
  */
 static void
 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	for (i = 0; i < adapter->intr.num_intrs; i++)
 		vmxnet3_enable_intr(adapter, i);
 	adapter->shared->devRead.intrConf.intrCtrl &=
 					cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
 }
 static void
 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	adapter->shared->devRead.intrConf.intrCtrl |=
 					cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
 	for (i = 0; i < adapter->intr.num_intrs; i++)
 		vmxnet3_disable_intr(adapter, i);
 }
 static void
 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
 {
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
 }
 static bool
 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 {
 	return tq->stopped;
 }
 static void
 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 {
 	tq->stopped = false;
 	netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
 }
 static void
 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 {
 	tq->stopped = false;
 	netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
 }
 static void
 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
 {
 	tq->stopped = true;
 	tq->num_stop++;
 	netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
 }
 /*
  * Check the link state. This may start or stop the tx queue.
  */
 static void
 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
 {
 	u32 ret;
 	int i;
 	unsigned long flags;
 	spin_lock_irqsave(&adapter->cmd_lock, flags);
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
 	ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 	adapter->link_speed = ret >> 16;
 	if (ret & 1) { /* Link is up. */
 		netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
 			    adapter->link_speed);
 		netif_carrier_on(adapter->netdev);
 		if (affectTxQueue) {
 			for (i = 0; i < adapter->num_tx_queues; i++)
 				vmxnet3_tq_start(&adapter->tx_queue[i],
 						 adapter);
 		}
 	} else {
 		netdev_info(adapter->netdev, "NIC Link is Down\n");
 		netif_carrier_off(adapter->netdev);
 		if (affectTxQueue) {
 			for (i = 0; i < adapter->num_tx_queues; i++)
 				vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
 		}
 	}
 }
 static void
 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	unsigned long flags;
 	u32 events = le32_to_cpu(adapter->shared->ecr);
 	if (!events)
 		return;
 	vmxnet3_ack_events(adapter, events);
 	/* Check if link state has changed */
 	if (events & VMXNET3_ECR_LINK)
 		vmxnet3_check_link(adapter, true);
 	/* Check if there is an error on xmit/recv queues */
 	if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
 		spin_lock_irqsave(&adapter->cmd_lock, flags);
 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 				       VMXNET3_CMD_GET_QUEUE_STATUS);
 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 		for (i = 0; i < adapter->num_tx_queues; i++)
 			if (adapter->tqd_start[i].status.stopped)
 				dev_err(&adapter->netdev->dev,
 					"%s: tq[%d] error 0x%x\n",
 					adapter->netdev->name, i, le32_to_cpu(
 					adapter->tqd_start[i].status.error));
 		for (i = 0; i < adapter->num_rx_queues; i++)
 			if (adapter->rqd_start[i].status.stopped)
 				dev_err(&adapter->netdev->dev,
 					"%s: rq[%d] error 0x%x\n",
 					adapter->netdev->name, i,
 					adapter->rqd_start[i].status.error);
 		schedule_work(&adapter->work);
 	}
 }
 #ifdef __BIG_ENDIAN_BITFIELD
 /*
  * The device expects the bitfields in shared structures to be written in
  * little endian. When CPU is big endian, the following routines are used to
  * correctly read and write into ABI.
  * The general technique used here is : double word bitfields are defined in
  * opposite order for big endian architecture. Then before reading them in
  * driver the complete double word is translated using le32_to_cpu. Similarly
  * After the driver writes into bitfields, cpu_to_le32 is used to translate the
  * double words into required format.
  * In order to avoid touching bits in shared structure more than once, temporary
  * descriptors are used. These are passed as srcDesc to following functions.
  */
 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
 				struct Vmxnet3_RxDesc *dstDesc)
 {
 	u32 *src = (u32 *)srcDesc + 2;
 	u32 *dst = (u32 *)dstDesc + 2;
 	dstDesc->addr = le64_to_cpu(srcDesc->addr);
 	*dst = le32_to_cpu(*src);
 	dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
 }
 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
 			       struct Vmxnet3_TxDesc *dstDesc)
 {
 	int i;
 	u32 *src = (u32 *)(srcDesc + 1);
 	u32 *dst = (u32 *)(dstDesc + 1);
 	/* Working backwards so that the gen bit is set at the end. */
 	for (i = 2; i > 0; i--) {
 		src--;
 		dst--;
 		*dst = cpu_to_le32(*src);
 	}
 }
 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
 				struct Vmxnet3_RxCompDesc *dstDesc)
 {
 	int i = 0;
 	u32 *src = (u32 *)srcDesc;
 	u32 *dst = (u32 *)dstDesc;
 	for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
 		*dst = le32_to_cpu(*src);
 		src++;
 		dst++;
 	}
 }
 /* Used to read bitfield values from double words. */
 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
 {
 	u32 temp = le32_to_cpu(*bitfield);
 	u32 mask = ((1 << size) - 1) << pos;
 	temp &= mask;
 	temp >>= pos;
 	return temp;
 }
 #endif  /* __BIG_ENDIAN_BITFIELD */
 #ifdef __BIG_ENDIAN_BITFIELD
 #   define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
 			txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
 			VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
 #   define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
 			txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
 			VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
 #   define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
 			VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
 			VMXNET3_TCD_GEN_SIZE)
 #   define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
 			VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
 			(dstrcd) = (tmp); \
 			vmxnet3_RxCompToCPU((rcd), (tmp)); \
 		} while (0)
 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
 			(dstrxd) = (tmp); \
 			vmxnet3_RxDescToCPU((rxd), (tmp)); \
 		} while (0)
 #else
 #   define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
 #   define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
 #   define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
 #   define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
 #endif /* __BIG_ENDIAN_BITFIELD  */
 static void
 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
 		     struct pci_dev *pdev)
 {
 	if (tbi->map_type == VMXNET3_MAP_SINGLE)
 		dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
 				 PCI_DMA_TODEVICE);
 	else if (tbi->map_type == VMXNET3_MAP_PAGE)
 		dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
 			       PCI_DMA_TODEVICE);
 	else
 		BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
 	tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
 }
 static int
 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
 		  struct pci_dev *pdev,	struct vmxnet3_adapter *adapter)
 {
 	struct sk_buff *skb;
 	int entries = 0;
 	/* no out of order completion */
 	BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
 	BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
 	skb = tq->buf_info[eop_idx].skb;
 	BUG_ON(skb == NULL);
 	tq->buf_info[eop_idx].skb = NULL;
 	VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
 	while (tq->tx_ring.next2comp != eop_idx) {
 		vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
 				     pdev);
 		/* update next2comp w/o tx_lock. Since we are marking more,
 		 * instead of less, tx ring entries avail, the worst case is
 		 * that the tx routine incorrectly re-queues a pkt due to
 		 * insufficient tx ring entries.
 		 */
 		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
 		entries++;
 	}
 	dev_kfree_skb_any(skb);
 	return entries;
 }
 static int
 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
 			struct vmxnet3_adapter *adapter)
 {
 	int completed = 0;
 	union Vmxnet3_GenericDesc *gdesc;
 	gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
 	while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
 		completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
 					       &gdesc->tcd), tq, adapter->pdev,
 					       adapter);
 		vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
 		gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
 	}
 	if (completed) {
 		spin_lock(&tq->tx_lock);
 		if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
 			     vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
 			     VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
 			     netif_carrier_ok(adapter->netdev))) {
 			vmxnet3_tq_wake(tq, adapter);
 		}
 		spin_unlock(&tq->tx_lock);
 	}
 	return completed;
 }
 static void
 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
 		   struct vmxnet3_adapter *adapter)
 {
 	int i;
 	while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
 		struct vmxnet3_tx_buf_info *tbi;
 		tbi = tq->buf_info + tq->tx_ring.next2comp;
 		vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
 		if (tbi->skb) {
 			dev_kfree_skb_any(tbi->skb);
 			tbi->skb = NULL;
 		}
 		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
 	}
 	/* sanity check, verify all buffers are indeed unmapped and freed */
 	for (i = 0; i < tq->tx_ring.size; i++) {
 		BUG_ON(tq->buf_info[i].skb != NULL ||
 		       tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
 	}
 	tq->tx_ring.gen = VMXNET3_INIT_GEN;
 	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
 	tq->comp_ring.gen = VMXNET3_INIT_GEN;
 	tq->comp_ring.next2proc = 0;
 }
 static void
 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
 		   struct vmxnet3_adapter *adapter)
 {
 	if (tq->tx_ring.base) {
 		dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
 				  sizeof(struct Vmxnet3_TxDesc),
 				  tq->tx_ring.base, tq->tx_ring.basePA);
 		tq->tx_ring.base = NULL;
 	}
 	if (tq->data_ring.base) {
 		dma_free_coherent(&adapter->pdev->dev, tq->data_ring.size *
 				  sizeof(struct Vmxnet3_TxDataDesc),
 				  tq->data_ring.base, tq->data_ring.basePA);
 		tq->data_ring.base = NULL;
 	}
 	if (tq->comp_ring.base) {
 		dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
 				  sizeof(struct Vmxnet3_TxCompDesc),
 				  tq->comp_ring.base, tq->comp_ring.basePA);
 		tq->comp_ring.base = NULL;
 	}
 	if (tq->buf_info) {
 		dma_free_coherent(&adapter->pdev->dev,
 				  tq->tx_ring.size * sizeof(tq->buf_info[0]),
 				  tq->buf_info, tq->buf_info_pa);
 		tq->buf_info = NULL;
 	}
 }
 /* Destroy all tx queues */
 void
 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	for (i = 0; i < adapter->num_tx_queues; i++)
 		vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
 }
 static void
 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
 		struct vmxnet3_adapter *adapter)
 {
 	int i;
 	/* reset the tx ring contents to 0 and reset the tx ring states */
 	memset(tq->tx_ring.base, 0, tq->tx_ring.size *
 	       sizeof(struct Vmxnet3_TxDesc));
 	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
 	tq->tx_ring.gen = VMXNET3_INIT_GEN;
 	memset(tq->data_ring.base, 0, tq->data_ring.size *
 	       sizeof(struct Vmxnet3_TxDataDesc));
 	/* reset the tx comp ring contents to 0 and reset comp ring states */
 	memset(tq->comp_ring.base, 0, tq->comp_ring.size *
 	       sizeof(struct Vmxnet3_TxCompDesc));
 	tq->comp_ring.next2proc = 0;
 	tq->comp_ring.gen = VMXNET3_INIT_GEN;
 	/* reset the bookkeeping data */
 	memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
 	for (i = 0; i < tq->tx_ring.size; i++)
 		tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
 	/* stats are not reset */
 }
 static int
 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
 		  struct vmxnet3_adapter *adapter)
 {
 	size_t sz;
 	BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
 	       tq->comp_ring.base || tq->buf_info);
 	tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
 			tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
 			&tq->tx_ring.basePA, GFP_KERNEL);
 	if (!tq->tx_ring.base) {
 		netdev_err(adapter->netdev, "failed to allocate tx ring\n");
 		goto err;
 	}
 	tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
 			tq->data_ring.size * sizeof(struct Vmxnet3_TxDataDesc),
 			&tq->data_ring.basePA, GFP_KERNEL);
 	if (!tq->data_ring.base) {
 		netdev_err(adapter->netdev, "failed to allocate data ring\n");
 		goto err;
 	}
 	tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
 			tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
 			&tq->comp_ring.basePA, GFP_KERNEL);
 	if (!tq->comp_ring.base) {
 		netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
 		goto err;
 	}
 	sz = tq->tx_ring.size * sizeof(tq->buf_info[0]);
 	tq->buf_info = dma_zalloc_coherent(&adapter->pdev->dev, sz,
 					   &tq->buf_info_pa, GFP_KERNEL);
 	if (!tq->buf_info)
 		goto err;
 	return 0;
 err:
 	vmxnet3_tq_destroy(tq, adapter);
 	return -ENOMEM;
 }
 static void
 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	for (i = 0; i < adapter->num_tx_queues; i++)
 		vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
 }
 /*
  *    starting from ring->next2fill, allocate rx buffers for the given ring
  *    of the rx queue and update the rx desc. stop after @num_to_alloc buffers
  *    are allocated or allocation fails
  */
 static int
 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
 			int num_to_alloc, struct vmxnet3_adapter *adapter)
 {
 	int num_allocated = 0;
 	struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
 	struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
 	u32 val;
 	while (num_allocated <= num_to_alloc) {
 		struct vmxnet3_rx_buf_info *rbi;
 		union Vmxnet3_GenericDesc *gd;
 		rbi = rbi_base + ring->next2fill;
 		gd = ring->base + ring->next2fill;
 		if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
 			if (rbi->skb == NULL) {
 				rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
 								       rbi->len,
 								       GFP_KERNEL);
 				if (unlikely(rbi->skb == NULL)) {
 					rq->stats.rx_buf_alloc_failure++;
 					break;
 				}
 				rbi->dma_addr = dma_map_single(
 						&adapter->pdev->dev,
 						rbi->skb->data, rbi->len,
 						PCI_DMA_FROMDEVICE);
 			} else {
 				/* rx buffer skipped by the device */
 			}
 			val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
 		} else {
 			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
 			       rbi->len  != PAGE_SIZE);
 			if (rbi->page == NULL) {
 				rbi->page = alloc_page(GFP_ATOMIC);
 				if (unlikely(rbi->page == NULL)) {
 					rq->stats.rx_buf_alloc_failure++;
 					break;
 				}
 				rbi->dma_addr = dma_map_page(
 						&adapter->pdev->dev,
 						rbi->page, 0, PAGE_SIZE,
 						PCI_DMA_FROMDEVICE);
 			} else {
 				/* rx buffers skipped by the device */
 			}
 			val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
 		}
 		BUG_ON(rbi->dma_addr == 0);
 		gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
 		gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
 					   | val | rbi->len);
 		/* Fill the last buffer but dont mark it ready, or else the
 		 * device will think that the queue is full */
 		if (num_allocated == num_to_alloc)
 			break;
 		gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
 		num_allocated++;
 		vmxnet3_cmd_ring_adv_next2fill(ring);
 	}
 	netdev_dbg(adapter->netdev,
 		"alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
 		num_allocated, ring->next2fill, ring->next2comp);
 	/* so that the device can distinguish a full ring and an empty ring */
 	BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
 	return num_allocated;
 }
 static void
 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
 		    struct vmxnet3_rx_buf_info *rbi)
 {
 	struct skb_frag_struct *frag = skb_shinfo(skb)->frags +
 		skb_shinfo(skb)->nr_frags;
 	BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
 	__skb_frag_set_page(frag, rbi->page);
 	frag->page_offset = 0;
 	skb_frag_size_set(frag, rcd->len);
 	skb->data_len += rcd->len;
 	skb->truesize += PAGE_SIZE;
 	skb_shinfo(skb)->nr_frags++;
 }
 static void
 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
 		struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
 		struct vmxnet3_adapter *adapter)
 {
 	u32 dw2, len;
 	unsigned long buf_offset;
 	int i;
 	union Vmxnet3_GenericDesc *gdesc;
 	struct vmxnet3_tx_buf_info *tbi = NULL;
 	BUG_ON(ctx->copy_size > skb_headlen(skb));
 	/* use the previous gen bit for the SOP desc */
 	dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
 	ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
 	gdesc = ctx->sop_txd; /* both loops below can be skipped */
 	/* no need to map the buffer if headers are copied */
 	if (ctx->copy_size) {
 		ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
 					tq->tx_ring.next2fill *
 					sizeof(struct Vmxnet3_TxDataDesc));
 		ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
 		ctx->sop_txd->dword[3] = 0;
 		tbi = tq->buf_info + tq->tx_ring.next2fill;
 		tbi->map_type = VMXNET3_MAP_NONE;
 		netdev_dbg(adapter->netdev,
 			"txd[%u]: 0x%Lx 0x%x 0x%x\n",
 			tq->tx_ring.next2fill,
 			le64_to_cpu(ctx->sop_txd->txd.addr),
 			ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
 		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 		/* use the right gen for non-SOP desc */
 		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 	}
 	/* linear part can use multiple tx desc if it's big */
 	len = skb_headlen(skb) - ctx->copy_size;
 	buf_offset = ctx->copy_size;
 	while (len) {
 		u32 buf_size;
 		if (len < VMXNET3_MAX_TX_BUF_SIZE) {
 			buf_size = len;
 			dw2 |= len;
 		} else {
 			buf_size = VMXNET3_MAX_TX_BUF_SIZE;
 			/* spec says that for TxDesc.len, 0 == 2^14 */
 		}
 		tbi = tq->buf_info + tq->tx_ring.next2fill;
 		tbi->map_type = VMXNET3_MAP_SINGLE;
 		tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
 				skb->data + buf_offset, buf_size,
 				PCI_DMA_TODEVICE);
 		tbi->len = buf_size;
 		gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
 		BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
 		gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
 		gdesc->dword[2] = cpu_to_le32(dw2);
 		gdesc->dword[3] = 0;
 		netdev_dbg(adapter->netdev,
 			"txd[%u]: 0x%Lx 0x%x 0x%x\n",
 			tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
 			le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
 		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 		len -= buf_size;
 		buf_offset += buf_size;
 	}
 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 		const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 		u32 buf_size;
 		buf_offset = 0;
 		len = skb_frag_size(frag);
 		while (len) {
 			tbi = tq->buf_info + tq->tx_ring.next2fill;
 			if (len < VMXNET3_MAX_TX_BUF_SIZE) {
 				buf_size = len;
 				dw2 |= len;
 			} else {
 				buf_size = VMXNET3_MAX_TX_BUF_SIZE;
 				/* spec says that for TxDesc.len, 0 == 2^14 */
 			}
 			tbi->map_type = VMXNET3_MAP_PAGE;
 			tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
 							 buf_offset, buf_size,
 							 DMA_TO_DEVICE);
 			tbi->len = buf_size;
 			gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
 			BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
 			gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
 			gdesc->dword[2] = cpu_to_le32(dw2);
 			gdesc->dword[3] = 0;
 			netdev_dbg(adapter->netdev,
 				"txd[%u]: 0x%llu %u %u\n",
 				tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
 				le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
 			vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
 			dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
 			len -= buf_size;
 			buf_offset += buf_size;
 		}
 	}
 	ctx->eop_txd = gdesc;
 	/* set the last buf_info for the pkt */
 	tbi->skb = skb;
 	tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
 }
 /* Init all tx queues */
 static void
 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	for (i = 0; i < adapter->num_tx_queues; i++)
 		vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
 }
 /*
  *    parse and copy relevant protocol headers:
  *      For a tso pkt, relevant headers are L2/3/4 including options
  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
  *      if it's a TCP/UDP pkt
  *
  * Returns:
  *    -1:  error happens during parsing
  *     0:  protocol headers parsed, but too big to be copied
  *     1:  protocol headers parsed and copied
  *
  * Other effects:
  *    1. related *ctx fields are updated.
  *    2. ctx->copy_size is # of bytes copied
  *    3. the portion copied is guaranteed to be in the linear part
  *
  */
 static int
 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 			   struct vmxnet3_tx_ctx *ctx,
 			   struct vmxnet3_adapter *adapter)
 {
 	struct Vmxnet3_TxDataDesc *tdd;
 	if (ctx->mss) {	/* TSO */
 		ctx->eth_ip_hdr_size = skb_transport_offset(skb);
 		ctx->l4_hdr_size = tcp_hdrlen(skb);
 		ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size;
 	} else {
 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
 			ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb);
 			if (ctx->ipv4) {
 				const struct iphdr *iph = ip_hdr(skb);
 				if (iph->protocol == IPPROTO_TCP)
 					ctx->l4_hdr_size = tcp_hdrlen(skb);
 				else if (iph->protocol == IPPROTO_UDP)
 					ctx->l4_hdr_size = sizeof(struct udphdr);
 				else
 					ctx->l4_hdr_size = 0;
 			} else {
 				/* for simplicity, don't copy L4 headers */
 				ctx->l4_hdr_size = 0;
 			}
 			ctx->copy_size = min(ctx->eth_ip_hdr_size +
 					 ctx->l4_hdr_size, skb->len);
 		} else {
 			ctx->eth_ip_hdr_size = 0;
 			ctx->l4_hdr_size = 0;
 			/* copy as much as allowed */
 			ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE
 					     , skb_headlen(skb));
 		}
 		/* make sure headers are accessible directly */
 		if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
 			goto err;
 	}
 	if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) {
 		tq->stats.oversized_hdr++;
 		ctx->copy_size = 0;
 		return 0;
 	}
 	tdd = tq->data_ring.base + tq->tx_ring.next2fill;
 	memcpy(tdd->data, skb->data, ctx->copy_size);
 	netdev_dbg(adapter->netdev,
 		"copy %u bytes to dataRing[%u]\n",
 		ctx->copy_size, tq->tx_ring.next2fill);
 	return 1;
 err:
 	return -1;
 }
 static void
 vmxnet3_prepare_tso(struct sk_buff *skb,
 		    struct vmxnet3_tx_ctx *ctx)
 {
 	struct tcphdr *tcph = tcp_hdr(skb);
 	if (ctx->ipv4) {
 		struct iphdr *iph = ip_hdr(skb);
 		iph->check = 0;
 		tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
 						 IPPROTO_TCP, 0);
 	} else {
 		struct ipv6hdr *iph = ipv6_hdr(skb);
 		tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
 					       IPPROTO_TCP, 0);
 	}
 }
 static int txd_estimate(const struct sk_buff *skb)
 {
 	int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
 	int i;
 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 		const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 		count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
 	}
 	return count;
 }
 /*
  * Transmits a pkt thru a given tq
  * Returns:
  *    NETDEV_TX_OK:      descriptors are setup successfully
  *    NETDEV_TX_OK:      error occurred, the pkt is dropped
  *    NETDEV_TX_BUSY:    tx ring is full, queue is stopped
  *
  * Side-effects:
  *    1. tx ring may be changed
  *    2. tq stats may be updated accordingly
  *    3. shared->txNumDeferred may be updated
  */
 static int
 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
 		struct vmxnet3_adapter *adapter, struct net_device *netdev)
 {
 	int ret;
 	u32 count;
 	unsigned long flags;
 	struct vmxnet3_tx_ctx ctx;
 	union Vmxnet3_GenericDesc *gdesc;
 #ifdef __BIG_ENDIAN_BITFIELD
 	/* Use temporary descriptor to avoid touching bits multiple times */
 	union Vmxnet3_GenericDesc tempTxDesc;
 #endif
 	count = txd_estimate(skb);
 	ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
 	ctx.mss = skb_shinfo(skb)->gso_size;
 	if (ctx.mss) {
 		if (skb_header_cloned(skb)) {
 			if (unlikely(pskb_expand_head(skb, 0, 0,
 						      GFP_ATOMIC) != 0)) {
 				tq->stats.drop_tso++;
 				goto drop_pkt;
 			}
 			tq->stats.copy_skb_header++;
 		}
 		vmxnet3_prepare_tso(skb, &ctx);
 	} else {
 		if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
 			/* non-tso pkts must not use more than
 			 * VMXNET3_MAX_TXD_PER_PKT entries
 			 */
 			if (skb_linearize(skb) != 0) {
 				tq->stats.drop_too_many_frags++;
 				goto drop_pkt;
 			}
 			tq->stats.linearized++;
 			/* recalculate the # of descriptors to use */
 			count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
 		}
 	}
 	spin_lock_irqsave(&tq->tx_lock, flags);
 	if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
 		tq->stats.tx_ring_full++;
 		netdev_dbg(adapter->netdev,
 			"tx queue stopped on %s, next2comp %u"
 			" next2fill %u\n", adapter->netdev->name,
 			tq->tx_ring.next2comp, tq->tx_ring.next2fill);
 		vmxnet3_tq_stop(tq, adapter);
 		spin_unlock_irqrestore(&tq->tx_lock, flags);
 		return NETDEV_TX_BUSY;
 	}
 	ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter);
 	if (ret >= 0) {
 		BUG_ON(ret <= 0 && ctx.copy_size != 0);
 		/* hdrs parsed, check against other limits */
 		if (ctx.mss) {
 			if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size >
 				     VMXNET3_MAX_TX_BUF_SIZE)) {
 				goto hdr_too_big;
 			}
 		} else {
 			if (skb->ip_summed == CHECKSUM_PARTIAL) {
 				if (unlikely(ctx.eth_ip_hdr_size +
 					     skb->csum_offset >
 					     VMXNET3_MAX_CSUM_OFFSET)) {
 					goto hdr_too_big;
 				}
 			}
 		}
 	} else {
 		tq->stats.drop_hdr_inspect_err++;
 		goto unlock_drop_pkt;
 	}
 	/* fill tx descs related to addr & len */
 	vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter);
 	/* setup the EOP desc */
 	ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
 	/* setup the SOP desc */
 #ifdef __BIG_ENDIAN_BITFIELD
 	gdesc = &tempTxDesc;
 	gdesc->dword[2] = ctx.sop_txd->dword[2];
 	gdesc->dword[3] = ctx.sop_txd->dword[3];
 #else
 	gdesc = ctx.sop_txd;
 #endif
 	if (ctx.mss) {
 		gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size;
 		gdesc->txd.om = VMXNET3_OM_TSO;
 		gdesc->txd.msscof = ctx.mss;
 		le32_add_cpu(&tq->shared->txNumDeferred, (skb->len -
 			     gdesc->txd.hlen + ctx.mss - 1) / ctx.mss);
 	} else {
 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
 			gdesc->txd.hlen = ctx.eth_ip_hdr_size;
 			gdesc->txd.om = VMXNET3_OM_CSUM;
 			gdesc->txd.msscof = ctx.eth_ip_hdr_size +
 					    skb->csum_offset;
 		} else {
 			gdesc->txd.om = 0;
 			gdesc->txd.msscof = 0;
 		}
 		le32_add_cpu(&tq->shared->txNumDeferred, 1);
 	}
 	if (vlan_tx_tag_present(skb)) {
 		gdesc->txd.ti = 1;
 		gdesc->txd.tci = vlan_tx_tag_get(skb);
 	}
 	/* finally flips the GEN bit of the SOP desc. */
 	gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
 						  VMXNET3_TXD_GEN);
 #ifdef __BIG_ENDIAN_BITFIELD
 	/* Finished updating in bitfields of Tx Desc, so write them in original
 	 * place.
 	 */
 	vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
 			   (struct Vmxnet3_TxDesc *)ctx.sop_txd);
 	gdesc = ctx.sop_txd;
 #endif
 	netdev_dbg(adapter->netdev,
 		"txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
 		(u32)(ctx.sop_txd -
 		tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
 		le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
 	spin_unlock_irqrestore(&tq->tx_lock, flags);
 	if (le32_to_cpu(tq->shared->txNumDeferred) >=
 					le32_to_cpu(tq->shared->txThreshold)) {
 		tq->shared->txNumDeferred = 0;
 		VMXNET3_WRITE_BAR0_REG(adapter,
 				       VMXNET3_REG_TXPROD + tq->qid * 8,
 				       tq->tx_ring.next2fill);
 	}
 	return NETDEV_TX_OK;
 hdr_too_big:
 	tq->stats.drop_oversized_hdr++;
 unlock_drop_pkt:
 	spin_unlock_irqrestore(&tq->tx_lock, flags);
 drop_pkt:
 	tq->stats.drop_total++;
 	dev_kfree_skb(skb);
 	return NETDEV_TX_OK;
 }
 static netdev_tx_t
 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 {
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
 	return vmxnet3_tq_xmit(skb,
 			       &adapter->tx_queue[skb->queue_mapping],
 			       adapter, netdev);
 }
 static void
 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
 		struct sk_buff *skb,
 		union Vmxnet3_GenericDesc *gdesc)
 {
 	if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
 		/* typical case: TCP/UDP over IP and both csums are correct */
 		if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) ==
 							VMXNET3_RCD_CSUM_OK) {
 			skb->ip_summed = CHECKSUM_UNNECESSARY;
 			BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp));
 			BUG_ON(!(gdesc->rcd.v4  || gdesc->rcd.v6));
 			BUG_ON(gdesc->rcd.frg);
 		} else {
 			if (gdesc->rcd.csum) {
 				skb->csum = htons(gdesc->rcd.csum);
 				skb->ip_summed = CHECKSUM_PARTIAL;
 			} else {
 				skb_checksum_none_assert(skb);
 			}
 		}
 	} else {
 		skb_checksum_none_assert(skb);
 	}
 }
 static void
 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
 		 struct vmxnet3_rx_ctx *ctx,  struct vmxnet3_adapter *adapter)
 {
 	rq->stats.drop_err++;
 	if (!rcd->fcs)
 		rq->stats.drop_fcs++;
 	rq->stats.drop_total++;
 	/*
 	 * We do not unmap and chain the rx buffer to the skb.
 	 * We basically pretend this buffer is not used and will be recycled
 	 * by vmxnet3_rq_alloc_rx_buf()
 	 */
 	/*
 	 * ctx->skb may be NULL if this is the first and the only one
 	 * desc for the pkt
 	 */
 	if (ctx->skb)
 		dev_kfree_skb_irq(ctx->skb);
 	ctx->skb = NULL;
 }
 static int
 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
 		       struct vmxnet3_adapter *adapter, int quota)
 {
 	static const u32 rxprod_reg[2] = {
 		VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2
 	};
 	u32 num_rxd = 0;
 	bool skip_page_frags = false;
 	struct Vmxnet3_RxCompDesc *rcd;
 	struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
 #ifdef __BIG_ENDIAN_BITFIELD
 	struct Vmxnet3_RxDesc rxCmdDesc;
 	struct Vmxnet3_RxCompDesc rxComp;
 #endif
 	vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
 			  &rxComp);
 	while (rcd->gen == rq->comp_ring.gen) {
 		struct vmxnet3_rx_buf_info *rbi;
 		struct sk_buff *skb, *new_skb = NULL;
 		struct page *new_page = NULL;
 		int num_to_alloc;
 		struct Vmxnet3_RxDesc *rxd;
 		u32 idx, ring_idx;
 		struct vmxnet3_cmd_ring	*ring = NULL;
 		if (num_rxd >= quota) {
 			/* we may stop even before we see the EOP desc of
 			 * the current pkt
 			 */
 			break;
 		}
 		num_rxd++;
 		BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2);
 		idx = rcd->rxdIdx;
 		ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1;
 		ring = rq->rx_ring + ring_idx;
 		vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
 				  &rxCmdDesc);
 		rbi = rq->buf_info[ring_idx] + idx;
 		BUG_ON(rxd->addr != rbi->dma_addr ||
 		       rxd->len != rbi->len);
 		if (unlikely(rcd->eop && rcd->err)) {
 			vmxnet3_rx_error(rq, rcd, ctx, adapter);
 			goto rcd_done;
 		}
 		if (rcd->sop) { /* first buf of the pkt */
 			BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
 			       rcd->rqID != rq->qid);
 			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
 			BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
 			if (unlikely(rcd->len == 0)) {
 				/* Pretend the rx buffer is skipped. */
 				BUG_ON(!(rcd->sop && rcd->eop));
 				netdev_dbg(adapter->netdev,
 					"rxRing[%u][%u] 0 length\n",
 					ring_idx, idx);
 				goto rcd_done;
 			}
 			skip_page_frags = false;
 			ctx->skb = rbi->skb;
 			new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
 							    rbi->len);
 			if (new_skb == NULL) {
 				/* Skb allocation failed, do not handover this
 				 * skb to stack. Reuse it. Drop the existing pkt
 				 */
 				rq->stats.rx_buf_alloc_failure++;
 				ctx->skb = NULL;
 				rq->stats.drop_total++;
 				skip_page_frags = true;
 				goto rcd_done;
 			}
 			dma_unmap_single(&adapter->pdev->dev, rbi->dma_addr,
 					 rbi->len,
 					 PCI_DMA_FROMDEVICE);
 #ifdef VMXNET3_RSS
 			if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
 			    (adapter->netdev->features & NETIF_F_RXHASH))
 				ctx->skb->rxhash = le32_to_cpu(rcd->rssHash);
 #endif
 			skb_put(ctx->skb, rcd->len);
 			/* Immediate refill */
 			rbi->skb = new_skb;
 			rbi->dma_addr = dma_map_single(&adapter->pdev->dev,
 						       rbi->skb->data, rbi->len,
 						       PCI_DMA_FROMDEVICE);
 			rxd->addr = cpu_to_le64(rbi->dma_addr);
 			rxd->len = rbi->len;
 		} else {
 			BUG_ON(ctx->skb == NULL && !skip_page_frags);
 			/* non SOP buffer must be type 1 in most cases */
 			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
 			BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
 			/* If an sop buffer was dropped, skip all
 			 * following non-sop fragments. They will be reused.
 			 */
 			if (skip_page_frags)
 				goto rcd_done;
 			new_page = alloc_page(GFP_ATOMIC);
 			if (unlikely(new_page == NULL)) {
 				/* Replacement page frag could not be allocated.
 				 * Reuse this page. Drop the pkt and free the
 				 * skb which contained this page as a frag. Skip
 				 * processing all the following non-sop frags.
 				 */
 				rq->stats.rx_buf_alloc_failure++;
 				dev_kfree_skb(ctx->skb);
 				ctx->skb = NULL;
 				skip_page_frags = true;
 				goto rcd_done;
 			}
 			if (rcd->len) {
 				dma_unmap_page(&adapter->pdev->dev,
 					       rbi->dma_addr, rbi->len,
 					       PCI_DMA_FROMDEVICE);
 				vmxnet3_append_frag(ctx->skb, rcd, rbi);
 			}
 			/* Immediate refill */
 			rbi->page = new_page;
 			rbi->dma_addr = dma_map_page(&adapter->pdev->dev,
 						     rbi->page,
 						     0, PAGE_SIZE,
 						     PCI_DMA_FROMDEVICE);
 			rxd->addr = cpu_to_le64(rbi->dma_addr);
 			rxd->len = rbi->len;
 		}
 		skb = ctx->skb;
 		if (rcd->eop) {
 			skb->len += skb->data_len;
 			vmxnet3_rx_csum(adapter, skb,
 					(union Vmxnet3_GenericDesc *)rcd);
 			skb->protocol = eth_type_trans(skb, adapter->netdev);
 			if (unlikely(rcd->ts))
 				__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
 			if (adapter->netdev->features & NETIF_F_LRO)
 				netif_receive_skb(skb);
 			else
 				napi_gro_receive(&rq->napi, skb);
 			ctx->skb = NULL;
 		}
 rcd_done:
 		/* device may have skipped some rx descs */
 		ring->next2comp = idx;
 		num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
 		ring = rq->rx_ring + ring_idx;
 		while (num_to_alloc) {
 			vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
 					  &rxCmdDesc);
 			BUG_ON(!rxd->addr);
 			/* Recv desc is ready to be used by the device */
 			rxd->gen = ring->gen;
 			vmxnet3_cmd_ring_adv_next2fill(ring);
 			num_to_alloc--;
 		}
 		/* if needed, update the register */
 		if (unlikely(rq->shared->updateRxProd)) {
 			VMXNET3_WRITE_BAR0_REG(adapter,
 					       rxprod_reg[ring_idx] + rq->qid * 8,
 					       ring->next2fill);
 		}
 		vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
 		vmxnet3_getRxComp(rcd,
 				  &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
 	}
 	return num_rxd;
 }
 static void
 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
 		   struct vmxnet3_adapter *adapter)
 {
 	u32 i, ring_idx;
 	struct Vmxnet3_RxDesc *rxd;
 	for (ring_idx = 0; ring_idx < 2; ring_idx++) {
 		for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
 #ifdef __BIG_ENDIAN_BITFIELD
 			struct Vmxnet3_RxDesc rxDesc;
 #endif
 			vmxnet3_getRxDesc(rxd,
 				&rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
 			if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
 					rq->buf_info[ring_idx][i].skb) {
 				dma_unmap_single(&adapter->pdev->dev, rxd->addr,
 						 rxd->len, PCI_DMA_FROMDEVICE);
 				dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
 				rq->buf_info[ring_idx][i].skb = NULL;
 			} else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
 					rq->buf_info[ring_idx][i].page) {
 				dma_unmap_page(&adapter->pdev->dev, rxd->addr,
 					       rxd->len, PCI_DMA_FROMDEVICE);
 				put_page(rq->buf_info[ring_idx][i].page);
 				rq->buf_info[ring_idx][i].page = NULL;
 			}
 		}
 		rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
 		rq->rx_ring[ring_idx].next2fill =
 					rq->rx_ring[ring_idx].next2comp = 0;
 	}
 	rq->comp_ring.gen = VMXNET3_INIT_GEN;
 	rq->comp_ring.next2proc = 0;
 }
 static void
 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	for (i = 0; i < adapter->num_rx_queues; i++)
 		vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
 }
 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
 			       struct vmxnet3_adapter *adapter)
 {
 	int i;
 	int j;
 	/* all rx buffers must have already been freed */
 	for (i = 0; i < 2; i++) {
 		if (rq->buf_info[i]) {
 			for (j = 0; j < rq->rx_ring[i].size; j++)
 				BUG_ON(rq->buf_info[i][j].page != NULL);
 		}
 	}
 	for (i = 0; i < 2; i++) {
 		if (rq->rx_ring[i].base) {
 			dma_free_coherent(&adapter->pdev->dev,
 					  rq->rx_ring[i].size
 					  * sizeof(struct Vmxnet3_RxDesc),
 					  rq->rx_ring[i].base,
 					  rq->rx_ring[i].basePA);
 			rq->rx_ring[i].base = NULL;
 		}
 		rq->buf_info[i] = NULL;
 	}
 	if (rq->comp_ring.base) {
 		dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
 				  * sizeof(struct Vmxnet3_RxCompDesc),
 				  rq->comp_ring.base, rq->comp_ring.basePA);
 		rq->comp_ring.base = NULL;
 	}
 	if (rq->buf_info[0]) {
 		size_t sz = sizeof(struct vmxnet3_rx_buf_info) *
 			(rq->rx_ring[0].size + rq->rx_ring[1].size);
 		dma_free_coherent(&adapter->pdev->dev, sz, rq->buf_info[0],
 				  rq->buf_info_pa);
 	}
 }
 static int
 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
 		struct vmxnet3_adapter  *adapter)
 {
 	int i;
 	/* initialize buf_info */
 	for (i = 0; i < rq->rx_ring[0].size; i++) {
 		/* 1st buf for a pkt is skbuff */
 		if (i % adapter->rx_buf_per_pkt == 0) {
 			rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
 			rq->buf_info[0][i].len = adapter->skb_buf_size;
 		} else { /* subsequent bufs for a pkt is frag */
 			rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
 			rq->buf_info[0][i].len = PAGE_SIZE;
 		}
 	}
 	for (i = 0; i < rq->rx_ring[1].size; i++) {
 		rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
 		rq->buf_info[1][i].len = PAGE_SIZE;
 	}
 	/* reset internal state and allocate buffers for both rings */
 	for (i = 0; i < 2; i++) {
 		rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
 		memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
 		       sizeof(struct Vmxnet3_RxDesc));
 		rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
 	}
 	if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
 				    adapter) == 0) {
 		/* at least has 1 rx buffer for the 1st ring */
 		return -ENOMEM;
 	}
 	vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
 	/* reset the comp ring */
 	rq->comp_ring.next2proc = 0;
 	memset(rq->comp_ring.base, 0, rq->comp_ring.size *
 	       sizeof(struct Vmxnet3_RxCompDesc));
 	rq->comp_ring.gen = VMXNET3_INIT_GEN;
 	/* reset rxctx */
 	rq->rx_ctx.skb = NULL;
 	/* stats are not reset */
 	return 0;
 }
 static int
 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
 {
 	int i, err = 0;
 	for (i = 0; i < adapter->num_rx_queues; i++) {
 		err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
 		if (unlikely(err)) {
 			dev_err(&adapter->netdev->dev, "%s: failed to "
 				"initialize rx queue%i\n",
 				adapter->netdev->name, i);
 			break;
 		}
 	}
 	return err;
 }
 static int
 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
 {
 	int i;
 	size_t sz;
 	struct vmxnet3_rx_buf_info *bi;
 	for (i = 0; i < 2; i++) {
 		sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
 		rq->rx_ring[i].base = dma_alloc_coherent(
 						&adapter->pdev->dev, sz,
 						&rq->rx_ring[i].basePA,
 						GFP_KERNEL);
 		if (!rq->rx_ring[i].base) {
 			netdev_err(adapter->netdev,
 				   "failed to allocate rx ring %d\n", i);
 			goto err;
 		}
 	}
 	sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
 	rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
 						&rq->comp_ring.basePA,
 						GFP_KERNEL);
 	if (!rq->comp_ring.base) {
 		netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
 		goto err;
 	}
 	sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size +
 						   rq->rx_ring[1].size);
 	bi = dma_zalloc_coherent(&adapter->pdev->dev, sz, &rq->buf_info_pa,
 				 GFP_KERNEL);
 	if (!bi)
 		goto err;
 	rq->buf_info[0] = bi;
 	rq->buf_info[1] = bi + rq->rx_ring[0].size;
 	return 0;
 err:
 	vmxnet3_rq_destroy(rq, adapter);
 	return -ENOMEM;
 }
 static int
 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
 {
 	int i, err = 0;
 	for (i = 0; i < adapter->num_rx_queues; i++) {
 		err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
 		if (unlikely(err)) {
 			dev_err(&adapter->netdev->dev,
 				"%s: failed to create rx queue%i\n",
 				adapter->netdev->name, i);
 			goto err_out;
 		}
 	}
 	return err;
 err_out:
 	vmxnet3_rq_destroy_all(adapter);
 	return err;
 }
 /* Multiple queue aware polling function for tx and rx */
 static int
 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
 {
 	int rcd_done = 0, i;
 	if (unlikely(adapter->shared->ecr))
 		vmxnet3_process_events(adapter);
 	for (i = 0; i < adapter->num_tx_queues; i++)
 		vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
 	for (i = 0; i < adapter->num_rx_queues; i++)
 		rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
 						   adapter, budget);
 	return rcd_done;
 }
 static int
 vmxnet3_poll(struct napi_struct *napi, int budget)
 {
 	struct vmxnet3_rx_queue *rx_queue = container_of(napi,
 					  struct vmxnet3_rx_queue, napi);
 	int rxd_done;
 	rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
 	if (rxd_done < budget) {
 		napi_complete(napi);
 		vmxnet3_enable_all_intrs(rx_queue->adapter);
 	}
 	return rxd_done;
 }
 /*
  * NAPI polling function for MSI-X mode with multiple Rx queues
  * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
  */
 static int
 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
 {
 	struct vmxnet3_rx_queue *rq = container_of(napi,
 						struct vmxnet3_rx_queue, napi);
 	struct vmxnet3_adapter *adapter = rq->adapter;
 	int rxd_done;
 	/* When sharing interrupt with corresponding tx queue, process
 	 * tx completions in that queue as well
 	 */
 	if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
 		struct vmxnet3_tx_queue *tq =
 				&adapter->tx_queue[rq - adapter->rx_queue];
 		vmxnet3_tq_tx_complete(tq, adapter);
 	}
 	rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
 	if (rxd_done < budget) {
 		napi_complete(napi);
 		vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
 	}
 	return rxd_done;
 }
 #ifdef CONFIG_PCI_MSI
 /*
  * Handle completion interrupts on tx queues
  * Returns whether or not the intr is handled
  */
 static irqreturn_t
 vmxnet3_msix_tx(int irq, void *data)
 {
 	struct vmxnet3_tx_queue *tq = data;
 	struct vmxnet3_adapter *adapter = tq->adapter;
 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
 		vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
 	/* Handle the case where only one irq is allocate for all tx queues */
 	if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
 		int i;
 		for (i = 0; i < adapter->num_tx_queues; i++) {
 			struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
 			vmxnet3_tq_tx_complete(txq, adapter);
 		}
 	} else {
 		vmxnet3_tq_tx_complete(tq, adapter);
 	}
 	vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
 	return IRQ_HANDLED;
 }
 /*
  * Handle completion interrupts on rx queues. Returns whether or not the
  * intr is handled
  */
 static irqreturn_t
 vmxnet3_msix_rx(int irq, void *data)
 {
 	struct vmxnet3_rx_queue *rq = data;
 	struct vmxnet3_adapter *adapter = rq->adapter;
 	/* disable intr if needed */
 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
 		vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
 	napi_schedule(&rq->napi);
 	return IRQ_HANDLED;
 }
 /*
  *----------------------------------------------------------------------------
  *
  * vmxnet3_msix_event --
  *
  *    vmxnet3 msix event intr handler
  *
  * Result:
  *    whether or not the intr is handled
  *
  *----------------------------------------------------------------------------
  */
 static irqreturn_t
 vmxnet3_msix_event(int irq, void *data)
 {
 	struct net_device *dev = data;
 	struct vmxnet3_adapter *adapter = netdev_priv(dev);
 	/* disable intr if needed */
 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
 		vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
 	if (adapter->shared->ecr)
 		vmxnet3_process_events(adapter);
 	vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
 	return IRQ_HANDLED;
 }
 #endif /* CONFIG_PCI_MSI  */
 /* Interrupt handler for vmxnet3  */
 static irqreturn_t
 vmxnet3_intr(int irq, void *dev_id)
 {
 	struct net_device *dev = dev_id;
 	struct vmxnet3_adapter *adapter = netdev_priv(dev);
 	if (adapter->intr.type == VMXNET3_IT_INTX) {
 		u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
 		if (unlikely(icr == 0))
 			/* not ours */
 			return IRQ_NONE;
 	}
 	/* disable intr if needed */
 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
 		vmxnet3_disable_all_intrs(adapter);
 	napi_schedule(&adapter->rx_queue[0].napi);
 	return IRQ_HANDLED;
 }
 #ifdef CONFIG_NET_POLL_CONTROLLER
 /* netpoll callback. */
 static void
 vmxnet3_netpoll(struct net_device *netdev)
 {
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
 		vmxnet3_disable_all_intrs(adapter);
 	vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size);
 	vmxnet3_enable_all_intrs(adapter);
 }
 #endif	/* CONFIG_NET_POLL_CONTROLLER */
 static int
 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
 {
 	struct vmxnet3_intr *intr = &adapter->intr;
 	int err = 0, i;
 	int vector = 0;
 #ifdef CONFIG_PCI_MSI
 	if (adapter->intr.type == VMXNET3_IT_MSIX) {
 		for (i = 0; i < adapter->num_tx_queues; i++) {
 			if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
 				sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
 					adapter->netdev->name, vector);
 				err = request_irq(
 					      intr->msix_entries[vector].vector,
 					      vmxnet3_msix_tx, 0,
 					      adapter->tx_queue[i].name,
 					      &adapter->tx_queue[i]);
 			} else {
 				sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
 					adapter->netdev->name, vector);
 			}
 			if (err) {
 				dev_err(&adapter->netdev->dev,
 					"Failed to request irq for MSIX, %s, "
 					"error %d\n",
 					adapter->tx_queue[i].name, err);
 				return err;
 			}
 			/* Handle the case where only 1 MSIx was allocated for
 			 * all tx queues */
 			if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
 				for (; i < adapter->num_tx_queues; i++)
 					adapter->tx_queue[i].comp_ring.intr_idx
 								= vector;
 				vector++;
 				break;
 			} else {
 				adapter->tx_queue[i].comp_ring.intr_idx
 								= vector++;
 			}
 		}
 		if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
 			vector = 0;
 		for (i = 0; i < adapter->num_rx_queues; i++) {
 			if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
 				sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
 					adapter->netdev->name, vector);
 			else
 				sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
 					adapter->netdev->name, vector);
 			err = request_irq(intr->msix_entries[vector].vector,
 					  vmxnet3_msix_rx, 0,
 					  adapter->rx_queue[i].name,
 					  &(adapter->rx_queue[i]));
 			if (err) {
 				netdev_err(adapter->netdev,
 					   "Failed to request irq for MSIX, "
 					   "%s, error %d\n",
 					   adapter->rx_queue[i].name, err);
 				return err;
 			}
 			adapter->rx_queue[i].comp_ring.intr_idx = vector++;
 		}
 		sprintf(intr->event_msi_vector_name, "%s-event-%d",
 			adapter->netdev->name, vector);
 		err = request_irq(intr->msix_entries[vector].vector,
 				  vmxnet3_msix_event, 0,
 				  intr->event_msi_vector_name, adapter->netdev);
 		intr->event_intr_idx = vector;
 	} else if (intr->type == VMXNET3_IT_MSI) {
 		adapter->num_rx_queues = 1;
 		err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
 				  adapter->netdev->name, adapter->netdev);
 	} else {
 #endif
 		adapter->num_rx_queues = 1;
 		err = request_irq(adapter->pdev->irq, vmxnet3_intr,
 				  IRQF_SHARED, adapter->netdev->name,
 				  adapter->netdev);
 #ifdef CONFIG_PCI_MSI
 	}
 #endif
 	intr->num_intrs = vector + 1;
 	if (err) {
 		netdev_err(adapter->netdev,
 			   "Failed to request irq (intr type:%d), error %d\n",
 			   intr->type, err);
 	} else {
 		/* Number of rx queues will not change after this */
 		for (i = 0; i < adapter->num_rx_queues; i++) {
 			struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
 			rq->qid = i;
 			rq->qid2 = i + adapter->num_rx_queues;
 		}
 		/* init our intr settings */
 		for (i = 0; i < intr->num_intrs; i++)
 			intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
 		if (adapter->intr.type != VMXNET3_IT_MSIX) {
 			adapter->intr.event_intr_idx = 0;
 			for (i = 0; i < adapter->num_tx_queues; i++)
 				adapter->tx_queue[i].comp_ring.intr_idx = 0;
 			adapter->rx_queue[0].comp_ring.intr_idx = 0;
 		}
 		netdev_info(adapter->netdev,
 			    "intr type %u, mode %u, %u vectors allocated\n",
 			    intr->type, intr->mask_mode, intr->num_intrs);
 	}
 	return err;
 }
 static void
 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
 {
 	struct vmxnet3_intr *intr = &adapter->intr;
 	BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
 	switch (intr->type) {
 #ifdef CONFIG_PCI_MSI
 	case VMXNET3_IT_MSIX:
 	{
 		int i, vector = 0;
 		if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
 			for (i = 0; i < adapter->num_tx_queues; i++) {
 				free_irq(intr->msix_entries[vector++].vector,
 					 &(adapter->tx_queue[i]));
 				if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
 					break;
 			}
 		}
 		for (i = 0; i < adapter->num_rx_queues; i++) {
 			free_irq(intr->msix_entries[vector++].vector,
 				 &(adapter->rx_queue[i]));
 		}
 		free_irq(intr->msix_entries[vector].vector,
 			 adapter->netdev);
 		BUG_ON(vector >= intr->num_intrs);
 		break;
 	}
 #endif
 	case VMXNET3_IT_MSI:
 		free_irq(adapter->pdev->irq, adapter->netdev);
 		break;
 	case VMXNET3_IT_INTX:
 		free_irq(adapter->pdev->irq, adapter->netdev);
 		break;
 	default:
 		BUG();
 	}
 }
 static void
 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
 {
 	u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
 	u16 vid;
 	/* allow untagged pkts */
 	VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
 		VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
 }
 static int
 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
 {
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	if (!(netdev->flags & IFF_PROMISC)) {
 		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
 		unsigned long flags;
 		VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
 		spin_lock_irqsave(&adapter->cmd_lock, flags);
 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 	}
 	set_bit(vid, adapter->active_vlans);
 	return 0;
 }
 static int
 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
 {
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	if (!(netdev->flags & IFF_PROMISC)) {
 		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
 		unsigned long flags;
 		VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
 		spin_lock_irqsave(&adapter->cmd_lock, flags);
 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 	}
 	clear_bit(vid, adapter->active_vlans);
 	return 0;
 }
 static u8 *
 vmxnet3_copy_mc(struct net_device *netdev)
 {
 	u8 *buf = NULL;
 	u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
 	/* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
 	if (sz <= 0xffff) {
 		/* We may be called with BH disabled */
 		buf = kmalloc(sz, GFP_ATOMIC);
 		if (buf) {
 			struct netdev_hw_addr *ha;
 			int i = 0;
 			netdev_for_each_mc_addr(ha, netdev)
 				memcpy(buf + i++ * ETH_ALEN, ha->addr,
 				       ETH_ALEN);
 		}
 	}
 	return buf;
 }
 static void
 vmxnet3_set_mc(struct net_device *netdev)
 {
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	unsigned long flags;
 	struct Vmxnet3_RxFilterConf *rxConf =
 					&adapter->shared->devRead.rxFilterConf;
 	u8 *new_table = NULL;
 	dma_addr_t new_table_pa = 0;
 	u32 new_mode = VMXNET3_RXM_UCAST;
 	if (netdev->flags & IFF_PROMISC) {
 		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
 		memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
 		new_mode |= VMXNET3_RXM_PROMISC;
 	} else {
 		vmxnet3_restore_vlan(adapter);
 	}
 	if (netdev->flags & IFF_BROADCAST)
 		new_mode |= VMXNET3_RXM_BCAST;
 	if (netdev->flags & IFF_ALLMULTI)
 		new_mode |= VMXNET3_RXM_ALL_MULTI;
 	else
 		if (!netdev_mc_empty(netdev)) {
 			new_table = vmxnet3_copy_mc(netdev);
 			if (new_table) {
 				new_mode |= VMXNET3_RXM_MCAST;
 				rxConf->mfTableLen = cpu_to_le16(
 					netdev_mc_count(netdev) * ETH_ALEN);
 				new_table_pa = dma_map_single(
 							&adapter->pdev->dev,
 							new_table,
 							rxConf->mfTableLen,
 							PCI_DMA_TODEVICE);
 				rxConf->mfTablePA = cpu_to_le64(new_table_pa);
 			} else {
 				netdev_info(netdev, "failed to copy mcast list"
 					    ", setting ALL_MULTI\n");
 				new_mode |= VMXNET3_RXM_ALL_MULTI;
 			}
 		}
 	if (!(new_mode & VMXNET3_RXM_MCAST)) {
 		rxConf->mfTableLen = 0;
 		rxConf->mfTablePA = 0;
 	}
 	spin_lock_irqsave(&adapter->cmd_lock, flags);
 	if (new_mode != rxConf->rxMode) {
 		rxConf->rxMode = cpu_to_le32(new_mode);
 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 				       VMXNET3_CMD_UPDATE_RX_MODE);
 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
 	}
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 			       VMXNET3_CMD_UPDATE_MAC_FILTERS);
 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 	if (new_table) {
 		dma_unmap_single(&adapter->pdev->dev, new_table_pa,
 				 rxConf->mfTableLen, PCI_DMA_TODEVICE);
 		kfree(new_table);
 	}
 }
 void
 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	for (i = 0; i < adapter->num_rx_queues; i++)
 		vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
 }
 /*
  *   Set up driver_shared based on settings in adapter.
  */
 static void
 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
 {
 	struct Vmxnet3_DriverShared *shared = adapter->shared;
 	struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
 	struct Vmxnet3_TxQueueConf *tqc;
 	struct Vmxnet3_RxQueueConf *rqc;
 	int i;
 	memset(shared, 0, sizeof(*shared));
 	/* driver settings */
 	shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
 	devRead->misc.driverInfo.version = cpu_to_le32(
 						VMXNET3_DRIVER_VERSION_NUM);
 	devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
 				VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
 	devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
 	*((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
 				*((u32 *)&devRead->misc.driverInfo.gos));
 	devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
 	devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
 	devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
 	devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
 	/* set up feature flags */
 	if (adapter->netdev->features & NETIF_F_RXCSUM)
 		devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
 	if (adapter->netdev->features & NETIF_F_LRO) {
 		devRead->misc.uptFeatures |= UPT1_F_LRO;
 		devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
 	}
 	if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
 		devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
 	devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
 	devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
 	devRead->misc.queueDescLen = cpu_to_le32(
 		adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
 		adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
 	/* tx queue settings */
 	devRead->misc.numTxQueues =  adapter->num_tx_queues;
 	for (i = 0; i < adapter->num_tx_queues; i++) {
 		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];
 		BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
 		tqc = &adapter->tqd_start[i].conf;
 		tqc->txRingBasePA   = cpu_to_le64(tq->tx_ring.basePA);
 		tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
 		tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
 		tqc->ddPA           = cpu_to_le64(tq->buf_info_pa);
 		tqc->txRingSize     = cpu_to_le32(tq->tx_ring.size);
 		tqc->dataRingSize   = cpu_to_le32(tq->data_ring.size);
 		tqc->compRingSize   = cpu_to_le32(tq->comp_ring.size);
 		tqc->ddLen          = cpu_to_le32(
 					sizeof(struct vmxnet3_tx_buf_info) *
 					tqc->txRingSize);
 		tqc->intrIdx        = tq->comp_ring.intr_idx;
 	}
 	/* rx queue settings */
 	devRead->misc.numRxQueues = adapter->num_rx_queues;
 	for (i = 0; i < adapter->num_rx_queues; i++) {
 		struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[i];
 		rqc = &adapter->rqd_start[i].conf;
 		rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
 		rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
 		rqc->compRingBasePA  = cpu_to_le64(rq->comp_ring.basePA);
 		rqc->ddPA            = cpu_to_le64(rq->buf_info_pa);
 		rqc->rxRingSize[0]   = cpu_to_le32(rq->rx_ring[0].size);
 		rqc->rxRingSize[1]   = cpu_to_le32(rq->rx_ring[1].size);
 		rqc->compRingSize    = cpu_to_le32(rq->comp_ring.size);
 		rqc->ddLen           = cpu_to_le32(
 					sizeof(struct vmxnet3_rx_buf_info) *
 					(rqc->rxRingSize[0] +
 					 rqc->rxRingSize[1]));
 		rqc->intrIdx         = rq->comp_ring.intr_idx;
 	}
 #ifdef VMXNET3_RSS
 	memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
 	if (adapter->rss) {
 		struct UPT1_RSSConf *rssConf = adapter->rss_conf;
 		static const uint8_t rss_key[UPT1_RSS_MAX_KEY_SIZE] = {
 			0x3b, 0x56, 0xd1, 0x56, 0x13, 0x4a, 0xe7, 0xac,
 			0xe8, 0x79, 0x09, 0x75, 0xe8, 0x65, 0x79, 0x28,
 			0x35, 0x12, 0xb9, 0x56, 0x7c, 0x76, 0x4b, 0x70,
 			0xd8, 0x56, 0xa3, 0x18, 0x9b, 0x0a, 0xee, 0xf3,
 			0x96, 0xa6, 0x9f, 0x8f, 0x9e, 0x8c, 0x90, 0xc9,
 		};
 		devRead->misc.uptFeatures |= UPT1_F_RSS;
 		devRead->misc.numRxQueues = adapter->num_rx_queues;
 		rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
 				    UPT1_RSS_HASH_TYPE_IPV4 |
 				    UPT1_RSS_HASH_TYPE_TCP_IPV6 |
 				    UPT1_RSS_HASH_TYPE_IPV6;
 		rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
 		rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
 		rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
 		memcpy(rssConf->hashKey, rss_key, sizeof(rss_key));
 		for (i = 0; i < rssConf->indTableSize; i++)
 			rssConf->indTable[i] = ethtool_rxfh_indir_default(
 				i, adapter->num_rx_queues);
 		devRead->rssConfDesc.confVer = 1;
 		devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
 		devRead->rssConfDesc.confPA =
 			cpu_to_le64(adapter->rss_conf_pa);
 	}
 #endif /* VMXNET3_RSS */
 	/* intr settings */
 	devRead->intrConf.autoMask = adapter->intr.mask_mode ==
 				     VMXNET3_IMM_AUTO;
 	devRead->intrConf.numIntrs = adapter->intr.num_intrs;
 	for (i = 0; i < adapter->intr.num_intrs; i++)
 		devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
 	devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
 	devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
 	/* rx filter settings */
 	devRead->rxFilterConf.rxMode = 0;
 	vmxnet3_restore_vlan(adapter);
 	vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
 	/* the rest are already zeroed */
 }
 int
 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
 {
 	int err, i;
 	u32 ret;
 	unsigned long flags;
 	netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
 		" ring sizes %u %u %u\n", adapter->netdev->name,
 		adapter->skb_buf_size, adapter->rx_buf_per_pkt,
 		adapter->tx_queue[0].tx_ring.size,
 		adapter->rx_queue[0].rx_ring[0].size,
 		adapter->rx_queue[0].rx_ring[1].size);
 	vmxnet3_tq_init_all(adapter);
 	err = vmxnet3_rq_init_all(adapter);
 	if (err) {
 		netdev_err(adapter->netdev,
 			   "Failed to init rx queue error %d\n", err);
 		goto rq_err;
 	}
 	err = vmxnet3_request_irqs(adapter);
 	if (err) {
 		netdev_err(adapter->netdev,
 			   "Failed to setup irq for error %d\n", err);
 		goto irq_err;
 	}
 	vmxnet3_setup_driver_shared(adapter);
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
 			       adapter->shared_pa));
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
 			       adapter->shared_pa));
 	spin_lock_irqsave(&adapter->cmd_lock, flags);
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 			       VMXNET3_CMD_ACTIVATE_DEV);
 	ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 	if (ret != 0) {
 		netdev_err(adapter->netdev,
 			   "Failed to activate dev: error %u\n", ret);
 		err = -EINVAL;
 		goto activate_err;
 	}
 	for (i = 0; i < adapter->num_rx_queues; i++) {
 		VMXNET3_WRITE_BAR0_REG(adapter,
 				VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN,
 				adapter->rx_queue[i].rx_ring[0].next2fill);
 		VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 +
 				(i * VMXNET3_REG_ALIGN)),
 				adapter->rx_queue[i].rx_ring[1].next2fill);
 	}
 	/* Apply the rx filter settins last. */
 	vmxnet3_set_mc(adapter->netdev);
 	/*
 	 * Check link state when first activating device. It will start the
 	 * tx queue if the link is up.
 	 */
 	vmxnet3_check_link(adapter, true);
 	for (i = 0; i < adapter->num_rx_queues; i++)
 		napi_enable(&adapter->rx_queue[i].napi);
 	vmxnet3_enable_all_intrs(adapter);
 	clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
 	return 0;
 activate_err:
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
 	vmxnet3_free_irqs(adapter);
 irq_err:
 rq_err:
 	/* free up buffers we allocated */
 	vmxnet3_rq_cleanup_all(adapter);
 	return err;
 }
 void
 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
 {
 	unsigned long flags;
 	spin_lock_irqsave(&adapter->cmd_lock, flags);
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 }
 int
 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	unsigned long flags;
 	if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
 		return 0;
 	spin_lock_irqsave(&adapter->cmd_lock, flags);
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 			       VMXNET3_CMD_QUIESCE_DEV);
 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 	vmxnet3_disable_all_intrs(adapter);
 	for (i = 0; i < adapter->num_rx_queues; i++)
 		napi_disable(&adapter->rx_queue[i].napi);
 	netif_tx_disable(adapter->netdev);
 	adapter->link_speed = 0;
 	netif_carrier_off(adapter->netdev);
 	vmxnet3_tq_cleanup_all(adapter);
 	vmxnet3_rq_cleanup_all(adapter);
 	vmxnet3_free_irqs(adapter);
 	return 0;
 }
 static void
 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
 {
 	u32 tmp;
 	tmp = *(u32 *)mac;
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
 	tmp = (mac[5] << 8) | mac[4];
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
 }
 static int
 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
 {
 	struct sockaddr *addr = p;
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 	vmxnet3_write_mac_addr(adapter, addr->sa_data);
 	return 0;
 }
 /* ==================== initialization and cleanup routines ============ */
 static int
 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64)
 {
 	int err;
 	unsigned long mmio_start, mmio_len;
 	struct pci_dev *pdev = adapter->pdev;
 	err = pci_enable_device(pdev);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
 		return err;
 	}
 	if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) {
 		if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) {
 			dev_err(&pdev->dev,
 				"pci_set_consistent_dma_mask failed\n");
 			err = -EIO;
 			goto err_set_mask;
 		}
 		*dma64 = true;
 	} else {
 		if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) {
 			dev_err(&pdev->dev,
 				"pci_set_dma_mask failed\n");
 			err = -EIO;
 			goto err_set_mask;
 		}
 		*dma64 = false;
 	}
 	err = pci_request_selected_regions(pdev, (1 << 2) - 1,
 					   vmxnet3_driver_name);
 	if (err) {
 		dev_err(&pdev->dev,
 			"Failed to request region for adapter: error %d\n", err);
 		goto err_set_mask;
 	}
 	pci_set_master(pdev);
 	mmio_start = pci_resource_start(pdev, 0);
 	mmio_len = pci_resource_len(pdev, 0);
 	adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
 	if (!adapter->hw_addr0) {
 		dev_err(&pdev->dev, "Failed to map bar0\n");
 		err = -EIO;
 		goto err_ioremap;
 	}
 	mmio_start = pci_resource_start(pdev, 1);
 	mmio_len = pci_resource_len(pdev, 1);
 	adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
 	if (!adapter->hw_addr1) {
 		dev_err(&pdev->dev, "Failed to map bar1\n");
 		err = -EIO;
 		goto err_bar1;
 	}
 	return 0;
 err_bar1:
 	iounmap(adapter->hw_addr0);
 err_ioremap:
 	pci_release_selected_regions(pdev, (1 << 2) - 1);
 err_set_mask:
 	pci_disable_device(pdev);
 	return err;
 }
 static void
 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
 {
 	BUG_ON(!adapter->pdev);
 	iounmap(adapter->hw_addr0);
 	iounmap(adapter->hw_addr1);
 	pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
 	pci_disable_device(adapter->pdev);
 }
 static void
 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
 {
 	size_t sz, i, ring0_size, ring1_size, comp_size;
 	struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[0];
 	if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
 				    VMXNET3_MAX_ETH_HDR_SIZE) {
 		adapter->skb_buf_size = adapter->netdev->mtu +
 					VMXNET3_MAX_ETH_HDR_SIZE;
 		if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
 			adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
 		adapter->rx_buf_per_pkt = 1;
 	} else {
 		adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
 		sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
 					    VMXNET3_MAX_ETH_HDR_SIZE;
 		adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
 	}
 	/*
 	 * for simplicity, force the ring0 size to be a multiple of
 	 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
 	 */
 	sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
 	ring0_size = adapter->rx_queue[0].rx_ring[0].size;
 	ring0_size = (ring0_size + sz - 1) / sz * sz;
 	ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
 			   sz * sz);
 	ring1_size = adapter->rx_queue[0].rx_ring[1].size;
 	comp_size = ring0_size + ring1_size;
 	for (i = 0; i < adapter->num_rx_queues; i++) {
 		rq = &adapter->rx_queue[i];
 		rq->rx_ring[0].size = ring0_size;
 		rq->rx_ring[1].size = ring1_size;
 		rq->comp_ring.size = comp_size;
 	}
 }
 int
 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
 		      u32 rx_ring_size, u32 rx_ring2_size)
 {
 	int err = 0, i;
 	for (i = 0; i < adapter->num_tx_queues; i++) {
 		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];
 		tq->tx_ring.size   = tx_ring_size;
 		tq->data_ring.size = tx_ring_size;
 		tq->comp_ring.size = tx_ring_size;
 		tq->shared = &adapter->tqd_start[i].ctrl;
 		tq->stopped = true;
 		tq->adapter = adapter;
 		tq->qid = i;
 		err = vmxnet3_tq_create(tq, adapter);
 		/*
 		 * Too late to change num_tx_queues. We cannot do away with
 		 * lesser number of queues than what we asked for
 		 */
 		if (err)
 			goto queue_err;
 	}
 	adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
 	adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
 	vmxnet3_adjust_rx_ring_size(adapter);
 	for (i = 0; i < adapter->num_rx_queues; i++) {
 		struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
 		/* qid and qid2 for rx queues will be assigned later when num
 		 * of rx queues is finalized after allocating intrs */
 		rq->shared = &adapter->rqd_start[i].ctrl;
 		rq->adapter = adapter;
 		err = vmxnet3_rq_create(rq, adapter);
 		if (err) {
 			if (i == 0) {
 				netdev_err(adapter->netdev,
 					   "Could not allocate any rx queues. "
 					   "Aborting.\n");
 				goto queue_err;
 			} else {
 				netdev_info(adapter->netdev,
 					    "Number of rx queues changed "
 					    "to : %d.\n", i);
 				adapter->num_rx_queues = i;
 				err = 0;
 				break;
 			}
 		}
 	}
 	return err;
 queue_err:
 	vmxnet3_tq_destroy_all(adapter);
 	return err;
 }
 static int
 vmxnet3_open(struct net_device *netdev)
 {
 	struct vmxnet3_adapter *adapter;
 	int err, i;
 	adapter = netdev_priv(netdev);
 	for (i = 0; i < adapter->num_tx_queues; i++)
 		spin_lock_init(&adapter->tx_queue[i].tx_lock);
 	err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE,
 				    VMXNET3_DEF_RX_RING_SIZE,
 				    VMXNET3_DEF_RX_RING_SIZE);
 	if (err)
 		goto queue_err;
 	err = vmxnet3_activate_dev(adapter);
 	if (err)
 		goto activate_err;
 	return 0;
 activate_err:
 	vmxnet3_rq_destroy_all(adapter);
 	vmxnet3_tq_destroy_all(adapter);
 queue_err:
 	return err;
 }
 static int
 vmxnet3_close(struct net_device *netdev)
 {
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	/*
 	 * Reset_work may be in the middle of resetting the device, wait for its
 	 * completion.
 	 */
 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
 		msleep(1);
 	vmxnet3_quiesce_dev(adapter);
 	vmxnet3_rq_destroy_all(adapter);
 	vmxnet3_tq_destroy_all(adapter);
 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
 	return 0;
 }
 void
 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
 {
 	int i;
 	/*
 	 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
 	 * vmxnet3_close() will deadlock.
 	 */
 	BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
 	/* we need to enable NAPI, otherwise dev_close will deadlock */
 	for (i = 0; i < adapter->num_rx_queues; i++)
 		napi_enable(&adapter->rx_queue[i].napi);
 	dev_close(adapter->netdev);
 }
 static int
 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
 {
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	int err = 0;
 	if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU)
 		return -EINVAL;
 	netdev->mtu = new_mtu;
 	/*
 	 * Reset_work may be in the middle of resetting the device, wait for its
 	 * completion.
 	 */
 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
 		msleep(1);
 	if (netif_running(netdev)) {
 		vmxnet3_quiesce_dev(adapter);
 		vmxnet3_reset_dev(adapter);
 		/* we need to re-create the rx queue based on the new mtu */
 		vmxnet3_rq_destroy_all(adapter);
 		vmxnet3_adjust_rx_ring_size(adapter);
 		err = vmxnet3_rq_create_all(adapter);
 		if (err) {
 			netdev_err(netdev,
 				   "failed to re-create rx queues, "
 				   " error %d. Closing it.\n", err);
 			goto out;
 		}
 		err = vmxnet3_activate_dev(adapter);
 		if (err) {
 			netdev_err(netdev,
 				   "failed to re-activate, error %d. "
 				   "Closing it\n", err);
 			goto out;
 		}
 	}
 out:
 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
 	if (err)
 		vmxnet3_force_close(adapter);
 	return err;
 }
 static void
 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64)
 {
 	struct net_device *netdev = adapter->netdev;
 	netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
 		NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
 		NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
 		NETIF_F_LRO;
 	if (dma64)
 		netdev->hw_features |= NETIF_F_HIGHDMA;
 	netdev->vlan_features = netdev->hw_features &
 				~(NETIF_F_HW_VLAN_CTAG_TX |
 				  NETIF_F_HW_VLAN_CTAG_RX);
 	netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
 }
 static void
 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
 {
 	u32 tmp;
 	tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
 	*(u32 *)mac = tmp;
 	tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
 	mac[4] = tmp & 0xff;
 	mac[5] = (tmp >> 8) & 0xff;
 }
 #ifdef CONFIG_PCI_MSI
 /*
  * Enable MSIx vectors.
  * Returns :
  *	0 on successful enabling of required vectors,
  *	VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
  *	 could be enabled.
  *	number of vectors which can be enabled otherwise (this number is smaller
  *	 than VMXNET3_LINUX_MIN_MSIX_VECT)
  */
 static int
 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter,
 			     int vectors)
 {
 	int err = 0, vector_threshold;
 	vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT;
 	while (vectors >= vector_threshold) {
 		err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries,
 				      vectors);
 		if (!err) {
 			adapter->intr.num_intrs = vectors;
 			return 0;
 		} else if (err < 0) {
 			dev_err(&adapter->netdev->dev,
 				   "Failed to enable MSI-X, error: %d\n", err);
 			vectors = 0;
 		} else if (err < vector_threshold) {
 			break;
 		} else {
 			/* If fails to enable required number of MSI-x vectors
 			 * try enabling minimum number of vectors required.
 			 */
 			dev_err(&adapter->netdev->dev,
 				"Failed to enable %d MSI-X, trying %d instead\n",
 				    vectors, vector_threshold);
 			vectors = vector_threshold;
 		}
 	}
 	dev_info(&adapter->pdev->dev,
 		 "Number of MSI-X interrupts which can be allocated "
 		 "is lower than min threshold required.\n");
 	return err;
 }
 #endif /* CONFIG_PCI_MSI */
 static void
 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
 {
 	u32 cfg;
 	unsigned long flags;
 	/* intr settings */
 	spin_lock_irqsave(&adapter->cmd_lock, flags);
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 			       VMXNET3_CMD_GET_CONF_INTR);
 	cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 	adapter->intr.type = cfg & 0x3;
 	adapter->intr.mask_mode = (cfg >> 2) & 0x3;
 	if (adapter->intr.type == VMXNET3_IT_AUTO) {
 		adapter->intr.type = VMXNET3_IT_MSIX;
 	}
 #ifdef CONFIG_PCI_MSI
 	if (adapter->intr.type == VMXNET3_IT_MSIX) {
 		int vector, err = 0;
 		adapter->intr.num_intrs = (adapter->share_intr ==
 					   VMXNET3_INTR_TXSHARE) ? 1 :
 					   adapter->num_tx_queues;
 		adapter->intr.num_intrs += (adapter->share_intr ==
 					   VMXNET3_INTR_BUDDYSHARE) ? 0 :
 					   adapter->num_rx_queues;
 		adapter->intr.num_intrs += 1;		/* for link event */
 		adapter->intr.num_intrs = (adapter->intr.num_intrs >
 					   VMXNET3_LINUX_MIN_MSIX_VECT
 					   ? adapter->intr.num_intrs :
 					   VMXNET3_LINUX_MIN_MSIX_VECT);
 		for (vector = 0; vector < adapter->intr.num_intrs; vector++)
 			adapter->intr.msix_entries[vector].entry = vector;
 		err = vmxnet3_acquire_msix_vectors(adapter,
 						   adapter->intr.num_intrs);
 		/* If we cannot allocate one MSIx vector per queue
 		 * then limit the number of rx queues to 1
 		 */
 		if (err == VMXNET3_LINUX_MIN_MSIX_VECT) {
 			if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
 			    || adapter->num_rx_queues != 1) {
 				adapter->share_intr = VMXNET3_INTR_TXSHARE;
 				netdev_err(adapter->netdev,
 					   "Number of rx queues : 1\n");
 				adapter->num_rx_queues = 1;
 				adapter->intr.num_intrs =
 						VMXNET3_LINUX_MIN_MSIX_VECT;
 			}
 			return;
 		}
 		if (!err)
 			return;
 		/* If we cannot allocate MSIx vectors use only one rx queue */
 		dev_info(&adapter->pdev->dev,
 			 "Failed to enable MSI-X, error %d. "
 			 "Limiting #rx queues to 1, try MSI.\n", err);
 		adapter->intr.type = VMXNET3_IT_MSI;
 	}
 	if (adapter->intr.type == VMXNET3_IT_MSI) {
 		int err;
 		err = pci_enable_msi(adapter->pdev);
 		if (!err) {
 			adapter->num_rx_queues = 1;
 			adapter->intr.num_intrs = 1;
 			return;
 		}
 	}
 #endif /* CONFIG_PCI_MSI */
 	adapter->num_rx_queues = 1;
 	dev_info(&adapter->netdev->dev,
 		 "Using INTx interrupt, #Rx queues: 1.\n");
 	adapter->intr.type = VMXNET3_IT_INTX;
 	/* INT-X related setting */
 	adapter->intr.num_intrs = 1;
 }
 static void
 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
 {
 	if (adapter->intr.type == VMXNET3_IT_MSIX)
 		pci_disable_msix(adapter->pdev);
 	else if (adapter->intr.type == VMXNET3_IT_MSI)
 		pci_disable_msi(adapter->pdev);
 	else
 		BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
 }
 static void
 vmxnet3_tx_timeout(struct net_device *netdev)
 {
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	adapter->tx_timeout_count++;
 	netdev_err(adapter->netdev, "tx hang\n");
 	schedule_work(&adapter->work);
 	netif_wake_queue(adapter->netdev);
 }
 static void
 vmxnet3_reset_work(struct work_struct *data)
 {
 	struct vmxnet3_adapter *adapter;
 	adapter = container_of(data, struct vmxnet3_adapter, work);
 	/* if another thread is resetting the device, no need to proceed */
 	if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
 		return;
 	/* if the device is closed, we must leave it alone */
 	rtnl_lock();
 	if (netif_running(adapter->netdev)) {
 		netdev_notice(adapter->netdev, "resetting\n");
 		vmxnet3_quiesce_dev(adapter);
 		vmxnet3_reset_dev(adapter);
 		vmxnet3_activate_dev(adapter);
 	} else {
 		netdev_info(adapter->netdev, "already closed\n");
 	}
 	rtnl_unlock();
 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
 }
 static int
 vmxnet3_probe_device(struct pci_dev *pdev,
 		     const struct pci_device_id *id)
 {
 	static const struct net_device_ops vmxnet3_netdev_ops = {
 		.ndo_open = vmxnet3_open,
 		.ndo_stop = vmxnet3_close,
 		.ndo_start_xmit = vmxnet3_xmit_frame,
 		.ndo_set_mac_address = vmxnet3_set_mac_addr,
 		.ndo_change_mtu = vmxnet3_change_mtu,
 		.ndo_set_features = vmxnet3_set_features,
 		.ndo_get_stats64 = vmxnet3_get_stats64,
 		.ndo_tx_timeout = vmxnet3_tx_timeout,
 		.ndo_set_rx_mode = vmxnet3_set_mc,
 		.ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
 		.ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
 #ifdef CONFIG_NET_POLL_CONTROLLER
 		.ndo_poll_controller = vmxnet3_netpoll,
 #endif
 	};
 	int err;
 	bool dma64 = false; /* stupid gcc */
 	u32 ver;
 	struct net_device *netdev;
 	struct vmxnet3_adapter *adapter;
 	u8 mac[ETH_ALEN];
 	int size;
 	int num_tx_queues;
 	int num_rx_queues;
 	if (!pci_msi_enabled())
 		enable_mq = 0;
 #ifdef VMXNET3_RSS
 	if (enable_mq)
 		num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
 				    (int)num_online_cpus());
 	else
 #endif
 		num_rx_queues = 1;
 	num_rx_queues = rounddown_pow_of_two(num_rx_queues);
 	if (enable_mq)
 		num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
 				    (int)num_online_cpus());
 	else
 		num_tx_queues = 1;
 	num_tx_queues = rounddown_pow_of_two(num_tx_queues);
 	netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
 				   max(num_tx_queues, num_rx_queues));
 	dev_info(&pdev->dev,
 		 "# of Tx queues : %d, # of Rx queues : %d\n",
 		 num_tx_queues, num_rx_queues);
 	if (!netdev)
 		return -ENOMEM;
 	pci_set_drvdata(pdev, netdev);
 	adapter = netdev_priv(netdev);
 	adapter->netdev = netdev;
 	adapter->pdev = pdev;
 	spin_lock_init(&adapter->cmd_lock);
 	adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
 					     sizeof(struct vmxnet3_adapter),
 					     PCI_DMA_TODEVICE);
 	adapter->shared = dma_alloc_coherent(
 				&adapter->pdev->dev,
 				sizeof(struct Vmxnet3_DriverShared),
 				&adapter->shared_pa, GFP_KERNEL);
 	if (!adapter->shared) {
 		dev_err(&pdev->dev, "Failed to allocate memory\n");
 		err = -ENOMEM;
 		goto err_alloc_shared;
 	}
 	adapter->num_rx_queues = num_rx_queues;
 	adapter->num_tx_queues = num_tx_queues;
 	adapter->rx_buf_per_pkt = 1;
 	size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
 	size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
 	adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
 						&adapter->queue_desc_pa,
 						GFP_KERNEL);
 	if (!adapter->tqd_start) {
 		dev_err(&pdev->dev, "Failed to allocate memory\n");
 		err = -ENOMEM;
 		goto err_alloc_queue_desc;
 	}
 	adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
 							    adapter->num_tx_queues);
 	adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
 					      sizeof(struct Vmxnet3_PMConf),
 					      &adapter->pm_conf_pa,
 					      GFP_KERNEL);
 	if (adapter->pm_conf == NULL) {
 		err = -ENOMEM;
 		goto err_alloc_pm;
 	}
 #ifdef VMXNET3_RSS
 	adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
 					       sizeof(struct UPT1_RSSConf),
 					       &adapter->rss_conf_pa,
 					       GFP_KERNEL);
 	if (adapter->rss_conf == NULL) {
 		err = -ENOMEM;
 		goto err_alloc_rss;
 	}
 #endif /* VMXNET3_RSS */
 	err = vmxnet3_alloc_pci_resources(adapter, &dma64);
 	if (err < 0)
 		goto err_alloc_pci;
 	ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
 	if (ver & 1) {
 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1);
 	} else {
 		dev_err(&pdev->dev,
 			"Incompatible h/w version (0x%x) for adapter\n", ver);
 		err = -EBUSY;
 		goto err_ver;
 	}
 	ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
 	if (ver & 1) {
 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
 	} else {
 		dev_err(&pdev->dev,
 			"Incompatible upt version (0x%x) for adapter\n", ver);
 		err = -EBUSY;
 		goto err_ver;
 	}
 	SET_NETDEV_DEV(netdev, &pdev->dev);
 	vmxnet3_declare_features(adapter, dma64);
 	if (adapter->num_tx_queues == adapter->num_rx_queues)
 		adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
 	else
 		adapter->share_intr = VMXNET3_INTR_DONTSHARE;
 	vmxnet3_alloc_intr_resources(adapter);
 #ifdef VMXNET3_RSS
 	if (adapter->num_rx_queues > 1 &&
 	    adapter->intr.type == VMXNET3_IT_MSIX) {
 		adapter->rss = true;
 		netdev->hw_features |= NETIF_F_RXHASH;
 		netdev->features |= NETIF_F_RXHASH;
 		dev_dbg(&pdev->dev, "RSS is enabled.\n");
 	} else {
 		adapter->rss = false;
 	}
 #endif
 	vmxnet3_read_mac_addr(adapter, mac);
 	memcpy(netdev->dev_addr,  mac, netdev->addr_len);
 	netdev->netdev_ops = &vmxnet3_netdev_ops;
 	vmxnet3_set_ethtool_ops(netdev);
 	netdev->watchdog_timeo = 5 * HZ;
 	INIT_WORK(&adapter->work, vmxnet3_reset_work);
 	set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
 	if (adapter->intr.type == VMXNET3_IT_MSIX) {
 		int i;
 		for (i = 0; i < adapter->num_rx_queues; i++) {
 			netif_napi_add(adapter->netdev,
 				       &adapter->rx_queue[i].napi,
 				       vmxnet3_poll_rx_only, 64);
 		}
 	} else {
 		netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
 			       vmxnet3_poll, 64);
 	}
 	netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
 	netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
 	netif_carrier_off(netdev);
 	err = register_netdev(netdev);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to register adapter\n");
 		goto err_register;
 	}
 	vmxnet3_check_link(adapter, false);
 	return 0;
 err_register:
 	vmxnet3_free_intr_resources(adapter);
 err_ver:
 	vmxnet3_free_pci_resources(adapter);
 err_alloc_pci:
 #ifdef VMXNET3_RSS
 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
 			  adapter->rss_conf, adapter->rss_conf_pa);
 err_alloc_rss:
 #endif
 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
 			  adapter->pm_conf, adapter->pm_conf_pa);
 err_alloc_pm:
 	dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
 			  adapter->queue_desc_pa);
 err_alloc_queue_desc:
 	dma_free_coherent(&adapter->pdev->dev,
 			  sizeof(struct Vmxnet3_DriverShared),
 			  adapter->shared, adapter->shared_pa);
 err_alloc_shared:
 	dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
 			 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
-	pci_set_drvdata(pdev, NULL);
 	free_netdev(netdev);
 	return err;
 }
 static void
 vmxnet3_remove_device(struct pci_dev *pdev)
 {
 	struct net_device *netdev = pci_get_drvdata(pdev);
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	int size = 0;
 	int num_rx_queues;
 #ifdef VMXNET3_RSS
 	if (enable_mq)
 		num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
 				    (int)num_online_cpus());
 	else
 #endif
 		num_rx_queues = 1;
 	num_rx_queues = rounddown_pow_of_two(num_rx_queues);
 	cancel_work_sync(&adapter->work);
 	unregister_netdev(netdev);
 	vmxnet3_free_intr_resources(adapter);
 	vmxnet3_free_pci_resources(adapter);
 #ifdef VMXNET3_RSS
 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
 			  adapter->rss_conf, adapter->rss_conf_pa);
 #endif
 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
 			  adapter->pm_conf, adapter->pm_conf_pa);
 	size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
 	size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
 	dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
 			  adapter->queue_desc_pa);
 	dma_free_coherent(&adapter->pdev->dev,
 			  sizeof(struct Vmxnet3_DriverShared),
 			  adapter->shared, adapter->shared_pa);
 	dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
 			 sizeof(struct vmxnet3_adapter), PCI_DMA_TODEVICE);
 	free_netdev(netdev);
 }
 #ifdef CONFIG_PM
 static int
 vmxnet3_suspend(struct device *device)
 {
 	struct pci_dev *pdev = to_pci_dev(device);
 	struct net_device *netdev = pci_get_drvdata(pdev);
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	struct Vmxnet3_PMConf *pmConf;
 	struct ethhdr *ehdr;
 	struct arphdr *ahdr;
 	u8 *arpreq;
 	struct in_device *in_dev;
 	struct in_ifaddr *ifa;
 	unsigned long flags;
 	int i = 0;
 	if (!netif_running(netdev))
 		return 0;
 	for (i = 0; i < adapter->num_rx_queues; i++)
 		napi_disable(&adapter->rx_queue[i].napi);
 	vmxnet3_disable_all_intrs(adapter);
 	vmxnet3_free_irqs(adapter);
 	vmxnet3_free_intr_resources(adapter);
 	netif_device_detach(netdev);
 	netif_tx_stop_all_queues(netdev);
 	/* Create wake-up filters. */
 	pmConf = adapter->pm_conf;
 	memset(pmConf, 0, sizeof(*pmConf));
 	if (adapter->wol & WAKE_UCAST) {
 		pmConf->filters[i].patternSize = ETH_ALEN;
 		pmConf->filters[i].maskSize = 1;
 		memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
 		pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
 		pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
 		i++;
 	}
 	if (adapter->wol & WAKE_ARP) {
 		in_dev = in_dev_get(netdev);
 		if (!in_dev)
 			goto skip_arp;
 		ifa = (struct in_ifaddr *)in_dev->ifa_list;
 		if (!ifa)
 			goto skip_arp;
 		pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
 			sizeof(struct arphdr) +		/* ARP header */
 			2 * ETH_ALEN +		/* 2 Ethernet addresses*/
 			2 * sizeof(u32);	/*2 IPv4 addresses */
 		pmConf->filters[i].maskSize =
 			(pmConf->filters[i].patternSize - 1) / 8 + 1;
 		/* ETH_P_ARP in Ethernet header. */
 		ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
 		ehdr->h_proto = htons(ETH_P_ARP);
 		/* ARPOP_REQUEST in ARP header. */
 		ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
 		ahdr->ar_op = htons(ARPOP_REQUEST);
 		arpreq = (u8 *)(ahdr + 1);
 		/* The Unicast IPv4 address in 'tip' field. */
 		arpreq += 2 * ETH_ALEN + sizeof(u32);
 		*(u32 *)arpreq = ifa->ifa_address;
 		/* The mask for the relevant bits. */
 		pmConf->filters[i].mask[0] = 0x00;
 		pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
 		pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
 		pmConf->filters[i].mask[3] = 0x00;
 		pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
 		pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
 		in_dev_put(in_dev);
 		pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
 		i++;
 	}
 skip_arp:
 	if (adapter->wol & WAKE_MAGIC)
 		pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
 	pmConf->numFilters = i;
 	adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
 	adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
 								  *pmConf));
 	adapter->shared->devRead.pmConfDesc.confPA =
 		cpu_to_le64(adapter->pm_conf_pa);
 	spin_lock_irqsave(&adapter->cmd_lock, flags);
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 			       VMXNET3_CMD_UPDATE_PMCFG);
 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 	pci_save_state(pdev);
 	pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
 			adapter->wol);
 	pci_disable_device(pdev);
 	pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
 	return 0;
 }
 static int
 vmxnet3_resume(struct device *device)
 {
 	int err, i = 0;
 	unsigned long flags;
 	struct pci_dev *pdev = to_pci_dev(device);
 	struct net_device *netdev = pci_get_drvdata(pdev);
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 	struct Vmxnet3_PMConf *pmConf;
 	if (!netif_running(netdev))
 		return 0;
 	/* Destroy wake-up filters. */
 	pmConf = adapter->pm_conf;
 	memset(pmConf, 0, sizeof(*pmConf));
 	adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
 	adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
 								  *pmConf));
 	adapter->shared->devRead.pmConfDesc.confPA =
 		cpu_to_le64(adapter->pm_conf_pa);
 	netif_device_attach(netdev);
 	pci_set_power_state(pdev, PCI_D0);
 	pci_restore_state(pdev);
 	err = pci_enable_device_mem(pdev);
 	if (err != 0)
 		return err;
 	pci_enable_wake(pdev, PCI_D0, 0);
 	spin_lock_irqsave(&adapter->cmd_lock, flags);
 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
 			       VMXNET3_CMD_UPDATE_PMCFG);
 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
 	vmxnet3_alloc_intr_resources(adapter);
 	vmxnet3_request_irqs(adapter);
 	for (i = 0; i < adapter->num_rx_queues; i++)
 		napi_enable(&adapter->rx_queue[i].napi);
 	vmxnet3_enable_all_intrs(adapter);
 	return 0;
 }
 static const struct dev_pm_ops vmxnet3_pm_ops = {
 	.suspend = vmxnet3_suspend,
 	.resume = vmxnet3_resume,
 };
 #endif
 static struct pci_driver vmxnet3_driver = {
 	.name		= vmxnet3_driver_name,
 	.id_table	= vmxnet3_pciid_table,
 	.probe		= vmxnet3_probe_device,
 	.remove		= vmxnet3_remove_device,
 #ifdef CONFIG_PM
 	.driver.pm	= &vmxnet3_pm_ops,
 #endif
 };
 static int __init
 vmxnet3_init_module(void)
 {
 	pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
 		VMXNET3_DRIVER_VERSION_REPORT);
 	return pci_register_driver(&vmxnet3_driver);
 }
 module_init(vmxnet3_init_module);
 static void
 vmxnet3_exit_module(void)
 {
 	pci_unregister_driver(&vmxnet3_driver);
 }
 module_exit(vmxnet3_exit_module);
 MODULE_AUTHOR("VMware, Inc.");
 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
 MODULE_LICENSE("GPL v2");
 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);