Doug / smarc-fsl-linux-kernel | Embedian Git Server

Commit 86c432ca5d6da90a26ac8d3e680f2268b502d9c5

Authored by Ben Hutchings 2011-09-01 20:09:29 +0800

Committed by David S. Miller 2011-09-16 05:31:28 +0800

Exists in master and in 6 other branches

Revert "sfc: Use write-combining to reduce TX latency" and follow-ups

This reverts commits 65f0b417dee94f779ce9b77102b7d73c93723b39,
d88d6b05fee3cc78e5b0273eb58c31201dcc6b76,
fcfa060468a4edcf776f0c1211d826d5de1668c1,
747df2258b1b9a2e25929ef496262c339c380009 and
867955f5682f7157fdafe8670804b9f8ea077bc7.

Depending on the processor model, write-combining may result in
reordering that the NIC will not tolerate.  This typically results
in a DMA error event and reset by the driver, logged as:

sfc 0000:0e:00.0: eth2: TX DMA Q reports TX_EV_PKT_ERR.
sfc 0000:0e:00.0: eth2: resetting (ALL)

Signed-off-by: Ben Hutchings <bhutchings@solarflare.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

Showing 7 changed files with 27 additions and 88 deletions Inline Diff

drivers/net/sfc/efx.c
drivers/net/sfc/io.h
drivers/net/sfc/mcdi.c
drivers/net/sfc/nic.c
drivers/net/sfc/nic.h
drivers/net/sfc/siena.c
drivers/net/sfc/workarounds.h

drivers/net/sfc/efx.c

Diff comments View file @ 86c432c

1	/****************************************************************************	1	/****************************************************************************
2	* Driver for Solarflare Solarstorm network controllers and boards	2	* Driver for Solarflare Solarstorm network controllers and boards
3	* Copyright 2005-2006 Fen Systems Ltd.	3	* Copyright 2005-2006 Fen Systems Ltd.
4	* Copyright 2005-2011 Solarflare Communications Inc.	4	* Copyright 2005-2011 Solarflare Communications Inc.
5	*	5	*
6	* This program is free software; you can redistribute it and/or modify it	6	* This program is free software; you can redistribute it and/or modify it
7	* under the terms of the GNU General Public License version 2 as published	7	* under the terms of the GNU General Public License version 2 as published
8	* by the Free Software Foundation, incorporated herein by reference.	8	* by the Free Software Foundation, incorporated herein by reference.
9	*/	9	*/
10		10
11	#include <linux/module.h>	11	#include <linux/module.h>
12	#include <linux/pci.h>	12	#include <linux/pci.h>
13	#include <linux/netdevice.h>	13	#include <linux/netdevice.h>
14	#include <linux/etherdevice.h>	14	#include <linux/etherdevice.h>
15	#include <linux/delay.h>	15	#include <linux/delay.h>
16	#include <linux/notifier.h>	16	#include <linux/notifier.h>
17	#include <linux/ip.h>	17	#include <linux/ip.h>
18	#include <linux/tcp.h>	18	#include <linux/tcp.h>
19	#include <linux/in.h>	19	#include <linux/in.h>
20	#include <linux/crc32.h>	20	#include <linux/crc32.h>
21	#include <linux/ethtool.h>	21	#include <linux/ethtool.h>
22	#include <linux/topology.h>	22	#include <linux/topology.h>
23	#include <linux/gfp.h>	23	#include <linux/gfp.h>
24	#include <linux/cpu_rmap.h>	24	#include <linux/cpu_rmap.h>
25	#include "net_driver.h"	25	#include "net_driver.h"
26	#include "efx.h"	26	#include "efx.h"
27	#include "nic.h"	27	#include "nic.h"
28		28
29	#include "mcdi.h"	29	#include "mcdi.h"
30	#include "workarounds.h"	30	#include "workarounds.h"
31		31
32	/**************************************************************************	32	/**************************************************************************
33	*	33	*
34	* Type name strings	34	* Type name strings
35	*	35	*
36	**************************************************************************	36	**************************************************************************
37	*/	37	*/
38		38
39	/* Loopback mode names (see LOOPBACK_MODE()) */	39	/* Loopback mode names (see LOOPBACK_MODE()) */
40	const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;	40	const unsigned int efx_loopback_mode_max = LOOPBACK_MAX;
41	const char *efx_loopback_mode_names[] = {	41	const char *efx_loopback_mode_names[] = {
42	[LOOPBACK_NONE] = "NONE",	42	[LOOPBACK_NONE] = "NONE",
43	[LOOPBACK_DATA] = "DATAPATH",	43	[LOOPBACK_DATA] = "DATAPATH",
44	[LOOPBACK_GMAC] = "GMAC",	44	[LOOPBACK_GMAC] = "GMAC",
45	[LOOPBACK_XGMII] = "XGMII",	45	[LOOPBACK_XGMII] = "XGMII",
46	[LOOPBACK_XGXS] = "XGXS",	46	[LOOPBACK_XGXS] = "XGXS",
47	[LOOPBACK_XAUI] = "XAUI",	47	[LOOPBACK_XAUI] = "XAUI",
48	[LOOPBACK_GMII] = "GMII",	48	[LOOPBACK_GMII] = "GMII",
49	[LOOPBACK_SGMII] = "SGMII",	49	[LOOPBACK_SGMII] = "SGMII",
50	[LOOPBACK_XGBR] = "XGBR",	50	[LOOPBACK_XGBR] = "XGBR",
51	[LOOPBACK_XFI] = "XFI",	51	[LOOPBACK_XFI] = "XFI",
52	[LOOPBACK_XAUI_FAR] = "XAUI_FAR",	52	[LOOPBACK_XAUI_FAR] = "XAUI_FAR",
53	[LOOPBACK_GMII_FAR] = "GMII_FAR",	53	[LOOPBACK_GMII_FAR] = "GMII_FAR",
54	[LOOPBACK_SGMII_FAR] = "SGMII_FAR",	54	[LOOPBACK_SGMII_FAR] = "SGMII_FAR",
55	[LOOPBACK_XFI_FAR] = "XFI_FAR",	55	[LOOPBACK_XFI_FAR] = "XFI_FAR",
56	[LOOPBACK_GPHY] = "GPHY",	56	[LOOPBACK_GPHY] = "GPHY",
57	[LOOPBACK_PHYXS] = "PHYXS",	57	[LOOPBACK_PHYXS] = "PHYXS",
58	[LOOPBACK_PCS] = "PCS",	58	[LOOPBACK_PCS] = "PCS",
59	[LOOPBACK_PMAPMD] = "PMA/PMD",	59	[LOOPBACK_PMAPMD] = "PMA/PMD",
60	[LOOPBACK_XPORT] = "XPORT",	60	[LOOPBACK_XPORT] = "XPORT",
61	[LOOPBACK_XGMII_WS] = "XGMII_WS",	61	[LOOPBACK_XGMII_WS] = "XGMII_WS",
62	[LOOPBACK_XAUI_WS] = "XAUI_WS",	62	[LOOPBACK_XAUI_WS] = "XAUI_WS",
63	[LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR",	63	[LOOPBACK_XAUI_WS_FAR] = "XAUI_WS_FAR",
64	[LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR",	64	[LOOPBACK_XAUI_WS_NEAR] = "XAUI_WS_NEAR",
65	[LOOPBACK_GMII_WS] = "GMII_WS",	65	[LOOPBACK_GMII_WS] = "GMII_WS",
66	[LOOPBACK_XFI_WS] = "XFI_WS",	66	[LOOPBACK_XFI_WS] = "XFI_WS",
67	[LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR",	67	[LOOPBACK_XFI_WS_FAR] = "XFI_WS_FAR",
68	[LOOPBACK_PHYXS_WS] = "PHYXS_WS",	68	[LOOPBACK_PHYXS_WS] = "PHYXS_WS",
69	};	69	};
70		70
71	const unsigned int efx_reset_type_max = RESET_TYPE_MAX;	71	const unsigned int efx_reset_type_max = RESET_TYPE_MAX;
72	const char *efx_reset_type_names[] = {	72	const char *efx_reset_type_names[] = {
73	[RESET_TYPE_INVISIBLE] = "INVISIBLE",	73	[RESET_TYPE_INVISIBLE] = "INVISIBLE",
74	[RESET_TYPE_ALL] = "ALL",	74	[RESET_TYPE_ALL] = "ALL",
75	[RESET_TYPE_WORLD] = "WORLD",	75	[RESET_TYPE_WORLD] = "WORLD",
76	[RESET_TYPE_DISABLE] = "DISABLE",	76	[RESET_TYPE_DISABLE] = "DISABLE",
77	[RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG",	77	[RESET_TYPE_TX_WATCHDOG] = "TX_WATCHDOG",
78	[RESET_TYPE_INT_ERROR] = "INT_ERROR",	78	[RESET_TYPE_INT_ERROR] = "INT_ERROR",
79	[RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY",	79	[RESET_TYPE_RX_RECOVERY] = "RX_RECOVERY",
80	[RESET_TYPE_RX_DESC_FETCH] = "RX_DESC_FETCH",	80	[RESET_TYPE_RX_DESC_FETCH] = "RX_DESC_FETCH",
81	[RESET_TYPE_TX_DESC_FETCH] = "TX_DESC_FETCH",	81	[RESET_TYPE_TX_DESC_FETCH] = "TX_DESC_FETCH",
82	[RESET_TYPE_TX_SKIP] = "TX_SKIP",	82	[RESET_TYPE_TX_SKIP] = "TX_SKIP",
83	[RESET_TYPE_MC_FAILURE] = "MC_FAILURE",	83	[RESET_TYPE_MC_FAILURE] = "MC_FAILURE",
84	};	84	};
85		85
86	#define EFX_MAX_MTU (9 * 1024)	86	#define EFX_MAX_MTU (9 * 1024)
87		87
88	/* Reset workqueue. If any NIC has a hardware failure then a reset will be	88	/* Reset workqueue. If any NIC has a hardware failure then a reset will be
89	* queued onto this work queue. This is not a per-nic work queue, because	89	* queued onto this work queue. This is not a per-nic work queue, because
90	* efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.	90	* efx_reset_work() acquires the rtnl lock, so resets are naturally serialised.
91	*/	91	*/
92	static struct workqueue_struct *reset_workqueue;	92	static struct workqueue_struct *reset_workqueue;
93		93
94	/**************************************************************************	94	/**************************************************************************
95	*	95	*
96	* Configurable values	96	* Configurable values
97	*	97	*
98	*************************************************************************/	98	*************************************************************************/
99		99
100	/*	100	/*
101	* Use separate channels for TX and RX events	101	* Use separate channels for TX and RX events
102	*	102	*
103	* Set this to 1 to use separate channels for TX and RX. It allows us	103	* Set this to 1 to use separate channels for TX and RX. It allows us
104	* to control interrupt affinity separately for TX and RX.	104	* to control interrupt affinity separately for TX and RX.
105	*	105	*
106	* This is only used in MSI-X interrupt mode	106	* This is only used in MSI-X interrupt mode
107	*/	107	*/
108	static unsigned int separate_tx_channels;	108	static unsigned int separate_tx_channels;
109	module_param(separate_tx_channels, uint, 0444);	109	module_param(separate_tx_channels, uint, 0444);
110	MODULE_PARM_DESC(separate_tx_channels,	110	MODULE_PARM_DESC(separate_tx_channels,
111	"Use separate channels for TX and RX");	111	"Use separate channels for TX and RX");
112		112
113	/* This is the weight assigned to each of the (per-channel) virtual	113	/* This is the weight assigned to each of the (per-channel) virtual
114	* NAPI devices.	114	* NAPI devices.
115	*/	115	*/
116	static int napi_weight = 64;	116	static int napi_weight = 64;
117		117
118	/* This is the time (in jiffies) between invocations of the hardware	118	/* This is the time (in jiffies) between invocations of the hardware
119	* monitor. On Falcon-based NICs, this will:	119	* monitor. On Falcon-based NICs, this will:
120	* - Check the on-board hardware monitor;	120	* - Check the on-board hardware monitor;
121	* - Poll the link state and reconfigure the hardware as necessary.	121	* - Poll the link state and reconfigure the hardware as necessary.
122	*/	122	*/
123	static unsigned int efx_monitor_interval = 1 * HZ;	123	static unsigned int efx_monitor_interval = 1 * HZ;
124		124
125	/* This controls whether or not the driver will initialise devices	125	/* This controls whether or not the driver will initialise devices
126	* with invalid MAC addresses stored in the EEPROM or flash. If true,	126	* with invalid MAC addresses stored in the EEPROM or flash. If true,
127	* such devices will be initialised with a random locally-generated	127	* such devices will be initialised with a random locally-generated
128	* MAC address. This allows for loading the sfc_mtd driver to	128	* MAC address. This allows for loading the sfc_mtd driver to
129	* reprogram the flash, even if the flash contents (including the MAC	129	* reprogram the flash, even if the flash contents (including the MAC
130	* address) have previously been erased.	130	* address) have previously been erased.
131	*/	131	*/
132	static unsigned int allow_bad_hwaddr;	132	static unsigned int allow_bad_hwaddr;
133		133
134	/* Initial interrupt moderation settings. They can be modified after	134	/* Initial interrupt moderation settings. They can be modified after
135	* module load with ethtool.	135	* module load with ethtool.
136	*	136	*
137	* The default for RX should strike a balance between increasing the	137	* The default for RX should strike a balance between increasing the
138	* round-trip latency and reducing overhead.	138	* round-trip latency and reducing overhead.
139	*/	139	*/
140	static unsigned int rx_irq_mod_usec = 60;	140	static unsigned int rx_irq_mod_usec = 60;
141		141
142	/* Initial interrupt moderation settings. They can be modified after	142	/* Initial interrupt moderation settings. They can be modified after
143	* module load with ethtool.	143	* module load with ethtool.
144	*	144	*
145	* This default is chosen to ensure that a 10G link does not go idle	145	* This default is chosen to ensure that a 10G link does not go idle
146	* while a TX queue is stopped after it has become full. A queue is	146	* while a TX queue is stopped after it has become full. A queue is
147	* restarted when it drops below half full. The time this takes (assuming	147	* restarted when it drops below half full. The time this takes (assuming
148	* worst case 3 descriptors per packet and 1024 descriptors) is	148	* worst case 3 descriptors per packet and 1024 descriptors) is
149	* 512 / 3 * 1.2 = 205 usec.	149	* 512 / 3 * 1.2 = 205 usec.
150	*/	150	*/
151	static unsigned int tx_irq_mod_usec = 150;	151	static unsigned int tx_irq_mod_usec = 150;
152		152
153	/* This is the first interrupt mode to try out of:	153	/* This is the first interrupt mode to try out of:
154	* 0 => MSI-X	154	* 0 => MSI-X
155	* 1 => MSI	155	* 1 => MSI
156	* 2 => legacy	156	* 2 => legacy
157	*/	157	*/
158	static unsigned int interrupt_mode;	158	static unsigned int interrupt_mode;
159		159
160	/* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),	160	/* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
161	* i.e. the number of CPUs among which we may distribute simultaneous	161	* i.e. the number of CPUs among which we may distribute simultaneous
162	* interrupt handling.	162	* interrupt handling.
163	*	163	*
164	* Cards without MSI-X will only target one CPU via legacy or MSI interrupt.	164	* Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
165	* The default (0) means to assign an interrupt to each package (level II cache)	165	* The default (0) means to assign an interrupt to each package (level II cache)
166	*/	166	*/
167	static unsigned int rss_cpus;	167	static unsigned int rss_cpus;
168	module_param(rss_cpus, uint, 0444);	168	module_param(rss_cpus, uint, 0444);
169	MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");	169	MODULE_PARM_DESC(rss_cpus, "Number of CPUs to use for Receive-Side Scaling");
170		170
171	static int phy_flash_cfg;	171	static int phy_flash_cfg;
172	module_param(phy_flash_cfg, int, 0644);	172	module_param(phy_flash_cfg, int, 0644);
173	MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");	173	MODULE_PARM_DESC(phy_flash_cfg, "Set PHYs into reflash mode initially");
174		174
175	static unsigned irq_adapt_low_thresh = 10000;	175	static unsigned irq_adapt_low_thresh = 10000;
176	module_param(irq_adapt_low_thresh, uint, 0644);	176	module_param(irq_adapt_low_thresh, uint, 0644);
177	MODULE_PARM_DESC(irq_adapt_low_thresh,	177	MODULE_PARM_DESC(irq_adapt_low_thresh,
178	"Threshold score for reducing IRQ moderation");	178	"Threshold score for reducing IRQ moderation");
179		179
180	static unsigned irq_adapt_high_thresh = 20000;	180	static unsigned irq_adapt_high_thresh = 20000;
181	module_param(irq_adapt_high_thresh, uint, 0644);	181	module_param(irq_adapt_high_thresh, uint, 0644);
182	MODULE_PARM_DESC(irq_adapt_high_thresh,	182	MODULE_PARM_DESC(irq_adapt_high_thresh,
183	"Threshold score for increasing IRQ moderation");	183	"Threshold score for increasing IRQ moderation");
184		184
185	static unsigned debug = (NETIF_MSG_DRV \| NETIF_MSG_PROBE \|	185	static unsigned debug = (NETIF_MSG_DRV \| NETIF_MSG_PROBE \|
186	NETIF_MSG_LINK \| NETIF_MSG_IFDOWN \|	186	NETIF_MSG_LINK \| NETIF_MSG_IFDOWN \|
187	NETIF_MSG_IFUP \| NETIF_MSG_RX_ERR \|	187	NETIF_MSG_IFUP \| NETIF_MSG_RX_ERR \|
188	NETIF_MSG_TX_ERR \| NETIF_MSG_HW);	188	NETIF_MSG_TX_ERR \| NETIF_MSG_HW);
189	module_param(debug, uint, 0);	189	module_param(debug, uint, 0);
190	MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");	190	MODULE_PARM_DESC(debug, "Bitmapped debugging message enable value");
191		191
192	/**************************************************************************	192	/**************************************************************************
193	*	193	*
194	* Utility functions and prototypes	194	* Utility functions and prototypes
195	*	195	*
196	*************************************************************************/	196	*************************************************************************/
197		197
198	static void efx_remove_channels(struct efx_nic *efx);	198	static void efx_remove_channels(struct efx_nic *efx);
199	static void efx_remove_port(struct efx_nic *efx);	199	static void efx_remove_port(struct efx_nic *efx);
200	static void efx_init_napi(struct efx_nic *efx);	200	static void efx_init_napi(struct efx_nic *efx);
201	static void efx_fini_napi(struct efx_nic *efx);	201	static void efx_fini_napi(struct efx_nic *efx);
202	static void efx_fini_napi_channel(struct efx_channel *channel);	202	static void efx_fini_napi_channel(struct efx_channel *channel);
203	static void efx_fini_struct(struct efx_nic *efx);	203	static void efx_fini_struct(struct efx_nic *efx);
204	static void efx_start_all(struct efx_nic *efx);	204	static void efx_start_all(struct efx_nic *efx);
205	static void efx_stop_all(struct efx_nic *efx);	205	static void efx_stop_all(struct efx_nic *efx);
206		206
207	#define EFX_ASSERT_RESET_SERIALISED(efx) \	207	#define EFX_ASSERT_RESET_SERIALISED(efx) \
208	do { \	208	do { \
209	if ((efx->state == STATE_RUNNING) \|\| \	209	if ((efx->state == STATE_RUNNING) \|\| \
210	(efx->state == STATE_DISABLED)) \	210	(efx->state == STATE_DISABLED)) \
211	ASSERT_RTNL(); \	211	ASSERT_RTNL(); \
212	} while (0)	212	} while (0)
213		213
214	/**************************************************************************	214	/**************************************************************************
215	*	215	*
216	* Event queue processing	216	* Event queue processing
217	*	217	*
218	*************************************************************************/	218	*************************************************************************/
219		219
220	/* Process channel's event queue	220	/* Process channel's event queue
221	*	221	*
222	* This function is responsible for processing the event queue of a	222	* This function is responsible for processing the event queue of a
223	* single channel. The caller must guarantee that this function will	223	* single channel. The caller must guarantee that this function will
224	* never be concurrently called more than once on the same channel,	224	* never be concurrently called more than once on the same channel,
225	* though different channels may be being processed concurrently.	225	* though different channels may be being processed concurrently.
226	*/	226	*/
227	static int efx_process_channel(struct efx_channel *channel, int budget)	227	static int efx_process_channel(struct efx_channel *channel, int budget)
228	{	228	{
229	struct efx_nic *efx = channel->efx;	229	struct efx_nic *efx = channel->efx;
230	int spent;	230	int spent;
231		231
232	if (unlikely(efx->reset_pending \|\| !channel->enabled))	232	if (unlikely(efx->reset_pending \|\| !channel->enabled))
233	return 0;	233	return 0;
234		234
235	spent = efx_nic_process_eventq(channel, budget);	235	spent = efx_nic_process_eventq(channel, budget);
236	if (spent == 0)	236	if (spent == 0)
237	return 0;	237	return 0;
238		238
239	/* Deliver last RX packet. */	239	/* Deliver last RX packet. */
240	if (channel->rx_pkt) {	240	if (channel->rx_pkt) {
241	__efx_rx_packet(channel, channel->rx_pkt,	241	__efx_rx_packet(channel, channel->rx_pkt,
242	channel->rx_pkt_csummed);	242	channel->rx_pkt_csummed);
243	channel->rx_pkt = NULL;	243	channel->rx_pkt = NULL;
244	}	244	}
245		245
246	efx_rx_strategy(channel);	246	efx_rx_strategy(channel);
247		247
248	efx_fast_push_rx_descriptors(efx_channel_get_rx_queue(channel));	248	efx_fast_push_rx_descriptors(efx_channel_get_rx_queue(channel));
249		249
250	return spent;	250	return spent;
251	}	251	}
252		252
253	/* Mark channel as finished processing	253	/* Mark channel as finished processing
254	*	254	*
255	* Note that since we will not receive further interrupts for this	255	* Note that since we will not receive further interrupts for this
256	* channel before we finish processing and call the eventq_read_ack()	256	* channel before we finish processing and call the eventq_read_ack()
257	* method, there is no need to use the interrupt hold-off timers.	257	* method, there is no need to use the interrupt hold-off timers.
258	*/	258	*/
259	static inline void efx_channel_processed(struct efx_channel *channel)	259	static inline void efx_channel_processed(struct efx_channel *channel)
260	{	260	{
261	/* The interrupt handler for this channel may set work_pending	261	/* The interrupt handler for this channel may set work_pending
262	* as soon as we acknowledge the events we've seen. Make sure	262	* as soon as we acknowledge the events we've seen. Make sure
263	* it's cleared before then. */	263	* it's cleared before then. */
264	channel->work_pending = false;	264	channel->work_pending = false;
265	smp_wmb();	265	smp_wmb();
266		266
267	efx_nic_eventq_read_ack(channel);	267	efx_nic_eventq_read_ack(channel);
268	}	268	}
269		269
270	/* NAPI poll handler	270	/* NAPI poll handler
271	*	271	*
272	* NAPI guarantees serialisation of polls of the same device, which	272	* NAPI guarantees serialisation of polls of the same device, which
273	* provides the guarantee required by efx_process_channel().	273	* provides the guarantee required by efx_process_channel().
274	*/	274	*/
275	static int efx_poll(struct napi_struct *napi, int budget)	275	static int efx_poll(struct napi_struct *napi, int budget)
276	{	276	{
277	struct efx_channel *channel =	277	struct efx_channel *channel =
278	container_of(napi, struct efx_channel, napi_str);	278	container_of(napi, struct efx_channel, napi_str);
279	struct efx_nic *efx = channel->efx;	279	struct efx_nic *efx = channel->efx;
280	int spent;	280	int spent;
281		281
282	netif_vdbg(efx, intr, efx->net_dev,	282	netif_vdbg(efx, intr, efx->net_dev,
283	"channel %d NAPI poll executing on CPU %d\n",	283	"channel %d NAPI poll executing on CPU %d\n",
284	channel->channel, raw_smp_processor_id());	284	channel->channel, raw_smp_processor_id());
285		285
286	spent = efx_process_channel(channel, budget);	286	spent = efx_process_channel(channel, budget);
287		287
288	if (spent < budget) {	288	if (spent < budget) {
289	if (channel->channel < efx->n_rx_channels &&	289	if (channel->channel < efx->n_rx_channels &&
290	efx->irq_rx_adaptive &&	290	efx->irq_rx_adaptive &&
291	unlikely(++channel->irq_count == 1000)) {	291	unlikely(++channel->irq_count == 1000)) {
292	if (unlikely(channel->irq_mod_score <	292	if (unlikely(channel->irq_mod_score <
293	irq_adapt_low_thresh)) {	293	irq_adapt_low_thresh)) {
294	if (channel->irq_moderation > 1) {	294	if (channel->irq_moderation > 1) {
295	channel->irq_moderation -= 1;	295	channel->irq_moderation -= 1;
296	efx->type->push_irq_moderation(channel);	296	efx->type->push_irq_moderation(channel);
297	}	297	}
298	} else if (unlikely(channel->irq_mod_score >	298	} else if (unlikely(channel->irq_mod_score >
299	irq_adapt_high_thresh)) {	299	irq_adapt_high_thresh)) {
300	if (channel->irq_moderation <	300	if (channel->irq_moderation <
301	efx->irq_rx_moderation) {	301	efx->irq_rx_moderation) {
302	channel->irq_moderation += 1;	302	channel->irq_moderation += 1;
303	efx->type->push_irq_moderation(channel);	303	efx->type->push_irq_moderation(channel);
304	}	304	}
305	}	305	}
306	channel->irq_count = 0;	306	channel->irq_count = 0;
307	channel->irq_mod_score = 0;	307	channel->irq_mod_score = 0;
308	}	308	}
309		309
310	efx_filter_rfs_expire(channel);	310	efx_filter_rfs_expire(channel);
311		311
312	/* There is no race here; although napi_disable() will	312	/* There is no race here; although napi_disable() will
313	* only wait for napi_complete(), this isn't a problem	313	* only wait for napi_complete(), this isn't a problem
314	* since efx_channel_processed() will have no effect if	314	* since efx_channel_processed() will have no effect if
315	* interrupts have already been disabled.	315	* interrupts have already been disabled.
316	*/	316	*/
317	napi_complete(napi);	317	napi_complete(napi);
318	efx_channel_processed(channel);	318	efx_channel_processed(channel);
319	}	319	}
320		320
321	return spent;	321	return spent;
322	}	322	}
323		323
324	/* Process the eventq of the specified channel immediately on this CPU	324	/* Process the eventq of the specified channel immediately on this CPU
325	*	325	*
326	* Disable hardware generated interrupts, wait for any existing	326	* Disable hardware generated interrupts, wait for any existing
327	* processing to finish, then directly poll (and ack ) the eventq.	327	* processing to finish, then directly poll (and ack ) the eventq.
328	* Finally reenable NAPI and interrupts.	328	* Finally reenable NAPI and interrupts.
329	*	329	*
330	* This is for use only during a loopback self-test. It must not	330	* This is for use only during a loopback self-test. It must not
331	* deliver any packets up the stack as this can result in deadlock.	331	* deliver any packets up the stack as this can result in deadlock.
332	*/	332	*/
333	void efx_process_channel_now(struct efx_channel *channel)	333	void efx_process_channel_now(struct efx_channel *channel)
334	{	334	{
335	struct efx_nic *efx = channel->efx;	335	struct efx_nic *efx = channel->efx;
336		336
337	BUG_ON(channel->channel >= efx->n_channels);	337	BUG_ON(channel->channel >= efx->n_channels);
338	BUG_ON(!channel->enabled);	338	BUG_ON(!channel->enabled);
339	BUG_ON(!efx->loopback_selftest);	339	BUG_ON(!efx->loopback_selftest);
340		340
341	/* Disable interrupts and wait for ISRs to complete */	341	/* Disable interrupts and wait for ISRs to complete */
342	efx_nic_disable_interrupts(efx);	342	efx_nic_disable_interrupts(efx);
343	if (efx->legacy_irq) {	343	if (efx->legacy_irq) {
344	synchronize_irq(efx->legacy_irq);	344	synchronize_irq(efx->legacy_irq);
345	efx->legacy_irq_enabled = false;	345	efx->legacy_irq_enabled = false;
346	}	346	}
347	if (channel->irq)	347	if (channel->irq)
348	synchronize_irq(channel->irq);	348	synchronize_irq(channel->irq);
349		349
350	/* Wait for any NAPI processing to complete */	350	/* Wait for any NAPI processing to complete */
351	napi_disable(&channel->napi_str);	351	napi_disable(&channel->napi_str);
352		352
353	/* Poll the channel */	353	/* Poll the channel */
354	efx_process_channel(channel, channel->eventq_mask + 1);	354	efx_process_channel(channel, channel->eventq_mask + 1);
355		355
356	/* Ack the eventq. This may cause an interrupt to be generated	356	/* Ack the eventq. This may cause an interrupt to be generated
357	* when they are reenabled */	357	* when they are reenabled */
358	efx_channel_processed(channel);	358	efx_channel_processed(channel);
359		359
360	napi_enable(&channel->napi_str);	360	napi_enable(&channel->napi_str);
361	if (efx->legacy_irq)	361	if (efx->legacy_irq)
362	efx->legacy_irq_enabled = true;	362	efx->legacy_irq_enabled = true;
363	efx_nic_enable_interrupts(efx);	363	efx_nic_enable_interrupts(efx);
364	}	364	}
365		365
366	/* Create event queue	366	/* Create event queue
367	* Event queue memory allocations are done only once. If the channel	367	* Event queue memory allocations are done only once. If the channel
368	* is reset, the memory buffer will be reused; this guards against	368	* is reset, the memory buffer will be reused; this guards against
369	* errors during channel reset and also simplifies interrupt handling.	369	* errors during channel reset and also simplifies interrupt handling.
370	*/	370	*/
371	static int efx_probe_eventq(struct efx_channel *channel)	371	static int efx_probe_eventq(struct efx_channel *channel)
372	{	372	{
373	struct efx_nic *efx = channel->efx;	373	struct efx_nic *efx = channel->efx;
374	unsigned long entries;	374	unsigned long entries;
375		375
376	netif_dbg(channel->efx, probe, channel->efx->net_dev,	376	netif_dbg(channel->efx, probe, channel->efx->net_dev,
377	"chan %d create event queue\n", channel->channel);	377	"chan %d create event queue\n", channel->channel);
378		378
379	/* Build an event queue with room for one event per tx and rx buffer,	379	/* Build an event queue with room for one event per tx and rx buffer,
380	* plus some extra for link state events and MCDI completions. */	380	* plus some extra for link state events and MCDI completions. */
381	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);	381	entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
382	EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);	382	EFX_BUG_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
383	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;	383	channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
384		384
385	return efx_nic_probe_eventq(channel);	385	return efx_nic_probe_eventq(channel);
386	}	386	}
387		387
388	/* Prepare channel's event queue */	388	/* Prepare channel's event queue */
389	static void efx_init_eventq(struct efx_channel *channel)	389	static void efx_init_eventq(struct efx_channel *channel)
390	{	390	{
391	netif_dbg(channel->efx, drv, channel->efx->net_dev,	391	netif_dbg(channel->efx, drv, channel->efx->net_dev,
392	"chan %d init event queue\n", channel->channel);	392	"chan %d init event queue\n", channel->channel);
393		393
394	channel->eventq_read_ptr = 0;	394	channel->eventq_read_ptr = 0;
395		395
396	efx_nic_init_eventq(channel);	396	efx_nic_init_eventq(channel);
397	}	397	}
398		398
399	static void efx_fini_eventq(struct efx_channel *channel)	399	static void efx_fini_eventq(struct efx_channel *channel)
400	{	400	{
401	netif_dbg(channel->efx, drv, channel->efx->net_dev,	401	netif_dbg(channel->efx, drv, channel->efx->net_dev,
402	"chan %d fini event queue\n", channel->channel);	402	"chan %d fini event queue\n", channel->channel);
403		403
404	efx_nic_fini_eventq(channel);	404	efx_nic_fini_eventq(channel);
405	}	405	}
406		406
407	static void efx_remove_eventq(struct efx_channel *channel)	407	static void efx_remove_eventq(struct efx_channel *channel)
408	{	408	{
409	netif_dbg(channel->efx, drv, channel->efx->net_dev,	409	netif_dbg(channel->efx, drv, channel->efx->net_dev,
410	"chan %d remove event queue\n", channel->channel);	410	"chan %d remove event queue\n", channel->channel);
411		411
412	efx_nic_remove_eventq(channel);	412	efx_nic_remove_eventq(channel);
413	}	413	}
414		414
415	/**************************************************************************	415	/**************************************************************************
416	*	416	*
417	* Channel handling	417	* Channel handling
418	*	418	*
419	*************************************************************************/	419	*************************************************************************/
420		420
421	/* Allocate and initialise a channel structure, optionally copying	421	/* Allocate and initialise a channel structure, optionally copying
422	* parameters (but not resources) from an old channel structure. */	422	* parameters (but not resources) from an old channel structure. */
423	static struct efx_channel *	423	static struct efx_channel *
424	efx_alloc_channel(struct efx_nic efx, int i, struct efx_channel old_channel)	424	efx_alloc_channel(struct efx_nic efx, int i, struct efx_channel old_channel)
425	{	425	{
426	struct efx_channel *channel;	426	struct efx_channel *channel;
427	struct efx_rx_queue *rx_queue;	427	struct efx_rx_queue *rx_queue;
428	struct efx_tx_queue *tx_queue;	428	struct efx_tx_queue *tx_queue;
429	int j;	429	int j;
430		430
431	if (old_channel) {	431	if (old_channel) {
432	channel = kmalloc(sizeof(*channel), GFP_KERNEL);	432	channel = kmalloc(sizeof(*channel), GFP_KERNEL);
433	if (!channel)	433	if (!channel)
434	return NULL;	434	return NULL;
435		435
436	channel = old_channel;	436	channel = old_channel;
437		437
438	channel->napi_dev = NULL;	438	channel->napi_dev = NULL;
439	memset(&channel->eventq, 0, sizeof(channel->eventq));	439	memset(&channel->eventq, 0, sizeof(channel->eventq));
440		440
441	rx_queue = &channel->rx_queue;	441	rx_queue = &channel->rx_queue;
442	rx_queue->buffer = NULL;	442	rx_queue->buffer = NULL;
443	memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));	443	memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
444		444
445	for (j = 0; j < EFX_TXQ_TYPES; j++) {	445	for (j = 0; j < EFX_TXQ_TYPES; j++) {
446	tx_queue = &channel->tx_queue[j];	446	tx_queue = &channel->tx_queue[j];
447	if (tx_queue->channel)	447	if (tx_queue->channel)
448	tx_queue->channel = channel;	448	tx_queue->channel = channel;
449	tx_queue->buffer = NULL;	449	tx_queue->buffer = NULL;
450	memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));	450	memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
451	}	451	}
452	} else {	452	} else {
453	channel = kzalloc(sizeof(*channel), GFP_KERNEL);	453	channel = kzalloc(sizeof(*channel), GFP_KERNEL);
454	if (!channel)	454	if (!channel)
455	return NULL;	455	return NULL;
456		456
457	channel->efx = efx;	457	channel->efx = efx;
458	channel->channel = i;	458	channel->channel = i;
459		459
460	for (j = 0; j < EFX_TXQ_TYPES; j++) {	460	for (j = 0; j < EFX_TXQ_TYPES; j++) {
461	tx_queue = &channel->tx_queue[j];	461	tx_queue = &channel->tx_queue[j];
462	tx_queue->efx = efx;	462	tx_queue->efx = efx;
463	tx_queue->queue = i * EFX_TXQ_TYPES + j;	463	tx_queue->queue = i * EFX_TXQ_TYPES + j;
464	tx_queue->channel = channel;	464	tx_queue->channel = channel;
465	}	465	}
466	}	466	}
467		467
468	rx_queue = &channel->rx_queue;	468	rx_queue = &channel->rx_queue;
469	rx_queue->efx = efx;	469	rx_queue->efx = efx;
470	setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill,	470	setup_timer(&rx_queue->slow_fill, efx_rx_slow_fill,
471	(unsigned long)rx_queue);	471	(unsigned long)rx_queue);
472		472
473	return channel;	473	return channel;
474	}	474	}
475		475
476	static int efx_probe_channel(struct efx_channel *channel)	476	static int efx_probe_channel(struct efx_channel *channel)
477	{	477	{
478	struct efx_tx_queue *tx_queue;	478	struct efx_tx_queue *tx_queue;
479	struct efx_rx_queue *rx_queue;	479	struct efx_rx_queue *rx_queue;
480	int rc;	480	int rc;
481		481
482	netif_dbg(channel->efx, probe, channel->efx->net_dev,	482	netif_dbg(channel->efx, probe, channel->efx->net_dev,
483	"creating channel %d\n", channel->channel);	483	"creating channel %d\n", channel->channel);
484		484
485	rc = efx_probe_eventq(channel);	485	rc = efx_probe_eventq(channel);
486	if (rc)	486	if (rc)
487	goto fail1;	487	goto fail1;
488		488
489	efx_for_each_channel_tx_queue(tx_queue, channel) {	489	efx_for_each_channel_tx_queue(tx_queue, channel) {
490	rc = efx_probe_tx_queue(tx_queue);	490	rc = efx_probe_tx_queue(tx_queue);
491	if (rc)	491	if (rc)
492	goto fail2;	492	goto fail2;
493	}	493	}
494		494
495	efx_for_each_channel_rx_queue(rx_queue, channel) {	495	efx_for_each_channel_rx_queue(rx_queue, channel) {
496	rc = efx_probe_rx_queue(rx_queue);	496	rc = efx_probe_rx_queue(rx_queue);
497	if (rc)	497	if (rc)
498	goto fail3;	498	goto fail3;
499	}	499	}
500		500
501	channel->n_rx_frm_trunc = 0;	501	channel->n_rx_frm_trunc = 0;
502		502
503	return 0;	503	return 0;
504		504
505	fail3:	505	fail3:
506	efx_for_each_channel_rx_queue(rx_queue, channel)	506	efx_for_each_channel_rx_queue(rx_queue, channel)
507	efx_remove_rx_queue(rx_queue);	507	efx_remove_rx_queue(rx_queue);
508	fail2:	508	fail2:
509	efx_for_each_channel_tx_queue(tx_queue, channel)	509	efx_for_each_channel_tx_queue(tx_queue, channel)
510	efx_remove_tx_queue(tx_queue);	510	efx_remove_tx_queue(tx_queue);
511	fail1:	511	fail1:
512	return rc;	512	return rc;
513	}	513	}
514		514
515		515
516	static void efx_set_channel_names(struct efx_nic *efx)	516	static void efx_set_channel_names(struct efx_nic *efx)
517	{	517	{
518	struct efx_channel *channel;	518	struct efx_channel *channel;
519	const char *type = "";	519	const char *type = "";
520	int number;	520	int number;
521		521
522	efx_for_each_channel(channel, efx) {	522	efx_for_each_channel(channel, efx) {
523	number = channel->channel;	523	number = channel->channel;
524	if (efx->n_channels > efx->n_rx_channels) {	524	if (efx->n_channels > efx->n_rx_channels) {
525	if (channel->channel < efx->n_rx_channels) {	525	if (channel->channel < efx->n_rx_channels) {
526	type = "-rx";	526	type = "-rx";
527	} else {	527	} else {
528	type = "-tx";	528	type = "-tx";
529	number -= efx->n_rx_channels;	529	number -= efx->n_rx_channels;
530	}	530	}
531	}	531	}
532	snprintf(efx->channel_name[channel->channel],	532	snprintf(efx->channel_name[channel->channel],
533	sizeof(efx->channel_name[0]),	533	sizeof(efx->channel_name[0]),
534	"%s%s-%d", efx->name, type, number);	534	"%s%s-%d", efx->name, type, number);
535	}	535	}
536	}	536	}
537		537
538	static int efx_probe_channels(struct efx_nic *efx)	538	static int efx_probe_channels(struct efx_nic *efx)
539	{	539	{
540	struct efx_channel *channel;	540	struct efx_channel *channel;
541	int rc;	541	int rc;
542		542
543	/* Restart special buffer allocation */	543	/* Restart special buffer allocation */
544	efx->next_buffer_table = 0;	544	efx->next_buffer_table = 0;
545		545
546	efx_for_each_channel(channel, efx) {	546	efx_for_each_channel(channel, efx) {
547	rc = efx_probe_channel(channel);	547	rc = efx_probe_channel(channel);
548	if (rc) {	548	if (rc) {
549	netif_err(efx, probe, efx->net_dev,	549	netif_err(efx, probe, efx->net_dev,
550	"failed to create channel %d\n",	550	"failed to create channel %d\n",
551	channel->channel);	551	channel->channel);
552	goto fail;	552	goto fail;
553	}	553	}
554	}	554	}
555	efx_set_channel_names(efx);	555	efx_set_channel_names(efx);
556		556
557	return 0;	557	return 0;
558		558
559	fail:	559	fail:
560	efx_remove_channels(efx);	560	efx_remove_channels(efx);
561	return rc;	561	return rc;
562	}	562	}
563		563
564	/* Channels are shutdown and reinitialised whilst the NIC is running	564	/* Channels are shutdown and reinitialised whilst the NIC is running
565	* to propagate configuration changes (mtu, checksum offload), or	565	* to propagate configuration changes (mtu, checksum offload), or
566	* to clear hardware error conditions	566	* to clear hardware error conditions
567	*/	567	*/
568	static void efx_init_channels(struct efx_nic *efx)	568	static void efx_init_channels(struct efx_nic *efx)
569	{	569	{
570	struct efx_tx_queue *tx_queue;	570	struct efx_tx_queue *tx_queue;
571	struct efx_rx_queue *rx_queue;	571	struct efx_rx_queue *rx_queue;
572	struct efx_channel *channel;	572	struct efx_channel *channel;
573		573
574	/* Calculate the rx buffer allocation parameters required to	574	/* Calculate the rx buffer allocation parameters required to
575	* support the current MTU, including padding for header	575	* support the current MTU, including padding for header
576	* alignment and overruns.	576	* alignment and overruns.
577	*/	577	*/
578	efx->rx_buffer_len = (max(EFX_PAGE_IP_ALIGN, NET_IP_ALIGN) +	578	efx->rx_buffer_len = (max(EFX_PAGE_IP_ALIGN, NET_IP_ALIGN) +
579	EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +	579	EFX_MAX_FRAME_LEN(efx->net_dev->mtu) +
580	efx->type->rx_buffer_hash_size +	580	efx->type->rx_buffer_hash_size +
581	efx->type->rx_buffer_padding);	581	efx->type->rx_buffer_padding);
582	efx->rx_buffer_order = get_order(efx->rx_buffer_len +	582	efx->rx_buffer_order = get_order(efx->rx_buffer_len +
583	sizeof(struct efx_rx_page_state));	583	sizeof(struct efx_rx_page_state));
584		584
585	/* Initialise the channels */	585	/* Initialise the channels */
586	efx_for_each_channel(channel, efx) {	586	efx_for_each_channel(channel, efx) {
587	netif_dbg(channel->efx, drv, channel->efx->net_dev,	587	netif_dbg(channel->efx, drv, channel->efx->net_dev,
588	"init chan %d\n", channel->channel);	588	"init chan %d\n", channel->channel);
589		589
590	efx_init_eventq(channel);	590	efx_init_eventq(channel);
591		591
592	efx_for_each_channel_tx_queue(tx_queue, channel)	592	efx_for_each_channel_tx_queue(tx_queue, channel)
593	efx_init_tx_queue(tx_queue);	593	efx_init_tx_queue(tx_queue);
594		594
595	/* The rx buffer allocation strategy is MTU dependent */	595	/* The rx buffer allocation strategy is MTU dependent */
596	efx_rx_strategy(channel);	596	efx_rx_strategy(channel);
597		597
598	efx_for_each_channel_rx_queue(rx_queue, channel)	598	efx_for_each_channel_rx_queue(rx_queue, channel)
599	efx_init_rx_queue(rx_queue);	599	efx_init_rx_queue(rx_queue);
600		600
601	WARN_ON(channel->rx_pkt != NULL);	601	WARN_ON(channel->rx_pkt != NULL);
602	efx_rx_strategy(channel);	602	efx_rx_strategy(channel);
603	}	603	}
604	}	604	}
605		605
606	/* This enables event queue processing and packet transmission.	606	/* This enables event queue processing and packet transmission.
607	*	607	*
608	* Note that this function is not allowed to fail, since that would	608	* Note that this function is not allowed to fail, since that would
609	* introduce too much complexity into the suspend/resume path.	609	* introduce too much complexity into the suspend/resume path.
610	*/	610	*/
611	static void efx_start_channel(struct efx_channel *channel)	611	static void efx_start_channel(struct efx_channel *channel)
612	{	612	{
613	struct efx_rx_queue *rx_queue;	613	struct efx_rx_queue *rx_queue;
614		614
615	netif_dbg(channel->efx, ifup, channel->efx->net_dev,	615	netif_dbg(channel->efx, ifup, channel->efx->net_dev,
616	"starting chan %d\n", channel->channel);	616	"starting chan %d\n", channel->channel);
617		617
618	/* The interrupt handler for this channel may set work_pending	618	/* The interrupt handler for this channel may set work_pending
619	* as soon as we enable it. Make sure it's cleared before	619	* as soon as we enable it. Make sure it's cleared before
620	* then. Similarly, make sure it sees the enabled flag set. */	620	* then. Similarly, make sure it sees the enabled flag set. */
621	channel->work_pending = false;	621	channel->work_pending = false;
622	channel->enabled = true;	622	channel->enabled = true;
623	smp_wmb();	623	smp_wmb();
624		624
625	/* Fill the queues before enabling NAPI */	625	/* Fill the queues before enabling NAPI */
626	efx_for_each_channel_rx_queue(rx_queue, channel)	626	efx_for_each_channel_rx_queue(rx_queue, channel)
627	efx_fast_push_rx_descriptors(rx_queue);	627	efx_fast_push_rx_descriptors(rx_queue);
628		628
629	napi_enable(&channel->napi_str);	629	napi_enable(&channel->napi_str);
630	}	630	}
631		631
632	/* This disables event queue processing and packet transmission.	632	/* This disables event queue processing and packet transmission.
633	* This function does not guarantee that all queue processing	633	* This function does not guarantee that all queue processing
634	* (e.g. RX refill) is complete.	634	* (e.g. RX refill) is complete.
635	*/	635	*/
636	static void efx_stop_channel(struct efx_channel *channel)	636	static void efx_stop_channel(struct efx_channel *channel)
637	{	637	{
638	if (!channel->enabled)	638	if (!channel->enabled)
639	return;	639	return;
640		640
641	netif_dbg(channel->efx, ifdown, channel->efx->net_dev,	641	netif_dbg(channel->efx, ifdown, channel->efx->net_dev,
642	"stop chan %d\n", channel->channel);	642	"stop chan %d\n", channel->channel);
643		643
644	channel->enabled = false;	644	channel->enabled = false;
645	napi_disable(&channel->napi_str);	645	napi_disable(&channel->napi_str);
646	}	646	}
647		647
648	static void efx_fini_channels(struct efx_nic *efx)	648	static void efx_fini_channels(struct efx_nic *efx)
649	{	649	{
650	struct efx_channel *channel;	650	struct efx_channel *channel;
651	struct efx_tx_queue *tx_queue;	651	struct efx_tx_queue *tx_queue;
652	struct efx_rx_queue *rx_queue;	652	struct efx_rx_queue *rx_queue;
653	int rc;	653	int rc;
654		654
655	EFX_ASSERT_RESET_SERIALISED(efx);	655	EFX_ASSERT_RESET_SERIALISED(efx);
656	BUG_ON(efx->port_enabled);	656	BUG_ON(efx->port_enabled);
657		657
658	rc = efx_nic_flush_queues(efx);	658	rc = efx_nic_flush_queues(efx);
659	if (rc && EFX_WORKAROUND_7803(efx)) {	659	if (rc && EFX_WORKAROUND_7803(efx)) {
660	/* Schedule a reset to recover from the flush failure. The	660	/* Schedule a reset to recover from the flush failure. The
661	* descriptor caches reference memory we're about to free,	661	* descriptor caches reference memory we're about to free,
662	* but falcon_reconfigure_mac_wrapper() won't reconnect	662	* but falcon_reconfigure_mac_wrapper() won't reconnect
663	* the MACs because of the pending reset. */	663	* the MACs because of the pending reset. */
664	netif_err(efx, drv, efx->net_dev,	664	netif_err(efx, drv, efx->net_dev,
665	"Resetting to recover from flush failure\n");	665	"Resetting to recover from flush failure\n");
666	efx_schedule_reset(efx, RESET_TYPE_ALL);	666	efx_schedule_reset(efx, RESET_TYPE_ALL);
667	} else if (rc) {	667	} else if (rc) {
668	netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");	668	netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
669	} else {	669	} else {
670	netif_dbg(efx, drv, efx->net_dev,	670	netif_dbg(efx, drv, efx->net_dev,
671	"successfully flushed all queues\n");	671	"successfully flushed all queues\n");
672	}	672	}
673		673
674	efx_for_each_channel(channel, efx) {	674	efx_for_each_channel(channel, efx) {
675	netif_dbg(channel->efx, drv, channel->efx->net_dev,	675	netif_dbg(channel->efx, drv, channel->efx->net_dev,
676	"shut down chan %d\n", channel->channel);	676	"shut down chan %d\n", channel->channel);
677		677
678	efx_for_each_channel_rx_queue(rx_queue, channel)	678	efx_for_each_channel_rx_queue(rx_queue, channel)
679	efx_fini_rx_queue(rx_queue);	679	efx_fini_rx_queue(rx_queue);
680	efx_for_each_possible_channel_tx_queue(tx_queue, channel)	680	efx_for_each_possible_channel_tx_queue(tx_queue, channel)
681	efx_fini_tx_queue(tx_queue);	681	efx_fini_tx_queue(tx_queue);
682	efx_fini_eventq(channel);	682	efx_fini_eventq(channel);
683	}	683	}
684	}	684	}
685		685
686	static void efx_remove_channel(struct efx_channel *channel)	686	static void efx_remove_channel(struct efx_channel *channel)
687	{	687	{
688	struct efx_tx_queue *tx_queue;	688	struct efx_tx_queue *tx_queue;
689	struct efx_rx_queue *rx_queue;	689	struct efx_rx_queue *rx_queue;
690		690
691	netif_dbg(channel->efx, drv, channel->efx->net_dev,	691	netif_dbg(channel->efx, drv, channel->efx->net_dev,
692	"destroy chan %d\n", channel->channel);	692	"destroy chan %d\n", channel->channel);
693		693
694	efx_for_each_channel_rx_queue(rx_queue, channel)	694	efx_for_each_channel_rx_queue(rx_queue, channel)
695	efx_remove_rx_queue(rx_queue);	695	efx_remove_rx_queue(rx_queue);
696	efx_for_each_possible_channel_tx_queue(tx_queue, channel)	696	efx_for_each_possible_channel_tx_queue(tx_queue, channel)
697	efx_remove_tx_queue(tx_queue);	697	efx_remove_tx_queue(tx_queue);
698	efx_remove_eventq(channel);	698	efx_remove_eventq(channel);
699	}	699	}
700		700
701	static void efx_remove_channels(struct efx_nic *efx)	701	static void efx_remove_channels(struct efx_nic *efx)
702	{	702	{
703	struct efx_channel *channel;	703	struct efx_channel *channel;
704		704
705	efx_for_each_channel(channel, efx)	705	efx_for_each_channel(channel, efx)
706	efx_remove_channel(channel);	706	efx_remove_channel(channel);
707	}	707	}
708		708
709	int	709	int
710	efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)	710	efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
711	{	711	{
712	struct efx_channel other_channel[EFX_MAX_CHANNELS], channel;	712	struct efx_channel other_channel[EFX_MAX_CHANNELS], channel;
713	u32 old_rxq_entries, old_txq_entries;	713	u32 old_rxq_entries, old_txq_entries;
714	unsigned i;	714	unsigned i;
715	int rc;	715	int rc;
716		716
717	efx_stop_all(efx);	717	efx_stop_all(efx);
718	efx_fini_channels(efx);	718	efx_fini_channels(efx);
719		719
720	/* Clone channels */	720	/* Clone channels */
721	memset(other_channel, 0, sizeof(other_channel));	721	memset(other_channel, 0, sizeof(other_channel));
722	for (i = 0; i < efx->n_channels; i++) {	722	for (i = 0; i < efx->n_channels; i++) {
723	channel = efx_alloc_channel(efx, i, efx->channel[i]);	723	channel = efx_alloc_channel(efx, i, efx->channel[i]);
724	if (!channel) {	724	if (!channel) {
725	rc = -ENOMEM;	725	rc = -ENOMEM;
726	goto out;	726	goto out;
727	}	727	}
728	other_channel[i] = channel;	728	other_channel[i] = channel;
729	}	729	}
730		730
731	/* Swap entry counts and channel pointers */	731	/* Swap entry counts and channel pointers */
732	old_rxq_entries = efx->rxq_entries;	732	old_rxq_entries = efx->rxq_entries;
733	old_txq_entries = efx->txq_entries;	733	old_txq_entries = efx->txq_entries;
734	efx->rxq_entries = rxq_entries;	734	efx->rxq_entries = rxq_entries;
735	efx->txq_entries = txq_entries;	735	efx->txq_entries = txq_entries;
736	for (i = 0; i < efx->n_channels; i++) {	736	for (i = 0; i < efx->n_channels; i++) {
737	channel = efx->channel[i];	737	channel = efx->channel[i];
738	efx->channel[i] = other_channel[i];	738	efx->channel[i] = other_channel[i];
739	other_channel[i] = channel;	739	other_channel[i] = channel;
740	}	740	}
741		741
742	rc = efx_probe_channels(efx);	742	rc = efx_probe_channels(efx);
743	if (rc)	743	if (rc)
744	goto rollback;	744	goto rollback;
745		745
746	efx_init_napi(efx);	746	efx_init_napi(efx);
747		747
748	/* Destroy old channels */	748	/* Destroy old channels */
749	for (i = 0; i < efx->n_channels; i++) {	749	for (i = 0; i < efx->n_channels; i++) {
750	efx_fini_napi_channel(other_channel[i]);	750	efx_fini_napi_channel(other_channel[i]);
751	efx_remove_channel(other_channel[i]);	751	efx_remove_channel(other_channel[i]);
752	}	752	}
753	out:	753	out:
754	/* Free unused channel structures */	754	/* Free unused channel structures */
755	for (i = 0; i < efx->n_channels; i++)	755	for (i = 0; i < efx->n_channels; i++)
756	kfree(other_channel[i]);	756	kfree(other_channel[i]);
757		757
758	efx_init_channels(efx);	758	efx_init_channels(efx);
759	efx_start_all(efx);	759	efx_start_all(efx);
760	return rc;	760	return rc;
761		761
762	rollback:	762	rollback:
763	/* Swap back */	763	/* Swap back */
764	efx->rxq_entries = old_rxq_entries;	764	efx->rxq_entries = old_rxq_entries;
765	efx->txq_entries = old_txq_entries;	765	efx->txq_entries = old_txq_entries;
766	for (i = 0; i < efx->n_channels; i++) {	766	for (i = 0; i < efx->n_channels; i++) {
767	channel = efx->channel[i];	767	channel = efx->channel[i];
768	efx->channel[i] = other_channel[i];	768	efx->channel[i] = other_channel[i];
769	other_channel[i] = channel;	769	other_channel[i] = channel;
770	}	770	}
771	goto out;	771	goto out;
772	}	772	}
773		773
774	void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)	774	void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue)
775	{	775	{
776	mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100));	776	mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(100));
777	}	777	}
778		778
779	/**************************************************************************	779	/**************************************************************************
780	*	780	*
781	* Port handling	781	* Port handling
782	*	782	*
783	**************************************************************************/	783	**************************************************************************/
784		784
785	/* This ensures that the kernel is kept informed (via	785	/* This ensures that the kernel is kept informed (via
786	* netif_carrier_on/off) of the link status, and also maintains the	786	* netif_carrier_on/off) of the link status, and also maintains the
787	* link status's stop on the port's TX queue.	787	* link status's stop on the port's TX queue.
788	*/	788	*/
789	void efx_link_status_changed(struct efx_nic *efx)	789	void efx_link_status_changed(struct efx_nic *efx)
790	{	790	{
791	struct efx_link_state *link_state = &efx->link_state;	791	struct efx_link_state *link_state = &efx->link_state;
792		792
793	/* SFC Bug 5356: A net_dev notifier is registered, so we must ensure	793	/* SFC Bug 5356: A net_dev notifier is registered, so we must ensure
794	* that no events are triggered between unregister_netdev() and the	794	* that no events are triggered between unregister_netdev() and the
795	* driver unloading. A more general condition is that NETDEV_CHANGE	795	* driver unloading. A more general condition is that NETDEV_CHANGE
796	* can only be generated between NETDEV_UP and NETDEV_DOWN */	796	* can only be generated between NETDEV_UP and NETDEV_DOWN */
797	if (!netif_running(efx->net_dev))	797	if (!netif_running(efx->net_dev))
798	return;	798	return;
799		799
800	if (link_state->up != netif_carrier_ok(efx->net_dev)) {	800	if (link_state->up != netif_carrier_ok(efx->net_dev)) {
801	efx->n_link_state_changes++;	801	efx->n_link_state_changes++;
802		802
803	if (link_state->up)	803	if (link_state->up)
804	netif_carrier_on(efx->net_dev);	804	netif_carrier_on(efx->net_dev);
805	else	805	else
806	netif_carrier_off(efx->net_dev);	806	netif_carrier_off(efx->net_dev);
807	}	807	}
808		808
809	/* Status message for kernel log */	809	/* Status message for kernel log */
810	if (link_state->up) {	810	if (link_state->up) {
811	netif_info(efx, link, efx->net_dev,	811	netif_info(efx, link, efx->net_dev,
812	"link up at %uMbps %s-duplex (MTU %d)%s\n",	812	"link up at %uMbps %s-duplex (MTU %d)%s\n",
813	link_state->speed, link_state->fd ? "full" : "half",	813	link_state->speed, link_state->fd ? "full" : "half",
814	efx->net_dev->mtu,	814	efx->net_dev->mtu,
815	(efx->promiscuous ? " [PROMISC]" : ""));	815	(efx->promiscuous ? " [PROMISC]" : ""));
816	} else {	816	} else {
817	netif_info(efx, link, efx->net_dev, "link down\n");	817	netif_info(efx, link, efx->net_dev, "link down\n");
818	}	818	}
819		819
820	}	820	}
821		821
822	void efx_link_set_advertising(struct efx_nic *efx, u32 advertising)	822	void efx_link_set_advertising(struct efx_nic *efx, u32 advertising)
823	{	823	{
824	efx->link_advertising = advertising;	824	efx->link_advertising = advertising;
825	if (advertising) {	825	if (advertising) {
826	if (advertising & ADVERTISED_Pause)	826	if (advertising & ADVERTISED_Pause)
827	efx->wanted_fc \|= (EFX_FC_TX \| EFX_FC_RX);	827	efx->wanted_fc \|= (EFX_FC_TX \| EFX_FC_RX);
828	else	828	else
829	efx->wanted_fc &= ~(EFX_FC_TX \| EFX_FC_RX);	829	efx->wanted_fc &= ~(EFX_FC_TX \| EFX_FC_RX);
830	if (advertising & ADVERTISED_Asym_Pause)	830	if (advertising & ADVERTISED_Asym_Pause)
831	efx->wanted_fc ^= EFX_FC_TX;	831	efx->wanted_fc ^= EFX_FC_TX;
832	}	832	}
833	}	833	}
834		834
835	void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)	835	void efx_link_set_wanted_fc(struct efx_nic *efx, u8 wanted_fc)
836	{	836	{
837	efx->wanted_fc = wanted_fc;	837	efx->wanted_fc = wanted_fc;
838	if (efx->link_advertising) {	838	if (efx->link_advertising) {
839	if (wanted_fc & EFX_FC_RX)	839	if (wanted_fc & EFX_FC_RX)
840	efx->link_advertising \|= (ADVERTISED_Pause \|	840	efx->link_advertising \|= (ADVERTISED_Pause \|
841	ADVERTISED_Asym_Pause);	841	ADVERTISED_Asym_Pause);
842	else	842	else
843	efx->link_advertising &= ~(ADVERTISED_Pause \|	843	efx->link_advertising &= ~(ADVERTISED_Pause \|
844	ADVERTISED_Asym_Pause);	844	ADVERTISED_Asym_Pause);
845	if (wanted_fc & EFX_FC_TX)	845	if (wanted_fc & EFX_FC_TX)
846	efx->link_advertising ^= ADVERTISED_Asym_Pause;	846	efx->link_advertising ^= ADVERTISED_Asym_Pause;
847	}	847	}
848	}	848	}
849		849
850	static void efx_fini_port(struct efx_nic *efx);	850	static void efx_fini_port(struct efx_nic *efx);
851		851
852	/* Push loopback/power/transmit disable settings to the PHY, and reconfigure	852	/* Push loopback/power/transmit disable settings to the PHY, and reconfigure
853	* the MAC appropriately. All other PHY configuration changes are pushed	853	* the MAC appropriately. All other PHY configuration changes are pushed
854	* through phy_op->set_settings(), and pushed asynchronously to the MAC	854	* through phy_op->set_settings(), and pushed asynchronously to the MAC
855	* through efx_monitor().	855	* through efx_monitor().
856	*	856	*
857	* Callers must hold the mac_lock	857	* Callers must hold the mac_lock
858	*/	858	*/
859	int __efx_reconfigure_port(struct efx_nic *efx)	859	int __efx_reconfigure_port(struct efx_nic *efx)
860	{	860	{
861	enum efx_phy_mode phy_mode;	861	enum efx_phy_mode phy_mode;
862	int rc;	862	int rc;
863		863
864	WARN_ON(!mutex_is_locked(&efx->mac_lock));	864	WARN_ON(!mutex_is_locked(&efx->mac_lock));
865		865
866	/* Serialise the promiscuous flag with efx_set_multicast_list. */	866	/* Serialise the promiscuous flag with efx_set_multicast_list. */
867	if (efx_dev_registered(efx)) {	867	if (efx_dev_registered(efx)) {
868	netif_addr_lock_bh(efx->net_dev);	868	netif_addr_lock_bh(efx->net_dev);
869	netif_addr_unlock_bh(efx->net_dev);	869	netif_addr_unlock_bh(efx->net_dev);
870	}	870	}
871		871
872	/* Disable PHY transmit in mac level loopbacks */	872	/* Disable PHY transmit in mac level loopbacks */
873	phy_mode = efx->phy_mode;	873	phy_mode = efx->phy_mode;
874	if (LOOPBACK_INTERNAL(efx))	874	if (LOOPBACK_INTERNAL(efx))
875	efx->phy_mode \|= PHY_MODE_TX_DISABLED;	875	efx->phy_mode \|= PHY_MODE_TX_DISABLED;
876	else	876	else
877	efx->phy_mode &= ~PHY_MODE_TX_DISABLED;	877	efx->phy_mode &= ~PHY_MODE_TX_DISABLED;
878		878
879	rc = efx->type->reconfigure_port(efx);	879	rc = efx->type->reconfigure_port(efx);
880		880
881	if (rc)	881	if (rc)
882	efx->phy_mode = phy_mode;	882	efx->phy_mode = phy_mode;
883		883
884	return rc;	884	return rc;
885	}	885	}
886		886
887	/* Reinitialise the MAC to pick up new PHY settings, even if the port is	887	/* Reinitialise the MAC to pick up new PHY settings, even if the port is
888	* disabled. */	888	* disabled. */
889	int efx_reconfigure_port(struct efx_nic *efx)	889	int efx_reconfigure_port(struct efx_nic *efx)
890	{	890	{
891	int rc;	891	int rc;
892		892
893	EFX_ASSERT_RESET_SERIALISED(efx);	893	EFX_ASSERT_RESET_SERIALISED(efx);
894		894
895	mutex_lock(&efx->mac_lock);	895	mutex_lock(&efx->mac_lock);
896	rc = __efx_reconfigure_port(efx);	896	rc = __efx_reconfigure_port(efx);
897	mutex_unlock(&efx->mac_lock);	897	mutex_unlock(&efx->mac_lock);
898		898
899	return rc;	899	return rc;
900	}	900	}
901		901
902	/* Asynchronous work item for changing MAC promiscuity and multicast	902	/* Asynchronous work item for changing MAC promiscuity and multicast
903	* hash. Avoid a drain/rx_ingress enable by reconfiguring the current	903	* hash. Avoid a drain/rx_ingress enable by reconfiguring the current
904	* MAC directly. */	904	* MAC directly. */
905	static void efx_mac_work(struct work_struct *data)	905	static void efx_mac_work(struct work_struct *data)
906	{	906	{
907	struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);	907	struct efx_nic *efx = container_of(data, struct efx_nic, mac_work);
908		908
909	mutex_lock(&efx->mac_lock);	909	mutex_lock(&efx->mac_lock);
910	if (efx->port_enabled) {	910	if (efx->port_enabled) {
911	efx->type->push_multicast_hash(efx);	911	efx->type->push_multicast_hash(efx);
912	efx->mac_op->reconfigure(efx);	912	efx->mac_op->reconfigure(efx);
913	}	913	}
914	mutex_unlock(&efx->mac_lock);	914	mutex_unlock(&efx->mac_lock);
915	}	915	}
916		916
917	static int efx_probe_port(struct efx_nic *efx)	917	static int efx_probe_port(struct efx_nic *efx)
918	{	918	{
919	unsigned char *perm_addr;	919	unsigned char *perm_addr;
920	int rc;	920	int rc;
921		921
922	netif_dbg(efx, probe, efx->net_dev, "create port\n");	922	netif_dbg(efx, probe, efx->net_dev, "create port\n");
923		923
924	if (phy_flash_cfg)	924	if (phy_flash_cfg)
925	efx->phy_mode = PHY_MODE_SPECIAL;	925	efx->phy_mode = PHY_MODE_SPECIAL;
926		926
927	/* Connect up MAC/PHY operations table */	927	/* Connect up MAC/PHY operations table */
928	rc = efx->type->probe_port(efx);	928	rc = efx->type->probe_port(efx);
929	if (rc)	929	if (rc)
930	return rc;	930	return rc;
931		931
932	/* Sanity check MAC address */	932	/* Sanity check MAC address */
933	perm_addr = efx->net_dev->perm_addr;	933	perm_addr = efx->net_dev->perm_addr;
934	if (is_valid_ether_addr(perm_addr)) {	934	if (is_valid_ether_addr(perm_addr)) {
935	memcpy(efx->net_dev->dev_addr, perm_addr, ETH_ALEN);	935	memcpy(efx->net_dev->dev_addr, perm_addr, ETH_ALEN);
936	} else {	936	} else {
937	netif_err(efx, probe, efx->net_dev, "invalid MAC address %pM\n",	937	netif_err(efx, probe, efx->net_dev, "invalid MAC address %pM\n",
938	perm_addr);	938	perm_addr);
939	if (!allow_bad_hwaddr) {	939	if (!allow_bad_hwaddr) {
940	rc = -EINVAL;	940	rc = -EINVAL;
941	goto err;	941	goto err;
942	}	942	}
943	random_ether_addr(efx->net_dev->dev_addr);	943	random_ether_addr(efx->net_dev->dev_addr);
944	netif_info(efx, probe, efx->net_dev,	944	netif_info(efx, probe, efx->net_dev,
945	"using locally-generated MAC %pM\n",	945	"using locally-generated MAC %pM\n",
946	efx->net_dev->dev_addr);	946	efx->net_dev->dev_addr);
947	}	947	}
948		948
949	return 0;	949	return 0;
950		950
951	err:	951	err:
952	efx->type->remove_port(efx);	952	efx->type->remove_port(efx);
953	return rc;	953	return rc;
954	}	954	}
955		955
956	static int efx_init_port(struct efx_nic *efx)	956	static int efx_init_port(struct efx_nic *efx)
957	{	957	{
958	int rc;	958	int rc;
959		959
960	netif_dbg(efx, drv, efx->net_dev, "init port\n");	960	netif_dbg(efx, drv, efx->net_dev, "init port\n");
961		961
962	mutex_lock(&efx->mac_lock);	962	mutex_lock(&efx->mac_lock);
963		963
964	rc = efx->phy_op->init(efx);	964	rc = efx->phy_op->init(efx);
965	if (rc)	965	if (rc)
966	goto fail1;	966	goto fail1;
967		967
968	efx->port_initialized = true;	968	efx->port_initialized = true;
969		969
970	/* Reconfigure the MAC before creating dma queues (required for	970	/* Reconfigure the MAC before creating dma queues (required for
971	* Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */	971	* Falcon/A1 where RX_INGR_EN/TX_DRAIN_EN isn't supported) */
972	efx->mac_op->reconfigure(efx);	972	efx->mac_op->reconfigure(efx);
973		973
974	/* Ensure the PHY advertises the correct flow control settings */	974	/* Ensure the PHY advertises the correct flow control settings */
975	rc = efx->phy_op->reconfigure(efx);	975	rc = efx->phy_op->reconfigure(efx);
976	if (rc)	976	if (rc)
977	goto fail2;	977	goto fail2;
978		978
979	mutex_unlock(&efx->mac_lock);	979	mutex_unlock(&efx->mac_lock);
980	return 0;	980	return 0;
981		981
982	fail2:	982	fail2:
983	efx->phy_op->fini(efx);	983	efx->phy_op->fini(efx);
984	fail1:	984	fail1:
985	mutex_unlock(&efx->mac_lock);	985	mutex_unlock(&efx->mac_lock);
986	return rc;	986	return rc;
987	}	987	}
988		988
989	static void efx_start_port(struct efx_nic *efx)	989	static void efx_start_port(struct efx_nic *efx)
990	{	990	{
991	netif_dbg(efx, ifup, efx->net_dev, "start port\n");	991	netif_dbg(efx, ifup, efx->net_dev, "start port\n");
992	BUG_ON(efx->port_enabled);	992	BUG_ON(efx->port_enabled);
993		993
994	mutex_lock(&efx->mac_lock);	994	mutex_lock(&efx->mac_lock);
995	efx->port_enabled = true;	995	efx->port_enabled = true;
996		996
997	/* efx_mac_work() might have been scheduled after efx_stop_port(),	997	/* efx_mac_work() might have been scheduled after efx_stop_port(),
998	* and then cancelled by efx_flush_all() */	998	* and then cancelled by efx_flush_all() */
999	efx->type->push_multicast_hash(efx);	999	efx->type->push_multicast_hash(efx);
1000	efx->mac_op->reconfigure(efx);	1000	efx->mac_op->reconfigure(efx);
1001		1001
1002	mutex_unlock(&efx->mac_lock);	1002	mutex_unlock(&efx->mac_lock);
1003	}	1003	}
1004		1004
1005	/* Prevent efx_mac_work() and efx_monitor() from working */	1005	/* Prevent efx_mac_work() and efx_monitor() from working */
1006	static void efx_stop_port(struct efx_nic *efx)	1006	static void efx_stop_port(struct efx_nic *efx)
1007	{	1007	{
1008	netif_dbg(efx, ifdown, efx->net_dev, "stop port\n");	1008	netif_dbg(efx, ifdown, efx->net_dev, "stop port\n");
1009		1009
1010	mutex_lock(&efx->mac_lock);	1010	mutex_lock(&efx->mac_lock);
1011	efx->port_enabled = false;	1011	efx->port_enabled = false;
1012	mutex_unlock(&efx->mac_lock);	1012	mutex_unlock(&efx->mac_lock);
1013		1013
1014	/* Serialise against efx_set_multicast_list() */	1014	/* Serialise against efx_set_multicast_list() */
1015	if (efx_dev_registered(efx)) {	1015	if (efx_dev_registered(efx)) {
1016	netif_addr_lock_bh(efx->net_dev);	1016	netif_addr_lock_bh(efx->net_dev);
1017	netif_addr_unlock_bh(efx->net_dev);	1017	netif_addr_unlock_bh(efx->net_dev);
1018	}	1018	}
1019	}	1019	}
1020		1020
1021	static void efx_fini_port(struct efx_nic *efx)	1021	static void efx_fini_port(struct efx_nic *efx)
1022	{	1022	{
1023	netif_dbg(efx, drv, efx->net_dev, "shut down port\n");	1023	netif_dbg(efx, drv, efx->net_dev, "shut down port\n");
1024		1024
1025	if (!efx->port_initialized)	1025	if (!efx->port_initialized)
1026	return;	1026	return;
1027		1027
1028	efx->phy_op->fini(efx);	1028	efx->phy_op->fini(efx);
1029	efx->port_initialized = false;	1029	efx->port_initialized = false;
1030		1030
1031	efx->link_state.up = false;	1031	efx->link_state.up = false;
1032	efx_link_status_changed(efx);	1032	efx_link_status_changed(efx);
1033	}	1033	}
1034		1034
1035	static void efx_remove_port(struct efx_nic *efx)	1035	static void efx_remove_port(struct efx_nic *efx)
1036	{	1036	{
1037	netif_dbg(efx, drv, efx->net_dev, "destroying port\n");	1037	netif_dbg(efx, drv, efx->net_dev, "destroying port\n");
1038		1038
1039	efx->type->remove_port(efx);	1039	efx->type->remove_port(efx);
1040	}	1040	}
1041		1041
1042	/**************************************************************************	1042	/**************************************************************************
1043	*	1043	*
1044	* NIC handling	1044	* NIC handling
1045	*	1045	*
1046	**************************************************************************/	1046	**************************************************************************/
1047		1047
1048	/* This configures the PCI device to enable I/O and DMA. */	1048	/* This configures the PCI device to enable I/O and DMA. */
1049	static int efx_init_io(struct efx_nic *efx)	1049	static int efx_init_io(struct efx_nic *efx)
1050	{	1050	{
1051	struct pci_dev *pci_dev = efx->pci_dev;	1051	struct pci_dev *pci_dev = efx->pci_dev;
1052	dma_addr_t dma_mask = efx->type->max_dma_mask;	1052	dma_addr_t dma_mask = efx->type->max_dma_mask;
1053	bool use_wc;
1054	int rc;	1053	int rc;
1055		1054
1056	netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");	1055	netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n");
1057		1056
1058	rc = pci_enable_device(pci_dev);	1057	rc = pci_enable_device(pci_dev);
1059	if (rc) {	1058	if (rc) {
1060	netif_err(efx, probe, efx->net_dev,	1059	netif_err(efx, probe, efx->net_dev,
1061	"failed to enable PCI device\n");	1060	"failed to enable PCI device\n");
1062	goto fail1;	1061	goto fail1;
1063	}	1062	}
1064		1063
1065	pci_set_master(pci_dev);	1064	pci_set_master(pci_dev);
1066		1065
1067	/* Set the PCI DMA mask. Try all possibilities from our	1066	/* Set the PCI DMA mask. Try all possibilities from our
1068	* genuine mask down to 32 bits, because some architectures	1067	* genuine mask down to 32 bits, because some architectures
1069	* (e.g. x86_64 with iommu_sac_force set) will allow 40 bit	1068	* (e.g. x86_64 with iommu_sac_force set) will allow 40 bit
1070	* masks event though they reject 46 bit masks.	1069	* masks event though they reject 46 bit masks.
1071	*/	1070	*/
1072	while (dma_mask > 0x7fffffffUL) {	1071	while (dma_mask > 0x7fffffffUL) {
1073	if (pci_dma_supported(pci_dev, dma_mask) &&	1072	if (pci_dma_supported(pci_dev, dma_mask) &&
1074	((rc = pci_set_dma_mask(pci_dev, dma_mask)) == 0))	1073	((rc = pci_set_dma_mask(pci_dev, dma_mask)) == 0))
1075	break;	1074	break;
1076	dma_mask >>= 1;	1075	dma_mask >>= 1;
1077	}	1076	}
1078	if (rc) {	1077	if (rc) {
1079	netif_err(efx, probe, efx->net_dev,	1078	netif_err(efx, probe, efx->net_dev,
1080	"could not find a suitable DMA mask\n");	1079	"could not find a suitable DMA mask\n");
1081	goto fail2;	1080	goto fail2;
1082	}	1081	}
1083	netif_dbg(efx, probe, efx->net_dev,	1082	netif_dbg(efx, probe, efx->net_dev,
1084	"using DMA mask %llx\n", (unsigned long long) dma_mask);	1083	"using DMA mask %llx\n", (unsigned long long) dma_mask);
1085	rc = pci_set_consistent_dma_mask(pci_dev, dma_mask);	1084	rc = pci_set_consistent_dma_mask(pci_dev, dma_mask);
1086	if (rc) {	1085	if (rc) {
1087	/* pci_set_consistent_dma_mask() is not allowed to	1086	/* pci_set_consistent_dma_mask() is not allowed to
1088	* fail with a mask that pci_set_dma_mask() accepted,	1087	* fail with a mask that pci_set_dma_mask() accepted,
1089	* but just in case...	1088	* but just in case...
1090	*/	1089	*/
1091	netif_err(efx, probe, efx->net_dev,	1090	netif_err(efx, probe, efx->net_dev,
1092	"failed to set consistent DMA mask\n");	1091	"failed to set consistent DMA mask\n");
1093	goto fail2;	1092	goto fail2;
1094	}	1093	}
1095		1094
1096	efx->membase_phys = pci_resource_start(efx->pci_dev, EFX_MEM_BAR);	1095	efx->membase_phys = pci_resource_start(efx->pci_dev, EFX_MEM_BAR);
1097	rc = pci_request_region(pci_dev, EFX_MEM_BAR, "sfc");	1096	rc = pci_request_region(pci_dev, EFX_MEM_BAR, "sfc");
1098	if (rc) {	1097	if (rc) {
1099	netif_err(efx, probe, efx->net_dev,	1098	netif_err(efx, probe, efx->net_dev,
1100	"request for memory BAR failed\n");	1099	"request for memory BAR failed\n");
1101	rc = -EIO;	1100	rc = -EIO;
1102	goto fail3;	1101	goto fail3;
1103	}	1102	}
1104		1103	efx->membase = ioremap_nocache(efx->membase_phys,
1105	/* bug22643: If SR-IOV is enabled then tx push over a write combined	1104	efx->type->mem_map_size);
1106	* mapping is unsafe. We need to disable write combining in this case.
1107	* MSI is unsupported when SR-IOV is enabled, and the firmware will
1108	* have removed the MSI capability. So write combining is safe if
1109	* there is an MSI capability.
1110	*/
1111	use_wc = (!EFX_WORKAROUND_22643(efx) \|\|
1112	pci_find_capability(pci_dev, PCI_CAP_ID_MSI));
1113	if (use_wc)
1114	efx->membase = ioremap_wc(efx->membase_phys,
1115	efx->type->mem_map_size);
1116	else
1117	efx->membase = ioremap_nocache(efx->membase_phys,
1118	efx->type->mem_map_size);
1119	if (!efx->membase) {	1105	if (!efx->membase) {
1120	netif_err(efx, probe, efx->net_dev,	1106	netif_err(efx, probe, efx->net_dev,
1121	"could not map memory BAR at %llx+%x\n",	1107	"could not map memory BAR at %llx+%x\n",
1122	(unsigned long long)efx->membase_phys,	1108	(unsigned long long)efx->membase_phys,
1123	efx->type->mem_map_size);	1109	efx->type->mem_map_size);
1124	rc = -ENOMEM;	1110	rc = -ENOMEM;
1125	goto fail4;	1111	goto fail4;
1126	}	1112	}
1127	netif_dbg(efx, probe, efx->net_dev,	1113	netif_dbg(efx, probe, efx->net_dev,
1128	"memory BAR at %llx+%x (virtual %p)\n",	1114	"memory BAR at %llx+%x (virtual %p)\n",
1129	(unsigned long long)efx->membase_phys,	1115	(unsigned long long)efx->membase_phys,
1130	efx->type->mem_map_size, efx->membase);	1116	efx->type->mem_map_size, efx->membase);
1131		1117
1132	return 0;	1118	return 0;
1133		1119
1134	fail4:	1120	fail4:
1135	pci_release_region(efx->pci_dev, EFX_MEM_BAR);	1121	pci_release_region(efx->pci_dev, EFX_MEM_BAR);
1136	fail3:	1122	fail3:
1137	efx->membase_phys = 0;	1123	efx->membase_phys = 0;
1138	fail2:	1124	fail2:
1139	pci_disable_device(efx->pci_dev);	1125	pci_disable_device(efx->pci_dev);
1140	fail1:	1126	fail1:
1141	return rc;	1127	return rc;
1142	}	1128	}
1143		1129
1144	static void efx_fini_io(struct efx_nic *efx)	1130	static void efx_fini_io(struct efx_nic *efx)
1145	{	1131	{
1146	netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n");	1132	netif_dbg(efx, drv, efx->net_dev, "shutting down I/O\n");
1147		1133
1148	if (efx->membase) {	1134	if (efx->membase) {
1149	iounmap(efx->membase);	1135	iounmap(efx->membase);
1150	efx->membase = NULL;	1136	efx->membase = NULL;
1151	}	1137	}
1152		1138
1153	if (efx->membase_phys) {	1139	if (efx->membase_phys) {
1154	pci_release_region(efx->pci_dev, EFX_MEM_BAR);	1140	pci_release_region(efx->pci_dev, EFX_MEM_BAR);
1155	efx->membase_phys = 0;	1141	efx->membase_phys = 0;
1156	}	1142	}
1157		1143
1158	pci_disable_device(efx->pci_dev);	1144	pci_disable_device(efx->pci_dev);
1159	}	1145	}
1160		1146
1161	/* Get number of channels wanted. Each channel will have its own IRQ,	1147	/* Get number of channels wanted. Each channel will have its own IRQ,
1162	* 1 RX queue and/or 2 TX queues. */	1148	* 1 RX queue and/or 2 TX queues. */
1163	static int efx_wanted_channels(void)	1149	static int efx_wanted_channels(void)
1164	{	1150	{
1165	cpumask_var_t core_mask;	1151	cpumask_var_t core_mask;
1166	int count;	1152	int count;
1167	int cpu;	1153	int cpu;
1168		1154
1169	if (rss_cpus)	1155	if (rss_cpus)
1170	return rss_cpus;	1156	return rss_cpus;
1171		1157
1172	if (unlikely(!zalloc_cpumask_var(&core_mask, GFP_KERNEL))) {	1158	if (unlikely(!zalloc_cpumask_var(&core_mask, GFP_KERNEL))) {
1173	printk(KERN_WARNING	1159	printk(KERN_WARNING
1174	"sfc: RSS disabled due to allocation failure\n");	1160	"sfc: RSS disabled due to allocation failure\n");
1175	return 1;	1161	return 1;
1176	}	1162	}
1177		1163
1178	count = 0;	1164	count = 0;
1179	for_each_online_cpu(cpu) {	1165	for_each_online_cpu(cpu) {
1180	if (!cpumask_test_cpu(cpu, core_mask)) {	1166	if (!cpumask_test_cpu(cpu, core_mask)) {
1181	++count;	1167	++count;
1182	cpumask_or(core_mask, core_mask,	1168	cpumask_or(core_mask, core_mask,
1183	topology_core_cpumask(cpu));	1169	topology_core_cpumask(cpu));
1184	}	1170	}
1185	}	1171	}
1186		1172
1187	free_cpumask_var(core_mask);	1173	free_cpumask_var(core_mask);
1188	return count;	1174	return count;
1189	}	1175	}
1190		1176
1191	static int	1177	static int
1192	efx_init_rx_cpu_rmap(struct efx_nic efx, struct msix_entry xentries)	1178	efx_init_rx_cpu_rmap(struct efx_nic efx, struct msix_entry xentries)
1193	{	1179	{
1194	#ifdef CONFIG_RFS_ACCEL	1180	#ifdef CONFIG_RFS_ACCEL
1195	int i, rc;	1181	int i, rc;
1196		1182
1197	efx->net_dev->rx_cpu_rmap = alloc_irq_cpu_rmap(efx->n_rx_channels);	1183	efx->net_dev->rx_cpu_rmap = alloc_irq_cpu_rmap(efx->n_rx_channels);
1198	if (!efx->net_dev->rx_cpu_rmap)	1184	if (!efx->net_dev->rx_cpu_rmap)
1199	return -ENOMEM;	1185	return -ENOMEM;
1200	for (i = 0; i < efx->n_rx_channels; i++) {	1186	for (i = 0; i < efx->n_rx_channels; i++) {
1201	rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap,	1187	rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap,
1202	xentries[i].vector);	1188	xentries[i].vector);
1203	if (rc) {	1189	if (rc) {
1204	free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);	1190	free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
1205	efx->net_dev->rx_cpu_rmap = NULL;	1191	efx->net_dev->rx_cpu_rmap = NULL;
1206	return rc;	1192	return rc;
1207	}	1193	}
1208	}	1194	}
1209	#endif	1195	#endif
1210	return 0;	1196	return 0;
1211	}	1197	}
1212		1198
1213	/* Probe the number and type of interrupts we are able to obtain, and	1199	/* Probe the number and type of interrupts we are able to obtain, and
1214	* the resulting numbers of channels and RX queues.	1200	* the resulting numbers of channels and RX queues.
1215	*/	1201	*/
1216	static int efx_probe_interrupts(struct efx_nic *efx)	1202	static int efx_probe_interrupts(struct efx_nic *efx)
1217	{	1203	{
1218	int max_channels =	1204	int max_channels =
1219	min_t(int, efx->type->phys_addr_channels, EFX_MAX_CHANNELS);	1205	min_t(int, efx->type->phys_addr_channels, EFX_MAX_CHANNELS);
1220	int rc, i;	1206	int rc, i;
1221		1207
1222	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {	1208	if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
1223	struct msix_entry xentries[EFX_MAX_CHANNELS];	1209	struct msix_entry xentries[EFX_MAX_CHANNELS];
1224	int n_channels;	1210	int n_channels;
1225		1211
1226	n_channels = efx_wanted_channels();	1212	n_channels = efx_wanted_channels();
1227	if (separate_tx_channels)	1213	if (separate_tx_channels)
1228	n_channels *= 2;	1214	n_channels *= 2;
1229	n_channels = min(n_channels, max_channels);	1215	n_channels = min(n_channels, max_channels);
1230		1216
1231	for (i = 0; i < n_channels; i++)	1217	for (i = 0; i < n_channels; i++)
1232	xentries[i].entry = i;	1218	xentries[i].entry = i;
1233	rc = pci_enable_msix(efx->pci_dev, xentries, n_channels);	1219	rc = pci_enable_msix(efx->pci_dev, xentries, n_channels);
1234	if (rc > 0) {	1220	if (rc > 0) {
1235	netif_err(efx, drv, efx->net_dev,	1221	netif_err(efx, drv, efx->net_dev,
1236	"WARNING: Insufficient MSI-X vectors"	1222	"WARNING: Insufficient MSI-X vectors"
1237	" available (%d < %d).\n", rc, n_channels);	1223	" available (%d < %d).\n", rc, n_channels);
1238	netif_err(efx, drv, efx->net_dev,	1224	netif_err(efx, drv, efx->net_dev,
1239	"WARNING: Performance may be reduced.\n");	1225	"WARNING: Performance may be reduced.\n");
1240	EFX_BUG_ON_PARANOID(rc >= n_channels);	1226	EFX_BUG_ON_PARANOID(rc >= n_channels);
1241	n_channels = rc;	1227	n_channels = rc;
1242	rc = pci_enable_msix(efx->pci_dev, xentries,	1228	rc = pci_enable_msix(efx->pci_dev, xentries,
1243	n_channels);	1229	n_channels);
1244	}	1230	}
1245		1231
1246	if (rc == 0) {	1232	if (rc == 0) {
1247	efx->n_channels = n_channels;	1233	efx->n_channels = n_channels;
1248	if (separate_tx_channels) {	1234	if (separate_tx_channels) {
1249	efx->n_tx_channels =	1235	efx->n_tx_channels =
1250	max(efx->n_channels / 2, 1U);	1236	max(efx->n_channels / 2, 1U);
1251	efx->n_rx_channels =	1237	efx->n_rx_channels =
1252	max(efx->n_channels -	1238	max(efx->n_channels -
1253	efx->n_tx_channels, 1U);	1239	efx->n_tx_channels, 1U);
1254	} else {	1240	} else {
1255	efx->n_tx_channels = efx->n_channels;	1241	efx->n_tx_channels = efx->n_channels;
1256	efx->n_rx_channels = efx->n_channels;	1242	efx->n_rx_channels = efx->n_channels;
1257	}	1243	}
1258	rc = efx_init_rx_cpu_rmap(efx, xentries);	1244	rc = efx_init_rx_cpu_rmap(efx, xentries);
1259	if (rc) {	1245	if (rc) {
1260	pci_disable_msix(efx->pci_dev);	1246	pci_disable_msix(efx->pci_dev);
1261	return rc;	1247	return rc;
1262	}	1248	}
1263	for (i = 0; i < n_channels; i++)	1249	for (i = 0; i < n_channels; i++)
1264	efx_get_channel(efx, i)->irq =	1250	efx_get_channel(efx, i)->irq =
1265	xentries[i].vector;	1251	xentries[i].vector;
1266	} else {	1252	} else {
1267	/* Fall back to single channel MSI */	1253	/* Fall back to single channel MSI */
1268	efx->interrupt_mode = EFX_INT_MODE_MSI;	1254	efx->interrupt_mode = EFX_INT_MODE_MSI;
1269	netif_err(efx, drv, efx->net_dev,	1255	netif_err(efx, drv, efx->net_dev,
1270	"could not enable MSI-X\n");	1256	"could not enable MSI-X\n");
1271	}	1257	}
1272	}	1258	}
1273		1259
1274	/* Try single interrupt MSI */	1260	/* Try single interrupt MSI */
1275	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {	1261	if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
1276	efx->n_channels = 1;	1262	efx->n_channels = 1;
1277	efx->n_rx_channels = 1;	1263	efx->n_rx_channels = 1;
1278	efx->n_tx_channels = 1;	1264	efx->n_tx_channels = 1;
1279	rc = pci_enable_msi(efx->pci_dev);	1265	rc = pci_enable_msi(efx->pci_dev);
1280	if (rc == 0) {	1266	if (rc == 0) {
1281	efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;	1267	efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
1282	} else {	1268	} else {
1283	netif_err(efx, drv, efx->net_dev,	1269	netif_err(efx, drv, efx->net_dev,
1284	"could not enable MSI\n");	1270	"could not enable MSI\n");
1285	efx->interrupt_mode = EFX_INT_MODE_LEGACY;	1271	efx->interrupt_mode = EFX_INT_MODE_LEGACY;
1286	}	1272	}
1287	}	1273	}
1288		1274
1289	/* Assume legacy interrupts */	1275	/* Assume legacy interrupts */
1290	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {	1276	if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
1291	efx->n_channels = 1 + (separate_tx_channels ? 1 : 0);	1277	efx->n_channels = 1 + (separate_tx_channels ? 1 : 0);
1292	efx->n_rx_channels = 1;	1278	efx->n_rx_channels = 1;
1293	efx->n_tx_channels = 1;	1279	efx->n_tx_channels = 1;
1294	efx->legacy_irq = efx->pci_dev->irq;	1280	efx->legacy_irq = efx->pci_dev->irq;
1295	}	1281	}
1296		1282
1297	return 0;	1283	return 0;
1298	}	1284	}
1299		1285
1300	static void efx_remove_interrupts(struct efx_nic *efx)	1286	static void efx_remove_interrupts(struct efx_nic *efx)
1301	{	1287	{
1302	struct efx_channel *channel;	1288	struct efx_channel *channel;
1303		1289
1304	/* Remove MSI/MSI-X interrupts */	1290	/* Remove MSI/MSI-X interrupts */
1305	efx_for_each_channel(channel, efx)	1291	efx_for_each_channel(channel, efx)
1306	channel->irq = 0;	1292	channel->irq = 0;
1307	pci_disable_msi(efx->pci_dev);	1293	pci_disable_msi(efx->pci_dev);
1308	pci_disable_msix(efx->pci_dev);	1294	pci_disable_msix(efx->pci_dev);
1309		1295
1310	/* Remove legacy interrupt */	1296	/* Remove legacy interrupt */
1311	efx->legacy_irq = 0;	1297	efx->legacy_irq = 0;
1312	}	1298	}
1313		1299
1314	static void efx_set_channels(struct efx_nic *efx)	1300	static void efx_set_channels(struct efx_nic *efx)
1315	{	1301	{
1316	struct efx_channel *channel;	1302	struct efx_channel *channel;
1317	struct efx_tx_queue *tx_queue;	1303	struct efx_tx_queue *tx_queue;
1318		1304
1319	efx->tx_channel_offset =	1305	efx->tx_channel_offset =
1320	separate_tx_channels ? efx->n_channels - efx->n_tx_channels : 0;	1306	separate_tx_channels ? efx->n_channels - efx->n_tx_channels : 0;
1321		1307
1322	/* We need to adjust the TX queue numbers if we have separate	1308	/* We need to adjust the TX queue numbers if we have separate
1323	* RX-only and TX-only channels.	1309	* RX-only and TX-only channels.
1324	*/	1310	*/
1325	efx_for_each_channel(channel, efx) {	1311	efx_for_each_channel(channel, efx) {
1326	efx_for_each_channel_tx_queue(tx_queue, channel)	1312	efx_for_each_channel_tx_queue(tx_queue, channel)
1327	tx_queue->queue -= (efx->tx_channel_offset *	1313	tx_queue->queue -= (efx->tx_channel_offset *
1328	EFX_TXQ_TYPES);	1314	EFX_TXQ_TYPES);
1329	}	1315	}
1330	}	1316	}
1331		1317
1332	static int efx_probe_nic(struct efx_nic *efx)	1318	static int efx_probe_nic(struct efx_nic *efx)
1333	{	1319	{
1334	size_t i;	1320	size_t i;
1335	int rc;	1321	int rc;
1336		1322
1337	netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");	1323	netif_dbg(efx, probe, efx->net_dev, "creating NIC\n");
1338		1324
1339	/* Carry out hardware-type specific initialisation */	1325	/* Carry out hardware-type specific initialisation */
1340	rc = efx->type->probe(efx);	1326	rc = efx->type->probe(efx);
1341	if (rc)	1327	if (rc)
1342	return rc;	1328	return rc;
1343		1329
1344	/* Determine the number of channels and queues by trying to hook	1330	/* Determine the number of channels and queues by trying to hook
1345	* in MSI-X interrupts. */	1331	* in MSI-X interrupts. */
1346	rc = efx_probe_interrupts(efx);	1332	rc = efx_probe_interrupts(efx);
1347	if (rc)	1333	if (rc)
1348	goto fail;	1334	goto fail;
1349		1335
1350	if (efx->n_channels > 1)	1336	if (efx->n_channels > 1)
1351	get_random_bytes(&efx->rx_hash_key, sizeof(efx->rx_hash_key));	1337	get_random_bytes(&efx->rx_hash_key, sizeof(efx->rx_hash_key));
1352	for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++)	1338	for (i = 0; i < ARRAY_SIZE(efx->rx_indir_table); i++)
1353	efx->rx_indir_table[i] = i % efx->n_rx_channels;	1339	efx->rx_indir_table[i] = i % efx->n_rx_channels;
1354		1340
1355	efx_set_channels(efx);	1341	efx_set_channels(efx);
1356	netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);	1342	netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
1357	netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);	1343	netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
1358		1344
1359	/* Initialise the interrupt moderation settings */	1345	/* Initialise the interrupt moderation settings */
1360	efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true);	1346	efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true);
1361		1347
1362	return 0;	1348	return 0;
1363		1349
1364	fail:	1350	fail:
1365	efx->type->remove(efx);	1351	efx->type->remove(efx);
1366	return rc;	1352	return rc;
1367	}	1353	}
1368		1354
1369	static void efx_remove_nic(struct efx_nic *efx)	1355	static void efx_remove_nic(struct efx_nic *efx)
1370	{	1356	{
1371	netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");	1357	netif_dbg(efx, drv, efx->net_dev, "destroying NIC\n");
1372		1358
1373	efx_remove_interrupts(efx);	1359	efx_remove_interrupts(efx);
1374	efx->type->remove(efx);	1360	efx->type->remove(efx);
1375	}	1361	}
1376		1362
1377	/**************************************************************************	1363	/**************************************************************************
1378	*	1364	*
1379	* NIC startup/shutdown	1365	* NIC startup/shutdown
1380	*	1366	*
1381	*************************************************************************/	1367	*************************************************************************/
1382		1368
1383	static int efx_probe_all(struct efx_nic *efx)	1369	static int efx_probe_all(struct efx_nic *efx)
1384	{	1370	{
1385	int rc;	1371	int rc;
1386		1372
1387	rc = efx_probe_nic(efx);	1373	rc = efx_probe_nic(efx);
1388	if (rc) {	1374	if (rc) {
1389	netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");	1375	netif_err(efx, probe, efx->net_dev, "failed to create NIC\n");
1390	goto fail1;	1376	goto fail1;
1391	}	1377	}
1392		1378
1393	rc = efx_probe_port(efx);	1379	rc = efx_probe_port(efx);
1394	if (rc) {	1380	if (rc) {
1395	netif_err(efx, probe, efx->net_dev, "failed to create port\n");	1381	netif_err(efx, probe, efx->net_dev, "failed to create port\n");
1396	goto fail2;	1382	goto fail2;
1397	}	1383	}
1398		1384
1399	efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;	1385	efx->rxq_entries = efx->txq_entries = EFX_DEFAULT_DMAQ_SIZE;
1400	rc = efx_probe_channels(efx);	1386	rc = efx_probe_channels(efx);
1401	if (rc)	1387	if (rc)
1402	goto fail3;	1388	goto fail3;
1403		1389
1404	rc = efx_probe_filters(efx);	1390	rc = efx_probe_filters(efx);
1405	if (rc) {	1391	if (rc) {
1406	netif_err(efx, probe, efx->net_dev,	1392	netif_err(efx, probe, efx->net_dev,
1407	"failed to create filter tables\n");	1393	"failed to create filter tables\n");
1408	goto fail4;	1394	goto fail4;
1409	}	1395	}
1410		1396
1411	return 0;	1397	return 0;
1412		1398
1413	fail4:	1399	fail4:
1414	efx_remove_channels(efx);	1400	efx_remove_channels(efx);
1415	fail3:	1401	fail3:
1416	efx_remove_port(efx);	1402	efx_remove_port(efx);
1417	fail2:	1403	fail2:
1418	efx_remove_nic(efx);	1404	efx_remove_nic(efx);
1419	fail1:	1405	fail1:
1420	return rc;	1406	return rc;
1421	}	1407	}
1422		1408
1423	/* Called after previous invocation(s) of efx_stop_all, restarts the	1409	/* Called after previous invocation(s) of efx_stop_all, restarts the
1424	* port, kernel transmit queue, NAPI processing and hardware interrupts,	1410	* port, kernel transmit queue, NAPI processing and hardware interrupts,
1425	* and ensures that the port is scheduled to be reconfigured.	1411	* and ensures that the port is scheduled to be reconfigured.
1426	* This function is safe to call multiple times when the NIC is in any	1412	* This function is safe to call multiple times when the NIC is in any
1427	* state. */	1413	* state. */
1428	static void efx_start_all(struct efx_nic *efx)	1414	static void efx_start_all(struct efx_nic *efx)
1429	{	1415	{
1430	struct efx_channel *channel;	1416	struct efx_channel *channel;
1431		1417
1432	EFX_ASSERT_RESET_SERIALISED(efx);	1418	EFX_ASSERT_RESET_SERIALISED(efx);
1433		1419
1434	/* Check that it is appropriate to restart the interface. All	1420	/* Check that it is appropriate to restart the interface. All
1435	* of these flags are safe to read under just the rtnl lock */	1421	* of these flags are safe to read under just the rtnl lock */
1436	if (efx->port_enabled)	1422	if (efx->port_enabled)
1437	return;	1423	return;
1438	if ((efx->state != STATE_RUNNING) && (efx->state != STATE_INIT))	1424	if ((efx->state != STATE_RUNNING) && (efx->state != STATE_INIT))
1439	return;	1425	return;
1440	if (efx_dev_registered(efx) && !netif_running(efx->net_dev))	1426	if (efx_dev_registered(efx) && !netif_running(efx->net_dev))
1441	return;	1427	return;
1442		1428
1443	/* Mark the port as enabled so port reconfigurations can start, then	1429	/* Mark the port as enabled so port reconfigurations can start, then
1444	* restart the transmit interface early so the watchdog timer stops */	1430	* restart the transmit interface early so the watchdog timer stops */
1445	efx_start_port(efx);	1431	efx_start_port(efx);
1446		1432
1447	if (efx_dev_registered(efx) && netif_device_present(efx->net_dev))	1433	if (efx_dev_registered(efx) && netif_device_present(efx->net_dev))
1448	netif_tx_wake_all_queues(efx->net_dev);	1434	netif_tx_wake_all_queues(efx->net_dev);
1449		1435
1450	efx_for_each_channel(channel, efx)	1436	efx_for_each_channel(channel, efx)
1451	efx_start_channel(channel);	1437	efx_start_channel(channel);
1452		1438
1453	if (efx->legacy_irq)	1439	if (efx->legacy_irq)
1454	efx->legacy_irq_enabled = true;	1440	efx->legacy_irq_enabled = true;
1455	efx_nic_enable_interrupts(efx);	1441	efx_nic_enable_interrupts(efx);
1456		1442
1457	/* Switch to event based MCDI completions after enabling interrupts.	1443	/* Switch to event based MCDI completions after enabling interrupts.
1458	* If a reset has been scheduled, then we need to stay in polled mode.	1444	* If a reset has been scheduled, then we need to stay in polled mode.
1459	* Rather than serialising efx_mcdi_mode_event() [which sleeps] and	1445	* Rather than serialising efx_mcdi_mode_event() [which sleeps] and
1460	* reset_pending [modified from an atomic context], we instead guarantee	1446	* reset_pending [modified from an atomic context], we instead guarantee
1461	* that efx_mcdi_mode_poll() isn't reverted erroneously */	1447	* that efx_mcdi_mode_poll() isn't reverted erroneously */
1462	efx_mcdi_mode_event(efx);	1448	efx_mcdi_mode_event(efx);
1463	if (efx->reset_pending)	1449	if (efx->reset_pending)
1464	efx_mcdi_mode_poll(efx);	1450	efx_mcdi_mode_poll(efx);
1465		1451
1466	/* Start the hardware monitor if there is one. Otherwise (we're link	1452	/* Start the hardware monitor if there is one. Otherwise (we're link
1467	* event driven), we have to poll the PHY because after an event queue	1453	* event driven), we have to poll the PHY because after an event queue
1468	* flush, we could have a missed a link state change */	1454	* flush, we could have a missed a link state change */
1469	if (efx->type->monitor != NULL) {	1455	if (efx->type->monitor != NULL) {
1470	queue_delayed_work(efx->workqueue, &efx->monitor_work,	1456	queue_delayed_work(efx->workqueue, &efx->monitor_work,
1471	efx_monitor_interval);	1457	efx_monitor_interval);
1472	} else {	1458	} else {
1473	mutex_lock(&efx->mac_lock);	1459	mutex_lock(&efx->mac_lock);
1474	if (efx->phy_op->poll(efx))	1460	if (efx->phy_op->poll(efx))
1475	efx_link_status_changed(efx);	1461	efx_link_status_changed(efx);
1476	mutex_unlock(&efx->mac_lock);	1462	mutex_unlock(&efx->mac_lock);
1477	}	1463	}
1478		1464
1479	efx->type->start_stats(efx);	1465	efx->type->start_stats(efx);
1480	}	1466	}
1481		1467
1482	/* Flush all delayed work. Should only be called when no more delayed work	1468	/* Flush all delayed work. Should only be called when no more delayed work
1483	* will be scheduled. This doesn't flush pending online resets (efx_reset),	1469	* will be scheduled. This doesn't flush pending online resets (efx_reset),
1484	* since we're holding the rtnl_lock at this point. */	1470	* since we're holding the rtnl_lock at this point. */
1485	static void efx_flush_all(struct efx_nic *efx)	1471	static void efx_flush_all(struct efx_nic *efx)
1486	{	1472	{
1487	/* Make sure the hardware monitor is stopped */	1473	/* Make sure the hardware monitor is stopped */
1488	cancel_delayed_work_sync(&efx->monitor_work);	1474	cancel_delayed_work_sync(&efx->monitor_work);
1489	/* Stop scheduled port reconfigurations */	1475	/* Stop scheduled port reconfigurations */
1490	cancel_work_sync(&efx->mac_work);	1476	cancel_work_sync(&efx->mac_work);
1491	}	1477	}
1492		1478
1493	/* Quiesce hardware and software without bringing the link down.	1479	/* Quiesce hardware and software without bringing the link down.
1494	* Safe to call multiple times, when the nic and interface is in any	1480	* Safe to call multiple times, when the nic and interface is in any
1495	* state. The caller is guaranteed to subsequently be in a position	1481	* state. The caller is guaranteed to subsequently be in a position
1496	* to modify any hardware and software state they see fit without	1482	* to modify any hardware and software state they see fit without
1497	* taking locks. */	1483	* taking locks. */
1498	static void efx_stop_all(struct efx_nic *efx)	1484	static void efx_stop_all(struct efx_nic *efx)
1499	{	1485	{
1500	struct efx_channel *channel;	1486	struct efx_channel *channel;
1501		1487
1502	EFX_ASSERT_RESET_SERIALISED(efx);	1488	EFX_ASSERT_RESET_SERIALISED(efx);
1503		1489
1504	/* port_enabled can be read safely under the rtnl lock */	1490	/* port_enabled can be read safely under the rtnl lock */
1505	if (!efx->port_enabled)	1491	if (!efx->port_enabled)
1506	return;	1492	return;
1507		1493
1508	efx->type->stop_stats(efx);	1494	efx->type->stop_stats(efx);
1509		1495
1510	/* Switch to MCDI polling on Siena before disabling interrupts */	1496	/* Switch to MCDI polling on Siena before disabling interrupts */
1511	efx_mcdi_mode_poll(efx);	1497	efx_mcdi_mode_poll(efx);
1512		1498
1513	/* Disable interrupts and wait for ISR to complete */	1499	/* Disable interrupts and wait for ISR to complete */
1514	efx_nic_disable_interrupts(efx);	1500	efx_nic_disable_interrupts(efx);
1515	if (efx->legacy_irq) {	1501	if (efx->legacy_irq) {
1516	synchronize_irq(efx->legacy_irq);	1502	synchronize_irq(efx->legacy_irq);
1517	efx->legacy_irq_enabled = false;	1503	efx->legacy_irq_enabled = false;
1518	}	1504	}
1519	efx_for_each_channel(channel, efx) {	1505	efx_for_each_channel(channel, efx) {
1520	if (channel->irq)	1506	if (channel->irq)
1521	synchronize_irq(channel->irq);	1507	synchronize_irq(channel->irq);
1522	}	1508	}
1523		1509
1524	/* Stop all NAPI processing and synchronous rx refills */	1510	/* Stop all NAPI processing and synchronous rx refills */
1525	efx_for_each_channel(channel, efx)	1511	efx_for_each_channel(channel, efx)
1526	efx_stop_channel(channel);	1512	efx_stop_channel(channel);
1527		1513
1528	/* Stop all asynchronous port reconfigurations. Since all	1514	/* Stop all asynchronous port reconfigurations. Since all
1529	* event processing has already been stopped, there is no	1515	* event processing has already been stopped, there is no
1530	* window to loose phy events */	1516	* window to loose phy events */
1531	efx_stop_port(efx);	1517	efx_stop_port(efx);
1532		1518
1533	/* Flush efx_mac_work(), refill_workqueue, monitor_work */	1519	/* Flush efx_mac_work(), refill_workqueue, monitor_work */
1534	efx_flush_all(efx);	1520	efx_flush_all(efx);
1535		1521
1536	/* Stop the kernel transmit interface late, so the watchdog	1522	/* Stop the kernel transmit interface late, so the watchdog
1537	* timer isn't ticking over the flush */	1523	* timer isn't ticking over the flush */
1538	if (efx_dev_registered(efx)) {	1524	if (efx_dev_registered(efx)) {
1539	netif_tx_stop_all_queues(efx->net_dev);	1525	netif_tx_stop_all_queues(efx->net_dev);
1540	netif_tx_lock_bh(efx->net_dev);	1526	netif_tx_lock_bh(efx->net_dev);
1541	netif_tx_unlock_bh(efx->net_dev);	1527	netif_tx_unlock_bh(efx->net_dev);
1542	}	1528	}
1543	}	1529	}
1544		1530
1545	static void efx_remove_all(struct efx_nic *efx)	1531	static void efx_remove_all(struct efx_nic *efx)
1546	{	1532	{
1547	efx_remove_filters(efx);	1533	efx_remove_filters(efx);
1548	efx_remove_channels(efx);	1534	efx_remove_channels(efx);
1549	efx_remove_port(efx);	1535	efx_remove_port(efx);
1550	efx_remove_nic(efx);	1536	efx_remove_nic(efx);
1551	}	1537	}
1552		1538
1553	/**************************************************************************	1539	/**************************************************************************
1554	*	1540	*
1555	* Interrupt moderation	1541	* Interrupt moderation
1556	*	1542	*
1557	**************************************************************************/	1543	**************************************************************************/
1558		1544
1559	static unsigned irq_mod_ticks(int usecs, int resolution)	1545	static unsigned irq_mod_ticks(int usecs, int resolution)
1560	{	1546	{
1561	if (usecs <= 0)	1547	if (usecs <= 0)
1562	return 0; /* cannot receive interrupts ahead of time :-) */	1548	return 0; /* cannot receive interrupts ahead of time :-) */
1563	if (usecs < resolution)	1549	if (usecs < resolution)
1564	return 1; /* never round down to 0 */	1550	return 1; /* never round down to 0 */
1565	return usecs / resolution;	1551	return usecs / resolution;
1566	}	1552	}
1567		1553
1568	/* Set interrupt moderation parameters */	1554	/* Set interrupt moderation parameters */
1569	void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs,	1555	void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs,
1570	bool rx_adaptive)	1556	bool rx_adaptive)
1571	{	1557	{
1572	struct efx_channel *channel;	1558	struct efx_channel *channel;
1573	unsigned tx_ticks = irq_mod_ticks(tx_usecs, EFX_IRQ_MOD_RESOLUTION);	1559	unsigned tx_ticks = irq_mod_ticks(tx_usecs, EFX_IRQ_MOD_RESOLUTION);
1574	unsigned rx_ticks = irq_mod_ticks(rx_usecs, EFX_IRQ_MOD_RESOLUTION);	1560	unsigned rx_ticks = irq_mod_ticks(rx_usecs, EFX_IRQ_MOD_RESOLUTION);
1575		1561
1576	EFX_ASSERT_RESET_SERIALISED(efx);	1562	EFX_ASSERT_RESET_SERIALISED(efx);
1577		1563
1578	efx->irq_rx_adaptive = rx_adaptive;	1564	efx->irq_rx_adaptive = rx_adaptive;
1579	efx->irq_rx_moderation = rx_ticks;	1565	efx->irq_rx_moderation = rx_ticks;
1580	efx_for_each_channel(channel, efx) {	1566	efx_for_each_channel(channel, efx) {
1581	if (efx_channel_has_rx_queue(channel))	1567	if (efx_channel_has_rx_queue(channel))
1582	channel->irq_moderation = rx_ticks;	1568	channel->irq_moderation = rx_ticks;
1583	else if (efx_channel_has_tx_queues(channel))	1569	else if (efx_channel_has_tx_queues(channel))
1584	channel->irq_moderation = tx_ticks;	1570	channel->irq_moderation = tx_ticks;
1585	}	1571	}
1586	}	1572	}
1587		1573
1588	/**************************************************************************	1574	/**************************************************************************
1589	*	1575	*
1590	* Hardware monitor	1576	* Hardware monitor
1591	*	1577	*
1592	**************************************************************************/	1578	**************************************************************************/
1593		1579
1594	/* Run periodically off the general workqueue */	1580	/* Run periodically off the general workqueue */
1595	static void efx_monitor(struct work_struct *data)	1581	static void efx_monitor(struct work_struct *data)
1596	{	1582	{
1597	struct efx_nic *efx = container_of(data, struct efx_nic,	1583	struct efx_nic *efx = container_of(data, struct efx_nic,
1598	monitor_work.work);	1584	monitor_work.work);
1599		1585
1600	netif_vdbg(efx, timer, efx->net_dev,	1586	netif_vdbg(efx, timer, efx->net_dev,
1601	"hardware monitor executing on CPU %d\n",	1587	"hardware monitor executing on CPU %d\n",
1602	raw_smp_processor_id());	1588	raw_smp_processor_id());
1603	BUG_ON(efx->type->monitor == NULL);	1589	BUG_ON(efx->type->monitor == NULL);
1604		1590
1605	/* If the mac_lock is already held then it is likely a port	1591	/* If the mac_lock is already held then it is likely a port
1606	* reconfiguration is already in place, which will likely do	1592	* reconfiguration is already in place, which will likely do
1607	* most of the work of monitor() anyway. */	1593	* most of the work of monitor() anyway. */
1608	if (mutex_trylock(&efx->mac_lock)) {	1594	if (mutex_trylock(&efx->mac_lock)) {
1609	if (efx->port_enabled)	1595	if (efx->port_enabled)
1610	efx->type->monitor(efx);	1596	efx->type->monitor(efx);
1611	mutex_unlock(&efx->mac_lock);	1597	mutex_unlock(&efx->mac_lock);
1612	}	1598	}
1613		1599
1614	queue_delayed_work(efx->workqueue, &efx->monitor_work,	1600	queue_delayed_work(efx->workqueue, &efx->monitor_work,
1615	efx_monitor_interval);	1601	efx_monitor_interval);
1616	}	1602	}
1617		1603
1618	/**************************************************************************	1604	/**************************************************************************
1619	*	1605	*
1620	* ioctls	1606	* ioctls
1621	*	1607	*
1622	*************************************************************************/	1608	*************************************************************************/
1623		1609
1624	/* Net device ioctl	1610	/* Net device ioctl
1625	* Context: process, rtnl_lock() held.	1611	* Context: process, rtnl_lock() held.
1626	*/	1612	*/
1627	static int efx_ioctl(struct net_device net_dev, struct ifreq ifr, int cmd)	1613	static int efx_ioctl(struct net_device net_dev, struct ifreq ifr, int cmd)
1628	{	1614	{
1629	struct efx_nic *efx = netdev_priv(net_dev);	1615	struct efx_nic *efx = netdev_priv(net_dev);
1630	struct mii_ioctl_data *data = if_mii(ifr);	1616	struct mii_ioctl_data *data = if_mii(ifr);
1631		1617
1632	EFX_ASSERT_RESET_SERIALISED(efx);	1618	EFX_ASSERT_RESET_SERIALISED(efx);
1633		1619
1634	/* Convert phy_id from older PRTAD/DEVAD format */	1620	/* Convert phy_id from older PRTAD/DEVAD format */
1635	if ((cmd == SIOCGMIIREG \|\| cmd == SIOCSMIIREG) &&	1621	if ((cmd == SIOCGMIIREG \|\| cmd == SIOCSMIIREG) &&
1636	(data->phy_id & 0xfc00) == 0x0400)	1622	(data->phy_id & 0xfc00) == 0x0400)
1637	data->phy_id ^= MDIO_PHY_ID_C45 \| 0x0400;	1623	data->phy_id ^= MDIO_PHY_ID_C45 \| 0x0400;
1638		1624
1639	return mdio_mii_ioctl(&efx->mdio, data, cmd);	1625	return mdio_mii_ioctl(&efx->mdio, data, cmd);
1640	}	1626	}
1641		1627
1642	/**************************************************************************	1628	/**************************************************************************
1643	*	1629	*
1644	* NAPI interface	1630	* NAPI interface
1645	*	1631	*
1646	**************************************************************************/	1632	**************************************************************************/
1647		1633
1648	static void efx_init_napi(struct efx_nic *efx)	1634	static void efx_init_napi(struct efx_nic *efx)
1649	{	1635	{
1650	struct efx_channel *channel;	1636	struct efx_channel *channel;
1651		1637
1652	efx_for_each_channel(channel, efx) {	1638	efx_for_each_channel(channel, efx) {
1653	channel->napi_dev = efx->net_dev;	1639	channel->napi_dev = efx->net_dev;
1654	netif_napi_add(channel->napi_dev, &channel->napi_str,	1640	netif_napi_add(channel->napi_dev, &channel->napi_str,
1655	efx_poll, napi_weight);	1641	efx_poll, napi_weight);
1656	}	1642	}
1657	}	1643	}
1658		1644
1659	static void efx_fini_napi_channel(struct efx_channel *channel)	1645	static void efx_fini_napi_channel(struct efx_channel *channel)
1660	{	1646	{
1661	if (channel->napi_dev)	1647	if (channel->napi_dev)
1662	netif_napi_del(&channel->napi_str);	1648	netif_napi_del(&channel->napi_str);
1663	channel->napi_dev = NULL;	1649	channel->napi_dev = NULL;
1664	}	1650	}
1665		1651
1666	static void efx_fini_napi(struct efx_nic *efx)	1652	static void efx_fini_napi(struct efx_nic *efx)
1667	{	1653	{
1668	struct efx_channel *channel;	1654	struct efx_channel *channel;
1669		1655
1670	efx_for_each_channel(channel, efx)	1656	efx_for_each_channel(channel, efx)
1671	efx_fini_napi_channel(channel);	1657	efx_fini_napi_channel(channel);
1672	}	1658	}
1673		1659
1674	/**************************************************************************	1660	/**************************************************************************
1675	*	1661	*
1676	* Kernel netpoll interface	1662	* Kernel netpoll interface
1677	*	1663	*
1678	*************************************************************************/	1664	*************************************************************************/
1679		1665
1680	#ifdef CONFIG_NET_POLL_CONTROLLER	1666	#ifdef CONFIG_NET_POLL_CONTROLLER
1681		1667
1682	/* Although in the common case interrupts will be disabled, this is not	1668	/* Although in the common case interrupts will be disabled, this is not
1683	* guaranteed. However, all our work happens inside the NAPI callback,	1669	* guaranteed. However, all our work happens inside the NAPI callback,
1684	* so no locking is required.	1670	* so no locking is required.
1685	*/	1671	*/
1686	static void efx_netpoll(struct net_device *net_dev)	1672	static void efx_netpoll(struct net_device *net_dev)
1687	{	1673	{
1688	struct efx_nic *efx = netdev_priv(net_dev);	1674	struct efx_nic *efx = netdev_priv(net_dev);
1689	struct efx_channel *channel;	1675	struct efx_channel *channel;
1690		1676
1691	efx_for_each_channel(channel, efx)	1677	efx_for_each_channel(channel, efx)
1692	efx_schedule_channel(channel);	1678	efx_schedule_channel(channel);
1693	}	1679	}
1694		1680
1695	#endif	1681	#endif
1696		1682
1697	/**************************************************************************	1683	/**************************************************************************
1698	*	1684	*
1699	* Kernel net device interface	1685	* Kernel net device interface
1700	*	1686	*
1701	*************************************************************************/	1687	*************************************************************************/
1702		1688
1703	/* Context: process, rtnl_lock() held. */	1689	/* Context: process, rtnl_lock() held. */
1704	static int efx_net_open(struct net_device *net_dev)	1690	static int efx_net_open(struct net_device *net_dev)
1705	{	1691	{
1706	struct efx_nic *efx = netdev_priv(net_dev);	1692	struct efx_nic *efx = netdev_priv(net_dev);
1707	EFX_ASSERT_RESET_SERIALISED(efx);	1693	EFX_ASSERT_RESET_SERIALISED(efx);
1708		1694
1709	netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",	1695	netif_dbg(efx, ifup, efx->net_dev, "opening device on CPU %d\n",
1710	raw_smp_processor_id());	1696	raw_smp_processor_id());
1711		1697
1712	if (efx->state == STATE_DISABLED)	1698	if (efx->state == STATE_DISABLED)
1713	return -EIO;	1699	return -EIO;
1714	if (efx->phy_mode & PHY_MODE_SPECIAL)	1700	if (efx->phy_mode & PHY_MODE_SPECIAL)
1715	return -EBUSY;	1701	return -EBUSY;
1716	if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))	1702	if (efx_mcdi_poll_reboot(efx) && efx_reset(efx, RESET_TYPE_ALL))
1717	return -EIO;	1703	return -EIO;
1718		1704
1719	/* Notify the kernel of the link state polled during driver load,	1705	/* Notify the kernel of the link state polled during driver load,
1720	* before the monitor starts running */	1706	* before the monitor starts running */
1721	efx_link_status_changed(efx);	1707	efx_link_status_changed(efx);
1722		1708
1723	efx_start_all(efx);	1709	efx_start_all(efx);
1724	return 0;	1710	return 0;
1725	}	1711	}
1726		1712
1727	/* Context: process, rtnl_lock() held.	1713	/* Context: process, rtnl_lock() held.
1728	* Note that the kernel will ignore our return code; this method	1714	* Note that the kernel will ignore our return code; this method
1729	* should really be a void.	1715	* should really be a void.
1730	*/	1716	*/
1731	static int efx_net_stop(struct net_device *net_dev)	1717	static int efx_net_stop(struct net_device *net_dev)
1732	{	1718	{
1733	struct efx_nic *efx = netdev_priv(net_dev);	1719	struct efx_nic *efx = netdev_priv(net_dev);
1734		1720
1735	netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",	1721	netif_dbg(efx, ifdown, efx->net_dev, "closing on CPU %d\n",
1736	raw_smp_processor_id());	1722	raw_smp_processor_id());
1737		1723
1738	if (efx->state != STATE_DISABLED) {	1724	if (efx->state != STATE_DISABLED) {
1739	/* Stop the device and flush all the channels */	1725	/* Stop the device and flush all the channels */
1740	efx_stop_all(efx);	1726	efx_stop_all(efx);
1741	efx_fini_channels(efx);	1727	efx_fini_channels(efx);
1742	efx_init_channels(efx);	1728	efx_init_channels(efx);
1743	}	1729	}
1744		1730
1745	return 0;	1731	return 0;
1746	}	1732	}
1747		1733
1748	/* Context: process, dev_base_lock or RTNL held, non-blocking. */	1734	/* Context: process, dev_base_lock or RTNL held, non-blocking. */
1749	static struct rtnl_link_stats64 efx_net_stats(struct net_device net_dev, struct rtnl_link_stats64 *stats)	1735	static struct rtnl_link_stats64 efx_net_stats(struct net_device net_dev, struct rtnl_link_stats64 *stats)
1750	{	1736	{
1751	struct efx_nic *efx = netdev_priv(net_dev);	1737	struct efx_nic *efx = netdev_priv(net_dev);
1752	struct efx_mac_stats *mac_stats = &efx->mac_stats;	1738	struct efx_mac_stats *mac_stats = &efx->mac_stats;
1753		1739
1754	spin_lock_bh(&efx->stats_lock);	1740	spin_lock_bh(&efx->stats_lock);
1755	efx->type->update_stats(efx);	1741	efx->type->update_stats(efx);
1756	spin_unlock_bh(&efx->stats_lock);	1742	spin_unlock_bh(&efx->stats_lock);
1757		1743
1758	stats->rx_packets = mac_stats->rx_packets;	1744	stats->rx_packets = mac_stats->rx_packets;
1759	stats->tx_packets = mac_stats->tx_packets;	1745	stats->tx_packets = mac_stats->tx_packets;
1760	stats->rx_bytes = mac_stats->rx_bytes;	1746	stats->rx_bytes = mac_stats->rx_bytes;
1761	stats->tx_bytes = mac_stats->tx_bytes;	1747	stats->tx_bytes = mac_stats->tx_bytes;
1762	stats->rx_dropped = efx->n_rx_nodesc_drop_cnt;	1748	stats->rx_dropped = efx->n_rx_nodesc_drop_cnt;
1763	stats->multicast = mac_stats->rx_multicast;	1749	stats->multicast = mac_stats->rx_multicast;
1764	stats->collisions = mac_stats->tx_collision;	1750	stats->collisions = mac_stats->tx_collision;
1765	stats->rx_length_errors = (mac_stats->rx_gtjumbo +	1751	stats->rx_length_errors = (mac_stats->rx_gtjumbo +
1766	mac_stats->rx_length_error);	1752	mac_stats->rx_length_error);
1767	stats->rx_crc_errors = mac_stats->rx_bad;	1753	stats->rx_crc_errors = mac_stats->rx_bad;
1768	stats->rx_frame_errors = mac_stats->rx_align_error;	1754	stats->rx_frame_errors = mac_stats->rx_align_error;
1769	stats->rx_fifo_errors = mac_stats->rx_overflow;	1755	stats->rx_fifo_errors = mac_stats->rx_overflow;
1770	stats->rx_missed_errors = mac_stats->rx_missed;	1756	stats->rx_missed_errors = mac_stats->rx_missed;
1771	stats->tx_window_errors = mac_stats->tx_late_collision;	1757	stats->tx_window_errors = mac_stats->tx_late_collision;
1772		1758
1773	stats->rx_errors = (stats->rx_length_errors +	1759	stats->rx_errors = (stats->rx_length_errors +
1774	stats->rx_crc_errors +	1760	stats->rx_crc_errors +
1775	stats->rx_frame_errors +	1761	stats->rx_frame_errors +
1776	mac_stats->rx_symbol_error);	1762	mac_stats->rx_symbol_error);
1777	stats->tx_errors = (stats->tx_window_errors +	1763	stats->tx_errors = (stats->tx_window_errors +
1778	mac_stats->tx_bad);	1764	mac_stats->tx_bad);
1779		1765
1780	return stats;	1766	return stats;
1781	}	1767	}
1782		1768
1783	/* Context: netif_tx_lock held, BHs disabled. */	1769	/* Context: netif_tx_lock held, BHs disabled. */
1784	static void efx_watchdog(struct net_device *net_dev)	1770	static void efx_watchdog(struct net_device *net_dev)
1785	{	1771	{
1786	struct efx_nic *efx = netdev_priv(net_dev);	1772	struct efx_nic *efx = netdev_priv(net_dev);
1787		1773
1788	netif_err(efx, tx_err, efx->net_dev,	1774	netif_err(efx, tx_err, efx->net_dev,
1789	"TX stuck with port_enabled=%d: resetting channels\n",	1775	"TX stuck with port_enabled=%d: resetting channels\n",
1790	efx->port_enabled);	1776	efx->port_enabled);
1791		1777
1792	efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);	1778	efx_schedule_reset(efx, RESET_TYPE_TX_WATCHDOG);
1793	}	1779	}
1794		1780
1795		1781
1796	/* Context: process, rtnl_lock() held. */	1782	/* Context: process, rtnl_lock() held. */
1797	static int efx_change_mtu(struct net_device *net_dev, int new_mtu)	1783	static int efx_change_mtu(struct net_device *net_dev, int new_mtu)
1798	{	1784	{
1799	struct efx_nic *efx = netdev_priv(net_dev);	1785	struct efx_nic *efx = netdev_priv(net_dev);
1800	int rc = 0;	1786	int rc = 0;
1801		1787
1802	EFX_ASSERT_RESET_SERIALISED(efx);	1788	EFX_ASSERT_RESET_SERIALISED(efx);
1803		1789
1804	if (new_mtu > EFX_MAX_MTU)	1790	if (new_mtu > EFX_MAX_MTU)
1805	return -EINVAL;	1791	return -EINVAL;
1806		1792
1807	efx_stop_all(efx);	1793	efx_stop_all(efx);
1808		1794
1809	netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);	1795	netif_dbg(efx, drv, efx->net_dev, "changing MTU to %d\n", new_mtu);
1810		1796
1811	efx_fini_channels(efx);	1797	efx_fini_channels(efx);
1812		1798
1813	mutex_lock(&efx->mac_lock);	1799	mutex_lock(&efx->mac_lock);
1814	/* Reconfigure the MAC before enabling the dma queues so that	1800	/* Reconfigure the MAC before enabling the dma queues so that
1815	* the RX buffers don't overflow */	1801	* the RX buffers don't overflow */
1816	net_dev->mtu = new_mtu;	1802	net_dev->mtu = new_mtu;
1817	efx->mac_op->reconfigure(efx);	1803	efx->mac_op->reconfigure(efx);
1818	mutex_unlock(&efx->mac_lock);	1804	mutex_unlock(&efx->mac_lock);
1819		1805
1820	efx_init_channels(efx);	1806	efx_init_channels(efx);
1821		1807
1822	efx_start_all(efx);	1808	efx_start_all(efx);
1823	return rc;	1809	return rc;
1824	}	1810	}
1825		1811
1826	static int efx_set_mac_address(struct net_device net_dev, void data)	1812	static int efx_set_mac_address(struct net_device net_dev, void data)
1827	{	1813	{
1828	struct efx_nic *efx = netdev_priv(net_dev);	1814	struct efx_nic *efx = netdev_priv(net_dev);
1829	struct sockaddr *addr = data;	1815	struct sockaddr *addr = data;
1830	char *new_addr = addr->sa_data;	1816	char *new_addr = addr->sa_data;
1831		1817
1832	EFX_ASSERT_RESET_SERIALISED(efx);	1818	EFX_ASSERT_RESET_SERIALISED(efx);
1833		1819
1834	if (!is_valid_ether_addr(new_addr)) {	1820	if (!is_valid_ether_addr(new_addr)) {
1835	netif_err(efx, drv, efx->net_dev,	1821	netif_err(efx, drv, efx->net_dev,
1836	"invalid ethernet MAC address requested: %pM\n",	1822	"invalid ethernet MAC address requested: %pM\n",
1837	new_addr);	1823	new_addr);
1838	return -EINVAL;	1824	return -EINVAL;
1839	}	1825	}
1840		1826
1841	memcpy(net_dev->dev_addr, new_addr, net_dev->addr_len);	1827	memcpy(net_dev->dev_addr, new_addr, net_dev->addr_len);
1842		1828
1843	/* Reconfigure the MAC */	1829	/* Reconfigure the MAC */
1844	mutex_lock(&efx->mac_lock);	1830	mutex_lock(&efx->mac_lock);
1845	efx->mac_op->reconfigure(efx);	1831	efx->mac_op->reconfigure(efx);
1846	mutex_unlock(&efx->mac_lock);	1832	mutex_unlock(&efx->mac_lock);
1847		1833
1848	return 0;	1834	return 0;
1849	}	1835	}
1850		1836
1851	/* Context: netif_addr_lock held, BHs disabled. */	1837	/* Context: netif_addr_lock held, BHs disabled. */
1852	static void efx_set_multicast_list(struct net_device *net_dev)	1838	static void efx_set_multicast_list(struct net_device *net_dev)
1853	{	1839	{
1854	struct efx_nic *efx = netdev_priv(net_dev);	1840	struct efx_nic *efx = netdev_priv(net_dev);
1855	struct netdev_hw_addr *ha;	1841	struct netdev_hw_addr *ha;
1856	union efx_multicast_hash *mc_hash = &efx->multicast_hash;	1842	union efx_multicast_hash *mc_hash = &efx->multicast_hash;
1857	u32 crc;	1843	u32 crc;
1858	int bit;	1844	int bit;
1859		1845
1860	efx->promiscuous = !!(net_dev->flags & IFF_PROMISC);	1846	efx->promiscuous = !!(net_dev->flags & IFF_PROMISC);
1861		1847
1862	/* Build multicast hash table */	1848	/* Build multicast hash table */
1863	if (efx->promiscuous \|\| (net_dev->flags & IFF_ALLMULTI)) {	1849	if (efx->promiscuous \|\| (net_dev->flags & IFF_ALLMULTI)) {
1864	memset(mc_hash, 0xff, sizeof(*mc_hash));	1850	memset(mc_hash, 0xff, sizeof(*mc_hash));
1865	} else {	1851	} else {
1866	memset(mc_hash, 0x00, sizeof(*mc_hash));	1852	memset(mc_hash, 0x00, sizeof(*mc_hash));
1867	netdev_for_each_mc_addr(ha, net_dev) {	1853	netdev_for_each_mc_addr(ha, net_dev) {
1868	crc = ether_crc_le(ETH_ALEN, ha->addr);	1854	crc = ether_crc_le(ETH_ALEN, ha->addr);
1869	bit = crc & (EFX_MCAST_HASH_ENTRIES - 1);	1855	bit = crc & (EFX_MCAST_HASH_ENTRIES - 1);
1870	set_bit_le(bit, mc_hash->byte);	1856	set_bit_le(bit, mc_hash->byte);
1871	}	1857	}
1872		1858
1873	/* Broadcast packets go through the multicast hash filter.	1859	/* Broadcast packets go through the multicast hash filter.
1874	* ether_crc_le() of the broadcast address is 0xbe2612ff	1860	* ether_crc_le() of the broadcast address is 0xbe2612ff
1875	* so we always add bit 0xff to the mask.	1861	* so we always add bit 0xff to the mask.
1876	*/	1862	*/
1877	set_bit_le(0xff, mc_hash->byte);	1863	set_bit_le(0xff, mc_hash->byte);
1878	}	1864	}
1879		1865
1880	if (efx->port_enabled)	1866	if (efx->port_enabled)
1881	queue_work(efx->workqueue, &efx->mac_work);	1867	queue_work(efx->workqueue, &efx->mac_work);
1882	/* Otherwise efx_start_port() will do this */	1868	/* Otherwise efx_start_port() will do this */
1883	}	1869	}
1884		1870
1885	static int efx_set_features(struct net_device *net_dev, u32 data)	1871	static int efx_set_features(struct net_device *net_dev, u32 data)
1886	{	1872	{
1887	struct efx_nic *efx = netdev_priv(net_dev);	1873	struct efx_nic *efx = netdev_priv(net_dev);
1888		1874
1889	/* If disabling RX n-tuple filtering, clear existing filters */	1875	/* If disabling RX n-tuple filtering, clear existing filters */
1890	if (net_dev->features & ~data & NETIF_F_NTUPLE)	1876	if (net_dev->features & ~data & NETIF_F_NTUPLE)
1891	efx_filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);	1877	efx_filter_clear_rx(efx, EFX_FILTER_PRI_MANUAL);
1892		1878
1893	return 0;	1879	return 0;
1894	}	1880	}
1895		1881
1896	static const struct net_device_ops efx_netdev_ops = {	1882	static const struct net_device_ops efx_netdev_ops = {
1897	.ndo_open = efx_net_open,	1883	.ndo_open = efx_net_open,
1898	.ndo_stop = efx_net_stop,	1884	.ndo_stop = efx_net_stop,
1899	.ndo_get_stats64 = efx_net_stats,	1885	.ndo_get_stats64 = efx_net_stats,
1900	.ndo_tx_timeout = efx_watchdog,	1886	.ndo_tx_timeout = efx_watchdog,
1901	.ndo_start_xmit = efx_hard_start_xmit,	1887	.ndo_start_xmit = efx_hard_start_xmit,
1902	.ndo_validate_addr = eth_validate_addr,	1888	.ndo_validate_addr = eth_validate_addr,
1903	.ndo_do_ioctl = efx_ioctl,	1889	.ndo_do_ioctl = efx_ioctl,
1904	.ndo_change_mtu = efx_change_mtu,	1890	.ndo_change_mtu = efx_change_mtu,
1905	.ndo_set_mac_address = efx_set_mac_address,	1891	.ndo_set_mac_address = efx_set_mac_address,
1906	.ndo_set_multicast_list = efx_set_multicast_list,	1892	.ndo_set_multicast_list = efx_set_multicast_list,
1907	.ndo_set_features = efx_set_features,	1893	.ndo_set_features = efx_set_features,
1908	#ifdef CONFIG_NET_POLL_CONTROLLER	1894	#ifdef CONFIG_NET_POLL_CONTROLLER
1909	.ndo_poll_controller = efx_netpoll,	1895	.ndo_poll_controller = efx_netpoll,
1910	#endif	1896	#endif
1911	.ndo_setup_tc = efx_setup_tc,	1897	.ndo_setup_tc = efx_setup_tc,
1912	#ifdef CONFIG_RFS_ACCEL	1898	#ifdef CONFIG_RFS_ACCEL
1913	.ndo_rx_flow_steer = efx_filter_rfs,	1899	.ndo_rx_flow_steer = efx_filter_rfs,
1914	#endif	1900	#endif
1915	};	1901	};
1916		1902
1917	static void efx_update_name(struct efx_nic *efx)	1903	static void efx_update_name(struct efx_nic *efx)
1918	{	1904	{
1919	strcpy(efx->name, efx->net_dev->name);	1905	strcpy(efx->name, efx->net_dev->name);
1920	efx_mtd_rename(efx);	1906	efx_mtd_rename(efx);
1921	efx_set_channel_names(efx);	1907	efx_set_channel_names(efx);
1922	}	1908	}
1923		1909
1924	static int efx_netdev_event(struct notifier_block *this,	1910	static int efx_netdev_event(struct notifier_block *this,
1925	unsigned long event, void *ptr)	1911	unsigned long event, void *ptr)
1926	{	1912	{
1927	struct net_device *net_dev = ptr;	1913	struct net_device *net_dev = ptr;
1928		1914
1929	if (net_dev->netdev_ops == &efx_netdev_ops &&	1915	if (net_dev->netdev_ops == &efx_netdev_ops &&
1930	event == NETDEV_CHANGENAME)	1916	event == NETDEV_CHANGENAME)
1931	efx_update_name(netdev_priv(net_dev));	1917	efx_update_name(netdev_priv(net_dev));
1932		1918
1933	return NOTIFY_DONE;	1919	return NOTIFY_DONE;
1934	}	1920	}
1935		1921
1936	static struct notifier_block efx_netdev_notifier = {	1922	static struct notifier_block efx_netdev_notifier = {
1937	.notifier_call = efx_netdev_event,	1923	.notifier_call = efx_netdev_event,
1938	};	1924	};
1939		1925
1940	static ssize_t	1926	static ssize_t
1941	show_phy_type(struct device dev, struct device_attribute attr, char *buf)	1927	show_phy_type(struct device dev, struct device_attribute attr, char *buf)
1942	{	1928	{
1943	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));	1929	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
1944	return sprintf(buf, "%d\n", efx->phy_type);	1930	return sprintf(buf, "%d\n", efx->phy_type);
1945	}	1931	}
1946	static DEVICE_ATTR(phy_type, 0644, show_phy_type, NULL);	1932	static DEVICE_ATTR(phy_type, 0644, show_phy_type, NULL);
1947		1933
1948	static int efx_register_netdev(struct efx_nic *efx)	1934	static int efx_register_netdev(struct efx_nic *efx)
1949	{	1935	{
1950	struct net_device *net_dev = efx->net_dev;	1936	struct net_device *net_dev = efx->net_dev;
1951	struct efx_channel *channel;	1937	struct efx_channel *channel;
1952	int rc;	1938	int rc;
1953		1939
1954	net_dev->watchdog_timeo = 5 * HZ;	1940	net_dev->watchdog_timeo = 5 * HZ;
1955	net_dev->irq = efx->pci_dev->irq;	1941	net_dev->irq = efx->pci_dev->irq;
1956	net_dev->netdev_ops = &efx_netdev_ops;	1942	net_dev->netdev_ops = &efx_netdev_ops;
1957	SET_ETHTOOL_OPS(net_dev, &efx_ethtool_ops);	1943	SET_ETHTOOL_OPS(net_dev, &efx_ethtool_ops);
1958		1944
1959	/* Clear MAC statistics */	1945	/* Clear MAC statistics */
1960	efx->mac_op->update_stats(efx);	1946	efx->mac_op->update_stats(efx);
1961	memset(&efx->mac_stats, 0, sizeof(efx->mac_stats));	1947	memset(&efx->mac_stats, 0, sizeof(efx->mac_stats));
1962		1948
1963	rtnl_lock();	1949	rtnl_lock();
1964		1950
1965	rc = dev_alloc_name(net_dev, net_dev->name);	1951	rc = dev_alloc_name(net_dev, net_dev->name);
1966	if (rc < 0)	1952	if (rc < 0)
1967	goto fail_locked;	1953	goto fail_locked;
1968	efx_update_name(efx);	1954	efx_update_name(efx);
1969		1955
1970	rc = register_netdevice(net_dev);	1956	rc = register_netdevice(net_dev);
1971	if (rc)	1957	if (rc)
1972	goto fail_locked;	1958	goto fail_locked;
1973		1959
1974	efx_for_each_channel(channel, efx) {	1960	efx_for_each_channel(channel, efx) {
1975	struct efx_tx_queue *tx_queue;	1961	struct efx_tx_queue *tx_queue;
1976	efx_for_each_channel_tx_queue(tx_queue, channel)	1962	efx_for_each_channel_tx_queue(tx_queue, channel)
1977	efx_init_tx_queue_core_txq(tx_queue);	1963	efx_init_tx_queue_core_txq(tx_queue);
1978	}	1964	}
1979		1965
1980	/* Always start with carrier off; PHY events will detect the link */	1966	/* Always start with carrier off; PHY events will detect the link */
1981	netif_carrier_off(efx->net_dev);	1967	netif_carrier_off(efx->net_dev);
1982		1968
1983	rtnl_unlock();	1969	rtnl_unlock();
1984		1970
1985	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);	1971	rc = device_create_file(&efx->pci_dev->dev, &dev_attr_phy_type);
1986	if (rc) {	1972	if (rc) {
1987	netif_err(efx, drv, efx->net_dev,	1973	netif_err(efx, drv, efx->net_dev,
1988	"failed to init net dev attributes\n");	1974	"failed to init net dev attributes\n");
1989	goto fail_registered;	1975	goto fail_registered;
1990	}	1976	}
1991		1977
1992	return 0;	1978	return 0;
1993		1979
1994	fail_locked:	1980	fail_locked:
1995	rtnl_unlock();	1981	rtnl_unlock();
1996	netif_err(efx, drv, efx->net_dev, "could not register net dev\n");	1982	netif_err(efx, drv, efx->net_dev, "could not register net dev\n");
1997	return rc;	1983	return rc;
1998		1984
1999	fail_registered:	1985	fail_registered:
2000	unregister_netdev(net_dev);	1986	unregister_netdev(net_dev);
2001	return rc;	1987	return rc;
2002	}	1988	}
2003		1989
2004	static void efx_unregister_netdev(struct efx_nic *efx)	1990	static void efx_unregister_netdev(struct efx_nic *efx)
2005	{	1991	{
2006	struct efx_channel *channel;	1992	struct efx_channel *channel;
2007	struct efx_tx_queue *tx_queue;	1993	struct efx_tx_queue *tx_queue;
2008		1994
2009	if (!efx->net_dev)	1995	if (!efx->net_dev)
2010	return;	1996	return;
2011		1997
2012	BUG_ON(netdev_priv(efx->net_dev) != efx);	1998	BUG_ON(netdev_priv(efx->net_dev) != efx);
2013		1999
2014	/* Free up any skbs still remaining. This has to happen before	2000	/* Free up any skbs still remaining. This has to happen before
2015	* we try to unregister the netdev as running their destructors	2001	* we try to unregister the netdev as running their destructors
2016	* may be needed to get the device ref. count to 0. */	2002	* may be needed to get the device ref. count to 0. */
2017	efx_for_each_channel(channel, efx) {	2003	efx_for_each_channel(channel, efx) {
2018	efx_for_each_channel_tx_queue(tx_queue, channel)	2004	efx_for_each_channel_tx_queue(tx_queue, channel)
2019	efx_release_tx_buffers(tx_queue);	2005	efx_release_tx_buffers(tx_queue);
2020	}	2006	}
2021		2007
2022	if (efx_dev_registered(efx)) {	2008	if (efx_dev_registered(efx)) {
2023	strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));	2009	strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
2024	device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);	2010	device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
2025	unregister_netdev(efx->net_dev);	2011	unregister_netdev(efx->net_dev);
2026	}	2012	}
2027	}	2013	}
2028		2014
2029	/**************************************************************************	2015	/**************************************************************************
2030	*	2016	*
2031	* Device reset and suspend	2017	* Device reset and suspend
2032	*	2018	*
2033	**************************************************************************/	2019	**************************************************************************/
2034		2020
2035	/* Tears down the entire software state and most of the hardware state	2021	/* Tears down the entire software state and most of the hardware state
2036	* before reset. */	2022	* before reset. */
2037	void efx_reset_down(struct efx_nic *efx, enum reset_type method)	2023	void efx_reset_down(struct efx_nic *efx, enum reset_type method)
2038	{	2024	{
2039	EFX_ASSERT_RESET_SERIALISED(efx);	2025	EFX_ASSERT_RESET_SERIALISED(efx);
2040		2026
2041	efx_stop_all(efx);	2027	efx_stop_all(efx);
2042	mutex_lock(&efx->mac_lock);	2028	mutex_lock(&efx->mac_lock);
2043		2029
2044	efx_fini_channels(efx);	2030	efx_fini_channels(efx);
2045	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE)	2031	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE)
2046	efx->phy_op->fini(efx);	2032	efx->phy_op->fini(efx);
2047	efx->type->fini(efx);	2033	efx->type->fini(efx);
2048	}	2034	}
2049		2035
2050	/* This function will always ensure that the locks acquired in	2036	/* This function will always ensure that the locks acquired in
2051	* efx_reset_down() are released. A failure return code indicates	2037	* efx_reset_down() are released. A failure return code indicates
2052	* that we were unable to reinitialise the hardware, and the	2038	* that we were unable to reinitialise the hardware, and the
2053	* driver should be disabled. If ok is false, then the rx and tx	2039	* driver should be disabled. If ok is false, then the rx and tx
2054	* engines are not restarted, pending a RESET_DISABLE. */	2040	* engines are not restarted, pending a RESET_DISABLE. */
2055	int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)	2041	int efx_reset_up(struct efx_nic *efx, enum reset_type method, bool ok)
2056	{	2042	{
2057	int rc;	2043	int rc;
2058		2044
2059	EFX_ASSERT_RESET_SERIALISED(efx);	2045	EFX_ASSERT_RESET_SERIALISED(efx);
2060		2046
2061	rc = efx->type->init(efx);	2047	rc = efx->type->init(efx);
2062	if (rc) {	2048	if (rc) {
2063	netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n");	2049	netif_err(efx, drv, efx->net_dev, "failed to initialise NIC\n");
2064	goto fail;	2050	goto fail;
2065	}	2051	}
2066		2052
2067	if (!ok)	2053	if (!ok)
2068	goto fail;	2054	goto fail;
2069		2055
2070	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) {	2056	if (efx->port_initialized && method != RESET_TYPE_INVISIBLE) {
2071	rc = efx->phy_op->init(efx);	2057	rc = efx->phy_op->init(efx);
2072	if (rc)	2058	if (rc)
2073	goto fail;	2059	goto fail;
2074	if (efx->phy_op->reconfigure(efx))	2060	if (efx->phy_op->reconfigure(efx))
2075	netif_err(efx, drv, efx->net_dev,	2061	netif_err(efx, drv, efx->net_dev,
2076	"could not restore PHY settings\n");	2062	"could not restore PHY settings\n");
2077	}	2063	}
2078		2064
2079	efx->mac_op->reconfigure(efx);	2065	efx->mac_op->reconfigure(efx);
2080		2066
2081	efx_init_channels(efx);	2067	efx_init_channels(efx);
2082	efx_restore_filters(efx);	2068	efx_restore_filters(efx);
2083		2069
2084	mutex_unlock(&efx->mac_lock);	2070	mutex_unlock(&efx->mac_lock);
2085		2071
2086	efx_start_all(efx);	2072	efx_start_all(efx);
2087		2073
2088	return 0;	2074	return 0;
2089		2075
2090	fail:	2076	fail:
2091	efx->port_initialized = false;	2077	efx->port_initialized = false;
2092		2078
2093	mutex_unlock(&efx->mac_lock);	2079	mutex_unlock(&efx->mac_lock);
2094		2080
2095	return rc;	2081	return rc;
2096	}	2082	}
2097		2083
2098	/* Reset the NIC using the specified method. Note that the reset may	2084	/* Reset the NIC using the specified method. Note that the reset may
2099	* fail, in which case the card will be left in an unusable state.	2085	* fail, in which case the card will be left in an unusable state.
2100	*	2086	*
2101	* Caller must hold the rtnl_lock.	2087	* Caller must hold the rtnl_lock.
2102	*/	2088	*/
2103	int efx_reset(struct efx_nic *efx, enum reset_type method)	2089	int efx_reset(struct efx_nic *efx, enum reset_type method)
2104	{	2090	{
2105	int rc, rc2;	2091	int rc, rc2;
2106	bool disabled;	2092	bool disabled;
2107		2093
2108	netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",	2094	netif_info(efx, drv, efx->net_dev, "resetting (%s)\n",
2109	RESET_TYPE(method));	2095	RESET_TYPE(method));
2110		2096
2111	netif_device_detach(efx->net_dev);	2097	netif_device_detach(efx->net_dev);
2112	efx_reset_down(efx, method);	2098	efx_reset_down(efx, method);
2113		2099
2114	rc = efx->type->reset(efx, method);	2100	rc = efx->type->reset(efx, method);
2115	if (rc) {	2101	if (rc) {
2116	netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n");	2102	netif_err(efx, drv, efx->net_dev, "failed to reset hardware\n");
2117	goto out;	2103	goto out;
2118	}	2104	}
2119		2105
2120	/* Clear flags for the scopes we covered. We assume the NIC and	2106	/* Clear flags for the scopes we covered. We assume the NIC and
2121	* driver are now quiescent so that there is no race here.	2107	* driver are now quiescent so that there is no race here.
2122	*/	2108	*/
2123	efx->reset_pending &= -(1 << (method + 1));	2109	efx->reset_pending &= -(1 << (method + 1));
2124		2110
2125	/* Reinitialise bus-mastering, which may have been turned off before	2111	/* Reinitialise bus-mastering, which may have been turned off before
2126	* the reset was scheduled. This is still appropriate, even in the	2112	* the reset was scheduled. This is still appropriate, even in the
2127	* RESET_TYPE_DISABLE since this driver generally assumes the hardware	2113	* RESET_TYPE_DISABLE since this driver generally assumes the hardware
2128	* can respond to requests. */	2114	* can respond to requests. */
2129	pci_set_master(efx->pci_dev);	2115	pci_set_master(efx->pci_dev);
2130		2116
2131	out:	2117	out:
2132	/* Leave device stopped if necessary */	2118	/* Leave device stopped if necessary */
2133	disabled = rc \|\| method == RESET_TYPE_DISABLE;	2119	disabled = rc \|\| method == RESET_TYPE_DISABLE;
2134	rc2 = efx_reset_up(efx, method, !disabled);	2120	rc2 = efx_reset_up(efx, method, !disabled);
2135	if (rc2) {	2121	if (rc2) {
2136	disabled = true;	2122	disabled = true;
2137	if (!rc)	2123	if (!rc)
2138	rc = rc2;	2124	rc = rc2;
2139	}	2125	}
2140		2126
2141	if (disabled) {	2127	if (disabled) {
2142	dev_close(efx->net_dev);	2128	dev_close(efx->net_dev);
2143	netif_err(efx, drv, efx->net_dev, "has been disabled\n");	2129	netif_err(efx, drv, efx->net_dev, "has been disabled\n");
2144	efx->state = STATE_DISABLED;	2130	efx->state = STATE_DISABLED;
2145	} else {	2131	} else {
2146	netif_dbg(efx, drv, efx->net_dev, "reset complete\n");	2132	netif_dbg(efx, drv, efx->net_dev, "reset complete\n");
2147	netif_device_attach(efx->net_dev);	2133	netif_device_attach(efx->net_dev);
2148	}	2134	}
2149	return rc;	2135	return rc;
2150	}	2136	}
2151		2137
2152	/* The worker thread exists so that code that cannot sleep can	2138	/* The worker thread exists so that code that cannot sleep can
2153	* schedule a reset for later.	2139	* schedule a reset for later.
2154	*/	2140	*/
2155	static void efx_reset_work(struct work_struct *data)	2141	static void efx_reset_work(struct work_struct *data)
2156	{	2142	{
2157	struct efx_nic *efx = container_of(data, struct efx_nic, reset_work);	2143	struct efx_nic *efx = container_of(data, struct efx_nic, reset_work);
2158	unsigned long pending = ACCESS_ONCE(efx->reset_pending);	2144	unsigned long pending = ACCESS_ONCE(efx->reset_pending);
2159		2145
2160	if (!pending)	2146	if (!pending)
2161	return;	2147	return;
2162		2148
2163	/* If we're not RUNNING then don't reset. Leave the reset_pending	2149	/* If we're not RUNNING then don't reset. Leave the reset_pending
2164	* flags set so that efx_pci_probe_main will be retried */	2150	* flags set so that efx_pci_probe_main will be retried */
2165	if (efx->state != STATE_RUNNING) {	2151	if (efx->state != STATE_RUNNING) {
2166	netif_info(efx, drv, efx->net_dev,	2152	netif_info(efx, drv, efx->net_dev,
2167	"scheduled reset quenched. NIC not RUNNING\n");	2153	"scheduled reset quenched. NIC not RUNNING\n");
2168	return;	2154	return;
2169	}	2155	}
2170		2156
2171	rtnl_lock();	2157	rtnl_lock();
2172	(void)efx_reset(efx, fls(pending) - 1);	2158	(void)efx_reset(efx, fls(pending) - 1);
2173	rtnl_unlock();	2159	rtnl_unlock();
2174	}	2160	}
2175		2161
2176	void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)	2162	void efx_schedule_reset(struct efx_nic *efx, enum reset_type type)
2177	{	2163	{
2178	enum reset_type method;	2164	enum reset_type method;
2179		2165
2180	switch (type) {	2166	switch (type) {
2181	case RESET_TYPE_INVISIBLE:	2167	case RESET_TYPE_INVISIBLE:
2182	case RESET_TYPE_ALL:	2168	case RESET_TYPE_ALL:
2183	case RESET_TYPE_WORLD:	2169	case RESET_TYPE_WORLD:
2184	case RESET_TYPE_DISABLE:	2170	case RESET_TYPE_DISABLE:
2185	method = type;	2171	method = type;
2186	netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n",	2172	netif_dbg(efx, drv, efx->net_dev, "scheduling %s reset\n",
2187	RESET_TYPE(method));	2173	RESET_TYPE(method));
2188	break;	2174	break;
2189	default:	2175	default:
2190	method = efx->type->map_reset_reason(type);	2176	method = efx->type->map_reset_reason(type);
2191	netif_dbg(efx, drv, efx->net_dev,	2177	netif_dbg(efx, drv, efx->net_dev,
2192	"scheduling %s reset for %s\n",	2178	"scheduling %s reset for %s\n",
2193	RESET_TYPE(method), RESET_TYPE(type));	2179	RESET_TYPE(method), RESET_TYPE(type));
2194	break;	2180	break;
2195	}	2181	}
2196		2182
2197	set_bit(method, &efx->reset_pending);	2183	set_bit(method, &efx->reset_pending);
2198		2184
2199	/* efx_process_channel() will no longer read events once a	2185	/* efx_process_channel() will no longer read events once a
2200	* reset is scheduled. So switch back to poll'd MCDI completions. */	2186	* reset is scheduled. So switch back to poll'd MCDI completions. */
2201	efx_mcdi_mode_poll(efx);	2187	efx_mcdi_mode_poll(efx);
2202		2188
2203	queue_work(reset_workqueue, &efx->reset_work);	2189	queue_work(reset_workqueue, &efx->reset_work);
2204	}	2190	}
2205		2191
2206	/**************************************************************************	2192	/**************************************************************************
2207	*	2193	*
2208	* List of NICs we support	2194	* List of NICs we support
2209	*	2195	*
2210	**************************************************************************/	2196	**************************************************************************/
2211		2197
2212	/* PCI device ID table */	2198	/* PCI device ID table */
2213	static DEFINE_PCI_DEVICE_TABLE(efx_pci_table) = {	2199	static DEFINE_PCI_DEVICE_TABLE(efx_pci_table) = {
2214	{PCI_DEVICE(EFX_VENDID_SFC, FALCON_A_P_DEVID),	2200	{PCI_DEVICE(EFX_VENDID_SFC, FALCON_A_P_DEVID),
2215	.driver_data = (unsigned long) &falcon_a1_nic_type},	2201	.driver_data = (unsigned long) &falcon_a1_nic_type},
2216	{PCI_DEVICE(EFX_VENDID_SFC, FALCON_B_P_DEVID),	2202	{PCI_DEVICE(EFX_VENDID_SFC, FALCON_B_P_DEVID),
2217	.driver_data = (unsigned long) &falcon_b0_nic_type},	2203	.driver_data = (unsigned long) &falcon_b0_nic_type},
2218	{PCI_DEVICE(EFX_VENDID_SFC, BETHPAGE_A_P_DEVID),	2204	{PCI_DEVICE(EFX_VENDID_SFC, BETHPAGE_A_P_DEVID),
2219	.driver_data = (unsigned long) &siena_a0_nic_type},	2205	.driver_data = (unsigned long) &siena_a0_nic_type},
2220	{PCI_DEVICE(EFX_VENDID_SFC, SIENA_A_P_DEVID),	2206	{PCI_DEVICE(EFX_VENDID_SFC, SIENA_A_P_DEVID),
2221	.driver_data = (unsigned long) &siena_a0_nic_type},	2207	.driver_data = (unsigned long) &siena_a0_nic_type},
2222	{0} /* end of list */	2208	{0} /* end of list */
2223	};	2209	};
2224		2210
2225	/**************************************************************************	2211	/**************************************************************************
2226	*	2212	*
2227	* Dummy PHY/MAC operations	2213	* Dummy PHY/MAC operations
2228	*	2214	*
2229	* Can be used for some unimplemented operations	2215	* Can be used for some unimplemented operations
2230	* Needed so all function pointers are valid and do not have to be tested	2216	* Needed so all function pointers are valid and do not have to be tested
2231	* before use	2217	* before use
2232	*	2218	*
2233	**************************************************************************/	2219	**************************************************************************/
2234	int efx_port_dummy_op_int(struct efx_nic *efx)	2220	int efx_port_dummy_op_int(struct efx_nic *efx)
2235	{	2221	{
2236	return 0;	2222	return 0;
2237	}	2223	}
2238	void efx_port_dummy_op_void(struct efx_nic *efx) {}	2224	void efx_port_dummy_op_void(struct efx_nic *efx) {}
2239		2225
2240	static bool efx_port_dummy_op_poll(struct efx_nic *efx)	2226	static bool efx_port_dummy_op_poll(struct efx_nic *efx)
2241	{	2227	{
2242	return false;	2228	return false;
2243	}	2229	}
2244		2230
2245	static const struct efx_phy_operations efx_dummy_phy_operations = {	2231	static const struct efx_phy_operations efx_dummy_phy_operations = {
2246	.init = efx_port_dummy_op_int,	2232	.init = efx_port_dummy_op_int,
2247	.reconfigure = efx_port_dummy_op_int,	2233	.reconfigure = efx_port_dummy_op_int,
2248	.poll = efx_port_dummy_op_poll,	2234	.poll = efx_port_dummy_op_poll,
2249	.fini = efx_port_dummy_op_void,	2235	.fini = efx_port_dummy_op_void,
2250	};	2236	};
2251		2237
2252	/**************************************************************************	2238	/**************************************************************************
2253	*	2239	*
2254	* Data housekeeping	2240	* Data housekeeping
2255	*	2241	*
2256	**************************************************************************/	2242	**************************************************************************/
2257		2243
2258	/* This zeroes out and then fills in the invariants in a struct	2244	/* This zeroes out and then fills in the invariants in a struct
2259	* efx_nic (including all sub-structures).	2245	* efx_nic (including all sub-structures).
2260	*/	2246	*/
2261	static int efx_init_struct(struct efx_nic efx, const struct efx_nic_type type,	2247	static int efx_init_struct(struct efx_nic efx, const struct efx_nic_type type,
2262	struct pci_dev pci_dev, struct net_device net_dev)	2248	struct pci_dev pci_dev, struct net_device net_dev)
2263	{	2249	{
2264	int i;	2250	int i;
2265		2251
2266	/* Initialise common structures */	2252	/* Initialise common structures */
2267	memset(efx, 0, sizeof(*efx));	2253	memset(efx, 0, sizeof(*efx));
2268	spin_lock_init(&efx->biu_lock);	2254	spin_lock_init(&efx->biu_lock);
2269	#ifdef CONFIG_SFC_MTD	2255	#ifdef CONFIG_SFC_MTD
2270	INIT_LIST_HEAD(&efx->mtd_list);	2256	INIT_LIST_HEAD(&efx->mtd_list);
2271	#endif	2257	#endif
2272	INIT_WORK(&efx->reset_work, efx_reset_work);	2258	INIT_WORK(&efx->reset_work, efx_reset_work);
2273	INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);	2259	INIT_DELAYED_WORK(&efx->monitor_work, efx_monitor);
2274	efx->pci_dev = pci_dev;	2260	efx->pci_dev = pci_dev;
2275	efx->msg_enable = debug;	2261	efx->msg_enable = debug;
2276	efx->state = STATE_INIT;	2262	efx->state = STATE_INIT;
2277	strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));	2263	strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
2278		2264
2279	efx->net_dev = net_dev;	2265	efx->net_dev = net_dev;
2280	spin_lock_init(&efx->stats_lock);	2266	spin_lock_init(&efx->stats_lock);
2281	mutex_init(&efx->mac_lock);	2267	mutex_init(&efx->mac_lock);
2282	efx->mac_op = type->default_mac_ops;	2268	efx->mac_op = type->default_mac_ops;
2283	efx->phy_op = &efx_dummy_phy_operations;	2269	efx->phy_op = &efx_dummy_phy_operations;
2284	efx->mdio.dev = net_dev;	2270	efx->mdio.dev = net_dev;
2285	INIT_WORK(&efx->mac_work, efx_mac_work);	2271	INIT_WORK(&efx->mac_work, efx_mac_work);
2286		2272
2287	for (i = 0; i < EFX_MAX_CHANNELS; i++) {	2273	for (i = 0; i < EFX_MAX_CHANNELS; i++) {
2288	efx->channel[i] = efx_alloc_channel(efx, i, NULL);	2274	efx->channel[i] = efx_alloc_channel(efx, i, NULL);
2289	if (!efx->channel[i])	2275	if (!efx->channel[i])
2290	goto fail;	2276	goto fail;
2291	}	2277	}
2292		2278
2293	efx->type = type;	2279	efx->type = type;
2294		2280
2295	EFX_BUG_ON_PARANOID(efx->type->phys_addr_channels > EFX_MAX_CHANNELS);	2281	EFX_BUG_ON_PARANOID(efx->type->phys_addr_channels > EFX_MAX_CHANNELS);
2296		2282
2297	/* Higher numbered interrupt modes are less capable! */	2283	/* Higher numbered interrupt modes are less capable! */
2298	efx->interrupt_mode = max(efx->type->max_interrupt_mode,	2284	efx->interrupt_mode = max(efx->type->max_interrupt_mode,
2299	interrupt_mode);	2285	interrupt_mode);
2300		2286
2301	/* Would be good to use the net_dev name, but we're too early */	2287	/* Would be good to use the net_dev name, but we're too early */
2302	snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",	2288	snprintf(efx->workqueue_name, sizeof(efx->workqueue_name), "sfc%s",
2303	pci_name(pci_dev));	2289	pci_name(pci_dev));
2304	efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);	2290	efx->workqueue = create_singlethread_workqueue(efx->workqueue_name);
2305	if (!efx->workqueue)	2291	if (!efx->workqueue)
2306	goto fail;	2292	goto fail;
2307		2293
2308	return 0;	2294	return 0;
2309		2295
2310	fail:	2296	fail:
2311	efx_fini_struct(efx);	2297	efx_fini_struct(efx);
2312	return -ENOMEM;	2298	return -ENOMEM;
2313	}	2299	}
2314		2300
2315	static void efx_fini_struct(struct efx_nic *efx)	2301	static void efx_fini_struct(struct efx_nic *efx)
2316	{	2302	{
2317	int i;	2303	int i;
2318		2304
2319	for (i = 0; i < EFX_MAX_CHANNELS; i++)	2305	for (i = 0; i < EFX_MAX_CHANNELS; i++)
2320	kfree(efx->channel[i]);	2306	kfree(efx->channel[i]);
2321		2307
2322	if (efx->workqueue) {	2308	if (efx->workqueue) {
2323	destroy_workqueue(efx->workqueue);	2309	destroy_workqueue(efx->workqueue);
2324	efx->workqueue = NULL;	2310	efx->workqueue = NULL;
2325	}	2311	}
2326	}	2312	}
2327		2313
2328	/**************************************************************************	2314	/**************************************************************************
2329	*	2315	*
2330	* PCI interface	2316	* PCI interface
2331	*	2317	*
2332	**************************************************************************/	2318	**************************************************************************/
2333		2319
2334	/* Main body of final NIC shutdown code	2320	/* Main body of final NIC shutdown code
2335	* This is called only at module unload (or hotplug removal).	2321	* This is called only at module unload (or hotplug removal).
2336	*/	2322	*/
2337	static void efx_pci_remove_main(struct efx_nic *efx)	2323	static void efx_pci_remove_main(struct efx_nic *efx)
2338	{	2324	{
2339	#ifdef CONFIG_RFS_ACCEL	2325	#ifdef CONFIG_RFS_ACCEL
2340	free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);	2326	free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap);
2341	efx->net_dev->rx_cpu_rmap = NULL;	2327	efx->net_dev->rx_cpu_rmap = NULL;
2342	#endif	2328	#endif
2343	efx_nic_fini_interrupt(efx);	2329	efx_nic_fini_interrupt(efx);
2344	efx_fini_channels(efx);	2330	efx_fini_channels(efx);
2345	efx_fini_port(efx);	2331	efx_fini_port(efx);
2346	efx->type->fini(efx);	2332	efx->type->fini(efx);
2347	efx_fini_napi(efx);	2333	efx_fini_napi(efx);
2348	efx_remove_all(efx);	2334	efx_remove_all(efx);
2349	}	2335	}
2350		2336
2351	/* Final NIC shutdown	2337	/* Final NIC shutdown
2352	* This is called only at module unload (or hotplug removal).	2338	* This is called only at module unload (or hotplug removal).
2353	*/	2339	*/
2354	static void efx_pci_remove(struct pci_dev *pci_dev)	2340	static void efx_pci_remove(struct pci_dev *pci_dev)
2355	{	2341	{
2356	struct efx_nic *efx;	2342	struct efx_nic *efx;
2357		2343
2358	efx = pci_get_drvdata(pci_dev);	2344	efx = pci_get_drvdata(pci_dev);
2359	if (!efx)	2345	if (!efx)
2360	return;	2346	return;
2361		2347
2362	/* Mark the NIC as fini, then stop the interface */	2348	/* Mark the NIC as fini, then stop the interface */
2363	rtnl_lock();	2349	rtnl_lock();
2364	efx->state = STATE_FINI;	2350	efx->state = STATE_FINI;
2365	dev_close(efx->net_dev);	2351	dev_close(efx->net_dev);
2366		2352
2367	/* Allow any queued efx_resets() to complete */	2353	/* Allow any queued efx_resets() to complete */
2368	rtnl_unlock();	2354	rtnl_unlock();
2369		2355
2370	efx_unregister_netdev(efx);	2356	efx_unregister_netdev(efx);
2371		2357
2372	efx_mtd_remove(efx);	2358	efx_mtd_remove(efx);
2373		2359
2374	/* Wait for any scheduled resets to complete. No more will be	2360	/* Wait for any scheduled resets to complete. No more will be
2375	* scheduled from this point because efx_stop_all() has been	2361	* scheduled from this point because efx_stop_all() has been
2376	* called, we are no longer registered with driverlink, and	2362	* called, we are no longer registered with driverlink, and
2377	* the net_device's have been removed. */	2363	* the net_device's have been removed. */
2378	cancel_work_sync(&efx->reset_work);	2364	cancel_work_sync(&efx->reset_work);
2379		2365
2380	efx_pci_remove_main(efx);	2366	efx_pci_remove_main(efx);
2381		2367
2382	efx_fini_io(efx);	2368	efx_fini_io(efx);
2383	netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");	2369	netif_dbg(efx, drv, efx->net_dev, "shutdown successful\n");
2384		2370
2385	pci_set_drvdata(pci_dev, NULL);	2371	pci_set_drvdata(pci_dev, NULL);
2386	efx_fini_struct(efx);	2372	efx_fini_struct(efx);
2387	free_netdev(efx->net_dev);	2373	free_netdev(efx->net_dev);
2388	};	2374	};
2389		2375
2390	/* Main body of NIC initialisation	2376	/* Main body of NIC initialisation
2391	* This is called at module load (or hotplug insertion, theoretically).	2377	* This is called at module load (or hotplug insertion, theoretically).
2392	*/	2378	*/
2393	static int efx_pci_probe_main(struct efx_nic *efx)	2379	static int efx_pci_probe_main(struct efx_nic *efx)
2394	{	2380	{
2395	int rc;	2381	int rc;
2396		2382
2397	/* Do start-of-day initialisation */	2383	/* Do start-of-day initialisation */
2398	rc = efx_probe_all(efx);	2384	rc = efx_probe_all(efx);
2399	if (rc)	2385	if (rc)
2400	goto fail1;	2386	goto fail1;
2401		2387
2402	efx_init_napi(efx);	2388	efx_init_napi(efx);
2403		2389
2404	rc = efx->type->init(efx);	2390	rc = efx->type->init(efx);
2405	if (rc) {	2391	if (rc) {
2406	netif_err(efx, probe, efx->net_dev,	2392	netif_err(efx, probe, efx->net_dev,
2407	"failed to initialise NIC\n");	2393	"failed to initialise NIC\n");
2408	goto fail3;	2394	goto fail3;
2409	}	2395	}
2410		2396
2411	rc = efx_init_port(efx);	2397	rc = efx_init_port(efx);
2412	if (rc) {	2398	if (rc) {
2413	netif_err(efx, probe, efx->net_dev,	2399	netif_err(efx, probe, efx->net_dev,
2414	"failed to initialise port\n");	2400	"failed to initialise port\n");
2415	goto fail4;	2401	goto fail4;
2416	}	2402	}
2417		2403
2418	efx_init_channels(efx);	2404	efx_init_channels(efx);
2419		2405
2420	rc = efx_nic_init_interrupt(efx);	2406	rc = efx_nic_init_interrupt(efx);
2421	if (rc)	2407	if (rc)
2422	goto fail5;	2408	goto fail5;
2423		2409
2424	return 0;	2410	return 0;
2425		2411
2426	fail5:	2412	fail5:
2427	efx_fini_channels(efx);	2413	efx_fini_channels(efx);
2428	efx_fini_port(efx);	2414	efx_fini_port(efx);
2429	fail4:	2415	fail4:
2430	efx->type->fini(efx);	2416	efx->type->fini(efx);
2431	fail3:	2417	fail3:
2432	efx_fini_napi(efx);	2418	efx_fini_napi(efx);
2433	efx_remove_all(efx);	2419	efx_remove_all(efx);
2434	fail1:	2420	fail1:
2435	return rc;	2421	return rc;
2436	}	2422	}
2437		2423
2438	/* NIC initialisation	2424	/* NIC initialisation
2439	*	2425	*
2440	* This is called at module load (or hotplug insertion,	2426	* This is called at module load (or hotplug insertion,
2441	* theoretically). It sets up PCI mappings, tests and resets the NIC,	2427	* theoretically). It sets up PCI mappings, tests and resets the NIC,
2442	* sets up and registers the network devices with the kernel and hooks	2428	* sets up and registers the network devices with the kernel and hooks
2443	* the interrupt service routine. It does not prepare the device for	2429	* the interrupt service routine. It does not prepare the device for
2444	* transmission; this is left to the first time one of the network	2430	* transmission; this is left to the first time one of the network
2445	* interfaces is brought up (i.e. efx_net_open).	2431	* interfaces is brought up (i.e. efx_net_open).
2446	*/	2432	*/
2447	static int __devinit efx_pci_probe(struct pci_dev *pci_dev,	2433	static int __devinit efx_pci_probe(struct pci_dev *pci_dev,
2448	const struct pci_device_id *entry)	2434	const struct pci_device_id *entry)
2449	{	2435	{
2450	const struct efx_nic_type type = (const struct efx_nic_type ) entry->driver_data;	2436	const struct efx_nic_type type = (const struct efx_nic_type ) entry->driver_data;
2451	struct net_device *net_dev;	2437	struct net_device *net_dev;
2452	struct efx_nic *efx;	2438	struct efx_nic *efx;
2453	int i, rc;	2439	int i, rc;
2454		2440
2455	/* Allocate and initialise a struct net_device and struct efx_nic */	2441	/* Allocate and initialise a struct net_device and struct efx_nic */
2456	net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,	2442	net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES,
2457	EFX_MAX_RX_QUEUES);	2443	EFX_MAX_RX_QUEUES);
2458	if (!net_dev)	2444	if (!net_dev)
2459	return -ENOMEM;	2445	return -ENOMEM;
2460	net_dev->features \|= (type->offload_features \| NETIF_F_SG \|	2446	net_dev->features \|= (type->offload_features \| NETIF_F_SG \|
2461	NETIF_F_HIGHDMA \| NETIF_F_TSO \|	2447	NETIF_F_HIGHDMA \| NETIF_F_TSO \|
2462	NETIF_F_RXCSUM);	2448	NETIF_F_RXCSUM);
2463	if (type->offload_features & NETIF_F_V6_CSUM)	2449	if (type->offload_features & NETIF_F_V6_CSUM)
2464	net_dev->features \|= NETIF_F_TSO6;	2450	net_dev->features \|= NETIF_F_TSO6;
2465	/* Mask for features that also apply to VLAN devices */	2451	/* Mask for features that also apply to VLAN devices */
2466	net_dev->vlan_features \|= (NETIF_F_ALL_CSUM \| NETIF_F_SG \|	2452	net_dev->vlan_features \|= (NETIF_F_ALL_CSUM \| NETIF_F_SG \|
2467	NETIF_F_HIGHDMA \| NETIF_F_ALL_TSO \|	2453	NETIF_F_HIGHDMA \| NETIF_F_ALL_TSO \|
2468	NETIF_F_RXCSUM);	2454	NETIF_F_RXCSUM);
2469	/* All offloads can be toggled */	2455	/* All offloads can be toggled */
2470	net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA;	2456	net_dev->hw_features = net_dev->features & ~NETIF_F_HIGHDMA;
2471	efx = netdev_priv(net_dev);	2457	efx = netdev_priv(net_dev);
2472	pci_set_drvdata(pci_dev, efx);	2458	pci_set_drvdata(pci_dev, efx);
2473	SET_NETDEV_DEV(net_dev, &pci_dev->dev);	2459	SET_NETDEV_DEV(net_dev, &pci_dev->dev);
2474	rc = efx_init_struct(efx, type, pci_dev, net_dev);	2460	rc = efx_init_struct(efx, type, pci_dev, net_dev);
2475	if (rc)	2461	if (rc)
2476	goto fail1;	2462	goto fail1;
2477		2463
2478	netif_info(efx, probe, efx->net_dev,	2464	netif_info(efx, probe, efx->net_dev,
2479	"Solarflare NIC detected\n");	2465	"Solarflare NIC detected\n");
2480		2466
2481	/* Set up basic I/O (BAR mappings etc) */	2467	/* Set up basic I/O (BAR mappings etc) */
2482	rc = efx_init_io(efx);	2468	rc = efx_init_io(efx);
2483	if (rc)	2469	if (rc)
2484	goto fail2;	2470	goto fail2;
2485		2471
2486	/* No serialisation is required with the reset path because	2472	/* No serialisation is required with the reset path because
2487	* we're in STATE_INIT. */	2473	* we're in STATE_INIT. */
2488	for (i = 0; i < 5; i++) {	2474	for (i = 0; i < 5; i++) {
2489	rc = efx_pci_probe_main(efx);	2475	rc = efx_pci_probe_main(efx);
2490		2476
2491	/* Serialise against efx_reset(). No more resets will be	2477	/* Serialise against efx_reset(). No more resets will be
2492	* scheduled since efx_stop_all() has been called, and we	2478	* scheduled since efx_stop_all() has been called, and we
2493	* have not and never have been registered with either	2479	* have not and never have been registered with either
2494	* the rtnetlink or driverlink layers. */	2480	* the rtnetlink or driverlink layers. */
2495	cancel_work_sync(&efx->reset_work);	2481	cancel_work_sync(&efx->reset_work);
2496		2482
2497	if (rc == 0) {	2483	if (rc == 0) {
2498	if (efx->reset_pending) {	2484	if (efx->reset_pending) {
2499	/* If there was a scheduled reset during	2485	/* If there was a scheduled reset during
2500	* probe, the NIC is probably hosed anyway */	2486	* probe, the NIC is probably hosed anyway */
2501	efx_pci_remove_main(efx);	2487	efx_pci_remove_main(efx);
2502	rc = -EIO;	2488	rc = -EIO;
2503	} else {	2489	} else {
2504	break;	2490	break;
2505	}	2491	}
2506	}	2492	}
2507		2493
2508	/* Retry if a recoverably reset event has been scheduled */	2494	/* Retry if a recoverably reset event has been scheduled */
2509	if (efx->reset_pending &	2495	if (efx->reset_pending &
2510	~(1 << RESET_TYPE_INVISIBLE \| 1 << RESET_TYPE_ALL) \|\|	2496	~(1 << RESET_TYPE_INVISIBLE \| 1 << RESET_TYPE_ALL) \|\|
2511	!efx->reset_pending)	2497	!efx->reset_pending)
2512	goto fail3;	2498	goto fail3;
2513		2499
2514	efx->reset_pending = 0;	2500	efx->reset_pending = 0;
2515	}	2501	}
2516		2502
2517	if (rc) {	2503	if (rc) {
2518	netif_err(efx, probe, efx->net_dev, "Could not reset NIC\n");	2504	netif_err(efx, probe, efx->net_dev, "Could not reset NIC\n");
2519	goto fail4;	2505	goto fail4;
2520	}	2506	}
2521		2507
2522	/* Switch to the running state before we expose the device to the OS,	2508	/* Switch to the running state before we expose the device to the OS,
2523	* so that dev_open()\|efx_start_all() will actually start the device */	2509	* so that dev_open()\|efx_start_all() will actually start the device */
2524	efx->state = STATE_RUNNING;	2510	efx->state = STATE_RUNNING;
2525		2511
2526	rc = efx_register_netdev(efx);	2512	rc = efx_register_netdev(efx);
2527	if (rc)	2513	if (rc)
2528	goto fail5;	2514	goto fail5;
2529		2515
2530	netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");	2516	netif_dbg(efx, probe, efx->net_dev, "initialisation successful\n");
2531		2517
2532	rtnl_lock();	2518	rtnl_lock();
2533	efx_mtd_probe(efx); /* allowed to fail */	2519	efx_mtd_probe(efx); /* allowed to fail */
2534	rtnl_unlock();	2520	rtnl_unlock();
2535	return 0;	2521	return 0;
2536		2522
2537	fail5:	2523	fail5:
2538	efx_pci_remove_main(efx);	2524	efx_pci_remove_main(efx);
2539	fail4:	2525	fail4:
2540	fail3:	2526	fail3:
2541	efx_fini_io(efx);	2527	efx_fini_io(efx);
2542	fail2:	2528	fail2:
2543	efx_fini_struct(efx);	2529	efx_fini_struct(efx);
2544	fail1:	2530	fail1:
2545	WARN_ON(rc > 0);	2531	WARN_ON(rc > 0);
2546	netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);	2532	netif_dbg(efx, drv, efx->net_dev, "initialisation failed. rc=%d\n", rc);
2547	free_netdev(net_dev);	2533	free_netdev(net_dev);
2548	return rc;	2534	return rc;
2549	}	2535	}
2550		2536
2551	static int efx_pm_freeze(struct device *dev)	2537	static int efx_pm_freeze(struct device *dev)
2552	{	2538	{
2553	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));	2539	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2554		2540
2555	efx->state = STATE_FINI;	2541	efx->state = STATE_FINI;
2556		2542
2557	netif_device_detach(efx->net_dev);	2543	netif_device_detach(efx->net_dev);
2558		2544
2559	efx_stop_all(efx);	2545	efx_stop_all(efx);
2560	efx_fini_channels(efx);	2546	efx_fini_channels(efx);
2561		2547
2562	return 0;	2548	return 0;
2563	}	2549	}
2564		2550
2565	static int efx_pm_thaw(struct device *dev)	2551	static int efx_pm_thaw(struct device *dev)
2566	{	2552	{
2567	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));	2553	struct efx_nic *efx = pci_get_drvdata(to_pci_dev(dev));
2568		2554
2569	efx->state = STATE_INIT;	2555	efx->state = STATE_INIT;
2570		2556
2571	efx_init_channels(efx);	2557	efx_init_channels(efx);
2572		2558
2573	mutex_lock(&efx->mac_lock);	2559	mutex_lock(&efx->mac_lock);
2574	efx->phy_op->reconfigure(efx);	2560	efx->phy_op->reconfigure(efx);
2575	mutex_unlock(&efx->mac_lock);	2561	mutex_unlock(&efx->mac_lock);
2576		2562
2577	efx_start_all(efx);	2563	efx_start_all(efx);
2578		2564
2579	netif_device_attach(efx->net_dev);	2565	netif_device_attach(efx->net_dev);
2580		2566
2581	efx->state = STATE_RUNNING;	2567	efx->state = STATE_RUNNING;
2582		2568
2583	efx->type->resume_wol(efx);	2569	efx->type->resume_wol(efx);
2584		2570
2585	/* Reschedule any quenched resets scheduled during efx_pm_freeze() */	2571	/* Reschedule any quenched resets scheduled during efx_pm_freeze() */
2586	queue_work(reset_workqueue, &efx->reset_work);	2572	queue_work(reset_workqueue, &efx->reset_work);
2587		2573
2588	return 0;	2574	return 0;
2589	}	2575	}
2590		2576
2591	static int efx_pm_poweroff(struct device *dev)	2577	static int efx_pm_poweroff(struct device *dev)
2592	{	2578	{
2593	struct pci_dev *pci_dev = to_pci_dev(dev);	2579	struct pci_dev *pci_dev = to_pci_dev(dev);
2594	struct efx_nic *efx = pci_get_drvdata(pci_dev);	2580	struct efx_nic *efx = pci_get_drvdata(pci_dev);
2595		2581
2596	efx->type->fini(efx);	2582	efx->type->fini(efx);
2597		2583
2598	efx->reset_pending = 0;	2584	efx->reset_pending = 0;
2599		2585
2600	pci_save_state(pci_dev);	2586	pci_save_state(pci_dev);
2601	return pci_set_power_state(pci_dev, PCI_D3hot);	2587	return pci_set_power_state(pci_dev, PCI_D3hot);
2602	}	2588	}
2603		2589
2604	/* Used for both resume and restore */	2590	/* Used for both resume and restore */
2605	static int efx_pm_resume(struct device *dev)	2591	static int efx_pm_resume(struct device *dev)
2606	{	2592	{
2607	struct pci_dev *pci_dev = to_pci_dev(dev);	2593	struct pci_dev *pci_dev = to_pci_dev(dev);
2608	struct efx_nic *efx = pci_get_drvdata(pci_dev);	2594	struct efx_nic *efx = pci_get_drvdata(pci_dev);
2609	int rc;	2595	int rc;
2610		2596
2611	rc = pci_set_power_state(pci_dev, PCI_D0);	2597	rc = pci_set_power_state(pci_dev, PCI_D0);
2612	if (rc)	2598	if (rc)
2613	return rc;	2599	return rc;
2614	pci_restore_state(pci_dev);	2600	pci_restore_state(pci_dev);
2615	rc = pci_enable_device(pci_dev);	2601	rc = pci_enable_device(pci_dev);
2616	if (rc)	2602	if (rc)
2617	return rc;	2603	return rc;
2618	pci_set_master(efx->pci_dev);	2604	pci_set_master(efx->pci_dev);
2619	rc = efx->type->reset(efx, RESET_TYPE_ALL);	2605	rc = efx->type->reset(efx, RESET_TYPE_ALL);
2620	if (rc)	2606	if (rc)
2621	return rc;	2607	return rc;
2622	rc = efx->type->init(efx);	2608	rc = efx->type->init(efx);
2623	if (rc)	2609	if (rc)
2624	return rc;	2610	return rc;
2625	efx_pm_thaw(dev);	2611	efx_pm_thaw(dev);
2626	return 0;	2612	return 0;
2627	}	2613	}
2628		2614
2629	static int efx_pm_suspend(struct device *dev)	2615	static int efx_pm_suspend(struct device *dev)
2630	{	2616	{
2631	int rc;	2617	int rc;
2632		2618
2633	efx_pm_freeze(dev);	2619	efx_pm_freeze(dev);
2634	rc = efx_pm_poweroff(dev);	2620	rc = efx_pm_poweroff(dev);
2635	if (rc)	2621	if (rc)
2636	efx_pm_resume(dev);	2622	efx_pm_resume(dev);
2637	return rc;	2623	return rc;
2638	}	2624	}
2639		2625
2640	static struct dev_pm_ops efx_pm_ops = {	2626	static struct dev_pm_ops efx_pm_ops = {
2641	.suspend = efx_pm_suspend,	2627	.suspend = efx_pm_suspend,
2642	.resume = efx_pm_resume,	2628	.resume = efx_pm_resume,
2643	.freeze = efx_pm_freeze,	2629	.freeze = efx_pm_freeze,
2644	.thaw = efx_pm_thaw,	2630	.thaw = efx_pm_thaw,
2645	.poweroff = efx_pm_poweroff,	2631	.poweroff = efx_pm_poweroff,
2646	.restore = efx_pm_resume,	2632	.restore = efx_pm_resume,
2647	};	2633	};
2648		2634
2649	static struct pci_driver efx_pci_driver = {	2635	static struct pci_driver efx_pci_driver = {
2650	.name = KBUILD_MODNAME,	2636	.name = KBUILD_MODNAME,
2651	.id_table = efx_pci_table,	2637	.id_table = efx_pci_table,
2652	.probe = efx_pci_probe,	2638	.probe = efx_pci_probe,
2653	.remove = efx_pci_remove,	2639	.remove = efx_pci_remove,
2654	.driver.pm = &efx_pm_ops,	2640	.driver.pm = &efx_pm_ops,
2655	};	2641	};
2656		2642
2657	/**************************************************************************	2643	/**************************************************************************
2658	*	2644	*
2659	* Kernel module interface	2645	* Kernel module interface
2660	*	2646	*
2661	*************************************************************************/	2647	*************************************************************************/
2662		2648
2663	module_param(interrupt_mode, uint, 0444);	2649	module_param(interrupt_mode, uint, 0444);
2664	MODULE_PARM_DESC(interrupt_mode,	2650	MODULE_PARM_DESC(interrupt_mode,
2665	"Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");	2651	"Interrupt mode (0=>MSIX 1=>MSI 2=>legacy)");
2666		2652
2667	static int __init efx_init_module(void)	2653	static int __init efx_init_module(void)
2668	{	2654	{
2669	int rc;	2655	int rc;
2670		2656
2671	printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");	2657	printk(KERN_INFO "Solarflare NET driver v" EFX_DRIVER_VERSION "\n");
2672		2658
2673	rc = register_netdevice_notifier(&efx_netdev_notifier);	2659	rc = register_netdevice_notifier(&efx_netdev_notifier);
2674	if (rc)	2660	if (rc)
2675	goto err_notifier;	2661	goto err_notifier;
2676		2662
2677	reset_workqueue = create_singlethread_workqueue("sfc_reset");	2663	reset_workqueue = create_singlethread_workqueue("sfc_reset");
2678	if (!reset_workqueue) {	2664	if (!reset_workqueue) {
2679	rc = -ENOMEM;	2665	rc = -ENOMEM;
2680	goto err_reset;	2666	goto err_reset;
2681	}	2667	}
2682		2668
2683	rc = pci_register_driver(&efx_pci_driver);	2669	rc = pci_register_driver(&efx_pci_driver);
2684	if (rc < 0)	2670	if (rc < 0)
2685	goto err_pci;	2671	goto err_pci;
2686		2672
2687	return 0;	2673	return 0;
2688		2674
2689	err_pci:	2675	err_pci:
2690	destroy_workqueue(reset_workqueue);	2676	destroy_workqueue(reset_workqueue);
2691	err_reset:	2677	err_reset:
2692	unregister_netdevice_notifier(&efx_netdev_notifier);	2678	unregister_netdevice_notifier(&efx_netdev_notifier);
2693	err_notifier:	2679	err_notifier:
2694	return rc;	2680	return rc;
2695	}	2681	}
2696		2682
2697	static void __exit efx_exit_module(void)	2683	static void __exit efx_exit_module(void)
2698	{	2684	{
2699	printk(KERN_INFO "Solarflare NET driver unloading\n");	2685	printk(KERN_INFO "Solarflare NET driver unloading\n");
2700		2686
2701	pci_unregister_driver(&efx_pci_driver);	2687	pci_unregister_driver(&efx_pci_driver);
2702	destroy_workqueue(reset_workqueue);	2688	destroy_workqueue(reset_workqueue);
2703	unregister_netdevice_notifier(&efx_netdev_notifier);	2689	unregister_netdevice_notifier(&efx_netdev_notifier);
2704		2690
2705	}	2691	}
2706		2692
2707	module_init(efx_init_module);	2693	module_init(efx_init_module);
2708	module_exit(efx_exit_module);	2694	module_exit(efx_exit_module);
2709		2695
2710	MODULE_AUTHOR("Solarflare Communications and "	2696	MODULE_AUTHOR("Solarflare Communications and "
2711	"Michael Brown <mbrown@fensystems.co.uk>");	2697	"Michael Brown <mbrown@fensystems.co.uk>");
2712	MODULE_DESCRIPTION("Solarflare Communications network driver");	2698	MODULE_DESCRIPTION("Solarflare Communications network driver");
2713	MODULE_LICENSE("GPL");	2699	MODULE_LICENSE("GPL");
2714	MODULE_DEVICE_TABLE(pci, efx_pci_table);	2700	MODULE_DEVICE_TABLE(pci, efx_pci_table);
2715		2701

drivers/net/sfc/io.h

Diff comments View file @ 86c432c

 /****************************************************************************
  * Driver for Solarflare Solarstorm network controllers and boards
  * Copyright 2005-2006 Fen Systems Ltd.
  * Copyright 2006-2010 Solarflare Communications Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
  * by the Free Software Foundation, incorporated herein by reference.
  */
 #ifndef EFX_IO_H
 #define EFX_IO_H
 #include <linux/io.h>
 #include <linux/spinlock.h>
 /**************************************************************************
  *
  * NIC register I/O
  *
  **************************************************************************
  *
  * Notes on locking strategy:
  *
  * Most CSRs are 128-bit (oword) and therefore cannot be read or
  * written atomically.  Access from the host is buffered by the Bus
  * Interface Unit (BIU).  Whenever the host reads from the lowest
  * address of such a register, or from the address of a different such
  * register, the BIU latches the register's value.  Subsequent reads
  * from higher addresses of the same register will read the latched
  * value.  Whenever the host writes part of such a register, the BIU
  * collects the written value and does not write to the underlying
  * register until all 4 dwords have been written.  A similar buffering
  * scheme applies to host access to the NIC's 64-bit SRAM.
  *
  * Access to different CSRs and 64-bit SRAM words must be serialised,
  * since interleaved access can result in lost writes or lost
  * information from read-to-clear fields.  We use efx_nic::biu_lock
  * for this.  (We could use separate locks for read and write, but
  * this is not normally a performance bottleneck.)
  *
  * The DMA descriptor pointers (RX_DESC_UPD and TX_DESC_UPD) are
  * 128-bit but are special-cased in the BIU to avoid the need for
  * locking in the host:
  *
  * - They are write-only.
  * - The semantics of writing to these registers are such that
  *   replacing the low 96 bits with zero does not affect functionality.
  * - If the host writes to the last dword address of such a register
  *   (i.e. the high 32 bits) the underlying register will always be
- *   written.  If the collector and the current write together do not
+ *   written.  If the collector does not hold values for the low 96
- *   provide values for all 128 bits of the register, the low 96 bits
+ *   bits of the register, they will be written as zero.  Writing to
- *   will be written as zero.
+ *   the last qword does not have this effect and must not be done.
  * - If the host writes to the address of any other part of such a
  *   register while the collector already holds values for some other
  *   register, the write is discarded and the collector maintains its
  *   current state.
  */
 #if BITS_PER_LONG == 64
 #define EFX_USE_QWORD_IO 1
 #endif
 #ifdef EFX_USE_QWORD_IO
 static inline void _efx_writeq(struct efx_nic *efx, __le64 value,
 				  unsigned int reg)
 {
 	__raw_writeq((__force u64)value, efx->membase + reg);
 }
 static inline __le64 _efx_readq(struct efx_nic *efx, unsigned int reg)
 {
 	return (__force __le64)__raw_readq(efx->membase + reg);
 }
 #endif
 static inline void _efx_writed(struct efx_nic *efx, __le32 value,
 				  unsigned int reg)
 {
 	__raw_writel((__force u32)value, efx->membase + reg);
 }
 static inline __le32 _efx_readd(struct efx_nic *efx, unsigned int reg)
 {
 	return (__force __le32)__raw_readl(efx->membase + reg);
 }
 /* Write a normal 128-bit CSR, locking as appropriate. */
 static inline void efx_writeo(struct efx_nic *efx, efx_oword_t *value,
 			      unsigned int reg)
 {
 	unsigned long flags __attribute__ ((unused));
 	netif_vdbg(efx, hw, efx->net_dev,
 		   "writing register %x with " EFX_OWORD_FMT "\n", reg,
 		   EFX_OWORD_VAL(*value));
 	spin_lock_irqsave(&efx->biu_lock, flags);
 #ifdef EFX_USE_QWORD_IO
 	_efx_writeq(efx, value->u64[0], reg + 0);
 	_efx_writeq(efx, value->u64[1], reg + 8);
 #else
 	_efx_writed(efx, value->u32[0], reg + 0);
 	_efx_writed(efx, value->u32[1], reg + 4);
 	_efx_writed(efx, value->u32[2], reg + 8);
 	_efx_writed(efx, value->u32[3], reg + 12);
 #endif
-	wmb();
 	mmiowb();
 	spin_unlock_irqrestore(&efx->biu_lock, flags);
 }
 /* Write 64-bit SRAM through the supplied mapping, locking as appropriate. */
 static inline void efx_sram_writeq(struct efx_nic *efx, void __iomem *membase,
 				   efx_qword_t *value, unsigned int index)
 {
 	unsigned int addr = index * sizeof(*value);
 	unsigned long flags __attribute__ ((unused));
 	netif_vdbg(efx, hw, efx->net_dev,
 		   "writing SRAM address %x with " EFX_QWORD_FMT "\n",
 		   addr, EFX_QWORD_VAL(*value));
 	spin_lock_irqsave(&efx->biu_lock, flags);
 #ifdef EFX_USE_QWORD_IO
 	__raw_writeq((__force u64)value->u64[0], membase + addr);
 #else
 	__raw_writel((__force u32)value->u32[0], membase + addr);
 	__raw_writel((__force u32)value->u32[1], membase + addr + 4);
 #endif
-	wmb();
 	mmiowb();
 	spin_unlock_irqrestore(&efx->biu_lock, flags);
 }
 /* Write a 32-bit CSR or the last dword of a special 128-bit CSR */
 static inline void efx_writed(struct efx_nic *efx, efx_dword_t *value,
 			      unsigned int reg)
 {
 	netif_vdbg(efx, hw, efx->net_dev,
 		   "writing register %x with "EFX_DWORD_FMT"\n",
 		   reg, EFX_DWORD_VAL(*value));
 	/* No lock required */
 	_efx_writed(efx, value->u32[0], reg);
-	wmb();
 }
 /* Read a 128-bit CSR, locking as appropriate. */
 static inline void efx_reado(struct efx_nic *efx, efx_oword_t *value,
 			     unsigned int reg)
 {
 	unsigned long flags __attribute__ ((unused));
 	spin_lock_irqsave(&efx->biu_lock, flags);
 	value->u32[0] = _efx_readd(efx, reg + 0);
-	rmb();
 	value->u32[1] = _efx_readd(efx, reg + 4);
 	value->u32[2] = _efx_readd(efx, reg + 8);
 	value->u32[3] = _efx_readd(efx, reg + 12);
 	spin_unlock_irqrestore(&efx->biu_lock, flags);
 	netif_vdbg(efx, hw, efx->net_dev,
 		   "read from register %x, got " EFX_OWORD_FMT "\n", reg,
 		   EFX_OWORD_VAL(*value));
 }
 /* Read 64-bit SRAM through the supplied mapping, locking as appropriate. */
 static inline void efx_sram_readq(struct efx_nic *efx, void __iomem *membase,
 				  efx_qword_t *value, unsigned int index)
 {
 	unsigned int addr = index * sizeof(*value);
 	unsigned long flags __attribute__ ((unused));
 	spin_lock_irqsave(&efx->biu_lock, flags);
 #ifdef EFX_USE_QWORD_IO
 	value->u64[0] = (__force __le64)__raw_readq(membase + addr);
 #else
 	value->u32[0] = (__force __le32)__raw_readl(membase + addr);
-	rmb();
 	value->u32[1] = (__force __le32)__raw_readl(membase + addr + 4);
 #endif
 	spin_unlock_irqrestore(&efx->biu_lock, flags);
 	netif_vdbg(efx, hw, efx->net_dev,
 		   "read from SRAM address %x, got "EFX_QWORD_FMT"\n",
 		   addr, EFX_QWORD_VAL(*value));
 }
 /* Read a 32-bit CSR or SRAM */
 static inline void efx_readd(struct efx_nic *efx, efx_dword_t *value,
 				unsigned int reg)
 {
 	value->u32[0] = _efx_readd(efx, reg);
 	netif_vdbg(efx, hw, efx->net_dev,
 		   "read from register %x, got "EFX_DWORD_FMT"\n",
 		   reg, EFX_DWORD_VAL(*value));
 }
 /* Write a 128-bit CSR forming part of a table */
 static inline void efx_writeo_table(struct efx_nic *efx, efx_oword_t *value,
 				      unsigned int reg, unsigned int index)
 {
 	efx_writeo(efx, value, reg + index * sizeof(efx_oword_t));
 }
 /* Read a 128-bit CSR forming part of a table */
 static inline void efx_reado_table(struct efx_nic *efx, efx_oword_t *value,
 				     unsigned int reg, unsigned int index)
 {
 	efx_reado(efx, value, reg + index * sizeof(efx_oword_t));
 }
 /* Write a 32-bit CSR forming part of a table, or 32-bit SRAM */
 static inline void efx_writed_table(struct efx_nic *efx, efx_dword_t *value,
 				       unsigned int reg, unsigned int index)
 {
 	efx_writed(efx, value, reg + index * sizeof(efx_oword_t));
 }
 /* Read a 32-bit CSR forming part of a table, or 32-bit SRAM */
 static inline void efx_readd_table(struct efx_nic *efx, efx_dword_t *value,
 				   unsigned int reg, unsigned int index)
 {
 	efx_readd(efx, value, reg + index * sizeof(efx_dword_t));
 }
 /* Page-mapped register block size */
 #define EFX_PAGE_BLOCK_SIZE 0x2000
 /* Calculate offset to page-mapped register block */
 #define EFX_PAGED_REG(page, reg) \
 	((page) * EFX_PAGE_BLOCK_SIZE + (reg))
 /* Write the whole of RX_DESC_UPD or TX_DESC_UPD */
 static inline void _efx_writeo_page(struct efx_nic *efx, efx_oword_t *value,
 				    unsigned int reg, unsigned int page)
 {
 	reg = EFX_PAGED_REG(page, reg);
 	netif_vdbg(efx, hw, efx->net_dev,
 		   "writing register %x with " EFX_OWORD_FMT "\n", reg,
 		   EFX_OWORD_VAL(*value));
 #ifdef EFX_USE_QWORD_IO
 	_efx_writeq(efx, value->u64[0], reg + 0);
-	_efx_writeq(efx, value->u64[1], reg + 8);
 #else
 	_efx_writed(efx, value->u32[0], reg + 0);
 	_efx_writed(efx, value->u32[1], reg + 4);
+#endif
 	_efx_writed(efx, value->u32[2], reg + 8);
 	_efx_writed(efx, value->u32[3], reg + 12);
-#endif
-	wmb();
 }
 #define efx_writeo_page(efx, value, reg, page)				\
 	_efx_writeo_page(efx, value,					\
 			 reg +						\
 			 BUILD_BUG_ON_ZERO((reg) != 0x830 && (reg) != 0xa10), \
 			 page)
 /* Write a page-mapped 32-bit CSR (EVQ_RPTR or the high bits of
  * RX_DESC_UPD or TX_DESC_UPD)
  */
 static inline void _efx_writed_page(struct efx_nic *efx, efx_dword_t *value,
 				    unsigned int reg, unsigned int page)
 {
 	efx_writed(efx, value, EFX_PAGED_REG(page, reg));
 }
 #define efx_writed_page(efx, value, reg, page)				\
 	_efx_writed_page(efx, value,					\
 			 reg +						\
 			 BUILD_BUG_ON_ZERO((reg) != 0x400 && (reg) != 0x83c \
 					   && (reg) != 0xa1c),		\
 			 page)
 /* Write TIMER_COMMAND.  This is a page-mapped 32-bit CSR, but a bug
  * in the BIU means that writes to TIMER_COMMAND[0] invalidate the
  * collector register.
  */
 static inline void _efx_writed_page_locked(struct efx_nic *efx,
 					   efx_dword_t *value,
 					   unsigned int reg,
 					   unsigned int page)
 {
 	unsigned long flags __attribute__ ((unused));
 	if (page == 0) {
 		spin_lock_irqsave(&efx->biu_lock, flags);
 		efx_writed(efx, value, EFX_PAGED_REG(page, reg));
 		spin_unlock_irqrestore(&efx->biu_lock, flags);
 	} else {
 		efx_writed(efx, value, EFX_PAGED_REG(page, reg));
 	}
 }
 #define efx_writed_page_locked(efx, value, reg, page)			\
 	_efx_writed_page_locked(efx, value,				\
 				reg + BUILD_BUG_ON_ZERO((reg) != 0x420), \
 				page)
 #endif /* EFX_IO_H */

drivers/net/sfc/mcdi.c

Diff comments View file @ 86c432c

 /****************************************************************************
  * Driver for Solarflare Solarstorm network controllers and boards
  * Copyright 2008-2011 Solarflare Communications Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
  * by the Free Software Foundation, incorporated herein by reference.
  */
 #include <linux/delay.h>
 #include "net_driver.h"
 #include "nic.h"
 #include "io.h"
 #include "regs.h"
 #include "mcdi_pcol.h"
 #include "phy.h"
 /**************************************************************************
  *
  * Management-Controller-to-Driver Interface
  *
  **************************************************************************
  */
 /* Software-defined structure to the shared-memory */
 #define CMD_NOTIFY_PORT0 0
 #define CMD_NOTIFY_PORT1 4
 #define CMD_PDU_PORT0    0x008
 #define CMD_PDU_PORT1    0x108
 #define REBOOT_FLAG_PORT0 0x3f8
 #define REBOOT_FLAG_PORT1 0x3fc
 #define MCDI_RPC_TIMEOUT       10 /*seconds */
 #define MCDI_PDU(efx)							\
 	(efx_port_num(efx) ? CMD_PDU_PORT1 : CMD_PDU_PORT0)
 #define MCDI_DOORBELL(efx)						\
 	(efx_port_num(efx) ? CMD_NOTIFY_PORT1 : CMD_NOTIFY_PORT0)
 #define MCDI_REBOOT_FLAG(efx)						\
 	(efx_port_num(efx) ? REBOOT_FLAG_PORT1 : REBOOT_FLAG_PORT0)
 #define SEQ_MASK							\
 	EFX_MASK32(EFX_WIDTH(MCDI_HEADER_SEQ))
 static inline struct efx_mcdi_iface *efx_mcdi(struct efx_nic *efx)
 {
 	struct siena_nic_data *nic_data;
 	EFX_BUG_ON_PARANOID(efx_nic_rev(efx) < EFX_REV_SIENA_A0);
 	nic_data = efx->nic_data;
 	return &nic_data->mcdi;
 }
-static inline void
-efx_mcdi_readd(struct efx_nic *efx, efx_dword_t *value, unsigned reg)
-{
-	struct siena_nic_data *nic_data = efx->nic_data;
-	value->u32[0] = (__force __le32)__raw_readl(nic_data->mcdi_smem + reg);
-}
-static inline void
-efx_mcdi_writed(struct efx_nic *efx, const efx_dword_t *value, unsigned reg)
-{
-	struct siena_nic_data *nic_data = efx->nic_data;
-	__raw_writel((__force u32)value->u32[0], nic_data->mcdi_smem + reg);
-}
 void efx_mcdi_init(struct efx_nic *efx)
 {
 	struct efx_mcdi_iface *mcdi;
 	if (efx_nic_rev(efx) < EFX_REV_SIENA_A0)
 		return;
 	mcdi = efx_mcdi(efx);
 	init_waitqueue_head(&mcdi->wq);
 	spin_lock_init(&mcdi->iface_lock);
 	atomic_set(&mcdi->state, MCDI_STATE_QUIESCENT);
 	mcdi->mode = MCDI_MODE_POLL;
 	(void) efx_mcdi_poll_reboot(efx);
 }
 static void efx_mcdi_copyin(struct efx_nic *efx, unsigned cmd,
 			    const u8 *inbuf, size_t inlen)
 {
 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
-	unsigned pdu = MCDI_PDU(efx);
+	unsigned pdu = FR_CZ_MC_TREG_SMEM + MCDI_PDU(efx);
-	unsigned doorbell = MCDI_DOORBELL(efx);
+	unsigned doorbell = FR_CZ_MC_TREG_SMEM + MCDI_DOORBELL(efx);
 	unsigned int i;
 	efx_dword_t hdr;
 	u32 xflags, seqno;
 	BUG_ON(atomic_read(&mcdi->state) == MCDI_STATE_QUIESCENT);
 	BUG_ON(inlen & 3 || inlen >= 0x100);
 	seqno = mcdi->seqno & SEQ_MASK;
 	xflags = 0;
 	if (mcdi->mode == MCDI_MODE_EVENTS)
 		xflags |= MCDI_HEADER_XFLAGS_EVREQ;
 	EFX_POPULATE_DWORD_6(hdr,
 			     MCDI_HEADER_RESPONSE, 0,
 			     MCDI_HEADER_RESYNC, 1,
 			     MCDI_HEADER_CODE, cmd,
 			     MCDI_HEADER_DATALEN, inlen,
 			     MCDI_HEADER_SEQ, seqno,
 			     MCDI_HEADER_XFLAGS, xflags);
-	efx_mcdi_writed(efx, &hdr, pdu);
+	efx_writed(efx, &hdr, pdu);
 	for (i = 0; i < inlen; i += 4)
-		efx_mcdi_writed(efx, (const efx_dword_t *)(inbuf + i),
+		_efx_writed(efx, *((__le32 *)(inbuf + i)), pdu + 4 + i);
-				pdu + 4 + i);
+	/* Ensure the payload is written out before the header */
+	wmb();
 	/* ring the doorbell with a distinctive value */
-	EFX_POPULATE_DWORD_1(hdr, EFX_DWORD_0, 0x45789abc);
+	_efx_writed(efx, (__force __le32) 0x45789abc, doorbell);
-	efx_mcdi_writed(efx, &hdr, doorbell);
 }
 static void efx_mcdi_copyout(struct efx_nic *efx, u8 *outbuf, size_t outlen)
 {
 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
-	unsigned int pdu = MCDI_PDU(efx);
+	unsigned int pdu = FR_CZ_MC_TREG_SMEM + MCDI_PDU(efx);
 	int i;
 	BUG_ON(atomic_read(&mcdi->state) == MCDI_STATE_QUIESCENT);
 	BUG_ON(outlen & 3 || outlen >= 0x100);
 	for (i = 0; i < outlen; i += 4)
-		efx_mcdi_readd(efx, (efx_dword_t *)(outbuf + i), pdu + 4 + i);
+		*((__le32 *)(outbuf + i)) = _efx_readd(efx, pdu + 4 + i);
 }
 static int efx_mcdi_poll(struct efx_nic *efx)
 {
 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
 	unsigned int time, finish;
 	unsigned int respseq, respcmd, error;
-	unsigned int pdu = MCDI_PDU(efx);
+	unsigned int pdu = FR_CZ_MC_TREG_SMEM + MCDI_PDU(efx);
 	unsigned int rc, spins;
 	efx_dword_t reg;
 	/* Check for a reboot atomically with respect to efx_mcdi_copyout() */
 	rc = -efx_mcdi_poll_reboot(efx);
 	if (rc)
 		goto out;
 	/* Poll for completion. Poll quickly (once a us) for the 1st jiffy,
 	 * because generally mcdi responses are fast. After that, back off
 	 * and poll once a jiffy (approximately)
 	 */
 	spins = TICK_USEC;
 	finish = get_seconds() + MCDI_RPC_TIMEOUT;
 	while (1) {
 		if (spins != 0) {
 			--spins;
 			udelay(1);
 		} else {
 			schedule_timeout_uninterruptible(1);
 		}
 		time = get_seconds();
-		efx_mcdi_readd(efx, &reg, pdu);
+		rmb();
+		efx_readd(efx, &reg, pdu);
 		/* All 1's indicates that shared memory is in reset (and is
 		 * not a valid header). Wait for it to come out reset before
 		 * completing the command */
 		if (EFX_DWORD_FIELD(reg, EFX_DWORD_0) != 0xffffffff &&
 		    EFX_DWORD_FIELD(reg, MCDI_HEADER_RESPONSE))
 			break;
 		if (time >= finish)
 			return -ETIMEDOUT;
 	}
 	mcdi->resplen = EFX_DWORD_FIELD(reg, MCDI_HEADER_DATALEN);
 	respseq = EFX_DWORD_FIELD(reg, MCDI_HEADER_SEQ);
 	respcmd = EFX_DWORD_FIELD(reg, MCDI_HEADER_CODE);
 	error = EFX_DWORD_FIELD(reg, MCDI_HEADER_ERROR);
 	if (error && mcdi->resplen == 0) {
 		netif_err(efx, hw, efx->net_dev, "MC rebooted\n");
 		rc = EIO;
 	} else if ((respseq ^ mcdi->seqno) & SEQ_MASK) {
 		netif_err(efx, hw, efx->net_dev,
 			  "MC response mismatch tx seq 0x%x rx seq 0x%x\n",
 			  respseq, mcdi->seqno);
 		rc = EIO;
 	} else if (error) {
-		efx_mcdi_readd(efx, &reg, pdu + 4);
+		efx_readd(efx, &reg, pdu + 4);
 		switch (EFX_DWORD_FIELD(reg, EFX_DWORD_0)) {
 #define TRANSLATE_ERROR(name)					\
 		case MC_CMD_ERR_ ## name:			\
 			rc = name;				\
 			break
 			TRANSLATE_ERROR(ENOENT);
 			TRANSLATE_ERROR(EINTR);
 			TRANSLATE_ERROR(EACCES);
 			TRANSLATE_ERROR(EBUSY);
 			TRANSLATE_ERROR(EINVAL);
 			TRANSLATE_ERROR(EDEADLK);
 			TRANSLATE_ERROR(ENOSYS);
 			TRANSLATE_ERROR(ETIME);
 #undef TRANSLATE_ERROR
 		default:
 			rc = EIO;
 			break;
 		}
 	} else
 		rc = 0;
 out:
 	mcdi->resprc = rc;
 	if (rc)
 		mcdi->resplen = 0;
 	/* Return rc=0 like wait_event_timeout() */
 	return 0;
 }
 /* Test and clear MC-rebooted flag for this port/function */
 int efx_mcdi_poll_reboot(struct efx_nic *efx)
 {
-	unsigned int addr = MCDI_REBOOT_FLAG(efx);
+	unsigned int addr = FR_CZ_MC_TREG_SMEM + MCDI_REBOOT_FLAG(efx);
 	efx_dword_t reg;
 	uint32_t value;
 	if (efx_nic_rev(efx) < EFX_REV_SIENA_A0)
 		return false;
-	efx_mcdi_readd(efx, &reg, addr);
+	efx_readd(efx, &reg, addr);
 	value = EFX_DWORD_FIELD(reg, EFX_DWORD_0);
 	if (value == 0)
 		return 0;
 	EFX_ZERO_DWORD(reg);
-	efx_mcdi_writed(efx, &reg, addr);
+	efx_writed(efx, &reg, addr);
 	if (value == MC_STATUS_DWORD_ASSERT)
 		return -EINTR;
 	else
 		return -EIO;
 }
 static void efx_mcdi_acquire(struct efx_mcdi_iface *mcdi)
 {
 	/* Wait until the interface becomes QUIESCENT and we win the race
 	 * to mark it RUNNING. */
 	wait_event(mcdi->wq,
 		   atomic_cmpxchg(&mcdi->state,
 				  MCDI_STATE_QUIESCENT,
 				  MCDI_STATE_RUNNING)
 		   == MCDI_STATE_QUIESCENT);
 }
 static int efx_mcdi_await_completion(struct efx_nic *efx)
 {
 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
 	if (wait_event_timeout(
 		    mcdi->wq,
 		    atomic_read(&mcdi->state) == MCDI_STATE_COMPLETED,
 		    msecs_to_jiffies(MCDI_RPC_TIMEOUT * 1000)) == 0)
 		return -ETIMEDOUT;
 	/* Check if efx_mcdi_set_mode() switched us back to polled completions.
 	 * In which case, poll for completions directly. If efx_mcdi_ev_cpl()
 	 * completed the request first, then we'll just end up completing the
 	 * request again, which is safe.
 	 *
 	 * We need an smp_rmb() to synchronise with efx_mcdi_mode_poll(), which
 	 * wait_event_timeout() implicitly provides.
 	 */
 	if (mcdi->mode == MCDI_MODE_POLL)
 		return efx_mcdi_poll(efx);
 	return 0;
 }
 static bool efx_mcdi_complete(struct efx_mcdi_iface *mcdi)
 {
 	/* If the interface is RUNNING, then move to COMPLETED and wake any
 	 * waiters. If the interface isn't in RUNNING then we've received a
 	 * duplicate completion after we've already transitioned back to
 	 * QUIESCENT. [A subsequent invocation would increment seqno, so would
 	 * have failed the seqno check].
 	 */
 	if (atomic_cmpxchg(&mcdi->state,
 			   MCDI_STATE_RUNNING,
 			   MCDI_STATE_COMPLETED) == MCDI_STATE_RUNNING) {
 		wake_up(&mcdi->wq);
 		return true;
 	}
 	return false;
 }
 static void efx_mcdi_release(struct efx_mcdi_iface *mcdi)
 {
 	atomic_set(&mcdi->state, MCDI_STATE_QUIESCENT);
 	wake_up(&mcdi->wq);
 }
 static void efx_mcdi_ev_cpl(struct efx_nic *efx, unsigned int seqno,
 			    unsigned int datalen, unsigned int errno)
 {
 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
 	bool wake = false;
 	spin_lock(&mcdi->iface_lock);
 	if ((seqno ^ mcdi->seqno) & SEQ_MASK) {
 		if (mcdi->credits)
 			/* The request has been cancelled */
 			--mcdi->credits;
 		else
 			netif_err(efx, hw, efx->net_dev,
 				  "MC response mismatch tx seq 0x%x rx "
 				  "seq 0x%x\n", seqno, mcdi->seqno);
 	} else {
 		mcdi->resprc = errno;
 		mcdi->resplen = datalen;
 		wake = true;
 	}
 	spin_unlock(&mcdi->iface_lock);
 	if (wake)
 		efx_mcdi_complete(mcdi);
 }
 /* Issue the given command by writing the data into the shared memory PDU,
  * ring the doorbell and wait for completion. Copyout the result. */
 int efx_mcdi_rpc(struct efx_nic *efx, unsigned cmd,
 		 const u8 *inbuf, size_t inlen, u8 *outbuf, size_t outlen,
 		 size_t *outlen_actual)
 {
 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
 	int rc;
 	BUG_ON(efx_nic_rev(efx) < EFX_REV_SIENA_A0);
 	efx_mcdi_acquire(mcdi);
 	/* Serialise with efx_mcdi_ev_cpl() and efx_mcdi_ev_death() */
 	spin_lock_bh(&mcdi->iface_lock);
 	++mcdi->seqno;
 	spin_unlock_bh(&mcdi->iface_lock);
 	efx_mcdi_copyin(efx, cmd, inbuf, inlen);
 	if (mcdi->mode == MCDI_MODE_POLL)
 		rc = efx_mcdi_poll(efx);
 	else
 		rc = efx_mcdi_await_completion(efx);
 	if (rc != 0) {
 		/* Close the race with efx_mcdi_ev_cpl() executing just too late
 		 * and completing a request we've just cancelled, by ensuring
 		 * that the seqno check therein fails.
 		 */
 		spin_lock_bh(&mcdi->iface_lock);
 		++mcdi->seqno;
 		++mcdi->credits;
 		spin_unlock_bh(&mcdi->iface_lock);
 		netif_err(efx, hw, efx->net_dev,
 			  "MC command 0x%x inlen %d mode %d timed out\n",
 			  cmd, (int)inlen, mcdi->mode);
 	} else {
 		size_t resplen;
 		/* At the very least we need a memory barrier here to ensure
 		 * we pick up changes from efx_mcdi_ev_cpl(). Protect against
 		 * a spurious efx_mcdi_ev_cpl() running concurrently by
 		 * acquiring the iface_lock. */
 		spin_lock_bh(&mcdi->iface_lock);
 		rc = -mcdi->resprc;
 		resplen = mcdi->resplen;
 		spin_unlock_bh(&mcdi->iface_lock);
 		if (rc == 0) {
 			efx_mcdi_copyout(efx, outbuf,
 					 min(outlen, mcdi->resplen + 3) & ~0x3);
 			if (outlen_actual != NULL)
 				*outlen_actual = resplen;
 		} else if (cmd == MC_CMD_REBOOT && rc == -EIO)
 			; /* Don't reset if MC_CMD_REBOOT returns EIO */
 		else if (rc == -EIO || rc == -EINTR) {
 			netif_err(efx, hw, efx->net_dev, "MC fatal error %d\n",
 				  -rc);
 			efx_schedule_reset(efx, RESET_TYPE_MC_FAILURE);
 		} else
 			netif_dbg(efx, hw, efx->net_dev,
 				  "MC command 0x%x inlen %d failed rc=%d\n",
 				  cmd, (int)inlen, -rc);
 	}
 	efx_mcdi_release(mcdi);
 	return rc;
 }
 void efx_mcdi_mode_poll(struct efx_nic *efx)
 {
 	struct efx_mcdi_iface *mcdi;
 	if (efx_nic_rev(efx) < EFX_REV_SIENA_A0)
 		return;
 	mcdi = efx_mcdi(efx);
 	if (mcdi->mode == MCDI_MODE_POLL)
 		return;
 	/* We can switch from event completion to polled completion, because
 	 * mcdi requests are always completed in shared memory. We do this by
 	 * switching the mode to POLL'd then completing the request.
 	 * efx_mcdi_await_completion() will then call efx_mcdi_poll().
 	 *
 	 * We need an smp_wmb() to synchronise with efx_mcdi_await_completion(),
 	 * which efx_mcdi_complete() provides for us.
 	 */
 	mcdi->mode = MCDI_MODE_POLL;
 	efx_mcdi_complete(mcdi);
 }
 void efx_mcdi_mode_event(struct efx_nic *efx)
 {
 	struct efx_mcdi_iface *mcdi;
 	if (efx_nic_rev(efx) < EFX_REV_SIENA_A0)
 		return;
 	mcdi = efx_mcdi(efx);
 	if (mcdi->mode == MCDI_MODE_EVENTS)
 		return;
 	/* We can't switch from polled to event completion in the middle of a
 	 * request, because the completion method is specified in the request.
 	 * So acquire the interface to serialise the requestors. We don't need
 	 * to acquire the iface_lock to change the mode here, but we do need a
 	 * write memory barrier ensure that efx_mcdi_rpc() sees it, which
 	 * efx_mcdi_acquire() provides.
 	 */
 	efx_mcdi_acquire(mcdi);
 	mcdi->mode = MCDI_MODE_EVENTS;
 	efx_mcdi_release(mcdi);
 }
 static void efx_mcdi_ev_death(struct efx_nic *efx, int rc)
 {
 	struct efx_mcdi_iface *mcdi = efx_mcdi(efx);
 	/* If there is an outstanding MCDI request, it has been terminated
 	 * either by a BADASSERT or REBOOT event. If the mcdi interface is
 	 * in polled mode, then do nothing because the MC reboot handler will
 	 * set the header correctly. However, if the mcdi interface is waiting
 	 * for a CMDDONE event it won't receive it [and since all MCDI events
 	 * are sent to the same queue, we can't be racing with
 	 * efx_mcdi_ev_cpl()]
 	 *
 	 * There's a race here with efx_mcdi_rpc(), because we might receive
 	 * a REBOOT event *before* the request has been copied out. In polled
 	 * mode (during startup) this is irrelevant, because efx_mcdi_complete()
 	 * is ignored. In event mode, this condition is just an edge-case of
 	 * receiving a REBOOT event after posting the MCDI request. Did the mc
 	 * reboot before or after the copyout? The best we can do always is
 	 * just return failure.
 	 */
 	spin_lock(&mcdi->iface_lock);
 	if (efx_mcdi_complete(mcdi)) {
 		if (mcdi->mode == MCDI_MODE_EVENTS) {
 			mcdi->resprc = rc;
 			mcdi->resplen = 0;
 			++mcdi->credits;
 		}
 	} else
 		/* Nobody was waiting for an MCDI request, so trigger a reset */
 		efx_schedule_reset(efx, RESET_TYPE_MC_FAILURE);
 	spin_unlock(&mcdi->iface_lock);
 }
 static unsigned int efx_mcdi_event_link_speed[] = {
 	[MCDI_EVENT_LINKCHANGE_SPEED_100M] = 100,
 	[MCDI_EVENT_LINKCHANGE_SPEED_1G] = 1000,
 	[MCDI_EVENT_LINKCHANGE_SPEED_10G] = 10000,
 };
 static void efx_mcdi_process_link_change(struct efx_nic *efx, efx_qword_t *ev)
 {
 	u32 flags, fcntl, speed, lpa;
 	speed = EFX_QWORD_FIELD(*ev, MCDI_EVENT_LINKCHANGE_SPEED);
 	EFX_BUG_ON_PARANOID(speed >= ARRAY_SIZE(efx_mcdi_event_link_speed));
 	speed = efx_mcdi_event_link_speed[speed];
 	flags = EFX_QWORD_FIELD(*ev, MCDI_EVENT_LINKCHANGE_LINK_FLAGS);
 	fcntl = EFX_QWORD_FIELD(*ev, MCDI_EVENT_LINKCHANGE_FCNTL);
 	lpa = EFX_QWORD_FIELD(*ev, MCDI_EVENT_LINKCHANGE_LP_CAP);
 	/* efx->link_state is only modified by efx_mcdi_phy_get_link(),
 	 * which is only run after flushing the event queues. Therefore, it
 	 * is safe to modify the link state outside of the mac_lock here.
 	 */
 	efx_mcdi_phy_decode_link(efx, &efx->link_state, speed, flags, fcntl);
 	efx_mcdi_phy_check_fcntl(efx, lpa);
 	efx_link_status_changed(efx);
 }
 static const char *sensor_names[] = {
 	[MC_CMD_SENSOR_CONTROLLER_TEMP] = "Controller temp. sensor",
 	[MC_CMD_SENSOR_PHY_COMMON_TEMP] = "PHY shared temp. sensor",
 	[MC_CMD_SENSOR_CONTROLLER_COOLING] = "Controller cooling",
 	[MC_CMD_SENSOR_PHY0_TEMP] = "PHY 0 temp. sensor",
 	[MC_CMD_SENSOR_PHY0_COOLING] = "PHY 0 cooling",
 	[MC_CMD_SENSOR_PHY1_TEMP] = "PHY 1 temp. sensor",
 	[MC_CMD_SENSOR_PHY1_COOLING] = "PHY 1 cooling",
 	[MC_CMD_SENSOR_IN_1V0] = "1.0V supply sensor",
 	[MC_CMD_SENSOR_IN_1V2] = "1.2V supply sensor",
 	[MC_CMD_SENSOR_IN_1V8] = "1.8V supply sensor",
 	[MC_CMD_SENSOR_IN_2V5] = "2.5V supply sensor",
 	[MC_CMD_SENSOR_IN_3V3] = "3.3V supply sensor",
 	[MC_CMD_SENSOR_IN_12V0] = "12V supply sensor"
 };
 static const char *sensor_status_names[] = {
 	[MC_CMD_SENSOR_STATE_OK] = "OK",
 	[MC_CMD_SENSOR_STATE_WARNING] = "Warning",
 	[MC_CMD_SENSOR_STATE_FATAL] = "Fatal",
 	[MC_CMD_SENSOR_STATE_BROKEN] = "Device failure",
 };
 static void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev)
 {
 	unsigned int monitor, state, value;
 	const char *name, *state_txt;
 	monitor = EFX_QWORD_FIELD(*ev, MCDI_EVENT_SENSOREVT_MONITOR);
 	state = EFX_QWORD_FIELD(*ev, MCDI_EVENT_SENSOREVT_STATE);
 	value = EFX_QWORD_FIELD(*ev, MCDI_EVENT_SENSOREVT_VALUE);
 	/* Deal gracefully with the board having more drivers than we
 	 * know about, but do not expect new sensor states. */
 	name = (monitor >= ARRAY_SIZE(sensor_names))
 				    ? "No sensor name available" :
 				    sensor_names[monitor];
 	EFX_BUG_ON_PARANOID(state >= ARRAY_SIZE(sensor_status_names));
 	state_txt = sensor_status_names[state];
 	netif_err(efx, hw, efx->net_dev,
 		  "Sensor %d (%s) reports condition '%s' for raw value %d\n",
 		  monitor, name, state_txt, value);
 }
 /* Called from  falcon_process_eventq for MCDI events */
 void efx_mcdi_process_event(struct efx_channel *channel,
 			    efx_qword_t *event)
 {
 	struct efx_nic *efx = channel->efx;
 	int code = EFX_QWORD_FIELD(*event, MCDI_EVENT_CODE);
 	u32 data = EFX_QWORD_FIELD(*event, MCDI_EVENT_DATA);
 	switch (code) {
 	case MCDI_EVENT_CODE_BADSSERT:
 		netif_err(efx, hw, efx->net_dev,
 			  "MC watchdog or assertion failure at 0x%x\n", data);
 		efx_mcdi_ev_death(efx, EINTR);
 		break;
 	case MCDI_EVENT_CODE_PMNOTICE:
 		netif_info(efx, wol, efx->net_dev, "MCDI PM event.\n");
 		break;
 	case MCDI_EVENT_CODE_CMDDONE:
 		efx_mcdi_ev_cpl(efx,
 				MCDI_EVENT_FIELD(*event, CMDDONE_SEQ),
 				MCDI_EVENT_FIELD(*event, CMDDONE_DATALEN),
 				MCDI_EVENT_FIELD(*event, CMDDONE_ERRNO));
 		break;
 	case MCDI_EVENT_CODE_LINKCHANGE:
 		efx_mcdi_process_link_change(efx, event);
 		break;
 	case MCDI_EVENT_CODE_SENSOREVT:
 		efx_mcdi_sensor_event(efx, event);
 		break;
 	case MCDI_EVENT_CODE_SCHEDERR:
 		netif_info(efx, hw, efx->net_dev,
 			   "MC Scheduler error address=0x%x\n", data);
 		break;
 	case MCDI_EVENT_CODE_REBOOT:
 		netif_info(efx, hw, efx->net_dev, "MC Reboot\n");
 		efx_mcdi_ev_death(efx, EIO);
 		break;
 	case MCDI_EVENT_CODE_MAC_STATS_DMA:
 		/* MAC stats are gather lazily.  We can ignore this. */
 		break;
 	default:
 		netif_err(efx, hw, efx->net_dev, "Unknown MCDI event 0x%x\n",
 			  code);
 	}
 }
 /**************************************************************************
  *
  * Specific request functions
  *
  **************************************************************************
  */
 void efx_mcdi_print_fwver(struct efx_nic *efx, char *buf, size_t len)
 {
 	u8 outbuf[ALIGN(MC_CMD_GET_VERSION_V1_OUT_LEN, 4)];
 	size_t outlength;
 	const __le16 *ver_words;
 	int rc;
 	BUILD_BUG_ON(MC_CMD_GET_VERSION_IN_LEN != 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_GET_VERSION, NULL, 0,
 			  outbuf, sizeof(outbuf), &outlength);
 	if (rc)
 		goto fail;
 	if (outlength < MC_CMD_GET_VERSION_V1_OUT_LEN) {
 		rc = -EIO;
 		goto fail;
 	}
 	ver_words = (__le16 *)MCDI_PTR(outbuf, GET_VERSION_OUT_VERSION);
 	snprintf(buf, len, "%u.%u.%u.%u",
 		 le16_to_cpu(ver_words[0]), le16_to_cpu(ver_words[1]),
 		 le16_to_cpu(ver_words[2]), le16_to_cpu(ver_words[3]));
 	return;
 fail:
 	netif_err(efx, probe, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	buf[0] = 0;
 }
 int efx_mcdi_drv_attach(struct efx_nic *efx, bool driver_operating,
 			bool *was_attached)
 {
 	u8 inbuf[MC_CMD_DRV_ATTACH_IN_LEN];
 	u8 outbuf[MC_CMD_DRV_ATTACH_OUT_LEN];
 	size_t outlen;
 	int rc;
 	MCDI_SET_DWORD(inbuf, DRV_ATTACH_IN_NEW_STATE,
 		       driver_operating ? 1 : 0);
 	MCDI_SET_DWORD(inbuf, DRV_ATTACH_IN_UPDATE, 1);
 	rc = efx_mcdi_rpc(efx, MC_CMD_DRV_ATTACH, inbuf, sizeof(inbuf),
 			  outbuf, sizeof(outbuf), &outlen);
 	if (rc)
 		goto fail;
 	if (outlen < MC_CMD_DRV_ATTACH_OUT_LEN) {
 		rc = -EIO;
 		goto fail;
 	}
 	if (was_attached != NULL)
 		*was_attached = MCDI_DWORD(outbuf, DRV_ATTACH_OUT_OLD_STATE);
 	return 0;
 fail:
 	netif_err(efx, probe, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int efx_mcdi_get_board_cfg(struct efx_nic *efx, u8 *mac_address,
 			   u16 *fw_subtype_list)
 {
 	uint8_t outbuf[MC_CMD_GET_BOARD_CFG_OUT_LEN];
 	size_t outlen;
 	int port_num = efx_port_num(efx);
 	int offset;
 	int rc;
 	BUILD_BUG_ON(MC_CMD_GET_BOARD_CFG_IN_LEN != 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_GET_BOARD_CFG, NULL, 0,
 			  outbuf, sizeof(outbuf), &outlen);
 	if (rc)
 		goto fail;
 	if (outlen < MC_CMD_GET_BOARD_CFG_OUT_LEN) {
 		rc = -EIO;
 		goto fail;
 	}
 	offset = (port_num)
 		? MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT1_OFST
 		: MC_CMD_GET_BOARD_CFG_OUT_MAC_ADDR_BASE_PORT0_OFST;
 	if (mac_address)
 		memcpy(mac_address, outbuf + offset, ETH_ALEN);
 	if (fw_subtype_list)
 		memcpy(fw_subtype_list,
 		       outbuf + MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_OFST,
 		       MC_CMD_GET_BOARD_CFG_OUT_FW_SUBTYPE_LIST_LEN);
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d len=%d\n",
 		  __func__, rc, (int)outlen);
 	return rc;
 }
 int efx_mcdi_log_ctrl(struct efx_nic *efx, bool evq, bool uart, u32 dest_evq)
 {
 	u8 inbuf[MC_CMD_LOG_CTRL_IN_LEN];
 	u32 dest = 0;
 	int rc;
 	if (uart)
 		dest |= MC_CMD_LOG_CTRL_IN_LOG_DEST_UART;
 	if (evq)
 		dest |= MC_CMD_LOG_CTRL_IN_LOG_DEST_EVQ;
 	MCDI_SET_DWORD(inbuf, LOG_CTRL_IN_LOG_DEST, dest);
 	MCDI_SET_DWORD(inbuf, LOG_CTRL_IN_LOG_DEST_EVQ, dest_evq);
 	BUILD_BUG_ON(MC_CMD_LOG_CTRL_OUT_LEN != 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_LOG_CTRL, inbuf, sizeof(inbuf),
 			  NULL, 0, NULL);
 	if (rc)
 		goto fail;
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int efx_mcdi_nvram_types(struct efx_nic *efx, u32 *nvram_types_out)
 {
 	u8 outbuf[MC_CMD_NVRAM_TYPES_OUT_LEN];
 	size_t outlen;
 	int rc;
 	BUILD_BUG_ON(MC_CMD_NVRAM_TYPES_IN_LEN != 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_TYPES, NULL, 0,
 			  outbuf, sizeof(outbuf), &outlen);
 	if (rc)
 		goto fail;
 	if (outlen < MC_CMD_NVRAM_TYPES_OUT_LEN) {
 		rc = -EIO;
 		goto fail;
 	}
 	*nvram_types_out = MCDI_DWORD(outbuf, NVRAM_TYPES_OUT_TYPES);
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n",
 		  __func__, rc);
 	return rc;
 }
 int efx_mcdi_nvram_info(struct efx_nic *efx, unsigned int type,
 			size_t *size_out, size_t *erase_size_out,
 			bool *protected_out)
 {
 	u8 inbuf[MC_CMD_NVRAM_INFO_IN_LEN];
 	u8 outbuf[MC_CMD_NVRAM_INFO_OUT_LEN];
 	size_t outlen;
 	int rc;
 	MCDI_SET_DWORD(inbuf, NVRAM_INFO_IN_TYPE, type);
 	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_INFO, inbuf, sizeof(inbuf),
 			  outbuf, sizeof(outbuf), &outlen);
 	if (rc)
 		goto fail;
 	if (outlen < MC_CMD_NVRAM_INFO_OUT_LEN) {
 		rc = -EIO;
 		goto fail;
 	}
 	*size_out = MCDI_DWORD(outbuf, NVRAM_INFO_OUT_SIZE);
 	*erase_size_out = MCDI_DWORD(outbuf, NVRAM_INFO_OUT_ERASESIZE);
 	*protected_out = !!(MCDI_DWORD(outbuf, NVRAM_INFO_OUT_FLAGS) &
 				(1 << MC_CMD_NVRAM_PROTECTED_LBN));
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int efx_mcdi_nvram_update_start(struct efx_nic *efx, unsigned int type)
 {
 	u8 inbuf[MC_CMD_NVRAM_UPDATE_START_IN_LEN];
 	int rc;
 	MCDI_SET_DWORD(inbuf, NVRAM_UPDATE_START_IN_TYPE, type);
 	BUILD_BUG_ON(MC_CMD_NVRAM_UPDATE_START_OUT_LEN != 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_UPDATE_START, inbuf, sizeof(inbuf),
 			  NULL, 0, NULL);
 	if (rc)
 		goto fail;
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int efx_mcdi_nvram_read(struct efx_nic *efx, unsigned int type,
 			loff_t offset, u8 *buffer, size_t length)
 {
 	u8 inbuf[MC_CMD_NVRAM_READ_IN_LEN];
 	u8 outbuf[MC_CMD_NVRAM_READ_OUT_LEN(EFX_MCDI_NVRAM_LEN_MAX)];
 	size_t outlen;
 	int rc;
 	MCDI_SET_DWORD(inbuf, NVRAM_READ_IN_TYPE, type);
 	MCDI_SET_DWORD(inbuf, NVRAM_READ_IN_OFFSET, offset);
 	MCDI_SET_DWORD(inbuf, NVRAM_READ_IN_LENGTH, length);
 	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_READ, inbuf, sizeof(inbuf),
 			  outbuf, sizeof(outbuf), &outlen);
 	if (rc)
 		goto fail;
 	memcpy(buffer, MCDI_PTR(outbuf, NVRAM_READ_OUT_READ_BUFFER), length);
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int efx_mcdi_nvram_write(struct efx_nic *efx, unsigned int type,
 			   loff_t offset, const u8 *buffer, size_t length)
 {
 	u8 inbuf[MC_CMD_NVRAM_WRITE_IN_LEN(EFX_MCDI_NVRAM_LEN_MAX)];
 	int rc;
 	MCDI_SET_DWORD(inbuf, NVRAM_WRITE_IN_TYPE, type);
 	MCDI_SET_DWORD(inbuf, NVRAM_WRITE_IN_OFFSET, offset);
 	MCDI_SET_DWORD(inbuf, NVRAM_WRITE_IN_LENGTH, length);
 	memcpy(MCDI_PTR(inbuf, NVRAM_WRITE_IN_WRITE_BUFFER), buffer, length);
 	BUILD_BUG_ON(MC_CMD_NVRAM_WRITE_OUT_LEN != 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_WRITE, inbuf,
 			  ALIGN(MC_CMD_NVRAM_WRITE_IN_LEN(length), 4),
 			  NULL, 0, NULL);
 	if (rc)
 		goto fail;
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int efx_mcdi_nvram_erase(struct efx_nic *efx, unsigned int type,
 			 loff_t offset, size_t length)
 {
 	u8 inbuf[MC_CMD_NVRAM_ERASE_IN_LEN];
 	int rc;
 	MCDI_SET_DWORD(inbuf, NVRAM_ERASE_IN_TYPE, type);
 	MCDI_SET_DWORD(inbuf, NVRAM_ERASE_IN_OFFSET, offset);
 	MCDI_SET_DWORD(inbuf, NVRAM_ERASE_IN_LENGTH, length);
 	BUILD_BUG_ON(MC_CMD_NVRAM_ERASE_OUT_LEN != 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_ERASE, inbuf, sizeof(inbuf),
 			  NULL, 0, NULL);
 	if (rc)
 		goto fail;
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int efx_mcdi_nvram_update_finish(struct efx_nic *efx, unsigned int type)
 {
 	u8 inbuf[MC_CMD_NVRAM_UPDATE_FINISH_IN_LEN];
 	int rc;
 	MCDI_SET_DWORD(inbuf, NVRAM_UPDATE_FINISH_IN_TYPE, type);
 	BUILD_BUG_ON(MC_CMD_NVRAM_UPDATE_FINISH_OUT_LEN != 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_UPDATE_FINISH, inbuf, sizeof(inbuf),
 			  NULL, 0, NULL);
 	if (rc)
 		goto fail;
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 static int efx_mcdi_nvram_test(struct efx_nic *efx, unsigned int type)
 {
 	u8 inbuf[MC_CMD_NVRAM_TEST_IN_LEN];
 	u8 outbuf[MC_CMD_NVRAM_TEST_OUT_LEN];
 	int rc;
 	MCDI_SET_DWORD(inbuf, NVRAM_TEST_IN_TYPE, type);
 	rc = efx_mcdi_rpc(efx, MC_CMD_NVRAM_TEST, inbuf, sizeof(inbuf),
 			  outbuf, sizeof(outbuf), NULL);
 	if (rc)
 		return rc;
 	switch (MCDI_DWORD(outbuf, NVRAM_TEST_OUT_RESULT)) {
 	case MC_CMD_NVRAM_TEST_PASS:
 	case MC_CMD_NVRAM_TEST_NOTSUPP:
 		return 0;
 	default:
 		return -EIO;
 	}
 }
 int efx_mcdi_nvram_test_all(struct efx_nic *efx)
 {
 	u32 nvram_types;
 	unsigned int type;
 	int rc;
 	rc = efx_mcdi_nvram_types(efx, &nvram_types);
 	if (rc)
 		goto fail1;
 	type = 0;
 	while (nvram_types != 0) {
 		if (nvram_types & 1) {
 			rc = efx_mcdi_nvram_test(efx, type);
 			if (rc)
 				goto fail2;
 		}
 		type++;
 		nvram_types >>= 1;
 	}
 	return 0;
 fail2:
 	netif_err(efx, hw, efx->net_dev, "%s: failed type=%u\n",
 		  __func__, type);
 fail1:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 static int efx_mcdi_read_assertion(struct efx_nic *efx)
 {
 	u8 inbuf[MC_CMD_GET_ASSERTS_IN_LEN];
 	u8 outbuf[MC_CMD_GET_ASSERTS_OUT_LEN];
 	unsigned int flags, index, ofst;
 	const char *reason;
 	size_t outlen;
 	int retry;
 	int rc;
 	/* Attempt to read any stored assertion state before we reboot
 	 * the mcfw out of the assertion handler. Retry twice, once
 	 * because a boot-time assertion might cause this command to fail
 	 * with EINTR. And once again because GET_ASSERTS can race with
 	 * MC_CMD_REBOOT running on the other port. */
 	retry = 2;
 	do {
 		MCDI_SET_DWORD(inbuf, GET_ASSERTS_IN_CLEAR, 1);
 		rc = efx_mcdi_rpc(efx, MC_CMD_GET_ASSERTS,
 				  inbuf, MC_CMD_GET_ASSERTS_IN_LEN,
 				  outbuf, sizeof(outbuf), &outlen);
 	} while ((rc == -EINTR || rc == -EIO) && retry-- > 0);
 	if (rc)
 		return rc;
 	if (outlen < MC_CMD_GET_ASSERTS_OUT_LEN)
 		return -EIO;
 	/* Print out any recorded assertion state */
 	flags = MCDI_DWORD(outbuf, GET_ASSERTS_OUT_GLOBAL_FLAGS);
 	if (flags == MC_CMD_GET_ASSERTS_FLAGS_NO_FAILS)
 		return 0;
 	reason = (flags == MC_CMD_GET_ASSERTS_FLAGS_SYS_FAIL)
 		? "system-level assertion"
 		: (flags == MC_CMD_GET_ASSERTS_FLAGS_THR_FAIL)
 		? "thread-level assertion"
 		: (flags == MC_CMD_GET_ASSERTS_FLAGS_WDOG_FIRED)
 		? "watchdog reset"
 		: "unknown assertion";
 	netif_err(efx, hw, efx->net_dev,
 		  "MCPU %s at PC = 0x%.8x in thread 0x%.8x\n", reason,
 		  MCDI_DWORD(outbuf, GET_ASSERTS_OUT_SAVED_PC_OFFS),
 		  MCDI_DWORD(outbuf, GET_ASSERTS_OUT_THREAD_OFFS));
 	/* Print out the registers */
 	ofst = MC_CMD_GET_ASSERTS_OUT_GP_REGS_OFFS_OFST;
 	for (index = 1; index < 32; index++) {
 		netif_err(efx, hw, efx->net_dev, "R%.2d (?): 0x%.8x\n", index,
 			MCDI_DWORD2(outbuf, ofst));
 		ofst += sizeof(efx_dword_t);
 	}
 	return 0;
 }
 static void efx_mcdi_exit_assertion(struct efx_nic *efx)
 {
 	u8 inbuf[MC_CMD_REBOOT_IN_LEN];
 	/* Atomically reboot the mcfw out of the assertion handler */
 	BUILD_BUG_ON(MC_CMD_REBOOT_OUT_LEN != 0);
 	MCDI_SET_DWORD(inbuf, REBOOT_IN_FLAGS,
 		       MC_CMD_REBOOT_FLAGS_AFTER_ASSERTION);
 	efx_mcdi_rpc(efx, MC_CMD_REBOOT, inbuf, MC_CMD_REBOOT_IN_LEN,
 		     NULL, 0, NULL);
 }
 int efx_mcdi_handle_assertion(struct efx_nic *efx)
 {
 	int rc;
 	rc = efx_mcdi_read_assertion(efx);
 	if (rc)
 		return rc;
 	efx_mcdi_exit_assertion(efx);
 	return 0;
 }
 void efx_mcdi_set_id_led(struct efx_nic *efx, enum efx_led_mode mode)
 {
 	u8 inbuf[MC_CMD_SET_ID_LED_IN_LEN];
 	int rc;
 	BUILD_BUG_ON(EFX_LED_OFF != MC_CMD_LED_OFF);
 	BUILD_BUG_ON(EFX_LED_ON != MC_CMD_LED_ON);
 	BUILD_BUG_ON(EFX_LED_DEFAULT != MC_CMD_LED_DEFAULT);
 	BUILD_BUG_ON(MC_CMD_SET_ID_LED_OUT_LEN != 0);
 	MCDI_SET_DWORD(inbuf, SET_ID_LED_IN_STATE, mode);
 	rc = efx_mcdi_rpc(efx, MC_CMD_SET_ID_LED, inbuf, sizeof(inbuf),
 			  NULL, 0, NULL);
 	if (rc)
 		netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n",
 			  __func__, rc);
 }
 int efx_mcdi_reset_port(struct efx_nic *efx)
 {
 	int rc = efx_mcdi_rpc(efx, MC_CMD_PORT_RESET, NULL, 0, NULL, 0, NULL);
 	if (rc)
 		netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n",
 			  __func__, rc);
 	return rc;
 }
 int efx_mcdi_reset_mc(struct efx_nic *efx)
 {
 	u8 inbuf[MC_CMD_REBOOT_IN_LEN];
 	int rc;
 	BUILD_BUG_ON(MC_CMD_REBOOT_OUT_LEN != 0);
 	MCDI_SET_DWORD(inbuf, REBOOT_IN_FLAGS, 0);
 	rc = efx_mcdi_rpc(efx, MC_CMD_REBOOT, inbuf, sizeof(inbuf),
 			  NULL, 0, NULL);
 	/* White is black, and up is down */
 	if (rc == -EIO)
 		return 0;
 	if (rc == 0)
 		rc = -EIO;
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 static int efx_mcdi_wol_filter_set(struct efx_nic *efx, u32 type,
 				   const u8 *mac, int *id_out)
 {
 	u8 inbuf[MC_CMD_WOL_FILTER_SET_IN_LEN];
 	u8 outbuf[MC_CMD_WOL_FILTER_SET_OUT_LEN];
 	size_t outlen;
 	int rc;
 	MCDI_SET_DWORD(inbuf, WOL_FILTER_SET_IN_WOL_TYPE, type);
 	MCDI_SET_DWORD(inbuf, WOL_FILTER_SET_IN_FILTER_MODE,
 		       MC_CMD_FILTER_MODE_SIMPLE);
 	memcpy(MCDI_PTR(inbuf, WOL_FILTER_SET_IN_MAGIC_MAC), mac, ETH_ALEN);
 	rc = efx_mcdi_rpc(efx, MC_CMD_WOL_FILTER_SET, inbuf, sizeof(inbuf),
 			  outbuf, sizeof(outbuf), &outlen);
 	if (rc)
 		goto fail;
 	if (outlen < MC_CMD_WOL_FILTER_SET_OUT_LEN) {
 		rc = -EIO;
 		goto fail;
 	}
 	*id_out = (int)MCDI_DWORD(outbuf, WOL_FILTER_SET_OUT_FILTER_ID);
 	return 0;
 fail:
 	*id_out = -1;
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int
 efx_mcdi_wol_filter_set_magic(struct efx_nic *efx,  const u8 *mac, int *id_out)
 {
 	return efx_mcdi_wol_filter_set(efx, MC_CMD_WOL_TYPE_MAGIC, mac, id_out);
 }
 int efx_mcdi_wol_filter_get_magic(struct efx_nic *efx, int *id_out)
 {
 	u8 outbuf[MC_CMD_WOL_FILTER_GET_OUT_LEN];
 	size_t outlen;
 	int rc;
 	rc = efx_mcdi_rpc(efx, MC_CMD_WOL_FILTER_GET, NULL, 0,
 			  outbuf, sizeof(outbuf), &outlen);
 	if (rc)
 		goto fail;
 	if (outlen < MC_CMD_WOL_FILTER_GET_OUT_LEN) {
 		rc = -EIO;
 		goto fail;
 	}
 	*id_out = (int)MCDI_DWORD(outbuf, WOL_FILTER_GET_OUT_FILTER_ID);
 	return 0;
 fail:
 	*id_out = -1;
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int efx_mcdi_wol_filter_remove(struct efx_nic *efx, int id)
 {
 	u8 inbuf[MC_CMD_WOL_FILTER_REMOVE_IN_LEN];
 	int rc;
 	MCDI_SET_DWORD(inbuf, WOL_FILTER_REMOVE_IN_FILTER_ID, (u32)id);
 	rc = efx_mcdi_rpc(efx, MC_CMD_WOL_FILTER_REMOVE, inbuf, sizeof(inbuf),
 			  NULL, 0, NULL);
 	if (rc)
 		goto fail;
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);
 	return rc;
 }
 int efx_mcdi_wol_filter_reset(struct efx_nic *efx)
 {
 	int rc;
 	rc = efx_mcdi_rpc(efx, MC_CMD_WOL_FILTER_RESET, NULL, 0, NULL, 0, NULL);
 	if (rc)
 		goto fail;
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc);

drivers/net/sfc/nic.c

Diff comments View file @ 86c432c

 /****************************************************************************
  * Driver for Solarflare Solarstorm network controllers and boards
  * Copyright 2005-2006 Fen Systems Ltd.
  * Copyright 2006-2011 Solarflare Communications Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
  * by the Free Software Foundation, incorporated herein by reference.
  */
 #include <linux/bitops.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
 #include <linux/pci.h>
 #include <linux/module.h>
 #include <linux/seq_file.h>
 #include "net_driver.h"
 #include "bitfield.h"
 #include "efx.h"
 #include "nic.h"
 #include "regs.h"
 #include "io.h"
 #include "workarounds.h"
 /**************************************************************************
  *
  * Configurable values
  *
  **************************************************************************
  */
 /* This is set to 16 for a good reason.  In summary, if larger than
  * 16, the descriptor cache holds more than a default socket
  * buffer's worth of packets (for UDP we can only have at most one
  * socket buffer's worth outstanding).  This combined with the fact
  * that we only get 1 TX event per descriptor cache means the NIC
  * goes idle.
  */
 #define TX_DC_ENTRIES 16
 #define TX_DC_ENTRIES_ORDER 1
 #define RX_DC_ENTRIES 64
 #define RX_DC_ENTRIES_ORDER 3
 /* If EFX_MAX_INT_ERRORS internal errors occur within
  * EFX_INT_ERROR_EXPIRE seconds, we consider the NIC broken and
  * disable it.
  */
 #define EFX_INT_ERROR_EXPIRE 3600
 #define EFX_MAX_INT_ERRORS 5
 /* We poll for events every FLUSH_INTERVAL ms, and check FLUSH_POLL_COUNT times
  */
 #define EFX_FLUSH_INTERVAL 10
 #define EFX_FLUSH_POLL_COUNT 100
 /* Size and alignment of special buffers (4KB) */
 #define EFX_BUF_SIZE 4096
 /* Depth of RX flush request fifo */
 #define EFX_RX_FLUSH_COUNT 4
 /* Generated event code for efx_generate_test_event() */
 #define EFX_CHANNEL_MAGIC_TEST(_channel)	\
 	(0x00010100 + (_channel)->channel)
 /* Generated event code for efx_generate_fill_event() */
 #define EFX_CHANNEL_MAGIC_FILL(_channel)	\
 	(0x00010200 + (_channel)->channel)
 /**************************************************************************
  *
  * Solarstorm hardware access
  *
  **************************************************************************/
 static inline void efx_write_buf_tbl(struct efx_nic *efx, efx_qword_t *value,
 				     unsigned int index)
 {
 	efx_sram_writeq(efx, efx->membase + efx->type->buf_tbl_base,
 			value, index);
 }
 /* Read the current event from the event queue */
 static inline efx_qword_t *efx_event(struct efx_channel *channel,
 				     unsigned int index)
 {
 	return ((efx_qword_t *) (channel->eventq.addr)) +
 		(index & channel->eventq_mask);
 }
 /* See if an event is present
  *
  * We check both the high and low dword of the event for all ones.  We
  * wrote all ones when we cleared the event, and no valid event can
  * have all ones in either its high or low dwords.  This approach is
  * robust against reordering.
  *
  * Note that using a single 64-bit comparison is incorrect; even
  * though the CPU read will be atomic, the DMA write may not be.
  */
 static inline int efx_event_present(efx_qword_t *event)
 {
 	return !(EFX_DWORD_IS_ALL_ONES(event->dword[0]) |
 		  EFX_DWORD_IS_ALL_ONES(event->dword[1]));
 }
 static bool efx_masked_compare_oword(const efx_oword_t *a, const efx_oword_t *b,
 				     const efx_oword_t *mask)
 {
 	return ((a->u64[0] ^ b->u64[0]) & mask->u64[0]) ||
 		((a->u64[1] ^ b->u64[1]) & mask->u64[1]);
 }
 int efx_nic_test_registers(struct efx_nic *efx,
 			   const struct efx_nic_register_test *regs,
 			   size_t n_regs)
 {
 	unsigned address = 0, i, j;
 	efx_oword_t mask, imask, original, reg, buf;
 	/* Falcon should be in loopback to isolate the XMAC from the PHY */
 	WARN_ON(!LOOPBACK_INTERNAL(efx));
 	for (i = 0; i < n_regs; ++i) {
 		address = regs[i].address;
 		mask = imask = regs[i].mask;
 		EFX_INVERT_OWORD(imask);
 		efx_reado(efx, &original, address);
 		/* bit sweep on and off */
 		for (j = 0; j < 128; j++) {
 			if (!EFX_EXTRACT_OWORD32(mask, j, j))
 				continue;
 			/* Test this testable bit can be set in isolation */
 			EFX_AND_OWORD(reg, original, mask);
 			EFX_SET_OWORD32(reg, j, j, 1);
 			efx_writeo(efx, &reg, address);
 			efx_reado(efx, &buf, address);
 			if (efx_masked_compare_oword(&reg, &buf, &mask))
 				goto fail;
 			/* Test this testable bit can be cleared in isolation */
 			EFX_OR_OWORD(reg, original, mask);
 			EFX_SET_OWORD32(reg, j, j, 0);
 			efx_writeo(efx, &reg, address);
 			efx_reado(efx, &buf, address);
 			if (efx_masked_compare_oword(&reg, &buf, &mask))
 				goto fail;
 		}
 		efx_writeo(efx, &original, address);
 	}
 	return 0;
 fail:
 	netif_err(efx, hw, efx->net_dev,
 		  "wrote "EFX_OWORD_FMT" read "EFX_OWORD_FMT
 		  " at address 0x%x mask "EFX_OWORD_FMT"\n", EFX_OWORD_VAL(reg),
 		  EFX_OWORD_VAL(buf), address, EFX_OWORD_VAL(mask));
 	return -EIO;
 }
 /**************************************************************************
  *
  * Special buffer handling
  * Special buffers are used for event queues and the TX and RX
  * descriptor rings.
  *
  *************************************************************************/
 /*
  * Initialise a special buffer
  *
  * This will define a buffer (previously allocated via
  * efx_alloc_special_buffer()) in the buffer table, allowing
  * it to be used for event queues, descriptor rings etc.
  */
 static void
 efx_init_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 {
 	efx_qword_t buf_desc;
 	int index;
 	dma_addr_t dma_addr;
 	int i;
 	EFX_BUG_ON_PARANOID(!buffer->addr);
 	/* Write buffer descriptors to NIC */
 	for (i = 0; i < buffer->entries; i++) {
 		index = buffer->index + i;
 		dma_addr = buffer->dma_addr + (i * 4096);
 		netif_dbg(efx, probe, efx->net_dev,
 			  "mapping special buffer %d at %llx\n",
 			  index, (unsigned long long)dma_addr);
 		EFX_POPULATE_QWORD_3(buf_desc,
 				     FRF_AZ_BUF_ADR_REGION, 0,
 				     FRF_AZ_BUF_ADR_FBUF, dma_addr >> 12,
 				     FRF_AZ_BUF_OWNER_ID_FBUF, 0);
 		efx_write_buf_tbl(efx, &buf_desc, index);
 	}
 }
 /* Unmaps a buffer and clears the buffer table entries */
 static void
 efx_fini_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 {
 	efx_oword_t buf_tbl_upd;
 	unsigned int start = buffer->index;
 	unsigned int end = (buffer->index + buffer->entries - 1);
 	if (!buffer->entries)
 		return;
 	netif_dbg(efx, hw, efx->net_dev, "unmapping special buffers %d-%d\n",
 		  buffer->index, buffer->index + buffer->entries - 1);
 	EFX_POPULATE_OWORD_4(buf_tbl_upd,
 			     FRF_AZ_BUF_UPD_CMD, 0,
 			     FRF_AZ_BUF_CLR_CMD, 1,
 			     FRF_AZ_BUF_CLR_END_ID, end,
 			     FRF_AZ_BUF_CLR_START_ID, start);
 	efx_writeo(efx, &buf_tbl_upd, FR_AZ_BUF_TBL_UPD);
 }
 /*
  * Allocate a new special buffer
  *
  * This allocates memory for a new buffer, clears it and allocates a
  * new buffer ID range.  It does not write into the buffer table.
  *
  * This call will allocate 4KB buffers, since 8KB buffers can't be
  * used for event queues and descriptor rings.
  */
 static int efx_alloc_special_buffer(struct efx_nic *efx,
 				    struct efx_special_buffer *buffer,
 				    unsigned int len)
 {
 	len = ALIGN(len, EFX_BUF_SIZE);
 	buffer->addr = dma_alloc_coherent(&efx->pci_dev->dev, len,
 					  &buffer->dma_addr, GFP_KERNEL);
 	if (!buffer->addr)
 		return -ENOMEM;
 	buffer->len = len;
 	buffer->entries = len / EFX_BUF_SIZE;
 	BUG_ON(buffer->dma_addr & (EFX_BUF_SIZE - 1));
 	/* All zeros is a potentially valid event so memset to 0xff */
 	memset(buffer->addr, 0xff, len);
 	/* Select new buffer ID */
 	buffer->index = efx->next_buffer_table;
 	efx->next_buffer_table += buffer->entries;
 	netif_dbg(efx, probe, efx->net_dev,
 		  "allocating special buffers %d-%d at %llx+%x "
 		  "(virt %p phys %llx)\n", buffer->index,
 		  buffer->index + buffer->entries - 1,
 		  (u64)buffer->dma_addr, len,
 		  buffer->addr, (u64)virt_to_phys(buffer->addr));
 	return 0;
 }
 static void
 efx_free_special_buffer(struct efx_nic *efx, struct efx_special_buffer *buffer)
 {
 	if (!buffer->addr)
 		return;
 	netif_dbg(efx, hw, efx->net_dev,
 		  "deallocating special buffers %d-%d at %llx+%x "
 		  "(virt %p phys %llx)\n", buffer->index,
 		  buffer->index + buffer->entries - 1,
 		  (u64)buffer->dma_addr, buffer->len,
 		  buffer->addr, (u64)virt_to_phys(buffer->addr));
 	dma_free_coherent(&efx->pci_dev->dev, buffer->len, buffer->addr,
 			  buffer->dma_addr);
 	buffer->addr = NULL;
 	buffer->entries = 0;
 }
 /**************************************************************************
  *
  * Generic buffer handling
  * These buffers are used for interrupt status and MAC stats
  *
  **************************************************************************/
 int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer,
 			 unsigned int len)
 {
 	buffer->addr = pci_alloc_consistent(efx->pci_dev, len,
 					    &buffer->dma_addr);
 	if (!buffer->addr)
 		return -ENOMEM;
 	buffer->len = len;
 	memset(buffer->addr, 0, len);
 	return 0;
 }
 void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer)
 {
 	if (buffer->addr) {
 		pci_free_consistent(efx->pci_dev, buffer->len,
 				    buffer->addr, buffer->dma_addr);
 		buffer->addr = NULL;
 	}
 }
 /**************************************************************************
  *
  * TX path
  *
  **************************************************************************/
 /* Returns a pointer to the specified transmit descriptor in the TX
  * descriptor queue belonging to the specified channel.
  */
 static inline efx_qword_t *
 efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)
 {
 	return ((efx_qword_t *) (tx_queue->txd.addr)) + index;
 }
 /* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */
 static inline void efx_notify_tx_desc(struct efx_tx_queue *tx_queue)
 {
 	unsigned write_ptr;
 	efx_dword_t reg;
 	write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 	EFX_POPULATE_DWORD_1(reg, FRF_AZ_TX_DESC_WPTR_DWORD, write_ptr);
 	efx_writed_page(tx_queue->efx, &reg,
 			FR_AZ_TX_DESC_UPD_DWORD_P0, tx_queue->queue);
 }
 /* Write pointer and first descriptor for TX descriptor ring */
 static inline void efx_push_tx_desc(struct efx_tx_queue *tx_queue,
 				    const efx_qword_t *txd)
 {
 	unsigned write_ptr;
 	efx_oword_t reg;
 	BUILD_BUG_ON(FRF_AZ_TX_DESC_LBN != 0);
 	BUILD_BUG_ON(FR_AA_TX_DESC_UPD_KER != FR_BZ_TX_DESC_UPD_P0);
 	write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 	EFX_POPULATE_OWORD_2(reg, FRF_AZ_TX_DESC_PUSH_CMD, true,
 			     FRF_AZ_TX_DESC_WPTR, write_ptr);
 	reg.qword[0] = *txd;
 	efx_writeo_page(tx_queue->efx, &reg,
 			FR_BZ_TX_DESC_UPD_P0, tx_queue->queue);
 }
 static inline bool
 efx_may_push_tx_desc(struct efx_tx_queue *tx_queue, unsigned int write_count)
 {
 	unsigned empty_read_count = ACCESS_ONCE(tx_queue->empty_read_count);
 	if (empty_read_count == 0)
 		return false;
 	tx_queue->empty_read_count = 0;
 	return ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID) == 0;
 }
 /* For each entry inserted into the software descriptor ring, create a
  * descriptor in the hardware TX descriptor ring (in host memory), and
  * write a doorbell.
  */
 void efx_nic_push_buffers(struct efx_tx_queue *tx_queue)
 {
 	struct efx_tx_buffer *buffer;
 	efx_qword_t *txd;
 	unsigned write_ptr;
 	unsigned old_write_count = tx_queue->write_count;
 	BUG_ON(tx_queue->write_count == tx_queue->insert_count);
 	do {
 		write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 		buffer = &tx_queue->buffer[write_ptr];
 		txd = efx_tx_desc(tx_queue, write_ptr);
 		++tx_queue->write_count;
 		/* Create TX descriptor ring entry */
 		EFX_POPULATE_QWORD_4(*txd,
 				     FSF_AZ_TX_KER_CONT, buffer->continuation,
 				     FSF_AZ_TX_KER_BYTE_COUNT, buffer->len,
 				     FSF_AZ_TX_KER_BUF_REGION, 0,
 				     FSF_AZ_TX_KER_BUF_ADDR, buffer->dma_addr);
 	} while (tx_queue->write_count != tx_queue->insert_count);
 	wmb(); /* Ensure descriptors are written before they are fetched */
 	if (efx_may_push_tx_desc(tx_queue, old_write_count)) {
 		txd = efx_tx_desc(tx_queue,
 				  old_write_count & tx_queue->ptr_mask);
 		efx_push_tx_desc(tx_queue, txd);
 		++tx_queue->pushes;
 	} else {
 		efx_notify_tx_desc(tx_queue);
 	}
 }
 /* Allocate hardware resources for a TX queue */
 int efx_nic_probe_tx(struct efx_tx_queue *tx_queue)
 {
 	struct efx_nic *efx = tx_queue->efx;
 	unsigned entries;
 	entries = tx_queue->ptr_mask + 1;
 	return efx_alloc_special_buffer(efx, &tx_queue->txd,
 					entries * sizeof(efx_qword_t));
 }
 void efx_nic_init_tx(struct efx_tx_queue *tx_queue)
 {
 	struct efx_nic *efx = tx_queue->efx;
 	efx_oword_t reg;
 	tx_queue->flushed = FLUSH_NONE;
 	/* Pin TX descriptor ring */
 	efx_init_special_buffer(efx, &tx_queue->txd);
 	/* Push TX descriptor ring to card */
 	EFX_POPULATE_OWORD_10(reg,
 			      FRF_AZ_TX_DESCQ_EN, 1,
 			      FRF_AZ_TX_ISCSI_DDIG_EN, 0,
 			      FRF_AZ_TX_ISCSI_HDIG_EN, 0,
 			      FRF_AZ_TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index,
 			      FRF_AZ_TX_DESCQ_EVQ_ID,
 			      tx_queue->channel->channel,
 			      FRF_AZ_TX_DESCQ_OWNER_ID, 0,
 			      FRF_AZ_TX_DESCQ_LABEL, tx_queue->queue,
 			      FRF_AZ_TX_DESCQ_SIZE,
 			      __ffs(tx_queue->txd.entries),
 			      FRF_AZ_TX_DESCQ_TYPE, 0,
 			      FRF_BZ_TX_NON_IP_DROP_DIS, 1);
 	if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 		int csum = tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD;
 		EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_IP_CHKSM_DIS, !csum);
 		EFX_SET_OWORD_FIELD(reg, FRF_BZ_TX_TCP_CHKSM_DIS,
 				    !csum);
 	}
 	efx_writeo_table(efx, &reg, efx->type->txd_ptr_tbl_base,
 			 tx_queue->queue);
 	if (efx_nic_rev(efx) < EFX_REV_FALCON_B0) {
 		/* Only 128 bits in this register */
 		BUILD_BUG_ON(EFX_MAX_TX_QUEUES > 128);
 		efx_reado(efx, &reg, FR_AA_TX_CHKSM_CFG);
 		if (tx_queue->queue & EFX_TXQ_TYPE_OFFLOAD)
 			clear_bit_le(tx_queue->queue, (void *)&reg);
 		else
 			set_bit_le(tx_queue->queue, (void *)&reg);
 		efx_writeo(efx, &reg, FR_AA_TX_CHKSM_CFG);
 	}
 	if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 		EFX_POPULATE_OWORD_1(reg,
 				     FRF_BZ_TX_PACE,
 				     (tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ?
 				     FFE_BZ_TX_PACE_OFF :
 				     FFE_BZ_TX_PACE_RESERVED);
 		efx_writeo_table(efx, &reg, FR_BZ_TX_PACE_TBL,
 				 tx_queue->queue);
 	}
 }
 static void efx_flush_tx_queue(struct efx_tx_queue *tx_queue)
 {
 	struct efx_nic *efx = tx_queue->efx;
 	efx_oword_t tx_flush_descq;
 	tx_queue->flushed = FLUSH_PENDING;
 	/* Post a flush command */
 	EFX_POPULATE_OWORD_2(tx_flush_descq,
 			     FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
 			     FRF_AZ_TX_FLUSH_DESCQ, tx_queue->queue);
 	efx_writeo(efx, &tx_flush_descq, FR_AZ_TX_FLUSH_DESCQ);
 }
 void efx_nic_fini_tx(struct efx_tx_queue *tx_queue)
 {
 	struct efx_nic *efx = tx_queue->efx;
 	efx_oword_t tx_desc_ptr;
 	/* The queue should have been flushed */
 	WARN_ON(tx_queue->flushed != FLUSH_DONE);
 	/* Remove TX descriptor ring from card */
 	EFX_ZERO_OWORD(tx_desc_ptr);
 	efx_writeo_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base,
 			 tx_queue->queue);
 	/* Unpin TX descriptor ring */
 	efx_fini_special_buffer(efx, &tx_queue->txd);
 }
 /* Free buffers backing TX queue */
 void efx_nic_remove_tx(struct efx_tx_queue *tx_queue)
 {
 	efx_free_special_buffer(tx_queue->efx, &tx_queue->txd);
 }
 /**************************************************************************
  *
  * RX path
  *
  **************************************************************************/
 /* Returns a pointer to the specified descriptor in the RX descriptor queue */
 static inline efx_qword_t *
 efx_rx_desc(struct efx_rx_queue *rx_queue, unsigned int index)
 {
 	return ((efx_qword_t *) (rx_queue->rxd.addr)) + index;
 }
 /* This creates an entry in the RX descriptor queue */
 static inline void
 efx_build_rx_desc(struct efx_rx_queue *rx_queue, unsigned index)
 {
 	struct efx_rx_buffer *rx_buf;
 	efx_qword_t *rxd;
 	rxd = efx_rx_desc(rx_queue, index);
 	rx_buf = efx_rx_buffer(rx_queue, index);
 	EFX_POPULATE_QWORD_3(*rxd,
 			     FSF_AZ_RX_KER_BUF_SIZE,
 			     rx_buf->len -
 			     rx_queue->efx->type->rx_buffer_padding,
 			     FSF_AZ_RX_KER_BUF_REGION, 0,
 			     FSF_AZ_RX_KER_BUF_ADDR, rx_buf->dma_addr);
 }
 /* This writes to the RX_DESC_WPTR register for the specified receive
  * descriptor ring.
  */
 void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue)
 {
 	struct efx_nic *efx = rx_queue->efx;
 	efx_dword_t reg;
 	unsigned write_ptr;
 	while (rx_queue->notified_count != rx_queue->added_count) {
 		efx_build_rx_desc(
 			rx_queue,
 			rx_queue->notified_count & rx_queue->ptr_mask);
 		++rx_queue->notified_count;
 	}
 	wmb();
 	write_ptr = rx_queue->added_count & rx_queue->ptr_mask;
 	EFX_POPULATE_DWORD_1(reg, FRF_AZ_RX_DESC_WPTR_DWORD, write_ptr);
 	efx_writed_page(efx, &reg, FR_AZ_RX_DESC_UPD_DWORD_P0,
 			efx_rx_queue_index(rx_queue));
 }
 int efx_nic_probe_rx(struct efx_rx_queue *rx_queue)
 {
 	struct efx_nic *efx = rx_queue->efx;
 	unsigned entries;
 	entries = rx_queue->ptr_mask + 1;
 	return efx_alloc_special_buffer(efx, &rx_queue->rxd,
 					entries * sizeof(efx_qword_t));
 }
 void efx_nic_init_rx(struct efx_rx_queue *rx_queue)
 {
 	efx_oword_t rx_desc_ptr;
 	struct efx_nic *efx = rx_queue->efx;
 	bool is_b0 = efx_nic_rev(efx) >= EFX_REV_FALCON_B0;
 	bool iscsi_digest_en = is_b0;
 	netif_dbg(efx, hw, efx->net_dev,
 		  "RX queue %d ring in special buffers %d-%d\n",
 		  efx_rx_queue_index(rx_queue), rx_queue->rxd.index,
 		  rx_queue->rxd.index + rx_queue->rxd.entries - 1);
 	rx_queue->flushed = FLUSH_NONE;
 	/* Pin RX descriptor ring */
 	efx_init_special_buffer(efx, &rx_queue->rxd);
 	/* Push RX descriptor ring to card */
 	EFX_POPULATE_OWORD_10(rx_desc_ptr,
 			      FRF_AZ_RX_ISCSI_DDIG_EN, iscsi_digest_en,
 			      FRF_AZ_RX_ISCSI_HDIG_EN, iscsi_digest_en,
 			      FRF_AZ_RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index,
 			      FRF_AZ_RX_DESCQ_EVQ_ID,
 			      efx_rx_queue_channel(rx_queue)->channel,
 			      FRF_AZ_RX_DESCQ_OWNER_ID, 0,
 			      FRF_AZ_RX_DESCQ_LABEL,
 			      efx_rx_queue_index(rx_queue),
 			      FRF_AZ_RX_DESCQ_SIZE,
 			      __ffs(rx_queue->rxd.entries),
 			      FRF_AZ_RX_DESCQ_TYPE, 0 /* kernel queue */ ,
 			      /* For >=B0 this is scatter so disable */
 			      FRF_AZ_RX_DESCQ_JUMBO, !is_b0,
 			      FRF_AZ_RX_DESCQ_EN, 1);
 	efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 			 efx_rx_queue_index(rx_queue));
 }
 static void efx_flush_rx_queue(struct efx_rx_queue *rx_queue)
 {
 	struct efx_nic *efx = rx_queue->efx;
 	efx_oword_t rx_flush_descq;
 	rx_queue->flushed = FLUSH_PENDING;
 	/* Post a flush command */
 	EFX_POPULATE_OWORD_2(rx_flush_descq,
 			     FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
 			     FRF_AZ_RX_FLUSH_DESCQ,
 			     efx_rx_queue_index(rx_queue));
 	efx_writeo(efx, &rx_flush_descq, FR_AZ_RX_FLUSH_DESCQ);
 }
 void efx_nic_fini_rx(struct efx_rx_queue *rx_queue)
 {
 	efx_oword_t rx_desc_ptr;
 	struct efx_nic *efx = rx_queue->efx;
 	/* The queue should already have been flushed */
 	WARN_ON(rx_queue->flushed != FLUSH_DONE);
 	/* Remove RX descriptor ring from card */
 	EFX_ZERO_OWORD(rx_desc_ptr);
 	efx_writeo_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base,
 			 efx_rx_queue_index(rx_queue));
 	/* Unpin RX descriptor ring */
 	efx_fini_special_buffer(efx, &rx_queue->rxd);
 }
 /* Free buffers backing RX queue */
 void efx_nic_remove_rx(struct efx_rx_queue *rx_queue)
 {
 	efx_free_special_buffer(rx_queue->efx, &rx_queue->rxd);
 }
 /**************************************************************************
  *
  * Event queue processing
  * Event queues are processed by per-channel tasklets.
  *
  **************************************************************************/
 /* Update a channel's event queue's read pointer (RPTR) register
  *
  * This writes the EVQ_RPTR_REG register for the specified channel's
  * event queue.
  */
 void efx_nic_eventq_read_ack(struct efx_channel *channel)
 {
 	efx_dword_t reg;
 	struct efx_nic *efx = channel->efx;
 	EFX_POPULATE_DWORD_1(reg, FRF_AZ_EVQ_RPTR,
 			     channel->eventq_read_ptr & channel->eventq_mask);
 	efx_writed_table(efx, &reg, efx->type->evq_rptr_tbl_base,
 			 channel->channel);
 }
 /* Use HW to insert a SW defined event */
 static void efx_generate_event(struct efx_channel *channel, efx_qword_t *event)
 {
 	efx_oword_t drv_ev_reg;
 	BUILD_BUG_ON(FRF_AZ_DRV_EV_DATA_LBN != 0 ||
 		     FRF_AZ_DRV_EV_DATA_WIDTH != 64);
 	drv_ev_reg.u32[0] = event->u32[0];
 	drv_ev_reg.u32[1] = event->u32[1];
 	drv_ev_reg.u32[2] = 0;
 	drv_ev_reg.u32[3] = 0;
 	EFX_SET_OWORD_FIELD(drv_ev_reg, FRF_AZ_DRV_EV_QID, channel->channel);
 	efx_writeo(channel->efx, &drv_ev_reg, FR_AZ_DRV_EV);
 }
 /* Handle a transmit completion event
  *
  * The NIC batches TX completion events; the message we receive is of
  * the form "complete all TX events up to this index".
  */
 static int
 efx_handle_tx_event(struct efx_channel *channel, efx_qword_t *event)
 {
 	unsigned int tx_ev_desc_ptr;
 	unsigned int tx_ev_q_label;
 	struct efx_tx_queue *tx_queue;
 	struct efx_nic *efx = channel->efx;
 	int tx_packets = 0;
 	if (likely(EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_COMP))) {
 		/* Transmit completion */
 		tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_DESC_PTR);
 		tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 		tx_queue = efx_channel_get_tx_queue(
 			channel, tx_ev_q_label % EFX_TXQ_TYPES);
 		tx_packets = ((tx_ev_desc_ptr - tx_queue->read_count) &
 			      tx_queue->ptr_mask);
 		channel->irq_mod_score += tx_packets;
 		efx_xmit_done(tx_queue, tx_ev_desc_ptr);
 	} else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_WQ_FF_FULL)) {
 		/* Rewrite the FIFO write pointer */
 		tx_ev_q_label = EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_Q_LABEL);
 		tx_queue = efx_channel_get_tx_queue(
 			channel, tx_ev_q_label % EFX_TXQ_TYPES);
 		if (efx_dev_registered(efx))
 			netif_tx_lock(efx->net_dev);
 		efx_notify_tx_desc(tx_queue);
 		if (efx_dev_registered(efx))
 			netif_tx_unlock(efx->net_dev);
 	} else if (EFX_QWORD_FIELD(*event, FSF_AZ_TX_EV_PKT_ERR) &&
 		   EFX_WORKAROUND_10727(efx)) {
 		efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
 	} else {
 		netif_err(efx, tx_err, efx->net_dev,
 			  "channel %d unexpected TX event "
 			  EFX_QWORD_FMT"\n", channel->channel,
 			  EFX_QWORD_VAL(*event));
 	}
 	return tx_packets;
 }
 /* Detect errors included in the rx_evt_pkt_ok bit. */
 static void efx_handle_rx_not_ok(struct efx_rx_queue *rx_queue,
 				 const efx_qword_t *event,
 				 bool *rx_ev_pkt_ok,
 				 bool *discard)
 {
 	struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
 	struct efx_nic *efx = rx_queue->efx;
 	bool rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err;
 	bool rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err;
 	bool rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc;
 	bool rx_ev_other_err, rx_ev_pause_frm;
 	bool rx_ev_hdr_type, rx_ev_mcast_pkt;
 	unsigned rx_ev_pkt_type;
 	rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 	rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 	rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_TOBE_DISC);
 	rx_ev_pkt_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_TYPE);
 	rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event,
 						 FSF_AZ_RX_EV_BUF_OWNER_ID_ERR);
 	rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event,
 						  FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR);
 	rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event,
 						   FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR);
 	rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_ETH_CRC_ERR);
 	rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_FRM_TRUNC);
 	rx_ev_drib_nib = ((efx_nic_rev(efx) >= EFX_REV_FALCON_B0) ?
 			  0 : EFX_QWORD_FIELD(*event, FSF_AA_RX_EV_DRIB_NIB));
 	rx_ev_pause_frm = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PAUSE_FRM_ERR);
 	/* Every error apart from tobe_disc and pause_frm */
 	rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err |
 			   rx_ev_buf_owner_id_err | rx_ev_eth_crc_err |
 			   rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err);
 	/* Count errors that are not in MAC stats.  Ignore expected
 	 * checksum errors during self-test. */
 	if (rx_ev_frm_trunc)
 		++channel->n_rx_frm_trunc;
 	else if (rx_ev_tobe_disc)
 		++channel->n_rx_tobe_disc;
 	else if (!efx->loopback_selftest) {
 		if (rx_ev_ip_hdr_chksum_err)
 			++channel->n_rx_ip_hdr_chksum_err;
 		else if (rx_ev_tcp_udp_chksum_err)
 			++channel->n_rx_tcp_udp_chksum_err;
 	}
 	/* The frame must be discarded if any of these are true. */
 	*discard = (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib |
 		    rx_ev_tobe_disc | rx_ev_pause_frm);
 	/* TOBE_DISC is expected on unicast mismatches; don't print out an
 	 * error message.  FRM_TRUNC indicates RXDP dropped the packet due
 	 * to a FIFO overflow.
 	 */
 #ifdef EFX_ENABLE_DEBUG
 	if (rx_ev_other_err && net_ratelimit()) {
 		netif_dbg(efx, rx_err, efx->net_dev,
 			  " RX queue %d unexpected RX event "
 			  EFX_QWORD_FMT "%s%s%s%s%s%s%s%s\n",
 			  efx_rx_queue_index(rx_queue), EFX_QWORD_VAL(*event),
 			  rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "",
 			  rx_ev_ip_hdr_chksum_err ?
 			  " [IP_HDR_CHKSUM_ERR]" : "",
 			  rx_ev_tcp_udp_chksum_err ?
 			  " [TCP_UDP_CHKSUM_ERR]" : "",
 			  rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "",
 			  rx_ev_frm_trunc ? " [FRM_TRUNC]" : "",
 			  rx_ev_drib_nib ? " [DRIB_NIB]" : "",
 			  rx_ev_tobe_disc ? " [TOBE_DISC]" : "",
 			  rx_ev_pause_frm ? " [PAUSE]" : "");
 	}
 #endif
 }
 /* Handle receive events that are not in-order. */
 static void
 efx_handle_rx_bad_index(struct efx_rx_queue *rx_queue, unsigned index)
 {
 	struct efx_nic *efx = rx_queue->efx;
 	unsigned expected, dropped;
 	expected = rx_queue->removed_count & rx_queue->ptr_mask;
 	dropped = (index - expected) & rx_queue->ptr_mask;
 	netif_info(efx, rx_err, efx->net_dev,
 		   "dropped %d events (index=%d expected=%d)\n",
 		   dropped, index, expected);
 	efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ?
 			   RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE);
 }
 /* Handle a packet received event
  *
  * The NIC gives a "discard" flag if it's a unicast packet with the
  * wrong destination address
  * Also "is multicast" and "matches multicast filter" flags can be used to
  * discard non-matching multicast packets.
  */
 static void
 efx_handle_rx_event(struct efx_channel *channel, const efx_qword_t *event)
 {
 	unsigned int rx_ev_desc_ptr, rx_ev_byte_cnt;
 	unsigned int rx_ev_hdr_type, rx_ev_mcast_pkt;
 	unsigned expected_ptr;
 	bool rx_ev_pkt_ok, discard = false, checksummed;
 	struct efx_rx_queue *rx_queue;
 	/* Basic packet information */
 	rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_BYTE_CNT);
 	rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_PKT_OK);
 	rx_ev_hdr_type = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_HDR_TYPE);
 	WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_JUMBO_CONT));
 	WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_SOP) != 1);
 	WARN_ON(EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_Q_LABEL) !=
 		channel->channel);
 	rx_queue = efx_channel_get_rx_queue(channel);
 	rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_DESC_PTR);
 	expected_ptr = rx_queue->removed_count & rx_queue->ptr_mask;
 	if (unlikely(rx_ev_desc_ptr != expected_ptr))
 		efx_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr);
 	if (likely(rx_ev_pkt_ok)) {
 		/* If packet is marked as OK and packet type is TCP/IP or
 		 * UDP/IP, then we can rely on the hardware checksum.
 		 */
 		checksummed =
 			rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_TCP ||
 			rx_ev_hdr_type == FSE_CZ_RX_EV_HDR_TYPE_IPV4V6_UDP;
 	} else {
 		efx_handle_rx_not_ok(rx_queue, event, &rx_ev_pkt_ok, &discard);
 		checksummed = false;
 	}
 	/* Detect multicast packets that didn't match the filter */
 	rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_PKT);
 	if (rx_ev_mcast_pkt) {
 		unsigned int rx_ev_mcast_hash_match =
 			EFX_QWORD_FIELD(*event, FSF_AZ_RX_EV_MCAST_HASH_MATCH);
 		if (unlikely(!rx_ev_mcast_hash_match)) {
 			++channel->n_rx_mcast_mismatch;
 			discard = true;
 		}
 	}
 	channel->irq_mod_score += 2;
 	/* Handle received packet */
 	efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt,
 		      checksummed, discard);
 }
 static void
 efx_handle_generated_event(struct efx_channel *channel, efx_qword_t *event)
 {
 	struct efx_nic *efx = channel->efx;
 	unsigned code;
 	code = EFX_QWORD_FIELD(*event, FSF_AZ_DRV_GEN_EV_MAGIC);
 	if (code == EFX_CHANNEL_MAGIC_TEST(channel))
 		; /* ignore */
 	else if (code == EFX_CHANNEL_MAGIC_FILL(channel))
 		/* The queue must be empty, so we won't receive any rx
 		 * events, so efx_process_channel() won't refill the
 		 * queue. Refill it here */
 		efx_fast_push_rx_descriptors(efx_channel_get_rx_queue(channel));
 	else
 		netif_dbg(efx, hw, efx->net_dev, "channel %d received "
 			  "generated event "EFX_QWORD_FMT"\n",
 			  channel->channel, EFX_QWORD_VAL(*event));
 }
 static void
 efx_handle_driver_event(struct efx_channel *channel, efx_qword_t *event)
 {
 	struct efx_nic *efx = channel->efx;
 	unsigned int ev_sub_code;
 	unsigned int ev_sub_data;
 	ev_sub_code = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBCODE);
 	ev_sub_data = EFX_QWORD_FIELD(*event, FSF_AZ_DRIVER_EV_SUBDATA);
 	switch (ev_sub_code) {
 	case FSE_AZ_TX_DESCQ_FLS_DONE_EV:
 		netif_vdbg(efx, hw, efx->net_dev, "channel %d TXQ %d flushed\n",
 			   channel->channel, ev_sub_data);
 		break;
 	case FSE_AZ_RX_DESCQ_FLS_DONE_EV:
 		netif_vdbg(efx, hw, efx->net_dev, "channel %d RXQ %d flushed\n",
 			   channel->channel, ev_sub_data);
 		break;
 	case FSE_AZ_EVQ_INIT_DONE_EV:
 		netif_dbg(efx, hw, efx->net_dev,
 			  "channel %d EVQ %d initialised\n",
 			  channel->channel, ev_sub_data);
 		break;
 	case FSE_AZ_SRM_UPD_DONE_EV:
 		netif_vdbg(efx, hw, efx->net_dev,
 			   "channel %d SRAM update done\n", channel->channel);
 		break;
 	case FSE_AZ_WAKE_UP_EV:
 		netif_vdbg(efx, hw, efx->net_dev,
 			   "channel %d RXQ %d wakeup event\n",
 			   channel->channel, ev_sub_data);
 		break;
 	case FSE_AZ_TIMER_EV:
 		netif_vdbg(efx, hw, efx->net_dev,
 			   "channel %d RX queue %d timer expired\n",
 			   channel->channel, ev_sub_data);
 		break;
 	case FSE_AA_RX_RECOVER_EV:
 		netif_err(efx, rx_err, efx->net_dev,
 			  "channel %d seen DRIVER RX_RESET event. "
 			"Resetting.\n", channel->channel);
 		atomic_inc(&efx->rx_reset);
 		efx_schedule_reset(efx,
 				   EFX_WORKAROUND_6555(efx) ?
 				   RESET_TYPE_RX_RECOVERY :
 				   RESET_TYPE_DISABLE);
 		break;
 	case FSE_BZ_RX_DSC_ERROR_EV:
 		netif_err(efx, rx_err, efx->net_dev,
 			  "RX DMA Q %d reports descriptor fetch error."
 			  " RX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
 		efx_schedule_reset(efx, RESET_TYPE_RX_DESC_FETCH);
 		break;
 	case FSE_BZ_TX_DSC_ERROR_EV:
 		netif_err(efx, tx_err, efx->net_dev,
 			  "TX DMA Q %d reports descriptor fetch error."
 			  " TX Q %d is disabled.\n", ev_sub_data, ev_sub_data);
 		efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH);
 		break;
 	default:
 		netif_vdbg(efx, hw, efx->net_dev,
 			   "channel %d unknown driver event code %d "
 			   "data %04x\n", channel->channel, ev_sub_code,
 			   ev_sub_data);
 		break;
 	}
 }
 int efx_nic_process_eventq(struct efx_channel *channel, int budget)
 {
 	struct efx_nic *efx = channel->efx;
 	unsigned int read_ptr;
 	efx_qword_t event, *p_event;
 	int ev_code;
 	int tx_packets = 0;
 	int spent = 0;
 	read_ptr = channel->eventq_read_ptr;
 	for (;;) {
 		p_event = efx_event(channel, read_ptr);
 		event = *p_event;
 		if (!efx_event_present(&event))
 			/* End of events */
 			break;
 		netif_vdbg(channel->efx, intr, channel->efx->net_dev,
 			   "channel %d event is "EFX_QWORD_FMT"\n",
 			   channel->channel, EFX_QWORD_VAL(event));
 		/* Clear this event by marking it all ones */
 		EFX_SET_QWORD(*p_event);
 		++read_ptr;
 		ev_code = EFX_QWORD_FIELD(event, FSF_AZ_EV_CODE);
 		switch (ev_code) {
 		case FSE_AZ_EV_CODE_RX_EV:
 			efx_handle_rx_event(channel, &event);
 			if (++spent == budget)
 				goto out;
 			break;
 		case FSE_AZ_EV_CODE_TX_EV:
 			tx_packets += efx_handle_tx_event(channel, &event);
 			if (tx_packets > efx->txq_entries) {
 				spent = budget;
 				goto out;
 			}
 			break;
 		case FSE_AZ_EV_CODE_DRV_GEN_EV:
 			efx_handle_generated_event(channel, &event);
 			break;
 		case FSE_AZ_EV_CODE_DRIVER_EV:
 			efx_handle_driver_event(channel, &event);
 			break;
 		case FSE_CZ_EV_CODE_MCDI_EV:
 			efx_mcdi_process_event(channel, &event);
 			break;
 		case FSE_AZ_EV_CODE_GLOBAL_EV:
 			if (efx->type->handle_global_event &&
 			    efx->type->handle_global_event(channel, &event))
 				break;
 			/* else fall through */
 		default:
 			netif_err(channel->efx, hw, channel->efx->net_dev,
 				  "channel %d unknown event type %d (data "
 				  EFX_QWORD_FMT ")\n", channel->channel,
 				  ev_code, EFX_QWORD_VAL(event));
 		}
 	}
 out:
 	channel->eventq_read_ptr = read_ptr;
 	return spent;
 }
 /* Check whether an event is present in the eventq at the current
  * read pointer.  Only useful for self-test.
  */
 bool efx_nic_event_present(struct efx_channel *channel)
 {
 	return efx_event_present(efx_event(channel, channel->eventq_read_ptr));
 }
 /* Allocate buffer table entries for event queue */
 int efx_nic_probe_eventq(struct efx_channel *channel)
 {
 	struct efx_nic *efx = channel->efx;
 	unsigned entries;
 	entries = channel->eventq_mask + 1;
 	return efx_alloc_special_buffer(efx, &channel->eventq,
 					entries * sizeof(efx_qword_t));
 }
 void efx_nic_init_eventq(struct efx_channel *channel)
 {
 	efx_oword_t reg;
 	struct efx_nic *efx = channel->efx;
 	netif_dbg(efx, hw, efx->net_dev,
 		  "channel %d event queue in special buffers %d-%d\n",
 		  channel->channel, channel->eventq.index,
 		  channel->eventq.index + channel->eventq.entries - 1);
 	if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) {
 		EFX_POPULATE_OWORD_3(reg,
 				     FRF_CZ_TIMER_Q_EN, 1,
 				     FRF_CZ_HOST_NOTIFY_MODE, 0,
 				     FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS);
 		efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);
 	}
 	/* Pin event queue buffer */
 	efx_init_special_buffer(efx, &channel->eventq);
 	/* Fill event queue with all ones (i.e. empty events) */
 	memset(channel->eventq.addr, 0xff, channel->eventq.len);
 	/* Push event queue to card */
 	EFX_POPULATE_OWORD_3(reg,
 			     FRF_AZ_EVQ_EN, 1,
 			     FRF_AZ_EVQ_SIZE, __ffs(channel->eventq.entries),
 			     FRF_AZ_EVQ_BUF_BASE_ID, channel->eventq.index);
 	efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,
 			 channel->channel);
 	efx->type->push_irq_moderation(channel);
 }
 void efx_nic_fini_eventq(struct efx_channel *channel)
 {
 	efx_oword_t reg;
 	struct efx_nic *efx = channel->efx;
 	/* Remove event queue from card */
 	EFX_ZERO_OWORD(reg);
 	efx_writeo_table(efx, &reg, efx->type->evq_ptr_tbl_base,
 			 channel->channel);
 	if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
 		efx_writeo_table(efx, &reg, FR_BZ_TIMER_TBL, channel->channel);
 	/* Unpin event queue */
 	efx_fini_special_buffer(efx, &channel->eventq);
 }
 /* Free buffers backing event queue */
 void efx_nic_remove_eventq(struct efx_channel *channel)
 {
 	efx_free_special_buffer(channel->efx, &channel->eventq);
 }
 void efx_nic_generate_test_event(struct efx_channel *channel)
 {
 	unsigned int magic = EFX_CHANNEL_MAGIC_TEST(channel);
 	efx_qword_t test_event;
 	EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE,
 			     FSE_AZ_EV_CODE_DRV_GEN_EV,
 			     FSF_AZ_DRV_GEN_EV_MAGIC, magic);
 	efx_generate_event(channel, &test_event);
 }
 void efx_nic_generate_fill_event(struct efx_channel *channel)
 {
 	unsigned int magic = EFX_CHANNEL_MAGIC_FILL(channel);
 	efx_qword_t test_event;
 	EFX_POPULATE_QWORD_2(test_event, FSF_AZ_EV_CODE,
 			     FSE_AZ_EV_CODE_DRV_GEN_EV,
 			     FSF_AZ_DRV_GEN_EV_MAGIC, magic);
 	efx_generate_event(channel, &test_event);
 }
 /**************************************************************************
  *
  * Flush handling
  *
  **************************************************************************/
 static void efx_poll_flush_events(struct efx_nic *efx)
 {
 	struct efx_channel *channel = efx_get_channel(efx, 0);
 	struct efx_tx_queue *tx_queue;
 	struct efx_rx_queue *rx_queue;
 	unsigned int read_ptr = channel->eventq_read_ptr;
 	unsigned int end_ptr = read_ptr + channel->eventq_mask - 1;
 	do {
 		efx_qword_t *event = efx_event(channel, read_ptr);
 		int ev_code, ev_sub_code, ev_queue;
 		bool ev_failed;
 		if (!efx_event_present(event))
 			break;
 		ev_code = EFX_QWORD_FIELD(*event, FSF_AZ_EV_CODE);
 		ev_sub_code = EFX_QWORD_FIELD(*event,
 					      FSF_AZ_DRIVER_EV_SUBCODE);
 		if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV &&
 		    ev_sub_code == FSE_AZ_TX_DESCQ_FLS_DONE_EV) {
 			ev_queue = EFX_QWORD_FIELD(*event,
 						   FSF_AZ_DRIVER_EV_SUBDATA);
 			if (ev_queue < EFX_TXQ_TYPES * efx->n_tx_channels) {
 				tx_queue = efx_get_tx_queue(
 					efx, ev_queue / EFX_TXQ_TYPES,
 					ev_queue % EFX_TXQ_TYPES);
 				tx_queue->flushed = FLUSH_DONE;
 			}
 		} else if (ev_code == FSE_AZ_EV_CODE_DRIVER_EV &&
 			   ev_sub_code == FSE_AZ_RX_DESCQ_FLS_DONE_EV) {
 			ev_queue = EFX_QWORD_FIELD(
 				*event, FSF_AZ_DRIVER_EV_RX_DESCQ_ID);
 			ev_failed = EFX_QWORD_FIELD(
 				*event, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL);
 			if (ev_queue < efx->n_rx_channels) {
 				rx_queue = efx_get_rx_queue(efx, ev_queue);
 				rx_queue->flushed =
 					ev_failed ? FLUSH_FAILED : FLUSH_DONE;
 			}
 		}
 		/* We're about to destroy the queue anyway, so
 		 * it's ok to throw away every non-flush event */
 		EFX_SET_QWORD(*event);
 		++read_ptr;
 	} while (read_ptr != end_ptr);
 	channel->eventq_read_ptr = read_ptr;
 }
 /* Handle tx and rx flushes at the same time, since they run in
  * parallel in the hardware and there's no reason for us to
  * serialise them */
 int efx_nic_flush_queues(struct efx_nic *efx)
 {
 	struct efx_channel *channel;
 	struct efx_rx_queue *rx_queue;
 	struct efx_tx_queue *tx_queue;
 	int i, tx_pending, rx_pending;
 	/* If necessary prepare the hardware for flushing */
 	efx->type->prepare_flush(efx);
 	/* Flush all tx queues in parallel */
 	efx_for_each_channel(channel, efx) {
 		efx_for_each_possible_channel_tx_queue(tx_queue, channel) {
 			if (tx_queue->initialised)
 				efx_flush_tx_queue(tx_queue);
 		}
 	}
 	/* The hardware supports four concurrent rx flushes, each of which may
 	 * need to be retried if there is an outstanding descriptor fetch */
 	for (i = 0; i < EFX_FLUSH_POLL_COUNT; ++i) {
 		rx_pending = tx_pending = 0;
 		efx_for_each_channel(channel, efx) {
 			efx_for_each_channel_rx_queue(rx_queue, channel) {
 				if (rx_queue->flushed == FLUSH_PENDING)
 					++rx_pending;
 			}
 		}
 		efx_for_each_channel(channel, efx) {
 			efx_for_each_channel_rx_queue(rx_queue, channel) {
 				if (rx_pending == EFX_RX_FLUSH_COUNT)
 					break;
 				if (rx_queue->flushed == FLUSH_FAILED ||
 				    rx_queue->flushed == FLUSH_NONE) {
 					efx_flush_rx_queue(rx_queue);
 					++rx_pending;
 				}
 			}
 			efx_for_each_possible_channel_tx_queue(tx_queue, channel) {
 				if (tx_queue->initialised &&
 				    tx_queue->flushed != FLUSH_DONE)
 					++tx_pending;
 			}
 		}
 		if (rx_pending == 0 && tx_pending == 0)
 			return 0;
 		msleep(EFX_FLUSH_INTERVAL);
 		efx_poll_flush_events(efx);
 	}
 	/* Mark the queues as all flushed. We're going to return failure
 	 * leading to a reset, or fake up success anyway */
 	efx_for_each_channel(channel, efx) {
 		efx_for_each_possible_channel_tx_queue(tx_queue, channel) {
 			if (tx_queue->initialised &&
 			    tx_queue->flushed != FLUSH_DONE)
 				netif_err(efx, hw, efx->net_dev,
 					  "tx queue %d flush command timed out\n",
 					  tx_queue->queue);
 			tx_queue->flushed = FLUSH_DONE;
 		}
 		efx_for_each_channel_rx_queue(rx_queue, channel) {
 			if (rx_queue->flushed != FLUSH_DONE)
 				netif_err(efx, hw, efx->net_dev,
 					  "rx queue %d flush command timed out\n",
 					  efx_rx_queue_index(rx_queue));
 			rx_queue->flushed = FLUSH_DONE;
 		}
 	}
 	return -ETIMEDOUT;
 }
 /**************************************************************************
  *
  * Hardware interrupts
  * The hardware interrupt handler does very little work; all the event
  * queue processing is carried out by per-channel tasklets.
  *
  **************************************************************************/
 /* Enable/disable/generate interrupts */
 static inline void efx_nic_interrupts(struct efx_nic *efx,
 				      bool enabled, bool force)
 {
 	efx_oword_t int_en_reg_ker;
 	EFX_POPULATE_OWORD_3(int_en_reg_ker,
 			     FRF_AZ_KER_INT_LEVE_SEL, efx->fatal_irq_level,
 			     FRF_AZ_KER_INT_KER, force,
 			     FRF_AZ_DRV_INT_EN_KER, enabled);
 	efx_writeo(efx, &int_en_reg_ker, FR_AZ_INT_EN_KER);
 }
 void efx_nic_enable_interrupts(struct efx_nic *efx)
 {
 	struct efx_channel *channel;
 	EFX_ZERO_OWORD(*((efx_oword_t *) efx->irq_status.addr));
 	wmb(); /* Ensure interrupt vector is clear before interrupts enabled */
 	/* Enable interrupts */
 	efx_nic_interrupts(efx, true, false);
 	/* Force processing of all the channels to get the EVQ RPTRs up to
 	   date */
 	efx_for_each_channel(channel, efx)
 		efx_schedule_channel(channel);
 }
 void efx_nic_disable_interrupts(struct efx_nic *efx)
 {
 	/* Disable interrupts */
 	efx_nic_interrupts(efx, false, false);
 }
 /* Generate a test interrupt
  * Interrupt must already have been enabled, otherwise nasty things
  * may happen.
  */
 void efx_nic_generate_interrupt(struct efx_nic *efx)
 {
 	efx_nic_interrupts(efx, true, true);
 }
 /* Process a fatal interrupt
  * Disable bus mastering ASAP and schedule a reset
  */
 irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx)
 {
 	struct falcon_nic_data *nic_data = efx->nic_data;
 	efx_oword_t *int_ker = efx->irq_status.addr;
 	efx_oword_t fatal_intr;
 	int error, mem_perr;
 	efx_reado(efx, &fatal_intr, FR_AZ_FATAL_INTR_KER);
 	error = EFX_OWORD_FIELD(fatal_intr, FRF_AZ_FATAL_INTR);
 	netif_err(efx, hw, efx->net_dev, "SYSTEM ERROR "EFX_OWORD_FMT" status "
 		  EFX_OWORD_FMT ": %s\n", EFX_OWORD_VAL(*int_ker),
 		  EFX_OWORD_VAL(fatal_intr),
 		  error ? "disabling bus mastering" : "no recognised error");
 	/* If this is a memory parity error dump which blocks are offending */
 	mem_perr = (EFX_OWORD_FIELD(fatal_intr, FRF_AZ_MEM_PERR_INT_KER) ||
 		    EFX_OWORD_FIELD(fatal_intr, FRF_AZ_SRM_PERR_INT_KER));
 	if (mem_perr) {
 		efx_oword_t reg;
 		efx_reado(efx, &reg, FR_AZ_MEM_STAT);
 		netif_err(efx, hw, efx->net_dev,
 			  "SYSTEM ERROR: memory parity error "EFX_OWORD_FMT"\n",
 			  EFX_OWORD_VAL(reg));
 	}
 	/* Disable both devices */
 	pci_clear_master(efx->pci_dev);
 	if (efx_nic_is_dual_func(efx))
 		pci_clear_master(nic_data->pci_dev2);
 	efx_nic_disable_interrupts(efx);
 	/* Count errors and reset or disable the NIC accordingly */
 	if (efx->int_error_count == 0 ||
 	    time_after(jiffies, efx->int_error_expire)) {
 		efx->int_error_count = 0;
 		efx->int_error_expire =
 			jiffies + EFX_INT_ERROR_EXPIRE * HZ;
 	}
 	if (++efx->int_error_count < EFX_MAX_INT_ERRORS) {
 		netif_err(efx, hw, efx->net_dev,
 			  "SYSTEM ERROR - reset scheduled\n");
 		efx_schedule_reset(efx, RESET_TYPE_INT_ERROR);
 	} else {
 		netif_err(efx, hw, efx->net_dev,
 			  "SYSTEM ERROR - max number of errors seen."
 			  "NIC will be disabled\n");
 		efx_schedule_reset(efx, RESET_TYPE_DISABLE);
 	}
 	return IRQ_HANDLED;
 }
 /* Handle a legacy interrupt
  * Acknowledges the interrupt and schedule event queue processing.
  */
 static irqreturn_t efx_legacy_interrupt(int irq, void *dev_id)
 {
 	struct efx_nic *efx = dev_id;
 	efx_oword_t *int_ker = efx->irq_status.addr;
 	irqreturn_t result = IRQ_NONE;
 	struct efx_channel *channel;
 	efx_dword_t reg;
 	u32 queues;
 	int syserr;
 	/* Could this be ours?  If interrupts are disabled then the
 	 * channel state may not be valid.
 	 */
 	if (!efx->legacy_irq_enabled)
 		return result;
 	/* Read the ISR which also ACKs the interrupts */
 	efx_readd(efx, &reg, FR_BZ_INT_ISR0);
 	queues = EFX_EXTRACT_DWORD(reg, 0, 31);
 	/* Check to see if we have a serious error condition */
 	if (queues & (1U << efx->fatal_irq_level)) {
 		syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
 		if (unlikely(syserr))
 			return efx_nic_fatal_interrupt(efx);
 	}
 	if (queues != 0) {
 		if (EFX_WORKAROUND_15783(efx))
 			efx->irq_zero_count = 0;
 		/* Schedule processing of any interrupting queues */
 		efx_for_each_channel(channel, efx) {
 			if (queues & 1)
 				efx_schedule_channel(channel);
 			queues >>= 1;
 		}
 		result = IRQ_HANDLED;
 	} else if (EFX_WORKAROUND_15783(efx)) {
 		efx_qword_t *event;
 		/* We can't return IRQ_HANDLED more than once on seeing ISR=0
 		 * because this might be a shared interrupt. */
 		if (efx->irq_zero_count++ == 0)
 			result = IRQ_HANDLED;
 		/* Ensure we schedule or rearm all event queues */
 		efx_for_each_channel(channel, efx) {
 			event = efx_event(channel, channel->eventq_read_ptr);
 			if (efx_event_present(event))
 				efx_schedule_channel(channel);
 			else
 				efx_nic_eventq_read_ack(channel);
 		}
 	}
 	if (result == IRQ_HANDLED) {
 		efx->last_irq_cpu = raw_smp_processor_id();
 		netif_vdbg(efx, intr, efx->net_dev,
 			   "IRQ %d on CPU %d status " EFX_DWORD_FMT "\n",
 			   irq, raw_smp_processor_id(), EFX_DWORD_VAL(reg));
 	}
 	return result;
 }
 /* Handle an MSI interrupt
  *
  * Handle an MSI hardware interrupt.  This routine schedules event
  * queue processing.  No interrupt acknowledgement cycle is necessary.
  * Also, we never need to check that the interrupt is for us, since
  * MSI interrupts cannot be shared.
  */
 static irqreturn_t efx_msi_interrupt(int irq, void *dev_id)
 {
 	struct efx_channel *channel = *(struct efx_channel **)dev_id;
 	struct efx_nic *efx = channel->efx;
 	efx_oword_t *int_ker = efx->irq_status.addr;
 	int syserr;
 	efx->last_irq_cpu = raw_smp_processor_id();
 	netif_vdbg(efx, intr, efx->net_dev,
 		   "IRQ %d on CPU %d status " EFX_OWORD_FMT "\n",
 		   irq, raw_smp_processor_id(), EFX_OWORD_VAL(*int_ker));
 	/* Check to see if we have a serious error condition */
 	if (channel->channel == efx->fatal_irq_level) {
 		syserr = EFX_OWORD_FIELD(*int_ker, FSF_AZ_NET_IVEC_FATAL_INT);
 		if (unlikely(syserr))
 			return efx_nic_fatal_interrupt(efx);
 	}
 	/* Schedule processing of the channel */
 	efx_schedule_channel(channel);
 	return IRQ_HANDLED;
 }
 /* Setup RSS indirection table.
  * This maps from the hash value of the packet to RXQ
  */
 void efx_nic_push_rx_indir_table(struct efx_nic *efx)
 {
 	size_t i = 0;
 	efx_dword_t dword;
 	if (efx_nic_rev(efx) < EFX_REV_FALCON_B0)
 		return;
 	BUILD_BUG_ON(ARRAY_SIZE(efx->rx_indir_table) !=
 		     FR_BZ_RX_INDIRECTION_TBL_ROWS);
 	for (i = 0; i < FR_BZ_RX_INDIRECTION_TBL_ROWS; i++) {
 		EFX_POPULATE_DWORD_1(dword, FRF_BZ_IT_QUEUE,
 				     efx->rx_indir_table[i]);
 		efx_writed_table(efx, &dword, FR_BZ_RX_INDIRECTION_TBL, i);
 	}
 }
 /* Hook interrupt handler(s)
  * Try MSI and then legacy interrupts.
  */
 int efx_nic_init_interrupt(struct efx_nic *efx)
 {
 	struct efx_channel *channel;
 	int rc;
 	if (!EFX_INT_MODE_USE_MSI(efx)) {
 		irq_handler_t handler;
 		if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
 			handler = efx_legacy_interrupt;
 		else
 			handler = falcon_legacy_interrupt_a1;
 		rc = request_irq(efx->legacy_irq, handler, IRQF_SHARED,
 				 efx->name, efx);
 		if (rc) {
 			netif_err(efx, drv, efx->net_dev,
 				  "failed to hook legacy IRQ %d\n",
 				  efx->pci_dev->irq);
 			goto fail1;
 		}
 		return 0;
 	}
 	/* Hook MSI or MSI-X interrupt */
 	efx_for_each_channel(channel, efx) {
 		rc = request_irq(channel->irq, efx_msi_interrupt,
 				 IRQF_PROBE_SHARED, /* Not shared */
 				 efx->channel_name[channel->channel],
 				 &efx->channel[channel->channel]);
 		if (rc) {
 			netif_err(efx, drv, efx->net_dev,
 				  "failed to hook IRQ %d\n", channel->irq);
 			goto fail2;
 		}
 	}
 	return 0;
  fail2:
 	efx_for_each_channel(channel, efx)
 		free_irq(channel->irq, &efx->channel[channel->channel]);
  fail1:
 	return rc;
 }
 void efx_nic_fini_interrupt(struct efx_nic *efx)
 {
 	struct efx_channel *channel;
 	efx_oword_t reg;
 	/* Disable MSI/MSI-X interrupts */
 	efx_for_each_channel(channel, efx) {
 		if (channel->irq)
 			free_irq(channel->irq, &efx->channel[channel->channel]);
 	}
 	/* ACK legacy interrupt */
 	if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
 		efx_reado(efx, &reg, FR_BZ_INT_ISR0);
 	else
 		falcon_irq_ack_a1(efx);
 	/* Disable legacy interrupt */
 	if (efx->legacy_irq)
 		free_irq(efx->legacy_irq, efx);
 }
 u32 efx_nic_fpga_ver(struct efx_nic *efx)
 {
 	efx_oword_t altera_build;
 	efx_reado(efx, &altera_build, FR_AZ_ALTERA_BUILD);
 	return EFX_OWORD_FIELD(altera_build, FRF_AZ_ALTERA_BUILD_VER);
 }
 void efx_nic_init_common(struct efx_nic *efx)
 {
 	efx_oword_t temp;
 	/* Set positions of descriptor caches in SRAM. */
 	EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_TX_DC_BASE_ADR,
 			     efx->type->tx_dc_base / 8);
 	efx_writeo(efx, &temp, FR_AZ_SRM_TX_DC_CFG);
 	EFX_POPULATE_OWORD_1(temp, FRF_AZ_SRM_RX_DC_BASE_ADR,
 			     efx->type->rx_dc_base / 8);
 	efx_writeo(efx, &temp, FR_AZ_SRM_RX_DC_CFG);
 	/* Set TX descriptor cache size. */
 	BUILD_BUG_ON(TX_DC_ENTRIES != (8 << TX_DC_ENTRIES_ORDER));
 	EFX_POPULATE_OWORD_1(temp, FRF_AZ_TX_DC_SIZE, TX_DC_ENTRIES_ORDER);
 	efx_writeo(efx, &temp, FR_AZ_TX_DC_CFG);
 	/* Set RX descriptor cache size.  Set low watermark to size-8, as
 	 * this allows most efficient prefetching.
 	 */
 	BUILD_BUG_ON(RX_DC_ENTRIES != (8 << RX_DC_ENTRIES_ORDER));
 	EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_SIZE, RX_DC_ENTRIES_ORDER);
 	efx_writeo(efx, &temp, FR_AZ_RX_DC_CFG);
 	EFX_POPULATE_OWORD_1(temp, FRF_AZ_RX_DC_PF_LWM, RX_DC_ENTRIES - 8);
 	efx_writeo(efx, &temp, FR_AZ_RX_DC_PF_WM);
 	/* Program INT_KER address */
 	EFX_POPULATE_OWORD_2(temp,
 			     FRF_AZ_NORM_INT_VEC_DIS_KER,
 			     EFX_INT_MODE_USE_MSI(efx),
 			     FRF_AZ_INT_ADR_KER, efx->irq_status.dma_addr);
 	efx_writeo(efx, &temp, FR_AZ_INT_ADR_KER);
 	if (EFX_WORKAROUND_17213(efx) && !EFX_INT_MODE_USE_MSI(efx))
 		/* Use an interrupt level unused by event queues */
 		efx->fatal_irq_level = 0x1f;
 	else
 		/* Use a valid MSI-X vector */
 		efx->fatal_irq_level = 0;
 	/* Enable all the genuinely fatal interrupts.  (They are still
 	 * masked by the overall interrupt mask, controlled by
 	 * falcon_interrupts()).
 	 *
 	 * Note: All other fatal interrupts are enabled
 	 */
 	EFX_POPULATE_OWORD_3(temp,
 			     FRF_AZ_ILL_ADR_INT_KER_EN, 1,
 			     FRF_AZ_RBUF_OWN_INT_KER_EN, 1,
 			     FRF_AZ_TBUF_OWN_INT_KER_EN, 1);
 	if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0)
 		EFX_SET_OWORD_FIELD(temp, FRF_CZ_SRAM_PERR_INT_P_KER_EN, 1);
 	EFX_INVERT_OWORD(temp);
 	efx_writeo(efx, &temp, FR_AZ_FATAL_INTR_KER);
 	efx_nic_push_rx_indir_table(efx);
 	/* Disable the ugly timer-based TX DMA backoff and allow TX DMA to be
 	 * controlled by the RX FIFO fill level. Set arbitration to one pkt/Q.
 	 */
 	efx_reado(efx, &temp, FR_AZ_TX_RESERVED);
 	EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER, 0xfe);
 	EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_RX_SPACER_EN, 1);
 	EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
 	EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PUSH_EN, 1);
 	EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_DIS_NON_IP_EV, 1);
 	/* Enable SW_EV to inherit in char driver - assume harmless here */
 	EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_SOFT_EVT_EN, 1);
 	/* Prefetch threshold 2 => fetch when descriptor cache half empty */
 	EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_THRESHOLD, 2);
 	/* Disable hardware watchdog which can misfire */
 	EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
 	/* Squash TX of packets of 16 bytes or less */
 	if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0)
 		EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
 	efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);
 	if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) {
 		EFX_POPULATE_OWORD_4(temp,
 				     /* Default values */
 				     FRF_BZ_TX_PACE_SB_NOT_AF, 0x15,
 				     FRF_BZ_TX_PACE_SB_AF, 0xb,
 				     FRF_BZ_TX_PACE_FB_BASE, 0,
 				     /* Allow large pace values in the
 				      * fast bin. */
 				     FRF_BZ_TX_PACE_BIN_TH,
 				     FFE_BZ_TX_PACE_RESERVED);
 		efx_writeo(efx, &temp, FR_BZ_TX_PACE);
 	}
 }
 /* Register dump */
 #define REGISTER_REVISION_A	1
 #define REGISTER_REVISION_B	2
 #define REGISTER_REVISION_C	3
 #define REGISTER_REVISION_Z	3	/* latest revision */
 struct efx_nic_reg {
 	u32 offset:24;
 	u32 min_revision:2, max_revision:2;
 };
 #define REGISTER(name, min_rev, max_rev) {				\
 	FR_ ## min_rev ## max_rev ## _ ## name,				\
 	REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev	\
 }
 #define REGISTER_AA(name) REGISTER(name, A, A)
 #define REGISTER_AB(name) REGISTER(name, A, B)
 #define REGISTER_AZ(name) REGISTER(name, A, Z)
 #define REGISTER_BB(name) REGISTER(name, B, B)
 #define REGISTER_BZ(name) REGISTER(name, B, Z)
 #define REGISTER_CZ(name) REGISTER(name, C, Z)
 static const struct efx_nic_reg efx_nic_regs[] = {
 	REGISTER_AZ(ADR_REGION),
 	REGISTER_AZ(INT_EN_KER),
 	REGISTER_BZ(INT_EN_CHAR),
 	REGISTER_AZ(INT_ADR_KER),
 	REGISTER_BZ(INT_ADR_CHAR),
 	/* INT_ACK_KER is WO */
 	/* INT_ISR0 is RC */
 	REGISTER_AZ(HW_INIT),
 	REGISTER_CZ(USR_EV_CFG),
 	REGISTER_AB(EE_SPI_HCMD),
 	REGISTER_AB(EE_SPI_HADR),
 	REGISTER_AB(EE_SPI_HDATA),
 	REGISTER_AB(EE_BASE_PAGE),
 	REGISTER_AB(EE_VPD_CFG0),
 	/* EE_VPD_SW_CNTL and EE_VPD_SW_DATA are not used */
 	/* PMBX_DBG_IADDR and PBMX_DBG_IDATA are indirect */
 	/* PCIE_CORE_INDIRECT is indirect */
 	REGISTER_AB(NIC_STAT),
 	REGISTER_AB(GPIO_CTL),
 	REGISTER_AB(GLB_CTL),
 	/* FATAL_INTR_KER and FATAL_INTR_CHAR are partly RC */
 	REGISTER_BZ(DP_CTRL),
 	REGISTER_AZ(MEM_STAT),
 	REGISTER_AZ(CS_DEBUG),
 	REGISTER_AZ(ALTERA_BUILD),
 	REGISTER_AZ(CSR_SPARE),
 	REGISTER_AB(PCIE_SD_CTL0123),
 	REGISTER_AB(PCIE_SD_CTL45),
 	REGISTER_AB(PCIE_PCS_CTL_STAT),
 	/* DEBUG_DATA_OUT is not used */
 	/* DRV_EV is WO */
 	REGISTER_AZ(EVQ_CTL),
 	REGISTER_AZ(EVQ_CNT1),
 	REGISTER_AZ(EVQ_CNT2),
 	REGISTER_AZ(BUF_TBL_CFG),
 	REGISTER_AZ(SRM_RX_DC_CFG),
 	REGISTER_AZ(SRM_TX_DC_CFG),
 	REGISTER_AZ(SRM_CFG),
 	/* BUF_TBL_UPD is WO */
 	REGISTER_AZ(SRM_UPD_EVQ),
 	REGISTER_AZ(SRAM_PARITY),
 	REGISTER_AZ(RX_CFG),
 	REGISTER_BZ(RX_FILTER_CTL),
 	/* RX_FLUSH_DESCQ is WO */
 	REGISTER_AZ(RX_DC_CFG),
 	REGISTER_AZ(RX_DC_PF_WM),
 	REGISTER_BZ(RX_RSS_TKEY),
 	/* RX_NODESC_DROP is RC */
 	REGISTER_AA(RX_SELF_RST),
 	/* RX_DEBUG, RX_PUSH_DROP are not used */
 	REGISTER_CZ(RX_RSS_IPV6_REG1),
 	REGISTER_CZ(RX_RSS_IPV6_REG2),
 	REGISTER_CZ(RX_RSS_IPV6_REG3),
 	/* TX_FLUSH_DESCQ is WO */
 	REGISTER_AZ(TX_DC_CFG),
 	REGISTER_AA(TX_CHKSM_CFG),
 	REGISTER_AZ(TX_CFG),
 	/* TX_PUSH_DROP is not used */
 	REGISTER_AZ(TX_RESERVED),
 	REGISTER_BZ(TX_PACE),
 	/* TX_PACE_DROP_QID is RC */
 	REGISTER_BB(TX_VLAN),
 	REGISTER_BZ(TX_IPFIL_PORTEN),
 	REGISTER_AB(MD_TXD),
 	REGISTER_AB(MD_RXD),
 	REGISTER_AB(MD_CS),
 	REGISTER_AB(MD_PHY_ADR),
 	REGISTER_AB(MD_ID),
 	/* MD_STAT is RC */
 	REGISTER_AB(MAC_STAT_DMA),
 	REGISTER_AB(MAC_CTRL),
 	REGISTER_BB(GEN_MODE),
 	REGISTER_AB(MAC_MC_HASH_REG0),
 	REGISTER_AB(MAC_MC_HASH_REG1),
 	REGISTER_AB(GM_CFG1),
 	REGISTER_AB(GM_CFG2),
 	/* GM_IPG and GM_HD are not used */
 	REGISTER_AB(GM_MAX_FLEN),
 	/* GM_TEST is not used */
 	REGISTER_AB(GM_ADR1),
 	REGISTER_AB(GM_ADR2),
 	REGISTER_AB(GMF_CFG0),
 	REGISTER_AB(GMF_CFG1),
 	REGISTER_AB(GMF_CFG2),
 	REGISTER_AB(GMF_CFG3),
 	REGISTER_AB(GMF_CFG4),
 	REGISTER_AB(GMF_CFG5),
 	REGISTER_BB(TX_SRC_MAC_CTL),
 	REGISTER_AB(XM_ADR_LO),
 	REGISTER_AB(XM_ADR_HI),
 	REGISTER_AB(XM_GLB_CFG),
 	REGISTER_AB(XM_TX_CFG),
 	REGISTER_AB(XM_RX_CFG),
 	REGISTER_AB(XM_MGT_INT_MASK),
 	REGISTER_AB(XM_FC),
 	REGISTER_AB(XM_PAUSE_TIME),
 	REGISTER_AB(XM_TX_PARAM),
 	REGISTER_AB(XM_RX_PARAM),
 	/* XM_MGT_INT_MSK (note no 'A') is RC */
 	REGISTER_AB(XX_PWR_RST),
 	REGISTER_AB(XX_SD_CTL),
 	REGISTER_AB(XX_TXDRV_CTL),
 	/* XX_PRBS_CTL, XX_PRBS_CHK and XX_PRBS_ERR are not used */
 	/* XX_CORE_STAT is partly RC */
 };
 struct efx_nic_reg_table {
 	u32 offset:24;
 	u32 min_revision:2, max_revision:2;
 	u32 step:6, rows:21;
 };
 #define REGISTER_TABLE_DIMENSIONS(_, offset, min_rev, max_rev, step, rows) { \
 	offset,								\
 	REGISTER_REVISION_ ## min_rev, REGISTER_REVISION_ ## max_rev,	\
 	step, rows							\
 }
 #define REGISTER_TABLE(name, min_rev, max_rev) 				\
 	REGISTER_TABLE_DIMENSIONS(					\
 		name, FR_ ## min_rev ## max_rev ## _ ## name,		\
 		min_rev, max_rev,					\
 		FR_ ## min_rev ## max_rev ## _ ## name ## _STEP,	\
 		FR_ ## min_rev ## max_rev ## _ ## name ## _ROWS)
 #define REGISTER_TABLE_AA(name) REGISTER_TABLE(name, A, A)
 #define REGISTER_TABLE_AZ(name) REGISTER_TABLE(name, A, Z)
 #define REGISTER_TABLE_BB(name) REGISTER_TABLE(name, B, B)
 #define REGISTER_TABLE_BZ(name) REGISTER_TABLE(name, B, Z)
 #define REGISTER_TABLE_BB_CZ(name)					\
 	REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, B, B,		\
 				  FR_BZ_ ## name ## _STEP,		\
 				  FR_BB_ ## name ## _ROWS),		\
 	REGISTER_TABLE_DIMENSIONS(name, FR_BZ_ ## name, C, Z,		\
 				  FR_BZ_ ## name ## _STEP,		\
 				  FR_CZ_ ## name ## _ROWS)
 #define REGISTER_TABLE_CZ(name) REGISTER_TABLE(name, C, Z)
 static const struct efx_nic_reg_table efx_nic_reg_tables[] = {
 	/* DRIVER is not used */
 	/* EVQ_RPTR, TIMER_COMMAND, USR_EV and {RX,TX}_DESC_UPD are WO */
 	REGISTER_TABLE_BB(TX_IPFIL_TBL),
 	REGISTER_TABLE_BB(TX_SRC_MAC_TBL),
 	REGISTER_TABLE_AA(RX_DESC_PTR_TBL_KER),
 	REGISTER_TABLE_BB_CZ(RX_DESC_PTR_TBL),
 	REGISTER_TABLE_AA(TX_DESC_PTR_TBL_KER),
 	REGISTER_TABLE_BB_CZ(TX_DESC_PTR_TBL),
 	REGISTER_TABLE_AA(EVQ_PTR_TBL_KER),
 	REGISTER_TABLE_BB_CZ(EVQ_PTR_TBL),
 	/* We can't reasonably read all of the buffer table (up to 8MB!).
 	 * However this driver will only use a few entries.  Reading
 	 * 1K entries allows for some expansion of queue count and
 	 * size before we need to change the version. */
 	REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL_KER, FR_AA_BUF_FULL_TBL_KER,
 				  A, A, 8, 1024),
 	REGISTER_TABLE_DIMENSIONS(BUF_FULL_TBL, FR_BZ_BUF_FULL_TBL,
 				  B, Z, 8, 1024),
 	REGISTER_TABLE_CZ(RX_MAC_FILTER_TBL0),
 	REGISTER_TABLE_BB_CZ(TIMER_TBL),
 	REGISTER_TABLE_BB_CZ(TX_PACE_TBL),
 	REGISTER_TABLE_BZ(RX_INDIRECTION_TBL),
 	/* TX_FILTER_TBL0 is huge and not used by this driver */
 	REGISTER_TABLE_CZ(TX_MAC_FILTER_TBL0),
 	REGISTER_TABLE_CZ(MC_TREG_SMEM),
 	/* MSIX_PBA_TABLE is not mapped */
 	/* SRM_DBG is not mapped (and is redundant with BUF_FLL_TBL) */
 	REGISTER_TABLE_BZ(RX_FILTER_TBL0),
 };
 size_t efx_nic_get_regs_len(struct efx_nic *efx)
 {
 	const struct efx_nic_reg *reg;
 	const struct efx_nic_reg_table *table;
 	size_t len = 0;
 	for (reg = efx_nic_regs;
 	     reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
 	     reg++)
 		if (efx->type->revision >= reg->min_revision &&
 		    efx->type->revision <= reg->max_revision)
 			len += sizeof(efx_oword_t);
 	for (table = efx_nic_reg_tables;
 	     table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
 	     table++)
 		if (efx->type->revision >= table->min_revision &&
 		    efx->type->revision <= table->max_revision)
 			len += table->rows * min_t(size_t, table->step, 16);
 	return len;
 }
 void efx_nic_get_regs(struct efx_nic *efx, void *buf)
 {
 	const struct efx_nic_reg *reg;
 	const struct efx_nic_reg_table *table;
 	for (reg = efx_nic_regs;
 	     reg < efx_nic_regs + ARRAY_SIZE(efx_nic_regs);
 	     reg++) {
 		if (efx->type->revision >= reg->min_revision &&
 		    efx->type->revision <= reg->max_revision) {
 			efx_reado(efx, (efx_oword_t *)buf, reg->offset);
 			buf += sizeof(efx_oword_t);
 		}
 	}
 	for (table = efx_nic_reg_tables;
 	     table < efx_nic_reg_tables + ARRAY_SIZE(efx_nic_reg_tables);
 	     table++) {
 		size_t size, i;
 		if (!(efx->type->revision >= table->min_revision &&
 		      efx->type->revision <= table->max_revision))
 			continue;
 		size = min_t(size_t, table->step, 16);
-		if (table->offset >= efx->type->mem_map_size) {
-			/* No longer mapped; return dummy data */
-			memcpy(buf, "\xde\xc0\xad\xde", 4);
-			buf += table->rows * size;
-			continue;
-		}
 		for (i = 0; i < table->rows; i++) {
 			switch (table->step) {
 			case 4: /* 32-bit register or SRAM */
 				efx_readd_table(efx, buf, table->offset, i);
 				break;
 			case 8: /* 64-bit SRAM */
 				efx_sram_readq(efx,
 					       efx->membase + table->offset,
 					       buf, i);
 				break;
 			case 16: /* 128-bit register */
 				efx_reado_table(efx, buf, table->offset, i);
 				break;
 			case 32: /* 128-bit register, interleaved */
 				efx_reado_table(efx, buf, table->offset, 2 * i);
 				break;
 			default:
 				WARN_ON(1);
 				return;
 			}
 			buf += size;
 		}
 	}
 }

drivers/net/sfc/nic.h

Diff comments View file @ 86c432c

 /****************************************************************************
  * Driver for Solarflare Solarstorm network controllers and boards
  * Copyright 2005-2006 Fen Systems Ltd.
  * Copyright 2006-2011 Solarflare Communications Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
  * by the Free Software Foundation, incorporated herein by reference.
  */
 #ifndef EFX_NIC_H
 #define EFX_NIC_H
 #include <linux/i2c-algo-bit.h>
 #include "net_driver.h"
 #include "efx.h"
 #include "mcdi.h"
 #include "spi.h"
 /*
  * Falcon hardware control
  */
 enum {
 	EFX_REV_FALCON_A0 = 0,
 	EFX_REV_FALCON_A1 = 1,
 	EFX_REV_FALCON_B0 = 2,
 	EFX_REV_SIENA_A0 = 3,
 };
 static inline int efx_nic_rev(struct efx_nic *efx)
 {
 	return efx->type->revision;
 }
 extern u32 efx_nic_fpga_ver(struct efx_nic *efx);
 static inline bool efx_nic_has_mc(struct efx_nic *efx)
 {
 	return efx_nic_rev(efx) >= EFX_REV_SIENA_A0;
 }
 /* NIC has two interlinked PCI functions for the same port. */
 static inline bool efx_nic_is_dual_func(struct efx_nic *efx)
 {
 	return efx_nic_rev(efx) < EFX_REV_FALCON_B0;
 }
 enum {
 	PHY_TYPE_NONE = 0,
 	PHY_TYPE_TXC43128 = 1,
 	PHY_TYPE_88E1111 = 2,
 	PHY_TYPE_SFX7101 = 3,
 	PHY_TYPE_QT2022C2 = 4,
 	PHY_TYPE_PM8358 = 6,
 	PHY_TYPE_SFT9001A = 8,
 	PHY_TYPE_QT2025C = 9,
 	PHY_TYPE_SFT9001B = 10,
 };
 #define FALCON_XMAC_LOOPBACKS			\
 	((1 << LOOPBACK_XGMII) |		\
 	 (1 << LOOPBACK_XGXS) |			\
 	 (1 << LOOPBACK_XAUI))
 #define FALCON_GMAC_LOOPBACKS			\
 	(1 << LOOPBACK_GMAC)
 /**
  * struct falcon_board_type - board operations and type information
  * @id: Board type id, as found in NVRAM
  * @ref_model: Model number of Solarflare reference design
  * @gen_type: Generic board type description
  * @init: Allocate resources and initialise peripheral hardware
  * @init_phy: Do board-specific PHY initialisation
  * @fini: Shut down hardware and free resources
  * @set_id_led: Set state of identifying LED or revert to automatic function
  * @monitor: Board-specific health check function
  */
 struct falcon_board_type {
 	u8 id;
 	const char *ref_model;
 	const char *gen_type;
 	int (*init) (struct efx_nic *nic);
 	void (*init_phy) (struct efx_nic *efx);
 	void (*fini) (struct efx_nic *nic);
 	void (*set_id_led) (struct efx_nic *efx, enum efx_led_mode mode);
 	int (*monitor) (struct efx_nic *nic);
 };
 /**
  * struct falcon_board - board information
  * @type: Type of board
  * @major: Major rev. ('A', 'B' ...)
  * @minor: Minor rev. (0, 1, ...)
  * @i2c_adap: I2C adapter for on-board peripherals
  * @i2c_data: Data for bit-banging algorithm
  * @hwmon_client: I2C client for hardware monitor
  * @ioexp_client: I2C client for power/port control
  */
 struct falcon_board {
 	const struct falcon_board_type *type;
 	int major;
 	int minor;
 	struct i2c_adapter i2c_adap;
 	struct i2c_algo_bit_data i2c_data;
 	struct i2c_client *hwmon_client, *ioexp_client;
 };
 /**
  * struct falcon_nic_data - Falcon NIC state
  * @pci_dev2: Secondary function of Falcon A
  * @board: Board state and functions
  * @stats_disable_count: Nest count for disabling statistics fetches
  * @stats_pending: Is there a pending DMA of MAC statistics.
  * @stats_timer: A timer for regularly fetching MAC statistics.
  * @stats_dma_done: Pointer to the flag which indicates DMA completion.
  * @spi_flash: SPI flash device
  * @spi_eeprom: SPI EEPROM device
  * @spi_lock: SPI bus lock
  * @mdio_lock: MDIO bus lock
  * @xmac_poll_required: XMAC link state needs polling
  */
 struct falcon_nic_data {
 	struct pci_dev *pci_dev2;
 	struct falcon_board board;
 	unsigned int stats_disable_count;
 	bool stats_pending;
 	struct timer_list stats_timer;
 	u32 *stats_dma_done;
 	struct efx_spi_device spi_flash;
 	struct efx_spi_device spi_eeprom;
 	struct mutex spi_lock;
 	struct mutex mdio_lock;
 	bool xmac_poll_required;
 };
 static inline struct falcon_board *falcon_board(struct efx_nic *efx)
 {
 	struct falcon_nic_data *data = efx->nic_data;
 	return &data->board;
 }
 /**
  * struct siena_nic_data - Siena NIC state
  * @mcdi: Management-Controller-to-Driver Interface
- * @mcdi_smem: MCDI shared memory mapping. The mapping is always uncacheable.
  * @wol_filter_id: Wake-on-LAN packet filter id
  */
 struct siena_nic_data {
 	struct efx_mcdi_iface mcdi;
-	void __iomem *mcdi_smem;
 	int wol_filter_id;
 };
 extern const struct efx_nic_type falcon_a1_nic_type;
 extern const struct efx_nic_type falcon_b0_nic_type;
 extern const struct efx_nic_type siena_a0_nic_type;
 /**************************************************************************
  *
  * Externs
  *
  **************************************************************************
  */
 extern int falcon_probe_board(struct efx_nic *efx, u16 revision_info);
 /* TX data path */
 extern int efx_nic_probe_tx(struct efx_tx_queue *tx_queue);
 extern void efx_nic_init_tx(struct efx_tx_queue *tx_queue);
 extern void efx_nic_fini_tx(struct efx_tx_queue *tx_queue);
 extern void efx_nic_remove_tx(struct efx_tx_queue *tx_queue);
 extern void efx_nic_push_buffers(struct efx_tx_queue *tx_queue);
 /* RX data path */
 extern int efx_nic_probe_rx(struct efx_rx_queue *rx_queue);
 extern void efx_nic_init_rx(struct efx_rx_queue *rx_queue);
 extern void efx_nic_fini_rx(struct efx_rx_queue *rx_queue);
 extern void efx_nic_remove_rx(struct efx_rx_queue *rx_queue);
 extern void efx_nic_notify_rx_desc(struct efx_rx_queue *rx_queue);
 /* Event data path */
 extern int efx_nic_probe_eventq(struct efx_channel *channel);
 extern void efx_nic_init_eventq(struct efx_channel *channel);
 extern void efx_nic_fini_eventq(struct efx_channel *channel);
 extern void efx_nic_remove_eventq(struct efx_channel *channel);
 extern int efx_nic_process_eventq(struct efx_channel *channel, int rx_quota);
 extern void efx_nic_eventq_read_ack(struct efx_channel *channel);
 extern bool efx_nic_event_present(struct efx_channel *channel);
 /* MAC/PHY */
 extern void falcon_drain_tx_fifo(struct efx_nic *efx);
 extern void falcon_reconfigure_mac_wrapper(struct efx_nic *efx);
 /* Interrupts and test events */
 extern int efx_nic_init_interrupt(struct efx_nic *efx);
 extern void efx_nic_enable_interrupts(struct efx_nic *efx);
 extern void efx_nic_generate_test_event(struct efx_channel *channel);
 extern void efx_nic_generate_fill_event(struct efx_channel *channel);
 extern void efx_nic_generate_interrupt(struct efx_nic *efx);
 extern void efx_nic_disable_interrupts(struct efx_nic *efx);
 extern void efx_nic_fini_interrupt(struct efx_nic *efx);
 extern irqreturn_t efx_nic_fatal_interrupt(struct efx_nic *efx);
 extern irqreturn_t falcon_legacy_interrupt_a1(int irq, void *dev_id);
 extern void falcon_irq_ack_a1(struct efx_nic *efx);
 #define EFX_IRQ_MOD_RESOLUTION 5
 /* Global Resources */
 extern int efx_nic_flush_queues(struct efx_nic *efx);
 extern void falcon_start_nic_stats(struct efx_nic *efx);
 extern void falcon_stop_nic_stats(struct efx_nic *efx);
 extern void falcon_setup_xaui(struct efx_nic *efx);
 extern int falcon_reset_xaui(struct efx_nic *efx);
 extern void efx_nic_init_common(struct efx_nic *efx);
 extern void efx_nic_push_rx_indir_table(struct efx_nic *efx);
 int efx_nic_alloc_buffer(struct efx_nic *efx, struct efx_buffer *buffer,
 			 unsigned int len);
 void efx_nic_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer);
 /* Tests */
 struct efx_nic_register_test {
 	unsigned address;
 	efx_oword_t mask;
 };
 extern int efx_nic_test_registers(struct efx_nic *efx,
 				  const struct efx_nic_register_test *regs,
 				  size_t n_regs);
 extern size_t efx_nic_get_regs_len(struct efx_nic *efx);
 extern void efx_nic_get_regs(struct efx_nic *efx, void *buf);
 /**************************************************************************
  *
  * Falcon MAC stats
  *
  **************************************************************************
  */
 #define FALCON_STAT_OFFSET(falcon_stat) EFX_VAL(falcon_stat, offset)
 #define FALCON_STAT_WIDTH(falcon_stat) EFX_VAL(falcon_stat, WIDTH)
 /* Retrieve statistic from statistics block */
 #define FALCON_STAT(efx, falcon_stat, efx_stat) do {		\
 	if (FALCON_STAT_WIDTH(falcon_stat) == 16)		\
 		(efx)->mac_stats.efx_stat += le16_to_cpu(	\
 			*((__force __le16 *)				\
 			  (efx->stats_buffer.addr +		\
 			   FALCON_STAT_OFFSET(falcon_stat))));	\
 	else if (FALCON_STAT_WIDTH(falcon_stat) == 32)		\
 		(efx)->mac_stats.efx_stat += le32_to_cpu(	\
 			*((__force __le32 *)				\
 			  (efx->stats_buffer.addr +		\
 			   FALCON_STAT_OFFSET(falcon_stat))));	\
 	else							\
 		(efx)->mac_stats.efx_stat += le64_to_cpu(	\
 			*((__force __le64 *)				\
 			  (efx->stats_buffer.addr +		\
 			   FALCON_STAT_OFFSET(falcon_stat))));	\
 	} while (0)
 #define FALCON_MAC_STATS_SIZE 0x100
 #define MAC_DATA_LBN 0
 #define MAC_DATA_WIDTH 32
 extern void efx_nic_generate_event(struct efx_channel *channel,
 				   efx_qword_t *event);
 extern void falcon_poll_xmac(struct efx_nic *efx);
 #endif /* EFX_NIC_H */

drivers/net/sfc/siena.c

Diff comments View file @ 86c432c

 /****************************************************************************
  * Driver for Solarflare Solarstorm network controllers and boards
  * Copyright 2005-2006 Fen Systems Ltd.
  * Copyright 2006-2010 Solarflare Communications Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
  * by the Free Software Foundation, incorporated herein by reference.
  */
 #include <linux/bitops.h>
 #include <linux/delay.h>
 #include <linux/pci.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/random.h>
 #include "net_driver.h"
 #include "bitfield.h"
 #include "efx.h"
 #include "nic.h"
 #include "mac.h"
 #include "spi.h"
 #include "regs.h"
 #include "io.h"
 #include "phy.h"
 #include "workarounds.h"
 #include "mcdi.h"
 #include "mcdi_pcol.h"
 /* Hardware control for SFC9000 family including SFL9021 (aka Siena). */
 static void siena_init_wol(struct efx_nic *efx);
 static void siena_push_irq_moderation(struct efx_channel *channel)
 {
 	efx_dword_t timer_cmd;
 	if (channel->irq_moderation)
 		EFX_POPULATE_DWORD_2(timer_cmd,
 				     FRF_CZ_TC_TIMER_MODE,
 				     FFE_CZ_TIMER_MODE_INT_HLDOFF,
 				     FRF_CZ_TC_TIMER_VAL,
 				     channel->irq_moderation - 1);
 	else
 		EFX_POPULATE_DWORD_2(timer_cmd,
 				     FRF_CZ_TC_TIMER_MODE,
 				     FFE_CZ_TIMER_MODE_DIS,
 				     FRF_CZ_TC_TIMER_VAL, 0);
 	efx_writed_page_locked(channel->efx, &timer_cmd, FR_BZ_TIMER_COMMAND_P0,
 			       channel->channel);
 }
 static void siena_push_multicast_hash(struct efx_nic *efx)
 {
 	WARN_ON(!mutex_is_locked(&efx->mac_lock));
 	efx_mcdi_rpc(efx, MC_CMD_SET_MCAST_HASH,
 		     efx->multicast_hash.byte, sizeof(efx->multicast_hash),
 		     NULL, 0, NULL);
 }
 static int siena_mdio_write(struct net_device *net_dev,
 			    int prtad, int devad, u16 addr, u16 value)
 {
 	struct efx_nic *efx = netdev_priv(net_dev);
 	uint32_t status;
 	int rc;
 	rc = efx_mcdi_mdio_write(efx, efx->mdio_bus, prtad, devad,
 				 addr, value, &status);
 	if (rc)
 		return rc;
 	if (status != MC_CMD_MDIO_STATUS_GOOD)
 		return -EIO;
 	return 0;
 }
 static int siena_mdio_read(struct net_device *net_dev,
 			   int prtad, int devad, u16 addr)
 {
 	struct efx_nic *efx = netdev_priv(net_dev);
 	uint16_t value;
 	uint32_t status;
 	int rc;
 	rc = efx_mcdi_mdio_read(efx, efx->mdio_bus, prtad, devad,
 				addr, &value, &status);
 	if (rc)
 		return rc;
 	if (status != MC_CMD_MDIO_STATUS_GOOD)
 		return -EIO;
 	return (int)value;
 }
 /* This call is responsible for hooking in the MAC and PHY operations */
 static int siena_probe_port(struct efx_nic *efx)
 {
 	int rc;
 	/* Hook in PHY operations table */
 	efx->phy_op = &efx_mcdi_phy_ops;
 	/* Set up MDIO structure for PHY */
 	efx->mdio.mode_support = MDIO_SUPPORTS_C45 | MDIO_EMULATE_C22;
 	efx->mdio.mdio_read = siena_mdio_read;
 	efx->mdio.mdio_write = siena_mdio_write;
 	/* Fill out MDIO structure, loopback modes, and initial link state */
 	rc = efx->phy_op->probe(efx);
 	if (rc != 0)
 		return rc;
 	/* Allocate buffer for stats */
 	rc = efx_nic_alloc_buffer(efx, &efx->stats_buffer,
 				  MC_CMD_MAC_NSTATS * sizeof(u64));
 	if (rc)
 		return rc;
 	netif_dbg(efx, probe, efx->net_dev,
 		  "stats buffer at %llx (virt %p phys %llx)\n",
 		  (u64)efx->stats_buffer.dma_addr,
 		  efx->stats_buffer.addr,
 		  (u64)virt_to_phys(efx->stats_buffer.addr));
 	efx_mcdi_mac_stats(efx, efx->stats_buffer.dma_addr, 0, 0, 1);
 	return 0;
 }
 static void siena_remove_port(struct efx_nic *efx)
 {
 	efx->phy_op->remove(efx);
 	efx_nic_free_buffer(efx, &efx->stats_buffer);
 }
 static const struct efx_nic_register_test siena_register_tests[] = {
 	{ FR_AZ_ADR_REGION,
 	  EFX_OWORD32(0x0003FFFF, 0x0003FFFF, 0x0003FFFF, 0x0003FFFF) },
 	{ FR_CZ_USR_EV_CFG,
 	  EFX_OWORD32(0x000103FF, 0x00000000, 0x00000000, 0x00000000) },
 	{ FR_AZ_RX_CFG,
 	  EFX_OWORD32(0xFFFFFFFE, 0xFFFFFFFF, 0x0003FFFF, 0x00000000) },
 	{ FR_AZ_TX_CFG,
 	  EFX_OWORD32(0x7FFF0037, 0xFFFF8000, 0xFFFFFFFF, 0x03FFFFFF) },
 	{ FR_AZ_TX_RESERVED,
 	  EFX_OWORD32(0xFFFEFE80, 0x1FFFFFFF, 0x020000FE, 0x007FFFFF) },
 	{ FR_AZ_SRM_TX_DC_CFG,
 	  EFX_OWORD32(0x001FFFFF, 0x00000000, 0x00000000, 0x00000000) },
 	{ FR_AZ_RX_DC_CFG,
 	  EFX_OWORD32(0x00000003, 0x00000000, 0x00000000, 0x00000000) },
 	{ FR_AZ_RX_DC_PF_WM,
 	  EFX_OWORD32(0x000003FF, 0x00000000, 0x00000000, 0x00000000) },
 	{ FR_BZ_DP_CTRL,
 	  EFX_OWORD32(0x00000FFF, 0x00000000, 0x00000000, 0x00000000) },
 	{ FR_BZ_RX_RSS_TKEY,
 	  EFX_OWORD32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF) },
 	{ FR_CZ_RX_RSS_IPV6_REG1,
 	  EFX_OWORD32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF) },
 	{ FR_CZ_RX_RSS_IPV6_REG2,
 	  EFX_OWORD32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF) },
 	{ FR_CZ_RX_RSS_IPV6_REG3,
 	  EFX_OWORD32(0xFFFFFFFF, 0xFFFFFFFF, 0x00000007, 0x00000000) },
 };
 static int siena_test_registers(struct efx_nic *efx)
 {
 	return efx_nic_test_registers(efx, siena_register_tests,
 				      ARRAY_SIZE(siena_register_tests));
 }
 /**************************************************************************
  *
  * Device reset
  *
  **************************************************************************
  */
 static enum reset_type siena_map_reset_reason(enum reset_type reason)
 {
 	return RESET_TYPE_ALL;
 }
 static int siena_map_reset_flags(u32 *flags)
 {
 	enum {
 		SIENA_RESET_PORT = (ETH_RESET_DMA | ETH_RESET_FILTER |
 				    ETH_RESET_OFFLOAD | ETH_RESET_MAC |
 				    ETH_RESET_PHY),
 		SIENA_RESET_MC = (SIENA_RESET_PORT |
 				  ETH_RESET_MGMT << ETH_RESET_SHARED_SHIFT),
 	};
 	if ((*flags & SIENA_RESET_MC) == SIENA_RESET_MC) {
 		*flags &= ~SIENA_RESET_MC;
 		return RESET_TYPE_WORLD;
 	}
 	if ((*flags & SIENA_RESET_PORT) == SIENA_RESET_PORT) {
 		*flags &= ~SIENA_RESET_PORT;
 		return RESET_TYPE_ALL;
 	}
 	/* no invisible reset implemented */
 	return -EINVAL;
 }
 static int siena_reset_hw(struct efx_nic *efx, enum reset_type method)
 {
 	int rc;
 	/* Recover from a failed assertion pre-reset */
 	rc = efx_mcdi_handle_assertion(efx);
 	if (rc)
 		return rc;
 	if (method == RESET_TYPE_WORLD)
 		return efx_mcdi_reset_mc(efx);
 	else
 		return efx_mcdi_reset_port(efx);
 }
 static int siena_probe_nvconfig(struct efx_nic *efx)
 {
 	return efx_mcdi_get_board_cfg(efx, efx->net_dev->perm_addr, NULL);
 }
 static int siena_probe_nic(struct efx_nic *efx)
 {
 	struct siena_nic_data *nic_data;
 	bool already_attached = 0;
 	efx_oword_t reg;
 	int rc;
 	/* Allocate storage for hardware specific data */
 	nic_data = kzalloc(sizeof(struct siena_nic_data), GFP_KERNEL);
 	if (!nic_data)
 		return -ENOMEM;
 	efx->nic_data = nic_data;
 	if (efx_nic_fpga_ver(efx) != 0) {
 		netif_err(efx, probe, efx->net_dev,
 			  "Siena FPGA not supported\n");
 		rc = -ENODEV;
 		goto fail1;
 	}
 	efx_reado(efx, &reg, FR_AZ_CS_DEBUG);
 	efx->net_dev->dev_id = EFX_OWORD_FIELD(reg, FRF_CZ_CS_PORT_NUM) - 1;
-	/* Initialise MCDI */
-	nic_data->mcdi_smem = ioremap_nocache(efx->membase_phys +
-					      FR_CZ_MC_TREG_SMEM,
-					      FR_CZ_MC_TREG_SMEM_STEP *
-					      FR_CZ_MC_TREG_SMEM_ROWS);
-	if (!nic_data->mcdi_smem) {
-		netif_err(efx, probe, efx->net_dev,
-			  "could not map MCDI at %llx+%x\n",
-			  (unsigned long long)efx->membase_phys +
-			  FR_CZ_MC_TREG_SMEM,
-			  FR_CZ_MC_TREG_SMEM_STEP * FR_CZ_MC_TREG_SMEM_ROWS);
-		rc = -ENOMEM;
-		goto fail1;
-	}
 	efx_mcdi_init(efx);
 	/* Recover from a failed assertion before probing */
 	rc = efx_mcdi_handle_assertion(efx);
 	if (rc)
-		goto fail2;
+		goto fail1;
 	/* Let the BMC know that the driver is now in charge of link and
 	 * filter settings. We must do this before we reset the NIC */
 	rc = efx_mcdi_drv_attach(efx, true, &already_attached);
 	if (rc) {
 		netif_err(efx, probe, efx->net_dev,
 			  "Unable to register driver with MCPU\n");
 		goto fail2;
 	}
 	if (already_attached)
 		/* Not a fatal error */
 		netif_err(efx, probe, efx->net_dev,
 			  "Host already registered with MCPU\n");
 	/* Now we can reset the NIC */
 	rc = siena_reset_hw(efx, RESET_TYPE_ALL);
 	if (rc) {
 		netif_err(efx, probe, efx->net_dev, "failed to reset NIC\n");
 		goto fail3;
 	}
 	siena_init_wol(efx);
 	/* Allocate memory for INT_KER */
 	rc = efx_nic_alloc_buffer(efx, &efx->irq_status, sizeof(efx_oword_t));
 	if (rc)
 		goto fail4;
 	BUG_ON(efx->irq_status.dma_addr & 0x0f);
 	netif_dbg(efx, probe, efx->net_dev,
 		  "INT_KER at %llx (virt %p phys %llx)\n",
 		  (unsigned long long)efx->irq_status.dma_addr,
 		  efx->irq_status.addr,
 		  (unsigned long long)virt_to_phys(efx->irq_status.addr));
 	/* Read in the non-volatile configuration */
 	rc = siena_probe_nvconfig(efx);
 	if (rc == -EINVAL) {
 		netif_err(efx, probe, efx->net_dev,
 			  "NVRAM is invalid therefore using defaults\n");
 		efx->phy_type = PHY_TYPE_NONE;
 		efx->mdio.prtad = MDIO_PRTAD_NONE;
 	} else if (rc) {
 		goto fail5;
 	}
 	return 0;
 fail5:
 	efx_nic_free_buffer(efx, &efx->irq_status);
 fail4:
 fail3:
 	efx_mcdi_drv_attach(efx, false, NULL);
 fail2:
-	iounmap(nic_data->mcdi_smem);
 fail1:
 	kfree(efx->nic_data);
 	return rc;
 }
 /* This call performs hardware-specific global initialisation, such as
  * defining the descriptor cache sizes and number of RSS channels.
  * It does not set up any buffers, descriptor rings or event queues.
  */
 static int siena_init_nic(struct efx_nic *efx)
 {
 	efx_oword_t temp;
 	int rc;
 	/* Recover from a failed assertion post-reset */
 	rc = efx_mcdi_handle_assertion(efx);
 	if (rc)
 		return rc;
 	/* Squash TX of packets of 16 bytes or less */
 	efx_reado(efx, &temp, FR_AZ_TX_RESERVED);
 	EFX_SET_OWORD_FIELD(temp, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
 	efx_writeo(efx, &temp, FR_AZ_TX_RESERVED);
 	/* Do not enable TX_NO_EOP_DISC_EN, since it limits packets to 16
 	 * descriptors (which is bad).
 	 */
 	efx_reado(efx, &temp, FR_AZ_TX_CFG);
 	EFX_SET_OWORD_FIELD(temp, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
 	EFX_SET_OWORD_FIELD(temp, FRF_CZ_TX_FILTER_EN_BIT, 1);
 	efx_writeo(efx, &temp, FR_AZ_TX_CFG);
 	efx_reado(efx, &temp, FR_AZ_RX_CFG);
 	EFX_SET_OWORD_FIELD(temp, FRF_BZ_RX_DESC_PUSH_EN, 0);
 	EFX_SET_OWORD_FIELD(temp, FRF_BZ_RX_INGR_EN, 1);
 	/* Enable hash insertion. This is broken for the 'Falcon' hash
 	 * if IPv6 hashing is also enabled, so also select Toeplitz
 	 * TCP/IPv4 and IPv4 hashes. */
 	EFX_SET_OWORD_FIELD(temp, FRF_BZ_RX_HASH_INSRT_HDR, 1);
 	EFX_SET_OWORD_FIELD(temp, FRF_BZ_RX_HASH_ALG, 1);
 	EFX_SET_OWORD_FIELD(temp, FRF_BZ_RX_IP_HASH, 1);
 	efx_writeo(efx, &temp, FR_AZ_RX_CFG);
 	/* Set hash key for IPv4 */
 	memcpy(&temp, efx->rx_hash_key, sizeof(temp));
 	efx_writeo(efx, &temp, FR_BZ_RX_RSS_TKEY);
 	/* Enable IPv6 RSS */
 	BUILD_BUG_ON(sizeof(efx->rx_hash_key) <
 		     2 * sizeof(temp) + FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH / 8 ||
 		     FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN != 0);
 	memcpy(&temp, efx->rx_hash_key, sizeof(temp));
 	efx_writeo(efx, &temp, FR_CZ_RX_RSS_IPV6_REG1);
 	memcpy(&temp, efx->rx_hash_key + sizeof(temp), sizeof(temp));
 	efx_writeo(efx, &temp, FR_CZ_RX_RSS_IPV6_REG2);
 	EFX_POPULATE_OWORD_2(temp, FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1,
 			     FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, 1);
 	memcpy(&temp, efx->rx_hash_key + 2 * sizeof(temp),
 	       FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH / 8);
 	efx_writeo(efx, &temp, FR_CZ_RX_RSS_IPV6_REG3);
 	/* Enable event logging */
 	rc = efx_mcdi_log_ctrl(efx, true, false, 0);
 	if (rc)
 		return rc;
 	/* Set destination of both TX and RX Flush events */
 	EFX_POPULATE_OWORD_1(temp, FRF_BZ_FLS_EVQ_ID, 0);
 	efx_writeo(efx, &temp, FR_BZ_DP_CTRL);
 	EFX_POPULATE_OWORD_1(temp, FRF_CZ_USREV_DIS, 1);
 	efx_writeo(efx, &temp, FR_CZ_USR_EV_CFG);
 	efx_nic_init_common(efx);
 	return 0;
 }
 static void siena_remove_nic(struct efx_nic *efx)
 {
-	struct siena_nic_data *nic_data = efx->nic_data;
 	efx_nic_free_buffer(efx, &efx->irq_status);
 	siena_reset_hw(efx, RESET_TYPE_ALL);
 	/* Relinquish the device back to the BMC */
 	if (efx_nic_has_mc(efx))
 		efx_mcdi_drv_attach(efx, false, NULL);
 	/* Tear down the private nic state */
-	iounmap(nic_data->mcdi_smem);
+	kfree(efx->nic_data);
-	kfree(nic_data);
 	efx->nic_data = NULL;
 }
 #define STATS_GENERATION_INVALID ((__force __le64)(-1))
 static int siena_try_update_nic_stats(struct efx_nic *efx)
 {
 	__le64 *dma_stats;
 	struct efx_mac_stats *mac_stats;
 	__le64 generation_start, generation_end;
 	mac_stats = &efx->mac_stats;
 	dma_stats = efx->stats_buffer.addr;
 	generation_end = dma_stats[MC_CMD_MAC_GENERATION_END];
 	if (generation_end == STATS_GENERATION_INVALID)
 		return 0;
 	rmb();
 #define MAC_STAT(M, D) \
 	mac_stats->M = le64_to_cpu(dma_stats[MC_CMD_MAC_ ## D])
 	MAC_STAT(tx_bytes, TX_BYTES);
 	MAC_STAT(tx_bad_bytes, TX_BAD_BYTES);
 	mac_stats->tx_good_bytes = (mac_stats->tx_bytes -
 				    mac_stats->tx_bad_bytes);
 	MAC_STAT(tx_packets, TX_PKTS);
 	MAC_STAT(tx_bad, TX_BAD_FCS_PKTS);
 	MAC_STAT(tx_pause, TX_PAUSE_PKTS);
 	MAC_STAT(tx_control, TX_CONTROL_PKTS);
 	MAC_STAT(tx_unicast, TX_UNICAST_PKTS);
 	MAC_STAT(tx_multicast, TX_MULTICAST_PKTS);
 	MAC_STAT(tx_broadcast, TX_BROADCAST_PKTS);
 	MAC_STAT(tx_lt64, TX_LT64_PKTS);
 	MAC_STAT(tx_64, TX_64_PKTS);
 	MAC_STAT(tx_65_to_127, TX_65_TO_127_PKTS);
 	MAC_STAT(tx_128_to_255, TX_128_TO_255_PKTS);
 	MAC_STAT(tx_256_to_511, TX_256_TO_511_PKTS);
 	MAC_STAT(tx_512_to_1023, TX_512_TO_1023_PKTS);
 	MAC_STAT(tx_1024_to_15xx, TX_1024_TO_15XX_PKTS);
 	MAC_STAT(tx_15xx_to_jumbo, TX_15XX_TO_JUMBO_PKTS);
 	MAC_STAT(tx_gtjumbo, TX_GTJUMBO_PKTS);
 	mac_stats->tx_collision = 0;
 	MAC_STAT(tx_single_collision, TX_SINGLE_COLLISION_PKTS);
 	MAC_STAT(tx_multiple_collision, TX_MULTIPLE_COLLISION_PKTS);
 	MAC_STAT(tx_excessive_collision, TX_EXCESSIVE_COLLISION_PKTS);
 	MAC_STAT(tx_deferred, TX_DEFERRED_PKTS);
 	MAC_STAT(tx_late_collision, TX_LATE_COLLISION_PKTS);
 	mac_stats->tx_collision = (mac_stats->tx_single_collision +
 				   mac_stats->tx_multiple_collision +
 				   mac_stats->tx_excessive_collision +
 				   mac_stats->tx_late_collision);
 	MAC_STAT(tx_excessive_deferred, TX_EXCESSIVE_DEFERRED_PKTS);
 	MAC_STAT(tx_non_tcpudp, TX_NON_TCPUDP_PKTS);
 	MAC_STAT(tx_mac_src_error, TX_MAC_SRC_ERR_PKTS);
 	MAC_STAT(tx_ip_src_error, TX_IP_SRC_ERR_PKTS);
 	MAC_STAT(rx_bytes, RX_BYTES);
 	MAC_STAT(rx_bad_bytes, RX_BAD_BYTES);
 	mac_stats->rx_good_bytes = (mac_stats->rx_bytes -
 				    mac_stats->rx_bad_bytes);
 	MAC_STAT(rx_packets, RX_PKTS);
 	MAC_STAT(rx_good, RX_GOOD_PKTS);
 	MAC_STAT(rx_bad, RX_BAD_FCS_PKTS);
 	MAC_STAT(rx_pause, RX_PAUSE_PKTS);
 	MAC_STAT(rx_control, RX_CONTROL_PKTS);
 	MAC_STAT(rx_unicast, RX_UNICAST_PKTS);
 	MAC_STAT(rx_multicast, RX_MULTICAST_PKTS);
 	MAC_STAT(rx_broadcast, RX_BROADCAST_PKTS);
 	MAC_STAT(rx_lt64, RX_UNDERSIZE_PKTS);
 	MAC_STAT(rx_64, RX_64_PKTS);
 	MAC_STAT(rx_65_to_127, RX_65_TO_127_PKTS);
 	MAC_STAT(rx_128_to_255, RX_128_TO_255_PKTS);
 	MAC_STAT(rx_256_to_511, RX_256_TO_511_PKTS);
 	MAC_STAT(rx_512_to_1023, RX_512_TO_1023_PKTS);
 	MAC_STAT(rx_1024_to_15xx, RX_1024_TO_15XX_PKTS);
 	MAC_STAT(rx_15xx_to_jumbo, RX_15XX_TO_JUMBO_PKTS);
 	MAC_STAT(rx_gtjumbo, RX_GTJUMBO_PKTS);
 	mac_stats->rx_bad_lt64 = 0;
 	mac_stats->rx_bad_64_to_15xx = 0;
 	mac_stats->rx_bad_15xx_to_jumbo = 0;
 	MAC_STAT(rx_bad_gtjumbo, RX_JABBER_PKTS);
 	MAC_STAT(rx_overflow, RX_OVERFLOW_PKTS);
 	mac_stats->rx_missed = 0;
 	MAC_STAT(rx_false_carrier, RX_FALSE_CARRIER_PKTS);
 	MAC_STAT(rx_symbol_error, RX_SYMBOL_ERROR_PKTS);
 	MAC_STAT(rx_align_error, RX_ALIGN_ERROR_PKTS);
 	MAC_STAT(rx_length_error, RX_LENGTH_ERROR_PKTS);
 	MAC_STAT(rx_internal_error, RX_INTERNAL_ERROR_PKTS);
 	mac_stats->rx_good_lt64 = 0;
 	efx->n_rx_nodesc_drop_cnt =
 		le64_to_cpu(dma_stats[MC_CMD_MAC_RX_NODESC_DROPS]);
 #undef MAC_STAT
 	rmb();
 	generation_start = dma_stats[MC_CMD_MAC_GENERATION_START];
 	if (generation_end != generation_start)
 		return -EAGAIN;
 	return 0;
 }
 static void siena_update_nic_stats(struct efx_nic *efx)
 {
 	int retry;
 	/* If we're unlucky enough to read statistics wduring the DMA, wait
 	 * up to 10ms for it to finish (typically takes <500us) */
 	for (retry = 0; retry < 100; ++retry) {
 		if (siena_try_update_nic_stats(efx) == 0)
 			return;
 		udelay(100);
 	}
 	/* Use the old values instead */
 }
 static void siena_start_nic_stats(struct efx_nic *efx)
 {
 	__le64 *dma_stats = efx->stats_buffer.addr;
 	dma_stats[MC_CMD_MAC_GENERATION_END] = STATS_GENERATION_INVALID;
 	efx_mcdi_mac_stats(efx, efx->stats_buffer.dma_addr,
 			   MC_CMD_MAC_NSTATS * sizeof(u64), 1, 0);
 }
 static void siena_stop_nic_stats(struct efx_nic *efx)
 {
 	efx_mcdi_mac_stats(efx, efx->stats_buffer.dma_addr, 0, 0, 0);
 }
 /**************************************************************************
  *
  * Wake on LAN
  *
  **************************************************************************
  */
 static void siena_get_wol(struct efx_nic *efx, struct ethtool_wolinfo *wol)
 {
 	struct siena_nic_data *nic_data = efx->nic_data;
 	wol->supported = WAKE_MAGIC;
 	if (nic_data->wol_filter_id != -1)
 		wol->wolopts = WAKE_MAGIC;
 	else
 		wol->wolopts = 0;
 	memset(&wol->sopass, 0, sizeof(wol->sopass));
 }
 static int siena_set_wol(struct efx_nic *efx, u32 type)
 {
 	struct siena_nic_data *nic_data = efx->nic_data;
 	int rc;
 	if (type & ~WAKE_MAGIC)
 		return -EINVAL;
 	if (type & WAKE_MAGIC) {
 		if (nic_data->wol_filter_id != -1)
 			efx_mcdi_wol_filter_remove(efx,
 						   nic_data->wol_filter_id);
 		rc = efx_mcdi_wol_filter_set_magic(efx, efx->net_dev->dev_addr,
 						   &nic_data->wol_filter_id);
 		if (rc)
 			goto fail;
 		pci_wake_from_d3(efx->pci_dev, true);
 	} else {
 		rc = efx_mcdi_wol_filter_reset(efx);
 		nic_data->wol_filter_id = -1;
 		pci_wake_from_d3(efx->pci_dev, false);
 		if (rc)
 			goto fail;
 	}
 	return 0;
  fail:
 	netif_err(efx, hw, efx->net_dev, "%s failed: type=%d rc=%d\n",
 		  __func__, type, rc);
 	return rc;
 }
 static void siena_init_wol(struct efx_nic *efx)
 {
 	struct siena_nic_data *nic_data = efx->nic_data;
 	int rc;
 	rc = efx_mcdi_wol_filter_get_magic(efx, &nic_data->wol_filter_id);
 	if (rc != 0) {
 		/* If it failed, attempt to get into a synchronised
 		 * state with MC by resetting any set WoL filters */
 		efx_mcdi_wol_filter_reset(efx);
 		nic_data->wol_filter_id = -1;
 	} else if (nic_data->wol_filter_id != -1) {
 		pci_wake_from_d3(efx->pci_dev, true);
 	}
 }
 /**************************************************************************
  *
  * Revision-dependent attributes used by efx.c and nic.c
  *
  **************************************************************************
  */
 const struct efx_nic_type siena_a0_nic_type = {
 	.probe = siena_probe_nic,
 	.remove = siena_remove_nic,
 	.init = siena_init_nic,
 	.fini = efx_port_dummy_op_void,
 	.monitor = NULL,
 	.map_reset_reason = siena_map_reset_reason,
 	.map_reset_flags = siena_map_reset_flags,
 	.reset = siena_reset_hw,
 	.probe_port = siena_probe_port,
 	.remove_port = siena_remove_port,
 	.prepare_flush = efx_port_dummy_op_void,
 	.update_stats = siena_update_nic_stats,
 	.start_stats = siena_start_nic_stats,
 	.stop_stats = siena_stop_nic_stats,
 	.set_id_led = efx_mcdi_set_id_led,
 	.push_irq_moderation = siena_push_irq_moderation,
 	.push_multicast_hash = siena_push_multicast_hash,
 	.reconfigure_port = efx_mcdi_phy_reconfigure,
 	.get_wol = siena_get_wol,
 	.set_wol = siena_set_wol,
 	.resume_wol = siena_init_wol,
 	.test_registers = siena_test_registers,
 	.test_nvram = efx_mcdi_nvram_test_all,
 	.default_mac_ops = &efx_mcdi_mac_operations,
 	.revision = EFX_REV_SIENA_A0,
-	.mem_map_size = FR_CZ_MC_TREG_SMEM, /* MC_TREG_SMEM mapped separately */
+	.mem_map_size = (FR_CZ_MC_TREG_SMEM +
+			 FR_CZ_MC_TREG_SMEM_STEP * FR_CZ_MC_TREG_SMEM_ROWS),
 	.txd_ptr_tbl_base = FR_BZ_TX_DESC_PTR_TBL,
 	.rxd_ptr_tbl_base = FR_BZ_RX_DESC_PTR_TBL,
 	.buf_tbl_base = FR_BZ_BUF_FULL_TBL,
 	.evq_ptr_tbl_base = FR_BZ_EVQ_PTR_TBL,
 	.evq_rptr_tbl_base = FR_BZ_EVQ_RPTR,
 	.max_dma_mask = DMA_BIT_MASK(FSF_AZ_TX_KER_BUF_ADDR_WIDTH),
 	.rx_buffer_hash_size = 0x10,
 	.rx_buffer_padding = 0,
 	.max_interrupt_mode = EFX_INT_MODE_MSIX,
 	.phys_addr_channels = 32, /* Hardware limit is 64, but the legacy
 				   * interrupt handler only supports 32
 				   * channels */
 	.tx_dc_base = 0x88000,
 	.rx_dc_base = 0x68000,
 	.offload_features = (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
 			     NETIF_F_RXHASH | NETIF_F_NTUPLE),
 };

drivers/net/sfc/workarounds.h

Diff comments View file @ 86c432c

 /****************************************************************************
  * Driver for Solarflare Solarstorm network controllers and boards
  * Copyright 2006-2010 Solarflare Communications Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
  * by the Free Software Foundation, incorporated herein by reference.
  */
 #ifndef EFX_WORKAROUNDS_H
 #define EFX_WORKAROUNDS_H
 /*
  * Hardware workarounds.
  * Bug numbers are from Solarflare's Bugzilla.
  */
 #define EFX_WORKAROUND_ALWAYS(efx) 1
 #define EFX_WORKAROUND_FALCON_A(efx) (efx_nic_rev(efx) <= EFX_REV_FALCON_A1)
 #define EFX_WORKAROUND_FALCON_AB(efx) (efx_nic_rev(efx) <= EFX_REV_FALCON_B0)
 #define EFX_WORKAROUND_SIENA(efx) (efx_nic_rev(efx) == EFX_REV_SIENA_A0)
 #define EFX_WORKAROUND_10G(efx) 1
 /* XAUI resets if link not detected */
 #define EFX_WORKAROUND_5147 EFX_WORKAROUND_ALWAYS
 /* RX PCIe double split performance issue */
 #define EFX_WORKAROUND_7575 EFX_WORKAROUND_ALWAYS
 /* Bit-bashed I2C reads cause performance drop */
 #define EFX_WORKAROUND_7884 EFX_WORKAROUND_10G
 /* TX_EV_PKT_ERR can be caused by a dangling TX descriptor
  * or a PCIe error (bug 11028) */
 #define EFX_WORKAROUND_10727 EFX_WORKAROUND_ALWAYS
 /* Transmit flow control may get disabled */
 #define EFX_WORKAROUND_11482 EFX_WORKAROUND_FALCON_AB
 /* Truncated IPv4 packets can confuse the TX packet parser */
 #define EFX_WORKAROUND_15592 EFX_WORKAROUND_FALCON_AB
 /* Legacy ISR read can return zero once */
 #define EFX_WORKAROUND_15783 EFX_WORKAROUND_ALWAYS
 /* Legacy interrupt storm when interrupt fifo fills */
 #define EFX_WORKAROUND_17213 EFX_WORKAROUND_SIENA
-/* Write combining and sriov=enabled are incompatible */
-#define EFX_WORKAROUND_22643 EFX_WORKAROUND_SIENA
 /* Spurious parity errors in TSORT buffers */
 #define EFX_WORKAROUND_5129 EFX_WORKAROUND_FALCON_A
 /* Unaligned read request >512 bytes after aligning may break TSORT */
 #define EFX_WORKAROUND_5391 EFX_WORKAROUND_FALCON_A
 /* iSCSI parsing errors */
 #define EFX_WORKAROUND_5583 EFX_WORKAROUND_FALCON_A
 /* RX events go missing */
 #define EFX_WORKAROUND_5676 EFX_WORKAROUND_FALCON_A
 /* RX_RESET on A1 */
 #define EFX_WORKAROUND_6555 EFX_WORKAROUND_FALCON_A
 /* Increase filter depth to avoid RX_RESET */
 #define EFX_WORKAROUND_7244 EFX_WORKAROUND_FALCON_A
 /* Flushes may never complete */
 #define EFX_WORKAROUND_7803 EFX_WORKAROUND_FALCON_AB
 /* Leak overlength packets rather than free */
 #define EFX_WORKAROUND_8071 EFX_WORKAROUND_FALCON_A
 #endif /* EFX_WORKAROUNDS_H */