enic: add Cisco 10G Ethernet NIC driver

Signed-off-by: Scott Feldman <scofeldm@cisco.com>
Signed-off-by: Jeff Garzik <jgarzik@redhat.com>

25 files changed, 5432 insertions, 0 deletions

diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile
new file mode 100644
index 00000000000..391c3bce5b7
--- /dev/null
+++ b/drivers/net/enic/Makefile
@@ -0,0 +1,5 @@
+obj-$(CONFIG_ENIC) := enic.o
+
+enic-y := enic_main.o vnic_cq.o vnic_intr.o vnic_wq.o \
+	enic_res.o vnic_dev.o vnic_rq.o
+
diff --git a/drivers/net/enic/cq_desc.h b/drivers/net/enic/cq_desc.h
new file mode 100644
index 00000000000..c036a8bfd04
--- /dev/null
+++ b/drivers/net/enic/cq_desc.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
+ *
+ * This program is free software; you may redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef _CQ_DESC_H_
+#define _CQ_DESC_H_
+
+/*
+ * Completion queue descriptor types
+ */
+enum cq_desc_types {
+	CQ_DESC_TYPE_WQ_ENET = 0,
+	CQ_DESC_TYPE_DESC_COPY = 1,
+	CQ_DESC_TYPE_WQ_EXCH = 2,
+	CQ_DESC_TYPE_RQ_ENET = 3,
+	CQ_DESC_TYPE_RQ_FCP = 4,
+};
+
+/* Completion queue descriptor: 16B
+ *
+ * All completion queues have this basic layout.  The
+ * type_specfic area is unique for each completion
+ * queue type.
+ */
+struct cq_desc {
+	__le16 completed_index;
+	__le16 q_number;
+	u8 type_specfic[11];
+	u8 type_color;
+};
+
+#define CQ_DESC_TYPE_BITS        7
+#define CQ_DESC_TYPE_MASK        ((1 << CQ_DESC_TYPE_BITS) - 1)
+#define CQ_DESC_COLOR_MASK       1
+#define CQ_DESC_Q_NUM_BITS       10
+#define CQ_DESC_Q_NUM_MASK       ((1 << CQ_DESC_Q_NUM_BITS) - 1)
+#define CQ_DESC_COMP_NDX_BITS    12
+#define CQ_DESC_COMP_NDX_MASK    ((1 << CQ_DESC_COMP_NDX_BITS) - 1)
+
+static inline void cq_desc_dec(const struct cq_desc *desc_arg,
+	u8 *type, u8 *color, u16 *q_number, u16 *completed_index)
+{
+	const struct cq_desc *desc = desc_arg;
+	const u8 type_color = desc->type_color;
+
+	*color = (type_color >> CQ_DESC_TYPE_BITS) & CQ_DESC_COLOR_MASK;
+
+	/*
+	 * Make sure color bit is read from desc *before* other fields
+	 * are read from desc.  Hardware guarantees color bit is last
+	 * bit (byte) written.  Adding the rmb() prevents the compiler
+	 * and/or CPU from reordering the reads which would potentially
+	 * result in reading stale values.
+	 */
+
+	rmb();
+
+	*type = type_color & CQ_DESC_TYPE_MASK;
+	*q_number = le16_to_cpu(desc->q_number) & CQ_DESC_Q_NUM_MASK;
+	*completed_index = le16_to_cpu(desc->completed_index) &
+		CQ_DESC_COMP_NDX_MASK;
+}
+
+#endif /* _CQ_DESC_H_ */
diff --git a/drivers/net/enic/cq_enet_desc.h b/drivers/net/enic/cq_enet_desc.h
new file mode 100644
index 00000000000..03dce9ed612
--- /dev/null
+++ b/drivers/net/enic/cq_enet_desc.h
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
+ *
+ * This program is free software; you may redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef _CQ_ENET_DESC_H_
+#define _CQ_ENET_DESC_H_
+
+#include "cq_desc.h"
+
+/* Ethernet completion queue descriptor: 16B */
+struct cq_enet_wq_desc {
+	__le16 completed_index;
+	__le16 q_number;
+	u8 reserved[11];
+	u8 type_color;
+};
+
+static inline void cq_enet_wq_desc_dec(struct cq_enet_wq_desc *desc,
+	u8 *type, u8 *color, u16 *q_number, u16 *completed_index)
+{
+	cq_desc_dec((struct cq_desc *)desc, type,
+		color, q_number, completed_index);
+}
+
+/* Completion queue descriptor: Ethernet receive queue, 16B */
+struct cq_enet_rq_desc {
+	__le16 completed_index_flags;
+	__le16 q_number_rss_type_flags;
+	__le32 rss_hash;
+	__le16 bytes_written_flags;
+	__le16 vlan;
+	__le16 checksum_fcoe;
+	u8 flags;
+	u8 type_color;
+};
+
+#define CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT          (0x1 << 12)
+#define CQ_ENET_RQ_DESC_FLAGS_FCOE                  (0x1 << 13)
+#define CQ_ENET_RQ_DESC_FLAGS_EOP                   (0x1 << 14)
+#define CQ_ENET_RQ_DESC_FLAGS_SOP                   (0x1 << 15)
+
+#define CQ_ENET_RQ_DESC_RSS_TYPE_BITS               4
+#define CQ_ENET_RQ_DESC_RSS_TYPE_MASK \
+	((1 << CQ_ENET_RQ_DESC_RSS_TYPE_BITS) - 1)
+#define CQ_ENET_RQ_DESC_RSS_TYPE_NONE               0
+#define CQ_ENET_RQ_DESC_RSS_TYPE_IPv4               1
+#define CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv4           2
+#define CQ_ENET_RQ_DESC_RSS_TYPE_IPv6               3
+#define CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6           4
+#define CQ_ENET_RQ_DESC_RSS_TYPE_IPv6_EX            5
+#define CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6_EX        6
+
+#define CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC         (0x1 << 14)
+
+#define CQ_ENET_RQ_DESC_BYTES_WRITTEN_BITS          14
+#define CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK \
+	((1 << CQ_ENET_RQ_DESC_BYTES_WRITTEN_BITS) - 1)
+#define CQ_ENET_RQ_DESC_FLAGS_TRUNCATED             (0x1 << 14)
+#define CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED         (0x1 << 15)
+
+#define CQ_ENET_RQ_DESC_FCOE_SOF_BITS               4
+#define CQ_ENET_RQ_DESC_FCOE_SOF_MASK \
+	((1 << CQ_ENET_RQ_DESC_FCOE_SOF_BITS) - 1)
+#define CQ_ENET_RQ_DESC_FCOE_EOF_BITS               8
+#define CQ_ENET_RQ_DESC_FCOE_EOF_MASK \
+	((1 << CQ_ENET_RQ_DESC_FCOE_EOF_BITS) - 1)
+#define CQ_ENET_RQ_DESC_FCOE_EOF_SHIFT              8
+
+#define CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK       (0x1 << 0)
+#define CQ_ENET_RQ_DESC_FCOE_FC_CRC_OK              (0x1 << 0)
+#define CQ_ENET_RQ_DESC_FLAGS_UDP                   (0x1 << 1)
+#define CQ_ENET_RQ_DESC_FCOE_ENC_ERROR              (0x1 << 1)
+#define CQ_ENET_RQ_DESC_FLAGS_TCP                   (0x1 << 2)
+#define CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK          (0x1 << 3)
+#define CQ_ENET_RQ_DESC_FLAGS_IPV6                  (0x1 << 4)
+#define CQ_ENET_RQ_DESC_FLAGS_IPV4                  (0x1 << 5)
+#define CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT         (0x1 << 6)
+#define CQ_ENET_RQ_DESC_FLAGS_FCS_OK                (0x1 << 7)
+
+static inline void cq_enet_rq_desc_dec(struct cq_enet_rq_desc *desc,
+	u8 *type, u8 *color, u16 *q_number, u16 *completed_index,
+	u8 *ingress_port, u8 *fcoe, u8 *eop, u8 *sop, u8 *rss_type,
+	u8 *csum_not_calc, u32 *rss_hash, u16 *bytes_written, u8 *packet_error,
+	u8 *vlan_stripped, u16 *vlan, u16 *checksum, u8 *fcoe_sof,
+	u8 *fcoe_fc_crc_ok, u8 *fcoe_enc_error, u8 *fcoe_eof,
+	u8 *tcp_udp_csum_ok, u8 *udp, u8 *tcp, u8 *ipv4_csum_ok,
+	u8 *ipv6, u8 *ipv4, u8 *ipv4_fragment, u8 *fcs_ok)
+{
+	u16 completed_index_flags = le16_to_cpu(desc->completed_index_flags);
+	u16 q_number_rss_type_flags =
+		le16_to_cpu(desc->q_number_rss_type_flags);
+	u16 bytes_written_flags = le16_to_cpu(desc->bytes_written_flags);
+
+	cq_desc_dec((struct cq_desc *)desc, type,
+		color, q_number, completed_index);
+
+	*ingress_port = (completed_index_flags &
+		CQ_ENET_RQ_DESC_FLAGS_INGRESS_PORT) ? 1 : 0;
+	*fcoe = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_FCOE) ?
+		1 : 0;
+	*eop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_EOP) ?
+		1 : 0;
+	*sop = (completed_index_flags & CQ_ENET_RQ_DESC_FLAGS_SOP) ?
+		1 : 0;
+
+	*rss_type = (u8)((q_number_rss_type_flags >> CQ_DESC_Q_NUM_BITS) &
+		CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
+	*csum_not_calc = (q_number_rss_type_flags &
+		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ? 1 : 0;
+
+	*rss_hash = le32_to_cpu(desc->rss_hash);
+
+	*bytes_written = bytes_written_flags &
+		CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
+	*packet_error = (bytes_written_flags &
+		CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ? 1 : 0;
+	*vlan_stripped = (bytes_written_flags &
+		CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) ? 1 : 0;
+
+	*vlan = le16_to_cpu(desc->vlan);
+
+	if (*fcoe) {
+		*fcoe_sof = (u8)(le16_to_cpu(desc->checksum_fcoe) &
+			CQ_ENET_RQ_DESC_FCOE_SOF_MASK);
+		*fcoe_fc_crc_ok = (desc->flags &
+			CQ_ENET_RQ_DESC_FCOE_FC_CRC_OK) ? 1 : 0;
+		*fcoe_enc_error = (desc->flags &
+			CQ_ENET_RQ_DESC_FCOE_ENC_ERROR) ? 1 : 0;
+		*fcoe_eof = (u8)((desc->checksum_fcoe >>
+			CQ_ENET_RQ_DESC_FCOE_EOF_SHIFT) &
+			CQ_ENET_RQ_DESC_FCOE_EOF_MASK);
+		*checksum = 0;
+	} else {
+		*fcoe_sof = 0;
+		*fcoe_fc_crc_ok = 0;
+		*fcoe_enc_error = 0;
+		*fcoe_eof = 0;
+		*checksum = le16_to_cpu(desc->checksum_fcoe);
+	}
+
+	*tcp_udp_csum_ok =
+		(desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ? 1 : 0;
+	*udp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_UDP) ? 1 : 0;
+	*tcp = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_TCP) ? 1 : 0;
+	*ipv4_csum_ok =
+		(desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ? 1 : 0;
+	*ipv6 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV6) ? 1 : 0;
+	*ipv4 = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4) ? 1 : 0;
+	*ipv4_fragment =
+		(desc->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT) ? 1 : 0;
+	*fcs_ok = (desc->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) ? 1 : 0;
+}
+
+#endif /* _CQ_ENET_DESC_H_ */
diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
new file mode 100644
index 00000000000..fb83c926da5
--- /dev/null
+++ b/drivers/net/enic/enic.h
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
+ *
+ * This program is free software; you may redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef _ENIC_H_
+#define _ENIC_H_
+
+#include <linux/inet_lro.h>
+
+#include "vnic_enet.h"
+#include "vnic_dev.h"
+#include "vnic_wq.h"
+#include "vnic_rq.h"
+#include "vnic_cq.h"
+#include "vnic_intr.h"
+#include "vnic_stats.h"
+#include "vnic_rss.h"
+
+#define DRV_NAME		"enic"
+#define DRV_DESCRIPTION		"Cisco 10G Ethernet Driver"
+#define DRV_VERSION		"0.0.1.18163.472"
+#define DRV_COPYRIGHT		"Copyright 2008 Cisco Systems, Inc"
+#define PFX			DRV_NAME ": "
+
+#define ENIC_LRO_MAX_DESC	8
+#define ENIC_LRO_MAX_AGGR	64
+
+enum enic_cq_index {
+	ENIC_CQ_RQ,
+	ENIC_CQ_WQ,
+	ENIC_CQ_MAX,
+};
+
+enum enic_intx_intr_index {
+	ENIC_INTX_WQ_RQ,
+	ENIC_INTX_ERR,
+	ENIC_INTX_NOTIFY,
+	ENIC_INTX_MAX,
+};
+
+enum enic_msix_intr_index {
+	ENIC_MSIX_RQ,
+	ENIC_MSIX_WQ,
+	ENIC_MSIX_ERR,
+	ENIC_MSIX_NOTIFY,
+	ENIC_MSIX_MAX,
+};
+
+struct enic_msix_entry {
+	int requested;
+	char devname[IFNAMSIZ];
+	irqreturn_t (*isr)(int, void *);
+	void *devid;
+};
+
+/* Per-instance private data structure */
+struct enic {
+	struct net_device *netdev;
+	struct pci_dev *pdev;
+	struct vnic_enet_config config;
+	struct vnic_dev_bar bar0;
+	struct vnic_dev *vdev;
+	struct net_device_stats net_stats;
+	struct timer_list notify_timer;
+	struct work_struct reset;
+	struct msix_entry msix_entry[ENIC_MSIX_MAX];
+	struct enic_msix_entry msix[ENIC_MSIX_MAX];
+	u32 msg_enable;
+	spinlock_t devcmd_lock;
+	u8 mac_addr[ETH_ALEN];
+	u8 mc_addr[ENIC_MULTICAST_PERFECT_FILTERS][ETH_ALEN];
+	unsigned int mc_count;
+	int csum_rx_enabled;
+	u32 port_mtu;
+
+	/* work queue cache line section */
+	____cacheline_aligned struct vnic_wq wq[1];
+	spinlock_t wq_lock[1];
+	unsigned int wq_count;
+	struct vlan_group *vlan_group;
+
+	/* receive queue cache line section */
+	____cacheline_aligned struct vnic_rq rq[1];
+	unsigned int rq_count;
+	int (*rq_alloc_buf)(struct vnic_rq *rq);
+	struct napi_struct napi;
+	struct net_lro_mgr lro_mgr;
+	struct net_lro_desc lro_desc[ENIC_LRO_MAX_DESC];
+
+	/* interrupt resource cache line section */
+	____cacheline_aligned struct vnic_intr intr[ENIC_MSIX_MAX];
+	unsigned int intr_count;
+	u32 __iomem *legacy_pba;		/* memory-mapped */
+
+	/* completion queue cache line section */
+	____cacheline_aligned struct vnic_cq cq[ENIC_CQ_MAX];
+	unsigned int cq_count;
+};
+
+#endif /* _ENIC_H_ */
diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
new file mode 100644
index 00000000000..4cf5ec76c99
--- /dev/null
+++ b/drivers/net/enic/enic_main.c
@@ -0,0 +1,1949 @@
+/*
+ * Copyright 2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
+ *
+ * This program is free software; you may redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; version 2 of the License.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/init.h>
+#include <linux/workqueue.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include <linux/ethtool.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/tcp.h>
+
+#include "cq_enet_desc.h"
+#include "vnic_dev.h"
+#include "vnic_intr.h"
+#include "vnic_stats.h"
+#include "enic_res.h"
+#include "enic.h"
+
+#define ENIC_NOTIFY_TIMER_PERIOD	(2 * HZ)
+#define ENIC_JUMBO_FIRST_BUF_SIZE	256
+
+/* Supported devices */
+static struct pci_device_id enic_id_table[] = {
+	{ PCI_VDEVICE(CISCO, 0x0043) },
+	{ 0, }	/* end of table */
+};
+
+MODULE_DESCRIPTION(DRV_DESCRIPTION);
+MODULE_AUTHOR("Scott Feldman <scofeldm@cisco.com>");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(DRV_VERSION);
+MODULE_DEVICE_TABLE(pci, enic_id_table);
+
+struct enic_stat {
+	char name[ETH_GSTRING_LEN];
+	unsigned int offset;
+};
+
+#define ENIC_TX_STAT(stat)	\
+	{ .name = #stat, .offset = offsetof(struct vnic_tx_stats, stat) / 8 }
+#define ENIC_RX_STAT(stat)	\
+	{ .name = #stat, .offset = offsetof(struct vnic_rx_stats, stat) / 8 }
+
+static const struct enic_stat enic_tx_stats[] = {
+	ENIC_TX_STAT(tx_frames_ok),
+	ENIC_TX_STAT(tx_unicast_frames_ok),
+	ENIC_TX_STAT(tx_multicast_frames_ok),
+	ENIC_TX_STAT(tx_broadcast_frames_ok),
+	ENIC_TX_STAT(tx_bytes_ok),
+	ENIC_TX_STAT(tx_unicast_bytes_ok),
+	ENIC_TX_STAT(tx_multicast_bytes_ok),
+	ENIC_TX_STAT(tx_broadcast_bytes_ok),
+	ENIC_TX_STAT(tx_drops),
+	ENIC_TX_STAT(tx_errors),
+	ENIC_TX_STAT(tx_tso),
+};
+
+static const struct enic_stat enic_rx_stats[] = {
+	ENIC_RX_STAT(rx_frames_ok),
+	ENIC_RX_STAT(rx_frames_total),
+	ENIC_RX_STAT(rx_unicast_frames_ok),
+	ENIC_RX_STAT(rx_multicast_frames_ok),
+	ENIC_RX_STAT(rx_broadcast_frames_ok),
+	ENIC_RX_STAT(rx_bytes_ok),
+	ENIC_RX_STAT(rx_unicast_bytes_ok),
+	ENIC_RX_STAT(rx_multicast_bytes_ok),
+	ENIC_RX_STAT(rx_broadcast_bytes_ok),
+	ENIC_RX_STAT(rx_drop),
+	ENIC_RX_STAT(rx_no_bufs),
+	ENIC_RX_STAT(rx_errors),
+	ENIC_RX_STAT(rx_rss),
+	ENIC_RX_STAT(rx_crc_errors),
+	ENIC_RX_STAT(rx_frames_64),
+	ENIC_RX_STAT(rx_frames_127),
+	ENIC_RX_STAT(rx_frames_255),
+	ENIC_RX_STAT(rx_frames_511),
+	ENIC_RX_STAT(rx_frames_1023),
+	ENIC_RX_STAT(rx_frames_1518),
+	ENIC_RX_STAT(rx_frames_to_max),
+};
+
+static const unsigned int enic_n_tx_stats = ARRAY_SIZE(enic_tx_stats);
+static const unsigned int enic_n_rx_stats = ARRAY_SIZE(enic_rx_stats);
+
+static int enic_get_settings(struct net_device *netdev,
+	struct ethtool_cmd *ecmd)
+{
+	struct enic *enic = netdev_priv(netdev);
+
+	ecmd->supported = (SUPPORTED_10000baseT_Full | SUPPORTED_FIBRE);
+	ecmd->advertising = (ADVERTISED_10000baseT_Full | ADVERTISED_FIBRE);
+	ecmd->port = PORT_FIBRE;
+	ecmd->transceiver = XCVR_EXTERNAL;
+
+	if (netif_carrier_ok(netdev)) {
+		ecmd->speed = vnic_dev_port_speed(enic->vdev);
+		ecmd->duplex = DUPLEX_FULL;
+	} else {
+		ecmd->speed = -1;
+		ecmd->duplex = -1;
+	}
+
+	ecmd->autoneg = AUTONEG_DISABLE;
+
+	return 0;
+}
+
+static void enic_get_drvinfo(struct net_device *netdev,
+	struct ethtool_drvinfo *drvinfo)
+{
+	struct enic *enic = netdev_priv(netdev);
+	struct vnic_devcmd_fw_info *fw_info;
+
+	spin_lock(&enic->devcmd_lock);
+	vnic_dev_fw_info(enic->vdev, &fw_info);
+	spin_unlock(&enic->devcmd_lock);
+
+	strncpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strncpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version));
+	strncpy(drvinfo->fw_version, fw_info->fw_version,
+		sizeof(drvinfo->fw_version));
+	strncpy(drvinfo->bus_info, pci_name(enic->pdev),
+		sizeof(drvinfo->bus_info));
+}
+
+static void enic_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
+{
+	unsigned int i;
+
+	switch (stringset) {
+	case ETH_SS_STATS:
+		for (i = 0; i < enic_n_tx_stats; i++) {
+			memcpy(data, enic_tx_stats[i].name, ETH_GSTRING_LEN);
+			data += ETH_GSTRING_LEN;
+		}
+		for (i = 0; i < enic_n_rx_stats; i++) {
+			memcpy(data, enic_rx_stats[i].name, ETH_GSTRING_LEN);
+			data += ETH_GSTRING_LEN;
+		}
+		break;
+	}
+}
+
+static int enic_get_stats_count(struct net_device *netdev)
+{
+	return enic_n_tx_stats + enic_n_rx_stats;
+}
+
+static void enic_get_ethtool_stats(struct net_device *netdev,
+	struct ethtool_stats *stats, u64 *data)
+{
+	struct enic *enic = netdev_priv(netdev);
+	struct vnic_stats *vstats;
+	unsigned int i;
+
+	spin_lock(&enic->devcmd_lock);
+	vnic_dev_stats_dump(enic->vdev, &vstats);
+	spin_unlock(&enic->devcmd_lock);
+
+	for (i = 0; i < enic_n_tx_stats; i++)
+		*(data++) = ((u64 *)&vstats->tx)[enic_tx_stats[i].offset];
+	for (i = 0; i < enic_n_rx_stats; i++)
+		*(data++) = ((u64 *)&vstats->rx)[enic_rx_stats[i].offset];
+}
+
+static u32 enic_get_rx_csum(struct net_device *netdev)
+{
+	struct enic *enic = netdev_priv(netdev);
+	return enic->csum_rx_enabled;
+}
+
+static int enic_set_rx_csum(struct net_device *netdev, u32 data)
+{
+	struct enic *enic = netdev_priv(netdev);
+
+	enic->csum_rx_enabled =
+		(data && ENIC_SETTING(enic, RXCSUM)) ? 1 : 0;
+
+	return 0;
+}
+
+static int enic_set_tx_csum(struct net_device *netdev, u32 data)
+{
+	struct enic *enic = netdev_priv(netdev);
+
+	if (data && ENIC_SETTING(enic, TXCSUM))
+		netdev->features |= NETIF_F_HW_CSUM;
+	else
+		netdev->features &= ~NETIF_F_HW_CSUM;
+
+	return 0;
+}
+
+static int enic_set_tso(struct net_device *netdev, u32 data)
+{
+	struct enic *enic = netdev_priv(netdev);
+
+	if (data && ENIC_SETTING(enic, TSO))
+		netdev->features |=
+			NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN;
+	else
+		netdev->features &=
+			~(NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN);
+
+	return 0;
+}
+
+static u32 enic_get_msglevel(struct net_device *netdev)
+{
+	struct enic *enic = netdev_priv(netdev);
+	return enic->msg_enable;
+}
+
+static void enic_set_msglevel(struct net_device *netdev, u32 value)
+{
+	struct enic *enic = netdev_priv(netdev);
+	enic->msg_enable = value;
+}
+
+static struct ethtool_ops enic_ethtool_ops = {
+	.get_settings = enic_get_settings,
+	.get_drvinfo = enic_get_drvinfo,
+	.get_msglevel = enic_get_msglevel,
+	.set_msglevel = enic_set_msglevel,
+	.get_link = ethtool_op_get_link,
+	.get_strings = enic_get_strings,
+	.get_stats_count = enic_get_stats_count,
+	.get_ethtool_stats = enic_get_ethtool_stats,
+	.get_rx_csum = enic_get_rx_csum,
+	.set_rx_csum = enic_set_rx_csum,
+	.get_tx_csum = ethtool_op_get_tx_csum,
+	.set_tx_csum = enic_set_tx_csum,
+	.get_sg = ethtool_op_get_sg,
+	.set_sg = ethtool_op_set_sg,
+	.get_tso = ethtool_op_get_tso,
+	.set_tso = enic_set_tso,
+};
+
+static void enic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf)
+{
+	struct enic *enic = vnic_dev_priv(wq->vdev);
+
+	if (buf->sop)
+		pci_unmap_single(enic->pdev, buf->dma_addr,
+			buf->len, PCI_DMA_TODEVICE);
+	else
+		pci_unmap_page(enic->pdev, buf->dma_addr,
+			buf->len, PCI_DMA_TODEVICE);
+
+	if (buf->os_buf)
+		dev_kfree_skb_any(buf->os_buf);
+}
+
+static void enic_wq_free_buf(struct vnic_wq *wq,
+	struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque)
+{
+	enic_free_wq_buf(wq, buf);
+}
+
+static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
+	u8 type, u16 q_number, u16 completed_index, void *opaque)
+{
+	struct enic *enic = vnic_dev_priv(vdev);
+
+	spin_lock(&enic->wq_lock[q_number]);
+
+	vnic_wq_service(&enic->wq[q_number], cq_desc,
+		completed_index, enic_wq_free_buf,
+		opaque);
+
+	if (netif_queue_stopped(enic->netdev) &&
+	    vnic_wq_desc_avail(&enic->wq[q_number]) >= MAX_SKB_FRAGS + 1)
+		netif_wake_queue(enic->netdev);
+
+	spin_unlock(&enic->wq_lock[q_number]);
+
+	return 0;
+}
+
+static void enic_log_q_error(struct enic *enic)
+{
+	unsigned int i;
+	u32 error_status;
+
+	for (i = 0; i < enic->wq_count; i++) {
+		error_status = vnic_wq_error_status(&enic->wq[i]);
+		if (error_status)
+			printk(KERN_ERR PFX "%s: WQ[%d] error_status %d\n",
+				enic->netdev->name, i, error_status);
+	}
+
+	for (i = 0; i < enic->rq_count; i++) {
+		error_status = vnic_rq_error_status(&enic->rq[i]);
+		if (error_status)
+			printk(KERN_ERR PFX "%s: RQ[%d] error_status %d\n",
+				enic->netdev->name, i, error_status);
+	}
+}
+
+static void enic_link_check(struct enic *enic)
+{
+	int link_status = vnic_dev_link_status(enic->vdev);
+	int carrier_ok = netif_carrier_ok(enic->netdev);
+
+	if (link_status && !carrier_ok) {
+		printk(KERN_INFO PFX "%s: Link UP\n", enic->netdev->name);
+		netif_carrier_on(enic->netdev);
+	} else if (!link_status && carrier_ok) {
+		printk(KERN_INFO PFX "%s: Link DOWN\n", enic->netdev->name);
+		netif_carrier_off(enic->netdev);
+	}
+}
+
+static void enic_mtu_check(struct enic *enic)
+{
+	u32 mtu = vnic_dev_mtu(enic->vdev);
+
+	if (mtu != enic->port_mtu) {
+		if (mtu < enic->netdev->mtu)
+			printk(KERN_WARNING PFX
+				"%s: interface MTU (%d) set higher "
+				"than switch port MTU (%d)\n",
+				enic->netdev->name, enic->netdev->mtu, mtu);
+		enic->port_mtu = mtu;
+	}
+}
+
+static void enic_msglvl_check(struct enic *enic)
+{
+	u32 msg_enable = vnic_dev_msg_lvl(enic->vdev);
+
+	if (msg_enable != enic->msg_enable) {
+		printk(KERN_INFO PFX "%s: msg lvl changed from 0x%x to 0x%x\n",
+			enic->netdev->name, enic->msg_enable, msg_enable);
+		enic->msg_enable = msg_enable;
+	}
+}
+
+static void enic_notify_check(struct enic *enic)
+{
+	enic_msglvl_check(enic);
+	enic_mtu_check(enic);
+	enic_link_check(enic);
+}
+
+#define ENIC_TEST_INTR(pba, i) (pba & (1 << i))
+
+static irqreturn_t enic_isr_legacy(int irq, void *data)
+{
+	struct net_device *netdev = data;
+	struct enic *enic = netdev_priv(netdev);
+	u32 pba;
+
+	vnic_intr_mask(&enic->intr[ENIC_INTX_WQ_RQ]);
+
+	pba = vnic_intr_legacy_pba(enic->legacy_pba);
+	if (!pba) {
+		vnic_intr_unmask(&enic->intr[ENIC_INTX_WQ_RQ]);
+		return IRQ_NONE;	/* not our interrupt */
+	}
+
+	if (ENIC_TEST_INTR(pba, ENIC_INTX_NOTIFY))
+		enic_notify_check(enic);
+
+	if (ENIC_TEST_INTR(pba, ENIC_INTX_ERR)) {
+		enic_log_q_error(enic);
+		/* schedule recovery from WQ/RQ error */
+		schedule_work(&enic->reset);
+		return IRQ_HANDLED;
+	}
+
+	if (ENIC_TEST_INTR(pba, ENIC_INTX_WQ_RQ)) {
+		if (netif_rx_schedule_prep(netdev, &enic->napi))
+			__netif_rx_schedule(netdev, &enic->napi);
+	} else {
+		vnic_intr_unmask(&enic->intr[ENIC_INTX_WQ_RQ]);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t enic_isr_msi(int irq, void *data)
+{
+	struct enic *enic = data;
+
+	/* With MSI, there is no sharing of interrupts, so this is
+	 * our interrupt and there is no need to ack it.  The device
+	 * is not providing per-vector masking, so the OS will not
+	 * write to PCI config space to mask/unmask the interrupt.
+	 * We're using mask_on_assertion for MSI, so the device
+	 * automatically masks the interrupt when the interrupt is
+	 * generated.  Later, when exiting polling, the interrupt
+	 * will be unmasked (see enic_poll).
+	 *
+	 * Also, the device uses the same PCIe Traffic Class (TC)
+	 * for Memory Write data and MSI, so there are no ordering
+	 * issues; the MSI will always arrive at the Root Complex
+	 * _after_ corresponding Memory Writes (i.e. descriptor
+	 * writes).
+	 */
+
+	netif_rx_schedule(enic->netdev, &enic->napi);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t enic_isr_msix_rq(int irq, void *data)
+{
+	struct enic *enic = data;
+
+	/* schedule NAPI polling for RQ cleanup */
+	netif_rx_schedule(enic->netdev, &enic->napi);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t enic_isr_msix_wq(int irq, void *data)
+{
+	struct enic *enic = data;
+	unsigned int wq_work_to_do = -1; /* no limit */
+	unsigned int wq_work_done;
+
+	wq_work_done = vnic_cq_service(&enic->cq[ENIC_CQ_WQ],
+		wq_work_to_do, enic_wq_service, NULL);
+
+	vnic_intr_return_credits(&enic->intr[ENIC_MSIX_WQ],
+		wq_work_done,
+		1 /* unmask intr */,
+		1 /* reset intr timer */);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t enic_isr_msix_err(int irq, void *data)
+{
+	struct enic *enic = data;
+
+	enic_log_q_error(enic);
+
+	/* schedule recovery from WQ/RQ error */
+	schedule_work(&enic->reset);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t enic_isr_msix_notify(int irq, void *data)
+{
+	struct enic *enic = data;
+
+	enic_notify_check(enic);
+	vnic_intr_unmask(&enic->intr[ENIC_MSIX_NOTIFY]);
+
+	return IRQ_HANDLED;
+}
+
+static inline void enic_queue_wq_skb_cont(struct enic *enic,
+	struct vnic_wq *wq, struct sk_buff *skb,
+	unsigned int len_left)
+{
+	skb_frag_t *frag;
+
+	/* Queue additional data fragments */
+	for (frag = skb_shinfo(skb)->frags; len_left; frag++) {
+		len_left -= frag->size;
+		enic_queue_wq_desc_cont(wq, skb,
+			pci_map_page(enic->pdev, frag->page,
+				frag->page_offset, frag->size,
+				PCI_DMA_TODEVICE),
+			frag->size,
+			(len_left == 0));	/* EOP? */
+	}
+}
+
+static inline void enic_queue_wq_skb_vlan(struct enic *enic,
+	struct vnic_wq *wq, struct sk_buff *skb,
+	int vlan_tag_insert, unsigned int vlan_tag)
+{
+	unsigned int head_len = skb_headlen(skb);
+	unsigned int len_left = skb->len - head_len;
+	int eop = (len_left == 0);
+
+	/* Queue the main skb fragment */
+	enic_queue_wq_desc(wq, skb,
+		pci_map_single(enic->pdev, skb->data,
+			head_len, PCI_DMA_TODEVICE),
+		head_len,
+		vlan_tag_insert, vlan_tag,