[NIU]: Add Sun Neptune ethernet driver.

With cleanup suggestions and bugs spotted by Stephen Hemminger,
Ingo Oeser, Matheos Worku, and Oliver Hartkopp.

Signed-off-by: David S. Miller <davem@davemloft.net>

4 files changed, 11169 insertions, 0 deletions

diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index 0e48b294200..9c635a237a9 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -2594,6 +2594,13 @@ config NETXEN_NIC
 	help
 	  This enables the support for NetXen's Gigabit Ethernet card.
 
+config NIU
+	tristate "Sun Neptune 10Gbit Ethernet support"
+	depends on PCI
+	help
+	  This enables support for cards based upon Sun's
+	  Neptune chipset.
+
 config PASEMI_MAC
 	tristate "PA Semi 1/10Gbit MAC"
 	depends on PPC64 && PCI
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index eb5c655e26d..d2e0f35da42 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -241,3 +241,4 @@ obj-$(CONFIG_NETCONSOLE) += netconsole.o
 obj-$(CONFIG_FS_ENET) += fs_enet/
 
 obj-$(CONFIG_NETXEN_NIC) += netxen/
+obj-$(CONFIG_NIU) += niu.o
diff --git a/drivers/net/niu.c b/drivers/net/niu.c
new file mode 100644
index 00000000000..43bfe7e6b6f
--- /dev/null
+++ b/drivers/net/niu.c
@@ -0,0 +1,7939 @@
+/* niu.c: Neptune ethernet driver.
+ *
+ * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/pci.h>
+#include <linux/dma-mapping.h>
+#include <linux/netdevice.h>
+#include <linux/ethtool.h>
+#include <linux/etherdevice.h>
+#include <linux/platform_device.h>
+#include <linux/delay.h>
+#include <linux/bitops.h>
+#include <linux/mii.h>
+#include <linux/if_ether.h>
+#include <linux/if_vlan.h>
+#include <linux/ip.h>
+#include <linux/in.h>
+#include <linux/ipv6.h>
+#include <linux/log2.h>
+#include <linux/jiffies.h>
+#include <linux/crc32.h>
+
+#include <linux/io.h>
+
+#ifdef CONFIG_SPARC64
+#include <linux/of_device.h>
+#endif
+
+#include "niu.h"
+
+#define DRV_MODULE_NAME		"niu"
+#define PFX DRV_MODULE_NAME	": "
+#define DRV_MODULE_VERSION	"0.5"
+#define DRV_MODULE_RELDATE	"October 5, 2007"
+
+static char version[] __devinitdata =
+	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
+
+MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
+MODULE_DESCRIPTION("NIU ethernet driver");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(DRV_MODULE_VERSION);
+
+#ifndef DMA_44BIT_MASK
+#define DMA_44BIT_MASK	0x00000fffffffffffULL
+#endif
+
+#ifndef readq
+static u64 readq(void __iomem *reg)
+{
+	return (((u64)readl(reg + 0x4UL) << 32) |
+		(u64)readl(reg));
+}
+
+static void writeq(u64 val, void __iomem *reg)
+{
+	writel(val & 0xffffffff, reg);
+	writel(val >> 32, reg + 0x4UL);
+}
+#endif
+
+static struct pci_device_id niu_pci_tbl[] = {
+	{PCI_DEVICE(PCI_VENDOR_ID_SUN, 0xabcd)},
+	{}
+};
+
+MODULE_DEVICE_TABLE(pci, niu_pci_tbl);
+
+#define NIU_TX_TIMEOUT			(5 * HZ)
+
+#define nr64(reg)		readq(np->regs + (reg))
+#define nw64(reg, val)		writeq((val), np->regs + (reg))
+
+#define nr64_mac(reg)		readq(np->mac_regs + (reg))
+#define nw64_mac(reg, val)	writeq((val), np->mac_regs + (reg))
+
+#define nr64_ipp(reg)		readq(np->regs + np->ipp_off + (reg))
+#define nw64_ipp(reg, val)	writeq((val), np->regs + np->ipp_off + (reg))
+
+#define nr64_pcs(reg)		readq(np->regs + np->pcs_off + (reg))
+#define nw64_pcs(reg, val)	writeq((val), np->regs + np->pcs_off + (reg))
+
+#define nr64_xpcs(reg)		readq(np->regs + np->xpcs_off + (reg))
+#define nw64_xpcs(reg, val)	writeq((val), np->regs + np->xpcs_off + (reg))
+
+#define NIU_MSG_DEFAULT (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
+
+static int niu_debug;
+static int debug = -1;
+module_param(debug, int, 0);
+MODULE_PARM_DESC(debug, "NIU debug level");
+
+#define niudbg(TYPE, f, a...) \
+do {	if ((np)->msg_enable & NETIF_MSG_##TYPE) \
+		printk(KERN_DEBUG PFX f, ## a); \
+} while (0)
+
+#define niuinfo(TYPE, f, a...) \
+do {	if ((np)->msg_enable & NETIF_MSG_##TYPE) \
+		printk(KERN_INFO PFX f, ## a); \
+} while (0)
+
+#define niuwarn(TYPE, f, a...) \
+do {	if ((np)->msg_enable & NETIF_MSG_##TYPE) \
+		printk(KERN_WARNING PFX f, ## a); \
+} while (0)
+
+#define niu_lock_parent(np, flags) \
+	spin_lock_irqsave(&np->parent->lock, flags)
+#define niu_unlock_parent(np, flags) \
+	spin_unlock_irqrestore(&np->parent->lock, flags)
+
+static int __niu_wait_bits_clear_mac(struct niu *np, unsigned long reg,
+				     u64 bits, int limit, int delay)
+{
+	while (--limit >= 0) {
+		u64 val = nr64_mac(reg);
+
+		if (!(val & bits))
+			break;
+		udelay(delay);
+	}
+	if (limit < 0)
+		return -ENODEV;
+	return 0;
+}
+
+static int __niu_set_and_wait_clear_mac(struct niu *np, unsigned long reg,
+					u64 bits, int limit, int delay,
+					const char *reg_name)
+{
+	int err;
+
+	nw64_mac(reg, bits);
+	err = __niu_wait_bits_clear_mac(np, reg, bits, limit, delay);
+	if (err)
+		dev_err(np->device, PFX "%s: bits (%llx) of register %s "
+			"would not clear, val[%llx]\n",
+			np->dev->name, (unsigned long long) bits, reg_name,
+			(unsigned long long) nr64_mac(reg));
+	return err;
+}
+
+#define niu_set_and_wait_clear_mac(NP, REG, BITS, LIMIT, DELAY, REG_NAME) \
+({	BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
+	__niu_set_and_wait_clear_mac(NP, REG, BITS, LIMIT, DELAY, REG_NAME); \
+})
+
+static int __niu_wait_bits_clear_ipp(struct niu *np, unsigned long reg,
+				     u64 bits, int limit, int delay)
+{
+	while (--limit >= 0) {
+		u64 val = nr64_ipp(reg);
+
+		if (!(val & bits))
+			break;
+		udelay(delay);
+	}
+	if (limit < 0)
+		return -ENODEV;
+	return 0;
+}
+
+static int __niu_set_and_wait_clear_ipp(struct niu *np, unsigned long reg,
+					u64 bits, int limit, int delay,
+					const char *reg_name)
+{
+	int err;
+	u64 val;
+
+	val = nr64_ipp(reg);
+	val |= bits;
+	nw64_ipp(reg, val);
+
+	err = __niu_wait_bits_clear_ipp(np, reg, bits, limit, delay);
+	if (err)
+		dev_err(np->device, PFX "%s: bits (%llx) of register %s "
+			"would not clear, val[%llx]\n",
+			np->dev->name, (unsigned long long) bits, reg_name,
+			(unsigned long long) nr64_ipp(reg));
+	return err;
+}
+
+#define niu_set_and_wait_clear_ipp(NP, REG, BITS, LIMIT, DELAY, REG_NAME) \
+({	BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
+	__niu_set_and_wait_clear_ipp(NP, REG, BITS, LIMIT, DELAY, REG_NAME); \
+})
+
+static int __niu_wait_bits_clear(struct niu *np, unsigned long reg,
+				 u64 bits, int limit, int delay)
+{
+	while (--limit >= 0) {
+		u64 val = nr64(reg);
+
+		if (!(val & bits))
+			break;
+		udelay(delay);
+	}
+	if (limit < 0)
+		return -ENODEV;
+	return 0;
+}
+
+#define niu_wait_bits_clear(NP, REG, BITS, LIMIT, DELAY) \
+({	BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
+	__niu_wait_bits_clear(NP, REG, BITS, LIMIT, DELAY); \
+})
+
+static int __niu_set_and_wait_clear(struct niu *np, unsigned long reg,
+				    u64 bits, int limit, int delay,
+				    const char *reg_name)
+{
+	int err;
+
+	nw64(reg, bits);
+	err = __niu_wait_bits_clear(np, reg, bits, limit, delay);
+	if (err)
+		dev_err(np->device, PFX "%s: bits (%llx) of register %s "
+			"would not clear, val[%llx]\n",
+			np->dev->name, (unsigned long long) bits, reg_name,
+			(unsigned long long) nr64(reg));
+	return err;
+}
+
+#define niu_set_and_wait_clear(NP, REG, BITS, LIMIT, DELAY, REG_NAME) \
+({	BUILD_BUG_ON(LIMIT <= 0 || DELAY < 0); \
+	__niu_set_and_wait_clear(NP, REG, BITS, LIMIT, DELAY, REG_NAME); \
+})
+
+static void niu_ldg_rearm(struct niu *np, struct niu_ldg *lp, int on)
+{
+	u64 val = (u64) lp->timer;
+
+	if (on)
+		val |= LDG_IMGMT_ARM;
+
+	nw64(LDG_IMGMT(lp->ldg_num), val);
+}
+
+static int niu_ldn_irq_enable(struct niu *np, int ldn, int on)
+{
+	unsigned long mask_reg, bits;
+	u64 val;
+
+	if (ldn < 0 || ldn > LDN_MAX)
+		return -EINVAL;
+
+	if (ldn < 64) {
+		mask_reg = LD_IM0(ldn);
+		bits = LD_IM0_MASK;
+	} else {
+		mask_reg = LD_IM1(ldn - 64);
+		bits = LD_IM1_MASK;
+	}
+
+	val = nr64(mask_reg);
+	if (on)
+		val &= ~bits;
+	else
+		val |= bits;
+	nw64(mask_reg, val);
+
+	return 0;
+}
+
+static int niu_enable_ldn_in_ldg(struct niu *np, struct niu_ldg *lp, int on)
+{
+	struct niu_parent *parent = np->parent;
+	int i;
+
+	for (i = 0; i <= LDN_MAX; i++) {
+		int err;
+
+		if (parent->ldg_map[i] != lp->ldg_num)
+			continue;
+
+		err = niu_ldn_irq_enable(np, i, on);
+		if (err)
+			return err;
+	}
+	return 0;
+}
+
+static int niu_enable_interrupts(struct niu *np, int on)
+{
+	int i;
+
+	for (i = 0; i < np->num_ldg; i++) {
+		struct niu_ldg *lp = &np->ldg[i];
+		int err;
+
+		err = niu_enable_ldn_in_ldg(np, lp, on);
+		if (err)
+			return err;
+	}
+	for (i = 0; i < np->num_ldg; i++)
+		niu_ldg_rearm(np, &np->ldg[i], on);
+
+	return 0;
+}
+
+static u32 phy_encode(u32 type, int port)
+{
+	return (type << (port * 2));
+}
+
+static u32 phy_decode(u32 val, int port)
+{
+	return (val >> (port * 2)) & PORT_TYPE_MASK;
+}
+
+static int mdio_wait(struct niu *np)
+{
+	int limit = 1000;
+	u64 val;
+
+	while (--limit > 0) {
+		val = nr64(MIF_FRAME_OUTPUT);
+		if ((val >> MIF_FRAME_OUTPUT_TA_SHIFT) & 0x1)
+			return val & MIF_FRAME_OUTPUT_DATA;
+
+		udelay(10);
+	}
+
+	return -ENODEV;
+}
+
+static int mdio_read(struct niu *np, int port, int dev, int reg)
+{
+	int err;
+
+	nw64(MIF_FRAME_OUTPUT, MDIO_ADDR_OP(port, dev, reg));
+	err = mdio_wait(np);
+	if (err < 0)
+		return err;
+
+	nw64(MIF_FRAME_OUTPUT, MDIO_READ_OP(port, dev));
+	return mdio_wait(np);
+}
+
+static int mdio_write(struct niu *np, int port, int dev, int reg, int data)
+{
+	int err;
+
+	nw64(MIF_FRAME_OUTPUT, MDIO_ADDR_OP(port, dev, reg));
+	err = mdio_wait(np);
+	if (err < 0)
+		return err;
+
+	nw64(MIF_FRAME_OUTPUT, MDIO_WRITE_OP(port, dev, data));
+	err = mdio_wait(np);
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+static int mii_read(struct niu *np, int port, int reg)
+{
+	nw64(MIF_FRAME_OUTPUT, MII_READ_OP(port, reg));
+	return mdio_wait(np);
+}
+
+static int mii_write(struct niu *np, int port, int reg, int data)
+{
+	int err;
+
+	nw64(MIF_FRAME_OUTPUT, MII_WRITE_OP(port, reg, data));
+	err = mdio_wait(np);
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+static int esr2_set_tx_cfg(struct niu *np, unsigned long channel, u32 val)
+{
+	int err;
+
+	err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
+			 ESR2_TI_PLL_TX_CFG_L(channel),
+			 val & 0xffff);
+	if (!err)
+		err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
+				 ESR2_TI_PLL_TX_CFG_H(channel),
+				 val >> 16);
+	return err;
+}
+
+static int esr2_set_rx_cfg(struct niu *np, unsigned long channel, u32 val)
+{
+	int err;
+
+	err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
+			 ESR2_TI_PLL_RX_CFG_L(channel),
+			 val & 0xffff);
+	if (!err)
+		err = mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
+				 ESR2_TI_PLL_RX_CFG_H(channel),
+				 val >> 16);
+	return err;
+}
+
+/* Mode is always 10G fiber.  */
+static int serdes_init_niu(struct niu *np)
+{
+	struct niu_link_config *lp = &np->link_config;
+	u32 tx_cfg, rx_cfg;
+	unsigned long i;
+
+	tx_cfg = (PLL_TX_CFG_ENTX | PLL_TX_CFG_SWING_1375MV);
+	rx_cfg = (PLL_RX_CFG_ENRX | PLL_RX_CFG_TERM_0P8VDDT |
+		  PLL_RX_CFG_ALIGN_ENA | PLL_RX_CFG_LOS_LTHRESH |
+		  PLL_RX_CFG_EQ_LP_ADAPTIVE);
+
+	if (lp->loopback_mode == LOOPBACK_PHY) {
+		u16 test_cfg = PLL_TEST_CFG_LOOPBACK_CML_DIS;
+
+		mdio_write(np, np->port, NIU_ESR2_DEV_ADDR,
+			   ESR2_TI_PLL_TEST_CFG_L, test_cfg);
+
+		tx_cfg |= PLL_TX_CFG_ENTEST;
+		rx_cfg |= PLL_RX_CFG_ENTEST;
+	}
+
+	/* Initialize all 4 lanes of the SERDES.  */
+	for (i = 0; i < 4; i++) {
+		int err = esr2_set_tx_cfg(np, i, tx_cfg);
+		if (err)
+			return err;
+	}
+
+	for (i = 0; i < 4; i++) {
+		int err = esr2_set_rx_cfg(np, i, rx_cfg);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int esr_read_rxtx_ctrl(struct niu *np, unsigned long chan, u32 *val)
+{
+	int err;
+
+	err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR, ESR_RXTX_CTRL_L(chan));
+	if (err >= 0) {
+		*val = (err & 0xffff);
+		err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
+				ESR_RXTX_CTRL_H(chan));
+		if (err >= 0)
+			*val |= ((err & 0xffff) << 16);
+		err = 0;
+	}
+	return err;
+}
+
+static int esr_read_glue0(struct niu *np, unsigned long chan, u32 *val)
+{
+	int err;
+
+	err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
+			ESR_GLUE_CTRL0_L(chan));
+	if (err >= 0) {
+		*val = (err & 0xffff);
+		err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
+				ESR_GLUE_CTRL0_H(chan));
+		if (err >= 0) {
+			*val |= ((err & 0xffff) << 16);
+			err = 0;
+		}
+	}
+	return err;
+}
+
+static int esr_read_reset(struct niu *np, u32 *val)
+{
+	int err;
+
+	err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
+			ESR_RXTX_RESET_CTRL_L);
+	if (err >= 0) {
+		*val = (err & 0xffff);
+		err = mdio_read(np, np->port, NIU_ESR_DEV_ADDR,
+				ESR_RXTX_RESET_CTRL_H);
+		if (err >= 0) {
+			*val |= ((err & 0xffff) << 16);
+			err = 0;
+		}
+	}
+	return err;
+}
+
+static int esr_write_rxtx_ctrl(struct niu *np, unsigned long chan, u32 val)
+{
+	int err;
+
+	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
+			 ESR_RXTX_CTRL_L(chan), val & 0xffff);
+	if (!err)
+		err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
+				 ESR_RXTX_CTRL_H(chan), (val >> 16));
+	return err;
+}
+
+static int esr_write_glue0(struct niu *np, unsigned long chan, u32 val)
+{
+	int err;
+
+	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
+			ESR_GLUE_CTRL0_L(chan), val & 0xffff);
+	if (!err)
+		err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
+				 ESR_GLUE_CTRL0_H(chan), (val >> 16));
+	return err;
+}
+
+static int esr_reset(struct niu *np)
+{
+	u32 reset;
+	int err;
+
+	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
+			 ESR_RXTX_RESET_CTRL_L, 0x0000);
+	if (err)
+		return err;
+	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
+			 ESR_RXTX_RESET_CTRL_H, 0xffff);
+	if (err)
+		return err;
+	udelay(200);
+
+	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
+			 ESR_RXTX_RESET_CTRL_L, 0xffff);
+	if (err)
+		return err;
+	udelay(200);
+
+	err = mdio_write(np, np->port, NIU_ESR_DEV_ADDR,
+			 ESR_RXTX_RESET_CTRL_H, 0x0000);
+	if (err)
+		return err;
+	udelay(200);
+
+	err = esr_read_reset(np, &reset);
+	if (err)
+		return err;
+	if (reset != 0) {
+		dev_err(np->device, PFX "Port %u ESR_RESET "
+			"did not clear [%08x]\n",
+			np->port, reset);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int serdes_init_10g(struct niu *np)
+{
+	struct niu_link_config *lp = &np->link_config;
+	unsigned long ctrl_reg, test_cfg_reg, i;
+	u64 ctrl_val, test_cfg_val, sig, mask, val;
+	int err;
+
+	switch (np->port) {
+	case 0:
+		ctrl_reg = ENET_SERDES_0_CTRL_CFG;
+		test_cfg_reg = ENET_SERDES_0_TEST_CFG;
+		break;
+	case 1:
+		ctrl_reg = ENET_SERDES_1_CTRL_CFG;
+		test_cfg_reg = ENET_SERDES_1_TEST_CFG;
+		break;
+
+	default:
+		return -EINVAL;
+	}
+	ctrl_val = (ENET_SERDES_CTRL_SDET_0 |
+		    ENET_SERDES_CTRL_SDET_1 |
+		    ENET_SERDES_CTRL_SDET_2 |
+		    ENET_SERDES_CTRL_SDET_3 |
+		    (0x5 << ENET_SERDES_CTRL_EMPH_0_SHIFT) |
+		    (0x5 << ENET_SERDES_CTRL_EMPH_1_SHIFT) |
+		    (0x5 << ENET_SERDES_CTRL_EMPH_2_SHIFT) |
+		    (0x5 << ENET_SERDES_CTRL_EMPH_3_SHIFT) |
+		    (0x1 << ENET_SERDES_CTRL_LADJ_0_SHIFT) |
+		    (0x1 << ENET_SERDES_CTRL_LADJ_1_SHIFT) |
+		    (0x1 << ENET_SERDES_CTRL_LADJ_2_SHIFT) |
+		    (0x1 << ENET_SERDES_CTRL_LADJ_3_SHIFT));
+	test_cfg_val = 0;
+
+	if (lp->loopback_mode == LOOPBACK_PHY) {
+		test_cfg_val |= ((ENET_TEST_MD_PAD_LOOPBACK <<
+				  ENET_SERDES_TEST_MD_0_SHIFT) |
+				 (ENET_TEST_MD_PAD_LOOPBACK <<
+				  ENET_SERDES_TEST_MD_1_SHIFT) |
+				 (ENET_TEST_MD_PAD_LOOPBACK <<
+				  ENET_SERDES_TEST_MD_2_SHIFT) |
+				 (ENET_TEST_MD_PAD_LOOPBACK <<
+				  ENET_SERDES_TEST_MD_3_SHIFT));
+	}
+
+	nw64(ctrl_reg, ctrl_val);
+	nw64(test_cfg_reg, test_cfg_val);
+
+	/* Initialize all 4 lanes of the SERDES.  */
+	for (i = 0; i < 4; i++) {
+		u32 rxtx_ctrl, glue0;
+
+		err = esr_read_rxtx_ctrl(np, i, &rxtx_ctrl);
+		if (err)
+			return err;
+		err = esr_read_glue0(np, i, &glue0);
+		if (err)
+			return err;
+
+		rxtx_ctrl &= ~(ESR_RXTX_CTRL_VMUXLO);
+		rxtx_ctrl |= (ESR_RXTX_CTRL_ENSTRETCH |
+			      (2 << ESR_RXTX_CTRL_VMUXLO_SHIFT));
+
+		glue0 &= ~(ESR_GLUE_CTRL0_SRATE |
+			   ESR_GLUE_CTRL0_THCNT |
+			   ESR_GLUE_CTRL0_BLTIME);
+		glue0 |= (ESR_GLUE_CTRL0_RXLOSENAB |
+			  (0xf << ESR_GLUE_CTRL0_SRATE_SHIFT) |
+			  (0xff << ESR_GLUE_CTRL0_THCNT_SHIFT) |
+			  (BLTIME_300_CYCLES <<
+			   ESR_GLUE_CTRL0_BLTIME_SHIFT));
+
+		err = esr_write_rxtx_ctrl(np, i, rxtx_ctrl);
+		if (err)
+			return err;
+		err = esr_write_glue0(np, i, glue0);
+		if (err)
+			return err;
+	}
+
+	err = esr_reset(np);
+	if (err)
+		return err;
+
+	sig = nr64(ESR_INT_SIGNALS);
+	switch (np->port) {
+	case 0:
+		mask = ESR_INT_SIGNALS_P0_BITS;
+		val = (ESR_INT_SRDY0_P0 |
+		       ESR_INT_DET0_P0 |
+		       ESR_INT_XSRDY_P0 |
+		       ESR_INT_XDP_P0_CH3 |
+		       ESR_INT_XDP_P0_CH2 |
+		       ESR_INT_XDP_P0_CH1 |
+		       ESR_INT_XDP_P0_CH0);
+		break;
+
+	case 1:
+		mask = ESR_INT_SIGNALS_P1_BITS;
+		val = (ESR_INT_SRDY0_P1 |
+		       ESR_INT_DET0_P1 |
+		       ESR_INT_XSRDY_P1 |
+		       ESR_INT_XDP_P1_CH3 |
+		       ESR_INT_XDP_P1_CH2 |
+		       ESR_INT_XDP_P1_CH1 |
+		       ESR_INT_XDP_P1_CH0);
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	if ((sig & mask) != val) {
+		dev_err(np->device, PFX "Port %u signal bits [%08x] are not "
+			"[%08x]\n", np->port, (int) (sig & mask), (int) val);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int serdes_init_1g(struct niu *np)
+{
+	u64 val;
+
+	val = nr64(ENET_SERDES_1_PLL_CFG);
+	val &= ~ENET_SERDES_PLL_FBDIV2;
+	switch (np->port) {
+	case 0:
+		val |= ENET_SERDES_PLL_HRATE0;
+		break;
+	case 1:
+		val |= ENET_SERDES_PLL_HRATE1;
+		break;
+	case 2:
+		val |= ENET_SERDES_PLL_HRATE2;
+		break;
+	case 3:
+		val |= ENET_SERDES_PLL_HRATE3;
+		break;
+	default:
+		return -EINVAL;
+	}
+	nw64(ENET_SERDES_1_PLL_CFG, val);
+
+	return 0;
+}
+
+static int bcm8704_reset(struct niu *np)
+{
+	int err, limit;
+
+	err = mdio_read(np, np->phy_addr,
+			BCM8704_PHYXS_DEV_ADDR, MII_BMCR);
+	if (err < 0)
+		return err;
+	err |= BMCR_RESET;
+	err = mdio_write(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
+			 MII_BMCR, err);
+	if (err)
+		return err;
+
+	limit = 1000;
+	while (--limit >= 0) {
+		err = mdio_read(np, np->phy_addr,
+				BCM8704_PHYXS_DEV_ADDR, MII_BMCR);
+		if (err < 0)
+			return err;
+		if (!(err & BMCR_RESET))
+			break;
+	}
+	if (limit < 0) {
+		dev_err(np->device, PFX "Port %u PHY will not reset "
+			"(bmcr=%04x)\n", np->port, (err & 0xffff));
+		return -ENODEV;
+	}
+	return 0;
+}
+
+/* When written, certain PHY registers need to be read back twice
+ * in order for the bits to settle properly.
+ */
+static int bcm8704_user_dev3_readback(struct niu *np, int reg)
+{
+	int err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, reg);
+	if (err < 0)
+		return err;
+	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, reg);
+	if (err < 0)
+		return err;
+	return 0;
+}
+
+static int bcm8704_init_user_dev3(struct niu *np)
+{
+	int err;
+
+	err = mdio_write(np, np->phy_addr,
+			 BCM8704_USER_DEV3_ADDR, BCM8704_USER_CONTROL,
+			 (USER_CONTROL_OPTXRST_LVL |
+			  USER_CONTROL_OPBIASFLT_LVL |
+			  USER_CONTROL_OBTMPFLT_LVL |
+			  USER_CONTROL_OPPRFLT_LVL |
+			  USER_CONTROL_OPTXFLT_LVL |
+			  USER_CONTROL_OPRXLOS_LVL |
+			  USER_CONTROL_OPRXFLT_LVL |
+			  USER_CONTROL_OPTXON_LVL |
+			  (0x3f << USER_CONTROL_RES1_SHIFT)));
+	if (err)
+		return err;
+
+	err = mdio_write(np, np->phy_addr,
+			 BCM8704_USER_DEV3_ADDR, BCM8704_USER_PMD_TX_CONTROL,
+			 (USER_PMD_TX_CTL_XFP_CLKEN |
+			  (1 << USER_PMD_TX_CTL_TX_DAC_TXD_SH) |
+			  (2 << USER_PMD_TX_CTL_TX_DAC_TXCK_SH) |
+			  USER_PMD_TX_CTL_TSCK_LPWREN));
+	if (err)
+		return err;
+
+	err = bcm8704_user_dev3_readback(np, BCM8704_USER_CONTROL);
+	if (err)
+		return err;
+	err = bcm8704_user_dev3_readback(np, BCM8704_USER_PMD_TX_CONTROL);
+	if (err)
+		return err;
+
+	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
+			BCM8704_USER_OPT_DIGITAL_CTRL);
+	if (err < 0)
+		return err;
+	err &= ~USER_ODIG_CTRL_GPIOS;
+	err |= (0x3 << USER_ODIG_CTRL_GPIOS_SHIFT);
+	err = mdio_write(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
+			 BCM8704_USER_OPT_DIGITAL_CTRL, err);
+	if (err)
+		return err;
+
+	mdelay(1000);
+
+	return 0;
+}
+
+static int xcvr_init_10g(struct niu *np)
+{
+	struct niu_link_config *lp = &np->link_config;
+	u16 analog_stat0, tx_alarm_status;
+	int err;
+	u64 val;
+
+	val = nr64_mac(XMAC_CONFIG);
+	val &= ~XMAC_CONFIG_LED_POLARITY;
+	val |= XMAC_CONFIG_FORCE_LED_ON;
+	nw64_mac(XMAC_CONFIG, val);
+
+	/* XXX shared resource, lock parent XXX */
+	val = nr64(MIF_CONFIG);
+	val |= MIF_CONFIG_INDIRECT_MODE;
+	nw64(MIF_CONFIG, val);
+
+	err = bcm8704_reset(np);
+	if (err)
+		return err;
+
+	err = bcm8704_init_user_dev3(np);
+	if (err)
+		return err;
+
+	err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
+			MII_BMCR);
+	if (err < 0)
+		return err;
+	err &= ~BMCR_LOOPBACK;
+
+	if (lp->loopback_mode == LOOPBACK_MAC)
+		err |= BMCR_LOOPBACK;
+
+	err = mdio_write(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
+			 MII_BMCR, err);
+	if (err)
+		return err;
+
+#if 1
+	err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,
+			MII_STAT1000);
+	if (err < 0)
+		return err;
+	pr_info(PFX "Port %u PMA_PMD(MII_STAT1000) [%04x]\n",
+		np->port, err);
+
+	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR, 0x20);
+	if (err < 0)
+		return err;
+	pr_info(PFX "Port %u USER_DEV3(0x20) [%04x]\n",
+		np->port, err);
+
+	err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
+			MII_NWAYTEST);
+	if (err < 0)
+		return err;
+	pr_info(PFX "Port %u PHYXS(MII_NWAYTEST) [%04x]\n",
+		np->port, err);
+#endif
+
+	/* XXX dig this out it might not be so useful XXX */
+	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
+			BCM8704_USER_ANALOG_STATUS0);
+	if (err < 0)
+		return err;
+	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
+			BCM8704_USER_ANALOG_STATUS0);
+	if (err < 0)
+		return err;
+	analog_stat0 = err;
+
+	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
+			BCM8704_USER_TX_ALARM_STATUS);
+	if (err < 0)
+		return err;
+	err = mdio_read(np, np->phy_addr, BCM8704_USER_DEV3_ADDR,
+			BCM8704_USER_TX_ALARM_STATUS);
+	if (err < 0)
+		return err;
+	tx_alarm_status = err;
+
+	if (analog_stat0 != 0x03fc) {
+		if ((analog_stat0 == 0x43bc) && (tx_alarm_status != 0)) {
+			pr_info(PFX "Port %u cable not connected "
+				"or bad cable.\n", np->port);
+		} else if (analog_stat0 == 0x639c) {
+			pr_info(PFX "Port %u optical module is bad "
+				"or missing.\n", np->port);
+		}
+	}
+
+	return 0;
+}
+
+static int mii_reset(struct niu *np)
+{
+	int limit, err;
+
+	err = mii_write(np, np->phy_addr, MII_BMCR, BMCR_RESET);
+	if (err)
+		return err;
+
+	limit = 1000;
+	while (--limit >= 0) {
+		udelay(500);
+		err = mii_read(np, np->phy_addr, MII_BMCR);
+		if (err < 0)
+			return err;
+		if (!(err & BMCR_RESET))
+			break;
+	}
+	if (limit < 0) {
+		dev_err(np->device, PFX "Port %u MII would not reset, "
+			"bmcr[%04x]\n", np->port, err);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int mii_init_common(struct niu *np)
+{
+	struct niu_link_config *lp = &np->link_config;
+	u16 bmcr, bmsr, adv, estat;
+	int err;
+
+	err = mii_reset(np);
+	if (err)
+		return err;
+
+	err = mii_read(np, np->phy_addr, MII_BMSR);
+	if (err < 0)
+		return err;
+	bmsr = err;
+
+	estat = 0;
+	if (bmsr & BMSR_ESTATEN) {
+		err = mii_read(np, np->phy_addr, MII_ESTATUS);
+		if (err < 0)
+			return err;
+		estat = err;
+	}
+
+	bmcr = 0;
+	err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);
+	if (err)
+		return err;
+
+	if (lp->loopback_mode == LOOPBACK_MAC) {
+		bmcr |= BMCR_LOOPBACK;
+		if (lp->active_speed == SPEED_1000)
+			bmcr |= BMCR_SPEED1000;
+		if (lp->active_duplex == DUPLEX_FULL)
+			bmcr |= BMCR_FULLDPLX;
+	}
+
+	if (lp->loopback_mode == LOOPBACK_PHY) {
+		u16 aux;
+
+		aux = (BCM5464R_AUX_CTL_EXT_LB |
+		       BCM5464R_AUX_CTL_WRITE_1);
+		err = mii_write(np, np->phy_addr, BCM5464R_AUX_CTL, aux);
+		if (err)
+			return err;
+	}
+
+	/* XXX configurable XXX */
+	/* XXX for now don't advertise half-duplex or asym pause... XXX */
+	adv = ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP;
+	if (bmsr & BMSR_10FULL)
+		adv |= ADVERTISE_10FULL;
+	if (bmsr & BMSR_100FULL)
+		adv |= ADVERTISE_100FULL;
+	err = mii_write(np, np->phy_addr, MII_ADVERTISE, adv);
+	if (err)
+		return err;
+
+	if (bmsr & BMSR_ESTATEN) {
+		u16 ctrl1000 = 0;
+
+		if (estat & ESTATUS_1000_TFULL)
+			ctrl1000 |= ADVERTISE_1000FULL;
+		err = mii_write(np, np->phy_addr, MII_CTRL1000, ctrl1000);
+		if (err)
+			return err;
+	}
+	bmcr |= (BMCR_ANENABLE | BMCR_ANRESTART);
+
+	err = mii_write(np, np->phy_addr, MII_BMCR, bmcr);
+	if (err)
+		return err;
+
+	err = mii_read(np, np->phy_addr, MII_BMCR);
+	if (err < 0)
+		return err;
+	err = mii_read(np, np->phy_addr, MII_BMSR);
+	if (err < 0)
+		return err;
+#if 0
+	pr_info(PFX "Port %u after MII init bmcr[%04x] bmsr[%04x]\n",
+		np->port, bmcr, bmsr);
+#endif
+
+	return 0;
+}
+
+static int xcvr_init_1g(struct niu *np)
+{
+	u64 val;
+
+	/* XXX shared resource, lock parent XXX */
+	val = nr64(MIF_CONFIG);
+	val &= ~MIF_CONFIG_INDIRECT_MODE;
+	nw64(MIF_CONFIG, val);
+
+	return mii_init_common(np);
+}
+
+static int niu_xcvr_init(struct niu *np)
+{
+	const struct niu_phy_ops *ops = np->phy_ops;
+	int err;
+
+	err = 0;
+	if (ops->xcvr_init)
+		err = ops->xcvr_init(np);
+
+	return err;
+}
+
+static int niu_serdes_init(struct niu *np)
+{
+	const struct niu_phy_ops *ops = np->phy_ops;
+	int err;
+
+	err = 0;
+	if (ops->serdes_init)
+		err = ops->serdes_init(np);
+
+	return err;
+}
+
+static void niu_init_xif(struct niu *);
+
+static int niu_link_status_common(struct niu *np, int link_up)
+{
+	struct niu_link_config *lp = &np->link_config;
+	struct net_device *dev = np->dev;
+	unsigned long flags;
+
+	if (!netif_carrier_ok(dev) && link_up) {
+		niuinfo(LINK, "%s: Link is up at %s, %s duplex\n",
+		       dev->name,
+		       (lp->active_speed == SPEED_10000 ?
+			"10Gb/sec" :
+			(lp->active_speed == SPEED_1000 ?
+			 "1Gb/sec" :
+			 (lp->active_speed == SPEED_100 ?
+			  "100Mbit/sec" : "10Mbit/sec"))),
+		       (lp->active_duplex == DUPLEX_FULL ?
+			"full" : "half"));
+
+		spin_lock_irqsave(&np->lock, flags);
+		niu_init_xif(np);
+		spin_unlock_irqrestore(&np->lock, flags);
+
+		netif_carrier_on(dev);
+	} else if (netif_carrier_ok(dev) && !link_up) {
+		niuwarn(LINK, "%s: Link is down\n", dev->name);
+		netif_carrier_off(dev);
+	}
+
+	return 0;
+}
+
+static int link_status_10g(struct niu *np, int *link_up_p)
+{
+	unsigned long flags;
+	int err, link_up;
+
+	link_up = 0;
+
+	spin_lock_irqsave(&np->lock, flags);
+
+	err = -EINVAL;
+	if (np->link_config.loopback_mode != LOOPBACK_DISABLED)
+		goto out;
+
+	err = mdio_read(np, np->phy_addr, BCM8704_PMA_PMD_DEV_ADDR,
+			BCM8704_PMD_RCV_SIGDET);
+	if (err < 0)
+		goto out;
+	if (!(err & PMD_RCV_SIGDET_GLOBAL)) {
+		err = 0;
+		goto out;
+	}
+
+	err = mdio_read(np, np->phy_addr, BCM8704_PCS_DEV_ADDR,
+			BCM8704_PCS_10G_R_STATUS);
+	if (err < 0)
+		goto out;
+	if (!(err & PCS_10G_R_STATUS_BLK_LOCK)) {
+		err = 0;
+		goto out;
+	}
+
+	err = mdio_read(np, np->phy_addr, BCM8704_PHYXS_DEV_ADDR,
+			BCM8704_PHYXS_XGXS_LANE_STAT);
+	if (err < 0)
+		goto out;
+
+	if (err != (PHYXS_XGXS_LANE_STAT_ALINGED |
+		    PHYXS_XGXS_LANE_STAT_MAGIC |
+		    PHYXS_XGXS_LANE_STAT_LANE3 |
+		    PHYXS_XGXS_LANE_STAT_LANE2 |
+		    PHYXS_XGXS_LANE_STAT_LANE1 |
+		    PHYXS_XGXS_LANE_STAT_LANE0)) {
+		err = 0;
+		goto out;
+	}
+
+	link_up = 1;
+	np->link_config.active_speed = SPEED_10000;
+	np->link_config.active_duplex = DUPLEX_FULL;
+	err = 0;
+
+out:
+	spin_unlock_irqrestore(&np->lock, flags);
+
+	*link_up_p = link_up;
+	return err;
+}
+
+static int link_status_1g(struct niu *np, int *link_up_p)
+{
+	u16 current_speed, bmsr;
+	unsigned long flags;
+	u8 current_duplex;
+	int err, link_up;
+
+	link_up = 0;
+	current_speed = SPEED_INVALID;
+	current_duplex = DUPLEX_INVALID;
+
+	spin_lock_irqsave(&np->lock, flags);
+
+	err = -EINVAL;
+	if (np->link_config.loopback_mode != LOOPBACK_DISABLED)
+		goto out;
+
+	err = mii_read(np, np->phy_addr, MII_BMSR);
+	if (err < 0)
+		goto out;
+
+	bmsr = err;
+	if (bmsr & BMSR_LSTATUS) {
+		u16 adv, lpa, common, estat;
+
+		err = mii_read(np, np->phy_addr, MII_ADVERTISE);
+		if (err < 0)
+			goto out;
+		adv = err;
+
+		err = mii_read(np, np->phy_addr, MII_LPA);
+		if (err < 0)
+			goto out;
+		lpa = err;
+
+		common = adv & lpa;
+
+		err = mii_read(np, np->phy_addr, MII_ESTATUS);
+		if (err < 0)
+			goto out;
+		estat = err;
+
+		link_up = 1;
+		if (estat & (ESTATUS_1000_TFULL | ESTATUS_1000_THALF)) {
+			current_speed = SPEED_1000;
+			if (estat & ESTATUS_1000_TFULL)
+				current_duplex = DUPLEX_FULL;
+			else
+				current_duplex = DUPLEX_HALF;
+		} else {
+			if (common & ADVERTISE_100BASE4) {
+				current_speed = SPEED_100;
+				current_duplex = DUPLEX_HALF;
+			} else if (common & ADVERTISE_100FULL) {
+				current_speed = SPEED_100;
+				current_duplex = DUPLEX_FULL;
+			} else if (common & ADVERTISE_100HALF) {
+				current_speed = SPEED_100;
+				current_duplex = DUPLEX_HALF;
+			} else if (common & ADVERTISE_10FULL) {
+				current_speed = SPEED_10;
+				current_duplex = DUPLEX_FULL;
+			} else if (common & ADVERTISE_10HALF) {
+				current_speed = SPEED_10;
+				current_duplex = DUPLEX_HALF;
+			} else
+				link_up = 0;
+		}
+	}
+	err = 0;
+
+out:
+	spin_unlock_irqrestore(&np->lock, flags);
+
+	*link_up_p = link_up;
+	return err;
+}
+
+static int niu_link_status(struct niu *np, int *link_up_p)
+{
+	const struct niu_phy_ops *ops = np->phy_ops;
+	int err;
+