diff options
Diffstat (limited to 'drivers/net/tg3.c')
| -rw-r--r-- | drivers/net/tg3.c | 1254 |
1 files changed, 853 insertions, 401 deletions
diff --git a/drivers/net/tg3.c b/drivers/net/tg3.c index ba5d3fe753b..3a74d216859 100644 --- a/drivers/net/tg3.c +++ b/drivers/net/tg3.c @@ -68,8 +68,8 @@ #define DRV_MODULE_NAME "tg3" #define PFX DRV_MODULE_NAME ": " -#define DRV_MODULE_VERSION "3.102" -#define DRV_MODULE_RELDATE "September 1, 2009" +#define DRV_MODULE_VERSION "3.105" +#define DRV_MODULE_RELDATE "December 2, 2009" #define TG3_DEF_MAC_MODE 0 #define TG3_DEF_RX_MODE 0 @@ -137,6 +137,12 @@ #define TG3_RX_STD_MAP_SZ TG3_RX_DMA_TO_MAP_SZ(TG3_RX_STD_DMA_SZ) #define TG3_RX_JMB_MAP_SZ TG3_RX_DMA_TO_MAP_SZ(TG3_RX_JMB_DMA_SZ) +#define TG3_RX_STD_BUFF_RING_SIZE \ + (sizeof(struct ring_info) * TG3_RX_RING_SIZE) + +#define TG3_RX_JMB_BUFF_RING_SIZE \ + (sizeof(struct ring_info) * TG3_RX_JUMBO_RING_SIZE) + /* minimum number of free TX descriptors required to wake up TX process */ #define TG3_TX_WAKEUP_THRESH(tnapi) ((tnapi)->tx_pending / 4) @@ -235,6 +241,9 @@ static struct pci_device_id tg3_pci_tbl[] = { {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57760)}, {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57790)}, {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_57788)}, + {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5717)}, + {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5718)}, + {PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, TG3PCI_DEVICE_TIGON3_5724)}, {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)}, {PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)}, {PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)}, @@ -396,7 +405,7 @@ static void tg3_write_indirect_mbox(struct tg3 *tp, u32 off, u32 val) TG3_64BIT_REG_LOW, val); return; } - if (off == (MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW)) { + if (off == TG3_RX_STD_PROD_IDX_REG) { pci_write_config_dword(tp->pdev, TG3PCI_STD_RING_PROD_IDX + TG3_64BIT_REG_LOW, val); return; @@ -937,9 +946,10 @@ static void tg3_mdio_config_5785(struct tg3 *tp) u32 val; struct phy_device *phydev; - phydev = tp->mdio_bus->phy_map[PHY_ADDR]; + phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]; switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) { case TG3_PHY_ID_BCM50610: + case TG3_PHY_ID_BCM50610M: val = MAC_PHYCFG2_50610_LED_MODES; break; case TG3_PHY_ID_BCMAC131: @@ -1031,7 +1041,7 @@ static void tg3_mdio_start(struct tg3 *tp) if (is_serdes) tp->phy_addr += 7; } else - tp->phy_addr = PHY_ADDR; + tp->phy_addr = TG3_PHY_MII_ADDR; if ((tp->tg3_flags3 & TG3_FLG3_MDIOBUS_INITED) && GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5785) @@ -1062,7 +1072,7 @@ static int tg3_mdio_init(struct tg3 *tp) tp->mdio_bus->read = &tg3_mdio_read; tp->mdio_bus->write = &tg3_mdio_write; tp->mdio_bus->reset = &tg3_mdio_reset; - tp->mdio_bus->phy_mask = ~(1 << PHY_ADDR); + tp->mdio_bus->phy_mask = ~(1 << TG3_PHY_MII_ADDR); tp->mdio_bus->irq = &tp->mdio_irq[0]; for (i = 0; i < PHY_MAX_ADDR; i++) @@ -1084,7 +1094,7 @@ static int tg3_mdio_init(struct tg3 *tp) return i; } - phydev = tp->mdio_bus->phy_map[PHY_ADDR]; + phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]; if (!phydev || !phydev->drv) { printk(KERN_WARNING "%s: No PHY devices\n", tp->dev->name); @@ -1096,8 +1106,14 @@ static int tg3_mdio_init(struct tg3 *tp) switch (phydev->drv->phy_id & phydev->drv->phy_id_mask) { case TG3_PHY_ID_BCM57780: phydev->interface = PHY_INTERFACE_MODE_GMII; + phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE; break; case TG3_PHY_ID_BCM50610: + case TG3_PHY_ID_BCM50610M: + phydev->dev_flags |= PHY_BRCM_CLEAR_RGMII_MODE | + PHY_BRCM_RX_REFCLK_UNUSED | + PHY_BRCM_DIS_TXCRXC_NOENRGY | + PHY_BRCM_AUTO_PWRDWN_ENABLE; if (tp->tg3_flags3 & TG3_FLG3_RGMII_STD_IBND_DISABLE) phydev->dev_flags |= PHY_BRCM_STD_IBND_DISABLE; if (tp->tg3_flags3 & TG3_FLG3_RGMII_EXT_IBND_RX_EN) @@ -1111,6 +1127,7 @@ static int tg3_mdio_init(struct tg3 *tp) case TG3_PHY_ID_RTL8201E: case TG3_PHY_ID_BCMAC131: phydev->interface = PHY_INTERFACE_MODE_MII; + phydev->dev_flags |= PHY_BRCM_AUTO_PWRDWN_ENABLE; tp->tg3_flags3 |= TG3_FLG3_PHY_IS_FET; break; } @@ -1311,7 +1328,7 @@ static void tg3_setup_flow_control(struct tg3 *tp, u32 lcladv, u32 rmtadv) u32 old_tx_mode = tp->tx_mode; if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) - autoneg = tp->mdio_bus->phy_map[PHY_ADDR]->autoneg; + autoneg = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]->autoneg; else autoneg = tp->link_config.autoneg; @@ -1348,7 +1365,7 @@ static void tg3_adjust_link(struct net_device *dev) u8 oldflowctrl, linkmesg = 0; u32 mac_mode, lcl_adv, rmt_adv; struct tg3 *tp = netdev_priv(dev); - struct phy_device *phydev = tp->mdio_bus->phy_map[PHY_ADDR]; + struct phy_device *phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]; spin_lock_bh(&tp->lock); @@ -1363,8 +1380,11 @@ static void tg3_adjust_link(struct net_device *dev) if (phydev->speed == SPEED_100 || phydev->speed == SPEED_10) mac_mode |= MAC_MODE_PORT_MODE_MII; - else + else if (phydev->speed == SPEED_1000 || + GET_ASIC_REV(tp->pci_chip_rev_id) != ASIC_REV_5785) mac_mode |= MAC_MODE_PORT_MODE_GMII; + else + mac_mode |= MAC_MODE_PORT_MODE_MII; if (phydev->duplex == DUPLEX_HALF) mac_mode |= MAC_MODE_HALF_DUPLEX; @@ -1434,7 +1454,7 @@ static int tg3_phy_init(struct tg3 *tp) /* Bring the PHY back to a known state. */ tg3_bmcr_reset(tp); - phydev = tp->mdio_bus->phy_map[PHY_ADDR]; + phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]; /* Attach the MAC to the PHY. */ phydev = phy_connect(tp->dev, dev_name(&phydev->dev), tg3_adjust_link, @@ -1461,7 +1481,7 @@ static int tg3_phy_init(struct tg3 *tp) SUPPORTED_Asym_Pause); break; default: - phy_disconnect(tp->mdio_bus->phy_map[PHY_ADDR]); + phy_disconnect(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]); return -EINVAL; } @@ -1479,7 +1499,7 @@ static void tg3_phy_start(struct tg3 *tp) if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED)) return; - phydev = tp->mdio_bus->phy_map[PHY_ADDR]; + phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]; if (tp->link_config.phy_is_low_power) { tp->link_config.phy_is_low_power = 0; @@ -1499,13 +1519,13 @@ static void tg3_phy_stop(struct tg3 *tp) if (!(tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED)) return; - phy_stop(tp->mdio_bus->phy_map[PHY_ADDR]); + phy_stop(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]); } static void tg3_phy_fini(struct tg3 *tp) { if (tp->tg3_flags3 & TG3_FLG3_PHY_CONNECTED) { - phy_disconnect(tp->mdio_bus->phy_map[PHY_ADDR]); + phy_disconnect(tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]); tp->tg3_flags3 &= ~TG3_FLG3_PHY_CONNECTED; } } @@ -2149,6 +2169,26 @@ static void tg3_power_down_phy(struct tg3 *tp, bool do_low_power) tw32_f(GRC_MISC_CFG, val | GRC_MISC_CFG_EPHY_IDDQ); udelay(40); return; + } else if (tp->tg3_flags3 & TG3_FLG3_PHY_IS_FET) { + u32 phytest; + if (!tg3_readphy(tp, MII_TG3_FET_TEST, &phytest)) { + u32 phy; + + tg3_writephy(tp, MII_ADVERTISE, 0); + tg3_writephy(tp, MII_BMCR, + BMCR_ANENABLE | BMCR_ANRESTART); + + tg3_writephy(tp, MII_TG3_FET_TEST, + phytest | MII_TG3_FET_SHADOW_EN); + if (!tg3_readphy(tp, MII_TG3_FET_SHDW_AUXMODE4, &phy)) { + phy |= MII_TG3_FET_SHDW_AUXMODE4_SBPD; + tg3_writephy(tp, + MII_TG3_FET_SHDW_AUXMODE4, + phy); + } + tg3_writephy(tp, MII_TG3_FET_TEST, phytest); + } + return; } else if (do_low_power) { tg3_writephy(tp, MII_TG3_EXT_CTRL, MII_TG3_EXT_CTRL_FORCE_LED_OFF); @@ -2218,7 +2258,7 @@ static void tg3_nvram_unlock(struct tg3 *tp) static void tg3_enable_nvram_access(struct tg3 *tp) { if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) && - !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) { + !(tp->tg3_flags3 & TG3_FLG3_PROTECTED_NVRAM)) { u32 nvaccess = tr32(NVRAM_ACCESS); tw32(NVRAM_ACCESS, nvaccess | ACCESS_ENABLE); @@ -2229,7 +2269,7 @@ static void tg3_enable_nvram_access(struct tg3 *tp) static void tg3_disable_nvram_access(struct tg3 *tp) { if ((tp->tg3_flags2 & TG3_FLG2_5750_PLUS) && - !(tp->tg3_flags2 & TG3_FLG2_PROTECTED_NVRAM)) { + !(tp->tg3_flags3 & TG3_FLG3_PROTECTED_NVRAM)) { u32 nvaccess = tr32(NVRAM_ACCESS); tw32(NVRAM_ACCESS, nvaccess & ~ACCESS_ENABLE); @@ -2474,7 +2514,7 @@ static int tg3_set_power_state(struct tg3 *tp, pci_power_t state) struct phy_device *phydev; u32 phyid, advertising; - phydev = tp->mdio_bus->phy_map[PHY_ADDR]; + phydev = tp->mdio_bus->phy_map[TG3_PHY_MII_ADDR]; tp->link_config.phy_is_low_power = 1; @@ -3243,15 +3283,6 @@ relink: pci_write_config_word(tp->pdev, tp->pcie_cap + PCI_EXP_LNKCTL, newlnkctl); - } else if (tp->tg3_flags3 & TG3_FLG3_TOGGLE_10_100_L1PLLPD) { - u32 newreg, oldreg = tr32(TG3_PCIE_LNKCTL); - if (tp->link_config.active_speed == SPEED_100 || - tp->link_config.active_speed == SPEED_10) - newreg = oldreg & ~TG3_PCIE_LNKCTL_L1_PLL_PD_EN; - else - newreg = oldreg | TG3_PCIE_LNKCTL_L1_PLL_PD_EN; - if (newreg != oldreg) - tw32(TG3_PCIE_LNKCTL, newreg); } if (current_link_up != netif_carrier_ok(tp->dev)) { @@ -4320,13 +4351,13 @@ static void tg3_tx(struct tg3_napi *tnapi) struct netdev_queue *txq; int index = tnapi - tp->napi; - if (tp->tg3_flags2 & TG3_FLG2_USING_MSIX) + if (tp->tg3_flags3 & TG3_FLG3_ENABLE_TSS) index--; txq = netdev_get_tx_queue(tp->dev, index); while (sw_idx != hw_idx) { - struct tx_ring_info *ri = &tnapi->tx_buffers[sw_idx]; + struct ring_info *ri = &tnapi->tx_buffers[sw_idx]; struct sk_buff *skb = ri->skb; int i, tx_bug = 0; @@ -4335,7 +4366,10 @@ static void tg3_tx(struct tg3_napi *tnapi) return; } - skb_dma_unmap(&tp->pdev->dev, skb, DMA_TO_DEVICE); + pci_unmap_single(tp->pdev, + pci_unmap_addr(ri, mapping), + skb_headlen(skb), + PCI_DMA_TODEVICE); ri->skb = NULL; @@ -4345,6 +4379,11 @@ static void tg3_tx(struct tg3_napi *tnapi) ri = &tnapi->tx_buffers[sw_idx]; if (unlikely(ri->skb != NULL || sw_idx == hw_idx)) tx_bug = 1; + + pci_unmap_page(tp->pdev, + pci_unmap_addr(ri, mapping), + skb_shinfo(skb)->frags[i].size, + PCI_DMA_TODEVICE); sw_idx = NEXT_TX(sw_idx); } @@ -4375,6 +4414,17 @@ static void tg3_tx(struct tg3_napi *tnapi) } } +static void tg3_rx_skb_free(struct tg3 *tp, struct ring_info *ri, u32 map_sz) +{ + if (!ri->skb) + return; + + pci_unmap_single(tp->pdev, pci_unmap_addr(ri, mapping), + map_sz, PCI_DMA_FROMDEVICE); + dev_kfree_skb_any(ri->skb); + ri->skb = NULL; +} + /* Returns size of skb allocated or < 0 on error. * * We only need to fill in the address because the other members @@ -4386,16 +4436,14 @@ static void tg3_tx(struct tg3_napi *tnapi) * buffers the cpu only reads the last cacheline of the RX descriptor * (to fetch the error flags, vlan tag, checksum, and opaque cookie). */ -static int tg3_alloc_rx_skb(struct tg3_napi *tnapi, u32 opaque_key, - int src_idx, u32 dest_idx_unmasked) +static int tg3_alloc_rx_skb(struct tg3 *tp, struct tg3_rx_prodring_set *tpr, + u32 opaque_key, u32 dest_idx_unmasked) { - struct tg3 *tp = tnapi->tp; struct tg3_rx_buffer_desc *desc; struct ring_info *map, *src_map; struct sk_buff *skb; dma_addr_t mapping; int skb_size, dest_idx; - struct tg3_rx_prodring_set *tpr = &tp->prodring[0]; src_map = NULL; switch (opaque_key) { @@ -4403,8 +4451,6 @@ static int tg3_alloc_rx_skb(struct tg3_napi *tnapi, u32 opaque_key, dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE; desc = &tpr->rx_std[dest_idx]; map = &tpr->rx_std_buffers[dest_idx]; - if (src_idx >= 0) - src_map = &tpr->rx_std_buffers[src_idx]; skb_size = tp->rx_pkt_map_sz; break; @@ -4412,8 +4458,6 @@ static int tg3_alloc_rx_skb(struct tg3_napi *tnapi, u32 opaque_key, dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE; desc = &tpr->rx_jmb[dest_idx].std; map = &tpr->rx_jmb_buffers[dest_idx]; - if (src_idx >= 0) - src_map = &tpr->rx_jmb_buffers[src_idx]; skb_size = TG3_RX_JMB_MAP_SZ; break; @@ -4435,13 +4479,14 @@ static int tg3_alloc_rx_skb(struct tg3_napi *tnapi, u32 opaque_key, mapping = pci_map_single(tp->pdev, skb->data, skb_size, PCI_DMA_FROMDEVICE); + if (pci_dma_mapping_error(tp->pdev, mapping)) { + dev_kfree_skb(skb); + return -EIO; + } map->skb = skb; pci_unmap_addr_set(map, mapping, mapping); - if (src_map != NULL) - src_map->skb = NULL; - desc->addr_hi = ((u64)mapping >> 32); desc->addr_lo = ((u64)mapping & 0xffffffff); @@ -4452,30 +4497,32 @@ static int tg3_alloc_rx_skb(struct tg3_napi *tnapi, u32 opaque_key, * members of the RX descriptor are invariant. See notes above * tg3_alloc_rx_skb for full details. */ -static void tg3_recycle_rx(struct tg3_napi *tnapi, u32 opaque_key, - int src_idx, u32 dest_idx_unmasked) +static void tg3_recycle_rx(struct tg3_napi *tnapi, + struct tg3_rx_prodring_set *dpr, + u32 opaque_key, int src_idx, + u32 dest_idx_unmasked) { struct tg3 *tp = tnapi->tp; struct tg3_rx_buffer_desc *src_desc, *dest_desc; struct ring_info *src_map, *dest_map; int dest_idx; - struct tg3_rx_prodring_set *tpr = &tp->prodring[0]; + struct tg3_rx_prodring_set *spr = &tp->prodring[0]; switch (opaque_key) { case RXD_OPAQUE_RING_STD: dest_idx = dest_idx_unmasked % TG3_RX_RING_SIZE; - dest_desc = &tpr->rx_std[dest_idx]; - dest_map = &tpr->rx_std_buffers[dest_idx]; - src_desc = &tpr->rx_std[src_idx]; - src_map = &tpr->rx_std_buffers[src_idx]; + dest_desc = &dpr->rx_std[dest_idx]; + dest_map = &dpr->rx_std_buffers[dest_idx]; + src_desc = &spr->rx_std[src_idx]; + src_map = &spr->rx_std_buffers[src_idx]; break; case RXD_OPAQUE_RING_JUMBO: dest_idx = dest_idx_unmasked % TG3_RX_JUMBO_RING_SIZE; - dest_desc = &tpr->rx_jmb[dest_idx].std; - dest_map = &tpr->rx_jmb_buffers[dest_idx]; - src_desc = &tpr->rx_jmb[src_idx].std; - src_map = &tpr->rx_jmb_buffers[src_idx]; + dest_desc = &dpr->rx_jmb[dest_idx].std; + dest_map = &dpr->rx_jmb_buffers[dest_idx]; + src_desc = &spr->rx_jmb[src_idx].std; + src_map = &spr->rx_jmb_buffers[src_idx]; break; default: @@ -4487,7 +4534,6 @@ static void tg3_recycle_rx(struct tg3_napi *tnapi, u32 opaque_key, pci_unmap_addr(src_map, mapping)); dest_desc->addr_hi = src_desc->addr_hi; dest_desc->addr_lo = src_desc->addr_lo; - src_map->skb = NULL; } @@ -4519,10 +4565,11 @@ static int tg3_rx(struct tg3_napi *tnapi, int budget) { struct tg3 *tp = tnapi->tp; u32 work_mask, rx_std_posted = 0; + u32 std_prod_idx, jmb_prod_idx; u32 sw_idx = tnapi->rx_rcb_ptr; u16 hw_idx; int received; - struct tg3_rx_prodring_set *tpr = &tp->prodring[0]; + struct tg3_rx_prodring_set *tpr = tnapi->prodring; hw_idx = *(tnapi->rx_rcb_prod_idx); /* @@ -4532,7 +4579,10 @@ static int tg3_rx(struct tg3_napi *tnapi, int budget) rmb(); work_mask = 0; received = 0; + std_prod_idx = tpr->rx_std_prod_idx; + jmb_prod_idx = tpr->rx_jmb_prod_idx; while (sw_idx != hw_idx && budget > 0) { + struct ring_info *ri; struct tg3_rx_buffer_desc *desc = &tnapi->rx_rcb[sw_idx]; unsigned int len; struct sk_buff *skb; @@ -4542,16 +4592,16 @@ static int tg3_rx(struct tg3_napi *tnapi, int budget) desc_idx = desc->opaque & RXD_OPAQUE_INDEX_MASK; opaque_key = desc->opaque & RXD_OPAQUE_RING_MASK; if (opaque_key == RXD_OPAQUE_RING_STD) { - struct ring_info *ri = &tpr->rx_std_buffers[desc_idx]; + ri = &tp->prodring[0].rx_std_buffers[desc_idx]; dma_addr = pci_unmap_addr(ri, mapping); skb = ri->skb; - post_ptr = &tpr->rx_std_ptr; + post_ptr = &std_prod_idx; rx_std_posted++; } else if (opaque_key == RXD_OPAQUE_RING_JUMBO) { - struct ring_info *ri = &tpr->rx_jmb_buffers[desc_idx]; + ri = &tp->prodring[0].rx_jmb_buffers[desc_idx]; dma_addr = pci_unmap_addr(ri, mapping); skb = ri->skb; - post_ptr = &tpr->rx_jmb_ptr; + post_ptr = &jmb_prod_idx; } else goto next_pkt_nopost; @@ -4560,7 +4610,7 @@ static int tg3_rx(struct tg3_napi *tnapi, int budget) if ((desc->err_vlan & RXD_ERR_MASK) != 0 && (desc->err_vlan != RXD_ERR_ODD_NIBBLE_RCVD_MII)) { drop_it: - tg3_recycle_rx(tnapi, opaque_key, + tg3_recycle_rx(tnapi, tpr, opaque_key, desc_idx, *post_ptr); drop_it_no_recycle: /* Other statistics kept track of by card. */ @@ -4571,20 +4621,21 @@ static int tg3_rx(struct tg3_napi *tnapi, int budget) len = ((desc->idx_len & RXD_LEN_MASK) >> RXD_LEN_SHIFT) - ETH_FCS_LEN; - if (len > RX_COPY_THRESHOLD - && tp->rx_offset == NET_IP_ALIGN - /* rx_offset will likely not equal NET_IP_ALIGN - * if this is a 5701 card running in PCI-X mode - * [see tg3_get_invariants()] - */ - ) { + if (len > RX_COPY_THRESHOLD && + tp->rx_offset == NET_IP_ALIGN) { + /* rx_offset will likely not equal NET_IP_ALIGN + * if this is a 5701 card running in PCI-X mode + * [see tg3_get_invariants()] + */ int skb_size; - skb_size = tg3_alloc_rx_skb(tnapi, opaque_key, - desc_idx, *post_ptr); + skb_size = tg3_alloc_rx_skb(tp, tpr, opaque_key, + *post_ptr); if (skb_size < 0) goto drop_it; + ri->skb = NULL; + pci_unmap_single(tp->pdev, dma_addr, skb_size, PCI_DMA_FROMDEVICE); @@ -4592,7 +4643,7 @@ static int tg3_rx(struct tg3_napi *tnapi, int budget) } else { struct sk_buff *copy_skb; - tg3_recycle_rx(tnapi, opaque_key, + tg3_recycle_rx(tnapi, tpr, opaque_key, desc_idx, *post_ptr); copy_skb = netdev_alloc_skb(tp->dev, @@ -4643,9 +4694,7 @@ next_pkt: if (unlikely(rx_std_posted >= tp->rx_std_max_post)) { u32 idx = *post_ptr % TG3_RX_RING_SIZE; - - tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + - TG3_64BIT_REG_LOW, idx); + tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG, idx); work_mask &= ~RXD_OPAQUE_RING_STD; rx_std_posted = 0; } @@ -4665,33 +4714,45 @@ next_pkt_nopost: tw32_rx_mbox(tnapi->consmbox, sw_idx); /* Refill RX ring(s). */ - if (work_mask & RXD_OPAQUE_RING_STD) { - sw_idx = tpr->rx_std_ptr % TG3_RX_RING_SIZE; - tw32_rx_mbox(MAILBOX_RCV_STD_PROD_IDX + TG3_64BIT_REG_LOW, - sw_idx); - } - if (work_mask & RXD_OPAQUE_RING_JUMBO) { - sw_idx = tpr->rx_jmb_ptr % TG3_RX_JUMBO_RING_SIZE; - tw32_rx_mbox(MAILBOX_RCV_JUMBO_PROD_IDX + TG3_64BIT_REG_LOW, - sw_idx); + if (!(tp->tg3_flags3 & TG3_FLG3_ENABLE_RSS) || tnapi == &tp->napi[1]) { + if (work_mask & RXD_OPAQUE_RING_STD) { + tpr->rx_std_prod_idx = std_prod_idx % TG3_RX_RING_SIZE; + tw32_rx_mbox(TG3_RX_STD_PROD_IDX_REG, + tpr->rx_std_prod_idx); + } + if (work_mask & RXD_OPAQUE_RING_JUMBO) { + tpr->rx_jmb_prod_idx = jmb_prod_idx % + TG3_RX_JUMBO_RING_SIZE; + tw32_rx_mbox(TG3_RX_JMB_PROD_IDX_REG, + tpr->rx_jmb_prod_idx); + } + mmiowb(); + } else if (work_mask) { + /* rx_std_buffers[] and rx_jmb_buffers[] entries must be + * updated before the producer indices can be updated. + */ + smp_wmb(); + + tpr->rx_std_prod_idx = std_prod_idx % TG3_RX_RING_SIZE; + tpr->rx_jmb_prod_idx = jmb_prod_idx % TG3_RX_JUMBO_RING_SIZE; + + napi_schedule(&tp->napi[1].napi); } - mmiowb(); return received; } -static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget) +static void tg3_poll_link(struct tg3 *tp) { - struct tg3 *tp = tnapi->tp; - struct tg3_hw_status *sblk = tnapi->hw_status; - /* handle link change and other phy events */ if (!(tp->tg3_flags & (TG3_FLAG_USE_LINKCHG_REG | TG3_FLAG_POLL_SERDES))) { + struct tg3_hw_status *sblk = tp->napi[0].hw_status; + if (sblk->status & SD_STATUS_LINK_CHG) { sblk->status = SD_STATUS_UPDATED | - (sblk->status & ~SD_STATUS_LINK_CHG); + (sblk->status & ~SD_STATUS_LINK_CHG); spin_lock(&tp->lock); if (tp->tg3_flags3 & TG3_FLG3_USE_PHYLIB) { tw32_f(MAC_STATUS, @@ -4705,6 +4766,98 @@ static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget) spin_unlock(&tp->lock); } } +} + +static void tg3_rx_prodring_xfer(struct tg3 *tp, + struct tg3_rx_prodring_set *dpr, + struct tg3_rx_prodring_set *spr) +{ + u32 si, di, cpycnt, src_prod_idx; + int i; + + while (1) { + src_prod_idx = spr->rx_std_prod_idx; + + /* Make sure updates to the rx_std_buffers[] entries and the + * standard producer index are seen in the correct order. + */ + smp_rmb(); + + if (spr->rx_std_cons_idx == src_prod_idx) + break; + + if (spr->rx_std_cons_idx < src_prod_idx) + cpycnt = src_prod_idx - spr->rx_std_cons_idx; + else + cpycnt = TG3_RX_RING_SIZE - spr->rx_std_cons_idx; + + cpycnt = min(cpycnt, TG3_RX_RING_SIZE - dpr->rx_std_prod_idx); + + si = spr->rx_std_cons_idx; + di = dpr->rx_std_prod_idx; + + memcpy(&dpr->rx_std_buffers[di], + &spr->rx_std_buffers[si], + cpycnt * sizeof(struct ring_info)); + + for (i = 0; i < cpycnt; i++, di++, si++) { + struct tg3_rx_buffer_desc *sbd, *dbd; + sbd = &spr->rx_std[si]; + dbd = &dpr->rx_std[di]; + dbd->addr_hi = sbd->addr_hi; + dbd->addr_lo = sbd->addr_lo; + } + + spr->rx_std_cons_idx = (spr->rx_std_cons_idx + cpycnt) % + TG3_RX_RING_SIZE; + dpr->rx_std_prod_idx = (dpr->rx_std_prod_idx + cpycnt) % + TG3_RX_RING_SIZE; + } + + while (1) { + src_prod_idx = spr->rx_jmb_prod_idx; + + /* Make sure updates to the rx_jmb_buffers[] entries and + * the jumbo producer index are seen in the correct order. + */ + smp_rmb(); + + if (spr->rx_jmb_cons_idx == src_prod_idx) + break; + + if (spr->rx_jmb_cons_idx < src_prod_idx) + cpycnt = src_prod_idx - spr->rx_jmb_cons_idx; + else + cpycnt = TG3_RX_JUMBO_RING_SIZE - spr->rx_jmb_cons_idx; + + cpycnt = min(cpycnt, + TG3_RX_JUMBO_RING_SIZE - dpr->rx_jmb_prod_idx); + + si = spr->rx_jmb_cons_idx; + di = dpr->rx_jmb_prod_idx; + + memcpy(&dpr->rx_jmb_buffers[di], + &spr->rx_jmb_buffers[si], + cpycnt * sizeof(struct ring_info)); + + for (i = 0; i < cpycnt; i++, di++, si++) { + struct tg3_rx_buffer_desc *sbd, *dbd; + sbd = &spr->rx_jmb[si].std; + dbd = &dpr->rx_jmb[di].std; + dbd->addr_hi = sbd->addr_hi; + dbd->addr_lo = sbd->addr_lo; + } + + spr->rx_jmb_cons_idx = (spr->rx_jmb_cons_idx + cpycnt) % + TG3_RX_JUMBO_RING_SIZE; + dpr->rx_jmb_prod_idx = (dpr->rx_jmb_prod_idx + cpycnt) % + TG3_RX_JUMBO_RING_SIZE; + } +} + +static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget) +{ + struct tg3 *tp = tnapi->tp; /* run TX completion thread */ if (tnapi->hw_status->idx[0].tx_consumer != tnapi->tx_cons) { @@ -4720,6 +4873,74 @@ static int tg3_poll_work(struct tg3_napi *tnapi, int work_done, int budget) if (*(tnapi->rx_rcb_prod_idx) != tnapi->rx_rcb_ptr) work_done += tg3_rx(tnapi, budget - work_done); + if ((tp->tg3_flags3 & TG3_FLG3_ENABLE_RSS) && tnapi == &tp->napi[1]) { + int i; + u32 std_prod_idx = tp->prodring[0].rx_std_prod_idx; + u32 jmb_prod_idx = tp->prodring[0].rx_jmb_prod_idx; + + for (i = 2; i < tp->irq_cnt; i++) + tg3_rx_prodring_xfer(tp, tnapi->prodring, + tp->napi[i].prodring); + + wmb(); + + if (std_prod_idx != tp->prodring[0].rx_std_prod_idx) { + u32 mbox = TG3_RX_STD_PROD_IDX_REG; + tw32_rx_mbox(mbox, tp->prodring[0].rx_std_prod_idx); + } + + if (jmb_prod_idx != tp->prodring[0].rx_jmb_prod_idx) { + u32 mbox = TG3_RX_JMB_PROD_IDX_REG; + tw32_rx_mbox(mbox, tp->prodring[0].rx_jmb_prod_idx); + } + + mmiowb(); + } + + return work_done; +} + +static int tg3_poll_msix(struct napi_struct *napi, int budget) +{ + struct tg3_napi *tnapi = container_of(napi, struct tg3_napi, napi); + struct tg3 *tp = tnapi->tp; + int work_done = 0; + struct tg3_hw_status *sblk = tnapi->hw_status; + + while (1) { + work_done = tg3_poll_work(tnapi, work_done, budget); + + if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING)) + goto tx_recovery; + + if (unlikely(work_done >= budget)) + break; + + /* tp->last_tag is used in tg3_restart_ints() below + * to tell the hw how much work has been processed, + * so we must read it before checking for more work. + */ + tnapi->last_tag = sblk->status_tag; + tnapi->last_irq_tag = tnapi->last_tag; + rmb(); + + /* check for RX/TX work to do */ + if (sblk->idx[0].tx_consumer == tnapi->tx_cons && + *(tnapi->rx_rcb_prod_idx) == tnapi->rx_rcb_ptr) { + napi_complete(napi); + /* Reenable interrupts. */ + tw32_mailbox(tnapi->int_mbox, tnapi->last_tag << 24); + mmiowb(); + break; + } + } + + return work_done; + +tx_recovery: + /* work_done is guaranteed to be less than budget. */ + napi_complete(napi); + schedule_work(&tp->reset_task); return work_done; } @@ -4731,6 +4952,8 @@ static int tg3_poll(struct napi_struct *napi, int budget) struct tg3_hw_status *sblk = tnapi->hw_status; while (1) { + tg3_poll_link(tp); + work_done = tg3_poll_work(tnapi, work_done, budget); if (unlikely(tp->tg3_flags & TG3_FLAG_TX_RECOVERY_PENDING)) @@ -5093,11 +5316,11 @@ static inline int tg3_40bit_overflow_test(struct tg3 *tp, dma_addr_t mapping, static void tg3_set_txd(struct tg3_napi *, int, dma_addr_t, int, u32, u32); /* Workaround 4GB and 40-bit hardware DMA bugs. */ -static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb, - u32 last_plus_one, u32 *start, - u32 base_flags, u32 mss) +static int tigon3_dma_hwbug_workaround(struct tg3_napi *tnapi, + struct sk_buff *skb, u32 last_plus_one, + u32 *start, u32 base_flags, u32 mss) { - struct tg3_napi *tnapi = &tp->napi[0]; + struct tg3 *tp = tnapi->tp; struct sk_buff *new_skb; dma_addr_t new_addr = 0; u32 entry = *start; @@ -5118,16 +5341,21 @@ static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb, } else { /* New SKB is guaranteed to be linear. */ entry = *start; - ret = skb_dma_map(&tp->pdev->dev, new_skb, DMA_TO_DEVICE); - new_addr = skb_shinfo(new_skb)->dma_head; + new_addr = pci_map_single(tp->pdev, new_skb->data, new_skb->len, + PCI_DMA_TODEVICE); + /* Make sure the mapping succeeded */ + if (pci_dma_mapping_error(tp->pdev, new_addr)) { + ret = -1; + dev_kfree_skb(new_skb); + new_skb = NULL; /* Make sure new skb does not cross any 4G boundaries. * Drop the packet if it does. */ - if (ret || tg3_4g_overflow_test(new_addr, new_skb->len)) { - if (!ret) - skb_dma_unmap(&tp->pdev->dev, new_skb, - DMA_TO_DEVICE); + } else if ((tp->tg3_flags3 & TG3_FLG3_4G_DMA_BNDRY_BUG) && + tg3_4g_overflow_test(new_addr, new_skb->len)) { + pci_unmap_single(tp->pdev, new_addr, new_skb->len, + PCI_DMA_TODEVICE); ret = -1; dev_kfree_skb(new_skb); new_skb = NULL; @@ -5141,15 +5369,28 @@ static int tigon3_dma_hwbug_workaround(struct tg3 *tp, struct sk_buff *skb, /* Now clean up the sw ring entries. */ i = 0; while (entry != last_plus_one) { + int len; + if (i == 0) - tnapi->tx_buffers[entry].skb = new_skb; + len = skb_headlen(skb); else + len = skb_shinfo(skb)->frags[i-1].size; + + pci_unmap_single(tp->pdev, + pci_unmap_addr(&tnapi->tx_buffers[entry], + mapping), + len, PCI_DMA_TODEVICE); + if (i == 0) { + tnapi->tx_buffers[entry].skb = new_skb; + pci_unmap_addr_set(&tnapi->tx_buffers[entry], mapping, + new_addr); + } else { tnapi->tx_buffers[entry].skb = NULL; + } entry = NEXT_TX(entry); i++; } - skb_dma_unmap(&tp->pdev->dev, skb, DMA_TO_DEVICE); dev_kfree_skb(skb); return ret; @@ -5179,21 +5420,22 @@ static void tg3_set_txd(struct tg3_napi *tnapi, int entry, } /* hard_start_xmit for devices that don't have any bugs and - * support TG3_FLG2_HW_TSO_2 only. + * support TG3_FLG2_HW_TSO_2 and TG3_FLG2_HW_TSO_3 only. */ static netdev_tx_t tg3_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct tg3 *tp = netdev_priv(dev); u32 len, entry, base_flags, mss; - struct skb_shared_info *sp; dma_addr_t mapping; struct tg3_napi *tnapi; struct netdev_queue *txq; + unsigned int i, last; + txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)); tnapi = &tp->napi[skb_get_queue_mapping(skb)]; - if (tp->tg3_flags2 & TG3_FLG2_USING_MSIX) + if (tp->tg3_flags3 & TG3_FLG3_ENABLE_TSS) tnapi++; /* We are running in BH disabled context with netif_tx_lock @@ -5238,7 +5480,7 @@ static netdev_tx_t tg3_start_xmit(struct sk_buff *skb, hdrlen = ip_tcp_len + tcp_opt_len; } - if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717) { + if (tp->tg3_flags2 & TG3_FLG2_HW_TSO_3) { mss |= (hdrlen & 0xc) << 12; if (hdrlen & 0x10) base_flags |= 0x00000010; @@ -5260,20 +5502,19 @@ static netdev_tx_t tg3_start_xmit(struct sk_buff *skb, (vlan_tx_tag_get(skb) << 16)); #endif - if (skb_dma_map(&tp->pdev->dev, skb, DMA_TO_DEVICE)) { + len = skb_headlen(skb); + + /* Queue skb data, a.k.a. the main skb fragment. */ + mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE); + if (pci_dma_mapping_error(tp->pdev, mapping)) { dev_kfree_skb(skb); goto out_unlock; } - sp = skb_shinfo(skb); - - mapping = sp->dma_head; - tnapi->tx_buffers[entry].skb = skb; + pci_unmap_addr_set(&tnapi->tx_buffers[entry], mapping, mapping); - len = skb_headlen(skb); - - if (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5717 && + if ((tp->tg3_flags3 & TG3_FLG3_USE_JUMBO_BDFLAG) && !mss && skb->len > ETH_DATA_LEN) base_flags |= TXD_FLAG_JMB_PKT; @@ -5284,15 +5525,21 @@ static netdev_tx_t tg3_start_xmit(struct sk_buff *skb, /* Now loop through additional data fragments, and queue them. */ if (skb_shinfo(skb)->nr_frags > 0) { - unsigned int i, last; - last = skb_shinfo(skb)->nr_frags - 1; for (i = 0; i <= last; i++) { skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; len = frag->size; - mapping = sp->dma_maps[i]; + mapping = pci_map_page(tp->pdev, + frag->page, + frag->page_offset, + len, PCI_DMA_TODEVICE); + if (pci_dma_mapping_error(tp->pdev, mapping)) + goto dma_error; + tnapi->tx_buffers[entry].skb = NULL; + pci_unmap_addr_set(&tnapi->tx_buffers[entry], mapping, + mapping); tg3_set_txd(tnapi, entry, mapping, len, base_flags, (i == last) | (mss << 1)); @@ -5315,6 +5562,27 @@ out_unlock: mmiowb(); return NETDEV_TX_OK; + +dma_error: + last = i; + entry = tnapi->tx_prod; + tnapi->tx_buffers[entry].skb = NULL; + pci_unmap_single(tp->pdev, + pci_unmap_addr(&tnapi->tx_buffers[entry], mapping), + skb_headlen(skb), + PCI_DMA_TODEVICE); + for (i = 0; i <= last; i++) { + skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; + entry = NEXT_TX(entry); + + pci_unmap_page(tp->pdev, + pci_unmap_addr(&tnapi->tx_buffers[entry], + mapping), + frag->size, PCI_DMA_TODEVICE); + } + + dev_kfree_skb(skb); + return NETDEV_TX_OK; } static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *, @@ -5362,12 +5630,17 @@ static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *skb, { struct tg3 *tp = netdev_priv(dev); u32 len, entry, base_flags, mss; - struct skb_shared_info *sp; int would_hit_hwbug; dma_addr_t mapping; - struct tg3_napi *tnapi = &tp->napi[0]; + struct tg3_napi *tnapi; + struct netdev_queue *txq; + unsigned int i, last; - len = skb_headlen(skb); + + txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb)); + tnapi = &tp->napi[skb_get_queue_mapping(skb)]; + if (tp->tg3_flags3 & TG3_FLG3_ENABLE_TSS) + tnapi++; /* We are running in BH disabled context with netif_tx_lock * and TX reclaim runs via tp->napi.poll inside of a software @@ -5375,8 +5648,8 @@ static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *skb, * no IRQ context deadlocks to worry about either. Rejoice! */ if (unlikely(tg3_tx_avail(tnapi) <= (skb_shinfo(skb)->nr_frags + 1))) { - if (!netif_queue_stopped(dev)) { - netif_stop_queue(dev); + if (!netif_tx_queue_stopped(txq)) { + netif_tx_stop_queue(txq); /* This is a hard error, log it. */ printk(KERN_ERR PFX "%s: BUG! Tx Ring full when " @@ -5389,10 +5662,10 @@ static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *skb, base_flags = 0; if (skb->ip_summed == CHECKSUM_PARTIAL) base_flags |= TXD_FLAG_TCPUDP_CSUM; - mss = 0; + if ((mss = skb_shinfo(skb)->gso_size) != 0) { struct iphdr *iph; - int tcp_opt_len, ip_tcp_len, hdr_len; + u32 tcp_opt_len, ip_tcp_len, hdr_len; if (skb_header_cloned(skb) && pskb_expand_head(skb, 0, 0, GFP_ATOMIC)) { @@ -5423,8 +5696,15 @@ static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *skb, IPPROTO_TCP, 0); - if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO) || - (GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705)) { + if (tp->tg3_flags2 & TG3_FLG2_HW_TSO_3) { + mss |= (hdr_len & 0xc) << 12; + if (hdr_len & 0x10) + base_flags |= 0x00000010; + base_flags |= (hdr_len & 0x3e0) << 5; + } else if (tp->tg3_flags2 & TG3_FLG2_HW_TSO_2) + mss |= hdr_len << 9; + else if ((tp->tg3_flags2 & TG3_FLG2_HW_TSO_1) || + GET_ASIC_REV(tp->pci_chip_rev_id) == ASIC_REV_5705) { if (tcp_opt_len || iph->ihl > 5) { int tsflags; @@ -5446,22 +5726,35 @@ static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *skb, (vlan_tx_tag_get(skb) << 16)); #endif - if (skb_dma_map(&tp->pdev->dev, skb, DMA_TO_DEVICE)) { + if ((tp->tg3_flags3 & TG3_FLG3_USE_JUMBO_BDFLAG) && + !mss && skb->len > ETH_DATA_LEN) + base_flags |= TXD_FLAG_JMB_PKT; + + len = skb_headlen(skb); + + mapping = pci_map_single(tp->pdev, skb->data, len, PCI_DMA_TODEVICE); + if (pci_dma_mapping_error(tp->pdev, mapping)) { dev_kfree_skb(skb); goto out_unlock; } - sp = skb_shinfo(skb); - - mapping = sp->dma_head; - tnapi->tx_buffers[entry].skb = skb; + pci_unmap_addr_set(&tnapi->tx_buffers[entry], mapping, mapping); would_hit_hwbug = 0; - if (tp->tg3_flags3 & TG3_FLG3_5701_DMA_BUG) + if ((tp->tg3_flags3 & TG3_FLG3_SHORT_DMA_BUG) && len <= 8) would_hit_hwbug = 1; - else if (tg3_4g_overflow_test(mapping, len)) + + if ((tp->tg3_flags3 & TG3_FLG3_4G_DMA_BNDRY_BUG) && + tg3_4g_overflow_test(mapping, len)) + would_hit_hwbug = 1; + + if ((tp->tg3_flags3 & TG3_FLG3_40BIT_DMA_LIMIT_BUG) && + tg3_40bit_overflow_test(tp, mapping, len)) + would_hit_hwbug = 1; + + if (tp->tg3_flags3 & TG3_FLG3_5701_DMA_BUG) would_hit_hwbug = 1; tg3_set_txd(tnapi, entry, mapping, len, base_flags, @@ -5471,21 +5764,32 @@ static netdev_tx_t tg3_start_xmit_dma_bug(struct sk_buff *skb, /* Now loop through additional data fragm |