diff options
Diffstat (limited to 'drivers/net/igb/igb_main.c')
| -rw-r--r-- | drivers/net/igb/igb_main.c | 3450 |
1 files changed, 1958 insertions, 1492 deletions
diff --git a/drivers/net/igb/igb_main.c b/drivers/net/igb/igb_main.c index 714c3a4a44e..16349ba6873 100644 --- a/drivers/net/igb/igb_main.c +++ b/drivers/net/igb/igb_main.c @@ -49,7 +49,7 @@ #endif #include "igb.h" -#define DRV_VERSION "1.3.16-k2" +#define DRV_VERSION "2.1.0-k2" char igb_driver_name[] = "igb"; char igb_driver_version[] = DRV_VERSION; static const char igb_driver_string[] = @@ -61,8 +61,14 @@ static const struct e1000_info *igb_info_tbl[] = { }; static struct pci_device_id igb_pci_tbl[] = { + { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 }, + { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 }, + { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 }, + { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 }, + { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 }, { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 }, { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 }, + { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 }, { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 }, { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 }, { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 }, @@ -81,6 +87,7 @@ static int igb_setup_all_tx_resources(struct igb_adapter *); static int igb_setup_all_rx_resources(struct igb_adapter *); static void igb_free_all_tx_resources(struct igb_adapter *); static void igb_free_all_rx_resources(struct igb_adapter *); +static void igb_setup_mrqc(struct igb_adapter *); void igb_update_stats(struct igb_adapter *); static int igb_probe(struct pci_dev *, const struct pci_device_id *); static void __devexit igb_remove(struct pci_dev *pdev); @@ -89,7 +96,6 @@ static int igb_open(struct net_device *); static int igb_close(struct net_device *); static void igb_configure_tx(struct igb_adapter *); static void igb_configure_rx(struct igb_adapter *); -static void igb_setup_rctl(struct igb_adapter *); static void igb_clean_all_tx_rings(struct igb_adapter *); static void igb_clean_all_rx_rings(struct igb_adapter *); static void igb_clean_tx_ring(struct igb_ring *); @@ -98,28 +104,22 @@ static void igb_set_rx_mode(struct net_device *); static void igb_update_phy_info(unsigned long); static void igb_watchdog(unsigned long); static void igb_watchdog_task(struct work_struct *); -static netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *, - struct net_device *, - struct igb_ring *); -static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, - struct net_device *); +static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *); static struct net_device_stats *igb_get_stats(struct net_device *); static int igb_change_mtu(struct net_device *, int); static int igb_set_mac(struct net_device *, void *); +static void igb_set_uta(struct igb_adapter *adapter); static irqreturn_t igb_intr(int irq, void *); static irqreturn_t igb_intr_msi(int irq, void *); static irqreturn_t igb_msix_other(int irq, void *); -static irqreturn_t igb_msix_rx(int irq, void *); -static irqreturn_t igb_msix_tx(int irq, void *); +static irqreturn_t igb_msix_ring(int irq, void *); #ifdef CONFIG_IGB_DCA -static void igb_update_rx_dca(struct igb_ring *); -static void igb_update_tx_dca(struct igb_ring *); +static void igb_update_dca(struct igb_q_vector *); static void igb_setup_dca(struct igb_adapter *); #endif /* CONFIG_IGB_DCA */ -static bool igb_clean_tx_irq(struct igb_ring *); +static bool igb_clean_tx_irq(struct igb_q_vector *); static int igb_poll(struct napi_struct *, int); -static bool igb_clean_rx_irq_adv(struct igb_ring *, int *, int); -static void igb_alloc_rx_buffers_adv(struct igb_ring *, int); +static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int); static int igb_ioctl(struct net_device *, struct ifreq *, int cmd); static void igb_tx_timeout(struct net_device *); static void igb_reset_task(struct work_struct *); @@ -127,57 +127,13 @@ static void igb_vlan_rx_register(struct net_device *, struct vlan_group *); static void igb_vlan_rx_add_vid(struct net_device *, u16); static void igb_vlan_rx_kill_vid(struct net_device *, u16); static void igb_restore_vlan(struct igb_adapter *); +static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8); static void igb_ping_all_vfs(struct igb_adapter *); static void igb_msg_task(struct igb_adapter *); -static int igb_rcv_msg_from_vf(struct igb_adapter *, u32); -static inline void igb_set_rah_pool(struct e1000_hw *, int , int); static void igb_vmm_control(struct igb_adapter *); -static int igb_set_vf_mac(struct igb_adapter *adapter, int, unsigned char *); +static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *); static void igb_restore_vf_multicasts(struct igb_adapter *adapter); -static inline void igb_set_vmolr(struct e1000_hw *hw, int vfn) -{ - u32 reg_data; - - reg_data = rd32(E1000_VMOLR(vfn)); - reg_data |= E1000_VMOLR_BAM | /* Accept broadcast */ - E1000_VMOLR_ROPE | /* Accept packets matched in UTA */ - E1000_VMOLR_ROMPE | /* Accept packets matched in MTA */ - E1000_VMOLR_AUPE | /* Accept untagged packets */ - E1000_VMOLR_STRVLAN; /* Strip vlan tags */ - wr32(E1000_VMOLR(vfn), reg_data); -} - -static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size, - int vfn) -{ - struct e1000_hw *hw = &adapter->hw; - u32 vmolr; - - /* if it isn't the PF check to see if VFs are enabled and - * increase the size to support vlan tags */ - if (vfn < adapter->vfs_allocated_count && - adapter->vf_data[vfn].vlans_enabled) - size += VLAN_TAG_SIZE; - - vmolr = rd32(E1000_VMOLR(vfn)); - vmolr &= ~E1000_VMOLR_RLPML_MASK; - vmolr |= size | E1000_VMOLR_LPE; - wr32(E1000_VMOLR(vfn), vmolr); - - return 0; -} - -static inline void igb_set_rah_pool(struct e1000_hw *hw, int pool, int entry) -{ - u32 reg_data; - - reg_data = rd32(E1000_RAH(entry)); - reg_data &= ~E1000_RAH_POOL_MASK; - reg_data |= E1000_RAH_POOL_1 << pool;; - wr32(E1000_RAH(entry), reg_data); -} - #ifdef CONFIG_PM static int igb_suspend(struct pci_dev *, pm_message_t); static int igb_resume(struct pci_dev *); @@ -228,46 +184,12 @@ static struct pci_driver igb_driver = { .err_handler = &igb_err_handler }; -static int global_quad_port_a; /* global quad port a indication */ - MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver"); MODULE_LICENSE("GPL"); MODULE_VERSION(DRV_VERSION); /** - * Scale the NIC clock cycle by a large factor so that - * relatively small clock corrections can be added or - * substracted at each clock tick. The drawbacks of a - * large factor are a) that the clock register overflows - * more quickly (not such a big deal) and b) that the - * increment per tick has to fit into 24 bits. - * - * Note that - * TIMINCA = IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS * - * IGB_TSYNC_SCALE - * TIMINCA += TIMINCA * adjustment [ppm] / 1e9 - * - * The base scale factor is intentionally a power of two - * so that the division in %struct timecounter can be done with - * a shift. - */ -#define IGB_TSYNC_SHIFT (19) -#define IGB_TSYNC_SCALE (1<<IGB_TSYNC_SHIFT) - -/** - * The duration of one clock cycle of the NIC. - * - * @todo This hard-coded value is part of the specification and might change - * in future hardware revisions. Add revision check. - */ -#define IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS 16 - -#if (IGB_TSYNC_SCALE * IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS) >= (1<<24) -# error IGB_TSYNC_SCALE and/or IGB_TSYNC_CYCLE_TIME_IN_NANOSECONDS are too large to fit into TIMINCA -#endif - -/** * igb_read_clock - read raw cycle counter (to be used by time counter) */ static cycle_t igb_read_clock(const struct cyclecounter *tc) @@ -275,11 +197,21 @@ static cycle_t igb_read_clock(const struct cyclecounter *tc) struct igb_adapter *adapter = container_of(tc, struct igb_adapter, cycles); struct e1000_hw *hw = &adapter->hw; - u64 stamp; + u64 stamp = 0; + int shift = 0; - stamp = rd32(E1000_SYSTIML); - stamp |= (u64)rd32(E1000_SYSTIMH) << 32ULL; + /* + * The timestamp latches on lowest register read. For the 82580 + * the lowest register is SYSTIMR instead of SYSTIML. However we never + * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it. + */ + if (hw->mac.type == e1000_82580) { + stamp = rd32(E1000_SYSTIMR) >> 8; + shift = IGB_82580_TSYNC_SHIFT; + } + stamp |= (u64)rd32(E1000_SYSTIML) << shift; + stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32); return stamp; } @@ -320,17 +252,6 @@ static char *igb_get_time_str(struct igb_adapter *adapter, #endif /** - * igb_desc_unused - calculate if we have unused descriptors - **/ -static int igb_desc_unused(struct igb_ring *ring) -{ - if (ring->next_to_clean > ring->next_to_use) - return ring->next_to_clean - ring->next_to_use - 1; - - return ring->count + ring->next_to_clean - ring->next_to_use - 1; -} - -/** * igb_init_module - Driver Registration Routine * * igb_init_module is the first routine called when the driver is @@ -344,12 +265,9 @@ static int __init igb_init_module(void) printk(KERN_INFO "%s\n", igb_copyright); - global_quad_port_a = 0; - #ifdef CONFIG_IGB_DCA dca_register_notify(&dca_notifier); #endif - ret = pci_register_driver(&igb_driver); return ret; } @@ -382,8 +300,8 @@ module_exit(igb_exit_module); **/ static void igb_cache_ring_register(struct igb_adapter *adapter) { - int i; - unsigned int rbase_offset = adapter->vfs_allocated_count; + int i = 0, j = 0; + u32 rbase_offset = adapter->vfs_allocated_count; switch (adapter->hw.mac.type) { case e1000_82576: @@ -392,23 +310,37 @@ static void igb_cache_ring_register(struct igb_adapter *adapter) * In order to avoid collision we start at the first free queue * and continue consuming queues in the same sequence */ - for (i = 0; i < adapter->num_rx_queues; i++) - adapter->rx_ring[i].reg_idx = rbase_offset + - Q_IDX_82576(i); - for (i = 0; i < adapter->num_tx_queues; i++) - adapter->tx_ring[i].reg_idx = rbase_offset + - Q_IDX_82576(i); - break; + if (adapter->vfs_allocated_count) { + for (; i < adapter->rss_queues; i++) + adapter->rx_ring[i].reg_idx = rbase_offset + + Q_IDX_82576(i); + for (; j < adapter->rss_queues; j++) + adapter->tx_ring[j].reg_idx = rbase_offset + + Q_IDX_82576(j); + } case e1000_82575: + case e1000_82580: default: - for (i = 0; i < adapter->num_rx_queues; i++) - adapter->rx_ring[i].reg_idx = i; - for (i = 0; i < adapter->num_tx_queues; i++) - adapter->tx_ring[i].reg_idx = i; + for (; i < adapter->num_rx_queues; i++) + adapter->rx_ring[i].reg_idx = rbase_offset + i; + for (; j < adapter->num_tx_queues; j++) + adapter->tx_ring[j].reg_idx = rbase_offset + j; break; } } +static void igb_free_queues(struct igb_adapter *adapter) +{ + kfree(adapter->tx_ring); + kfree(adapter->rx_ring); + + adapter->tx_ring = NULL; + adapter->rx_ring = NULL; + + adapter->num_rx_queues = 0; + adapter->num_tx_queues = 0; +} + /** * igb_alloc_queues - Allocate memory for all rings * @adapter: board private structure to initialize @@ -423,59 +355,61 @@ static int igb_alloc_queues(struct igb_adapter *adapter) adapter->tx_ring = kcalloc(adapter->num_tx_queues, sizeof(struct igb_ring), GFP_KERNEL); if (!adapter->tx_ring) - return -ENOMEM; + goto err; adapter->rx_ring = kcalloc(adapter->num_rx_queues, sizeof(struct igb_ring), GFP_KERNEL); - if (!adapter->rx_ring) { - kfree(adapter->tx_ring); - return -ENOMEM; - } - - adapter->rx_ring->buddy = adapter->tx_ring; + if (!adapter->rx_ring) + goto err; for (i = 0; i < adapter->num_tx_queues; i++) { struct igb_ring *ring = &(adapter->tx_ring[i]); ring->count = adapter->tx_ring_count; - ring->adapter = adapter; ring->queue_index = i; + ring->pdev = adapter->pdev; + ring->netdev = adapter->netdev; + /* For 82575, context index must be unique per ring. */ + if (adapter->hw.mac.type == e1000_82575) + ring->flags = IGB_RING_FLAG_TX_CTX_IDX; } + for (i = 0; i < adapter->num_rx_queues; i++) { struct igb_ring *ring = &(adapter->rx_ring[i]); ring->count = adapter->rx_ring_count; - ring->adapter = adapter; ring->queue_index = i; - ring->itr_register = E1000_ITR; - - /* set a default napi handler for each rx_ring */ - netif_napi_add(adapter->netdev, &ring->napi, igb_poll, 64); + ring->pdev = adapter->pdev; + ring->netdev = adapter->netdev; + ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE; + ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */ + /* set flag indicating ring supports SCTP checksum offload */ + if (adapter->hw.mac.type >= e1000_82576) + ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM; } igb_cache_ring_register(adapter); - return 0; -} - -static void igb_free_queues(struct igb_adapter *adapter) -{ - int i; - for (i = 0; i < adapter->num_rx_queues; i++) - netif_napi_del(&adapter->rx_ring[i].napi); + return 0; - adapter->num_rx_queues = 0; - adapter->num_tx_queues = 0; +err: + igb_free_queues(adapter); - kfree(adapter->tx_ring); - kfree(adapter->rx_ring); + return -ENOMEM; } #define IGB_N0_QUEUE -1 -static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue, - int tx_queue, int msix_vector) +static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector) { u32 msixbm = 0; + struct igb_adapter *adapter = q_vector->adapter; struct e1000_hw *hw = &adapter->hw; u32 ivar, index; + int rx_queue = IGB_N0_QUEUE; + int tx_queue = IGB_N0_QUEUE; + + if (q_vector->rx_ring) + rx_queue = q_vector->rx_ring->reg_idx; + if (q_vector->tx_ring) + tx_queue = q_vector->tx_ring->reg_idx; switch (hw->mac.type) { case e1000_82575: @@ -483,16 +417,12 @@ static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue, bitmask for the EICR/EIMS/EIMC registers. To assign one or more queues to a vector, we write the appropriate bits into the MSIXBM register for that vector. */ - if (rx_queue > IGB_N0_QUEUE) { + if (rx_queue > IGB_N0_QUEUE) msixbm = E1000_EICR_RX_QUEUE0 << rx_queue; - adapter->rx_ring[rx_queue].eims_value = msixbm; - } - if (tx_queue > IGB_N0_QUEUE) { + if (tx_queue > IGB_N0_QUEUE) msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue; - adapter->tx_ring[tx_queue].eims_value = - E1000_EICR_TX_QUEUE0 << tx_queue; - } array_wr32(E1000_MSIXBM(0), msix_vector, msixbm); + q_vector->eims_value = msixbm; break; case e1000_82576: /* 82576 uses a table-based method for assigning vectors. @@ -500,7 +430,40 @@ static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue, a vector number along with a "valid" bit. Sadly, the layout of the table is somewhat counterintuitive. */ if (rx_queue > IGB_N0_QUEUE) { - index = (rx_queue >> 1) + adapter->vfs_allocated_count; + index = (rx_queue & 0x7); + ivar = array_rd32(E1000_IVAR0, index); + if (rx_queue < 8) { + /* vector goes into low byte of register */ + ivar = ivar & 0xFFFFFF00; + ivar |= msix_vector | E1000_IVAR_VALID; + } else { + /* vector goes into third byte of register */ + ivar = ivar & 0xFF00FFFF; + ivar |= (msix_vector | E1000_IVAR_VALID) << 16; + } + array_wr32(E1000_IVAR0, index, ivar); + } + if (tx_queue > IGB_N0_QUEUE) { + index = (tx_queue & 0x7); + ivar = array_rd32(E1000_IVAR0, index); + if (tx_queue < 8) { + /* vector goes into second byte of register */ + ivar = ivar & 0xFFFF00FF; + ivar |= (msix_vector | E1000_IVAR_VALID) << 8; + } else { + /* vector goes into high byte of register */ + ivar = ivar & 0x00FFFFFF; + ivar |= (msix_vector | E1000_IVAR_VALID) << 24; + } + array_wr32(E1000_IVAR0, index, ivar); + } + q_vector->eims_value = 1 << msix_vector; + break; + case e1000_82580: + /* 82580 uses the same table-based approach as 82576 but has fewer + entries as a result we carry over for queues greater than 4. */ + if (rx_queue > IGB_N0_QUEUE) { + index = (rx_queue >> 1); ivar = array_rd32(E1000_IVAR0, index); if (rx_queue & 0x1) { /* vector goes into third byte of register */ @@ -511,11 +474,10 @@ static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue, ivar = ivar & 0xFFFFFF00; ivar |= msix_vector | E1000_IVAR_VALID; } - adapter->rx_ring[rx_queue].eims_value= 1 << msix_vector; array_wr32(E1000_IVAR0, index, ivar); } if (tx_queue > IGB_N0_QUEUE) { - index = (tx_queue >> 1) + adapter->vfs_allocated_count; + index = (tx_queue >> 1); ivar = array_rd32(E1000_IVAR0, index); if (tx_queue & 0x1) { /* vector goes into high byte of register */ @@ -526,9 +488,9 @@ static void igb_assign_vector(struct igb_adapter *adapter, int rx_queue, ivar = ivar & 0xFFFF00FF; ivar |= (msix_vector | E1000_IVAR_VALID) << 8; } - adapter->tx_ring[tx_queue].eims_value= 1 << msix_vector; array_wr32(E1000_IVAR0, index, ivar); } + q_vector->eims_value = 1 << msix_vector; break; default: BUG(); @@ -549,43 +511,10 @@ static void igb_configure_msix(struct igb_adapter *adapter) struct e1000_hw *hw = &adapter->hw; adapter->eims_enable_mask = 0; - if (hw->mac.type == e1000_82576) - /* Turn on MSI-X capability first, or our settings - * won't stick. And it will take days to debug. */ - wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE | - E1000_GPIE_PBA | E1000_GPIE_EIAME | - E1000_GPIE_NSICR); - - for (i = 0; i < adapter->num_tx_queues; i++) { - struct igb_ring *tx_ring = &adapter->tx_ring[i]; - igb_assign_vector(adapter, IGB_N0_QUEUE, i, vector++); - adapter->eims_enable_mask |= tx_ring->eims_value; - if (tx_ring->itr_val) - writel(tx_ring->itr_val, - hw->hw_addr + tx_ring->itr_register); - else - writel(1, hw->hw_addr + tx_ring->itr_register); - } - - for (i = 0; i < adapter->num_rx_queues; i++) { - struct igb_ring *rx_ring = &adapter->rx_ring[i]; - rx_ring->buddy = NULL; - igb_assign_vector(adapter, i, IGB_N0_QUEUE, vector++); - adapter->eims_enable_mask |= rx_ring->eims_value; - if (rx_ring->itr_val) - writel(rx_ring->itr_val, - hw->hw_addr + rx_ring->itr_register); - else - writel(1, hw->hw_addr + rx_ring->itr_register); - } - /* set vector for other causes, i.e. link changes */ switch (hw->mac.type) { case e1000_82575: - array_wr32(E1000_MSIXBM(0), vector++, - E1000_EIMS_OTHER); - tmp = rd32(E1000_CTRL_EXT); /* enable MSI-X PBA support*/ tmp |= E1000_CTRL_EXT_PBA_CLR; @@ -595,22 +524,41 @@ static void igb_configure_msix(struct igb_adapter *adapter) tmp |= E1000_CTRL_EXT_IRCA; wr32(E1000_CTRL_EXT, tmp); - adapter->eims_enable_mask |= E1000_EIMS_OTHER; + + /* enable msix_other interrupt */ + array_wr32(E1000_MSIXBM(0), vector++, + E1000_EIMS_OTHER); adapter->eims_other = E1000_EIMS_OTHER; break; case e1000_82576: + case e1000_82580: + /* Turn on MSI-X capability first, or our settings + * won't stick. And it will take days to debug. */ + wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE | + E1000_GPIE_PBA | E1000_GPIE_EIAME | + E1000_GPIE_NSICR); + + /* enable msix_other interrupt */ + adapter->eims_other = 1 << vector; tmp = (vector++ | E1000_IVAR_VALID) << 8; - wr32(E1000_IVAR_MISC, tmp); - adapter->eims_enable_mask = (1 << (vector)) - 1; - adapter->eims_other = 1 << (vector - 1); + wr32(E1000_IVAR_MISC, tmp); break; default: /* do nothing, since nothing else supports MSI-X */ break; } /* switch (hw->mac.type) */ + + adapter->eims_enable_mask |= adapter->eims_other; + + for (i = 0; i < adapter->num_q_vectors; i++) { + struct igb_q_vector *q_vector = adapter->q_vector[i]; + igb_assign_vector(q_vector, vector++); + adapter->eims_enable_mask |= q_vector->eims_value; + } + wrfl(); } @@ -623,43 +571,40 @@ static void igb_configure_msix(struct igb_adapter *adapter) static int igb_request_msix(struct igb_adapter *adapter) { struct net_device *netdev = adapter->netdev; + struct e1000_hw *hw = &adapter->hw; int i, err = 0, vector = 0; - vector = 0; - - for (i = 0; i < adapter->num_tx_queues; i++) { - struct igb_ring *ring = &(adapter->tx_ring[i]); - sprintf(ring->name, "%s-tx-%d", netdev->name, i); - err = request_irq(adapter->msix_entries[vector].vector, - &igb_msix_tx, 0, ring->name, - &(adapter->tx_ring[i])); - if (err) - goto out; - ring->itr_register = E1000_EITR(0) + (vector << 2); - ring->itr_val = 976; /* ~4000 ints/sec */ - vector++; - } - for (i = 0; i < adapter->num_rx_queues; i++) { - struct igb_ring *ring = &(adapter->rx_ring[i]); - if (strlen(netdev->name) < (IFNAMSIZ - 5)) - sprintf(ring->name, "%s-rx-%d", netdev->name, i); + err = request_irq(adapter->msix_entries[vector].vector, + igb_msix_other, 0, netdev->name, adapter); + if (err) + goto out; + vector++; + + for (i = 0; i < adapter->num_q_vectors; i++) { + struct igb_q_vector *q_vector = adapter->q_vector[i]; + + q_vector->itr_register = hw->hw_addr + E1000_EITR(vector); + + if (q_vector->rx_ring && q_vector->tx_ring) + sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, + q_vector->rx_ring->queue_index); + else if (q_vector->tx_ring) + sprintf(q_vector->name, "%s-tx-%u", netdev->name, + q_vector->tx_ring->queue_index); + else if (q_vector->rx_ring) + sprintf(q_vector->name, "%s-rx-%u", netdev->name, + q_vector->rx_ring->queue_index); else - memcpy(ring->name, netdev->name, IFNAMSIZ); + sprintf(q_vector->name, "%s-unused", netdev->name); + err = request_irq(adapter->msix_entries[vector].vector, - &igb_msix_rx, 0, ring->name, - &(adapter->rx_ring[i])); + igb_msix_ring, 0, q_vector->name, + q_vector); if (err) goto out; - ring->itr_register = E1000_EITR(0) + (vector << 2); - ring->itr_val = adapter->itr; vector++; } - err = request_irq(adapter->msix_entries[vector].vector, - &igb_msix_other, 0, netdev->name, netdev); - if (err) - goto out; - igb_configure_msix(adapter); return 0; out: @@ -672,11 +617,44 @@ static void igb_reset_interrupt_capability(struct igb_adapter *adapter) pci_disable_msix(adapter->pdev); kfree(adapter->msix_entries); adapter->msix_entries = NULL; - } else if (adapter->flags & IGB_FLAG_HAS_MSI) + } else if (adapter->flags & IGB_FLAG_HAS_MSI) { pci_disable_msi(adapter->pdev); - return; + } +} + +/** + * igb_free_q_vectors - Free memory allocated for interrupt vectors + * @adapter: board private structure to initialize + * + * This function frees the memory allocated to the q_vectors. In addition if + * NAPI is enabled it will delete any references to the NAPI struct prior + * to freeing the q_vector. + **/ +static void igb_free_q_vectors(struct igb_adapter *adapter) +{ + int v_idx; + + for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) { + struct igb_q_vector *q_vector = adapter->q_vector[v_idx]; + adapter->q_vector[v_idx] = NULL; + netif_napi_del(&q_vector->napi); + kfree(q_vector); + } + adapter->num_q_vectors = 0; } +/** + * igb_clear_interrupt_scheme - reset the device to a state of no interrupts + * + * This function resets the device so that it has 0 rx queues, tx queues, and + * MSI-X interrupts allocated. + */ +static void igb_clear_interrupt_scheme(struct igb_adapter *adapter) +{ + igb_free_queues(adapter); + igb_free_q_vectors(adapter); + igb_reset_interrupt_capability(adapter); +} /** * igb_set_interrupt_capability - set MSI or MSI-X if supported @@ -690,11 +668,21 @@ static void igb_set_interrupt_capability(struct igb_adapter *adapter) int numvecs, i; /* Number of supported queues. */ - /* Having more queues than CPUs doesn't make sense. */ - adapter->num_rx_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus()); - adapter->num_tx_queues = min_t(u32, IGB_MAX_TX_QUEUES, num_online_cpus()); + adapter->num_rx_queues = adapter->rss_queues; + adapter->num_tx_queues = adapter->rss_queues; + + /* start with one vector for every rx queue */ + numvecs = adapter->num_rx_queues; + + /* if tx handler is seperate add 1 for every tx queue */ + if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS)) + numvecs += adapter->num_tx_queues; + + /* store the number of vectors reserved for queues */ + adapter->num_q_vectors = numvecs; - numvecs = adapter->num_tx_queues + adapter->num_rx_queues + 1; + /* add 1 vector for link status interrupts */ + numvecs++; adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), GFP_KERNEL); if (!adapter->msix_entries) @@ -728,8 +716,12 @@ msi_only: dev_info(&adapter->pdev->dev, "IOV Disabled\n"); } #endif + adapter->vfs_allocated_count = 0; + adapter->rss_queues = 1; + adapter->flags |= IGB_FLAG_QUEUE_PAIRS; adapter->num_rx_queues = 1; adapter->num_tx_queues = 1; + adapter->num_q_vectors = 1; if (!pci_enable_msi(adapter->pdev)) adapter->flags |= IGB_FLAG_HAS_MSI; out: @@ -739,6 +731,143 @@ out: } /** + * igb_alloc_q_vectors - Allocate memory for interrupt vectors + * @adapter: board private structure to initialize + * + * We allocate one q_vector per queue interrupt. If allocation fails we + * return -ENOMEM. + **/ +static int igb_alloc_q_vectors(struct igb_adapter *adapter) +{ + struct igb_q_vector *q_vector; + struct e1000_hw *hw = &adapter->hw; + int v_idx; + + for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) { + q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL); + if (!q_vector) + goto err_out; + q_vector->adapter = adapter; + q_vector->itr_shift = (hw->mac.type == e1000_82575) ? 16 : 0; + q_vector->itr_register = hw->hw_addr + E1000_EITR(0); + q_vector->itr_val = IGB_START_ITR; + q_vector->set_itr = 1; + netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64); + adapter->q_vector[v_idx] = q_vector; + } + return 0; + +err_out: + while (v_idx) { + v_idx--; + q_vector = adapter->q_vector[v_idx]; + netif_napi_del(&q_vector->napi); + kfree(q_vector); + adapter->q_vector[v_idx] = NULL; + } + return -ENOMEM; +} + +static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter, + int ring_idx, int v_idx) +{ + struct igb_q_vector *q_vector; + + q_vector = adapter->q_vector[v_idx]; + q_vector->rx_ring = &adapter->rx_ring[ring_idx]; + q_vector->rx_ring->q_vector = q_vector; + q_vector->itr_val = adapter->rx_itr_setting; + if (q_vector->itr_val && q_vector->itr_val <= 3) + q_vector->itr_val = IGB_START_ITR; +} + +static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter, + int ring_idx, int v_idx) +{ + struct igb_q_vector *q_vector; + + q_vector = adapter->q_vector[v_idx]; + q_vector->tx_ring = &adapter->tx_ring[ring_idx]; + q_vector->tx_ring->q_vector = q_vector; + q_vector->itr_val = adapter->tx_itr_setting; + if (q_vector->itr_val && q_vector->itr_val <= 3) + q_vector->itr_val = IGB_START_ITR; +} + +/** + * igb_map_ring_to_vector - maps allocated queues to vectors + * + * This function maps the recently allocated queues to vectors. + **/ +static int igb_map_ring_to_vector(struct igb_adapter *adapter) +{ + int i; + int v_idx = 0; + + if ((adapter->num_q_vectors < adapter->num_rx_queues) || + (adapter->num_q_vectors < adapter->num_tx_queues)) + return -ENOMEM; + + if (adapter->num_q_vectors >= + (adapter->num_rx_queues + adapter->num_tx_queues)) { + for (i = 0; i < adapter->num_rx_queues; i++) + igb_map_rx_ring_to_vector(adapter, i, v_idx++); + for (i = 0; i < adapter->num_tx_queues; i++) + igb_map_tx_ring_to_vector(adapter, i, v_idx++); + } else { + for (i = 0; i < adapter->num_rx_queues; i++) { + if (i < adapter->num_tx_queues) + igb_map_tx_ring_to_vector(adapter, i, v_idx); + igb_map_rx_ring_to_vector(adapter, i, v_idx++); + } + for (; i < adapter->num_tx_queues; i++) + igb_map_tx_ring_to_vector(adapter, i, v_idx++); + } + return 0; +} + +/** + * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors + * + * This function initializes the interrupts and allocates all of the queues. + **/ +static int igb_init_interrupt_scheme(struct igb_adapter *adapter) +{ + struct pci_dev *pdev = adapter->pdev; + int err; + + igb_set_interrupt_capability(adapter); + + err = igb_alloc_q_vectors(adapter); + if (err) { + dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); + goto err_alloc_q_vectors; + } + + err = igb_alloc_queues(adapter); + if (err) { + dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); + goto err_alloc_queues; + } + + err = igb_map_ring_to_vector(adapter); + if (err) { + dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n"); + goto err_map_queues; + } + + + return 0; +err_map_queues: + igb_free_queues(adapter); +err_alloc_queues: + igb_free_q_vectors(adapter); +err_alloc_q_vectors: + igb_reset_interrupt_capability(adapter); + return err; +} + +/** * igb_request_irq - initialize interrupts * * Attempts to configure interrupts using the best available @@ -747,6 +876,7 @@ out: static int igb_request_irq(struct igb_adapter *adapter) { struct net_device *netdev = adapter->netdev; + struct pci_dev *pdev = adapter->pdev; struct e1000_hw *hw = &adapter->hw; int err = 0; @@ -755,19 +885,38 @@ static int igb_request_irq(struct igb_adapter *adapter) if (!err) goto request_done; /* fall back to MSI */ - igb_reset_interrupt_capability(adapter); + igb_clear_interrupt_scheme(adapter); if (!pci_enable_msi(adapter->pdev)) adapter->flags |= IGB_FLAG_HAS_MSI; igb_free_all_tx_resources(adapter); igb_free_all_rx_resources(adapter); + adapter->num_tx_queues = 1; adapter->num_rx_queues = 1; - igb_alloc_queues(adapter); + adapter->num_q_vectors = 1; + err = igb_alloc_q_vectors(adapter); + if (err) { + dev_err(&pdev->dev, + "Unable to allocate memory for vectors\n"); + goto request_done; + } + err = igb_alloc_queues(adapter); + if (err) { + dev_err(&pdev->dev, + "Unable to allocate memory for queues\n"); + igb_free_q_vectors(adapter); + goto request_done; + } + igb_setup_all_tx_resources(adapter); + igb_setup_all_rx_resources(adapter); } else { switch (hw->mac.type) { case e1000_82575: wr32(E1000_MSIXBM(0), - (E1000_EICR_RX_QUEUE0 | E1000_EIMS_OTHER)); + (E1000_EICR_RX_QUEUE0 | + E1000_EICR_TX_QUEUE0 | + E1000_EIMS_OTHER)); break; + case e1000_82580: case e1000_82576: wr32(E1000_IVAR0, E1000_IVAR_VALID); break; @@ -777,17 +926,18 @@ static int igb_request_irq(struct igb_adapter *adapter) } if (adapter->flags & IGB_FLAG_HAS_MSI) { - err = request_irq(adapter->pdev->irq, &igb_intr_msi, 0, - netdev->name, netdev); + err = request_irq(adapter->pdev->irq, igb_intr_msi, 0, + netdev->name, adapter); if (!err) goto request_done; + /* fall back to legacy interrupts */ igb_reset_interrupt_capability(adapter); adapter->flags &= ~IGB_FLAG_HAS_MSI; } - err = request_irq(adapter->pdev->irq, &igb_intr, IRQF_SHARED, - netdev->name, netdev); + err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED, + netdev->name, adapter); if (err) dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n", @@ -799,23 +949,19 @@ request_done: static void igb_free_irq(struct igb_adapter *adapter) { - struct net_device *netdev = adapter->netdev; - if (adapter->msix_entries) { int vector = 0, i; - for (i = 0; i < adapter->num_tx_queues; i++) - free_irq(adapter->msix_entries[vector++].vector, - &(adapter->tx_ring[i])); - for (i = 0; i < adapter->num_rx_queues; i++) - free_irq(adapter->msix_entries[vector++].vector, - &(adapter->rx_ring[i])); + free_irq(adapter->msix_entries[vector++].vector, adapter); - free_irq(adapter->msix_entries[vector++].vector, netdev); - return; + for (i = 0; i < adapter->num_q_vectors; i++) { + struct igb_q_vector *q_vector = adapter->q_vector[i]; + free_irq(adapter->msix_entries[vector++].vector, + q_vector); + } + } else { + free_irq(adapter->pdev->irq, adapter); } - - free_irq(adapter->pdev->irq, netdev); } /** @@ -826,6 +972,11 @@ static void igb_irq_disable(struct igb_adapter *adapter) { struct e1000_hw *hw = &adapter->hw; + /* + * we need to be careful when disabling interrupts. The VFs are also + * mapped into these registers and so clearing the bits can cause + * issues on the VF drivers so we only need to clear what we set + */ if (adapter->msix_entries) { u32 regval = rd32(E1000_EIAM); wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask); @@ -849,41 +1000,47 @@ static void igb_irq_enable(struct igb_adapter *adapter) struct e1000_hw *hw = &adapter->hw; if (adapter->msix_entries) { + u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC; u32 regval = rd32(E1000_EIAC); wr32(E1000_EIAC, regval | adapter->eims_enable_mask); regval = rd32(E1000_EIAM); wr32(E1000_EIAM, regval | adapter->eims_enable_mask); wr32(E1000_EIMS, adapter->eims_enable_mask); - if (adapter->vfs_allocated_count) + if (adapter->vfs_allocated_count) { wr32(E1000_MBVFIMR, 0xFF); - wr32(E1000_IMS, (E1000_IMS_LSC | E1000_IMS_VMMB | - E1000_IMS_DOUTSYNC)); + ims |= E1000_IMS_VMMB; + } + if (adapter->hw.mac.type == e1000_82580) + ims |= E1000_IMS_DRSTA; + + wr32(E1000_IMS, ims); } else { - wr32(E1000_IMS, IMS_ENABLE_MASK); - wr32(E1000_IAM, IMS_ENABLE_MASK); + wr32(E1000_IMS, IMS_ENABLE_MASK | + E1000_IMS_DRSTA); + wr32(E1000_IAM, IMS_ENABLE_MASK | + E1000_IMS_DRSTA); } } static void igb_update_mng_vlan(struct igb_adapter *adapter) { - struct net_device *netdev = adapter->netdev; + struct e1000_hw *hw = &adapter->hw; u16 vid = adapter->hw.mng_cookie.vlan_id; u16 old_vid = adapter->mng_vlan_id; - if (adapter->vlgrp) { - if (!vlan_group_get_device(adapter->vlgrp, vid)) { - if (adapter->hw.mng_cookie.status & - E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { - igb_vlan_rx_add_vid(netdev, vid); - adapter->mng_vlan_id = vid; - } else - adapter->mng_vlan_id = IGB_MNG_VLAN_NONE; - if ((old_vid != (u16)IGB_MNG_VLAN_NONE) && - (vid != old_vid) && - !vlan_group_get_device(adapter->vlgrp, old_vid)) - igb_vlan_rx_kill_vid(netdev, old_vid); - } else - adapter->mng_vlan_id = vid; + if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) { + /* add VID to filter table */ + igb_vfta_set(hw, vid, true); + adapter->mng_vlan_id = vid; + } else { + adapter->mng_vlan_id = IGB_MNG_VLAN_NONE; + } + + if ((old_vid != (u16)IGB_MNG_VLAN_NONE) && + (vid != old_vid) && + !vlan_group_get_device(adapter->vlgrp, old_vid)) { + /* remove VID from filter table */ + igb_vfta_set(hw, old_vid, false); } } @@ -907,7 +1064,6 @@ static void igb_release_hw_control(struct igb_adapter *adapter) |