diff options
Diffstat (limited to 'drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c')
| -rw-r--r-- | drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c | 1809 |
1 files changed, 1088 insertions, 721 deletions
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index de1ad506665..75467f83772 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -1,7 +1,7 @@ /******************************************************************************* Intel 82599 Virtual Function driver - Copyright(c) 1999 - 2012 Intel Corporation. + Copyright(c) 1999 - 2014 Intel Corporation. This program is free software; you can redistribute it and/or modify it under the terms and conditions of the GNU General Public License, @@ -58,7 +58,7 @@ const char ixgbevf_driver_name[] = "ixgbevf"; static const char ixgbevf_driver_string[] = "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; -#define DRV_VERSION "2.6.0-k" +#define DRV_VERSION "2.12.1-k" const char ixgbevf_driver_version[] = DRV_VERSION; static char ixgbevf_copyright[] = "Copyright (c) 2009 - 2012 Intel Corporation."; @@ -76,19 +76,16 @@ static const struct ixgbevf_info *ixgbevf_info_tbl[] = { * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, * Class, Class Mask, private data (not used) } */ -static struct pci_device_id ixgbevf_pci_tbl[] = { - {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), - board_82599_vf}, - {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), - board_X540_vf}, - +static DEFINE_PCI_DEVICE_TABLE(ixgbevf_pci_tbl) = { + {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, + {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, /* required last entry */ {0, } }; MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); -MODULE_DESCRIPTION("Intel(R) 82599 Virtual Function Driver"); +MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); MODULE_LICENSE("GPL"); MODULE_VERSION(DRV_VERSION); @@ -98,12 +95,59 @@ module_param(debug, int, 0); MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); /* forward decls */ +static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); +static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); + +static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) +{ + struct ixgbevf_adapter *adapter = hw->back; + + if (!hw->hw_addr) + return; + hw->hw_addr = NULL; + dev_err(&adapter->pdev->dev, "Adapter removed\n"); + if (test_bit(__IXGBEVF_WORK_INIT, &adapter->state)) + schedule_work(&adapter->watchdog_task); +} + +static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) +{ + u32 value; + + /* The following check not only optimizes a bit by not + * performing a read on the status register when the + * register just read was a status register read that + * returned IXGBE_FAILED_READ_REG. It also blocks any + * potential recursion. + */ + if (reg == IXGBE_VFSTATUS) { + ixgbevf_remove_adapter(hw); + return; + } + value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); + if (value == IXGBE_FAILED_READ_REG) + ixgbevf_remove_adapter(hw); +} + +u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) +{ + u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr); + u32 value; -static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw, - struct ixgbevf_ring *rx_ring, + if (IXGBE_REMOVED(reg_addr)) + return IXGBE_FAILED_READ_REG; + value = readl(reg_addr + reg); + if (unlikely(value == IXGBE_FAILED_READ_REG)) + ixgbevf_check_remove(hw, reg); + return value; +} + +static inline void ixgbevf_release_rx_desc(struct ixgbevf_ring *rx_ring, u32 val) { + rx_ring->next_to_use = val; + /* * Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only @@ -111,7 +155,7 @@ static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw, * such as IA-64). */ wmb(); - IXGBE_WRITE_REG(hw, IXGBE_VFRDT(rx_ring->reg_idx), val); + ixgbevf_write_tail(rx_ring, val); } /** @@ -120,7 +164,6 @@ static inline void ixgbevf_release_rx_desc(struct ixgbe_hw *hw, * @direction: 0 for Rx, 1 for Tx, -1 for other causes * @queue: queue to map the corresponding interrupt to * @msix_vector: the vector to map to the corresponding queue - * */ static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, u8 queue, u8 msix_vector) @@ -146,28 +189,25 @@ static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, } static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer - *tx_buffer_info) -{ - if (tx_buffer_info->dma) { - if (tx_buffer_info->mapped_as_page) - dma_unmap_page(tx_ring->dev, - tx_buffer_info->dma, - tx_buffer_info->length, - DMA_TO_DEVICE); - else + struct ixgbevf_tx_buffer *tx_buffer) +{ + if (tx_buffer->skb) { + dev_kfree_skb_any(tx_buffer->skb); + if (dma_unmap_len(tx_buffer, len)) dma_unmap_single(tx_ring->dev, - tx_buffer_info->dma, - tx_buffer_info->length, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), DMA_TO_DEVICE); - tx_buffer_info->dma = 0; - } - if (tx_buffer_info->skb) { - dev_kfree_skb_any(tx_buffer_info->skb); - tx_buffer_info->skb = NULL; + } else if (dma_unmap_len(tx_buffer, len)) { + dma_unmap_page(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); } - tx_buffer_info->time_stamp = 0; - /* tx_buffer_info must be completely set up in the transmit path */ + tx_buffer->next_to_watch = NULL; + tx_buffer->skb = NULL; + dma_unmap_len_set(tx_buffer, len, 0); + /* tx_buffer must be completely set up in the transmit path */ } #define IXGBE_MAX_TXD_PWR 14 @@ -188,85 +228,118 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring) { struct ixgbevf_adapter *adapter = q_vector->adapter; - union ixgbe_adv_tx_desc *tx_desc, *eop_desc; - struct ixgbevf_tx_buffer *tx_buffer_info; - unsigned int i, eop, count = 0; + struct ixgbevf_tx_buffer *tx_buffer; + union ixgbe_adv_tx_desc *tx_desc; unsigned int total_bytes = 0, total_packets = 0; + unsigned int budget = tx_ring->count / 2; + unsigned int i = tx_ring->next_to_clean; if (test_bit(__IXGBEVF_DOWN, &adapter->state)) return true; - i = tx_ring->next_to_clean; - eop = tx_ring->tx_buffer_info[i].next_to_watch; - eop_desc = IXGBEVF_TX_DESC(tx_ring, eop); - - while ((eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)) && - (count < tx_ring->count)) { - bool cleaned = false; - rmb(); /* read buffer_info after eop_desc */ - /* eop could change between read and DD-check */ - if (unlikely(eop != tx_ring->tx_buffer_info[i].next_to_watch)) - goto cont_loop; - for ( ; !cleaned; count++) { - struct sk_buff *skb; - tx_desc = IXGBEVF_TX_DESC(tx_ring, i); - tx_buffer_info = &tx_ring->tx_buffer_info[i]; - cleaned = (i == eop); - skb = tx_buffer_info->skb; - - if (cleaned && skb) { - unsigned int segs, bytecount; - - /* gso_segs is currently only valid for tcp */ - segs = skb_shinfo(skb)->gso_segs ?: 1; - /* multiply data chunks by size of headers */ - bytecount = ((segs - 1) * skb_headlen(skb)) + - skb->len; - total_packets += segs; - total_bytes += bytecount; - } + tx_buffer = &tx_ring->tx_buffer_info[i]; + tx_desc = IXGBEVF_TX_DESC(tx_ring, i); + i -= tx_ring->count; + + do { + union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; + + /* if next_to_watch is not set then there is no work pending */ + if (!eop_desc) + break; + + /* prevent any other reads prior to eop_desc */ + read_barrier_depends(); - ixgbevf_unmap_and_free_tx_resource(tx_ring, - tx_buffer_info); + /* if DD is not set pending work has not been completed */ + if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) + break; + + /* clear next_to_watch to prevent false hangs */ + tx_buffer->next_to_watch = NULL; + + /* update the statistics for this packet */ + total_bytes += tx_buffer->bytecount; + total_packets += tx_buffer->gso_segs; + + /* free the skb */ + dev_kfree_skb_any(tx_buffer->skb); - tx_desc->wb.status = 0; + /* unmap skb header data */ + dma_unmap_single(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + /* clear tx_buffer data */ + tx_buffer->skb = NULL; + dma_unmap_len_set(tx_buffer, len, 0); + + /* unmap remaining buffers */ + while (tx_desc != eop_desc) { + tx_buffer++; + tx_desc++; i++; - if (i == tx_ring->count) - i = 0; + if (unlikely(!i)) { + i -= tx_ring->count; + tx_buffer = tx_ring->tx_buffer_info; + tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); + } + + /* unmap any remaining paged data */ + if (dma_unmap_len(tx_buffer, len)) { + dma_unmap_page(tx_ring->dev, + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); + dma_unmap_len_set(tx_buffer, len, 0); + } } -cont_loop: - eop = tx_ring->tx_buffer_info[i].next_to_watch; - eop_desc = IXGBEVF_TX_DESC(tx_ring, eop); - } + /* move us one more past the eop_desc for start of next pkt */ + tx_buffer++; + tx_desc++; + i++; + if (unlikely(!i)) { + i -= tx_ring->count; + tx_buffer = tx_ring->tx_buffer_info; + tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); + } + /* issue prefetch for next Tx descriptor */ + prefetch(tx_desc); + + /* update budget accounting */ + budget--; + } while (likely(budget)); + + i += tx_ring->count; tx_ring->next_to_clean = i; + u64_stats_update_begin(&tx_ring->syncp); + tx_ring->stats.bytes += total_bytes; + tx_ring->stats.packets += total_packets; + u64_stats_update_end(&tx_ring->syncp); + q_vector->tx.total_bytes += total_bytes; + q_vector->tx.total_packets += total_packets; #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) - if (unlikely(count && netif_carrier_ok(tx_ring->netdev) && - (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { + if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && + (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { /* Make sure that anybody stopping the queue after this * sees the new next_to_clean. */ smp_mb(); + if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) && !test_bit(__IXGBEVF_DOWN, &adapter->state)) { netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index); - ++adapter->restart_queue; + ++tx_ring->tx_stats.restart_queue; } } - u64_stats_update_begin(&tx_ring->syncp); - tx_ring->total_bytes += total_bytes; - tx_ring->total_packets += total_packets; - u64_stats_update_end(&tx_ring->syncp); - q_vector->tx.total_bytes += total_bytes; - q_vector->tx.total_packets += total_packets; - - return count < tx_ring->count; + return !!budget; } /** @@ -285,19 +358,45 @@ static void ixgbevf_receive_skb(struct ixgbevf_q_vector *q_vector, u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan); if (is_vlan && test_bit(tag & VLAN_VID_MASK, adapter->active_vlans)) - __vlan_hwaccel_put_tag(skb, tag); + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag); + + if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL)) + napi_gro_receive(&q_vector->napi, skb); + else + netif_rx(skb); +} + +/** + * ixgbevf_rx_skb - Helper function to determine proper Rx method + * @q_vector: structure containing interrupt and ring information + * @skb: packet to send up + * @status: hardware indication of status of receive + * @rx_desc: rx descriptor + **/ +static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, + struct sk_buff *skb, u8 status, + union ixgbe_adv_rx_desc *rx_desc) +{ +#ifdef CONFIG_NET_RX_BUSY_POLL + skb_mark_napi_id(skb, &q_vector->napi); + + if (ixgbevf_qv_busy_polling(q_vector)) { + netif_receive_skb(skb); + /* exit early if we busy polled */ + return; + } +#endif /* CONFIG_NET_RX_BUSY_POLL */ - napi_gro_receive(&q_vector->napi, skb); + ixgbevf_receive_skb(q_vector, skb, status, rx_desc); } /** * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum - * @adapter: address of board private structure + * @ring: pointer to Rx descriptor ring structure * @status_err: hardware indication of status of receive * @skb: skb currently being received and modified **/ -static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *ring, +static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, u32 status_err, struct sk_buff *skb) { skb_checksum_none_assert(skb); @@ -309,7 +408,7 @@ static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter, /* if IP and error */ if ((status_err & IXGBE_RXD_STAT_IPCS) && (status_err & IXGBE_RXDADV_ERR_IPE)) { - adapter->hw_csum_rx_error++; + ring->rx_stats.csum_err++; return; } @@ -317,62 +416,60 @@ static inline void ixgbevf_rx_checksum(struct ixgbevf_adapter *adapter, return; if (status_err & IXGBE_RXDADV_ERR_TCPE) { - adapter->hw_csum_rx_error++; + ring->rx_stats.csum_err++; return; } /* It must be a TCP or UDP packet with a valid checksum */ skb->ip_summed = CHECKSUM_UNNECESSARY; - adapter->hw_csum_rx_good++; } /** * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split - * @adapter: address of board private structure + * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on **/ -static void ixgbevf_alloc_rx_buffers(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring, +static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, int cleaned_count) { - struct pci_dev *pdev = adapter->pdev; union ixgbe_adv_rx_desc *rx_desc; struct ixgbevf_rx_buffer *bi; - struct sk_buff *skb; unsigned int i = rx_ring->next_to_use; - bi = &rx_ring->rx_buffer_info[i]; - while (cleaned_count--) { rx_desc = IXGBEVF_RX_DESC(rx_ring, i); - skb = bi->skb; - if (!skb) { + bi = &rx_ring->rx_buffer_info[i]; + + if (!bi->skb) { + struct sk_buff *skb; + skb = netdev_alloc_skb_ip_align(rx_ring->netdev, rx_ring->rx_buf_len); - if (!skb) { - adapter->alloc_rx_buff_failed++; + if (!skb) goto no_buffers; - } + bi->skb = skb; - } - if (!bi->dma) { - bi->dma = dma_map_single(&pdev->dev, skb->data, + + bi->dma = dma_map_single(rx_ring->dev, skb->data, rx_ring->rx_buf_len, DMA_FROM_DEVICE); + if (dma_mapping_error(rx_ring->dev, bi->dma)) { + dev_kfree_skb(skb); + bi->skb = NULL; + dev_err(rx_ring->dev, "Rx DMA map failed\n"); + break; + } } rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); i++; if (i == rx_ring->count) i = 0; - bi = &rx_ring->rx_buffer_info[i]; } no_buffers: - if (rx_ring->next_to_use != i) { - rx_ring->next_to_use = i; - - ixgbevf_release_rx_desc(&adapter->hw, rx_ring, i); - } + rx_ring->rx_stats.alloc_rx_buff_failed++; + if (rx_ring->next_to_use != i) + ixgbevf_release_rx_desc(rx_ring, i); } static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, @@ -383,12 +480,10 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); } -static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, - struct ixgbevf_ring *rx_ring, - int budget) +static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, + struct ixgbevf_ring *rx_ring, + int budget) { - struct ixgbevf_adapter *adapter = q_vector->adapter; - struct pci_dev *pdev = adapter->pdev; union ixgbe_adv_rx_desc *rx_desc, *next_rxd; struct ixgbevf_rx_buffer *rx_buffer_info, *next_buffer; struct sk_buff *skb; @@ -414,7 +509,7 @@ static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, rx_buffer_info->skb = NULL; if (rx_buffer_info->dma) { - dma_unmap_single(&pdev->dev, rx_buffer_info->dma, + dma_unmap_single(rx_ring->dev, rx_buffer_info->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); rx_buffer_info->dma = 0; @@ -434,7 +529,7 @@ static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, if (!(staterr & IXGBE_RXD_STAT_EOP)) { skb->next = next_buffer->skb; IXGBE_CB(skb->next)->prev = skb; - adapter->non_eop_descs++; + rx_ring->rx_stats.non_eop_descs++; goto next_desc; } @@ -454,32 +549,33 @@ static bool ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, goto next_desc; } - ixgbevf_rx_checksum(adapter, rx_ring, staterr, skb); + ixgbevf_rx_checksum(rx_ring, staterr, skb); /* probably a little skewed due to removing CRC */ total_rx_bytes += skb->len; total_rx_packets++; - /* - * Work around issue of some types of VM to VM loop back - * packets not getting split correctly + skb->protocol = eth_type_trans(skb, rx_ring->netdev); + + /* Workaround hardware that can't do proper VEPA multicast + * source pruning. */ - if (staterr & IXGBE_RXD_STAT_LB) { - u32 header_fixup_len = skb_headlen(skb); - if (header_fixup_len < 14) - skb_push(skb, header_fixup_len); + if ((skb->pkt_type == PACKET_BROADCAST || + skb->pkt_type == PACKET_MULTICAST) && + ether_addr_equal(rx_ring->netdev->dev_addr, + eth_hdr(skb)->h_source)) { + dev_kfree_skb_irq(skb); + goto next_desc; } - skb->protocol = eth_type_trans(skb, rx_ring->netdev); - ixgbevf_receive_skb(q_vector, skb, staterr, rx_desc); + ixgbevf_rx_skb(q_vector, skb, staterr, rx_desc); next_desc: rx_desc->wb.upper.status_error = 0; /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { - ixgbevf_alloc_rx_buffers(adapter, rx_ring, - cleaned_count); + ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); cleaned_count = 0; } @@ -491,19 +587,19 @@ next_desc: } rx_ring->next_to_clean = i; - cleaned_count = IXGBE_DESC_UNUSED(rx_ring); + cleaned_count = ixgbevf_desc_unused(rx_ring); if (cleaned_count) - ixgbevf_alloc_rx_buffers(adapter, rx_ring, cleaned_count); + ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); u64_stats_update_begin(&rx_ring->syncp); - rx_ring->total_packets += total_rx_packets; - rx_ring->total_bytes += total_rx_bytes; + rx_ring->stats.packets += total_rx_packets; + rx_ring->stats.bytes += total_rx_bytes; u64_stats_update_end(&rx_ring->syncp); q_vector->rx.total_packets += total_rx_packets; q_vector->rx.total_bytes += total_rx_bytes; - return !!budget; + return total_rx_packets; } /** @@ -526,6 +622,11 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) ixgbevf_for_each_ring(ring, q_vector->tx) clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring); +#ifdef CONFIG_NET_RX_BUSY_POLL + if (!ixgbevf_qv_lock_napi(q_vector)) + return budget; +#endif + /* attempt to distribute budget to each queue fairly, but don't allow * the budget to go below 1 because we'll exit polling */ if (q_vector->rx.count > 1) @@ -533,9 +634,16 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) else per_ring_budget = budget; + adapter->flags |= IXGBE_FLAG_IN_NETPOLL; ixgbevf_for_each_ring(ring, q_vector->rx) - clean_complete &= ixgbevf_clean_rx_irq(q_vector, ring, - per_ring_budget); + clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring, + per_ring_budget) + < per_ring_budget); + adapter->flags &= ~IXGBE_FLAG_IN_NETPOLL; + +#ifdef CONFIG_NET_RX_BUSY_POLL + ixgbevf_qv_unlock_napi(q_vector); +#endif /* If all work not completed, return budget and keep polling */ if (!clean_complete) @@ -544,7 +652,8 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) napi_complete(napi); if (adapter->rx_itr_setting & 1) ixgbevf_set_itr(q_vector); - if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) + if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && + !test_bit(__IXGBEVF_REMOVING, &adapter->state)) ixgbevf_irq_enable_queues(adapter, 1 << q_vector->v_idx); @@ -555,7 +664,7 @@ static int ixgbevf_poll(struct napi_struct *napi, int budget) * ixgbevf_write_eitr - write VTEITR register in hardware specific way * @q_vector: structure containing interrupt and ring information */ -static void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) +void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) { struct ixgbevf_adapter *adapter = q_vector->adapter; struct ixgbe_hw *hw = &adapter->hw; @@ -571,6 +680,40 @@ static void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); } +#ifdef CONFIG_NET_RX_BUSY_POLL +/* must be called with local_bh_disable()d */ +static int ixgbevf_busy_poll_recv(struct napi_struct *napi) +{ + struct ixgbevf_q_vector *q_vector = + container_of(napi, struct ixgbevf_q_vector, napi); + struct ixgbevf_adapter *adapter = q_vector->adapter; + struct ixgbevf_ring *ring; + int found = 0; + + if (test_bit(__IXGBEVF_DOWN, &adapter->state)) + return LL_FLUSH_FAILED; + + if (!ixgbevf_qv_lock_poll(q_vector)) + return LL_FLUSH_BUSY; + + ixgbevf_for_each_ring(ring, q_vector->rx) { + found = ixgbevf_clean_rx_irq(q_vector, ring, 4); +#ifdef BP_EXTENDED_STATS + if (found) + ring->stats.cleaned += found; + else + ring->stats.misses++; +#endif + if (found) + break; + } + + ixgbevf_qv_unlock_poll(q_vector); + + return found; +} +#endif /* CONFIG_NET_RX_BUSY_POLL */ + /** * ixgbevf_configure_msix - Configure MSI-X hardware * @adapter: board private structure @@ -735,7 +878,8 @@ static irqreturn_t ixgbevf_msix_other(int irq, void *data) hw->mac.get_link_status = 1; - if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) + if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && + !test_bit(__IXGBEVF_REMOVING, &adapter->state)) mod_timer(&adapter->watchdog_timer, jiffies); IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); @@ -743,7 +887,6 @@ static irqreturn_t ixgbevf_msix_other(int irq, void *data) return IRQ_HANDLED; } - /** * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) * @irq: unused @@ -765,8 +908,8 @@ static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, { struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; - a->rx_ring[r_idx].next = q_vector->rx.ring; - q_vector->rx.ring = &a->rx_ring[r_idx]; + a->rx_ring[r_idx]->next = q_vector->rx.ring; + q_vector->rx.ring = a->rx_ring[r_idx]; q_vector->rx.count++; } @@ -775,8 +918,8 @@ static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, { struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; - a->tx_ring[t_idx].next = q_vector->tx.ring; - q_vector->tx.ring = &a->tx_ring[t_idx]; + a->tx_ring[t_idx]->next = q_vector->tx.ring; + q_vector->tx.ring = a->tx_ring[t_idx]; q_vector->tx.count++; } @@ -901,9 +1044,17 @@ free_queue_irqs: free_irq(adapter->msix_entries[vector].vector, adapter->q_vector[vector]); } - pci_disable_msix(adapter->pdev); - kfree(adapter->msix_entries); - adapter->msix_entries = NULL; + /* This failure is non-recoverable - it indicates the system is + * out of MSIX vector resources and the VF driver cannot run + * without them. Set the number of msix vectors to zero + * indicating that not enough can be allocated. The error + * will be returned to the user indicating device open failed. + * Any further attempts to force the driver to open will also + * fail. The only way to recover is to unload the driver and + * reload it again. If the system has recovered some MSIX + * vectors then it may succeed. + */ + adapter->num_msix_vectors = 0; return err; } @@ -996,6 +1147,70 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) } /** + * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset + * @adapter: board private structure + * @ring: structure containing ring specific data + * + * Configure the Tx descriptor ring after a reset. + **/ +static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) +{ + struct ixgbe_hw *hw = &adapter->hw; + u64 tdba = ring->dma; + int wait_loop = 10; + u32 txdctl = IXGBE_TXDCTL_ENABLE; + u8 reg_idx = ring->reg_idx; + + /* disable queue to avoid issues while updating state */ + IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); + IXGBE_WRITE_FLUSH(hw); + + IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); + IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); + IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), + ring->count * sizeof(union ixgbe_adv_tx_desc)); + + /* disable head writeback */ + IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); + IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); + + /* enable relaxed ordering */ + IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), + (IXGBE_DCA_TXCTRL_DESC_RRO_EN | + IXGBE_DCA_TXCTRL_DATA_RRO_EN)); + + /* reset head and tail pointers */ + IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); + IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); + ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); + + /* reset ntu and ntc to place SW in sync with hardwdare */ + ring->next_to_clean = 0; + ring->next_to_use = 0; + + /* In order to avoid issues WTHRESH + PTHRESH should always be equal + * to or less than the number of on chip descriptors, which is + * currently 40. + */ + txdctl |= (8 << 16); /* WTHRESH = 8 */ + + /* Setting PTHRESH to 32 both improves performance */ + txdctl |= (1 << 8) | /* HTHRESH = 1 */ + 32; /* PTHRESH = 32 */ + + IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); + + /* poll to verify queue is enabled */ + do { + usleep_range(1000, 2000); + txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); + } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); + if (!wait_loop) + pr_err("Could not enable Tx Queue %d\n", reg_idx); +} + +/** * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset * @adapter: board private structure * @@ -1003,31 +1218,11 @@ static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) **/ static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) { - u64 tdba; - struct ixgbe_hw *hw = &adapter->hw; - u32 i, j, tdlen, txctrl; + u32 i; /* Setup the HW Tx Head and Tail descriptor pointers */ - for (i = 0; i < adapter->num_tx_queues; i++) { - struct ixgbevf_ring *ring = &adapter->tx_ring[i]; - j = ring->reg_idx; - tdba = ring->dma; - tdlen = ring->count * sizeof(union ixgbe_adv_tx_desc); - IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(j), - (tdba & DMA_BIT_MASK(32))); - IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(j), (tdba >> 32)); - IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(j), tdlen); - IXGBE_WRITE_REG(hw, IXGBE_VFTDH(j), 0); - IXGBE_WRITE_REG(hw, IXGBE_VFTDT(j), 0); - adapter->tx_ring[i].head = IXGBE_VFTDH(j); - adapter->tx_ring[i].tail = IXGBE_VFTDT(j); - /* Disable Tx Head Writeback RO bit, since this hoses - * bookkeeping if things aren't delivered in order. - */ - txctrl = IXGBE_READ_REG(hw, IXGBE_VFDCA_TXCTRL(j)); - txctrl &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; - IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(j), txctrl); - } + for (i = 0; i < adapter->num_tx_queues; i++) + ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); } #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 @@ -1038,7 +1233,7 @@ static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) struct ixgbe_hw *hw = &adapter->hw; u32 srrctl; - rx_ring = &adapter->rx_ring[index]; + rx_ring = adapter->rx_ring[index]; srrctl = IXGBE_SRRCTL_DROP_EN; @@ -1050,6 +1245,21 @@ static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); } +static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) +{ + struct ixgbe_hw *hw = &adapter->hw; + + /* PSRTYPE must be initialized in 82599 */ + u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | + IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | + IXGBE_PSRTYPE_L2HDR; + + if (adapter->num_rx_queues > 1) + psrtype |= 1 << 29; + + IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); +} + static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; @@ -1065,23 +1275,113 @@ static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter) max_frame += VLAN_HLEN; /* - * Make best use of allocation by using all but 1K of a - * power of 2 allocation that will be used for skb->head. + * Allocate buffer sizes that fit well into 32K and + * take into account max frame size of 9.5K */ if ((hw->mac.type == ixgbe_mac_X540_vf) && (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)) rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE; - else if (max_frame <= IXGBEVF_RXBUFFER_3K) - rx_buf_len = IXGBEVF_RXBUFFER_3K; - else if (max_frame <= IXGBEVF_RXBUFFER_7K) - rx_buf_len = IXGBEVF_RXBUFFER_7K; - else if (max_frame <= IXGBEVF_RXBUFFER_15K) - rx_buf_len = IXGBEVF_RXBUFFER_15K; + else if (max_frame <= IXGBEVF_RXBUFFER_2K) + rx_buf_len = IXGBEVF_RXBUFFER_2K; + else if (max_frame <= IXGBEVF_RXBUFFER_4K) + rx_buf_len = IXGBEVF_RXBUFFER_4K; + else if (max_frame <= IXGBEVF_RXBUFFER_8K) + rx_buf_len = IXGBEVF_RXBUFFER_8K; else - rx_buf_len = IXGBEVF_MAX_RXBUFFER; + rx_buf_len = IXGBEVF_RXBUFFER_10K; for (i = 0; i < adapter->num_rx_queues; i++) - adapter->rx_ring[i].rx_buf_len = rx_buf_len; + adapter->rx_ring[i]->rx_buf_len = rx_buf_len; +} + +#define IXGBEVF_MAX_RX_DESC_POLL 10 +static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) +{ + struct ixgbe_hw *hw = &adapter->hw; + int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; + u32 rxdctl; + u8 reg_idx = ring->reg_idx; + + if (IXGBE_REMOVED(hw->hw_addr)) + return; + rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); + rxdctl &= ~IXGBE_RXDCTL_ENABLE; + + /* write value back with RXDCTL.ENABLE bit cleared */ + IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); + + /* the hardware may take up to 100us to really disable the rx queue */ + do { + udelay(10); + rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); + } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); + + if (!wait_loop) + pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", + reg_idx); +} + +static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) +{ + struct ixgbe_hw *hw = &adapter->hw; + int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; + u32 rxdctl; + u8 reg_idx = ring->reg_idx; + + if (IXGBE_REMOVED(hw->hw_addr)) + return; + do { + usleep_range(1000, 2000); + rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); + } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); + + if (!wait_loop) + pr_err("RXDCTL.ENABLE queue %d not set while polling\n", + reg_idx); +} + +static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, + struct ixgbevf_ring *ring) +{ + struct ixgbe_hw *hw = &adapter->hw; + u64 rdba = ring->dma; + u32 rxdctl; + u8 reg_idx = ring->reg_idx; + + /* disable queue to avoid issues while updating state */ + rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); + ixgbevf_disable_rx_queue(adapter, ring); + + IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); + IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); + IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), + ring->count * sizeof(union ixgbe_adv_rx_desc)); + + /* enable relaxed ordering */ + IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), + IXGBE_DCA_RXCTRL_DESC_RRO_EN); + + /* reset head and tail pointers */ + IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); + IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); + ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); + + /* reset ntu and ntc to place SW in sync with hardwdare */ + ring->next_to_clean = 0; + ring->next_to_use = 0; + + ixgbevf_configure_srrctl(adapter, reg_idx); + + /* prevent DMA from exceeding buffer space available */ + rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; + rxdctl |= ring->rx_buf_len | IXGBE_RXDCTL_RLPML_EN; + rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; + IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); + + ixgbevf_rx_desc_queue_enable(adapter, ring); + ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); } /** @@ -1092,51 +1392,32 @@ static void ixgbevf_set_rx_buffer_len(struct ixgbevf_adapter *adapter) **/ static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) { - u64 rdba; - struct ixgbe_hw *hw = &adapter->hw; - int i, j; - u32 rdlen; + int i; - /* PSRTYPE must be initialized in 82599 */ - IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, 0); + ixgbevf_setup_psrtype(adapter); /* set_rx_buffer_len must be called before ring initialization */ ixgbevf_set_rx_buffer_len(adapter); - rdlen = adapter->rx_ring[0].count * sizeof(union ixgbe_adv_rx_desc); /* Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ - for (i = 0; i < adapter->num_rx_queues; i++) { - rdba = adapter->rx_ring[i].dma; - j = adapter->rx_ring[i].reg_idx; - IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(j), - (rdba & DMA_BIT_MASK(32))); - IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(j), (rdba >> 32)); - IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(j), rdlen); - IXGBE_WRITE_REG(hw, IXGBE_VFRDH(j), 0); - IXGBE_WRITE_REG(hw, IXGBE_VFRDT(j), 0); - adapter->rx_ring[i].head = IXGBE_VFRDH(j); - adapter->rx_ring[i].tail = IXGBE_VFRDT(j); - - ixgbevf_configure_srrctl(adapter, j); - } + for (i = 0; i < adapter->num_rx_queues; i++) + ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]); } -static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) +static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, + __be16 proto, u16 vid) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; int err; - if (!hw->mac.ops.set_vfta) - return -EOPNOTSUPP; - - spin_lock(&adapter->mbx_lock); + spin_lock_bh(&adapter->mbx_lock); /* add VID to filter table */ err = hw->mac.ops.set_vfta(hw, vid, 0, true); - spin_unlock(&adapter->mbx_lock); + spin_unlock_bh(&adapter->mbx_lock); /* translate error return types so error makes sense */ if (err == IXGBE_ERR_MBX) @@ -1150,19 +1431,19 @@ static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, u16 vid) return err; } -static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) +static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, + __be16 proto, u16 vid) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; int err = -EOPNOTSUPP; - spin_lock(&adapter->mbx_lock); + spin_lock_bh(&adapter->mbx_lock); /* remove VID from filter table */ - if (hw->mac.ops.set_vfta) - err = hw->mac.ops.set_vfta(hw, vid, 0, false); + err = hw->mac.ops.set_vfta(hw, vid, 0, false); - spin_unlock(&adapter->mbx_lock); + spin_unlock_bh(&adapter->mbx_lock); clear_bit(vid, adapter->active_vlans); @@ -1174,7 +1455,8 @@ static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) u16 vid; for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) - ixgbevf_vlan_rx_add_vid(adapter->netdev, vid); + ixgbevf_vlan_rx_add_vid(adapter->netdev, + htons(ETH_P_8021Q), vid); } static int ixgbevf_write_uc_addr_list(struct net_device *netdev) @@ -1206,27 +1488,27 @@ static int ixgbevf_write_uc_addr_list(struct net_device *netdev) } /** - * ixgbevf_set_rx_mode - Multicast set + * ixgbevf_set_rx_mode - Multicast and unicast set * @netdev: network interface device structure * * The set_rx_method entry point is called whenever the multicast address - * list or the network interface flags are updated. This routine is - * responsible for configuring the hardware for proper multicast mode. + * list, unicast address list or the network interface flags are updated. + * This routine is responsible for configuring the hardware for proper + * multicast mode and configuring requested unicast filters. **/ static void ixgbevf_set_rx_mode(struct net_device *netdev) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; - spin_lock(&adapter->mbx_lock); + spin_lock_bh(&adapter->mbx_lock); /* reprogram multicast list */ - if (hw->mac.ops.update_mc_addr_list) - hw->mac.ops.update_mc_addr_list(hw, netdev); + hw->mac.ops.update_mc_addr_list(hw, netdev); ixgbevf_write_uc_addr_list(netdev); - spin_unlock(&adapter->mbx_lock); + spin_unlock_bh(&adapter->mbx_lock); } static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) @@ -1237,6 +1519,9 @@ static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) for (q_idx = 0; q_idx < q_vectors; q_idx++) { q_vector = adapter->q_vector[q_idx]; +#ifdef CONFIG_NET_RX_BUSY_POLL + ixgbevf_qv_init_lock(adapter->q_vector[q_idx]); +#endif napi_enable(&q_vector->napi); } } @@ -1250,48 +1535,63 @@ static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) for (q_idx = 0; q_idx < q_vectors; q_idx++) { q_vector = adapter->q_vector[q_idx]; napi_disable(&q_vector->napi); +#ifdef CONFIG_NET_RX_BUSY_POLL + while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) { + pr_info("QV %d locked\n", q_idx); + usleep_range(1000, 20000); + } +#endif /* CONFIG_NET_RX_BUSY_POLL */ } } -static void ixgbevf_configure(struct ixgbevf_adapter *adapter) +static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) { - struct net_device *netdev = adapter->netdev; - int i; + struct ixgbe_hw *hw = &adapter->hw; + unsigned int def_q = 0; + unsigned int num_tcs = 0; + unsigned int num_rx_queues = 1; + int err; - ixgbevf_set_rx_mode(netdev); + spin_lock_bh(&adapter->mbx_lock); - ixgbevf_restore_vlan(adapter); + /* fetch queue configuration from the PF */ + err = ixgbevf_get_queues(hw, &num_tcs, &def_q); - ixgbevf_configure_tx(adapter); - ixgbevf_configure_rx(adapter); - for (i = 0; i < adapter->num_rx_queues; i++) { - struct ixgbevf_ring *ring = &adapter->rx_ring[i]; - ixgbevf_alloc_rx_buffers(adapter, ring, - IXGBE_DESC_UNUSED(ring)); + spin_unlock_bh(&adapter->mbx_lock); + + if (err) + return err; + + if (num_tcs > 1) { + /* update default Tx ring register index */ + adapter->tx_ring[0]->reg_idx = def_q; + + /* we need as many queues as traffic classes */ + num_rx_queues = num_tcs; } + + /* if we have a bad config abort request queue reset */ + if (adapter->num_rx_queues != num_rx_queues) { + /* force mailbox timeout to prevent further messages */ + hw->mbx.timeout = 0; + + /* wait for watchdog to come around and bail us out */ + adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; + } + + return 0; } -#define IXGBE_MAX_RX_DESC_POLL 10 -static inline void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, - int rxr) +static void ixgbevf_configure(struct ixgbevf_adapter *adapter) { - struct ixgbe_hw *hw = &adapter->hw; - int j = adapter->rx_ring[rxr].reg_idx; - int k; + ixgbevf_configure_dcb(adapter); - for (k = 0; k < IXGBE_MAX_RX_DESC_POLL; k++) { - if (IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)) & IXGBE_RXDCTL_ENABLE) - break; - else - msleep(1); - } - if (k >= IXGBE_MAX_RX_DESC_POLL) { - hw_dbg(hw, "RXDCTL.ENABLE on Rx queue %d " - "not set within the polling period\n", rxr); - } + ixgbevf_set_rx_mode(adapter->netdev); - ixgbevf_release_rx_desc(&adapter->hw, &adapter->rx_ring[rxr], - (adapter->rx_ring[rxr].count - 1)); + ixgbevf_restore_vlan(adapter); + + ixgbevf_configure_tx(adapter); + ixgbevf_configure_rx(adapter); } static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) @@ -1335,11 +1635,12 @@ static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; - int api[] = { ixgbe_mbox_api_10, + int api[] = { ixgbe_mbox_api_11, + ixgbe_mbox_api_10, ixgbe_mbox_api_unknown }; int err = 0, idx = 0; - spin_lock(&adapter->mbx_lock); + spin_lock_bh(&adapter->mbx_lock); while (api[idx] != ixgbe_mbox_api_unknown) { err = ixgbevf_negotiate_api_version(hw, api[idx]); @@ -1348,58 +1649,26 @@ static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) idx++; } - spin_unlock(&adapter->mbx_lock); + spin_unlock_bh(&adapter->mbx_lock); } static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) { struct net_device *netdev = adapter->netdev; struct ixgbe_hw *hw = &adapter->hw; - int i, j = 0; - int num_rx_rings = adapter->num_rx_queues; - u32 txdctl, rxdctl; - - for (i = 0; i < adapter->num_tx_queues; i++) { - j = adapter->tx_ring[i].reg_idx; - txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); - /* enable WTHRESH=8 descriptors, to encourage burst writeback */ - txdctl |= (8 << 16); - IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); - } - - for (i = 0; i < adapter->num_tx_queues; i++) { - j = adapter->tx_ring[i].reg_idx; - txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); - txdctl |= IXGBE_TXDCTL_ENABLE; - IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), txdctl); - } - - for (i = 0; i < num_rx_rings; i++) { - j = adapter->rx_ring[i].reg_idx; - rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(j)); - rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; - if (hw->mac.type == ixgbe_mac_X540_vf) { - rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK; - rxdctl |= ((netdev->mtu + ETH_HLEN + ETH_FCS_LEN) | - IXGBE_RXDCTL_RLPML_EN); - } - IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(j), rxdctl); - ixgbevf_rx_desc_queue_enable(adapter, i); - } ixgbevf_configure_msix(adapter); - spin_lock(&adapter->mbx_lock); + spin_lock_bh(&adapter->mbx_lock); - if (hw->mac.ops.set_rar) { - if (is_valid_ether_addr(hw->mac.addr)) - hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); - else - hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); - } + if (is_valid_ether_addr(hw->mac.addr)) + hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); + else + hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); - spin_unlock(&adapter->mbx_lock); + spin_unlock_bh(&adapter->mbx_lock); + smp_mb__before_atomic(); clear_bit(__IXGBEVF_DOWN, &adapter->state); ixgbevf_napi_enable_all(adapter); @@ -1417,8 +1686,6 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; - ixgbevf_negotiate_api(adapter); - ixgbevf_configure(adapter); ixgbevf_up_complete(adapter); @@ -1431,13 +1698,10 @@ void ixgbevf_up(struct ixgbevf_adapter *adapter) /** * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue - * @adapter: board private structure * @rx_ring: ring to free buffers from **/ -static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring) +static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) { - struct pci_dev *pdev = adapter->pdev; unsigned long size; unsigned int i; @@ -1450,7 +1714,7 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter, rx_buffer_info = &rx_ring->rx_buffer_info[i]; if (rx_buffer_info->dma) { - dma_unmap_single(&pdev->dev, rx_buffer_info->dma, + dma_unmap_single(rx_ring->dev, rx_buffer_info->dma, rx_ring->rx_buf_len, DMA_FROM_DEVICE); rx_buffer_info->dma = 0; @@ -1471,23 +1735,13 @@ static void ixgbevf_clean_rx_ring(struct ixgbevf_adapter *adapter, /* Zero out the descriptor ring */ memset(rx_ring->desc, 0, rx_ring->size); - - rx_ring->next_to_clean = 0; - rx_ring->next_to_use = 0; - - if (rx_ring->head) - writel(0, adapter->hw.hw_addr + rx_ring->head); - if (rx_ring->tail) - writel(0, adapter->hw.hw_addr + rx_ring->tail); } /** * ixgbevf_clean_tx_ring - Free Tx Buffers - * @adapter: board private structure * @tx_ring: ring to be cleaned **/ -static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *tx_ring) +static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) { struct ixgbevf_tx_buffer *tx_buffer_info; unsigned long size; @@ -1497,7 +1751,6 @@ static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter, return; /* Free all the Tx ring sk_buffs */ - for (i = 0; i < tx_ring->count; i++) { tx_buffer_info = &tx_ring->tx_buffer_info[i]; ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); @@ -1507,14 +1760,6 @@ static void ixgbevf_clean_tx_ring(struct ixgbevf_adapter *adapter, memset(tx_ring->tx_buffer_info, 0, size); memset(tx_ring->desc, 0, tx_ring->size); - - tx_ring->next_to_use = 0; - tx_ring->next_to_clean = 0; - - if (tx_ring->head) - writel(0, adapter->hw.hw_addr + tx_ring->head); - if (tx_ring->tail) - writel(0, adapter->hw.hw_addr + tx_ring->tail); } /** @@ -1526,7 +1771,7 @@ static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) int i; for (i = 0; i < adapter->num_rx_queues; i++) - ixgbevf_clean_rx_ring(adapter, &adapter->rx_ring[i]); + ixgbevf_clean_rx_ring(adapter->rx_ring[i]); } /** @@ -1538,19 +1783,22 @@ static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) int i; for (i = 0; i < adapter->num_tx_queues; i++) - ixgbevf_clean_tx_ring(adapter, &adapter->tx_ring[i]); + ixgbevf_clean_tx_ring(adapter->tx_ring[i]); } void ixgbevf_down(struct ixgbevf_adapter *adapter) { struct net_device *netdev = adapter->netdev; struct ixgbe_hw *hw = &adapter->hw; - u32 txdctl; - int i, j; + int i; /* signal that we are down to the interrupt handler */ - set_bit(__IXGBEVF_DOWN, &adapter->state); - /* disable receives */ + if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) + return; /* do nothing if already down */ + + /* disable all enabled rx queues */ + for (i = 0; i < adapter->num_rx_queues; i++) + ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); netif_tx_disable(netdev); @@ -1571,10 +1819,10 @@ void ixgbevf_down(struct ixgbevf_adapter *adapter) /* disable transmits in the hardware now that interrupts are off */ for (i = 0; i < adapter->num_tx_queues; i++) { - j = adapter->tx_ring[i].reg_idx; - txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(j)); - IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(j), - (txdctl & ~IXGBE_TXDCTL_ENABLE)); + u8 reg_idx = adapter->tx_ring[i]->reg_idx; + + IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), + IXGBE_TXDCTL_SWFLSH); } netif_carrier_off(netdev); @@ -1593,13 +1841,6 @@ void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) msleep(1); - /* - * Check if PF is up before re-init. If not then skip until - * later when the PF is up and ready to service requests from - * the VF via mailbox. If the VF is up and running then the - * watchdog task will continue to schedule reset tasks until - * the PF is up and running. - */ ixgbevf_down(adapter); ixgbevf_up(adapter); @@ -1611,14 +1852,12 @@ void ixgbevf_reset(struct ixgbevf_adapter *adapter) struct ixgbe_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; - spin_lock(&adapter->mbx_lock); - - if (hw->mac.ops.reset_hw(hw)) + if (hw->mac.ops.reset_hw(hw)) { hw_dbg(hw, "PF still resetting\n"); - else + } else { hw->mac.ops.init_hw(hw); - - spin_unlock(&adapter->mbx_lock); + ixgbevf_negotiate_api(adapter); + } if (is_valid_ether_addr(adapter->hw.mac.addr)) { memcpy(netdev->dev_addr, adapter->hw.mac.addr, @@ -1628,10 +1867,10 @@ void ixgbevf_reset(struct ixgbevf_adapter *adapter) } } -static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, - int vectors) +static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, + int vectors) { - int err, vector_threshold; + int vector_threshold; /* We'll want at least 2 (vector_threshold): * 1) TxQ[0] + RxQ[0] handler @@ -1644,34 +1883,24 @@ static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, * Right now, we simply care about how many we'll get; we'll * set them up later while requesting irq's. */ - while (vectors >= vector_threshold) { - err = pci_enable_msix(adapter->pdev, adapter->msix_entries, - vectors); - if (!err) /* Success in acquiring all requested vectors. */ - break; - else if (err < 0) - vectors = 0; /* Nasty failure, quit now */ - else /* err == number of vectors we should try again with */ - vectors = err; - } + vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, + vector_threshold, vectors); - if (vectors < vector_threshold) { - /* Can't allocate enough MSI-X interrupts? Oh well. - * This just means we'll go with either a single MSI - * vector or fall back to legacy interrupts. - */ - hw_dbg(&adapter->hw, - "Unable to allocate MSI-X interrupts\n"); + if (vectors < 0) { + dev_err(&adapter->pdev->dev, + "Unable to allocate MSI-X interrupts\n"); kfree(adapter->msix_entries); adapter->msix_entries = NULL; - } else { - /* - * Adjust for only the vectors we'll use, which is minimum - * of max_msix_q_vectors + NON_Q_VECTORS, or the number of - * vectors we were allocated. - */ - adapter->num_msix_vectors = vectors; + return vectors; } + + /* Adjust for only the vectors we'll use, which is minimum + * of max_msix_q_vectors + NON_Q_VECTORS, or the number of + * vectors we were allocated. + */ + adapter->num_msix_vectors = vectors; + + return 0; } /** @@ -1687,9 +1916,28 @@ static void ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, **/ static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) { + struct ixgbe_hw *hw = &adapter->hw; + unsigned int def_q = 0; + unsigned int num_tcs = 0; + int err; + /* Start with base case */ adapter->num_rx_queues = 1; adapter->num_tx_queues = 1; + + spin_lock_bh(&adapter->mbx_lock); + + /* fetch queue configuration from the PF */ + err = ixgbevf_get_queues(hw, &num_tcs, &def_q); + + spin_unlock_bh(&adapter->mbx_lock); + + if (err) + return; + + /* we need as many queues as traffic classes */ + if (num_tcs > 1) + adapter->num_rx_queues = num_tcs; } /** @@ -1702,39 +1950,50 @@ static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) **/ static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) { - int i; + struct ixgbevf_ring *ring; + int rx = 0, tx = 0; - adapter->tx_ring = kcalloc(adapter->num_tx_queues, - sizeof(struct ixgbevf_ring), GFP_KERNEL); - if (!adapter->tx_ring) - goto err_tx_ring_allocation; + for (; tx < adapter->num_tx_queues; tx++) { + ring = kzalloc(sizeof(*ring), GFP_KERNEL); + if (!ring) + goto err_allocation; - adapter->rx_ring = kcalloc(adapter->num_rx_queues, - sizeof(struct ixgbevf_ring), GFP_KERNEL); - if (!adapter->rx_ring) - goto err_rx_ring_allocation; + ring->dev = &adapter->pdev->dev; + ring->netdev = adapter->netdev; + ring->count = adapter->tx_ring_count; + ring->queue_index = tx; + ring->reg_idx = tx; - for (i = 0; i < adapter->num_tx_queues; i++) { - adapter->tx_ring[i].count = adapter->tx_ring_count; - adapter->tx_ring[i].queue_index = i; - adapter->tx_ring[i].reg_idx = i; - adapter->tx_ring[i].dev = &adapter->pdev->dev; - adapter->tx_ring[i].netdev = adapter->netdev; + adapter->tx_ring[tx] = ring; } - for (i = 0; i < adapter->num_rx_queues; i++) { - adapter->rx_ring[i].count = adapter->rx_ring_count; - adapter->rx_ring[i].queue_index = i; - adapter->rx_ring[i].reg_idx = i; - adapter->rx_ring[i].dev = &adapter->pdev->dev; - adapter->rx_ring[i].netdev = adapter->netdev; + for (; rx < adapter->num_rx_queues; rx++) { + ring = kzalloc(sizeof(*ring), GFP_KERNEL); + if (!ring) + goto err_allocation; + + ring->dev = &adapter->pdev->dev; + ring->netdev = adapter->netdev; + + ring->count = adapter->rx_ring_count; + ring->queue_index = rx; + ring->reg_idx = rx; + + adapter->rx_ring[rx] = ring; } return 0; -err_rx_ring_allocation: - kfree(adapter->tx_ring); -err_tx_ring_allocation: +err_allocation: + while (tx) { + kfree(adapter->tx_ring[--tx]); + adapter->tx_ring[tx] = NULL; + } + + while (rx) { + kfree(adapter->rx_ring[--rx]); + adapter->rx_ring[rx] = NULL; + } return -ENOMEM; } @@ -1774,7 +2033,9 @@ static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) for (vector = 0; vector < v_budget; vector++) adapter->msix_entries[vector].entry = vector; - ixgbevf_acquire_msix_vectors(adapter, v_budget); + err = ixgbevf_acquire_msix_vectors(adapter, v_budget); + if (err) + goto out; err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); if (err) @@ -1808,6 +2069,9 @@ static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) q_vector->v_idx = q_idx; netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll, 64); +#ifdef CONFIG_NET_RX_BUSY_POLL + napi_hash_add(&q_vector->napi); +#endif adapter->q_vector[q_idx] = q_vector; } @@ -1817,6 +2081,9 @@ err_out: while (q_idx) { q_idx--; q_vector = adapter->q_vector[q_idx]; +#ifdef CONFIG_NET_RX_BUSY_POLL + napi_hash_del(&q_vector->napi); +#endif netif_napi_del(&q_vector->napi); kfree(q_vector); adapter->q_vector[q_idx] = NULL; @@ -1834,18 +2101,16 @@ err_out: **/ static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) { - int q_idx, num_q_vectors; - int napi_vectors; - - num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; - napi_vectors = adapter->num_rx_queues; + int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; adapter->q_vector[q_idx] = NULL; - if (q_idx < napi_vectors) - netif_napi_del(&q_vector->napi); +#ifdef CONFIG_NET_RX_BUSY_POLL + napi_hash_del(&q_vector->napi); +#endif + netif_napi_del(&q_vector->napi); kfree(q_vector); } } @@ -1919,6 +2184,17 @@ err_set_interrupt: **/ static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) { + int i; + + for (i = 0; i < adapter->num_tx_queues; i++) { + kfree(adapter->tx_ring[i]); + adapter->tx_ring[i] = NULL; + } + for (i = 0; i < adapter->num_rx_queues; i++) { + kfree(adapter->rx_ring[i]); + adapter->rx_ring[i] = NULL; + } + adapter->num_tx_queues = 0; adapter->num_rx_queues = 0; @@ -1935,10 +2211,11 @@ static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) * Fields are initialized based on PCI device information and * OS network device settings (MTU size). **/ -static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter) +static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct pci_dev *pdev = adapter->pdev; + struct net_device *netdev = adapter->netdev; int err; /* PCI config space info */ @@ -1950,27 +2227,39 @@ static int __devinit ixgbevf_sw_init(struct ixgbevf_adapter *adapter) hw->subsystem_device_id = pdev->subsystem_device; hw->mbx.ops.init_params(hw); - hw->mac.max_tx_queues = MAX_TX_QUEUES; - hw->mac.max_rx_queues = MAX_RX_QUEUES; + + /* assume legacy case in which PF would only give VF 2 queues */ + hw->mac.max_tx_queues = 2; + hw->mac.max_rx_queues = 2; + + /* lock to protect mailbox accesses */ + spin_lock_init(&adapter->mbx_lock); + err = hw->mac.ops.reset_hw(hw); if (err) { dev_info(&pdev->dev, - "PF still in reset state, assigning new address\n"); - eth_hw_addr_random(adapter->netdev); - memcpy(adapter->hw.mac.addr, adapter->netdev->dev_addr, - adapter->netdev->addr_len); + "PF still in reset state. Is the PF interface up?\n"); } else { err = hw->mac.ops.init_hw(hw); if (err) { pr_err("init_shared_code failed: %d\n", err); goto out; } - memcpy(adapter->netdev->dev_addr, adapter->hw.mac.addr, - adapter->netdev->addr_len); + ixgbevf_negotiate_api(adapter); + err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); + if (err) + dev_info(&pdev->dev, "Error reading MAC address\n"); + else if (is_zero_ether_addr(adapter->hw.mac.addr)) + dev_info(&pdev->dev, + "MAC address not assigned by administrator.\n"); + memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len); } - /* lock to protect mailbox accesses */ - spin_lock_init(&adapter->mbx_lock); + if (!is_valid_ether_addr(netdev->dev_addr)) { + dev_info(&pdev->dev, "Assigning random MAC address\n"); + eth_hw_addr_random(netdev); + memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len); + } /* Enable dynamic interrupt throttling rates */ adapter->rx_itr_setting = 1; @@ -2016,6 +2305,10 @@ out: void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; + int i; + + if (!adapter->link_up) + return; UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, adapter->stats.vfgprc); @@ -2029,6 +2322,12 @@ void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) adapter->stats.vfgotc); UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, adapter->stats.vfmprc); + + for (i = 0; i < adapter->num_rx_queues; i++) { + adapter->hw_csum_rx_error += + adapter->rx_ring[i]->hw_csum_rx_error; + adapter->rx_ring[i]->hw_csum_rx_error = 0; + } } /** @@ -2082,6 +2381,7 @@ static void ixgbevf_reset_task(struct work_struct *work) /* If we're already down or resetting, just bail */ if (test_bit(__IXGBEVF_DOWN, &adapter->state) || + test_bit(__IXGBEVF_REMOVING, &adapter->state) || test_bit(__IXGBEVF_RESETTING, &adapter->state)) return; @@ -2103,6 +2403,17 @@ static void ixgbevf_watchdog_task(struct work_struct *work) struct ixgbe_hw *hw = &adapter->hw; u32 link_speed = adapter->link_speed; bool link_up = adapter->link_up; + s32 need_reset; + + if (IXGBE_REMOVED(hw->hw_addr)) { + if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { + rtnl_lock(); + ixgbevf_down(adapter); + rtnl_unlock(); + } + return; + } + ixgbevf_queue_reset_subtask(adapter); adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK; @@ -2110,38 +2421,42 @@ static void ixgbevf_watchdog_task(struct work_struct *work) * Always check the link on the watchdog because we have * no LSC interrupt */ - if (hw->mac.ops.check_link) { - s32 need_reset; - - spin_lock(&adapter->mbx_lock); + spin_lock_bh(&adapter->mbx_lock); - need_reset = hw->mac.ops.check_link(hw, &link_speed, - &link_up, false); + need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); - spin_unlock(&adapter->mbx_lock); + spin_unlock_bh(&adapter->mbx_lock); - if (need_reset) { - adapter->link_up = link_up; - adapter->link_speed = link_speed; - netif_carrier_off(netdev); - netif_tx_stop_all_queues(netdev); - schedule_work(&adapter->reset_task); - goto pf_has_reset; - } - } else { - /* always assume link is up, if no check link - * function */ - link_speed = IXGBE_LINK_SPEED_10GB_FULL; - link_up = true; + if (need_reset) { + adapter->link_up = link_up; + adapter->link_speed = link_speed; + netif_carrier_off(netdev); + netif_tx_stop_all_queues(netdev); + schedule_work(&adapter->reset_task); + goto pf_has_reset; } adapter->link_up = link_up; adapter->link_speed = link_speed; if (link_up) { if (!netif_carrier_ok(netdev)) { - hw_dbg(&adapter->hw, "NIC Link is Up, %u Gbps\n", - (link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? - 10 : 1); + char *link_speed_string; + switch (link_speed) { + case IXGBE_LINK_SPEED_10GB_FULL: + link_speed_string = "10 Gbps"; + break; + case IXGBE_LINK_SPEED_1GB_FULL: + link_speed_string = "1 Gbps"; + break; + case IXGBE_LINK_SPEED_100_FULL: + link_speed_string = "100 Mbps"; + break; + default: + link_speed_string = "unknown speed"; + break; + } + dev_info(&adapter->pdev->dev, + "NIC Link is Up, %s\n", link_speed_string); netif_carrier_on(netdev); netif_tx_wake_all_queues(netdev); } @@ -2149,7 +2464,7 @@ static void ixgbevf_watchdog_task(struct work_struct *work) adapter->link_up = false; adapter->link_speed = 0; if (netif_carrier_ok(netdev)) { - hw_dbg(&adapter->hw, "NIC Link is Down\n"); + dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); netif_carrier_off(netdev); netif_tx_stop_all_queues(netdev); } @@ -2159,7 +2474,8 @@ static void ixgbevf_watchdog_task(struct work_struct *work) pf_has_reset: /* Reset the timer */ - if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) + if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && + !test_bit(__IXGBEVF_REMOVING, &adapter->state)) mod_timer(&adapter->watchdog_timer, round_jiffies(jiffies + (2 * HZ))); @@ -2168,22 +2484,22 @@ pf_has_reset: /** * ixgbevf_free_tx_resources - Free Tx Resources per Queue - * @adapter: board private structure * @tx_ring: Tx descriptor ring for a specific queue * * Free all transmit software resources **/ -void ixgbevf_free_tx_resources(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *tx_ring) +void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) { - struct pci_dev *pdev = adapter->pdev; - - ixgbevf_clean_tx_ring(adapter, tx_ring); + ixgbevf_clean_tx_ring(tx_ring); vfree(tx_ring->tx_buffer_info); tx_ring->tx_buffer_info = NULL; - dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc, + /* if not set, then don't free */ + if (!tx_ring->desc) + return; + + dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, tx_ring->dma); tx_ring->desc = NULL; @@ -2200,23 +2516,18 @@ static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) int i; for (i = 0; i < adapter->num_tx_queues; i++) - if (adapter->tx_ring[i].desc) - ixgbevf_free_tx_resources(adapter, - &adapter->tx_ring[i]); - + if (adapter->tx_ring[i]->desc) + ixgbevf_free_tx_resources(adapter->tx_ring[i]); } /** * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) - * @adapter: board private structure * @tx_ring: tx descriptor ring (for a specific queue) to setup * * Return 0 on success, negative on failure **/ -int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *tx_ring) +int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) { - struct pci_dev *pdev = adapter->pdev; int size; size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; @@ -2228,13 +2539,11 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_adapter *adapter, tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); tx_ring->size = ALIGN(tx_ring->size, 4096); - tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size, + tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, &tx_ring->dma, GFP_KERNEL); if (!tx_ring->desc) goto err; - tx_ring->next_to_use = 0; - tx_ring->next_to_clean = 0; return 0; err: @@ -2260,7 +2569,7 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) int i, err = 0; for (i = 0; i < adapter->num_tx_queues; i++) { - err = ixgbevf_setup_tx_resources(adapter, &adapter->tx_ring[i]); + err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); if (!err) continue; hw_dbg(&adapter->hw, @@ -2273,43 +2582,34 @@ static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) /** * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) - * @adapter: board private structure * @rx_ring: rx descriptor ring (for a specific queue) to setup * * Returns 0 on success, negative on failure **/ -int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring) +int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring) { - struct pci_dev *pdev = adapter->pdev; int size; size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; rx_ring->rx_buffer_info = vzalloc(size); if (!rx_ring->rx_buffer_info) - goto alloc_failed; + goto err; /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); - rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size, + rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, &rx_ring->dma, GFP_KERNEL); - if (!rx_ring->desc) { - hw_dbg(&adapter->hw, - "Unable to allocate memory for " - "the receive descriptor ring\n"); - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; - goto alloc_failed; - } - - rx_ring->next_to_clean = 0; - rx_ring->next_to_use = 0; + if (!rx_ring->desc) + goto err; return 0; -alloc_failed: +err: + vfree(rx_ring->rx_buffer_info); + rx_ring->rx_buffer_info = NULL; + dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); return -ENOMEM; } @@ -2328,7 +2628,7 @@ static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) int i, err = 0; for (i = 0; i < adapter->num_rx_queues; i++) { - err = ixgbevf_setup_rx_resources(adapter, &adapter->rx_ring[i]); + err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]); if (!err) continue; hw_dbg(&adapter->hw, @@ -2340,22 +2640,18 @@ static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) /** * ixgbevf_free_rx_resources - Free Rx Resources - * @adapter: board private structure * @rx_ring: ring to clean the resources from * * Free all receive software resources **/ -void ixgbevf_free_rx_resources(struct ixgbevf_adapter *adapter, - struct ixgbevf_ring *rx_ring) +void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) { - struct pci_dev *pdev = adapter->pdev; - - ixgbevf_clean_rx_ring(adapter, rx_ring); + ixgbevf_clean_rx_ring(rx_ring); vfree(rx_ring->rx_buffer_info); rx_ring->rx_buffer_info = NULL; - dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc, + dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; @@ -2372,9 +2668,8 @@ static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) int i; for (i = 0; i < adapter->num_rx_queues; i++) - if (adapter->rx_ring[i].desc) - ixgbevf_free_rx_resources(adapter, - &adapter->rx_ring[i]); + if (adapter->rx_ring[i]->desc) + ixgbevf_free_rx_resources(adapter->rx_ring[i]); } /** @@ -2395,6 +2690,15 @@ static int ixgbevf_open(struct net_device *netdev) struct ixgbe_hw *hw = &adapter->hw; int err; + /* A previous failure to open the device because of a lack of + * available MSIX vector resources may have reset the number + * of msix vectors variable to zero. The only way to recover + * is to unload/reload the driver and hope that the system has + * been able to recover some MSIX vector resources. + */ + if (!adapter->num_msix_vectors) + return -ENOMEM; + /* disallow open during test */ if (test_bit(__IXGBEVF_TESTING, &adapter->state)) return -EBUSY; @@ -2411,8 +2715,6 @@ static int ixgbevf_open(struct net_device *netdev) } } - ixgbevf_negotiate_api(adapter); - /* allocate transmit descriptors */ err = ixgbevf_setup_all_tx_resources(adapter); if (err) @@ -2446,7 +2748,6 @@ static int ixgbevf_open(struct net_device *netdev) err_req_irq: ixgbevf_down(adapter); - ixgbevf_free_irq(adapter); err_setup_rx: ixgbevf_free_all_rx_resources(adapter); err_setup_tx: @@ -2482,6 +2783,34 @@ static int ixgbevf_close(struct net_device *netdev) return 0; } +static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) +{ + struct net_device *dev = adapter->netdev; + + if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED)) + return; + + adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED; + + /* if interface is down do nothing */ + if (test_bit(__IXGBEVF_DOWN, &adapter->state) || + test_bit(__IXGBEVF_RESETTING, &adapter->state)) + return; + + /* Hardware has to reinitialize queues and interrupts to + * match packet buffer alignment. Unfortunately, the + * hardware is not flexible enough to do this dynamically. + */ + if (netif_running(dev)) + ixgbevf_close(dev); + + ixgbevf_clear_interrupt_scheme(adapter); + ixgbevf_init_interrupt_scheme(adapter); + + if (netif_running(dev)) + ixgbevf_open(dev); +} + static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, u32 vlan_macip_lens, u32 type_tucmd, u32 mss_l4len_idx) @@ -2504,19 +2833,23 @@ static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, } static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, - struct sk_buff *skb, u32 tx_flags, u8 *hdr_len) + struct ixgbevf_tx_buffer *first, + u8 *hdr_len) { + struct sk_buff *skb = first->skb; u32 vlan_macip_lens, type_tucmd; u32 mss_l4len_idx, l4len; + int err; + + if (skb->ip_summed != CHECKSUM_PARTIAL) + return 0; if (!skb_is_gso(skb)) return 0; - if (skb_header_cloned(skb)) { - int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); - if (err) - return err; - } + err = skb_cow_head(skb, 0); + if (err < 0) + return err; /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; @@ -2530,12 +2863,17 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, IPPROTO_TCP, 0); type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; + first->tx_flags |= IXGBE_TX_FLAGS_TSO | + IXGBE_TX_FLAGS_CSUM | + IXGBE_TX_FLAGS_IPV4; } else if (skb_is_gso_v6(skb)) { ipv6_hdr(skb)->payload_len = 0; tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); + first->tx_flags |= IXGBE_TX_FLAGS_TSO | + IXGBE_TX_FLAGS_CSUM; } /* compute header lengths */ @@ -2543,6 +2881,10 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, *hdr_len += l4len; *hdr_len = skb_transport_offset(skb) + l4len; + /* update gso size and bytecount with header size */ + first->gso_segs = skb_shinfo(skb)->gso_segs; + first->bytecount += (first->gso_segs - 1) * *hdr_len; + /* mss_l4len_id: use 1 as index for TSO */ mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT; mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; @@ -2551,7 +2893,7 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ vlan_macip_lens = skb_network_header_len(skb); vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; - vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; + vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx); @@ -2559,12 +2901,10 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, return 1; } -static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, - struct sk_buff *skb, u32 tx_flags) +static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, + struct ixgbevf_tx_buffer *first) { - - - + struct sk_buff *skb = first->skb; u32 vlan_macip_lens = 0; u32 mss_l4len_idx = 0; u32 type_tucmd = 0; @@ -2572,12 +2912,12 @@ static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, if (skb->ip_summed == CHECKSUM_PARTIAL) { u8 l4_hdr = 0; switch (skb->protocol) { - case __constant_htons(ETH_P_IP): + case htons(ETH_P_IP): vlan_macip_lens |= skb_network_header_len(skb); type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; l4_hdr = ip_hdr(skb)->protocol; break; - case __constant_htons(ETH_P_IPV6): + case htons(ETH_P_IPV6): vlan_macip_lens |= skb_network_header_len(skb); l4_hdr = ipv6_hdr(skb)->nexthdr; break; @@ -2585,7 +2925,7 @@ static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, if (unlikely(net_ratelimit())) { dev_warn(tx_ring->dev, "partial checksum but proto=%x!\n", - skb->protocol); + first->protocol); } break; } @@ -2613,180 +2953,190 @@ static bool ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, } break; } + + /* update TX checksum flag */ + first->tx_flags |= IXGBE_TX_FLAGS_CSUM; } /* vlan_macip_lens: MACLEN, VLAN tag */ vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; - vlan_macip_lens |= tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; + vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx); +} + +static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) +{ + /* set type for advanced descriptor with frame checksum insertion */ + __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_IFCS | + IXGBE_ADVTXD_DCMD_DEXT); + + /* set HW vlan bit if vlan is present */ + if (tx_flags & IXGBE_TX_FLAGS_VLAN) + cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); + + /* set segmentation enable bits for TSO/FSO */ + if (tx_flags & IXGBE_TX_FLAGS_TSO) + cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); - return (skb->ip_summed == CHECKSUM_PARTIAL); + return cmd_type; } -static int ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, - struct sk_buff *skb, u32 tx_flags, - unsigned int first) +static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, + u32 tx_flags, unsigned int paylen) { - struct ixgbevf_tx_buffer *tx_buffer_info; - unsigned int len; - unsigned int total = skb->len; - unsigned int offset = 0, size; - int count = 0; - unsigned int nr_frags = skb_shinfo(skb)->nr_frags; - unsigned int f; - int i; + __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); - i = tx_ring->next_to_use; + /* enable L4 checksum for TSO and TX checksum offload */ + if (tx_flags & IXGBE_TX_FLAGS_CSUM) + olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); - len = min(skb_headlen(skb), total); - while (len) { - tx_buffer_info = &tx_ring->tx_buffer_info[i]; - size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); - - tx_buffer_info->length = size; - tx_buffer_info->mapped_as_page = false; - tx_buffer_info->dma = dma_map_single(tx_ring->dev, - skb->data + offset, - size, DMA_TO_DEVICE); - if (dma_mapping_error(tx_ring->dev, tx_buffer_info->dma)) - goto dma_error; - tx_buffer_info->next_to_watch = i; + /* enble IPv4 checksum for TSO */ + if (tx_flags & IXGBE_TX_FLAGS_IPV4) + olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); - len -= size; - total -= size; - offset += size; - count++; - i++; - if (i == tx_ring->count) - i = 0; - } + /* use index 1 context for TSO/FSO/FCOE */ + if (tx_flags & IXGBE_TX_FLAGS_TSO) + olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT); - for (f = 0; f < nr_frags; f++) { - const struct skb_frag_struct *frag; + /* Check Context must be set if Tx switch is enabled, which it + * always is for case where virtual functions are running + */ + olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); - frag = &skb_shinfo(skb)->frags[f]; - len = min((unsigned int)skb_frag_size(frag), total); - offset = 0; + tx_desc->read.olinfo_status = olinfo_status; +} - while (len) { - tx_buffer_info = &tx_ring->tx_buffer_info[i]; - size = min(len, (unsigned int)IXGBE_MAX_DATA_PER_TXD); +static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, + struct ixgbevf_tx_buffer *first, + const u8 hdr_len) +{ + dma_addr_t dma; + struct sk_buff *skb = first->skb; + struct ixgbevf_tx_buffer *tx_buffer; + union ixgbe_adv_tx_desc *tx_desc; + struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; + unsigned int data_len = skb->data_len; + unsigned int size = skb_headlen(skb); + unsigned int paylen = skb->len - hdr_len; + u32 tx_flags = first->tx_flags; + __le32 cmd_type; + u16 i = tx_ring->next_to_use; - tx_buffer_info->length = size; - tx_buffer_info->dma = - skb_frag_dma_map(tx_ring->dev, frag, - offset, size, DMA_TO_DEVICE); - tx_buffer_info->mapped_as_page = true; - if (dma_mapping_error(tx_ring->dev, - tx_buffer_info->dma)) - goto dma_error; - tx_buffer_info->next_to_watch = i; + tx_desc = IXGBEVF_TX_DESC(tx_ring, i); - len -= size; - total -= size; - offset += size; - count++; - i++; - if (i == tx_ring->count) - i = 0; - } - if (total == 0) - break; - } + ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen); + cmd_type = ixgbevf_tx_cmd_type(tx_flags); - if (i == 0) - i = tx_ring->count - 1; - else - i = i - 1; - tx_ring->tx_buffer_info[i].skb = skb; - tx_ring->tx_buffer_info[first].next_to_watch = i; - tx_ring->tx_buffer_info[first].time_stamp = jiffies; + dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); + if (dma_mapping_error(tx_ring->dev, dma)) + goto dma_error; - return count; + /* record length, and DMA address */ + dma_unmap_len_set(first, len, size); + dma_unmap_addr_set(first, dma, dma); -dma_error: - dev_err(tx_ring->dev, "TX DMA map failed\n"); + tx_desc->read.buffer_addr = cpu_to_le64(dma); - /* clear timestamp and dma mappings for failed tx_buffer_info map */ - tx_buffer_info->dma = 0; - tx_buffer_info->next_to_watch = 0; - count--; + for (;;) { + while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { + tx_desc->read.cmd_type_len = + cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); - /* clear timestamp and dma mappings for remaining portion of packet */ - while (count >= 0) { - count--; - i--; - if (i < 0) - i += tx_ring->count; - tx_buffer_info = &tx_ring->tx_buffer_info[i]; - ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); - } + i++; + tx_desc++; + if (i == tx_ring->count) { + tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); + i = 0; + } - return count; -} + dma += IXGBE_MAX_DATA_PER_TXD; + size -= IXGBE_MAX_DATA_PER_TXD; -static void ixgbevf_tx_queue(struct ixgbevf_ring *tx_ring, int tx_flags, - int count, u32 paylen, u8 hdr_len) -{ - union ixgbe_adv_tx_desc *tx_desc = NULL; - struct ixgbevf_tx_buffer *tx_buffer_info; - u32 olinfo_status = 0, cmd_type_len = 0; - unsigned int i; + tx_desc->read.buffer_addr = cpu_to_le64(dma); + tx_desc->read.olinfo_status = 0; + } - u32 txd_cmd = IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS | IXGBE_TXD_CMD_IFCS; + if (likely(!data_len)) + break; - cmd_type_len |= IXGBE_ADVTXD_DTYP_DATA; + tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); - cmd_type_len |= IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT; + i++; + tx_desc++; + if (i == tx_ring->count) { + tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); + i = 0; + } - if (tx_flags & IXGBE_TX_FLAGS_VLAN) - cmd_type_len |= IXGBE_ADVTXD_DCMD_VLE; + size = skb_frag_size(frag); + data_len -= size; - if (tx_flags & IXGBE_TX_FLAGS_CSUM) - olinfo_status |= IXGBE_ADVTXD_POPTS_TXSM; + dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, + DMA_TO_DEVICE); + if (dma_mapping_error(tx_ring->dev, dma)) + goto dma_error; - if (tx_flags & IXGBE_TX_FLAGS_TSO) { - cmd_type_len |= IXGBE_ADVTXD_DCMD_TSE; + tx_buffer = &tx_ring->tx_buffer_info[i]; + dma_unmap_len_set(tx_buffer, len, size); + dma_unmap_addr_set(tx_buffer, dma, dma); - /* use index 1 context for tso */ - olinfo_status |= (1 << IXGBE_ADVTXD_IDX_SHIFT); - if (tx_flags & IXGBE_TX_FLAGS_IPV4) - olinfo_status |= IXGBE_ADVTXD_POPTS_IXSM; + tx_desc->read.buffer_addr = cpu_to_le64(dma); + tx_desc->read.olinfo_status = 0; + frag++; } - /* - * Check Context must be set if Tx switch is enabled, which it - * always is for case where virtual functions are running + /* write last descriptor with RS and EOP bits */ + cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); + tx_desc->read.cmd_type_len = cmd_type; + + /* set the timestamp */ + first->time_stamp = jiffies; + + /* Force memory writes to complete before letting h/w know there + * are new descriptors to fetch. (Only applicable for weak-ordered + * memory model archs, such as IA-64). + * + * We also need this memory barrier (wmb) to make certain all of the + * status bits have been updated before next_to_watch is written. */ - olinfo_status |= IXGBE_ADVTXD_CC; + wmb(); - olinfo_status |= ((paylen - hdr_len) << IXGBE_ADVTXD_PAYLEN_SHIFT); + /* set next_to_watch value indicating a packet is present */ + first->next_to_watch = tx_desc; - i = tx_ring->next_to_use; - while (count--) { - tx_buffer_info = &tx_ring->tx_buffer_info[i]; - tx_desc = IXGBEVF_TX_DESC(tx_ring, i); - tx_desc->read.buffer_addr = cpu_to_le64(tx_buffer_info->dma); - tx_desc->read.cmd_type_len = - cpu_to_le32(cmd_type_len | tx_buffer_info->length); - tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status); - i++; - if (i == tx_ring->count) - i = 0; - } + i++; + if (i == tx_ring->count) + i = 0; + + tx_ring->next_to_use = i; - tx_desc->read.cmd_type_len |= cpu_to_le32(txd_cmd); + /* notify HW of packet */ + ixgbevf_write_tail(tx_ring, i); + + return; +dma_error: + dev_err(tx_ring->dev, "TX DMA map failed\n"); + + /* clear dma mappings for failed tx_buffer_info map */ + for (;;) { + tx_buffer = &tx_ring->tx_buffer_info[i]; + ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer); + if (tx_buffer == first) + break; + if (i == 0) + i = tx_ring->count; + i--; + } tx_ring->next_to_use = i; } static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) { - struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); - netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); /* Herbert's original patch had: * smp_mb__after_netif_stop_queue(); @@ -2795,18 +3145,19 @@ static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) /* We need to check again in a case another CPU has just * made room available. */ - if (likely(IXGBE_DESC_UNUSED(tx_ring) < size)) + if (likely(ixgbevf_desc_unused(tx_ring) < size)) return -EBUSY; /* A reprieve! - use start_queue because it doesn't call schedule */ netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); - ++adapter->restart_queue; + ++tx_ring->tx_stats.restart_queue; + return 0; } static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) { - if (likely(IXGBE_DESC_UNUSED(tx_ring) >= size)) + if (likely(ixgbevf_desc_unused(tx_ring) >= size)) return 0; return __ixgbevf_maybe_stop_tx(tx_ring, size); } @@ -2814,17 +3165,23 @@ static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); + struct ixgbevf_tx_buffer *first; struct ixgbevf_ring *tx_ring; - unsigned int first; - unsigned int tx_flags = 0; - u8 hdr_len = 0; - int r_idx = 0, tso; + int tso; + u32 tx_flags = 0; u16 count = TXD_USE_COUNT(skb_headlen(skb)); #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD unsigned short f; #endif + u8 hdr_len = 0; + u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); + + if (!dst_mac || is_link_local_ether_addr(dst_mac)) { + dev_kfree_skb(skb); + return NETDEV_TX_OK; + } - tx_ring = &adapter->tx_ring[r_idx]; + tx_ring = adapter->tx_ring[skb->queue_mapping]; /* * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, @@ -2840,45 +3197,41 @@ static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) count += skb_shinfo(skb)->nr_frags; #endif if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { - adapter->tx_busy++; + tx_ring->tx_stats.tx_busy++; return NETDEV_TX_BUSY; } + /* record the location of the first descriptor for this packet */ + first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; + first->skb = skb; + first->bytecount = skb->len; + first->gso_segs = 1; + if (vlan_tx_tag_present(skb)) { tx_flags |= vlan_tx_tag_get(skb); tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; tx_flags |= IXGBE_TX_FLAGS_VLAN; } - first = tx_ring->next_to_use; + /* record initial flags and protocol */ + first->tx_flags = tx_flags; + first->protocol = vlan_get_protocol(skb); - if (skb->protocol == htons(ETH_P_IP)) - tx_flags |= IXGBE_TX_FLAGS_IPV4; - tso = ixgbevf_tso(tx_ring, skb, tx_flags, &hdr_len); - if (tso < 0) { - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; - } + tso = ixgbevf_tso(tx_ring, first, &hdr_len); + if (tso < 0) + goto out_drop; + else if (!tso) + ixgbevf_tx_csum(tx_ring, first); - if (tso) - tx_flags |= IXGBE_TX_FLAGS_TSO | IXGBE_TX_FLAGS_CSUM; - else if (ixgbevf_tx_csum(tx_ring, skb, tx_flags)) - tx_flags |= IXGBE_TX_FLAGS_CSUM; + ixgbevf_tx_map(tx_ring, first, hdr_len); - ixgbevf_tx_queue(tx_ring, tx_flags, - ixgbevf_tx_map(tx_ring, skb, tx_flags, first), - skb->len, hdr_len); - /* - * Force memory writes to complete before letting h/w - * know there are new descriptors to fetch. (Only - * applicable for weak-ordered memory model archs, - * such as IA-64). - */ - wmb(); + ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); - writel(tx_ring->next_to_use, adapter->hw.hw_addr + tx_ring->tail); + return NETDEV_TX_OK; - ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); +out_drop: + dev_kfree_skb_any(first->skb); + first->skb = NULL; return NETDEV_TX_OK; } @@ -2902,12 +3255,11 @@ static int ixgbevf_set_mac(struct net_device *netdev, void *p) memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len); - spin_lock(&adapter->mbx_lock); + spin_lock_bh(&adapter->mbx_lock); - if (hw->mac.ops.set_rar) - hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); + hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); - spin_unlock(&adapter->mbx_lock); + spin_unlock_bh(&adapter->mbx_lock); return 0; } @@ -2925,8 +3277,15 @@ static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; - if (adapter->hw.mac.type == ixgbe_mac_X540_vf) + switch (adapter->hw.api_version) { + case ixgbe_mbox_api_11: max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; + break; + default: + if (adapter->hw.mac.type == ixgbe_mac_X540_vf) + max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; + break; + } /* MTU < 68 is an error and causes problems on some kernels */ if ((new_mtu < 68) || (max_frame > max_possible_frame)) @@ -2970,7 +3329,8 @@ static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) return retval; #endif - pci_disable_device(pdev); + if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) + pci_disable_device(pdev); return 0; } @@ -2978,11 +3338,10 @@ static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) #ifdef CONFIG_PM static int ixgbevf_resume(struct pci_dev *pdev) { - struct ixgbevf_adapter *adapter = pci_get_drvdata(pdev); - struct net_device *netdev = adapter->netdev; + struct net_device *netdev = pci_get_drvdata(pdev); + struct ixgbevf_adapter *adapter = netdev_priv(netdev); u32 err; - pci_set_power_state(pdev, PCI_D0); pci_restore_state(pdev); /* * pci_restore_state clears dev->state_saved so call @@ -2995,8 +3354,12 @@ static int ixgbevf_resume(struct pci_dev *pdev) dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); return err; } + smp_mb__before_atomic(); + clear_bit(__IXGBEVF_DISABLED, &adapter->state); pci_set_master(pdev); + ixgbevf_reset(adapter); + rtnl_lock(); err = ixgbevf_init_interrupt_scheme(adapter); rtnl_unlock(); @@ -3005,8 +3368,6 @@ static int ixgbevf_resume(struct pci_dev *pdev) return err; } - ixgbevf_reset(adapter); - if (netif_running(netdev)) { err = ixgbevf_open(netdev); if (err) @@ -3038,23 +3399,23 @@ static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; for (i = 0; i < adapter->num_rx_queues; i++) { - ring = &adapter->rx_ring[i]; + ring = adapter->rx_ring[i]; do { - start = u64_stats_fetch_begin_bh(&ring->syncp); - bytes = ring->total_bytes; - packets = ring->total_packets; - } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); + start = u64_stats_fetch_begin_irq(&ring->syncp); + bytes = ring->stats.bytes; + packets = ring->stats.packets; + } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); stats->rx_bytes += bytes; stats->rx_packets += packets; } for (i = 0; i < adapter->num_tx_queues; i++) { - ring = &adapter->tx_ring[i]; + ring = adapter->tx_ring[i]; do { - start = u64_stats_fetch_begin_bh(&ring->syncp); - bytes = ring->total_bytes; - packets = ring->total_packets; - } while (u64_stats_fetch_retry_bh(&ring->syncp, start)); + start = u64_stats_fetch_begin_irq(&ring->syncp); + bytes = ring->stats.bytes; + packets = ring->stats.packets; + } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); stats->tx_bytes += bytes; stats->tx_packets += packets; } @@ -3074,6 +3435,9 @@ static const struct net_device_ops ixgbevf_netdev_ops = { .ndo_tx_timeout = ixgbevf_tx_timeout, .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, +#ifdef CONFIG_NET_RX_BUSY_POLL + .ndo_busy_poll = ixgbevf_busy_poll_recv, +#endif }; static void ixgbevf_assign_netdev_ops(struct net_device *dev) @@ -3094,8 +3458,7 @@ static void ixgbevf_assign_netdev_ops(struct net_device *dev) * The OS initialization, configuring of the adapter private structure, * and a hardware reset occur. **/ -static int __devinit ixgbevf_probe(struct pci_dev *pdev, - const struct pci_device_id *ent) +static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct net_device *netdev; struct ixgbevf_adapter *adapter = NULL; @@ -3108,19 +3471,14 @@ static int __devinit ixgbevf_probe(struct pci_dev *pdev, if (err) return err; - if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && - !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { + if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { pci_using_dac = 1; } else { - err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (err) { - err = dma_set_coherent_mask(&pdev->dev, - DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "No usable DMA " - "configuration, aborting\n"); - goto err_dma; - } + dev_err(&pdev->dev, "No usable DMA " + "configuration, aborting\n"); + goto err_dma; } pci_using_dac = 0; } @@ -3159,6 +3517,7 @@ static int __devinit ixgbevf_probe(struct pci_dev *pdev, hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pci_resource_len(pdev, 0)); + adapter->io_addr = hw->hw_addr; if (!hw->hw_addr) { err = -EIO; goto err_ioremap; @@ -3181,8 +3540,6 @@ static int __devinit ixgbevf_probe(struct pci_dev *pdev, goto err_sw_init; /* The HW MAC address was set and/or determined in sw_init */ - memcpy(netdev->perm_addr, adapter->hw.mac.addr, netdev->addr_len); - if (!is_valid_ether_addr(netdev->dev_addr)) { pr_err("invalid MAC address\n"); err = -EIO; @@ -3197,9 +3554,9 @@ static int __devinit ixgbevf_probe(struct pci_dev *pdev, NETIF_F_RXCSUM; netdev->features = netdev->hw_features | - NETIF_F_HW_VLAN_TX | - NETIF_F_HW_VLAN_RX | - NETIF_F_HW_VLAN_FILTER; + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_FILTER; netdev->vlan_features |= NETIF_F_TSO; netdev->vlan_features |= NETIF_F_TSO6; @@ -3216,17 +3573,18 @@ static int __devinit ixgbevf_probe(struct pci_dev *pdev, adapter->watchdog_timer.function = ixgbevf_watchdog; adapter->watchdog_timer.data = (unsigned long)adapter; + if (IXGBE_REMOVED(hw->hw_addr)) { + err = -EIO; + goto err_sw_init; + } INIT_WORK(&adapter->reset_task, ixgbevf_reset_task); INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task); + set_bit(__IXGBEVF_WORK_INIT, &adapter->state); err = ixgbevf_init_interrupt_scheme(adapter); if (err) goto err_sw_init; - /* pick up the PCI bus settings for reporting later */ - if (hw->mac.ops.get_bus_info) - hw->mac.ops.get_bus_info(hw); - strcpy(netdev->name, "eth%d"); err = register_netdev(netdev); @@ -3250,14 +3608,15 @@ err_register: ixgbevf_clear_interrupt_scheme(adapter); err_sw_init: ixgbevf_reset_interrupt_capability(adapter); - iounmap(hw->hw_addr); + iounmap(adapter->io_addr); err_ioremap: free_netdev(netdev); err_alloc_etherdev: pci_release_regions(pdev); err_pci_reg: err_dma: - pci_disable_device(pdev); + if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) + pci_disable_device(pdev); return err; } @@ -3270,12 +3629,12 @@ err_dma: * Hot-Plug event, or because the driver is going to be removed from * memory. **/ -static void __devexit ixgbevf_remove(struct pci_dev *pdev) +static void ixgbevf_remove(struct pci_dev *pdev) { struct net_device *netdev = pci_get_drvdata(pdev); struct ixgbevf_adapter *adapter = netdev_priv(netdev); - set_bit(__IXGBEVF_DOWN, &adapter->state); + set_bit(__IXGBEVF_REMOVING, &adapter->state); del_timer_sync(&adapter->watchdog_timer); @@ -3288,17 +3647,15 @@ static void __devexit ixgbevf_remove(struct pci_dev *pdev) ixgbevf_clear_interrupt_scheme(adapter); ixgbevf_reset_interrupt_capability(adapter); - iounmap(adapter->hw.hw_addr); + iounmap(adapter->io_addr); pci_release_regions(pdev); hw_dbg(&adapter->hw, "Remove complete\n"); - kfree(adapter->tx_ring); - kfree(adapter->rx_ring); - free_netdev(netdev); - pci_disable_device(pdev); + if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) + pci_disable_device(pdev); } /** @@ -3315,15 +3672,23 @@ static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, struct net_device *netdev = pci_get_drvdata(pdev); struct ixgbevf_adapter *adapter = netdev_priv(netdev); + if (!test_bit(__IXGBEVF_WORK_INIT, &adapter->state)) + return PCI_ERS_RESULT_DISCONNECT; + + rtnl_lock(); netif_device_detach(netdev); - if (state == pci_channel_io_perm_failure) + if (state == pci_channel_io_perm_failure) { + rtnl_unlock(); return PCI_ERS_RESULT_DISCONNECT; + } if (netif_running(netdev)) ixgbevf_down(adapter); - pci_disable_device(pdev); + if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) + pci_disable_device(pdev); + rtnl_unlock(); /* Request a slot slot reset. */ return PCI_ERS_RESULT_NEED_RESET; @@ -3347,6 +3712,8 @@ static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) return PCI_ERS_RESULT_DISCONNECT; } + smp_mb__before_atomic(); + clear_bit(__IXGBEVF_DISABLED, &adapter->state); pci_set_master(pdev); ixgbevf_reset(adapter); @@ -3384,7 +3751,7 @@ static struct pci_driver ixgbevf_driver = { .name = ixgbevf_driver_name, .id_table = ixgbevf_pci_tbl, .probe = ixgbevf_probe, - .remove = __devexit_p(ixgbevf_remove), + .remove = ixgbevf_remove, #ifdef CONFIG_PM /* Power Management Hooks */ .suspend = ixgbevf_suspend, |