diff options
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_txrx.c | 1817 |
1 files changed, 1817 insertions, 0 deletions
diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c new file mode 100644 index 00000000000..49d2cfa9b0c --- /dev/null +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -0,0 +1,1817 @@ +/******************************************************************************* + * + * Intel Ethernet Controller XL710 Family Linux Driver + * Copyright(c) 2013 Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * + * The full GNU General Public License is included in this distribution in + * the file called "COPYING". + * + * Contact Information: + * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> + * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 + * + ******************************************************************************/ + +#include "i40e.h" + +static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size, + u32 td_tag) +{ + return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA | + ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) | + ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) | + ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) | + ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT)); +} + +/** + * i40e_program_fdir_filter - Program a Flow Director filter + * @fdir_input: Packet data that will be filter parameters + * @pf: The pf pointer + * @add: True for add/update, False for remove + **/ +int i40e_program_fdir_filter(struct i40e_fdir_data *fdir_data, + struct i40e_pf *pf, bool add) +{ + struct i40e_filter_program_desc *fdir_desc; + struct i40e_tx_buffer *tx_buf; + struct i40e_tx_desc *tx_desc; + struct i40e_ring *tx_ring; + struct i40e_vsi *vsi; + struct device *dev; + dma_addr_t dma; + u32 td_cmd = 0; + u16 i; + + /* find existing FDIR VSI */ + vsi = NULL; + for (i = 0; i < pf->hw.func_caps.num_vsis; i++) + if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) + vsi = pf->vsi[i]; + if (!vsi) + return -ENOENT; + + tx_ring = &vsi->tx_rings[0]; + dev = tx_ring->dev; + + dma = dma_map_single(dev, fdir_data->raw_packet, + I40E_FDIR_MAX_RAW_PACKET_LOOKUP, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma)) + goto dma_fail; + + /* grab the next descriptor */ + fdir_desc = I40E_TX_FDIRDESC(tx_ring, tx_ring->next_to_use); + tx_buf = &tx_ring->tx_bi[tx_ring->next_to_use]; + tx_ring->next_to_use++; + if (tx_ring->next_to_use == tx_ring->count) + tx_ring->next_to_use = 0; + + fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32((fdir_data->q_index + << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) + & I40E_TXD_FLTR_QW0_QINDEX_MASK); + + fdir_desc->qindex_flex_ptype_vsi |= cpu_to_le32((fdir_data->flex_off + << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT) + & I40E_TXD_FLTR_QW0_FLEXOFF_MASK); + + fdir_desc->qindex_flex_ptype_vsi |= cpu_to_le32((fdir_data->pctype + << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) + & I40E_TXD_FLTR_QW0_PCTYPE_MASK); + + /* Use LAN VSI Id if not programmed by user */ + if (fdir_data->dest_vsi == 0) + fdir_desc->qindex_flex_ptype_vsi |= + cpu_to_le32((pf->vsi[pf->lan_vsi]->id) + << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT); + else + fdir_desc->qindex_flex_ptype_vsi |= + cpu_to_le32((fdir_data->dest_vsi + << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT) + & I40E_TXD_FLTR_QW0_DEST_VSI_MASK); + + fdir_desc->dtype_cmd_cntindex = + cpu_to_le32(I40E_TX_DESC_DTYPE_FILTER_PROG); + + if (add) + fdir_desc->dtype_cmd_cntindex |= cpu_to_le32( + I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE + << I40E_TXD_FLTR_QW1_PCMD_SHIFT); + else + fdir_desc->dtype_cmd_cntindex |= cpu_to_le32( + I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE + << I40E_TXD_FLTR_QW1_PCMD_SHIFT); + + fdir_desc->dtype_cmd_cntindex |= cpu_to_le32((fdir_data->dest_ctl + << I40E_TXD_FLTR_QW1_DEST_SHIFT) + & I40E_TXD_FLTR_QW1_DEST_MASK); + + fdir_desc->dtype_cmd_cntindex |= cpu_to_le32( + (fdir_data->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT) + & I40E_TXD_FLTR_QW1_FD_STATUS_MASK); + + if (fdir_data->cnt_index != 0) { + fdir_desc->dtype_cmd_cntindex |= + cpu_to_le32(I40E_TXD_FLTR_QW1_CNT_ENA_MASK); + fdir_desc->dtype_cmd_cntindex |= + cpu_to_le32((fdir_data->cnt_index + << I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) + & I40E_TXD_FLTR_QW1_CNTINDEX_MASK); + } + + fdir_desc->fd_id = cpu_to_le32(fdir_data->fd_id); + + /* Now program a dummy descriptor */ + tx_desc = I40E_TX_DESC(tx_ring, tx_ring->next_to_use); + tx_buf = &tx_ring->tx_bi[tx_ring->next_to_use]; + tx_ring->next_to_use++; + if (tx_ring->next_to_use == tx_ring->count) + tx_ring->next_to_use = 0; + + tx_desc->buffer_addr = cpu_to_le64(dma); + td_cmd = I40E_TX_DESC_CMD_EOP | + I40E_TX_DESC_CMD_RS | + I40E_TX_DESC_CMD_DUMMY; + + tx_desc->cmd_type_offset_bsz = + build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_LOOKUP, 0); + + /* Mark the data descriptor to be watched */ + tx_buf->next_to_watch = tx_desc; + + /* Force memory writes to complete before letting h/w + * know there are new descriptors to fetch. (Only + * applicable for weak-ordered memory model archs, + * such as IA-64). + */ + wmb(); + + writel(tx_ring->next_to_use, tx_ring->tail); + return 0; + +dma_fail: + return -1; +} + +/** + * i40e_fd_handle_status - check the Programming Status for FD + * @rx_ring: the Rx ring for this descriptor + * @qw: the descriptor data + * @prog_id: the id originally used for programming + * + * This is used to verify if the FD programming or invalidation + * requested by SW to the HW is successful or not and take actions accordingly. + **/ +static void i40e_fd_handle_status(struct i40e_ring *rx_ring, u32 qw, u8 prog_id) +{ + struct pci_dev *pdev = rx_ring->vsi->back->pdev; + u32 error; + + error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >> + I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT; + + /* for now just print the Status */ + dev_info(&pdev->dev, "FD programming id %02x, Status %08x\n", + prog_id, error); +} + +/** + * i40e_unmap_tx_resource - Release a Tx buffer + * @ring: the ring that owns the buffer + * @tx_buffer: the buffer to free + **/ +static inline void i40e_unmap_tx_resource(struct i40e_ring *ring, + struct i40e_tx_buffer *tx_buffer) +{ + if (tx_buffer->dma) { + if (tx_buffer->tx_flags & I40E_TX_FLAGS_MAPPED_AS_PAGE) + dma_unmap_page(ring->dev, + tx_buffer->dma, + tx_buffer->length, + DMA_TO_DEVICE); + else + dma_unmap_single(ring->dev, + tx_buffer->dma, + tx_buffer->length, + DMA_TO_DEVICE); + } + tx_buffer->dma = 0; + tx_buffer->time_stamp = 0; +} + +/** + * i40e_clean_tx_ring - Free any empty Tx buffers + * @tx_ring: ring to be cleaned + **/ +void i40e_clean_tx_ring(struct i40e_ring *tx_ring) +{ + struct i40e_tx_buffer *tx_buffer; + unsigned long bi_size; + u16 i; + + /* ring already cleared, nothing to do */ + if (!tx_ring->tx_bi) + return; + + /* Free all the Tx ring sk_buffs */ + for (i = 0; i < tx_ring->count; i++) { + tx_buffer = &tx_ring->tx_bi[i]; + i40e_unmap_tx_resource(tx_ring, tx_buffer); + if (tx_buffer->skb) + dev_kfree_skb_any(tx_buffer->skb); + tx_buffer->skb = NULL; + } + + bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; + memset(tx_ring->tx_bi, 0, bi_size); + + /* Zero out the descriptor ring */ + memset(tx_ring->desc, 0, tx_ring->size); + + tx_ring->next_to_use = 0; + tx_ring->next_to_clean = 0; +} + +/** + * i40e_free_tx_resources - Free Tx resources per queue + * @tx_ring: Tx descriptor ring for a specific queue + * + * Free all transmit software resources + **/ +void i40e_free_tx_resources(struct i40e_ring *tx_ring) +{ + i40e_clean_tx_ring(tx_ring); + kfree(tx_ring->tx_bi); + tx_ring->tx_bi = NULL; + + if (tx_ring->desc) { + dma_free_coherent(tx_ring->dev, tx_ring->size, + tx_ring->desc, tx_ring->dma); + tx_ring->desc = NULL; + } +} + +/** + * i40e_get_tx_pending - how many tx descriptors not processed + * @tx_ring: the ring of descriptors + * + * Since there is no access to the ring head register + * in XL710, we need to use our local copies + **/ +static u32 i40e_get_tx_pending(struct i40e_ring *ring) +{ + u32 ntu = ((ring->next_to_clean <= ring->next_to_use) + ? ring->next_to_use + : ring->next_to_use + ring->count); + return ntu - ring->next_to_clean; +} + +/** + * i40e_check_tx_hang - Is there a hang in the Tx queue + * @tx_ring: the ring of descriptors + **/ +static bool i40e_check_tx_hang(struct i40e_ring *tx_ring) +{ + u32 tx_pending = i40e_get_tx_pending(tx_ring); + bool ret = false; + + clear_check_for_tx_hang(tx_ring); + + /* Check for a hung queue, but be thorough. This verifies + * that a transmit has been completed since the previous + * check AND there is at least one packet pending. The + * ARMED bit is set to indicate a potential hang. The + * bit is cleared if a pause frame is received to remove + * false hang detection due to PFC or 802.3x frames. By + * requiring this to fail twice we avoid races with + * PFC clearing the ARMED bit and conditions where we + * run the check_tx_hang logic with a transmit completion + * pending but without time to complete it yet. + */ + if ((tx_ring->tx_stats.tx_done_old == tx_ring->tx_stats.packets) && + tx_pending) { + /* make sure it is true for two checks in a row */ + ret = test_and_set_bit(__I40E_HANG_CHECK_ARMED, + &tx_ring->state); + } else { + /* update completed stats and disarm the hang check */ + tx_ring->tx_stats.tx_done_old = tx_ring->tx_stats.packets; + clear_bit(__I40E_HANG_CHECK_ARMED, &tx_ring->state); + } + + return ret; +} + +/** + * i40e_clean_tx_irq - Reclaim resources after transmit completes + * @tx_ring: tx ring to clean + * @budget: how many cleans we're allowed + * + * Returns true if there's any budget left (e.g. the clean is finished) + **/ +static bool i40e_clean_tx_irq(struct i40e_ring *tx_ring, int budget) +{ + u16 i = tx_ring->next_to_clean; + struct i40e_tx_buffer *tx_buf; + struct i40e_tx_desc *tx_desc; + unsigned int total_packets = 0; + unsigned int total_bytes = 0; + + tx_buf = &tx_ring->tx_bi[i]; + tx_desc = I40E_TX_DESC(tx_ring, i); + + for (; budget; budget--) { + struct i40e_tx_desc *eop_desc; + + eop_desc = tx_buf->next_to_watch; + + /* if next_to_watch is not set then there is no work pending */ + if (!eop_desc) + break; + + /* if the descriptor isn't done, no work yet to do */ + if (!(eop_desc->cmd_type_offset_bsz & + cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) + break; + + /* count the packet as being completed */ + tx_ring->tx_stats.completed++; + tx_buf->next_to_watch = NULL; + tx_buf->time_stamp = 0; + + /* set memory barrier before eop_desc is verified */ + rmb(); + + do { + i40e_unmap_tx_resource(tx_ring, tx_buf); + + /* clear dtype status */ + tx_desc->cmd_type_offset_bsz &= + ~cpu_to_le64(I40E_TXD_QW1_DTYPE_MASK); + + if (likely(tx_desc == eop_desc)) { + eop_desc = NULL; + + dev_kfree_skb_any(tx_buf->skb); + tx_buf->skb = NULL; + + total_bytes += tx_buf->bytecount; + total_packets += tx_buf->gso_segs; + } + + tx_buf++; + tx_desc++; + i++; + if (unlikely(i == tx_ring->count)) { + i = 0; + tx_buf = tx_ring->tx_bi; + tx_desc = I40E_TX_DESC(tx_ring, 0); + } + } while (eop_desc); + } + + tx_ring->next_to_clean = i; + tx_ring->tx_stats.bytes += total_bytes; + tx_ring->tx_stats.packets += total_packets; + tx_ring->q_vector->tx.total_bytes += total_bytes; + tx_ring->q_vector->tx.total_packets += total_packets; + if (check_for_tx_hang(tx_ring) && i40e_check_tx_hang(tx_ring)) { + /* schedule immediate reset if we believe we hung */ + dev_info(tx_ring->dev, "Detected Tx Unit Hang\n" + " VSI <%d>\n" + " Tx Queue <%d>\n" + " next_to_use <%x>\n" + " next_to_clean <%x>\n", + tx_ring->vsi->seid, + tx_ring->queue_index, + tx_ring->next_to_use, i); + dev_info(tx_ring->dev, "tx_bi[next_to_clean]\n" + " time_stamp <%lx>\n" + " jiffies <%lx>\n", + tx_ring->tx_bi[i].time_stamp, jiffies); + + netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); + + dev_info(tx_ring->dev, + "tx hang detected on queue %d, resetting adapter\n", + tx_ring->queue_index); + + tx_ring->netdev->netdev_ops->ndo_tx_timeout(tx_ring->netdev); + + /* the adapter is about to reset, no point in enabling stuff */ + return true; + } + +#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) + if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && + (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { + /* Make sure that anybody stopping the queue after this + * sees the new next_to_clean. + */ + smp_mb(); + if (__netif_subqueue_stopped(tx_ring->netdev, + tx_ring->queue_index) && + !test_bit(__I40E_DOWN, &tx_ring->vsi->state)) { + netif_wake_subqueue(tx_ring->netdev, + tx_ring->queue_index); + ++tx_ring->tx_stats.restart_queue; + } + } + + return budget > 0; +} + +/** + * i40e_set_new_dynamic_itr - Find new ITR level + * @rc: structure containing ring performance data + * + * Stores a new ITR value based on packets and byte counts during + * the last interrupt. The advantage of per interrupt computation + * is faster updates and more accurate ITR for the current traffic + * pattern. Constants in this function were computed based on + * theoretical maximum wire speed and thresholds were set based on + * testing data as well as attempting to minimize response time + * while increasing bulk throughput. + **/ +static void i40e_set_new_dynamic_itr(struct i40e_ring_container *rc) +{ + enum i40e_latency_range new_latency_range = rc->latency_range; + u32 new_itr = rc->itr; + int bytes_per_int; + + if (rc->total_packets == 0 || !rc->itr) + return; + + /* simple throttlerate management + * 0-10MB/s lowest (100000 ints/s) + * 10-20MB/s low (20000 ints/s) + * 20-1249MB/s bulk (8000 ints/s) + */ + bytes_per_int = rc->total_bytes / rc->itr; + switch (rc->itr) { + case I40E_LOWEST_LATENCY: + if (bytes_per_int > 10) + new_latency_range = I40E_LOW_LATENCY; + break; + case I40E_LOW_LATENCY: + if (bytes_per_int > 20) + new_latency_range = I40E_BULK_LATENCY; + else if (bytes_per_int <= 10) + new_latency_range = I40E_LOWEST_LATENCY; + break; + case I40E_BULK_LATENCY: + if (bytes_per_int <= 20) + rc->latency_range = I40E_LOW_LATENCY; + break; + } + + switch (new_latency_range) { + case I40E_LOWEST_LATENCY: + new_itr = I40E_ITR_100K; + break; + case I40E_LOW_LATENCY: + new_itr = I40E_ITR_20K; + break; + case I40E_BULK_LATENCY: + new_itr = I40E_ITR_8K; + break; + default: + break; + } + + if (new_itr != rc->itr) { + /* do an exponential smoothing */ + new_itr = (10 * new_itr * rc->itr) / + ((9 * new_itr) + rc->itr); + rc->itr = new_itr & I40E_MAX_ITR; + } + + rc->total_bytes = 0; + rc->total_packets = 0; +} + +/** + * i40e_update_dynamic_itr - Adjust ITR based on bytes per int + * @q_vector: the vector to adjust + **/ +static void i40e_update_dynamic_itr(struct i40e_q_vector *q_vector) +{ + u16 vector = q_vector->vsi->base_vector + q_vector->v_idx; + struct i40e_hw *hw = &q_vector->vsi->back->hw; + u32 reg_addr; + u16 old_itr; + + reg_addr = I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1); + old_itr = q_vector->rx.itr; + i40e_set_new_dynamic_itr(&q_vector->rx); + if (old_itr != q_vector->rx.itr) + wr32(hw, reg_addr, q_vector->rx.itr); + + reg_addr = I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1); + old_itr = q_vector->tx.itr; + i40e_set_new_dynamic_itr(&q_vector->tx); + if (old_itr != q_vector->tx.itr) + wr32(hw, reg_addr, q_vector->tx.itr); + + i40e_flush(hw); +} + +/** + * i40e_clean_programming_status - clean the programming status descriptor + * @rx_ring: the rx ring that has this descriptor + * @rx_desc: the rx descriptor written back by HW + * + * Flow director should handle FD_FILTER_STATUS to check its filter programming + * status being successful or not and take actions accordingly. FCoE should + * handle its context/filter programming/invalidation status and take actions. + * + **/ +static void i40e_clean_programming_status(struct i40e_ring *rx_ring, + union i40e_rx_desc *rx_desc) +{ + u64 qw; + u8 id; + + qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len); + id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >> + I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT; + + if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) + i40e_fd_handle_status(rx_ring, qw, id); +} + +/** + * i40e_setup_tx_descriptors - Allocate the Tx descriptors + * @tx_ring: the tx ring to set up + * + * Return 0 on success, negative on error + **/ +int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring) +{ + struct device *dev = tx_ring->dev; + int bi_size; + + if (!dev) + return -ENOMEM; + + bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; + tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL); + if (!tx_ring->tx_bi) + goto err; + + /* round up to nearest 4K */ + tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc); + tx_ring->size = ALIGN(tx_ring->size, 4096); + tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, + &tx_ring->dma, GFP_KERNEL); + if (!tx_ring->desc) { + dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", + tx_ring->size); + goto err; + } + + tx_ring->next_to_use = 0; + tx_ring->next_to_clean = 0; + return 0; + +err: + kfree(tx_ring->tx_bi); + tx_ring->tx_bi = NULL; + return -ENOMEM; +} + +/** + * i40e_clean_rx_ring - Free Rx buffers + * @rx_ring: ring to be cleaned + **/ +void i40e_clean_rx_ring(struct i40e_ring *rx_ring) +{ + struct device *dev = rx_ring->dev; + struct i40e_rx_buffer *rx_bi; + unsigned long bi_size; + u16 i; + + /* ring already cleared, nothing to do */ + if (!rx_ring->rx_bi) + return; + + /* Free all the Rx ring sk_buffs */ + for (i = 0; i < rx_ring->count; i++) { + rx_bi = &rx_ring->rx_bi[i]; + if (rx_bi->dma) { + dma_unmap_single(dev, + rx_bi->dma, + rx_ring->rx_buf_len, + DMA_FROM_DEVICE); + rx_bi->dma = 0; + } + if (rx_bi->skb) { + dev_kfree_skb(rx_bi->skb); + rx_bi->skb = NULL; + } + if (rx_bi->page) { + if (rx_bi->page_dma) { + dma_unmap_page(dev, + rx_bi->page_dma, + PAGE_SIZE / 2, + DMA_FROM_DEVICE); + rx_bi->page_dma = 0; + } + __free_page(rx_bi->page); + rx_bi->page = NULL; + rx_bi->page_offset = 0; + } + } + + bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; + memset(rx_ring->rx_bi, 0, bi_size); + + /* Zero out the descriptor ring */ + memset(rx_ring->desc, 0, rx_ring->size); + + rx_ring->next_to_clean = 0; + rx_ring->next_to_use = 0; +} + +/** + * i40e_free_rx_resources - Free Rx resources + * @rx_ring: ring to clean the resources from + * + * Free all receive software resources + **/ +void i40e_free_rx_resources(struct i40e_ring *rx_ring) +{ + i40e_clean_rx_ring(rx_ring); + kfree(rx_ring->rx_bi); + rx_ring->rx_bi = NULL; + + if (rx_ring->desc) { + dma_free_coherent(rx_ring->dev, rx_ring->size, + rx_ring->desc, rx_ring->dma); + rx_ring->desc = NULL; + } +} + +/** + * i40e_setup_rx_descriptors - Allocate Rx descriptors + * @rx_ring: Rx descriptor ring (for a specific queue) to setup + * + * Returns 0 on success, negative on failure + **/ +int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) +{ + struct device *dev = rx_ring->dev; + int bi_size; + + bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; + rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL); + if (!rx_ring->rx_bi) + goto err; + + /* Round up to nearest 4K */ + rx_ring->size = ring_is_16byte_desc_enabled(rx_ring) + ? rx_ring->count * sizeof(union i40e_16byte_rx_desc) + : rx_ring->count * sizeof(union i40e_32byte_rx_desc); + rx_ring->size = ALIGN(rx_ring->size, 4096); + rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, + &rx_ring->dma, GFP_KERNEL); + + if (!rx_ring->desc) { + dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", + rx_ring->size); + goto err; + } + + rx_ring->next_to_clean = 0; + rx_ring->next_to_use = 0; + + return 0; +err: + kfree(rx_ring->rx_bi); + rx_ring->rx_bi = NULL; + return -ENOMEM; +} + +/** + * i40e_release_rx_desc - Store the new tail and head values + * @rx_ring: ring to bump + * @val: new head index + **/ +static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val) +{ + rx_ring->next_to_use = val; + /* Force memory writes to complete before letting h/w + * know there are new descriptors to fetch. (Only + * applicable for weak-ordered memory model archs, + * such as IA-64). + */ + wmb(); + writel(val, rx_ring->tail); +} + +/** + * i40e_alloc_rx_buffers - Replace used receive buffers; packet split + * @rx_ring: ring to place buffers on + * @cleaned_count: number of buffers to replace + **/ +void i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count) +{ + u16 i = rx_ring->next_to_use; + union i40e_rx_desc *rx_desc; + struct i40e_rx_buffer *bi; + struct sk_buff *skb; + + /* do nothing if no valid netdev defined */ + if (!rx_ring->netdev || !cleaned_count) + return; + + while (cleaned_count--) { + rx_desc = I40E_RX_DESC(rx_ring, i); + bi = &rx_ring->rx_bi[i]; + skb = bi->skb; + + if (!skb) { + skb = netdev_alloc_skb_ip_align(rx_ring->netdev, + rx_ring->rx_buf_len); + if (!skb) { + rx_ring->rx_stats.alloc_rx_buff_failed++; + goto no_buffers; + } + /* initialize queue mapping */ + skb_record_rx_queue(skb, rx_ring->queue_index); + bi->skb = skb; + } + + if (!bi->dma) { + bi->dma = dma_map_single(rx_ring->dev, + skb->data, + rx_ring->rx_buf_len, + DMA_FROM_DEVICE); + if (dma_mapping_error(rx_ring->dev, bi->dma)) { + rx_ring->rx_stats.alloc_rx_buff_failed++; + bi->dma = 0; + goto no_buffers; + } + } + + if (ring_is_ps_enabled(rx_ring)) { + if (!bi->page) { + bi->page = alloc_page(GFP_ATOMIC); + if (!bi->page) { + rx_ring->rx_stats.alloc_rx_page_failed++; + goto no_buffers; + } + } + + if (!bi->page_dma) { + /* use a half page if we're re-using */ + bi->page_offset ^= PAGE_SIZE / 2; + bi->page_dma = dma_map_page(rx_ring->dev, + bi->page, + bi->page_offset, + PAGE_SIZE / 2, + DMA_FROM_DEVICE); + if (dma_mapping_error(rx_ring->dev, + bi->page_dma)) { + rx_ring->rx_stats.alloc_rx_page_failed++; + bi->page_dma = 0; + goto no_buffers; + } + } + + /* Refresh the desc even if buffer_addrs didn't change + * because each write-back erases this info. + */ + rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); + rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); + } else { + rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); + rx_desc->read.hdr_addr = 0; + } + i++; + if (i == rx_ring->count) + i = 0; + } + +no_buffers: + if (rx_ring->next_to_use != i) + i40e_release_rx_desc(rx_ring, i); +} + +/** + * i40e_receive_skb - Send a completed packet up the stack + * @rx_ring: rx ring in play + * @skb: packet to send up + * @vlan_tag: vlan tag for packet + **/ +static void i40e_receive_skb(struct i40e_ring *rx_ring, + struct sk_buff *skb, u16 vlan_tag) +{ + struct i40e_q_vector *q_vector = rx_ring->q_vector; + struct i40e_vsi *vsi = rx_ring->vsi; + u64 flags = vsi->back->flags; + + if (vlan_tag & VLAN_VID_MASK) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag); + + if (flags & I40E_FLAG_IN_NETPOLL) + netif_rx(skb); + else + napi_gro_receive(&q_vector->napi, skb); +} + +/** + * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum + * @vsi: the VSI we care about + * @skb: skb currently being received and modified + * @rx_status: status value of last descriptor in packet + * @rx_error: error value of last descriptor in packet + **/ +static inline void i40e_rx_checksum(struct i40e_vsi *vsi, + struct sk_buff *skb, + u32 rx_status, + u32 rx_error) +{ + skb->ip_summed = CHECKSUM_NONE; + + /* Rx csum enabled and ip headers found? */ + if (!(vsi->netdev->features & NETIF_F_RXCSUM && + rx_status & (1 << I40E_RX_DESC_STATUS_L3L4P_SHIFT))) + return; + + /* IP or L4 checksum error */ + if (rx_error & ((1 << I40E_RX_DESC_ERROR_IPE_SHIFT) | + (1 << I40E_RX_DESC_ERROR_L4E_SHIFT))) { + vsi->back->hw_csum_rx_error++; + return; + } + + skb->ip_summed = CHECKSUM_UNNECESSARY; +} + +/** + * i40e_rx_hash - returns the hash value from the Rx descriptor + * @ring: descriptor ring + * @rx_desc: specific descriptor + **/ +static inline u32 i40e_rx_hash(struct i40e_ring *ring, + union i40e_rx_desc *rx_desc) +{ + if (ring->netdev->features & NETIF_F_RXHASH) { + if ((le64_to_cpu(rx_desc->wb.qword1.status_error_len) >> + I40E_RX_DESC_STATUS_FLTSTAT_SHIFT) & + I40E_RX_DESC_FLTSTAT_RSS_HASH) + return le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss); + } + return 0; +} + +/** + * i40e_clean_rx_irq - Reclaim resources after receive completes + * @rx_ring: rx ring to clean + * @budget: how many cleans we're allowed + * + * Returns true if there's any budget left (e.g. the clean is finished) + **/ +static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) +{ + unsigned int total_rx_bytes = 0, total_rx_packets = 0; + u16 rx_packet_len, rx_header_len, rx_sph, rx_hbo; + u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); + const int current_node = numa_node_id(); + struct i40e_vsi *vsi = rx_ring->vsi; + u16 i = rx_ring->next_to_clean; + union i40e_rx_desc *rx_desc; + u32 rx_error, rx_status; + u64 qword; + + rx_desc = I40E_RX_DESC(rx_ring, i); + qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); + rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) + >> I40E_RXD_QW1_STATUS_SHIFT; + + while (rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)) { + union i40e_rx_desc *next_rxd; + struct i40e_rx_buffer *rx_bi; + struct sk_buff *skb; + u16 vlan_tag; + if (i40e_rx_is_programming_status(qword)) { + i40e_clean_programming_status(rx_ring, rx_desc); + I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd); + goto next_desc; + } + rx_bi = &rx_ring->rx_bi[i]; + skb = rx_bi->skb; + prefetch(skb->data); + + rx_packet_len = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) + >> I40E_RXD_QW1_LENGTH_PBUF_SHIFT; + rx_header_len = (qword & I40E_RXD_QW1_LENGTH_HBUF_MASK) + >> I40E_RXD_QW1_LENGTH_HBUF_SHIFT; + rx_sph = (qword & I40E_RXD_QW1_LENGTH_SPH_MASK) + >> I40E_RXD_QW1_LENGTH_SPH_SHIFT; + + rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) + >> I40E_RXD_QW1_ERROR_SHIFT; + rx_hbo = rx_error & (1 << I40E_RX_DESC_ERROR_HBO_SHIFT); + rx_error &= ~(1 << I40E_RX_DESC_ERROR_HBO_SHIFT); + + rx_bi->skb = NULL; + + /* This memory barrier is needed to keep us from reading + * any other fields out of the rx_desc until we know the + * STATUS_DD bit is set + */ + rmb(); + + /* Get the header and possibly the whole packet + * If this is an skb from previous receive dma will be 0 + */ + if (rx_bi->dma) { + u16 len; + + if (rx_hbo) + len = I40E_RX_HDR_SIZE; + else if (rx_sph) + len = rx_header_len; + else if (rx_packet_len) + len = rx_packet_len; /* 1buf/no split found */ + else + len = rx_header_len; /* split always mode */ + + skb_put(skb, len); + dma_unmap_single(rx_ring->dev, + rx_bi->dma, + rx_ring->rx_buf_len, + DMA_FROM_DEVICE); + rx_bi->dma = 0; + } + + /* Get the rest of the data if this was a header split */ + if (ring_is_ps_enabled(rx_ring) && rx_packet_len) { + + skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, + rx_bi->page, + rx_bi->page_offset, + rx_packet_len); + + skb->len += rx_packet_len; + skb->data_len += rx_packet_len; + skb->truesize += rx_packet_len; + + if ((page_count(rx_bi->page) == 1) && + (page_to_nid(rx_bi->page) == current_node)) + get_page(rx_bi->page); + else + rx_bi->page = NULL; + + dma_unmap_page(rx_ring->dev, + rx_bi->page_dma, + PAGE_SIZE / 2, + DMA_FROM_DEVICE); + rx_bi->page_dma = 0; + } + I40E_RX_NEXT_DESC_PREFETCH(rx_ring, i, next_rxd); + + if (unlikely( + !(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT)))) { + struct i40e_rx_buffer *next_buffer; + + next_buffer = &rx_ring->rx_bi[i]; + + if (ring_is_ps_enabled(rx_ring)) { + rx_bi->skb = next_buffer->skb; + rx_bi->dma = next_buffer->dma; + next_buffer->skb = skb; + next_buffer->dma = 0; + } + rx_ring->rx_stats.non_eop_descs++; + goto next_desc; + } + + /* ERR_MASK will only have valid bits if EOP set */ + if (unlikely(rx_error & (1 << I40E_RX_DESC_ERROR_RXE_SHIFT))) { + dev_kfree_skb_any(skb); + goto next_desc; + } + + skb->rxhash = i40e_rx_hash(rx_ring, rx_desc); + i40e_rx_checksum(vsi, skb, rx_status, rx_error); + + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; + total_rx_packets++; + + skb->protocol = eth_type_trans(skb, rx_ring->netdev); + vlan_tag = rx_status & (1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT) + ? le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) + : 0; + i40e_receive_skb(rx_ring, skb, vlan_tag); + + rx_ring->netdev->last_rx = jiffies; + budget--; +next_desc: + rx_desc->wb.qword1.status_error_len = 0; + if (!budget) + break; + + cleaned_count++; + /* return some buffers to hardware, one at a time is too slow */ + if (cleaned_count >= I40E_RX_BUFFER_WRITE) { + i40e_alloc_rx_buffers(rx_ring, cleaned_count); + cleaned_count = 0; + } + + /* use prefetched values */ + rx_desc = next_rxd; + qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); + rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) + >> I40E_RXD_QW1_STATUS_SHIFT; + } + + rx_ring->next_to_clean = i; + rx_ring->rx_stats.packets += total_rx_packets; + rx_ring->rx_stats.bytes += total_rx_bytes; + rx_ring->q_vector->rx.total_packets += total_rx_packets; + rx_ring->q_vector->rx.total_bytes += total_rx_bytes; + + if (cleaned_count) + i40e_alloc_rx_buffers(rx_ring, cleaned_count); + + return budget > 0; +} + +/** + * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine + * @napi: napi struct with our devices info in it + * @budget: amount of work driver is allowed to do this pass, in packets + * + * This function will clean all queues associated with a q_vector. + * + * Returns the amount of work done + **/ +int i40e_napi_poll(struct napi_struct *napi, int budget) +{ + struct i40e_q_vector *q_vector = + container_of(napi, struct i40e_q_vector, napi); + struct i40e_vsi *vsi = q_vector->vsi; + bool clean_complete = true; + int budget_per_ring; + int i; + + if (test_bit(__I40E_DOWN, &vsi->state)) { + napi_complete(napi); + return 0; + } + + /* We attempt to distribute budget to each Rx queue fairly, but don't + * allow the budget to go below 1 because that would exit polling early. + * Since the actual Tx work is minimal, we can give the Tx a larger + * budget and be more aggressive about cleaning up the Tx descriptors. + */ + budget_per_ring = max(budget/q_vector->num_ringpairs, 1); + for (i = 0; i < q_vector->num_ringpairs; i++) { + clean_complete &= i40e_clean_tx_irq(q_vector->tx.ring[i], + vsi->work_limit); + clean_complete &= i40e_clean_rx_irq(q_vector->rx.ring[i], + budget_per_ring); + } + + /* If work not completed, return budget and polling will return */ + if (!clean_complete) + return budget; + + /* Work is done so exit the polling mode and re-enable the interrupt */ + napi_complete(napi); + if (ITR_IS_DYNAMIC(vsi->rx_itr_setting) || + ITR_IS_DYNAMIC(vsi->tx_itr_setting)) + i40e_update_dynamic_itr(q_vector); + + if (!test_bit(__I40E_DOWN, &vsi->state)) { + if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { + i40e_irq_dynamic_enable(vsi, + q_vector->v_idx + vsi->base_vector); + } else { + struct i40e_hw *hw = &vsi->back->hw; + /* We re-enable the queue 0 cause, but + * don't worry about dynamic_enable + * because we left it on for the other + * possible interrupts during napi + */ + u32 qval = rd32(hw, I40E_QINT_RQCTL(0)); + qval |= I40E_QINT_RQCTL_CAUSE_ENA_MASK; + wr32(hw, I40E_QINT_RQCTL(0), qval); + + qval = rd32(hw, I40E_QINT_TQCTL(0)); + qval |= I40E_QINT_TQCTL_CAUSE_ENA_MASK; + wr32(hw, I40E_QINT_TQCTL(0), qval); + i40e_flush(hw); + } + } + + return 0; +} + +/** + * i40e_atr - Add a Flow Director ATR filter + * @tx_ring: ring to add programming descriptor to + * @skb: send buffer + * @flags: send flags + * @protocol: wire protocol + **/ +static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb, + u32 flags, __be16 protocol) +{ + struct i40e_filter_program_desc *fdir_desc; + struct i40e_pf *pf = tx_ring->vsi->back; + union { + unsigned char *network; + struct iphdr *ipv4; + struct ipv6hdr *ipv6; + } hdr; + struct tcphdr *th; + unsigned int hlen; + u32 flex_ptype, dtype_cmd; + + /* make sure ATR is enabled */ + if (!(pf->flags |