diff options
Diffstat (limited to 'drivers/net/sfc/falcon.c')
| -rw-r--r-- | drivers/net/sfc/falcon.c | 2722 |
1 files changed, 2722 insertions, 0 deletions
diff --git a/drivers/net/sfc/falcon.c b/drivers/net/sfc/falcon.c new file mode 100644 index 00000000000..46db549ce58 --- /dev/null +++ b/drivers/net/sfc/falcon.c @@ -0,0 +1,2722 @@ +/**************************************************************************** + * Driver for Solarflare Solarstorm network controllers and boards + * Copyright 2005-2006 Fen Systems Ltd. + * Copyright 2006-2008 Solarflare Communications Inc. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, incorporated herein by reference. + */ + +#include <linux/bitops.h> +#include <linux/delay.h> +#include <linux/pci.h> +#include <linux/module.h> +#include <linux/seq_file.h> +#include "net_driver.h" +#include "bitfield.h" +#include "efx.h" +#include "mac.h" +#include "gmii.h" +#include "spi.h" +#include "falcon.h" +#include "falcon_hwdefs.h" +#include "falcon_io.h" +#include "mdio_10g.h" +#include "phy.h" +#include "boards.h" +#include "workarounds.h" + +/* Falcon hardware control. + * Falcon is the internal codename for the SFC4000 controller that is + * present in SFE400X evaluation boards + */ + +/** + * struct falcon_nic_data - Falcon NIC state + * @next_buffer_table: First available buffer table id + * @pci_dev2: The secondary PCI device if present + */ +struct falcon_nic_data { + unsigned next_buffer_table; + struct pci_dev *pci_dev2; +}; + +/************************************************************************** + * + * Configurable values + * + ************************************************************************** + */ + +static int disable_dma_stats; + +/* This is set to 16 for a good reason. In summary, if larger than + * 16, the descriptor cache holds more than a default socket + * buffer's worth of packets (for UDP we can only have at most one + * socket buffer's worth outstanding). This combined with the fact + * that we only get 1 TX event per descriptor cache means the NIC + * goes idle. + */ +#define TX_DC_ENTRIES 16 +#define TX_DC_ENTRIES_ORDER 0 +#define TX_DC_BASE 0x130000 + +#define RX_DC_ENTRIES 64 +#define RX_DC_ENTRIES_ORDER 2 +#define RX_DC_BASE 0x100000 + +/* RX FIFO XOFF watermark + * + * When the amount of the RX FIFO increases used increases past this + * watermark send XOFF. Only used if RX flow control is enabled (ethtool -A) + * This also has an effect on RX/TX arbitration + */ +static int rx_xoff_thresh_bytes = -1; +module_param(rx_xoff_thresh_bytes, int, 0644); +MODULE_PARM_DESC(rx_xoff_thresh_bytes, "RX fifo XOFF threshold"); + +/* RX FIFO XON watermark + * + * When the amount of the RX FIFO used decreases below this + * watermark send XON. Only used if TX flow control is enabled (ethtool -A) + * This also has an effect on RX/TX arbitration + */ +static int rx_xon_thresh_bytes = -1; +module_param(rx_xon_thresh_bytes, int, 0644); +MODULE_PARM_DESC(rx_xon_thresh_bytes, "RX fifo XON threshold"); + +/* TX descriptor ring size - min 512 max 4k */ +#define FALCON_TXD_RING_ORDER TX_DESCQ_SIZE_1K +#define FALCON_TXD_RING_SIZE 1024 +#define FALCON_TXD_RING_MASK (FALCON_TXD_RING_SIZE - 1) + +/* RX descriptor ring size - min 512 max 4k */ +#define FALCON_RXD_RING_ORDER RX_DESCQ_SIZE_1K +#define FALCON_RXD_RING_SIZE 1024 +#define FALCON_RXD_RING_MASK (FALCON_RXD_RING_SIZE - 1) + +/* Event queue size - max 32k */ +#define FALCON_EVQ_ORDER EVQ_SIZE_4K +#define FALCON_EVQ_SIZE 4096 +#define FALCON_EVQ_MASK (FALCON_EVQ_SIZE - 1) + +/* Max number of internal errors. After this resets will not be performed */ +#define FALCON_MAX_INT_ERRORS 4 + +/* Maximum period that we wait for flush events. If the flush event + * doesn't arrive in this period of time then we check if the queue + * was disabled anyway. */ +#define FALCON_FLUSH_TIMEOUT 10 /* 10ms */ + +/************************************************************************** + * + * Falcon constants + * + ************************************************************************** + */ + +/* DMA address mask (up to 46-bit, avoiding compiler warnings) + * + * Note that it is possible to have a platform with 64-bit longs and + * 32-bit DMA addresses, or vice versa. EFX_DMA_MASK takes care of the + * platform DMA mask. + */ +#if BITS_PER_LONG == 64 +#define FALCON_DMA_MASK EFX_DMA_MASK(0x00003fffffffffffUL) +#else +#define FALCON_DMA_MASK EFX_DMA_MASK(0x00003fffffffffffULL) +#endif + +/* TX DMA length mask (13-bit) */ +#define FALCON_TX_DMA_MASK (4096 - 1) + +/* Size and alignment of special buffers (4KB) */ +#define FALCON_BUF_SIZE 4096 + +/* Dummy SRAM size code */ +#define SRM_NB_BSZ_ONCHIP_ONLY (-1) + +/* Be nice if these (or equiv.) were in linux/pci_regs.h, but they're not. */ +#define PCI_EXP_DEVCAP_PWR_VAL_LBN 18 +#define PCI_EXP_DEVCAP_PWR_SCL_LBN 26 +#define PCI_EXP_DEVCTL_PAYLOAD_LBN 5 +#define PCI_EXP_LNKSTA_LNK_WID 0x3f0 +#define PCI_EXP_LNKSTA_LNK_WID_LBN 4 + +#define FALCON_IS_DUAL_FUNC(efx) \ + (FALCON_REV(efx) < FALCON_REV_B0) + +/************************************************************************** + * + * Falcon hardware access + * + **************************************************************************/ + +/* Read the current event from the event queue */ +static inline efx_qword_t *falcon_event(struct efx_channel *channel, + unsigned int index) +{ + return (((efx_qword_t *) (channel->eventq.addr)) + index); +} + +/* See if an event is present + * + * We check both the high and low dword of the event for all ones. We + * wrote all ones when we cleared the event, and no valid event can + * have all ones in either its high or low dwords. This approach is + * robust against reordering. + * + * Note that using a single 64-bit comparison is incorrect; even + * though the CPU read will be atomic, the DMA write may not be. + */ +static inline int falcon_event_present(efx_qword_t *event) +{ + return (!(EFX_DWORD_IS_ALL_ONES(event->dword[0]) | + EFX_DWORD_IS_ALL_ONES(event->dword[1]))); +} + +/************************************************************************** + * + * I2C bus - this is a bit-bashing interface using GPIO pins + * Note that it uses the output enables to tristate the outputs + * SDA is the data pin and SCL is the clock + * + ************************************************************************** + */ +static void falcon_setsdascl(struct efx_i2c_interface *i2c) +{ + efx_oword_t reg; + + falcon_read(i2c->efx, ®, GPIO_CTL_REG_KER); + EFX_SET_OWORD_FIELD(reg, GPIO0_OEN, (i2c->scl ? 0 : 1)); + EFX_SET_OWORD_FIELD(reg, GPIO3_OEN, (i2c->sda ? 0 : 1)); + falcon_write(i2c->efx, ®, GPIO_CTL_REG_KER); +} + +static int falcon_getsda(struct efx_i2c_interface *i2c) +{ + efx_oword_t reg; + + falcon_read(i2c->efx, ®, GPIO_CTL_REG_KER); + return EFX_OWORD_FIELD(reg, GPIO3_IN); +} + +static int falcon_getscl(struct efx_i2c_interface *i2c) +{ + efx_oword_t reg; + + falcon_read(i2c->efx, ®, GPIO_CTL_REG_KER); + return EFX_DWORD_FIELD(reg, GPIO0_IN); +} + +static struct efx_i2c_bit_operations falcon_i2c_bit_operations = { + .setsda = falcon_setsdascl, + .setscl = falcon_setsdascl, + .getsda = falcon_getsda, + .getscl = falcon_getscl, + .udelay = 100, + .mdelay = 10, +}; + +/************************************************************************** + * + * Falcon special buffer handling + * Special buffers are used for event queues and the TX and RX + * descriptor rings. + * + *************************************************************************/ + +/* + * Initialise a Falcon special buffer + * + * This will define a buffer (previously allocated via + * falcon_alloc_special_buffer()) in Falcon's buffer table, allowing + * it to be used for event queues, descriptor rings etc. + */ +static int +falcon_init_special_buffer(struct efx_nic *efx, + struct efx_special_buffer *buffer) +{ + efx_qword_t buf_desc; + int index; + dma_addr_t dma_addr; + int i; + + EFX_BUG_ON_PARANOID(!buffer->addr); + + /* Write buffer descriptors to NIC */ + for (i = 0; i < buffer->entries; i++) { + index = buffer->index + i; + dma_addr = buffer->dma_addr + (i * 4096); + EFX_LOG(efx, "mapping special buffer %d at %llx\n", + index, (unsigned long long)dma_addr); + EFX_POPULATE_QWORD_4(buf_desc, + IP_DAT_BUF_SIZE, IP_DAT_BUF_SIZE_4K, + BUF_ADR_REGION, 0, + BUF_ADR_FBUF, (dma_addr >> 12), + BUF_OWNER_ID_FBUF, 0); + falcon_write_sram(efx, &buf_desc, index); + } + + return 0; +} + +/* Unmaps a buffer from Falcon and clears the buffer table entries */ +static void +falcon_fini_special_buffer(struct efx_nic *efx, + struct efx_special_buffer *buffer) +{ + efx_oword_t buf_tbl_upd; + unsigned int start = buffer->index; + unsigned int end = (buffer->index + buffer->entries - 1); + + if (!buffer->entries) + return; + + EFX_LOG(efx, "unmapping special buffers %d-%d\n", + buffer->index, buffer->index + buffer->entries - 1); + + EFX_POPULATE_OWORD_4(buf_tbl_upd, + BUF_UPD_CMD, 0, + BUF_CLR_CMD, 1, + BUF_CLR_END_ID, end, + BUF_CLR_START_ID, start); + falcon_write(efx, &buf_tbl_upd, BUF_TBL_UPD_REG_KER); +} + +/* + * Allocate a new Falcon special buffer + * + * This allocates memory for a new buffer, clears it and allocates a + * new buffer ID range. It does not write into Falcon's buffer table. + * + * This call will allocate 4KB buffers, since Falcon can't use 8KB + * buffers for event queues and descriptor rings. + */ +static int falcon_alloc_special_buffer(struct efx_nic *efx, + struct efx_special_buffer *buffer, + unsigned int len) +{ + struct falcon_nic_data *nic_data = efx->nic_data; + + len = ALIGN(len, FALCON_BUF_SIZE); + + buffer->addr = pci_alloc_consistent(efx->pci_dev, len, + &buffer->dma_addr); + if (!buffer->addr) + return -ENOMEM; + buffer->len = len; + buffer->entries = len / FALCON_BUF_SIZE; + BUG_ON(buffer->dma_addr & (FALCON_BUF_SIZE - 1)); + + /* All zeros is a potentially valid event so memset to 0xff */ + memset(buffer->addr, 0xff, len); + + /* Select new buffer ID */ + buffer->index = nic_data->next_buffer_table; + nic_data->next_buffer_table += buffer->entries; + + EFX_LOG(efx, "allocating special buffers %d-%d at %llx+%x " + "(virt %p phys %lx)\n", buffer->index, + buffer->index + buffer->entries - 1, + (unsigned long long)buffer->dma_addr, len, + buffer->addr, virt_to_phys(buffer->addr)); + + return 0; +} + +static void falcon_free_special_buffer(struct efx_nic *efx, + struct efx_special_buffer *buffer) +{ + if (!buffer->addr) + return; + + EFX_LOG(efx, "deallocating special buffers %d-%d at %llx+%x " + "(virt %p phys %lx)\n", buffer->index, + buffer->index + buffer->entries - 1, + (unsigned long long)buffer->dma_addr, buffer->len, + buffer->addr, virt_to_phys(buffer->addr)); + + pci_free_consistent(efx->pci_dev, buffer->len, buffer->addr, + buffer->dma_addr); + buffer->addr = NULL; + buffer->entries = 0; +} + +/************************************************************************** + * + * Falcon generic buffer handling + * These buffers are used for interrupt status and MAC stats + * + **************************************************************************/ + +static int falcon_alloc_buffer(struct efx_nic *efx, + struct efx_buffer *buffer, unsigned int len) +{ + buffer->addr = pci_alloc_consistent(efx->pci_dev, len, + &buffer->dma_addr); + if (!buffer->addr) + return -ENOMEM; + buffer->len = len; + memset(buffer->addr, 0, len); + return 0; +} + +static void falcon_free_buffer(struct efx_nic *efx, struct efx_buffer *buffer) +{ + if (buffer->addr) { + pci_free_consistent(efx->pci_dev, buffer->len, + buffer->addr, buffer->dma_addr); + buffer->addr = NULL; + } +} + +/************************************************************************** + * + * Falcon TX path + * + **************************************************************************/ + +/* Returns a pointer to the specified transmit descriptor in the TX + * descriptor queue belonging to the specified channel. + */ +static inline efx_qword_t *falcon_tx_desc(struct efx_tx_queue *tx_queue, + unsigned int index) +{ + return (((efx_qword_t *) (tx_queue->txd.addr)) + index); +} + +/* This writes to the TX_DESC_WPTR; write pointer for TX descriptor ring */ +static inline void falcon_notify_tx_desc(struct efx_tx_queue *tx_queue) +{ + unsigned write_ptr; + efx_dword_t reg; + + write_ptr = tx_queue->write_count & FALCON_TXD_RING_MASK; + EFX_POPULATE_DWORD_1(reg, TX_DESC_WPTR_DWORD, write_ptr); + falcon_writel_page(tx_queue->efx, ®, + TX_DESC_UPD_REG_KER_DWORD, tx_queue->queue); +} + + +/* For each entry inserted into the software descriptor ring, create a + * descriptor in the hardware TX descriptor ring (in host memory), and + * write a doorbell. + */ +void falcon_push_buffers(struct efx_tx_queue *tx_queue) +{ + + struct efx_tx_buffer *buffer; + efx_qword_t *txd; + unsigned write_ptr; + + BUG_ON(tx_queue->write_count == tx_queue->insert_count); + + do { + write_ptr = tx_queue->write_count & FALCON_TXD_RING_MASK; + buffer = &tx_queue->buffer[write_ptr]; + txd = falcon_tx_desc(tx_queue, write_ptr); + ++tx_queue->write_count; + + /* Create TX descriptor ring entry */ + EFX_POPULATE_QWORD_5(*txd, + TX_KER_PORT, 0, + TX_KER_CONT, buffer->continuation, + TX_KER_BYTE_CNT, buffer->len, + TX_KER_BUF_REGION, 0, + TX_KER_BUF_ADR, buffer->dma_addr); + } while (tx_queue->write_count != tx_queue->insert_count); + + wmb(); /* Ensure descriptors are written before they are fetched */ + falcon_notify_tx_desc(tx_queue); +} + +/* Allocate hardware resources for a TX queue */ +int falcon_probe_tx(struct efx_tx_queue *tx_queue) +{ + struct efx_nic *efx = tx_queue->efx; + return falcon_alloc_special_buffer(efx, &tx_queue->txd, + FALCON_TXD_RING_SIZE * + sizeof(efx_qword_t)); +} + +int falcon_init_tx(struct efx_tx_queue *tx_queue) +{ + efx_oword_t tx_desc_ptr; + struct efx_nic *efx = tx_queue->efx; + int rc; + + /* Pin TX descriptor ring */ + rc = falcon_init_special_buffer(efx, &tx_queue->txd); + if (rc) + return rc; + + /* Push TX descriptor ring to card */ + EFX_POPULATE_OWORD_10(tx_desc_ptr, + TX_DESCQ_EN, 1, + TX_ISCSI_DDIG_EN, 0, + TX_ISCSI_HDIG_EN, 0, + TX_DESCQ_BUF_BASE_ID, tx_queue->txd.index, + TX_DESCQ_EVQ_ID, tx_queue->channel->evqnum, + TX_DESCQ_OWNER_ID, 0, + TX_DESCQ_LABEL, tx_queue->queue, + TX_DESCQ_SIZE, FALCON_TXD_RING_ORDER, + TX_DESCQ_TYPE, 0, + TX_NON_IP_DROP_DIS_B0, 1); + + if (FALCON_REV(efx) >= FALCON_REV_B0) { + int csum = !(efx->net_dev->features & NETIF_F_IP_CSUM); + EFX_SET_OWORD_FIELD(tx_desc_ptr, TX_IP_CHKSM_DIS_B0, csum); + EFX_SET_OWORD_FIELD(tx_desc_ptr, TX_TCP_CHKSM_DIS_B0, csum); + } + + falcon_write_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base, + tx_queue->queue); + + if (FALCON_REV(efx) < FALCON_REV_B0) { + efx_oword_t reg; + + BUG_ON(tx_queue->queue >= 128); /* HW limit */ + + falcon_read(efx, ®, TX_CHKSM_CFG_REG_KER_A1); + if (efx->net_dev->features & NETIF_F_IP_CSUM) + clear_bit_le(tx_queue->queue, (void *)®); + else + set_bit_le(tx_queue->queue, (void *)®); + falcon_write(efx, ®, TX_CHKSM_CFG_REG_KER_A1); + } + + return 0; +} + +static int falcon_flush_tx_queue(struct efx_tx_queue *tx_queue) +{ + struct efx_nic *efx = tx_queue->efx; + struct efx_channel *channel = &efx->channel[0]; + efx_oword_t tx_flush_descq; + unsigned int read_ptr, i; + + /* Post a flush command */ + EFX_POPULATE_OWORD_2(tx_flush_descq, + TX_FLUSH_DESCQ_CMD, 1, + TX_FLUSH_DESCQ, tx_queue->queue); + falcon_write(efx, &tx_flush_descq, TX_FLUSH_DESCQ_REG_KER); + msleep(FALCON_FLUSH_TIMEOUT); + + if (EFX_WORKAROUND_7803(efx)) + return 0; + + /* Look for a flush completed event */ + read_ptr = channel->eventq_read_ptr; + for (i = 0; i < FALCON_EVQ_SIZE; ++i) { + efx_qword_t *event = falcon_event(channel, read_ptr); + int ev_code, ev_sub_code, ev_queue; + if (!falcon_event_present(event)) + break; + + ev_code = EFX_QWORD_FIELD(*event, EV_CODE); + ev_sub_code = EFX_QWORD_FIELD(*event, DRIVER_EV_SUB_CODE); + ev_queue = EFX_QWORD_FIELD(*event, DRIVER_EV_TX_DESCQ_ID); + if ((ev_sub_code == TX_DESCQ_FLS_DONE_EV_DECODE) && + (ev_queue == tx_queue->queue)) { + EFX_LOG(efx, "tx queue %d flush command succesful\n", + tx_queue->queue); + return 0; + } + + read_ptr = (read_ptr + 1) & FALCON_EVQ_MASK; + } + + if (EFX_WORKAROUND_11557(efx)) { + efx_oword_t reg; + int enabled; + + falcon_read_table(efx, ®, efx->type->txd_ptr_tbl_base, + tx_queue->queue); + enabled = EFX_OWORD_FIELD(reg, TX_DESCQ_EN); + if (!enabled) { + EFX_LOG(efx, "tx queue %d disabled without a " + "flush event seen\n", tx_queue->queue); + return 0; + } + } + + EFX_ERR(efx, "tx queue %d flush command timed out\n", tx_queue->queue); + return -ETIMEDOUT; +} + +void falcon_fini_tx(struct efx_tx_queue *tx_queue) +{ + struct efx_nic *efx = tx_queue->efx; + efx_oword_t tx_desc_ptr; + + /* Stop the hardware using the queue */ + if (falcon_flush_tx_queue(tx_queue)) + EFX_ERR(efx, "failed to flush tx queue %d\n", tx_queue->queue); + + /* Remove TX descriptor ring from card */ + EFX_ZERO_OWORD(tx_desc_ptr); + falcon_write_table(efx, &tx_desc_ptr, efx->type->txd_ptr_tbl_base, + tx_queue->queue); + + /* Unpin TX descriptor ring */ + falcon_fini_special_buffer(efx, &tx_queue->txd); +} + +/* Free buffers backing TX queue */ +void falcon_remove_tx(struct efx_tx_queue *tx_queue) +{ + falcon_free_special_buffer(tx_queue->efx, &tx_queue->txd); +} + +/************************************************************************** + * + * Falcon RX path + * + **************************************************************************/ + +/* Returns a pointer to the specified descriptor in the RX descriptor queue */ +static inline efx_qword_t *falcon_rx_desc(struct efx_rx_queue *rx_queue, + unsigned int index) +{ + return (((efx_qword_t *) (rx_queue->rxd.addr)) + index); +} + +/* This creates an entry in the RX descriptor queue */ +static inline void falcon_build_rx_desc(struct efx_rx_queue *rx_queue, + unsigned index) +{ + struct efx_rx_buffer *rx_buf; + efx_qword_t *rxd; + + rxd = falcon_rx_desc(rx_queue, index); + rx_buf = efx_rx_buffer(rx_queue, index); + EFX_POPULATE_QWORD_3(*rxd, + RX_KER_BUF_SIZE, + rx_buf->len - + rx_queue->efx->type->rx_buffer_padding, + RX_KER_BUF_REGION, 0, + RX_KER_BUF_ADR, rx_buf->dma_addr); +} + +/* This writes to the RX_DESC_WPTR register for the specified receive + * descriptor ring. + */ +void falcon_notify_rx_desc(struct efx_rx_queue *rx_queue) +{ + efx_dword_t reg; + unsigned write_ptr; + + while (rx_queue->notified_count != rx_queue->added_count) { + falcon_build_rx_desc(rx_queue, + rx_queue->notified_count & + FALCON_RXD_RING_MASK); + ++rx_queue->notified_count; + } + + wmb(); + write_ptr = rx_queue->added_count & FALCON_RXD_RING_MASK; + EFX_POPULATE_DWORD_1(reg, RX_DESC_WPTR_DWORD, write_ptr); + falcon_writel_page(rx_queue->efx, ®, + RX_DESC_UPD_REG_KER_DWORD, rx_queue->queue); +} + +int falcon_probe_rx(struct efx_rx_queue *rx_queue) +{ + struct efx_nic *efx = rx_queue->efx; + return falcon_alloc_special_buffer(efx, &rx_queue->rxd, + FALCON_RXD_RING_SIZE * + sizeof(efx_qword_t)); +} + +int falcon_init_rx(struct efx_rx_queue *rx_queue) +{ + efx_oword_t rx_desc_ptr; + struct efx_nic *efx = rx_queue->efx; + int rc; + int is_b0 = FALCON_REV(efx) >= FALCON_REV_B0; + int iscsi_digest_en = is_b0; + + EFX_LOG(efx, "RX queue %d ring in special buffers %d-%d\n", + rx_queue->queue, rx_queue->rxd.index, + rx_queue->rxd.index + rx_queue->rxd.entries - 1); + + /* Pin RX descriptor ring */ + rc = falcon_init_special_buffer(efx, &rx_queue->rxd); + if (rc) + return rc; + + /* Push RX descriptor ring to card */ + EFX_POPULATE_OWORD_10(rx_desc_ptr, + RX_ISCSI_DDIG_EN, iscsi_digest_en, + RX_ISCSI_HDIG_EN, iscsi_digest_en, + RX_DESCQ_BUF_BASE_ID, rx_queue->rxd.index, + RX_DESCQ_EVQ_ID, rx_queue->channel->evqnum, + RX_DESCQ_OWNER_ID, 0, + RX_DESCQ_LABEL, rx_queue->queue, + RX_DESCQ_SIZE, FALCON_RXD_RING_ORDER, + RX_DESCQ_TYPE, 0 /* kernel queue */ , + /* For >=B0 this is scatter so disable */ + RX_DESCQ_JUMBO, !is_b0, + RX_DESCQ_EN, 1); + falcon_write_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, + rx_queue->queue); + return 0; +} + +static int falcon_flush_rx_queue(struct efx_rx_queue *rx_queue) +{ + struct efx_nic *efx = rx_queue->efx; + struct efx_channel *channel = &efx->channel[0]; + unsigned int read_ptr, i; + efx_oword_t rx_flush_descq; + + /* Post a flush command */ + EFX_POPULATE_OWORD_2(rx_flush_descq, + RX_FLUSH_DESCQ_CMD, 1, + RX_FLUSH_DESCQ, rx_queue->queue); + falcon_write(efx, &rx_flush_descq, RX_FLUSH_DESCQ_REG_KER); + msleep(FALCON_FLUSH_TIMEOUT); + + if (EFX_WORKAROUND_7803(efx)) + return 0; + + /* Look for a flush completed event */ + read_ptr = channel->eventq_read_ptr; + for (i = 0; i < FALCON_EVQ_SIZE; ++i) { + efx_qword_t *event = falcon_event(channel, read_ptr); + int ev_code, ev_sub_code, ev_queue, ev_failed; + if (!falcon_event_present(event)) + break; + + ev_code = EFX_QWORD_FIELD(*event, EV_CODE); + ev_sub_code = EFX_QWORD_FIELD(*event, DRIVER_EV_SUB_CODE); + ev_queue = EFX_QWORD_FIELD(*event, DRIVER_EV_RX_DESCQ_ID); + ev_failed = EFX_QWORD_FIELD(*event, DRIVER_EV_RX_FLUSH_FAIL); + + if ((ev_sub_code == RX_DESCQ_FLS_DONE_EV_DECODE) && + (ev_queue == rx_queue->queue)) { + if (ev_failed) { + EFX_INFO(efx, "rx queue %d flush command " + "failed\n", rx_queue->queue); + return -EAGAIN; + } else { + EFX_LOG(efx, "rx queue %d flush command " + "succesful\n", rx_queue->queue); + return 0; + } + } + + read_ptr = (read_ptr + 1) & FALCON_EVQ_MASK; + } + + if (EFX_WORKAROUND_11557(efx)) { + efx_oword_t reg; + int enabled; + + falcon_read_table(efx, ®, efx->type->rxd_ptr_tbl_base, + rx_queue->queue); + enabled = EFX_OWORD_FIELD(reg, RX_DESCQ_EN); + if (!enabled) { + EFX_LOG(efx, "rx queue %d disabled without a " + "flush event seen\n", rx_queue->queue); + return 0; + } + } + + EFX_ERR(efx, "rx queue %d flush command timed out\n", rx_queue->queue); + return -ETIMEDOUT; +} + +void falcon_fini_rx(struct efx_rx_queue *rx_queue) +{ + efx_oword_t rx_desc_ptr; + struct efx_nic *efx = rx_queue->efx; + int i, rc; + + /* Try and flush the rx queue. This may need to be repeated */ + for (i = 0; i < 5; i++) { + rc = falcon_flush_rx_queue(rx_queue); + if (rc == -EAGAIN) + continue; + break; + } + if (rc) + EFX_ERR(efx, "failed to flush rx queue %d\n", rx_queue->queue); + + /* Remove RX descriptor ring from card */ + EFX_ZERO_OWORD(rx_desc_ptr); + falcon_write_table(efx, &rx_desc_ptr, efx->type->rxd_ptr_tbl_base, + rx_queue->queue); + + /* Unpin RX descriptor ring */ + falcon_fini_special_buffer(efx, &rx_queue->rxd); +} + +/* Free buffers backing RX queue */ +void falcon_remove_rx(struct efx_rx_queue *rx_queue) +{ + falcon_free_special_buffer(rx_queue->efx, &rx_queue->rxd); +} + +/************************************************************************** + * + * Falcon event queue processing + * Event queues are processed by per-channel tasklets. + * + **************************************************************************/ + +/* Update a channel's event queue's read pointer (RPTR) register + * + * This writes the EVQ_RPTR_REG register for the specified channel's + * event queue. + * + * Note that EVQ_RPTR_REG contains the index of the "last read" event, + * whereas channel->eventq_read_ptr contains the index of the "next to + * read" event. + */ +void falcon_eventq_read_ack(struct efx_channel *channel) +{ + efx_dword_t reg; + struct efx_nic *efx = channel->efx; + + EFX_POPULATE_DWORD_1(reg, EVQ_RPTR_DWORD, channel->eventq_read_ptr); + falcon_writel_table(efx, ®, efx->type->evq_rptr_tbl_base, + channel->evqnum); +} + +/* Use HW to insert a SW defined event */ +void falcon_generate_event(struct efx_channel *channel, efx_qword_t *event) +{ + efx_oword_t drv_ev_reg; + + EFX_POPULATE_OWORD_2(drv_ev_reg, + DRV_EV_QID, channel->evqnum, + DRV_EV_DATA, + EFX_QWORD_FIELD64(*event, WHOLE_EVENT)); + falcon_write(channel->efx, &drv_ev_reg, DRV_EV_REG_KER); +} + +/* Handle a transmit completion event + * + * Falcon batches TX completion events; the message we receive is of + * the form "complete all TX events up to this index". + */ +static inline void falcon_handle_tx_event(struct efx_channel *channel, + efx_qword_t *event) +{ + unsigned int tx_ev_desc_ptr; + unsigned int tx_ev_q_label; + struct efx_tx_queue *tx_queue; + struct efx_nic *efx = channel->efx; + + if (likely(EFX_QWORD_FIELD(*event, TX_EV_COMP))) { + /* Transmit completion */ + tx_ev_desc_ptr = EFX_QWORD_FIELD(*event, TX_EV_DESC_PTR); + tx_ev_q_label = EFX_QWORD_FIELD(*event, TX_EV_Q_LABEL); + tx_queue = &efx->tx_queue[tx_ev_q_label]; + efx_xmit_done(tx_queue, tx_ev_desc_ptr); + } else if (EFX_QWORD_FIELD(*event, TX_EV_WQ_FF_FULL)) { + /* Rewrite the FIFO write pointer */ + tx_ev_q_label = EFX_QWORD_FIELD(*event, TX_EV_Q_LABEL); + tx_queue = &efx->tx_queue[tx_ev_q_label]; + + if (NET_DEV_REGISTERED(efx)) + netif_tx_lock(efx->net_dev); + falcon_notify_tx_desc(tx_queue); + if (NET_DEV_REGISTERED(efx)) + netif_tx_unlock(efx->net_dev); + } else if (EFX_QWORD_FIELD(*event, TX_EV_PKT_ERR) && + EFX_WORKAROUND_10727(efx)) { + efx_schedule_reset(efx, RESET_TYPE_TX_DESC_FETCH); + } else { + EFX_ERR(efx, "channel %d unexpected TX event " + EFX_QWORD_FMT"\n", channel->channel, + EFX_QWORD_VAL(*event)); + } +} + +/* Check received packet's destination MAC address. */ +static int check_dest_mac(struct efx_rx_queue *rx_queue, + const efx_qword_t *event) +{ + struct efx_rx_buffer *rx_buf; + struct efx_nic *efx = rx_queue->efx; + int rx_ev_desc_ptr; + struct ethhdr *eh; + + if (efx->promiscuous) + return 1; + + rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, RX_EV_DESC_PTR); + rx_buf = efx_rx_buffer(rx_queue, rx_ev_desc_ptr); + eh = (struct ethhdr *)rx_buf->data; + if (memcmp(eh->h_dest, efx->net_dev->dev_addr, ETH_ALEN)) + return 0; + return 1; +} + +/* Detect errors included in the rx_evt_pkt_ok bit. */ +static void falcon_handle_rx_not_ok(struct efx_rx_queue *rx_queue, + const efx_qword_t *event, + unsigned *rx_ev_pkt_ok, + int *discard, int byte_count) +{ + struct efx_nic *efx = rx_queue->efx; + unsigned rx_ev_buf_owner_id_err, rx_ev_ip_hdr_chksum_err; + unsigned rx_ev_tcp_udp_chksum_err, rx_ev_eth_crc_err; + unsigned rx_ev_frm_trunc, rx_ev_drib_nib, rx_ev_tobe_disc; + unsigned rx_ev_pkt_type, rx_ev_other_err, rx_ev_pause_frm; + unsigned rx_ev_ip_frag_err, rx_ev_hdr_type, rx_ev_mcast_pkt; + int snap, non_ip; + + rx_ev_hdr_type = EFX_QWORD_FIELD(*event, RX_EV_HDR_TYPE); + rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, RX_EV_MCAST_PKT); + rx_ev_tobe_disc = EFX_QWORD_FIELD(*event, RX_EV_TOBE_DISC); + rx_ev_pkt_type = EFX_QWORD_FIELD(*event, RX_EV_PKT_TYPE); + rx_ev_buf_owner_id_err = EFX_QWORD_FIELD(*event, + RX_EV_BUF_OWNER_ID_ERR); + rx_ev_ip_frag_err = EFX_QWORD_FIELD(*event, RX_EV_IF_FRAG_ERR); + rx_ev_ip_hdr_chksum_err = EFX_QWORD_FIELD(*event, + RX_EV_IP_HDR_CHKSUM_ERR); + rx_ev_tcp_udp_chksum_err = EFX_QWORD_FIELD(*event, + RX_EV_TCP_UDP_CHKSUM_ERR); + rx_ev_eth_crc_err = EFX_QWORD_FIELD(*event, RX_EV_ETH_CRC_ERR); + rx_ev_frm_trunc = EFX_QWORD_FIELD(*event, RX_EV_FRM_TRUNC); + rx_ev_drib_nib = ((FALCON_REV(efx) >= FALCON_REV_B0) ? + 0 : EFX_QWORD_FIELD(*event, RX_EV_DRIB_NIB)); + rx_ev_pause_frm = EFX_QWORD_FIELD(*event, RX_EV_PAUSE_FRM_ERR); + + /* Every error apart from tobe_disc and pause_frm */ + rx_ev_other_err = (rx_ev_drib_nib | rx_ev_tcp_udp_chksum_err | + rx_ev_buf_owner_id_err | rx_ev_eth_crc_err | + rx_ev_frm_trunc | rx_ev_ip_hdr_chksum_err); + + snap = (rx_ev_pkt_type == RX_EV_PKT_TYPE_LLC_DECODE) || + (rx_ev_pkt_type == RX_EV_PKT_TYPE_VLAN_LLC_DECODE); + non_ip = (rx_ev_hdr_type == RX_EV_HDR_TYPE_NON_IP_DECODE); + + /* SFC bug 5475/8970: The Falcon XMAC incorrectly calculates the + * length field of an LLC frame, which sets TOBE_DISC. We could set + * PASS_LEN_ERR, but we want the MAC to filter out short frames (to + * protect the RX block). + * + * bug5475 - LLC/SNAP: Falcon identifies SNAP packets. + * bug8970 - LLC/noSNAP: Falcon does not provide an LLC flag. + * LLC can't encapsulate IP, so by definition + * these packets are NON_IP. + * + * Unicast mismatch will also cause TOBE_DISC, so the driver needs + * to check this. + */ + if (EFX_WORKAROUND_5475(efx) && rx_ev_tobe_disc && (snap || non_ip)) { + /* If all the other flags are zero then we can state the + * entire packet is ok, which will flag to the kernel not + * to recalculate checksums. + */ + if (!(non_ip | rx_ev_other_err | rx_ev_pause_frm)) + *rx_ev_pkt_ok = 1; + + rx_ev_tobe_disc = 0; + + /* TOBE_DISC is set for unicast mismatch. But given that + * we can't trust TOBE_DISC here, we must validate the dest + * MAC address ourselves. + */ + if (!rx_ev_mcast_pkt && !check_dest_mac(rx_queue, event)) + rx_ev_tobe_disc = 1; + } + + /* Count errors that are not in MAC stats. */ + if (rx_ev_frm_trunc) + ++rx_queue->channel->n_rx_frm_trunc; + else if (rx_ev_tobe_disc) + ++rx_queue->channel->n_rx_tobe_disc; + else if (rx_ev_ip_hdr_chksum_err) + ++rx_queue->channel->n_rx_ip_hdr_chksum_err; + else if (rx_ev_tcp_udp_chksum_err) + ++rx_queue->channel->n_rx_tcp_udp_chksum_err; + if (rx_ev_ip_frag_err) + ++rx_queue->channel->n_rx_ip_frag_err; + + /* The frame must be discarded if any of these are true. */ + *discard = (rx_ev_eth_crc_err | rx_ev_frm_trunc | rx_ev_drib_nib | + rx_ev_tobe_disc | rx_ev_pause_frm); + + /* TOBE_DISC is expected on unicast mismatches; don't print out an + * error message. FRM_TRUNC indicates RXDP dropped the packet due + * to a FIFO overflow. + */ +#ifdef EFX_ENABLE_DEBUG + if (rx_ev_other_err) { + EFX_INFO_RL(efx, " RX queue %d unexpected RX event " + EFX_QWORD_FMT "%s%s%s%s%s%s%s%s%s\n", + rx_queue->queue, EFX_QWORD_VAL(*event), + rx_ev_buf_owner_id_err ? " [OWNER_ID_ERR]" : "", + rx_ev_ip_hdr_chksum_err ? + " [IP_HDR_CHKSUM_ERR]" : "", + rx_ev_tcp_udp_chksum_err ? + " [TCP_UDP_CHKSUM_ERR]" : "", + rx_ev_eth_crc_err ? " [ETH_CRC_ERR]" : "", + rx_ev_frm_trunc ? " [FRM_TRUNC]" : "", + rx_ev_drib_nib ? " [DRIB_NIB]" : "", + rx_ev_tobe_disc ? " [TOBE_DISC]" : "", + rx_ev_pause_frm ? " [PAUSE]" : "", + snap ? " [SNAP/LLC]" : ""); + } +#endif + + if (unlikely(rx_ev_eth_crc_err && EFX_WORKAROUND_10750(efx) && + efx->phy_type == PHY_TYPE_10XPRESS)) + tenxpress_crc_err(efx); +} + +/* Handle receive events that are not in-order. */ +static void falcon_handle_rx_bad_index(struct efx_rx_queue *rx_queue, + unsigned index) +{ + struct efx_nic *efx = rx_queue->efx; + unsigned expected, dropped; + + expected = rx_queue->removed_count & FALCON_RXD_RING_MASK; + dropped = ((index + FALCON_RXD_RING_SIZE - expected) & + FALCON_RXD_RING_MASK); + EFX_INFO(efx, "dropped %d events (index=%d expected=%d)\n", + dropped, index, expected); + + efx_schedule_reset(efx, EFX_WORKAROUND_5676(efx) ? + RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE); +} + +/* Handle a packet received event + * + * Falcon silicon gives a "discard" flag if it's a unicast packet with the + * wrong destination address + * Also "is multicast" and "matches multicast filter" flags can be used to + * discard non-matching multicast packets. + */ +static inline int falcon_handle_rx_event(struct efx_channel *channel, + const efx_qword_t *event) +{ + unsigned int rx_ev_q_label, rx_ev_desc_ptr, rx_ev_byte_cnt; + unsigned int rx_ev_pkt_ok, rx_ev_hdr_type, rx_ev_mcast_pkt; + unsigned expected_ptr; + int discard = 0, checksummed; + struct efx_rx_queue *rx_queue; + struct efx_nic *efx = channel->efx; + + /* Basic packet information */ + rx_ev_byte_cnt = EFX_QWORD_FIELD(*event, RX_EV_BYTE_CNT); + rx_ev_pkt_ok = EFX_QWORD_FIELD(*event, RX_EV_PKT_OK); + rx_ev_hdr_type = EFX_QWORD_FIELD(*event, RX_EV_HDR_TYPE); + WARN_ON(EFX_QWORD_FIELD(*event, RX_EV_JUMBO_CONT)); + WARN_ON(EFX_QWORD_FIELD(*event, RX_EV_SOP) != 1); + + rx_ev_q_label = EFX_QWORD_FIELD(*event, RX_EV_Q_LABEL); + rx_queue = &efx->rx_queue[rx_ev_q_label]; + + rx_ev_desc_ptr = EFX_QWORD_FIELD(*event, RX_EV_DESC_PTR); + expected_ptr = rx_queue->removed_count & FALCON_RXD_RING_MASK; + if (unlikely(rx_ev_desc_ptr != expected_ptr)) { + falcon_handle_rx_bad_index(rx_queue, rx_ev_desc_ptr); + return rx_ev_q_label; + } + + if (likely(rx_ev_pkt_ok)) { + /* If packet is marked as OK and packet type is TCP/IPv4 or + * UDP/IPv4, then we can rely on the hardware checksum. + */ + checksummed = RX_EV_HDR_TYPE_HAS_CHECKSUMS(rx_ev_hdr_type); + } else { + falcon_handle_rx_not_ok(rx_queue, event, &rx_ev_pkt_ok, + &discard, rx_ev_byte_cnt); + checksummed = 0; + } + + /* Detect multicast packets that didn't match the filter */ + rx_ev_mcast_pkt = EFX_QWORD_FIELD(*event, RX_EV_MCAST_PKT); + if (rx_ev_mcast_pkt) { + unsigned int rx_ev_mcast_hash_match = + EFX_QWORD_FIELD(*event, RX_EV_MCAST_HASH_MATCH); + + if (unlikely(!rx_ev_mcast_hash_match)) + discard = 1; + } + + /* Handle received packet */ + efx_rx_packet(rx_queue, rx_ev_desc_ptr, rx_ev_byte_cnt, + checksummed, discard); + + return rx_ev_q_label; +} + +/* Global events are basically PHY events */ +static void falcon_handle_global_event(struct efx_channel *channel, + efx_qword_t *event) +{ + struct efx_nic *efx = channel->efx; + int is_phy_event = 0, handled = 0; + + /* Check for interrupt on either port. Some boards have a + * single PHY wired to the interrupt line for port 1. */ + if (EFX_QWORD_FIELD(*event, G_PHY0_INTR) || + EFX_QWORD_FIELD(*event, G_PHY1_INTR) || + EFX_QWORD_FIELD(*event, XG_PHY_INTR)) + is_phy_event = 1; + + if ((FALCON_REV(efx) >= FALCON_REV_B0) && + EFX_OWORD_FIELD(*event, XG_MNT_INTR_B0)) + is_phy_event = 1; + + if (is_phy_event) { + efx->phy_op->clear_interrupt(efx); + queue_work(efx->workqueue, &efx->reconfigure_work); + handled = 1; + } + + if (EFX_QWORD_FIELD_VER(efx, *event, RX_RECOVERY)) { + EFX_ERR(efx, "channel %d seen global RX_RESET " + "event. Resetting.\n", channel->channel); + + atomic_inc(&efx->rx_reset); + efx_schedule_reset(efx, EFX_WORKAROUND_6555(efx) ? + RESET_TYPE_RX_RECOVERY : RESET_TYPE_DISABLE); + handled = 1; + } + + if (!handled) + EFX_ERR(efx, "channel %d unknown global event " + EFX_QWORD_FMT "\n", channel->channel, + EFX_QWORD_VAL(*event)); +} + +static vo |