diff options
Diffstat (limited to 'drivers/net/sfc')
31 files changed, 650 insertions, 352 deletions
diff --git a/drivers/net/sfc/efx.c b/drivers/net/sfc/efx.c index 002bac74384..a3c2aab53de 100644 --- a/drivers/net/sfc/efx.c +++ b/drivers/net/sfc/efx.c @@ -1,7 +1,7 @@ /**************************************************************************** * Driver for Solarflare Solarstorm network controllers and boards * Copyright 2005-2006 Fen Systems Ltd. - * Copyright 2005-2009 Solarflare Communications Inc. + * Copyright 2005-2011 Solarflare Communications Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -21,6 +21,7 @@ #include <linux/ethtool.h> #include <linux/topology.h> #include <linux/gfp.h> +#include <linux/cpu_rmap.h> #include "net_driver.h" #include "efx.h" #include "nic.h" @@ -307,6 +308,8 @@ static int efx_poll(struct napi_struct *napi, int budget) channel->irq_mod_score = 0; } + efx_filter_rfs_expire(channel); + /* There is no race here; although napi_disable() will * only wait for napi_complete(), this isn't a problem * since efx_channel_processed() will have no effect if @@ -325,7 +328,8 @@ static int efx_poll(struct napi_struct *napi, int budget) * processing to finish, then directly poll (and ack ) the eventq. * Finally reenable NAPI and interrupts. * - * Since we are touching interrupts the caller should hold the suspend lock + * This is for use only during a loopback self-test. It must not + * deliver any packets up the stack as this can result in deadlock. */ void efx_process_channel_now(struct efx_channel *channel) { @@ -333,6 +337,7 @@ void efx_process_channel_now(struct efx_channel *channel) BUG_ON(channel->channel >= efx->n_channels); BUG_ON(!channel->enabled); + BUG_ON(!efx->loopback_selftest); /* Disable interrupts and wait for ISRs to complete */ efx_nic_disable_interrupts(efx); @@ -673,7 +678,7 @@ static void efx_fini_channels(struct efx_nic *efx) efx_for_each_channel_rx_queue(rx_queue, channel) efx_fini_rx_queue(rx_queue); - efx_for_each_channel_tx_queue(tx_queue, channel) + efx_for_each_possible_channel_tx_queue(tx_queue, channel) efx_fini_tx_queue(tx_queue); efx_fini_eventq(channel); } @@ -689,7 +694,7 @@ static void efx_remove_channel(struct efx_channel *channel) efx_for_each_channel_rx_queue(rx_queue, channel) efx_remove_rx_queue(rx_queue); - efx_for_each_channel_tx_queue(tx_queue, channel) + efx_for_each_possible_channel_tx_queue(tx_queue, channel) efx_remove_tx_queue(tx_queue); efx_remove_eventq(channel); } @@ -1051,6 +1056,7 @@ static int efx_init_io(struct efx_nic *efx) { struct pci_dev *pci_dev = efx->pci_dev; dma_addr_t dma_mask = efx->type->max_dma_mask; + bool use_wc; int rc; netif_dbg(efx, probe, efx->net_dev, "initialising I/O\n"); @@ -1101,8 +1107,21 @@ static int efx_init_io(struct efx_nic *efx) rc = -EIO; goto fail3; } - efx->membase = ioremap_nocache(efx->membase_phys, - efx->type->mem_map_size); + + /* bug22643: If SR-IOV is enabled then tx push over a write combined + * mapping is unsafe. We need to disable write combining in this case. + * MSI is unsupported when SR-IOV is enabled, and the firmware will + * have removed the MSI capability. So write combining is safe if + * there is an MSI capability. + */ + use_wc = (!EFX_WORKAROUND_22643(efx) || + pci_find_capability(pci_dev, PCI_CAP_ID_MSI)); + if (use_wc) + efx->membase = ioremap_wc(efx->membase_phys, + efx->type->mem_map_size); + else + efx->membase = ioremap_nocache(efx->membase_phys, + efx->type->mem_map_size); if (!efx->membase) { netif_err(efx, probe, efx->net_dev, "could not map memory BAR at %llx+%x\n", @@ -1175,10 +1194,32 @@ static int efx_wanted_channels(void) return count; } +static int +efx_init_rx_cpu_rmap(struct efx_nic *efx, struct msix_entry *xentries) +{ +#ifdef CONFIG_RFS_ACCEL + int i, rc; + + efx->net_dev->rx_cpu_rmap = alloc_irq_cpu_rmap(efx->n_rx_channels); + if (!efx->net_dev->rx_cpu_rmap) + return -ENOMEM; + for (i = 0; i < efx->n_rx_channels; i++) { + rc = irq_cpu_rmap_add(efx->net_dev->rx_cpu_rmap, + xentries[i].vector); + if (rc) { + free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap); + efx->net_dev->rx_cpu_rmap = NULL; + return rc; + } + } +#endif + return 0; +} + /* Probe the number and type of interrupts we are able to obtain, and * the resulting numbers of channels and RX queues. */ -static void efx_probe_interrupts(struct efx_nic *efx) +static int efx_probe_interrupts(struct efx_nic *efx) { int max_channels = min_t(int, efx->type->phys_addr_channels, EFX_MAX_CHANNELS); @@ -1220,6 +1261,11 @@ static void efx_probe_interrupts(struct efx_nic *efx) efx->n_tx_channels = efx->n_channels; efx->n_rx_channels = efx->n_channels; } + rc = efx_init_rx_cpu_rmap(efx, xentries); + if (rc) { + pci_disable_msix(efx->pci_dev); + return rc; + } for (i = 0; i < n_channels; i++) efx_get_channel(efx, i)->irq = xentries[i].vector; @@ -1253,6 +1299,8 @@ static void efx_probe_interrupts(struct efx_nic *efx) efx->n_tx_channels = 1; efx->legacy_irq = efx->pci_dev->irq; } + + return 0; } static void efx_remove_interrupts(struct efx_nic *efx) @@ -1271,21 +1319,8 @@ static void efx_remove_interrupts(struct efx_nic *efx) static void efx_set_channels(struct efx_nic *efx) { - struct efx_channel *channel; - struct efx_tx_queue *tx_queue; - efx->tx_channel_offset = separate_tx_channels ? efx->n_channels - efx->n_tx_channels : 0; - - /* Channel pointers were set in efx_init_struct() but we now - * need to clear them for TX queues in any RX-only channels. */ - efx_for_each_channel(channel, efx) { - if (channel->channel - efx->tx_channel_offset >= - efx->n_tx_channels) { - efx_for_each_channel_tx_queue(tx_queue, channel) - tx_queue->channel = NULL; - } - } } static int efx_probe_nic(struct efx_nic *efx) @@ -1302,7 +1337,9 @@ static int efx_probe_nic(struct efx_nic *efx) /* Determine the number of channels and queues by trying to hook * in MSI-X interrupts. */ - efx_probe_interrupts(efx); + rc = efx_probe_interrupts(efx); + if (rc) + goto fail; if (efx->n_channels > 1) get_random_bytes(&efx->rx_hash_key, sizeof(efx->rx_hash_key)); @@ -1317,6 +1354,10 @@ static int efx_probe_nic(struct efx_nic *efx) efx_init_irq_moderation(efx, tx_irq_mod_usec, rx_irq_mod_usec, true); return 0; + +fail: + efx->type->remove(efx); + return rc; } static void efx_remove_nic(struct efx_nic *efx) @@ -1397,7 +1438,7 @@ static void efx_start_all(struct efx_nic *efx) * restart the transmit interface early so the watchdog timer stops */ efx_start_port(efx); - if (efx_dev_registered(efx)) + if (efx_dev_registered(efx) && !efx->port_inhibited) netif_tx_wake_all_queues(efx->net_dev); efx_for_each_channel(channel, efx) @@ -1531,9 +1572,9 @@ void efx_init_irq_moderation(struct efx_nic *efx, int tx_usecs, int rx_usecs, efx->irq_rx_adaptive = rx_adaptive; efx->irq_rx_moderation = rx_ticks; efx_for_each_channel(channel, efx) { - if (efx_channel_get_rx_queue(channel)) + if (efx_channel_has_rx_queue(channel)) channel->irq_moderation = rx_ticks; - else if (efx_channel_get_tx_queue(channel, 0)) + else if (efx_channel_has_tx_queues(channel)) channel->irq_moderation = tx_ticks; } } @@ -1849,6 +1890,10 @@ static const struct net_device_ops efx_netdev_ops = { #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = efx_netpoll, #endif + .ndo_setup_tc = efx_setup_tc, +#ifdef CONFIG_RFS_ACCEL + .ndo_rx_flow_steer = efx_filter_rfs, +#endif }; static void efx_update_name(struct efx_nic *efx) @@ -1910,10 +1955,8 @@ static int efx_register_netdev(struct efx_nic *efx) efx_for_each_channel(channel, efx) { struct efx_tx_queue *tx_queue; - efx_for_each_channel_tx_queue(tx_queue, channel) { - tx_queue->core_txq = netdev_get_tx_queue( - efx->net_dev, tx_queue->queue / EFX_TXQ_TYPES); - } + efx_for_each_channel_tx_queue(tx_queue, channel) + efx_init_tx_queue_core_txq(tx_queue); } /* Always start with carrier off; PHY events will detect the link */ @@ -2288,6 +2331,10 @@ static void efx_fini_struct(struct efx_nic *efx) */ static void efx_pci_remove_main(struct efx_nic *efx) { +#ifdef CONFIG_RFS_ACCEL + free_irq_cpu_rmap(efx->net_dev->rx_cpu_rmap); + efx->net_dev->rx_cpu_rmap = NULL; +#endif efx_nic_fini_interrupt(efx); efx_fini_channels(efx); efx_fini_port(efx); @@ -2401,7 +2448,8 @@ static int __devinit efx_pci_probe(struct pci_dev *pci_dev, int i, rc; /* Allocate and initialise a struct net_device and struct efx_nic */ - net_dev = alloc_etherdev_mq(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES); + net_dev = alloc_etherdev_mqs(sizeof(*efx), EFX_MAX_CORE_TX_QUEUES, + EFX_MAX_RX_QUEUES); if (!net_dev) return -ENOMEM; net_dev->features |= (type->offload_features | NETIF_F_SG | diff --git a/drivers/net/sfc/efx.h b/drivers/net/sfc/efx.h index d43a7e5212b..3d83a1f74fe 100644 --- a/drivers/net/sfc/efx.h +++ b/drivers/net/sfc/efx.h @@ -1,7 +1,7 @@ /**************************************************************************** * Driver for Solarflare Solarstorm network controllers and boards * Copyright 2005-2006 Fen Systems Ltd. - * Copyright 2006-2009 Solarflare Communications Inc. + * Copyright 2006-2010 Solarflare Communications Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -29,6 +29,7 @@ extern int efx_probe_tx_queue(struct efx_tx_queue *tx_queue); extern void efx_remove_tx_queue(struct efx_tx_queue *tx_queue); extern void efx_init_tx_queue(struct efx_tx_queue *tx_queue); +extern void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue); extern void efx_fini_tx_queue(struct efx_tx_queue *tx_queue); extern void efx_release_tx_buffers(struct efx_tx_queue *tx_queue); extern netdev_tx_t @@ -36,6 +37,7 @@ efx_hard_start_xmit(struct sk_buff *skb, struct net_device *net_dev); extern netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb); extern void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index); +extern int efx_setup_tc(struct net_device *net_dev, u8 num_tc); /* RX */ extern int efx_probe_rx_queue(struct efx_rx_queue *rx_queue); @@ -74,6 +76,21 @@ extern int efx_filter_remove_filter(struct efx_nic *efx, struct efx_filter_spec *spec); extern void efx_filter_clear_rx(struct efx_nic *efx, enum efx_filter_priority priority); +#ifdef CONFIG_RFS_ACCEL +extern int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, + u16 rxq_index, u32 flow_id); +extern bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota); +static inline void efx_filter_rfs_expire(struct efx_channel *channel) +{ + if (channel->rfs_filters_added >= 60 && + __efx_filter_rfs_expire(channel->efx, 100)) + channel->rfs_filters_added -= 60; +} +#define efx_filter_rfs_enabled() 1 +#else +static inline void efx_filter_rfs_expire(struct efx_channel *channel) {} +#define efx_filter_rfs_enabled() 0 +#endif /* Channels */ extern void efx_process_channel_now(struct efx_channel *channel); diff --git a/drivers/net/sfc/ethtool.c b/drivers/net/sfc/ethtool.c index ca886d98bdc..807178ef65a 100644 --- a/drivers/net/sfc/ethtool.c +++ b/drivers/net/sfc/ethtool.c @@ -1,7 +1,7 @@ /**************************************************************************** * Driver for Solarflare Solarstorm network controllers and boards * Copyright 2005-2006 Fen Systems Ltd. - * Copyright 2006-2009 Solarflare Communications Inc. + * Copyright 2006-2010 Solarflare Communications Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -28,7 +28,8 @@ struct efx_ethtool_stat { enum { EFX_ETHTOOL_STAT_SOURCE_mac_stats, EFX_ETHTOOL_STAT_SOURCE_nic, - EFX_ETHTOOL_STAT_SOURCE_channel + EFX_ETHTOOL_STAT_SOURCE_channel, + EFX_ETHTOOL_STAT_SOURCE_tx_queue } source; unsigned offset; u64(*get_stat) (void *field); /* Reader function */ @@ -86,6 +87,10 @@ static u64 efx_get_atomic_stat(void *field) EFX_ETHTOOL_STAT(field, channel, n_##field, \ unsigned int, efx_get_uint_stat) +#define EFX_ETHTOOL_UINT_TXQ_STAT(field) \ + EFX_ETHTOOL_STAT(tx_##field, tx_queue, field, \ + unsigned int, efx_get_uint_stat) + static struct efx_ethtool_stat efx_ethtool_stats[] = { EFX_ETHTOOL_U64_MAC_STAT(tx_bytes), EFX_ETHTOOL_U64_MAC_STAT(tx_good_bytes), @@ -116,6 +121,10 @@ static struct efx_ethtool_stat efx_ethtool_stats[] = { EFX_ETHTOOL_ULONG_MAC_STAT(tx_non_tcpudp), EFX_ETHTOOL_ULONG_MAC_STAT(tx_mac_src_error), EFX_ETHTOOL_ULONG_MAC_STAT(tx_ip_src_error), + EFX_ETHTOOL_UINT_TXQ_STAT(tso_bursts), + EFX_ETHTOOL_UINT_TXQ_STAT(tso_long_headers), + EFX_ETHTOOL_UINT_TXQ_STAT(tso_packets), + EFX_ETHTOOL_UINT_TXQ_STAT(pushes), EFX_ETHTOOL_U64_MAC_STAT(rx_bytes), EFX_ETHTOOL_U64_MAC_STAT(rx_good_bytes), EFX_ETHTOOL_U64_MAC_STAT(rx_bad_bytes), @@ -237,8 +246,8 @@ static void efx_ethtool_get_drvinfo(struct net_device *net_dev, strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); strlcpy(info->version, EFX_DRIVER_VERSION, sizeof(info->version)); if (efx_nic_rev(efx) >= EFX_REV_SIENA_A0) - siena_print_fwver(efx, info->fw_version, - sizeof(info->fw_version)); + efx_mcdi_print_fwver(efx, info->fw_version, + sizeof(info->fw_version)); strlcpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info)); } @@ -470,6 +479,7 @@ static void efx_ethtool_get_stats(struct net_device *net_dev, struct efx_mac_stats *mac_stats = &efx->mac_stats; struct efx_ethtool_stat *stat; struct efx_channel *channel; + struct efx_tx_queue *tx_queue; struct rtnl_link_stats64 temp; int i; @@ -495,6 +505,15 @@ static void efx_ethtool_get_stats(struct net_device *net_dev, data[i] += stat->get_stat((void *)channel + stat->offset); break; + case EFX_ETHTOOL_STAT_SOURCE_tx_queue: + data[i] = 0; + efx_for_each_channel(channel, efx) { + efx_for_each_channel_tx_queue(tx_queue, channel) + data[i] += + stat->get_stat((void *)tx_queue + + stat->offset); + } + break; } } } @@ -502,7 +521,7 @@ static void efx_ethtool_get_stats(struct net_device *net_dev, static int efx_ethtool_set_tso(struct net_device *net_dev, u32 enable) { struct efx_nic *efx __attribute__ ((unused)) = netdev_priv(net_dev); - unsigned long features; + u32 features; features = NETIF_F_TSO; if (efx->type->offload_features & NETIF_F_V6_CSUM) @@ -519,7 +538,7 @@ static int efx_ethtool_set_tso(struct net_device *net_dev, u32 enable) static int efx_ethtool_set_tx_csum(struct net_device *net_dev, u32 enable) { struct efx_nic *efx = netdev_priv(net_dev); - unsigned long features = efx->type->offload_features & NETIF_F_ALL_CSUM; + u32 features = efx->type->offload_features & NETIF_F_ALL_CSUM; if (enable) net_dev->features |= features; @@ -635,7 +654,7 @@ static int efx_ethtool_get_coalesce(struct net_device *net_dev, /* Find lowest IRQ moderation across all used TX queues */ coalesce->tx_coalesce_usecs_irq = ~((u32) 0); efx_for_each_channel(channel, efx) { - if (!efx_channel_get_tx_queue(channel, 0)) + if (!efx_channel_has_tx_queues(channel)) continue; if (channel->irq_moderation < coalesce->tx_coalesce_usecs_irq) { if (channel->channel < efx->n_rx_channels) @@ -680,8 +699,8 @@ static int efx_ethtool_set_coalesce(struct net_device *net_dev, /* If the channel is shared only allow RX parameters to be set */ efx_for_each_channel(channel, efx) { - if (efx_channel_get_rx_queue(channel) && - efx_channel_get_tx_queue(channel, 0) && + if (efx_channel_has_rx_queue(channel) && + efx_channel_has_tx_queues(channel) && tx_usecs) { netif_err(efx, drv, efx->net_dev, "Channel is shared. " "Only RX coalescing may be set\n"); diff --git a/drivers/net/sfc/falcon.c b/drivers/net/sfc/falcon.c index 61ddd2c6e75..d96b23769bd 100644 --- a/drivers/net/sfc/falcon.c +++ b/drivers/net/sfc/falcon.c @@ -1,7 +1,7 @@ /**************************************************************************** * Driver for Solarflare Solarstorm network controllers and boards * Copyright 2005-2006 Fen Systems Ltd. - * Copyright 2006-2009 Solarflare Communications Inc. + * Copyright 2006-2010 Solarflare Communications Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -692,7 +692,7 @@ static int falcon_gmii_wait(struct efx_nic *efx) efx_oword_t md_stat; int count; - /* wait upto 50ms - taken max from datasheet */ + /* wait up to 50ms - taken max from datasheet */ for (count = 0; count < 5000; count++) { efx_reado(efx, &md_stat, FR_AB_MD_STAT); if (EFX_OWORD_FIELD(md_stat, FRF_AB_MD_BSY) == 0) { @@ -1221,7 +1221,7 @@ static int falcon_reset_sram(struct efx_nic *efx) return 0; } - } while (++count < 20); /* wait upto 0.4 sec */ + } while (++count < 20); /* wait up to 0.4 sec */ netif_err(efx, hw, efx->net_dev, "timed out waiting for SRAM reset\n"); return -ETIMEDOUT; @@ -1478,36 +1478,26 @@ static void falcon_init_rx_cfg(struct efx_nic *efx) /* RX control FIFO thresholds (32 entries) */ const unsigned ctrl_xon_thr = 20; const unsigned ctrl_xoff_thr = 25; - /* RX data FIFO thresholds (256-byte units; size varies) */ - int data_xon_thr = efx_nic_rx_xon_thresh >> 8; - int data_xoff_thr = efx_nic_rx_xoff_thresh >> 8; efx_oword_t reg; efx_reado(efx, ®, FR_AZ_RX_CFG); if (efx_nic_rev(efx) <= EFX_REV_FALCON_A1) { /* Data FIFO size is 5.5K */ - if (data_xon_thr < 0) - data_xon_thr = 512 >> 8; - if (data_xoff_thr < 0) - data_xoff_thr = 2048 >> 8; EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_DESC_PUSH_EN, 0); EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_USR_BUF_SIZE, huge_buf_size); - EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XON_MAC_TH, data_xon_thr); - EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XOFF_MAC_TH, data_xoff_thr); + EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XON_MAC_TH, 512 >> 8); + EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XOFF_MAC_TH, 2048 >> 8); EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XON_TX_TH, ctrl_xon_thr); EFX_SET_OWORD_FIELD(reg, FRF_AA_RX_XOFF_TX_TH, ctrl_xoff_thr); } else { /* Data FIFO size is 80K; register fields moved */ - if (data_xon_thr < 0) - data_xon_thr = 27648 >> 8; /* ~3*max MTU */ - if (data_xoff_thr < 0) - data_xoff_thr = 54272 >> 8; /* ~80Kb - 3*max MTU */ EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_DESC_PUSH_EN, 0); EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_USR_BUF_SIZE, huge_buf_size); - EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XON_MAC_TH, data_xon_thr); - EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XOFF_MAC_TH, data_xoff_thr); + /* Send XON and XOFF at ~3 * max MTU away from empty/full */ + EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XON_MAC_TH, 27648 >> 8); + EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XOFF_MAC_TH, 54272 >> 8); EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XON_TX_TH, ctrl_xon_thr); EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_XOFF_TX_TH, ctrl_xoff_thr); EFX_SET_OWORD_FIELD(reg, FRF_BZ_RX_INGR_EN, 1); diff --git a/drivers/net/sfc/falcon_boards.c b/drivers/net/sfc/falcon_boards.c index 2dd16f0b3ce..b9cc846811d 100644 --- a/drivers/net/sfc/falcon_boards.c +++ b/drivers/net/sfc/falcon_boards.c @@ -1,6 +1,6 @@ /**************************************************************************** * Driver for Solarflare Solarstorm network controllers and boards - * Copyright 2007-2009 Solarflare Communications Inc. + * Copyright 2007-2010 Solarflare Communications Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published diff --git a/drivers/net/sfc/falcon_xmac.c b/drivers/net/sfc/falcon_xmac.c index b49e8439464..2c9ee5db3bf 100644 --- a/drivers/net/sfc/falcon_xmac.c +++ b/drivers/net/sfc/falcon_xmac.c @@ -1,7 +1,7 @@ /**************************************************************************** * Driver for Solarflare Solarstorm network controllers and boards * Copyright 2005-2006 Fen Systems Ltd. - * Copyright 2006-2009 Solarflare Communications Inc. + * Copyright 2006-2010 Solarflare Communications Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published diff --git a/drivers/net/sfc/filter.c b/drivers/net/sfc/filter.c index d4722c41c4c..95a980fd63d 100644 --- a/drivers/net/sfc/filter.c +++ b/drivers/net/sfc/filter.c @@ -8,6 +8,7 @@ */ #include <linux/in.h> +#include <net/ip.h> #include "efx.h" #include "filter.h" #include "io.h" @@ -27,6 +28,10 @@ */ #define FILTER_CTL_SRCH_MAX 200 +/* Don't try very hard to find space for performance hints, as this is + * counter-productive. */ +#define FILTER_CTL_SRCH_HINT_MAX 5 + enum efx_filter_table_id { EFX_FILTER_TABLE_RX_IP = 0, EFX_FILTER_TABLE_RX_MAC, @@ -47,6 +52,10 @@ struct efx_filter_table { struct efx_filter_state { spinlock_t lock; struct efx_filter_table table[EFX_FILTER_TABLE_COUNT]; +#ifdef CONFIG_RFS_ACCEL + u32 *rps_flow_id; + unsigned rps_expire_index; +#endif }; /* The filter hash function is LFSR polynomial x^16 + x^3 + 1 of a 32-bit @@ -325,15 +334,16 @@ static int efx_filter_search(struct efx_filter_table *table, struct efx_filter_spec *spec, u32 key, bool for_insert, int *depth_required) { - unsigned hash, incr, filter_idx, depth; + unsigned hash, incr, filter_idx, depth, depth_max; struct efx_filter_spec *cmp; hash = efx_filter_hash(key); incr = efx_filter_increment(key); + depth_max = (spec->priority <= EFX_FILTER_PRI_HINT ? + FILTER_CTL_SRCH_HINT_MAX : FILTER_CTL_SRCH_MAX); for (depth = 1, filter_idx = hash & (table->size - 1); - depth <= FILTER_CTL_SRCH_MAX && - test_bit(filter_idx, table->used_bitmap); + depth <= depth_max && test_bit(filter_idx, table->used_bitmap); ++depth) { cmp = &table->spec[filter_idx]; if (efx_filter_equal(spec, cmp)) @@ -342,7 +352,7 @@ static int efx_filter_search(struct efx_filter_table *table, } if (!for_insert) return -ENOENT; - if (depth > FILTER_CTL_SRCH_MAX) + if (depth > depth_max) return -EBUSY; found: *depth_required = depth; @@ -562,6 +572,13 @@ int efx_probe_filters(struct efx_nic *efx) spin_lock_init(&state->lock); if (efx_nic_rev(efx) >= EFX_REV_FALCON_B0) { +#ifdef CONFIG_RFS_ACCEL + state->rps_flow_id = kcalloc(FR_BZ_RX_FILTER_TBL0_ROWS, + sizeof(*state->rps_flow_id), + GFP_KERNEL); + if (!state->rps_flow_id) + goto fail; +#endif table = &state->table[EFX_FILTER_TABLE_RX_IP]; table->id = EFX_FILTER_TABLE_RX_IP; table->offset = FR_BZ_RX_FILTER_TBL0; @@ -607,5 +624,97 @@ void efx_remove_filters(struct efx_nic *efx) kfree(state->table[table_id].used_bitmap); vfree(state->table[table_id].spec); } +#ifdef CONFIG_RFS_ACCEL + kfree(state->rps_flow_id); +#endif kfree(state); } + +#ifdef CONFIG_RFS_ACCEL + +int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, + u16 rxq_index, u32 flow_id) +{ + struct efx_nic *efx = netdev_priv(net_dev); + struct efx_channel *channel; + struct efx_filter_state *state = efx->filter_state; + struct efx_filter_spec spec; + const struct iphdr *ip; + const __be16 *ports; + int nhoff; + int rc; + + nhoff = skb_network_offset(skb); + + if (skb->protocol != htons(ETH_P_IP)) + return -EPROTONOSUPPORT; + + /* RFS must validate the IP header length before calling us */ + EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + sizeof(*ip))); + ip = (const struct iphdr *)(skb->data + nhoff); + if (ip->frag_off & htons(IP_MF | IP_OFFSET)) + return -EPROTONOSUPPORT; + EFX_BUG_ON_PARANOID(!pskb_may_pull(skb, nhoff + 4 * ip->ihl + 4)); + ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); + + efx_filter_init_rx(&spec, EFX_FILTER_PRI_HINT, 0, rxq_index); + rc = efx_filter_set_ipv4_full(&spec, ip->protocol, + ip->daddr, ports[1], ip->saddr, ports[0]); + if (rc) + return rc; + + rc = efx_filter_insert_filter(efx, &spec, true); + if (rc < 0) + return rc; + + /* Remember this so we can check whether to expire the filter later */ + state->rps_flow_id[rc] = flow_id; + channel = efx_get_channel(efx, skb_get_rx_queue(skb)); + ++channel->rfs_filters_added; + + netif_info(efx, rx_status, efx->net_dev, + "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d]\n", + (ip->protocol == IPPROTO_TCP) ? "TCP" : "UDP", + &ip->saddr, ntohs(ports[0]), &ip->daddr, ntohs(ports[1]), + rxq_index, flow_id, rc); + + return rc; +} + +bool __efx_filter_rfs_expire(struct efx_nic *efx, unsigned quota) +{ + struct efx_filter_state *state = efx->filter_state; + struct efx_filter_table *table = &state->table[EFX_FILTER_TABLE_RX_IP]; + unsigned mask = table->size - 1; + unsigned index; + unsigned stop; + + if (!spin_trylock_bh(&state->lock)) + return false; + + index = state->rps_expire_index; + stop = (index + quota) & mask; + + while (index != stop) { + if (test_bit(index, table->used_bitmap) && + table->spec[index].priority == EFX_FILTER_PRI_HINT && + rps_may_expire_flow(efx->net_dev, + table->spec[index].dmaq_id, + state->rps_flow_id[index], index)) { + netif_info(efx, rx_status, efx->net_dev, + "expiring filter %d [flow %u]\n", + index, state->rps_flow_id[index]); + efx_filter_table_clear_entry(efx, table, index); + } + index = (index + 1) & mask; + } + + state->rps_expire_index = stop; + if (table->used == 0) + efx_filter_table_reset_search_depth(table); + + spin_unlock_bh(&state->lock); + return true; +} + +#endif /* CONFIG_RFS_ACCEL */ diff --git a/drivers/net/sfc/io.h b/drivers/net/sfc/io.h index 6da4ae20a03..cc978803d48 100644 --- a/drivers/net/sfc/io.h +++ b/drivers/net/sfc/io.h @@ -1,7 +1,7 @@ /**************************************************************************** * Driver for Solarflare Solarstorm network controllers and boards * Copyright 2005-2006 Fen Systems Ltd. - * Copyright 2006-2009 Solarflare Communications Inc. + * Copyright 2006-2010 Solarflare Communications Inc. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -48,9 +48,9 @@ * replacing the low 96 bits with zero does not affect functionality. * - If the host writes to the last dword address of such a register * (i.e. the high 32 bits) the underlying register will always be - * written. If the collector does not hold values for the low 96 - * bits of the register, they will be written as zero. Writing to - * the last qword does not have this effect and must not be done. + * written. If the collector and the current write together do not + * provide values for all 128 bits of the register, the low 96 bits + * will be written as zero. * - If the host writes to the address of any other part of such a * register while the collector already holds values for some other * register, the write is discarded and the collector maintains its @@ -103,6 +103,7 @@ static inline void efx_writeo(struct efx_nic *efx, efx_oword_t *value, _efx_writed(efx, value->u32[2], reg + 8); _efx_writed(efx, value->u32[3], reg + 12); #endif + wmb(); mmiowb(); spin_unlock_irqrestore(&efx->biu_lock, flags); } @@ -125,6 +126,7 @@ static inline void efx_sram_writeq(struct efx_nic *efx, void __iomem *membase, __raw_writel((__force u32)value->u32[0], membase + addr); __raw_writel((__force u32)value->u32[1], membase + addr + 4); #endif + wmb(); mmiowb(); spin_unlock_irqrestore(&efx->biu_lock, flags); } @@ -139,6 +141,7 @@ static inline void efx_writed(struct efx_nic *efx, efx_dword_t *value, /* No lock required */ _efx_writed(efx, value->u32[0], reg); + wmb(); } /* Read a 128-bit CSR, locking as appropriate. */ @@ -149,6 +152,7 @@ static inline void efx_reado(struct efx_nic *efx, efx_oword_t *value, spin_lock_irqsave(&efx->biu_lock, flags); value->u32[0] = _efx_readd(efx, reg + 0); + rmb(); value->u32[1] = _efx_readd(efx, reg + 4); value->u32[2] = _efx_readd(efx, reg + 8); value->u32[3] = _efx_readd(efx, reg + 12); @@ -171,6 +175,7 @@ static inline void efx_sram_readq(struct efx_nic *efx, void __iomem *membase, value->u64[0] = (__force __le64)__raw_readq(membase + addr); #else value->u32[0] = (__force __le32)__raw_readl(membase + addr);< |