diff options
Diffstat (limited to 'drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c')
| -rw-r--r-- | drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c | 1263 |
1 files changed, 1081 insertions, 182 deletions
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c index 6f82812d0fa..e45bf09af0c 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c @@ -1,23 +1,31 @@ +/* + * QLogic qlcnic NIC Driver + * Copyright (c) 2009-2013 QLogic Corporation + * + * See LICENSE.qlcnic for copyright and licensing details. + */ + #include <linux/netdevice.h> #include <linux/if_vlan.h> #include <net/ip.h> #include <linux/ipv6.h> +#include <net/checksum.h> #include "qlcnic.h" -#define QLCNIC_MAC_HASH(MAC)\ - ((((MAC) & 0x70000) >> 0x10) | (((MAC) & 0x70000000000ULL) >> 0x25)) +#define QLCNIC_TX_ETHER_PKT 0x01 +#define QLCNIC_TX_TCP_PKT 0x02 +#define QLCNIC_TX_UDP_PKT 0x03 +#define QLCNIC_TX_IP_PKT 0x04 +#define QLCNIC_TX_TCP_LSO 0x05 +#define QLCNIC_TX_TCP_LSO6 0x06 +#define QLCNIC_TX_ENCAP_PKT 0x07 +#define QLCNIC_TX_ENCAP_LSO 0x08 +#define QLCNIC_TX_TCPV6_PKT 0x0b +#define QLCNIC_TX_UDPV6_PKT 0x0c -#define TX_ETHER_PKT 0x01 -#define TX_TCP_PKT 0x02 -#define TX_UDP_PKT 0x03 -#define TX_IP_PKT 0x04 -#define TX_TCP_LSO 0x05 -#define TX_TCP_LSO6 0x06 -#define TX_TCPV6_PKT 0x0b -#define TX_UDPV6_PKT 0x0c -#define FLAGS_VLAN_TAGGED 0x10 -#define FLAGS_VLAN_OOB 0x40 +#define QLCNIC_FLAGS_VLAN_TAGGED 0x10 +#define QLCNIC_FLAGS_VLAN_OOB 0x40 #define qlcnic_set_tx_vlan_tci(cmd_desc, v) \ (cmd_desc)->vlan_TCI = cpu_to_le16(v); @@ -84,6 +92,8 @@ #define qlcnic_get_lro_sts_mss(sts_data1) \ ((sts_data1 >> 32) & 0x0FFFF) +#define qlcnic_83xx_get_lro_sts_mss(sts) ((sts) & 0xffff) + /* opcode field in status_desc */ #define QLCNIC_SYN_OFFLOAD 0x03 #define QLCNIC_RXPKT_DESC 0x04 @@ -91,18 +101,180 @@ #define QLCNIC_RESPONSE_DESC 0x05 #define QLCNIC_LRO_DESC 0x12 +#define QLCNIC_TX_POLL_BUDGET 128 +#define QLCNIC_TCP_HDR_SIZE 20 +#define QLCNIC_TCP_TS_OPTION_SIZE 12 +#define QLCNIC_FETCH_RING_ID(handle) ((handle) >> 63) +#define QLCNIC_DESC_OWNER_FW cpu_to_le64(STATUS_OWNER_PHANTOM) + +#define QLCNIC_TCP_TS_HDR_SIZE (QLCNIC_TCP_HDR_SIZE + QLCNIC_TCP_TS_OPTION_SIZE) + /* for status field in status_desc */ #define STATUS_CKSUM_LOOP 0 #define STATUS_CKSUM_OK 2 -static void qlcnic_change_filter(struct qlcnic_adapter *adapter, - u64 uaddr, __le16 vlan_id, - struct qlcnic_host_tx_ring *tx_ring) +#define qlcnic_83xx_pktln(sts) ((sts >> 32) & 0x3FFF) +#define qlcnic_83xx_hndl(sts) ((sts >> 48) & 0x7FFF) +#define qlcnic_83xx_csum_status(sts) ((sts >> 39) & 7) +#define qlcnic_83xx_opcode(sts) ((sts >> 42) & 0xF) +#define qlcnic_83xx_vlan_tag(sts) (((sts) >> 48) & 0xFFFF) +#define qlcnic_83xx_lro_pktln(sts) (((sts) >> 32) & 0x3FFF) +#define qlcnic_83xx_l2_hdr_off(sts) (((sts) >> 16) & 0xFF) +#define qlcnic_83xx_l4_hdr_off(sts) (((sts) >> 24) & 0xFF) +#define qlcnic_83xx_pkt_cnt(sts) (((sts) >> 16) & 0x7) +#define qlcnic_83xx_is_tstamp(sts) (((sts) >> 40) & 1) +#define qlcnic_83xx_is_psh_bit(sts) (((sts) >> 41) & 1) +#define qlcnic_83xx_is_ip_align(sts) (((sts) >> 46) & 1) +#define qlcnic_83xx_has_vlan_tag(sts) (((sts) >> 47) & 1) + +static int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, + int max); + +static struct sk_buff *qlcnic_process_rxbuf(struct qlcnic_adapter *, + struct qlcnic_host_rds_ring *, + u16, u16); + +static inline u8 qlcnic_mac_hash(u64 mac, u16 vlan) +{ + return (u8)((mac & 0xff) ^ ((mac >> 40) & 0xff) ^ (vlan & 0xff)); +} + +static inline u32 qlcnic_get_ref_handle(struct qlcnic_adapter *adapter, + u16 handle, u8 ring_id) +{ + if (qlcnic_83xx_check(adapter)) + return handle | (ring_id << 15); + else + return handle; +} + +static inline int qlcnic_82xx_is_lb_pkt(u64 sts_data) +{ + return (qlcnic_get_sts_status(sts_data) == STATUS_CKSUM_LOOP) ? 1 : 0; +} + +static void qlcnic_delete_rx_list_mac(struct qlcnic_adapter *adapter, + struct qlcnic_filter *fil, + void *addr, u16 vlan_id) +{ + int ret; + u8 op; + + op = vlan_id ? QLCNIC_MAC_VLAN_ADD : QLCNIC_MAC_ADD; + ret = qlcnic_sre_macaddr_change(adapter, addr, vlan_id, op); + if (ret) + return; + + op = vlan_id ? QLCNIC_MAC_VLAN_DEL : QLCNIC_MAC_DEL; + ret = qlcnic_sre_macaddr_change(adapter, addr, vlan_id, op); + if (!ret) { + hlist_del(&fil->fnode); + adapter->rx_fhash.fnum--; + } +} + +static struct qlcnic_filter *qlcnic_find_mac_filter(struct hlist_head *head, + void *addr, u16 vlan_id) +{ + struct qlcnic_filter *tmp_fil = NULL; + struct hlist_node *n; + + hlist_for_each_entry_safe(tmp_fil, n, head, fnode) { + if (ether_addr_equal(tmp_fil->faddr, addr) && + tmp_fil->vlan_id == vlan_id) + return tmp_fil; + } + + return NULL; +} + +static void qlcnic_add_lb_filter(struct qlcnic_adapter *adapter, + struct sk_buff *skb, int loopback_pkt, u16 vlan_id) +{ + struct ethhdr *phdr = (struct ethhdr *)(skb->data); + struct qlcnic_filter *fil, *tmp_fil; + struct hlist_head *head; + unsigned long time; + u64 src_addr = 0; + u8 hindex, op; + int ret; + + if (!qlcnic_sriov_pf_check(adapter) || (vlan_id == 0xffff)) + vlan_id = 0; + + memcpy(&src_addr, phdr->h_source, ETH_ALEN); + hindex = qlcnic_mac_hash(src_addr, vlan_id) & + (adapter->fhash.fbucket_size - 1); + + if (loopback_pkt) { + if (adapter->rx_fhash.fnum >= adapter->rx_fhash.fmax) + return; + + head = &(adapter->rx_fhash.fhead[hindex]); + + tmp_fil = qlcnic_find_mac_filter(head, &src_addr, vlan_id); + if (tmp_fil) { + time = tmp_fil->ftime; + if (time_after(jiffies, QLCNIC_READD_AGE * HZ + time)) + tmp_fil->ftime = jiffies; + return; + } + + fil = kzalloc(sizeof(struct qlcnic_filter), GFP_ATOMIC); + if (!fil) + return; + + fil->ftime = jiffies; + memcpy(fil->faddr, &src_addr, ETH_ALEN); + fil->vlan_id = vlan_id; + spin_lock(&adapter->rx_mac_learn_lock); + hlist_add_head(&(fil->fnode), head); + adapter->rx_fhash.fnum++; + spin_unlock(&adapter->rx_mac_learn_lock); + } else { + head = &adapter->fhash.fhead[hindex]; + + spin_lock(&adapter->mac_learn_lock); + + tmp_fil = qlcnic_find_mac_filter(head, &src_addr, vlan_id); + if (tmp_fil) { + op = vlan_id ? QLCNIC_MAC_VLAN_DEL : QLCNIC_MAC_DEL; + ret = qlcnic_sre_macaddr_change(adapter, + (u8 *)&src_addr, + vlan_id, op); + if (!ret) { + hlist_del(&tmp_fil->fnode); + adapter->fhash.fnum--; + } + + spin_unlock(&adapter->mac_learn_lock); + + return; + } + + spin_unlock(&adapter->mac_learn_lock); + + head = &adapter->rx_fhash.fhead[hindex]; + + spin_lock(&adapter->rx_mac_learn_lock); + + tmp_fil = qlcnic_find_mac_filter(head, &src_addr, vlan_id); + if (tmp_fil) + qlcnic_delete_rx_list_mac(adapter, tmp_fil, &src_addr, + vlan_id); + + spin_unlock(&adapter->rx_mac_learn_lock); + } +} + +void qlcnic_82xx_change_filter(struct qlcnic_adapter *adapter, u64 *uaddr, + u16 vlan_id) { struct cmd_desc_type0 *hwdesc; struct qlcnic_nic_req *req; struct qlcnic_mac_req *mac_req; struct qlcnic_vlan_req *vlan_req; + struct qlcnic_host_tx_ring *tx_ring = adapter->tx_ring; u32 producer; u64 word; @@ -118,100 +290,200 @@ static void qlcnic_change_filter(struct qlcnic_adapter *adapter, mac_req = (struct qlcnic_mac_req *)&(req->words[0]); mac_req->op = vlan_id ? QLCNIC_MAC_VLAN_ADD : QLCNIC_MAC_ADD; - memcpy(mac_req->mac_addr, &uaddr, ETH_ALEN); + memcpy(mac_req->mac_addr, uaddr, ETH_ALEN); vlan_req = (struct qlcnic_vlan_req *)&req->words[1]; - vlan_req->vlan_id = vlan_id; + vlan_req->vlan_id = cpu_to_le16(vlan_id); tx_ring->producer = get_next_index(producer, tx_ring->num_desc); smp_mb(); } static void qlcnic_send_filter(struct qlcnic_adapter *adapter, - struct qlcnic_host_tx_ring *tx_ring, struct cmd_desc_type0 *first_desc, struct sk_buff *skb) { + struct vlan_ethhdr *vh = (struct vlan_ethhdr *)(skb->data); struct ethhdr *phdr = (struct ethhdr *)(skb->data); + u16 protocol = ntohs(skb->protocol); struct qlcnic_filter *fil, *tmp_fil; - struct hlist_node *tmp_hnode, *n; struct hlist_head *head; + struct hlist_node *n; u64 src_addr = 0; - __le16 vlan_id = 0; - u8 hindex; + u16 vlan_id = 0; + u8 hindex, hval; if (ether_addr_equal(phdr->h_source, adapter->mac_addr)) return; - if (adapter->fhash.fnum >= adapter->fhash.fmax) - return; + if (adapter->flags & QLCNIC_VLAN_FILTERING) { + if (protocol == ETH_P_8021Q) { + vh = (struct vlan_ethhdr *)skb->data; + vlan_id = ntohs(vh->h_vlan_TCI); + } else if (vlan_tx_tag_present(skb)) { + vlan_id = vlan_tx_tag_get(skb); + } + } - /* Only NPAR capable devices support vlan based learning*/ - if (adapter->flags & QLCNIC_ESWITCH_ENABLED) - vlan_id = first_desc->vlan_TCI; memcpy(&src_addr, phdr->h_source, ETH_ALEN); - hindex = QLCNIC_MAC_HASH(src_addr) & (QLCNIC_LB_MAX_FILTERS - 1); + hval = qlcnic_mac_hash(src_addr, vlan_id); + hindex = hval & (adapter->fhash.fbucket_size - 1); head = &(adapter->fhash.fhead[hindex]); - hlist_for_each_entry_safe(tmp_fil, tmp_hnode, n, head, fnode) { - if (!memcmp(tmp_fil->faddr, &src_addr, ETH_ALEN) && - tmp_fil->vlan_id == vlan_id) { - + hlist_for_each_entry_safe(tmp_fil, n, head, fnode) { + if (ether_addr_equal(tmp_fil->faddr, (u8 *)&src_addr) && + tmp_fil->vlan_id == vlan_id) { if (jiffies > (QLCNIC_READD_AGE * HZ + tmp_fil->ftime)) - qlcnic_change_filter(adapter, src_addr, vlan_id, - tx_ring); + qlcnic_change_filter(adapter, &src_addr, + vlan_id); tmp_fil->ftime = jiffies; return; } } + if (unlikely(adapter->fhash.fnum >= adapter->fhash.fmax)) { + adapter->stats.mac_filter_limit_overrun++; + return; + } + fil = kzalloc(sizeof(struct qlcnic_filter), GFP_ATOMIC); if (!fil) return; - qlcnic_change_filter(adapter, src_addr, vlan_id, tx_ring); - + qlcnic_change_filter(adapter, &src_addr, vlan_id); fil->ftime = jiffies; fil->vlan_id = vlan_id; memcpy(fil->faddr, &src_addr, ETH_ALEN); - spin_lock(&adapter->mac_learn_lock); - hlist_add_head(&(fil->fnode), head); adapter->fhash.fnum++; - spin_unlock(&adapter->mac_learn_lock); } +#define QLCNIC_ENCAP_VXLAN_PKT BIT_0 +#define QLCNIC_ENCAP_OUTER_L3_IP6 BIT_1 +#define QLCNIC_ENCAP_INNER_L3_IP6 BIT_2 +#define QLCNIC_ENCAP_INNER_L4_UDP BIT_3 +#define QLCNIC_ENCAP_DO_L3_CSUM BIT_4 +#define QLCNIC_ENCAP_DO_L4_CSUM BIT_5 + +static int qlcnic_tx_encap_pkt(struct qlcnic_adapter *adapter, + struct cmd_desc_type0 *first_desc, + struct sk_buff *skb, + struct qlcnic_host_tx_ring *tx_ring) +{ + u8 opcode = 0, inner_hdr_len = 0, outer_hdr_len = 0, total_hdr_len = 0; + int copied, copy_len, descr_size; + u32 producer = tx_ring->producer; + struct cmd_desc_type0 *hwdesc; + u16 flags = 0, encap_descr = 0; + + opcode = QLCNIC_TX_ETHER_PKT; + encap_descr = QLCNIC_ENCAP_VXLAN_PKT; + + if (skb_is_gso(skb)) { + inner_hdr_len = skb_inner_transport_header(skb) + + inner_tcp_hdrlen(skb) - + skb_inner_mac_header(skb); + + /* VXLAN header size = 8 */ + outer_hdr_len = skb_transport_offset(skb) + 8 + + sizeof(struct udphdr); + first_desc->outer_hdr_length = outer_hdr_len; + total_hdr_len = inner_hdr_len + outer_hdr_len; + encap_descr |= QLCNIC_ENCAP_DO_L3_CSUM | + QLCNIC_ENCAP_DO_L4_CSUM; + first_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); + first_desc->hdr_length = inner_hdr_len; + + /* Copy inner and outer headers in Tx descriptor(s) + * If total_hdr_len > cmd_desc_type0, use multiple + * descriptors + */ + copied = 0; + descr_size = (int)sizeof(struct cmd_desc_type0); + while (copied < total_hdr_len) { + copy_len = min(descr_size, (total_hdr_len - copied)); + hwdesc = &tx_ring->desc_head[producer]; + tx_ring->cmd_buf_arr[producer].skb = NULL; + skb_copy_from_linear_data_offset(skb, copied, + (char *)hwdesc, + copy_len); + copied += copy_len; + producer = get_next_index(producer, tx_ring->num_desc); + } + + tx_ring->producer = producer; + + /* Make sure updated tx_ring->producer is visible + * for qlcnic_tx_avail() + */ + smp_mb(); + adapter->stats.encap_lso_frames++; + + opcode = QLCNIC_TX_ENCAP_LSO; + } else if (skb->ip_summed == CHECKSUM_PARTIAL) { + if (inner_ip_hdr(skb)->version == 6) { + if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_UDP) + encap_descr |= QLCNIC_ENCAP_INNER_L4_UDP; + } else { + if (inner_ip_hdr(skb)->protocol == IPPROTO_UDP) + encap_descr |= QLCNIC_ENCAP_INNER_L4_UDP; + } + + adapter->stats.encap_tx_csummed++; + opcode = QLCNIC_TX_ENCAP_PKT; + } + + /* Prepare first 16 bits of byte offset 16 of Tx descriptor */ + if (ip_hdr(skb)->version == 6) + encap_descr |= QLCNIC_ENCAP_OUTER_L3_IP6; + + /* outer IP header's size in 32bit words size*/ + encap_descr |= (skb_network_header_len(skb) >> 2) << 6; + + /* outer IP header offset */ + encap_descr |= skb_network_offset(skb) << 10; + first_desc->encap_descr = cpu_to_le16(encap_descr); + + first_desc->tcp_hdr_offset = skb_inner_transport_header(skb) - + skb->data; + first_desc->ip_hdr_offset = skb_inner_network_offset(skb); + + qlcnic_set_tx_flags_opcode(first_desc, flags, opcode); + + return 0; +} + static int qlcnic_tx_pkt(struct qlcnic_adapter *adapter, - struct cmd_desc_type0 *first_desc, struct sk_buff *skb) + struct cmd_desc_type0 *first_desc, struct sk_buff *skb, + struct qlcnic_host_tx_ring *tx_ring) { u8 l4proto, opcode = 0, hdr_len = 0; u16 flags = 0, vlan_tci = 0; int copied, offset, copy_len, size; struct cmd_desc_type0 *hwdesc; struct vlan_ethhdr *vh; - struct qlcnic_host_tx_ring *tx_ring = adapter->tx_ring; u16 protocol = ntohs(skb->protocol); u32 producer = tx_ring->producer; if (protocol == ETH_P_8021Q) { vh = (struct vlan_ethhdr *)skb->data; - flags = FLAGS_VLAN_TAGGED; + flags = QLCNIC_FLAGS_VLAN_TAGGED; vlan_tci = ntohs(vh->h_vlan_TCI); protocol = ntohs(vh->h_vlan_encapsulated_proto); } else if (vlan_tx_tag_present(skb)) { - flags = FLAGS_VLAN_OOB; + flags = QLCNIC_FLAGS_VLAN_OOB; vlan_tci = vlan_tx_tag_get(skb); } - if (unlikely(adapter->pvid)) { + if (unlikely(adapter->tx_pvid)) { if (vlan_tci && !(adapter->flags & QLCNIC_TAGGING_ENABLED)) return -EIO; if (vlan_tci && (adapter->flags & QLCNIC_TAGGING_ENABLED)) goto set_flags; - flags = FLAGS_VLAN_OOB; - vlan_tci = adapter->pvid; + flags = QLCNIC_FLAGS_VLAN_OOB; + vlan_tci = adapter->tx_pvid; } set_flags: qlcnic_set_tx_vlan_tci(first_desc, vlan_tci); @@ -221,26 +493,26 @@ set_flags: flags |= BIT_0; memcpy(&first_desc->eth_addr, skb->data, ETH_ALEN); } - opcode = TX_ETHER_PKT; - if ((adapter->netdev->features & (NETIF_F_TSO | NETIF_F_TSO6)) && - skb_shinfo(skb)->gso_size > 0) { + opcode = QLCNIC_TX_ETHER_PKT; + if (skb_is_gso(skb)) { hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); first_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); - first_desc->total_hdr_length = hdr_len; - opcode = (protocol == ETH_P_IPV6) ? TX_TCP_LSO6 : TX_TCP_LSO; + first_desc->hdr_length = hdr_len; + opcode = (protocol == ETH_P_IPV6) ? QLCNIC_TX_TCP_LSO6 : + QLCNIC_TX_TCP_LSO; /* For LSO, we need to copy the MAC/IP/TCP headers into * the descriptor ring */ copied = 0; offset = 2; - if (flags & FLAGS_VLAN_OOB) { - first_desc->total_hdr_length += VLAN_HLEN; + if (flags & QLCNIC_FLAGS_VLAN_OOB) { + first_desc->hdr_length += VLAN_HLEN; first_desc->tcp_hdr_offset = VLAN_HLEN; first_desc->ip_hdr_offset = VLAN_HLEN; /* Only in case of TSO on vlan device */ - flags |= FLAGS_VLAN_TAGGED; + flags |= QLCNIC_FLAGS_VLAN_TAGGED; /* Create a TSO vlan header template for firmware */ hwdesc = &tx_ring->desc_head[producer]; @@ -284,16 +556,16 @@ set_flags: l4proto = ip_hdr(skb)->protocol; if (l4proto == IPPROTO_TCP) - opcode = TX_TCP_PKT; + opcode = QLCNIC_TX_TCP_PKT; else if (l4proto == IPPROTO_UDP) - opcode = TX_UDP_PKT; + opcode = QLCNIC_TX_UDP_PKT; } else if (protocol == ETH_P_IPV6) { l4proto = ipv6_hdr(skb)->nexthdr; if (l4proto == IPPROTO_TCP) - opcode = TX_TCPV6_PKT; + opcode = QLCNIC_TX_TCPV6_PKT; else if (l4proto == IPPROTO_UDP) - opcode = TX_UDPV6_PKT; + opcode = QLCNIC_TX_UDPV6_PKT; } } first_desc->tcp_hdr_offset += skb_transport_offset(skb); @@ -375,7 +647,7 @@ static inline void qlcnic_clear_cmddesc(u64 *desc) netdev_tx_t qlcnic_xmit_frame(struct sk_buff *skb, struct net_device *netdev) { struct qlcnic_adapter *adapter = netdev_priv(netdev); - struct qlcnic_host_tx_ring *tx_ring = adapter->tx_ring; + struct qlcnic_host_tx_ring *tx_ring; struct qlcnic_cmd_buffer *pbuf; struct qlcnic_skb_frag *buffrag; struct cmd_desc_type0 *hwdesc, *first_desc; @@ -383,11 +655,11 @@ netdev_tx_t qlcnic_xmit_frame(struct sk_buff *skb, struct net_device *netdev) struct ethhdr *phdr; int i, k, frag_count, delta = 0; u32 producer, num_txd; - - num_txd = tx_ring->num_desc; + u16 protocol; + bool l4_is_udp = false; if (!test_bit(__QLCNIC_DEV_UP, &adapter->state)) { - netif_stop_queue(netdev); + netif_tx_stop_all_queues(netdev); return NETDEV_TX_BUSY; } @@ -397,7 +669,11 @@ netdev_tx_t qlcnic_xmit_frame(struct sk_buff *skb, struct net_device *netdev) goto drop_packet; } + tx_ring = &adapter->tx_ring[skb_get_queue_mapping(skb)]; + num_txd = tx_ring->num_desc; + frag_count = skb_shinfo(skb)->nr_frags + 1; + /* 14 frags supported for normal packet and * 32 frags supported for TSO packet */ @@ -412,11 +688,11 @@ netdev_tx_t qlcnic_xmit_frame(struct sk_buff *skb, struct net_device *netdev) } if (unlikely(qlcnic_tx_avail(tx_ring) <= TX_STOP_THRESH)) { - netif_stop_queue(netdev); + netif_tx_stop_queue(tx_ring->txq); if (qlcnic_tx_avail(tx_ring) > TX_STOP_THRESH) { - netif_start_queue(netdev); + netif_tx_start_queue(tx_ring->txq); } else { - adapter->stats.xmit_off++; + tx_ring->tx_stats.xmit_off++; return NETDEV_TX_BUSY; } } @@ -471,14 +747,29 @@ netdev_tx_t qlcnic_xmit_frame(struct sk_buff *skb, struct net_device *netdev) tx_ring->producer = get_next_index(producer, num_txd); smp_mb(); - if (unlikely(qlcnic_tx_pkt(adapter, first_desc, skb))) - goto unwind_buff; + protocol = ntohs(skb->protocol); + if (protocol == ETH_P_IP) + l4_is_udp = ip_hdr(skb)->protocol == IPPROTO_UDP; + else if (protocol == ETH_P_IPV6) + l4_is_udp = ipv6_hdr(skb)->nexthdr == IPPROTO_UDP; + + /* Check if it is a VXLAN packet */ + if (!skb->encapsulation || !l4_is_udp || + !qlcnic_encap_tx_offload(adapter)) { + if (unlikely(qlcnic_tx_pkt(adapter, first_desc, skb, + tx_ring))) + goto unwind_buff; + } else { + if (unlikely(qlcnic_tx_encap_pkt(adapter, first_desc, + skb, tx_ring))) + goto unwind_buff; + } - if (adapter->mac_learn) - qlcnic_send_filter(adapter, tx_ring, first_desc, skb); + if (adapter->drv_mac_learn) + qlcnic_send_filter(adapter, first_desc, skb); - adapter->stats.txbytes += skb->len; - adapter->stats.xmitcalled++; + tx_ring->tx_stats.tx_bytes += skb->len; + tx_ring->tx_stats.xmit_called++; qlcnic_update_cmd_producer(tx_ring); @@ -499,17 +790,20 @@ void qlcnic_advert_link_change(struct qlcnic_adapter *adapter, int linkup) if (adapter->ahw->linkup && !linkup) { netdev_info(netdev, "NIC Link is down\n"); adapter->ahw->linkup = 0; - if (netif_running(netdev)) { - netif_carrier_off(netdev); - netif_stop_queue(netdev); - } + netif_carrier_off(netdev); } else if (!adapter->ahw->linkup && linkup) { - netdev_info(netdev, "NIC Link is up\n"); adapter->ahw->linkup = 1; - if (netif_running(netdev)) { - netif_carrier_on(netdev); - netif_wake_queue(netdev); + + /* Do not advertise Link up to the stack if device + * is in loopback mode + */ + if (qlcnic_83xx_check(adapter) && adapter->ahw->lb_mode) { + netdev_info(netdev, "NIC Link is up for loopback test\n"); + return; } + + netdev_info(netdev, "NIC Link is up\n"); + netif_carrier_on(netdev); } } @@ -528,8 +822,8 @@ static int qlcnic_alloc_rx_skb(struct qlcnic_adapter *adapter, } skb_reserve(skb, NET_IP_ALIGN); - dma = pci_map_single(pdev, skb->data, rds_ring->dma_size, - PCI_DMA_FROMDEVICE); + dma = pci_map_single(pdev, skb->data, + rds_ring->dma_size, PCI_DMA_FROMDEVICE); if (pci_dma_mapping_error(pdev, dma)) { adapter->stats.rx_dma_map_error++; @@ -544,12 +838,13 @@ static int qlcnic_alloc_rx_skb(struct qlcnic_adapter *adapter, } static void qlcnic_post_rx_buffers_nodb(struct qlcnic_adapter *adapter, - struct qlcnic_host_rds_ring *rds_ring) + struct qlcnic_host_rds_ring *rds_ring, + u8 ring_id) { struct rcv_desc *pdesc; struct qlcnic_rx_buffer *buffer; int count = 0; - uint32_t producer; + uint32_t producer, handle; struct list_head *head; if (!spin_trylock(&rds_ring->lock)) @@ -557,7 +852,6 @@ static void qlcnic_post_rx_buffers_nodb(struct qlcnic_adapter *adapter, producer = rds_ring->producer; head = &rds_ring->free_list; - while (!list_empty(head)) { buffer = list_entry(head->next, struct qlcnic_rx_buffer, list); @@ -565,28 +859,29 @@ static void qlcnic_post_rx_buffers_nodb(struct qlcnic_adapter *adapter, if (qlcnic_alloc_rx_skb(adapter, rds_ring, buffer)) break; } - count++; list_del(&buffer->list); /* make a rcv descriptor */ pdesc = &rds_ring->desc_head[producer]; - pdesc->reference_handle = cpu_to_le16(buffer->ref_handle); + handle = qlcnic_get_ref_handle(adapter, + buffer->ref_handle, ring_id); + pdesc->reference_handle = cpu_to_le16(handle); pdesc->buffer_length = cpu_to_le32(rds_ring->dma_size); pdesc->addr_buffer = cpu_to_le64(buffer->dma); producer = get_next_index(producer, rds_ring->num_desc); } - if (count) { rds_ring->producer = producer; writel((producer - 1) & (rds_ring->num_desc - 1), rds_ring->crb_rcv_producer); } - spin_unlock(&rds_ring->lock); } -static int qlcnic_process_cmd_ring(struct qlcnic_adapter *adapter) +static int qlcnic_process_cmd_ring(struct qlcnic_adapter *adapter, + struct qlcnic_host_tx_ring *tx_ring, + int budget) { u32 sw_consumer, hw_consumer; int i, done, count = 0; @@ -594,9 +889,8 @@ static int qlcnic_process_cmd_ring(struct qlcnic_adapter *adapter) struct pci_dev *pdev = adapter->pdev; struct net_device *netdev = adapter->netdev; struct qlcnic_skb_frag *frag; - struct qlcnic_host_tx_ring *tx_ring = adapter->tx_ring; - if (!spin_trylock(&adapter->tx_clean_lock)) + if (!spin_trylock(&tx_ring->tx_clean_lock)) return 1; sw_consumer = tx_ring->sw_consumer; @@ -615,26 +909,25 @@ static int qlcnic_process_cmd_ring(struct qlcnic_adapter *adapter) PCI_DMA_TODEVICE); frag->dma = 0ULL; } - - adapter->stats.xmitfinished++; + tx_ring->tx_stats.xmit_finished++; dev_kfree_skb_any(buffer->skb); buffer->skb = NULL; } sw_consumer = get_next_index(sw_consumer, tx_ring->num_desc); - if (++count >= MAX_STATUS_HANDLE) + if (++count >= budget) break; } - if (count && netif_running(netdev)) { - tx_ring->sw_consumer = sw_consumer; + tx_ring->sw_consumer = sw_consumer; + if (count && netif_running(netdev)) { smp_mb(); - - if (netif_queue_stopped(netdev) && netif_carrier_ok(netdev)) { + if (netif_tx_queue_stopped(tx_ring->txq) && + netif_carrier_ok(netdev)) { if (qlcnic_tx_avail(tx_ring) > TX_STOP_THRESH) { - netif_wake_queue(netdev); - adapter->stats.xmit_on++; + netif_tx_wake_queue(tx_ring->txq); + tx_ring->tx_stats.xmit_on++; } } adapter->tx_timeo_cnt = 0; @@ -655,27 +948,50 @@ static int qlcnic_process_cmd_ring(struct qlcnic_adapter *adapter) hw_consumer = le32_to_cpu(*(tx_ring->hw_consumer)); done = (sw_consumer == hw_consumer); - spin_unlock(&adapter->tx_clean_lock); + spin_unlock(&tx_ring->tx_clean_lock); return done; } static int qlcnic_poll(struct napi_struct *napi, int budget) { + int tx_complete, work_done; struct qlcnic_host_sds_ring *sds_ring; struct qlcnic_adapter *adapter; - int tx_complete, work_done; + struct qlcnic_host_tx_ring *tx_ring; sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi); adapter = sds_ring->adapter; + tx_ring = sds_ring->tx_ring; - tx_complete = qlcnic_process_cmd_ring(adapter); + tx_complete = qlcnic_process_cmd_ring(adapter, tx_ring, + budget); work_done = qlcnic_process_rcv_ring(sds_ring, budget); - if ((work_done < budget) && tx_complete) { napi_complete(&sds_ring->napi); + if (test_bit(__QLCNIC_DEV_UP, &adapter->state)) { + qlcnic_enable_sds_intr(adapter, sds_ring); + qlcnic_enable_tx_intr(adapter, tx_ring); + } + } + + return work_done; +} + +static int qlcnic_tx_poll(struct napi_struct *napi, int budget) +{ + struct qlcnic_host_tx_ring *tx_ring; + struct qlcnic_adapter *adapter; + int work_done; + + tx_ring = container_of(napi, struct qlcnic_host_tx_ring, napi); + adapter = tx_ring->adapter; + + work_done = qlcnic_process_cmd_ring(adapter, tx_ring, budget); + if (work_done) { + napi_complete(&tx_ring->napi); if (test_bit(__QLCNIC_DEV_UP, &adapter->state)) - qlcnic_enable_int(sds_ring); + qlcnic_enable_tx_intr(adapter, tx_ring); } return work_done; @@ -695,7 +1011,7 @@ static int qlcnic_rx_poll(struct napi_struct *napi, int budget) if (work_done < budget) { napi_complete(&sds_ring->napi); if (test_bit(__QLCNIC_DEV_UP, &adapter->state)) - qlcnic_enable_int(sds_ring); + qlcnic_enable_sds_intr(adapter, sds_ring); } return work_done; @@ -785,45 +1101,45 @@ static void qlcnic_handle_fw_message(int desc_cnt, int index, break; case 1: dev_info(dev, "loopback already in progress\n"); - adapter->ahw->diag_cnt = -QLCNIC_TEST_IN_PROGRESS; + adapter->ahw->diag_cnt = -EINPROGRESS; break; case 2: dev_info(dev, "loopback cable is not connected\n"); - adapter->ahw->diag_cnt = -QLCNIC_LB_CABLE_NOT_CONN; + adapter->ahw->diag_cnt = -ENODEV; break; default: dev_info(dev, "loopback configure request failed, err %x\n", ret); - adapter->ahw->diag_cnt = -QLCNIC_UNDEFINED_ERROR; + adapter->ahw->diag_cnt = -EIO; break; } break; + case QLCNIC_C2H_OPCODE_GET_DCB_AEN: + qlcnic_dcb_aen_handler(adapter->dcb, (void *)&msg); + break; default: break; } } -static struct sk_buff * -qlcnic_process_rxbuf(struct qlcnic_adapter *adapter, - struct qlcnic_host_rds_ring *rds_ring, u16 index, - u16 cksum) +static struct sk_buff *qlcnic_process_rxbuf(struct qlcnic_adapter *adapter, + struct qlcnic_host_rds_ring *ring, + u16 index, u16 cksum) { struct qlcnic_rx_buffer *buffer; struct sk_buff *skb; - buffer = &rds_ring->rx_buf_arr[index]; - + buffer = &ring->rx_buf_arr[index]; if (unlikely(buffer->skb == NULL)) { WARN_ON(1); return NULL; } - pci_unmap_single(adapter->pdev, buffer->dma, rds_ring->dma_size, + pci_unmap_single(adapter->pdev, buffer->dma, ring->dma_size, PCI_DMA_FROMDEVICE); skb = buffer->skb; - if (likely((adapter->netdev->features & NETIF_F_RXCSUM) && (cksum == STATUS_CKSUM_OK || cksum == STATUS_CKSUM_LOOP))) { adapter->stats.csummed++; @@ -832,6 +1148,7 @@ qlcnic_process_rxbuf(struct qlcnic_adapter *adapter, skb_checksum_none_assert(skb); } + buffer->skb = NULL; return skb; @@ -847,10 +1164,10 @@ static inline int qlcnic_check_rx_tagging(struct qlcnic_adapter *adapter, memmove(skb->data + VLAN_HLEN, eth_hdr, ETH_ALEN * 2); skb_pull(skb, VLAN_HLEN); } - if (!adapter->pvid) + if (!adapter->rx_pvid) return 0; - if (*vlan_tag == adapter->pvid) { + if (*vlan_tag == adapter->rx_pvid) { /* Outer vlan tag. Packet should follow non-vlan path */ *vlan_tag = 0xffff; return 0; @@ -871,8 +1188,8 @@ qlcnic_process_rcv(struct qlcnic_adapter *adapter, struct qlcnic_rx_buffer *buffer; struct sk_buff *skb; struct qlcnic_host_rds_ring *rds_ring; - int index, length, cksum, pkt_offset; - u16 vid = 0xffff; + int index, length, cksum, pkt_offset, is_lb_pkt; + u16 vid = 0xffff, t_vid; if (unlikely(ring >= adapter->max_rds_rings)) return NULL; @@ -892,6 +1209,12 @@ qlcnic_process_rcv(struct qlcnic_adapter *adapter, if (!skb) return buffer; + if (adapter->rx_mac_learn) { + t_vid = 0; + is_lb_pkt = qlcnic_82xx_is_lb_pkt(sts_data0); + qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, t_vid); + } + if (length > rds_ring->skb_size) skb_put(skb, rds_ring->skb_size); else @@ -909,7 +1232,7 @@ qlcnic_process_rcv(struct qlcnic_adapter *adapter, skb->protocol = eth_type_trans(skb, netdev); if (vid != 0xffff) - __vlan_hwaccel_put_tag(skb, vid); + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); napi_gro_receive(&sds_ring->napi, skb); @@ -933,19 +1256,20 @@ qlcnic_process_lro(struct qlcnic_adapter *adapter, struct sk_buff *skb; struct qlcnic_host_rds_ring *rds_ring; struct iphdr *iph; + struct ipv6hdr *ipv6h; struct tcphdr *th; bool push, timestamp; - int index, l2_hdr_offset, l4_hdr_offset; - u16 lro_length, length, data_offset, vid = 0xffff; + int index, l2_hdr_offset, l4_hdr_offset, is_lb_pkt; + u16 lro_length, length, data_offset, t_vid, vid = 0xffff; u32 seq_number; - if (unlikely(ring > adapter->max_rds_rings)) + if (unlikely(ring >= adapter->max_rds_rings)) return NULL; rds_ring = &recv_ctx->rds_rings[ring]; index = qlcnic_get_lro_sts_refhandle(sts_data0); - if (unlikely(index > rds_ring->num_desc)) + if (unlikely(index >= rds_ring->num_desc)) return NULL; buffer = &rds_ring->rx_buf_arr[index]; @@ -961,6 +1285,12 @@ qlcnic_process_lro(struct qlcnic_adapter *adapter, if (!skb) return buffer; + if (adapter->rx_mac_learn) { + t_vid = 0; + is_lb_pkt = qlcnic_82xx_is_lb_pkt(sts_data0); + qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, t_vid); + } + if (timestamp) data_offset = l4_hdr_offset + QLC_TCP_TS_HDR_SIZE; else @@ -976,21 +1306,34 @@ qlcnic_process_lro(struct qlcnic_adapter *adapter, } skb->protocol = eth_type_trans(skb, netdev); - iph = (struct iphdr *)skb->data; - th = (struct tcphdr *)(skb->data + (iph->ihl << 2)); - length = (iph->ihl << 2) + (th->doff << 2) + lro_length; - iph->tot_len = htons(length); - iph->check = 0; - iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl); + + if (ntohs(skb->protocol) == ETH_P_IPV6) { + ipv6h = (struct ipv6hdr *)skb->data; + th = (struct tcphdr *)(skb->data + sizeof(struct ipv6hdr)); + length = (th->doff << 2) + lro_length; + ipv6h->payload_len = htons(length); + } else { + iph = (struct iphdr *)skb->data; + th = (struct tcphdr *)(skb->data + (iph->ihl << 2)); + length = (iph->ihl << 2) + (th->doff << 2) + lro_length; + csum_replace2(&iph->check, iph->tot_len, htons(length)); + iph->tot_len = htons(length); + } + th->psh = push; th->seq = htonl(seq_number); length = skb->len; - if (adapter->flags & QLCNIC_FW_LRO_MSS_CAP) + if (adapter->flags & QLCNIC_FW_LRO_MSS_CAP) { skb_shinfo(skb)->gso_size = qlcnic_get_lro_sts_mss(sts_data1); + if (skb->protocol == htons(ETH_P_IPV6)) + skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; + else + skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; + } if (vid != 0xffff) - __vlan_hwaccel_put_tag(skb, vid); + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); netif_receive_skb(skb); adapter->stats.lro_pkts++; @@ -999,16 +1342,16 @@ qlcnic_process_lro(struct qlcnic_adapter *adapter, return buffer; } -int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, int max) +static int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, int max) { struct qlcnic_host_rds_ring *rds_ring; struct qlcnic_adapter *adapter = sds_ring->adapter; struct list_head *cur; struct status_desc *desc; struct qlcnic_rx_buffer *rxbuf; + int opcode, desc_cnt, count = 0; u64 sts_data0, sts_data1; - __le64 owner_phantom = cpu_to_le64(STATUS_OWNER_PHANTOM); - int opcode, ring, desc_cnt, count = 0; + u8 ring; u32 consumer = sds_ring->consumer; while (count < max) { @@ -1020,7 +1363,6 @@ int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, int max) desc_cnt = qlcnic_get_sts_desc_cnt(sts_data0); opcode = qlcnic_get_sts_opcode(sts_data0); - switch (opcode) { case QLCNIC_RXPKT_DESC: case QLCNIC_OLD_RXPKT_DESC: @@ -1040,18 +1382,16 @@ int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, int max) default: goto skip; } - WARN_ON(desc_cnt > 1); if (likely(rxbuf)) list_add_tail(&rxbuf->list, &sds_ring->free_list[ring]); else adapter->stats.null_rxbuf++; - skip: for (; desc_cnt > 0; desc_cnt--) { desc = &sds_ring->desc_head[consumer]; - desc->status_desc_data[0] = owner_phantom; + desc->status_desc_data[0] = QLCNIC_DESC_OWNER_FW; consumer = get_next_index(consumer, sds_ring->num_desc); } count++; @@ -1059,7 +1399,6 @@ skip: for (ring = 0; ring < adapter->max_rds_rings; ring++) { rds_ring = &adapter->recv_ctx->rds_rings[ring]; - if (!list_empty(&sds_ring->free_list[ring])) { list_for_each(cur, &sds_ring->free_list[ring]) { rxbuf = list_entry(cur, struct qlcnic_rx_buffer, @@ -1072,7 +1411,7 @@ skip: spin_unlock(&rds_ring->lock); } - qlcnic_post_rx_buffers_nodb(adapter, rds_ring); + qlcnic_post_rx_buffers_nodb(adapter, rds_ring, ring); } if (count) { @@ -1084,12 +1423,12 @@ skip: } void qlcnic_post_rx_buffers(struct qlcnic_adapter *adapter, - struct qlcnic_host_rds_ring *rds_ring) + struct qlcnic_host_rds_ring *rds_ring, u8 ring_id) { struct rcv_desc *pdesc; struct qlcnic_rx_buffer *buffer; int count = 0; - u32 producer; + u32 producer, handle; struct list_head *head; producer = rds_ring->producer; @@ -1110,7 +1449,9 @@ void qlcnic_post_rx_buffers(struct qlcnic_adapter *adapter, /* make a rcv descriptor */ pdesc = &rds_ring->desc_head[producer]; pdesc->addr_buffer = cpu_to_le64(buffer->dma); - pdesc->reference_handle = cpu_to_le16(buffer->ref_handle); + handle = qlcnic_get_ref_handle(adapter, buffer->ref_handle, + ring_id); + pdesc->reference_handle = cpu_to_le16(handle); pdesc->buffer_length = cpu_to_le32(rds_ring->dma_size); producer = get_next_index(producer, rds_ring->num_desc); } @@ -1180,7 +1521,7 @@ static void qlcnic_process_rcv_diag(struct qlcnic_adapter *adapter, int ring, return; } -void qlcnic_process_rcv_ring_diag(struct qlcnic_host_sds_ring *sds_ring) +void qlcnic_82xx_process_rcv_ring_diag(struct qlcnic_host_sds_ring *sds_ring) { struct qlcnic_adapter *adapter = sds_ring->adapter; struct status_desc *desc; @@ -1217,93 +1558,651 @@ void qlcnic_process_rcv_ring_diag(struct qlcnic_host_sds_ring *sds_ring) writel(consumer, sds_ring->crb_sts_consumer); } -void qlcnic_fetch_mac(u32 off1, u32 off2, u8 alt_mac, u8 *mac) +int qlcnic_82xx_napi_add(struct qlcnic_adapter *adapter, + struct net_device *netdev) { - u32 mac_low, mac_high; - int i; + int ring; + struct qlcnic_host_sds_ring *sds_ring; + struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; + struct qlcnic_host_tx_ring *tx_ring; - mac_low = off1; - mac_high = off2; + if (qlcnic_alloc_sds_rings(recv_ctx, adapter->drv_sds_rings)) + return -ENOMEM; - if (alt_mac) { - mac_low |= (mac_low >> 16) | (mac_high << 16); - mac_high >>= 16; + for (ring = 0; ring < adapter->drv_sds_rings; ring++) { + sds_ring = &recv_ctx->sds_rings[ring]; + if (qlcnic_check_multi_tx(adapter) && + !adapter->ahw->diag_test) { + netif_napi_add(netdev, &sds_ring->napi, qlcnic_rx_poll, + NAPI_POLL_WEIGHT); + } else { + if (ring == (adapter->drv_sds_rings - 1)) + netif_napi_add(netdev, &sds_ring->napi, + qlcnic_poll, + NAPI_POLL_WEIGHT); + else + netif_napi_add(netdev, &sds_ring->napi, + qlcnic_rx_poll, + NAPI_POLL_WEIGHT); + } + } + + if (qlcnic_alloc_tx_rings(adapter, netdev)) { + qlcnic_free_sds_rings(recv_ctx); + return -ENOMEM; } - for (i = 0; i < 2; i++) - mac[i] = (u8)(mac_high >> ((1 - i) * 8)); - for (i = 2; i < 6; i++) - mac[i] = (u8)(mac_low >> ((5 - i) * 8)); + if (qlcnic_check_multi_tx(adapter) && !adapter->ahw->diag_test) { + for (ring = 0; ring < adapter->drv_tx_rings; ring++) { + tx_ring = &adapter->tx_ring[ring]; + netif_napi_add(netdev, &tx_ring->napi, qlcnic_tx_poll, + NAPI_POLL_WEIGHT); + } + } + + return 0; } -int qlcnic_napi_add(struct qlcnic_adapter *adapter, struct net_device *netdev) +void qlcnic_82xx_napi_del(struct qlcnic_adapter *adapter) { - int ring, max_sds_rings; + int ring; struct qlcnic_host_sds_ring *sds_ring; struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; + struct qlcnic_host_tx_ring *tx_ring; - if (qlcnic_alloc_sds_rings(recv_ctx, adapter->max_sds_rings)) - return -ENOMEM; + for (ring = 0; ring < adapter->drv_sds_rings; ring++) { + sds_ring = &recv_ctx->sds_rings[ring]; + netif_napi_del(&sds_ring->napi); + } + + qlcnic_free_sds_rings(adapter->recv_ctx); - max_sds_rings = adapter->max_sds_rings; + if (qlcnic_check_multi_tx(adapter) && !adapter->ahw->diag_test) { + for (ring = 0; ring < adapter->drv_tx_rings; ring++) { + tx_ring = &adapter->tx_ring[ring]; + netif_napi_del(&tx_ring->napi); + } + } - for (ring = 0; ring < adapter->max_sds_rings; ring++) { - sds_ring = &recv_ctx->sds_rings[ring]; + qlcnic_free_tx_rings(adapter); +} - if (ring == max_sds_rings - 1) - netif_napi_add(netdev, &sds_ring->napi, qlcnic_poll, - QLCNIC_NETDEV_WEIGHT / max_sds_rings); - else - netif_napi_add(netdev, &sds_ring->napi, qlcnic_rx_poll, - QLCNIC_NETDEV_WEIGHT*2); +void qlcnic_82xx_napi_enable(struct qlcnic_adapter *adapter) +{ + int ring; + struct qlcnic_host_sds_ring *sds_ring; + struct qlcnic_host_tx_ring *tx_ring; + struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; + + if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC) + return; + + for (ring = 0; ring < adapter->drv_sds_rings; ring++) { + sds_ring = &recv_ctx->sds_rings[ring]; + napi_enable(&sds_ring->napi); + qlcnic_enable_sds_intr(adapter, sds_ring); } - return 0; + if (qlcnic_check_multi_tx(adapter) && + (adapter->flags & QLCNIC_MSIX_ENABLED) && + !adapter->ahw->diag_test) { + for (ring = 0; ring < adapter->drv_tx_rings; ring++) { + tx_ring = &adapter->tx_ring[ring]; + napi_enable(&tx_ring->napi); + qlcnic_enable_tx_intr(adapter, tx_ring); + } + } } -void qlcnic_napi_del(struct qlcnic_adapter *adapter) +void qlcnic_82xx_napi_disable(struct qlcnic_adapter *adapter) { int ring; struct qlcnic_host_sds_ring *sds_ring; + struct qlcnic_host_tx_ring *tx_ring; struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; - for (ring = 0; ring < adapter->max_sds_rings; ring++) { + if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC) + return; + + for (ring = 0; ring < adapter->drv_sds_rings; ring++) { sds_ring = &recv_ctx->sds_rings[ring]; - netif_napi_del(&sds_ring->napi); + qlcnic_disable_sds_intr(adapter, sds_ring); + napi_synchronize(&sds_ring->napi); + napi_disable(&sds_ring->napi); } - qlcnic_free_sds_rings(adapter->recv_ctx); + if ((adapter->flags & QLCNIC_MSIX_ENABLED) && + !adapter->ahw->diag_test && + qlcnic_check_multi_tx(adapter)) { + for (ring = 0; ring < adapter->drv_tx_rings; ring++) { + tx_ring = &adapter->tx_ring[ring]; + qlcnic_disable_tx_intr(adapter, tx_ring); + napi_synchronize(&tx_ring->napi); + napi_disable(&tx_ring->napi); + } + } +} + +#define QLC_83XX_NORMAL_LB_PKT (1ULL << 36) +#define QLC_83XX_LRO_LB_PKT (1ULL << 46) + +static inline int qlcnic_83xx_is_lb_pkt(u64 sts_data, int lro_pkt) +{ + if (lro_pkt) + return (sts_data & QLC_83XX_LRO_LB_PKT) ? 1 : 0; + else + return (sts_data & QLC_83XX_NORMAL_LB_PKT) ? 1 : 0; +} + +#define QLCNIC_ENCAP_LENGTH_MASK 0x7f + +static inline u8 qlcnic_encap_length(u64 sts_data) +{ + return sts_data & QLCNIC_ENCAP_LENGTH_MASK; +} + +static struct qlcnic_rx_buffer * +qlcnic_83xx_process_rcv(struct qlcnic_adapter *adapter, + struct qlcnic_host_sds_ring *sds_ring, + u8 ring, u64 sts_data[]) +{ + struct net_device *netdev = adapter->netdev; + struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; + struct qlcnic_rx_buffer *buffer; + struct sk_buff *skb; + struct qlcnic_host_rds_ring *rds_ring; + int index, length, cksum, is_lb_pkt; + u16 vid = 0xffff; + int err; + + if (unlikely(ring >= adapter->max_rds_rings)) + return NULL; + + rds_ring = &recv_ctx->rds_rings[ring]; + + index = qlcnic_83xx_hndl(sts_data[0]); + if (unlikely(index >= rds_ring->num_desc)) + return NULL; + + buffer = &rds_ring->rx_buf_arr[index]; + length = qlcnic_83xx_pktln(sts_data[0]); + cksum = qlcnic_83xx_csum_status(sts_data[1]); + skb = qlcnic_process_rxbuf(adapter, rds_ring, index, cksum); + if (!skb) + return buffer; + + if (length > rds_ring->skb_size) + skb_put(skb, rds_ring->skb_size); + else + skb_put(skb, length); + + err = qlcnic_check_rx_tagging(adapter, skb, &vid); + + if (adapter->rx_mac_learn) { + is_lb_pkt = qlcnic_83xx_is_lb_pkt(sts_data[1], 0); + qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, vid); + } + + if (unlikely(err)) { + adapter->stats.rxdropped++; + dev_kfree_skb(skb); + return buffer; + } + + skb->protocol = eth_type_trans(skb, netdev); + + if (qlcnic_encap_length(sts_data[1]) && + skb->ip_summed == CHECKSUM_UNNECESSARY) { + skb->encapsulation = 1; + adapter->stats.encap_rx_csummed++; + } + + if (vid != 0xffff) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); + + napi_gro_receive(&sds_ring->napi, skb); + + adapter->stats.rx_pkts++; + adapter->stats.rxbytes += length; + + return buffer; +} + +static struct qlcnic_rx_buffer * +qlcnic_83xx_process_lro(struct qlcnic_adapter *adapter, + u8 ring, u64 sts_data[]) +{ + struct net_device *netdev = adapter->netdev; + struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; + struct qlcnic_rx_buffer *buffer; + struct sk_buff *skb; + struct qlcnic_host_rds_ring *rds_ring; + struct iphdr *iph; + struct ipv6hdr *ipv6h; + struct tcphdr *th; + bool push; + int l2_hdr_offset, l4_hdr_offset; + int index, is_lb_pkt; + u16 lro_length, length, data_offset, gso_size; + u16 vid = 0xffff; + int err; + + if (unlikely(ring >= adapter->max_rds_rings)) + return NULL; + + rds_ring = &recv_ctx->rds_rings[ring]; + + index = qlcnic_83xx_hndl(sts_data[0]); + if (unlikely(index >= rds_ring->num_desc)) + return NULL; + + buffer = &rds_ring->rx_buf_arr[index]; + + lro_length = qlcnic_83xx_lro_pktln(sts_data[0]); + l2_hdr_offset = qlcnic_83xx_l2_hdr_off(sts_data[1]); + l4_hdr_offset = qlcnic_83xx_l4_hdr_off(sts_data[1]); + push = qlcnic_83xx_is_psh_bit(sts_data[1]); + + skb = qlcnic_process_rxbuf(adapter, rds_ring, index, STATUS_CKSUM_OK); + if (!skb) + return buffer; + + if (qlcnic_83xx_is_tstamp(sts_data[1])) + data_offset = l4_hdr_offset + QLCNIC_TCP_TS_HDR_SIZE; + else + data_offset = l4_hdr_offset + QLCNIC_TCP_HDR_SIZE; + + skb_put(skb, lro_length + data_offset); + skb_pull(skb, l2_hdr_offset); + + err = qlcnic_check_rx_tagging(adapter, skb, &vid); + + if (adapter->rx_mac_learn) { + is_lb_pkt = qlcnic_83xx_is_lb_pkt(sts_data[1], 1); + qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, vid); + } + + if (unlikely(err)) { + adapter->stats.rxdropped++; + dev_kfree_skb(skb); + return buffer; + } + + skb->protocol = eth_type_trans(skb, netdev); + if (ntohs(skb->protocol) == ETH_P_IPV6) { + ipv6h = (struct ipv6hdr *)skb->data; + th = (struct tcphdr *)(skb->data + sizeof(struct ipv6hdr)); + + length = (th->doff << 2) + lro_length; + ipv6h->payload_len = htons(length); + } else { + iph = (struct iphdr *)skb->data; + th = (struct tcphdr *)(skb->data + (iph->ihl << 2)); + length = (iph->ihl << 2) + (th->doff << 2) + lro_length; + csum_replace2(&iph->check, iph->tot_len, htons(length)); + iph->tot_len = htons(length); + } + + th->psh = push; + length = skb->len; + + if (adapter->flags & QLCNIC_FW_LRO_MSS_CAP) { + gso_size = qlcnic_83xx_get_lro_sts_mss(sts_data[0]); + skb_shinfo(skb)->gso_size = gso_size; + if (skb->protocol == htons(ETH_P_IPV6)) + skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6; + else + skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; + } + + if (vid != 0xffff) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); + + netif_receive_skb(skb); + + adapter->stats.lro_pkts++; + adapter->stats.lrobytes += length; + return buffer; +} + +static int qlcnic_83xx_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, + int max) +{ + struct qlcnic_host_rds_ring *rds_ring; + struct qlcnic_adapter *adapter = sds_ring->adapter; + struct list_head *cur; + struct status_desc *desc; + struct qlcnic_rx_buffer *rxbuf = NULL; + u8 ring; + u64 sts_data[2]; + int count = 0, opcode; + u32 consumer = sds_ring->consumer; + + while (count < max) { + desc = &sds_ring->desc_head[consumer]; + sts_data[1] = le64_to_cpu(desc->status_desc_data[1]); + opcode = qlcnic_83xx_opcode(sts_data[1]); + if (!opcode) + break; + sts_data[0] = le64_to_cpu(desc->status_desc_data[0]); + ring = QLCNIC_FETCH_RING_ID(sts_data[0]); + + switch (opcode) { + case QLC_83XX_REG_DESC: + rxbuf = qlcnic_83xx_process_rcv(adapter, sds_ring, + ring, sts_data); + break; + case QLC_83XX_LRO_DESC: + rxbuf = qlcnic_83xx_process_lro(adapter, ring, + sts_data); + break; + default: + dev_info(&adapter->pdev->dev, + "Unknown opcode: 0x%x\n", opcode); + goto skip; + } + + if (likely(rxbuf)) + list_add_tail(&rxbuf->list, &sds_ring->free_list[ring]); + else + adapter->stats.null_rxbuf++; +skip: + desc = &sds_ring->desc_head[consumer]; + /* Reset the descriptor */ + desc->status_desc_data[1] = 0; + consumer = get_next_index(consumer, sds_ring->num_desc); + count++; + } + for (ring = 0; ring < adapter->max_rds_rings; ring++) { + rds_ring = &adapter->recv_ctx->rds_rings[ring]; + if (!list_empty(&sds_ring->free_list[ring])) { + list_for_each(cur, &sds_ring->free_list[ring]) { + rxbuf = list_entry(cur, struct qlcnic_rx_buffer, + list); + qlcnic_alloc_rx_skb(adapter, rds_ring, rxbuf); + } + spin_lock(&rds_ring->lock); + list_splice_tail_init(&sds_ring->free_list[ring], + &rds_ring->free_list); + spin_unlock(&rds_ring->lock); + } + qlcnic_post_rx_buffers_nodb(adapter, rds_ring, ring); + } + if (count) { + sds_ring->consumer = consumer; + writel(consumer, sds_ring->crb_sts_consumer); + } + return count; } -void qlcnic_napi_enable(struct qlcnic_adapter *adapter) +static int qlcnic_83xx_msix_sriov_vf_poll(struct napi_struct *napi, int budget) +{ + int tx_complete; + int work_done; + struct qlcnic_host_sds_ring *sds_ring; + struct qlcnic_adapter *adapter; + struct qlcnic_host_tx_ring *tx_ring; + + sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi); + adapter = sds_ring->adapter; + /* tx ring count = 1 */ + tx_ring = adapter->tx_ring; + + tx_complete = qlcnic_process_cmd_ring(adapter, tx_ring, budget); + work_done = qlcnic_83xx_process_rcv_ring(sds_ring, budget); + if ((work_done < budget) && tx_complete) { + napi_complete(&sds_ring->napi); + qlcnic_enable_sds_intr(adapter, sds_ring); + } + + return work_done; +} + +static int qlcnic_83xx_poll(struct napi_struct *napi, int budget) +{ + int tx_complete; + int work_done; + struct qlcnic_host_sds_ring *sds_ring; + struct qlcnic_adapter *adapter; + struct qlcnic_host_tx_ring *tx_ring; + + sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi); + adapter = sds_ring->adapter; + /* tx ring count = 1 */ + tx_ring = adapter->tx_ring; + + tx_complete = qlcnic_process_cmd_ring(adapter, tx_ring, budget); + work_done = qlcnic_83xx_process_rcv_ring(sds_ring, budget); + if ((work_done < budget) && tx_complete) { + napi_complete(&sds_ring->napi); + qlcnic_enable_sds_intr(adapter, sds_ring); + } + + return work_done; +} + +static int qlcnic_83xx_msix_tx_poll(struct napi_struct *napi, int budget) +{ + int work_done; + struct qlcnic_host_tx_ring *tx_ring; + struct qlcnic_adapter *adapter; + + budget = QLCNIC_TX_POLL_BUDGET; + tx_ring = container_of(napi, struct qlcnic_host_tx_ring, napi); + adapter = tx_ring->adapter; + work_done = qlcnic_process_cmd_ring(adapter, tx_ring, budget); + if (work_done) { + napi_complete(&tx_ring->napi); + if (test_bit(__QLCNIC_DEV_UP , &adapter->state)) + qlcnic_enable_tx_intr(adapter, tx_ring); + } + + return work_done; +} + +static int qlcnic_83xx_rx_poll(struct napi_struct *napi, int budget) +{ + int work_done; + struct qlcnic_host_sds_ring *sds_ring; + struct qlcnic_adapter *adapter; + + sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi); + adapter = sds_ring->adapter; + work_done = qlcnic_83xx_process_rcv_ring(sds_ring, budget); + if (work_done < budget) { + napi_complete(&sds_ring->napi); + if (test_bit(__QLCNIC_DEV_UP, &adapter->state)) + qlcnic_enable_sds_intr(adapter, sds_ring); + } + + return work_done; +} + +void qlcnic_83xx_napi_enable(struct qlcnic_adapter *adapter) { int ring; struct qlcnic_host_sds_ring *sds_ring; + struct qlcnic_host_tx_ring *tx_ring; struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC) return; - for (ring = 0; ring < adapter->max_sds_rings; ring++) { + for (ring = 0; ring < adapter->drv_sds_rings; ring++) { sds_ring = &recv_ctx->sds_rings[ring]; napi_enable(&sds_ring->napi); - qlcnic_enable_int(sds_ring); + if (adapter->flags & QLCNIC_MSIX_ENABLED) + qlcnic_enable_sds_intr(adapter, sds_ring); + } + + if ((adapter->flags & QLCNIC_MSIX_ENABLED) && + !(adapter->flags & QLCNIC_TX_INTR_SHARED)) { + for (ring = 0; ring < adapter->drv_tx_rings; ring++) { + tx_ring = &adapter->tx_ring[ring]; + napi_enable(&tx_ring->napi); + qlcnic_enable_tx_intr(adapter, tx_ring); + } } } -void qlcnic_napi_disable(struct qlcnic_adapter *adapter) +void qlcnic_83xx_napi_disable(struct qlcnic_adapter *adapter) { int ring; struct qlcnic_host_sds_ring *sds_ring; struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; + struct qlcnic_host_tx_ring *tx_ring; if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC) return; - for (ring = 0; ring < adapter->max_sds_rings; ring++) { + for (ring = 0; ring < adapter->drv_sds_rings; ring++) { sds_ring = &recv_ctx->sds_rings[ring]; - qlcnic_disable_int(sds_ring); + if (adapter->flags & QLCNIC_MSIX_ENABLED) + qlcnic_disable_sds_intr(adapter, sds_ring); napi_synchronize(&sds_ring->napi); napi_disable(&sds_ring->napi); } + + if ((adapter->flags & QLCNIC_MSIX_ENABLED) && + !(adapter->flags & QLCNIC_TX_INTR_SHARED)) { + for (ring = 0; ring < adapter->drv_tx_rings; ring++) { + tx_ring = &adapter->tx_ring[ring]; + qlcnic_disable_tx_intr(adapter, tx_ring); + napi_synchronize(&tx_ring->napi); + napi_disable(&tx_ring->napi); + } + } +} + +int qlcnic_83xx_napi_add(struct qlcnic_adapter *adapter, + struct net_device *netdev) +{ + int ring; + struct qlcnic_host_sds_ring *sds_ring; + struct qlcnic_host_tx_ring *tx_ring; + struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; + + if (qlcnic_alloc_sds_rings(recv_ctx, adapter->drv_sds_rings)) + return -ENOMEM; + + for (ring = 0; ring < adapter->drv_sds_rings; ring++) { + sds_ring = &recv_ctx->sds_rings[ring]; + if (adapter->flags & QLCNIC_MSIX_ENABLED) { + if (!(adapter->flags & QLCNIC_TX_INTR_SHARED)) + netif_napi_add(netdev, &sds_ring->napi, + qlcnic_83xx_rx_poll, + NAPI_POLL_WEIGHT); + else + netif_napi_add(netdev, &sds_ring->napi, + qlcnic_83xx_msix_sriov_vf_poll, + NAPI_POLL_WEIGHT); + + } else { + netif_napi_add(netdev, &sds_ring->napi, + qlcnic_83xx_poll, + NAPI_POLL_WEIGHT); + } + } + + if (qlcnic_alloc_tx_rings(adapter, netdev)) { + qlcnic_free_sds_rings(recv_ctx); + return -ENOMEM; + } + + if ((adapter->flags & QLCNIC_MSIX_ENABLED) && + !(adapter->flags & QLCNIC_TX_INTR_SHARED)) { + for (ring = 0; ring < adapter->drv_tx_rings; ring++) { + tx_ring = &adapter->tx_ring[ring]; + netif_napi_add(netdev, &tx_ring->napi, + qlcnic_83xx_msix_tx_poll, + NAPI_POLL_WEIGHT); + } + } + + return 0; +} + +void qlcnic_83xx_napi_del(struct qlcnic_adapter *adapter) +{ + int ring; + struct qlcnic_host_sds_ring *sds_ring; + struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; + struct qlcnic_host_tx_ring *tx_ring; + + for (ring = 0; ring < adapter->drv_sds_rings; ring++) { + sds_ring = &recv_ctx->sds_rings[ring]; + netif_napi_del(&sds_ring->napi); + } + + qlcnic_free_sds_rings(adapter->recv_ctx); + + if ((adapter->flags & QLCNIC_MSIX_ENABLED) && + !(adapter->flags & QLCNIC_TX_INTR_SHARED)) { + for (ring = 0; ring < adapter->drv_tx_rings; ring++) { + tx_ring = &adapter->tx_ring[ring]; + netif_napi_del(&tx_ring->napi); + } + } + + qlcnic_free_tx_rings(adapter); +} + +static void qlcnic_83xx_process_rcv_diag(struct qlcnic_adapter *adapter, + int ring, u64 sts_data[]) +{ + struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx; + struct sk_buff *skb; + struct qlcnic_host_rds_ring *rds_ring; + int index, length; + + if (unlikely(ring >= adapter->max_rds_rings)) + return; + + rds_ring = &recv_ctx->rds_rings[ring]; + index = qlcnic_83xx_hndl(sts_data[0]); + if (unlikely(index >= rds_ring->num_desc)) + return; + + length = qlcnic_83xx_pktln(sts_data[0]); + + skb = qlcnic_process_rxbuf(adapter, rds_ring, index, STATUS_CKSUM_OK); + if (!skb) + return; + + if (length > rds_ring->skb_size) + skb_put(skb, rds_ring->skb_size); + else + skb_put(skb, length); + + if (!qlcnic_check_loopback_buff(skb->data, adapter->mac_addr)) + adapter->ahw->diag_cnt++; + else + dump_skb(skb, adapter); + + dev_kfree_skb_any(skb); + return; +} + +void qlcnic_83xx_process_rcv_ring_diag(struct qlcnic_host_sds_ring *sds_ring) +{ + struct qlcnic_adapter *adapter = sds_ring->adapter; + struct status_desc *desc; + u64 sts_data[2]; + int ring, opcode; + u32 consumer = sds_ring->consumer; + + desc = &sds_ring->desc_head[consumer]; + sts_data[0] = le64_to_cpu(desc->status_desc_data[0]); + sts_data[1] = le64_to_cpu(desc->status_desc_data[1]); + opcode = qlcnic_83xx_opcode(sts_data[1]); + if (!opcode) + return; + + ring = QLCNIC_FETCH_RING_ID(qlcnic_83xx_hndl(sts_data[0])); + qlcnic_83xx_process_rcv_diag(adapter, ring, sts_data); + desc = &sds_ring->desc_head[consumer]; + desc->status_desc_data[0] = cpu_to_le64(STATUS_OWNER_PHANTOM); + consumer = get_next_index(consumer, sds_ring->num_desc); + sds_ring->consumer = consumer; + writel(consumer, sds_ring->crb_sts_consumer); } |