diff options
Diffstat (limited to 'drivers/net/ethernet/brocade/bna/bnad.c')
| -rw-r--r-- | drivers/net/ethernet/brocade/bna/bnad.c | 1758 |
1 files changed, 1047 insertions, 711 deletions
diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c index be7d91e4b78..3a77f9ead00 100644 --- a/drivers/net/ethernet/brocade/bna/bnad.c +++ b/drivers/net/ethernet/brocade/bna/bnad.c @@ -52,7 +52,7 @@ MODULE_PARM_DESC(bna_debugfs_enable, "Enables debugfs feature, default=1," /* * Global variables */ -u32 bnad_rxqs_per_cq = 2; +static u32 bnad_rxqs_per_cq = 2; static u32 bna_id; static struct mutex bnad_list_mutex; static LIST_HEAD(bnad_list); @@ -61,27 +61,19 @@ static const u8 bnad_bcast_addr[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; /* * Local MACROS */ -#define BNAD_TX_UNMAPQ_DEPTH (bnad->txq_depth * 2) - -#define BNAD_RX_UNMAPQ_DEPTH (bnad->rxq_depth) - #define BNAD_GET_MBOX_IRQ(_bnad) \ (((_bnad)->cfg_flags & BNAD_CF_MSIX) ? \ ((_bnad)->msix_table[BNAD_MAILBOX_MSIX_INDEX].vector) : \ ((_bnad)->pcidev->irq)) -#define BNAD_FILL_UNMAPQ_MEM_REQ(_res_info, _num, _depth) \ +#define BNAD_FILL_UNMAPQ_MEM_REQ(_res_info, _num, _size) \ do { \ (_res_info)->res_type = BNA_RES_T_MEM; \ (_res_info)->res_u.mem_info.mem_type = BNA_MEM_T_KVA; \ (_res_info)->res_u.mem_info.num = (_num); \ - (_res_info)->res_u.mem_info.len = \ - sizeof(struct bnad_unmap_q) + \ - (sizeof(struct bnad_skb_unmap) * ((_depth) - 1)); \ + (_res_info)->res_u.mem_info.len = (_size); \ } while (0) -#define BNAD_TXRX_SYNC_MDELAY 250 /* 250 msecs */ - static void bnad_add_to_list(struct bnad *bnad) { @@ -103,49 +95,61 @@ bnad_remove_from_list(struct bnad *bnad) * Reinitialize completions in CQ, once Rx is taken down */ static void -bnad_cq_cmpl_init(struct bnad *bnad, struct bna_ccb *ccb) +bnad_cq_cleanup(struct bnad *bnad, struct bna_ccb *ccb) { - struct bna_cq_entry *cmpl, *next_cmpl; - unsigned int wi_range, wis = 0, ccb_prod = 0; + struct bna_cq_entry *cmpl; int i; - BNA_CQ_QPGE_PTR_GET(ccb_prod, ccb->sw_qpt, cmpl, - wi_range); - for (i = 0; i < ccb->q_depth; i++) { - wis++; - if (likely(--wi_range)) - next_cmpl = cmpl + 1; - else { - BNA_QE_INDX_ADD(ccb_prod, wis, ccb->q_depth); - wis = 0; - BNA_CQ_QPGE_PTR_GET(ccb_prod, ccb->sw_qpt, - next_cmpl, wi_range); - } + cmpl = &((struct bna_cq_entry *)ccb->sw_q)[i]; cmpl->valid = 0; - cmpl = next_cmpl; } } +/* Tx Datapath functions */ + + +/* Caller should ensure that the entry at unmap_q[index] is valid */ static u32 -bnad_pci_unmap_skb(struct device *pdev, struct bnad_skb_unmap *array, - u32 index, u32 depth, struct sk_buff *skb, u32 frag) +bnad_tx_buff_unmap(struct bnad *bnad, + struct bnad_tx_unmap *unmap_q, + u32 q_depth, u32 index) { - int j; - array[index].skb = NULL; - - dma_unmap_single(pdev, dma_unmap_addr(&array[index], dma_addr), - skb_headlen(skb), DMA_TO_DEVICE); - dma_unmap_addr_set(&array[index], dma_addr, 0); - BNA_QE_INDX_ADD(index, 1, depth); + struct bnad_tx_unmap *unmap; + struct sk_buff *skb; + int vector, nvecs; + + unmap = &unmap_q[index]; + nvecs = unmap->nvecs; + + skb = unmap->skb; + unmap->skb = NULL; + unmap->nvecs = 0; + dma_unmap_single(&bnad->pcidev->dev, + dma_unmap_addr(&unmap->vectors[0], dma_addr), + skb_headlen(skb), DMA_TO_DEVICE); + dma_unmap_addr_set(&unmap->vectors[0], dma_addr, 0); + nvecs--; + + vector = 0; + while (nvecs) { + vector++; + if (vector == BFI_TX_MAX_VECTORS_PER_WI) { + vector = 0; + BNA_QE_INDX_INC(index, q_depth); + unmap = &unmap_q[index]; + } - for (j = 0; j < frag; j++) { - dma_unmap_page(pdev, dma_unmap_addr(&array[index], dma_addr), - skb_frag_size(&skb_shinfo(skb)->frags[j]), DMA_TO_DEVICE); - dma_unmap_addr_set(&array[index], dma_addr, 0); - BNA_QE_INDX_ADD(index, 1, depth); + dma_unmap_page(&bnad->pcidev->dev, + dma_unmap_addr(&unmap->vectors[vector], dma_addr), + dma_unmap_len(&unmap->vectors[vector], dma_len), + DMA_TO_DEVICE); + dma_unmap_addr_set(&unmap->vectors[vector], dma_addr, 0); + nvecs--; } + BNA_QE_INDX_INC(index, q_depth); + return index; } @@ -155,87 +159,64 @@ bnad_pci_unmap_skb(struct device *pdev, struct bnad_skb_unmap *array, * so DMA unmap & freeing is fine. */ static void -bnad_free_all_txbufs(struct bnad *bnad, - struct bna_tcb *tcb) +bnad_txq_cleanup(struct bnad *bnad, struct bna_tcb *tcb) { - u32 unmap_cons; - struct bnad_unmap_q *unmap_q = tcb->unmap_q; - struct bnad_skb_unmap *unmap_array; - struct sk_buff *skb = NULL; - int q; - - unmap_array = unmap_q->unmap_array; + struct bnad_tx_unmap *unmap_q = tcb->unmap_q; + struct sk_buff *skb; + int i; - for (q = 0; q < unmap_q->q_depth; q++) { - skb = unmap_array[q].skb; + for (i = 0; i < tcb->q_depth; i++) { + skb = unmap_q[i].skb; if (!skb) continue; - - unmap_cons = q; - unmap_cons = bnad_pci_unmap_skb(&bnad->pcidev->dev, unmap_array, - unmap_cons, unmap_q->q_depth, skb, - skb_shinfo(skb)->nr_frags); + bnad_tx_buff_unmap(bnad, unmap_q, tcb->q_depth, i); dev_kfree_skb_any(skb); } } -/* Data Path Handlers */ - /* - * bnad_free_txbufs : Frees the Tx bufs on Tx completion + * bnad_txcmpl_process : Frees the Tx bufs on Tx completion * Can be called in a) Interrupt context * b) Sending context - * c) Tasklet context */ static u32 -bnad_free_txbufs(struct bnad *bnad, - struct bna_tcb *tcb) +bnad_txcmpl_process(struct bnad *bnad, struct bna_tcb *tcb) { - u32 unmap_cons, sent_packets = 0, sent_bytes = 0; - u16 wis, updated_hw_cons; - struct bnad_unmap_q *unmap_q = tcb->unmap_q; - struct bnad_skb_unmap *unmap_array; - struct sk_buff *skb; + u32 sent_packets = 0, sent_bytes = 0; + u32 wis, unmap_wis, hw_cons, cons, q_depth; + struct bnad_tx_unmap *unmap_q = tcb->unmap_q; + struct bnad_tx_unmap *unmap; + struct sk_buff *skb; - /* - * Just return if TX is stopped. This check is useful - * when bnad_free_txbufs() runs out of a tasklet scheduled - * before bnad_cb_tx_cleanup() cleared BNAD_TXQ_TX_STARTED bit - * but this routine runs actually after the cleanup has been - * executed. - */ + /* Just return if TX is stopped */ if (!test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) return 0; - updated_hw_cons = *(tcb->hw_consumer_index); - - wis = BNA_Q_INDEX_CHANGE(tcb->consumer_index, - updated_hw_cons, tcb->q_depth); + hw_cons = *(tcb->hw_consumer_index); + cons = tcb->consumer_index; + q_depth = tcb->q_depth; + wis = BNA_Q_INDEX_CHANGE(cons, hw_cons, q_depth); BUG_ON(!(wis <= BNA_QE_IN_USE_CNT(tcb, tcb->q_depth))); - unmap_array = unmap_q->unmap_array; - unmap_cons = unmap_q->consumer_index; - - prefetch(&unmap_array[unmap_cons + 1]); while (wis) { - skb = unmap_array[unmap_cons].skb; + unmap = &unmap_q[cons]; + + skb = unmap->skb; sent_packets++; sent_bytes += skb->len; - wis -= BNA_TXQ_WI_NEEDED(1 + skb_shinfo(skb)->nr_frags); - unmap_cons = bnad_pci_unmap_skb(&bnad->pcidev->dev, unmap_array, - unmap_cons, unmap_q->q_depth, skb, - skb_shinfo(skb)->nr_frags); + unmap_wis = BNA_TXQ_WI_NEEDED(unmap->nvecs); + wis -= unmap_wis; + cons = bnad_tx_buff_unmap(bnad, unmap_q, q_depth, cons); dev_kfree_skb_any(skb); } /* Update consumer pointers. */ - tcb->consumer_index = updated_hw_cons; - unmap_q->consumer_index = unmap_cons; + tcb->consumer_index = hw_cons; tcb->txq->tx_packets += sent_packets; tcb->txq->tx_bytes += sent_bytes; @@ -243,57 +224,8 @@ bnad_free_txbufs(struct bnad *bnad, return sent_packets; } -/* Tx Free Tasklet function */ -/* Frees for all the tcb's in all the Tx's */ -/* - * Scheduled from sending context, so that - * the fat Tx lock is not held for too long - * in the sending context. - */ -static void -bnad_tx_free_tasklet(unsigned long bnad_ptr) -{ - struct bnad *bnad = (struct bnad *)bnad_ptr; - struct bna_tcb *tcb; - u32 acked = 0; - int i, j; - - for (i = 0; i < bnad->num_tx; i++) { - for (j = 0; j < bnad->num_txq_per_tx; j++) { - tcb = bnad->tx_info[i].tcb[j]; - if (!tcb) - continue; - if (((u16) (*tcb->hw_consumer_index) != - tcb->consumer_index) && - (!test_and_set_bit(BNAD_TXQ_FREE_SENT, - &tcb->flags))) { - acked = bnad_free_txbufs(bnad, tcb); - if (likely(test_bit(BNAD_TXQ_TX_STARTED, - &tcb->flags))) - bna_ib_ack(tcb->i_dbell, acked); - smp_mb__before_clear_bit(); - clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); - } - if (unlikely(!test_bit(BNAD_TXQ_TX_STARTED, - &tcb->flags))) - continue; - if (netif_queue_stopped(bnad->netdev)) { - if (acked && netif_carrier_ok(bnad->netdev) && - BNA_QE_FREE_CNT(tcb, tcb->q_depth) >= - BNAD_NETIF_WAKE_THRESHOLD) { - netif_wake_queue(bnad->netdev); - /* TODO */ - /* Counters for individual TxQs? */ - BNAD_UPDATE_CTR(bnad, - netif_queue_wakeup); - } - } - } - } -} - static u32 -bnad_tx(struct bnad *bnad, struct bna_tcb *tcb) +bnad_tx_complete(struct bnad *bnad, struct bna_tcb *tcb) { struct net_device *netdev = bnad->netdev; u32 sent = 0; @@ -301,7 +233,7 @@ bnad_tx(struct bnad *bnad, struct bna_tcb *tcb) if (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) return 0; - sent = bnad_free_txbufs(bnad, tcb); + sent = bnad_txcmpl_process(bnad, tcb); if (sent) { if (netif_queue_stopped(netdev) && netif_carrier_ok(netdev) && @@ -317,7 +249,7 @@ bnad_tx(struct bnad *bnad, struct bna_tcb *tcb) if (likely(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) bna_ib_ack(tcb->i_dbell, sent); - smp_mb__before_clear_bit(); + smp_mb__before_atomic(); clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); return sent; @@ -330,142 +262,358 @@ bnad_msix_tx(int irq, void *data) struct bna_tcb *tcb = (struct bna_tcb *)data; struct bnad *bnad = tcb->bnad; - bnad_tx(bnad, tcb); + bnad_tx_complete(bnad, tcb); return IRQ_HANDLED; } -static void -bnad_reset_rcb(struct bnad *bnad, struct bna_rcb *rcb) +static inline void +bnad_rxq_alloc_uninit(struct bnad *bnad, struct bna_rcb *rcb) +{ + struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; + + unmap_q->reuse_pi = -1; + unmap_q->alloc_order = -1; + unmap_q->map_size = 0; + unmap_q->type = BNAD_RXBUF_NONE; +} + +/* Default is page-based allocation. Multi-buffer support - TBD */ +static int +bnad_rxq_alloc_init(struct bnad *bnad, struct bna_rcb *rcb) { - struct bnad_unmap_q *unmap_q = rcb->unmap_q; + struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; + int order; - rcb->producer_index = 0; - rcb->consumer_index = 0; + bnad_rxq_alloc_uninit(bnad, rcb); - unmap_q->producer_index = 0; - unmap_q->consumer_index = 0; + order = get_order(rcb->rxq->buffer_size); + + unmap_q->type = BNAD_RXBUF_PAGE; + + if (bna_is_small_rxq(rcb->id)) { + unmap_q->alloc_order = 0; + unmap_q->map_size = rcb->rxq->buffer_size; + } else { + if (rcb->rxq->multi_buffer) { + unmap_q->alloc_order = 0; + unmap_q->map_size = rcb->rxq->buffer_size; + unmap_q->type = BNAD_RXBUF_MULTI_BUFF; + } else { + unmap_q->alloc_order = order; + unmap_q->map_size = + (rcb->rxq->buffer_size > 2048) ? + PAGE_SIZE << order : 2048; + } + } + + BUG_ON(((PAGE_SIZE << order) % unmap_q->map_size)); + + return 0; +} + +static inline void +bnad_rxq_cleanup_page(struct bnad *bnad, struct bnad_rx_unmap *unmap) +{ + if (!unmap->page) + return; + + dma_unmap_page(&bnad->pcidev->dev, + dma_unmap_addr(&unmap->vector, dma_addr), + unmap->vector.len, DMA_FROM_DEVICE); + put_page(unmap->page); + unmap->page = NULL; + dma_unmap_addr_set(&unmap->vector, dma_addr, 0); + unmap->vector.len = 0; +} + +static inline void +bnad_rxq_cleanup_skb(struct bnad *bnad, struct bnad_rx_unmap *unmap) +{ + if (!unmap->skb) + return; + + dma_unmap_single(&bnad->pcidev->dev, + dma_unmap_addr(&unmap->vector, dma_addr), + unmap->vector.len, DMA_FROM_DEVICE); + dev_kfree_skb_any(unmap->skb); + unmap->skb = NULL; + dma_unmap_addr_set(&unmap->vector, dma_addr, 0); + unmap->vector.len = 0; } static void -bnad_free_all_rxbufs(struct bnad *bnad, struct bna_rcb *rcb) +bnad_rxq_cleanup(struct bnad *bnad, struct bna_rcb *rcb) { - struct bnad_unmap_q *unmap_q; - struct bnad_skb_unmap *unmap_array; - struct sk_buff *skb; - int unmap_cons; + struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; + int i; - unmap_q = rcb->unmap_q; - unmap_array = unmap_q->unmap_array; - for (unmap_cons = 0; unmap_cons < unmap_q->q_depth; unmap_cons++) { - skb = unmap_array[unmap_cons].skb; - if (!skb) - continue; - unmap_array[unmap_cons].skb = NULL; - dma_unmap_single(&bnad->pcidev->dev, - dma_unmap_addr(&unmap_array[unmap_cons], - dma_addr), - rcb->rxq->buffer_size, - DMA_FROM_DEVICE); - dev_kfree_skb(skb); + for (i = 0; i < rcb->q_depth; i++) { + struct bnad_rx_unmap *unmap = &unmap_q->unmap[i]; + + if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) + bnad_rxq_cleanup_skb(bnad, unmap); + else + bnad_rxq_cleanup_page(bnad, unmap); } - bnad_reset_rcb(bnad, rcb); + bnad_rxq_alloc_uninit(bnad, rcb); } -static void -bnad_alloc_n_post_rxbufs(struct bnad *bnad, struct bna_rcb *rcb) +static u32 +bnad_rxq_refill_page(struct bnad *bnad, struct bna_rcb *rcb, u32 nalloc) { - u16 to_alloc, alloced, unmap_prod, wi_range; - struct bnad_unmap_q *unmap_q = rcb->unmap_q; - struct bnad_skb_unmap *unmap_array; + u32 alloced, prod, q_depth; + struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; + struct bnad_rx_unmap *unmap, *prev; struct bna_rxq_entry *rxent; - struct sk_buff *skb; + struct page *page; + u32 page_offset, alloc_size; dma_addr_t dma_addr; + prod = rcb->producer_index; + q_depth = rcb->q_depth; + + alloc_size = PAGE_SIZE << unmap_q->alloc_order; alloced = 0; - to_alloc = - BNA_QE_FREE_CNT(unmap_q, unmap_q->q_depth); - unmap_array = unmap_q->unmap_array; - unmap_prod = unmap_q->producer_index; + while (nalloc--) { + unmap = &unmap_q->unmap[prod]; + + if (unmap_q->reuse_pi < 0) { + page = alloc_pages(GFP_ATOMIC | __GFP_COMP, + unmap_q->alloc_order); + page_offset = 0; + } else { + prev = &unmap_q->unmap[unmap_q->reuse_pi]; + page = prev->page; + page_offset = prev->page_offset + unmap_q->map_size; + get_page(page); + } + + if (unlikely(!page)) { + BNAD_UPDATE_CTR(bnad, rxbuf_alloc_failed); + rcb->rxq->rxbuf_alloc_failed++; + goto finishing; + } + + dma_addr = dma_map_page(&bnad->pcidev->dev, page, page_offset, + unmap_q->map_size, DMA_FROM_DEVICE); + + unmap->page = page; + unmap->page_offset = page_offset; + dma_unmap_addr_set(&unmap->vector, dma_addr, dma_addr); + unmap->vector.len = unmap_q->map_size; + page_offset += unmap_q->map_size; + + if (page_offset < alloc_size) + unmap_q->reuse_pi = prod; + else + unmap_q->reuse_pi = -1; + + rxent = &((struct bna_rxq_entry *)rcb->sw_q)[prod]; + BNA_SET_DMA_ADDR(dma_addr, &rxent->host_addr); + BNA_QE_INDX_INC(prod, q_depth); + alloced++; + } + +finishing: + if (likely(alloced)) { + rcb->producer_index = prod; + smp_mb(); + if (likely(test_bit(BNAD_RXQ_POST_OK, &rcb->flags))) + bna_rxq_prod_indx_doorbell(rcb); + } + + return alloced; +} + +static u32 +bnad_rxq_refill_skb(struct bnad *bnad, struct bna_rcb *rcb, u32 nalloc) +{ + u32 alloced, prod, q_depth, buff_sz; + struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; + struct bnad_rx_unmap *unmap; + struct bna_rxq_entry *rxent; + struct sk_buff *skb; + dma_addr_t dma_addr; + + buff_sz = rcb->rxq->buffer_size; + prod = rcb->producer_index; + q_depth = rcb->q_depth; - BNA_RXQ_QPGE_PTR_GET(unmap_prod, rcb->sw_qpt, rxent, wi_range); + alloced = 0; + while (nalloc--) { + unmap = &unmap_q->unmap[prod]; + + skb = netdev_alloc_skb_ip_align(bnad->netdev, buff_sz); - while (to_alloc--) { - if (!wi_range) - BNA_RXQ_QPGE_PTR_GET(unmap_prod, rcb->sw_qpt, rxent, - wi_range); - skb = netdev_alloc_skb_ip_align(bnad->netdev, - rcb->rxq->buffer_size); if (unlikely(!skb)) { BNAD_UPDATE_CTR(bnad, rxbuf_alloc_failed); rcb->rxq->rxbuf_alloc_failed++; goto finishing; } - unmap_array[unmap_prod].skb = skb; dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data, - rcb->rxq->buffer_size, - DMA_FROM_DEVICE); - dma_unmap_addr_set(&unmap_array[unmap_prod], dma_addr, - dma_addr); - BNA_SET_DMA_ADDR(dma_addr, &rxent->host_addr); - BNA_QE_INDX_ADD(unmap_prod, 1, unmap_q->q_depth); + buff_sz, DMA_FROM_DEVICE); + + unmap->skb = skb; + dma_unmap_addr_set(&unmap->vector, dma_addr, dma_addr); + unmap->vector.len = buff_sz; - rxent++; - wi_range--; + rxent = &((struct bna_rxq_entry *)rcb->sw_q)[prod]; + BNA_SET_DMA_ADDR(dma_addr, &rxent->host_addr); + BNA_QE_INDX_INC(prod, q_depth); alloced++; } finishing: if (likely(alloced)) { - unmap_q->producer_index = unmap_prod; - rcb->producer_index = unmap_prod; + rcb->producer_index = prod; smp_mb(); if (likely(test_bit(BNAD_RXQ_POST_OK, &rcb->flags))) bna_rxq_prod_indx_doorbell(rcb); } + + return alloced; } static inline void -bnad_refill_rxq(struct bnad *bnad, struct bna_rcb *rcb) +bnad_rxq_post(struct bnad *bnad, struct bna_rcb *rcb) { - struct bnad_unmap_q *unmap_q = rcb->unmap_q; + struct bnad_rx_unmap_q *unmap_q = rcb->unmap_q; + u32 to_alloc; + + to_alloc = BNA_QE_FREE_CNT(rcb, rcb->q_depth); + if (!(to_alloc >> BNAD_RXQ_REFILL_THRESHOLD_SHIFT)) + return; + + if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) + bnad_rxq_refill_skb(bnad, rcb, to_alloc); + else + bnad_rxq_refill_page(bnad, rcb, to_alloc); +} + +#define flags_cksum_prot_mask (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \ + BNA_CQ_EF_IPV6 | \ + BNA_CQ_EF_TCP | BNA_CQ_EF_UDP | \ + BNA_CQ_EF_L4_CKSUM_OK) + +#define flags_tcp4 (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \ + BNA_CQ_EF_TCP | BNA_CQ_EF_L4_CKSUM_OK) +#define flags_tcp6 (BNA_CQ_EF_IPV6 | \ + BNA_CQ_EF_TCP | BNA_CQ_EF_L4_CKSUM_OK) +#define flags_udp4 (BNA_CQ_EF_IPV4 | BNA_CQ_EF_L3_CKSUM_OK | \ + BNA_CQ_EF_UDP | BNA_CQ_EF_L4_CKSUM_OK) +#define flags_udp6 (BNA_CQ_EF_IPV6 | \ + BNA_CQ_EF_UDP | BNA_CQ_EF_L4_CKSUM_OK) + +static void +bnad_cq_drop_packet(struct bnad *bnad, struct bna_rcb *rcb, + u32 sop_ci, u32 nvecs) +{ + struct bnad_rx_unmap_q *unmap_q; + struct bnad_rx_unmap *unmap; + u32 ci, vec; + + unmap_q = rcb->unmap_q; + for (vec = 0, ci = sop_ci; vec < nvecs; vec++) { + unmap = &unmap_q->unmap[ci]; + BNA_QE_INDX_INC(ci, rcb->q_depth); - if (!test_and_set_bit(BNAD_RXQ_REFILL, &rcb->flags)) { - if (BNA_QE_FREE_CNT(unmap_q, unmap_q->q_depth) - >> BNAD_RXQ_REFILL_THRESHOLD_SHIFT) - bnad_alloc_n_post_rxbufs(bnad, rcb); - smp_mb__before_clear_bit(); - clear_bit(BNAD_RXQ_REFILL, &rcb->flags); + if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) + bnad_rxq_cleanup_skb(bnad, unmap); + else + bnad_rxq_cleanup_page(bnad, unmap); + } +} + +static void +bnad_cq_setup_skb_frags(struct bna_rcb *rcb, struct sk_buff *skb, + u32 sop_ci, u32 nvecs, u32 last_fraglen) +{ + struct bnad *bnad; + u32 ci, vec, len, totlen = 0; + struct bnad_rx_unmap_q *unmap_q; + struct bnad_rx_unmap *unmap; + + unmap_q = rcb->unmap_q; + bnad = rcb->bnad; + + /* prefetch header */ + prefetch(page_address(unmap_q->unmap[sop_ci].page) + + unmap_q->unmap[sop_ci].page_offset); + + for (vec = 1, ci = sop_ci; vec <= nvecs; vec++) { + unmap = &unmap_q->unmap[ci]; + BNA_QE_INDX_INC(ci, rcb->q_depth); + + dma_unmap_page(&bnad->pcidev->dev, + dma_unmap_addr(&unmap->vector, dma_addr), + unmap->vector.len, DMA_FROM_DEVICE); + + len = (vec == nvecs) ? + last_fraglen : unmap->vector.len; + totlen += len; + + skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, + unmap->page, unmap->page_offset, len); + + unmap->page = NULL; + unmap->vector.len = 0; } + + skb->len += totlen; + skb->data_len += totlen; + skb->truesize += totlen; +} + +static inline void +bnad_cq_setup_skb(struct bnad *bnad, struct sk_buff *skb, + struct bnad_rx_unmap *unmap, u32 len) +{ + prefetch(skb->data); + + dma_unmap_single(&bnad->pcidev->dev, + dma_unmap_addr(&unmap->vector, dma_addr), + unmap->vector.len, DMA_FROM_DEVICE); + + skb_put(skb, len); + skb->protocol = eth_type_trans(skb, bnad->netdev); + + unmap->skb = NULL; + unmap->vector.len = 0; } static u32 -bnad_poll_cq(struct bnad *bnad, struct bna_ccb *ccb, int budget) +bnad_cq_process(struct bnad *bnad, struct bna_ccb *ccb, int budget) { - struct bna_cq_entry *cmpl, *next_cmpl; + struct bna_cq_entry *cq, *cmpl, *next_cmpl; struct bna_rcb *rcb = NULL; - unsigned int wi_range, packets = 0, wis = 0; - struct bnad_unmap_q *unmap_q; - struct bnad_skb_unmap *unmap_array; - struct sk_buff *skb; - u32 flags, unmap_cons; + struct bnad_rx_unmap_q *unmap_q; + struct bnad_rx_unmap *unmap = NULL; + struct sk_buff *skb = NULL; struct bna_pkt_rate *pkt_rt = &ccb->pkt_rate; - struct bnad_rx_ctrl *rx_ctrl = (struct bnad_rx_ctrl *)(ccb->ctrl); + struct bnad_rx_ctrl *rx_ctrl = ccb->ctrl; + u32 packets = 0, len = 0, totlen = 0; + u32 pi, vec, sop_ci = 0, nvecs = 0; + u32 flags, masked_flags; - set_bit(BNAD_FP_IN_RX_PATH, &rx_ctrl->flags); + prefetch(bnad->netdev); - if (!test_bit(BNAD_RXQ_STARTED, &ccb->rcb[0]->flags)) { - clear_bit(BNAD_FP_IN_RX_PATH, &rx_ctrl->flags); - return 0; - } + cq = ccb->sw_q; + cmpl = &cq[ccb->producer_index]; + + while (packets < budget) { + if (!cmpl->valid) + break; + /* The 'valid' field is set by the adapter, only after writing + * the other fields of completion entry. Hence, do not load + * other fields of completion entry *before* the 'valid' is + * loaded. Adding the rmb() here prevents the compiler and/or + * CPU from reordering the reads which would potentially result + * in reading stale values in completion entry. + */ + rmb(); - prefetch(bnad->netdev); - BNA_CQ_QPGE_PTR_GET(ccb->producer_index, ccb->sw_qpt, cmpl, - wi_range); - BUG_ON(!(wi_range <= ccb->q_depth)); - while (cmpl->valid && packets < budget) { - packets++; BNA_UPDATE_PKT_CNT(pkt_rt, ntohs(cmpl->length)); if (bna_is_small_rxq(cmpl->rxq_id)) @@ -474,84 +622,117 @@ bnad_poll_cq(struct bnad *bnad, struct bna_ccb *ccb, int budget) rcb = ccb->rcb[0]; unmap_q = rcb->unmap_q; - unmap_array = unmap_q->unmap_array; - unmap_cons = unmap_q->consumer_index; - - skb = unmap_array[unmap_cons].skb; - BUG_ON(!(skb)); - unmap_array[unmap_cons].skb = NULL; - dma_unmap_single(&bnad->pcidev->dev, - dma_unmap_addr(&unmap_array[unmap_cons], - dma_addr), - rcb->rxq->buffer_size, - DMA_FROM_DEVICE); - BNA_QE_INDX_ADD(unmap_q->consumer_index, 1, unmap_q->q_depth); - - /* Should be more efficient ? Performance ? */ - BNA_QE_INDX_ADD(rcb->consumer_index, 1, rcb->q_depth); - - wis++; - if (likely(--wi_range)) - next_cmpl = cmpl + 1; - else { - BNA_QE_INDX_ADD(ccb->producer_index, wis, ccb->q_depth); - wis = 0; - BNA_CQ_QPGE_PTR_GET(ccb->producer_index, ccb->sw_qpt, - next_cmpl, wi_range); - BUG_ON(!(wi_range <= ccb->q_depth)); + + /* start of packet ci */ + sop_ci = rcb->consumer_index; + + if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) { + unmap = &unmap_q->unmap[sop_ci]; + skb = unmap->skb; + } else { + skb = napi_get_frags(&rx_ctrl->napi); + if (unlikely(!skb)) + break; } - prefetch(next_cmpl); + prefetch(skb); flags = ntohl(cmpl->flags); - if (unlikely - (flags & - (BNA_CQ_EF_MAC_ERROR | BNA_CQ_EF_FCS_ERROR | - BNA_CQ_EF_TOO_LONG))) { - dev_kfree_skb_any(skb); + len = ntohs(cmpl->length); + totlen = len; + nvecs = 1; + + /* Check all the completions for this frame. + * busy-wait doesn't help much, break here. + */ + if (BNAD_RXBUF_IS_MULTI_BUFF(unmap_q->type) && + (flags & BNA_CQ_EF_EOP) == 0) { + pi = ccb->producer_index; + do { + BNA_QE_INDX_INC(pi, ccb->q_depth); + next_cmpl = &cq[pi]; + + if (!next_cmpl->valid) + break; + /* The 'valid' field is set by the adapter, only + * after writing the other fields of completion + * entry. Hence, do not load other fields of + * completion entry *before* the 'valid' is + * loaded. Adding the rmb() here prevents the + * compiler and/or CPU from reordering the reads + * which would potentially result in reading + * stale values in completion entry. + */ + rmb(); + + len = ntohs(next_cmpl->length); + flags = ntohl(next_cmpl->flags); + + nvecs++; + totlen += len; + } while ((flags & BNA_CQ_EF_EOP) == 0); + + if (!next_cmpl->valid) + break; + } + + /* TODO: BNA_CQ_EF_LOCAL ? */ + if (unlikely(flags & (BNA_CQ_EF_MAC_ERROR | + BNA_CQ_EF_FCS_ERROR | + BNA_CQ_EF_TOO_LONG))) { + bnad_cq_drop_packet(bnad, rcb, sop_ci, nvecs); rcb->rxq->rx_packets_with_error++; + goto next; } - skb_put(skb, ntohs(cmpl->length)); + if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) + bnad_cq_setup_skb(bnad, skb, unmap, len); + else + bnad_cq_setup_skb_frags(rcb, skb, sop_ci, nvecs, len); + + packets++; + rcb->rxq->rx_packets++; + rcb->rxq->rx_bytes += totlen; + ccb->bytes_per_intr += totlen; + + masked_flags = flags & flags_cksum_prot_mask; + if (likely ((bnad->netdev->features & NETIF_F_RXCSUM) && - (((flags & BNA_CQ_EF_IPV4) && - (flags & BNA_CQ_EF_L3_CKSUM_OK)) || - (flags & BNA_CQ_EF_IPV6)) && - (flags & (BNA_CQ_EF_TCP | BNA_CQ_EF_UDP)) && - (flags & BNA_CQ_EF_L4_CKSUM_OK))) + ((masked_flags == flags_tcp4) || + (masked_flags == flags_udp4) || + (masked_flags == flags_tcp6) || + (masked_flags == flags_udp6)))) skb->ip_summed = CHECKSUM_UNNECESSARY; else skb_checksum_none_assert(skb); - rcb->rxq->rx_packets++; - rcb->rxq->rx_bytes += skb->len; - skb->protocol = eth_type_trans(skb, bnad->netdev); + if ((flags & BNA_CQ_EF_VLAN) && + (bnad->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), ntohs(cmpl->vlan_tag)); - if (flags & BNA_CQ_EF_VLAN) - __vlan_hwaccel_put_tag(skb, ntohs(cmpl->vlan_tag)); - - if (skb->ip_summed == CHECKSUM_UNNECESSARY) - napi_gro_receive(&rx_ctrl->napi, skb); - else { + if (BNAD_RXBUF_IS_SK_BUFF(unmap_q->type)) netif_receive_skb(skb); - } + else + napi_gro_frags(&rx_ctrl->napi); next: - cmpl->valid = 0; - cmpl = next_cmpl; + BNA_QE_INDX_ADD(rcb->consumer_index, nvecs, rcb->q_depth); + for (vec = 0; vec < nvecs; vec++) { + cmpl = &cq[ccb->producer_index]; + cmpl->valid = 0; + BNA_QE_INDX_INC(ccb->producer_index, ccb->q_depth); + } + cmpl = &cq[ccb->producer_index]; } - BNA_QE_INDX_ADD(ccb->producer_index, wis, ccb->q_depth); - + napi_gro_flush(&rx_ctrl->napi, false); if (likely(test_bit(BNAD_RXQ_STARTED, &ccb->rcb[0]->flags))) bna_ib_ack_disable_irq(ccb->i_dbell, packets); - bnad_refill_rxq(bnad, ccb->rcb[0]); + bnad_rxq_post(bnad, ccb->rcb[0]); if (ccb->rcb[1]) - bnad_refill_rxq(bnad, ccb->rcb[1]); - - clear_bit(BNAD_FP_IN_RX_PATH, &rx_ctrl->flags); + bnad_rxq_post(bnad, ccb->rcb[1]); return packets; } @@ -646,7 +827,7 @@ bnad_isr(int irq, void *data) for (j = 0; j < bnad->num_txq_per_tx; j++) { tcb = bnad->tx_info[i].tcb[j]; if (tcb && test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) - bnad_tx(bnad, bnad->tx_info[i].tcb[j]); + bnad_tx_complete(bnad, bnad->tx_info[i].tcb[j]); } } /* Rx processing */ @@ -826,12 +1007,9 @@ bnad_cb_tcb_setup(struct bnad *bnad, struct bna_tcb *tcb) { struct bnad_tx_info *tx_info = (struct bnad_tx_info *)tcb->txq->tx->priv; - struct bnad_unmap_q *unmap_q = tcb->unmap_q; + tcb->priv = tcb; tx_info->tcb[tcb->id] = tcb; - unmap_q->producer_index = 0; - unmap_q->consumer_index = 0; - unmap_q->q_depth = BNAD_TX_UNMAPQ_DEPTH; } static void @@ -839,36 +1017,9 @@ bnad_cb_tcb_destroy(struct bnad *bnad, struct bna_tcb *tcb) { struct bnad_tx_info *tx_info = (struct bnad_tx_info *)tcb->txq->tx->priv; - struct bnad_unmap_q *unmap_q = tcb->unmap_q; - - while (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) - cpu_relax(); - - bnad_free_all_txbufs(bnad, tcb); - - unmap_q->producer_index = 0; - unmap_q->consumer_index = 0; - - smp_mb__before_clear_bit(); - clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); tx_info->tcb[tcb->id] = NULL; -} - -static void -bnad_cb_rcb_setup(struct bnad *bnad, struct bna_rcb *rcb) -{ - struct bnad_unmap_q *unmap_q = rcb->unmap_q; - - unmap_q->producer_index = 0; - unmap_q->consumer_index = 0; - unmap_q->q_depth = BNAD_RX_UNMAPQ_DEPTH; -} - -static void -bnad_cb_rcb_destroy(struct bnad *bnad, struct bna_rcb *rcb) -{ - bnad_free_all_rxbufs(bnad, rcb); + tcb->priv = NULL; } static void @@ -916,7 +1067,6 @@ bnad_cb_tx_resume(struct bnad *bnad, struct bna_tx *tx) { struct bnad_tx_info *tx_info = (struct bnad_tx_info *)tx->priv; struct bna_tcb *tcb; - struct bnad_unmap_q *unmap_q; u32 txq_id; int i; @@ -926,23 +1076,9 @@ bnad_cb_tx_resume(struct bnad *bnad, struct bna_tx *tx) continue; txq_id = tcb->id; - unmap_q = tcb->unmap_q; - - if (test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)) - continue; - - while (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) - cpu_relax(); - - bnad_free_all_txbufs(bnad, tcb); - - unmap_q->producer_index = 0; - unmap_q->consumer_index = 0; - - smp_mb__before_clear_bit(); - clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); - + BUG_ON(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags)); set_bit(BNAD_TXQ_TX_STARTED, &tcb->flags); + BUG_ON(*(tcb->hw_consumer_index) != 0); if (netif_carrier_ok(bnad->netdev)) { printk(KERN_INFO "bna: %s %d TXQ_STARTED\n", @@ -963,6 +1099,48 @@ bnad_cb_tx_resume(struct bnad *bnad, struct bna_tx *tx) } } +/* + * Free all TxQs buffers and then notify TX_E_CLEANUP_DONE to Tx fsm. + */ +static void +bnad_tx_cleanup(struct delayed_work *work) +{ + struct bnad_tx_info *tx_info = + container_of(work, struct bnad_tx_info, tx_cleanup_work); + struct bnad *bnad = NULL; + struct bna_tcb *tcb; + unsigned long flags; + u32 i, pending = 0; + + for (i = 0; i < BNAD_MAX_TXQ_PER_TX; i++) { + tcb = tx_info->tcb[i]; + if (!tcb) + continue; + + bnad = tcb->bnad; + + if (test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) { + pending++; + continue; + } + + bnad_txq_cleanup(bnad, tcb); + + smp_mb__before_atomic(); + clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); + } + + if (pending) { + queue_delayed_work(bnad->work_q, &tx_info->tx_cleanup_work, + msecs_to_jiffies(1)); + return; + } + + spin_lock_irqsave(&bnad->bna_lock, flags); + bna_tx_cleanup_complete(tx_info->tx); + spin_unlock_irqrestore(&bnad->bna_lock, flags); +} + static void bnad_cb_tx_cleanup(struct bnad *bnad, struct bna_tx *tx) { @@ -976,8 +1154,7 @@ bnad_cb_tx_cleanup(struct bnad *bnad, struct bna_tx *tx) continue; } - mdelay(BNAD_TXRX_SYNC_MDELAY); - bna_tx_cleanup_complete(tx); + queue_delayed_work(bnad->work_q, &tx_info->tx_cleanup_work, 0); } static void @@ -1001,6 +1178,44 @@ bnad_cb_rx_stall(struct bnad *bnad, struct bna_rx *rx) } } +/* + * Free all RxQs buffers and then notify RX_E_CLEANUP_DONE to Rx fsm. + */ +static void +bnad_rx_cleanup(void *work) +{ + struct bnad_rx_info *rx_info = + container_of(work, struct bnad_rx_info, rx_cleanup_work); + struct bnad_rx_ctrl *rx_ctrl; + struct bnad *bnad = NULL; + unsigned long flags; + u32 i; + + for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { + rx_ctrl = &rx_info->rx_ctrl[i]; + + if (!rx_ctrl->ccb) + continue; + + bnad = rx_ctrl->ccb->bnad; + + /* + * Wait till the poll handler has exited + * and nothing can be scheduled anymore + */ + napi_disable(&rx_ctrl->napi); + + bnad_cq_cleanup(bnad, rx_ctrl->ccb); + bnad_rxq_cleanup(bnad, rx_ctrl->ccb->rcb[0]); + if (rx_ctrl->ccb->rcb[1]) + bnad_rxq_cleanup(bnad, rx_ctrl->ccb->rcb[1]); + } + + spin_lock_irqsave(&bnad->bna_lock, flags); + bna_rx_cleanup_complete(rx_info->rx); + spin_unlock_irqrestore(&bnad->bna_lock, flags); +} + static void bnad_cb_rx_cleanup(struct bnad *bnad, struct bna_rx *rx) { @@ -1009,8 +1224,6 @@ bnad_cb_rx_cleanup(struct bnad *bnad, struct bna_rx *rx) struct bnad_rx_ctrl *rx_ctrl; int i; - mdelay(BNAD_TXRX_SYNC_MDELAY); - for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { rx_ctrl = &rx_info->rx_ctrl[i]; ccb = rx_ctrl->ccb; @@ -1021,12 +1234,9 @@ bnad_cb_rx_cleanup(struct bnad *bnad, struct bna_rx *rx) if (ccb->rcb[1]) clear_bit(BNAD_RXQ_STARTED, &ccb->rcb[1]->flags); - - while (test_bit(BNAD_FP_IN_RX_PATH, &rx_ctrl->flags)) - cpu_relax(); } - bna_rx_cleanup_complete(rx); + queue_work(bnad->work_q, &rx_info->rx_cleanup_work); } static void @@ -1036,9 +1246,7 @@ bnad_cb_rx_post(struct bnad *bnad, struct bna_rx *rx) struct bna_ccb *ccb; struct bna_rcb *rcb; struct bnad_rx_ctrl *rx_ctrl; - struct bnad_unmap_q *unmap_q; - int i; - int j; + int i, j; for (i = 0; i < BNAD_MAX_RXP_PER_RX; i++) { rx_ctrl = &rx_info->rx_ctrl[i]; @@ -1046,27 +1254,17 @@ bnad_cb_rx_post(struct bnad *bnad, struct bna_rx *rx) if (!ccb) continue; - bnad_cq_cmpl_init(bnad, ccb); + napi_enable(&rx_ctrl->napi); for (j = 0; j < BNAD_MAX_RXQ_PER_RXP; j++) { rcb = ccb->rcb[j]; if (!rcb) continue; - bnad_free_all_rxbufs(bnad, rcb); + bnad_rxq_alloc_init(bnad, rcb); set_bit(BNAD_RXQ_STARTED, &rcb->flags); set_bit(BNAD_RXQ_POST_OK, &rcb->flags); - unmap_q = rcb->unmap_q; - - /* Now allocate & post buffers for this RCB */ - /* !!Allocation in callback context */ - if (!test_and_set_bit(BNAD_RXQ_REFILL, &rcb->flags)) { - if (BNA_QE_FREE_CNT(unmap_q, unmap_q->q_depth) - >> BNAD_RXQ_REFILL_THRESHOLD_SHIFT) - bnad_alloc_n_post_rxbufs(bnad, rcb); - smp_mb__before_clear_bit(); - clear_bit(BNAD_RXQ_REFILL, &rcb->flags); - } + bnad_rxq_post(bnad, rcb); } } } @@ -1168,9 +1366,8 @@ bnad_mem_alloc(struct bnad *bnad, mem_info->mdl[i].len = mem_info->len; mem_info->mdl[i].kva = dma_alloc_coherent(&bnad->pcidev->dev, - mem_info->len, &dma_pa, - GFP_KERNEL); - + mem_info->len, &dma_pa, + GFP_KERNEL); if (mem_info->mdl[i].kva == NULL) goto err_return; @@ -1317,8 +1514,7 @@ bnad_txrx_irq_alloc(struct bnad *bnad, enum bnad_intr_source src, return 0; } -/** - * NOTE: Should be called for MSIX only +/* NOTE: Should be called for MSIX only * Unregisters Tx MSIX vector(s) from the kernel */ static void @@ -1337,8 +1533,7 @@ bnad_tx_msix_unregister(struct bnad *bnad, struct bnad_tx_info *tx_info, } } -/** - * NOTE: Should be called for MSIX only +/* NOTE: Should be called for MSIX only * Registers Tx MSIX vector(s) and ISR(s), cookie with the kernel */ static int @@ -1369,8 +1564,7 @@ err_return: return -1; } -/** - * NOTE: Should be called for MSIX only +/* NOTE: Should be called for MSIX only * Unregisters Rx MSIX vector(s) from the kernel */ static void @@ -1390,8 +1584,7 @@ bnad_rx_msix_unregister(struct bnad *bnad, struct bnad_rx_info *rx_info, } } -/** - * NOTE: Should be called for MSIX only +/* NOTE: Should be called for MSIX only * Registers Tx MSIX vector(s) and ISR(s), cookie with the kernel */ static int @@ -1687,7 +1880,7 @@ bnad_napi_poll_rx(struct napi_struct *napi, int budget) if (!netif_carrier_ok(bnad->netdev)) goto poll_exit; - rcvd = bnad_poll_cq(bnad, rx_ctrl->ccb, budget); + rcvd = bnad_cq_process(bnad, rx_ctrl->ccb, budget); if (rcvd >= budget) return rcvd; @@ -1704,7 +1897,7 @@ poll_exit: #define BNAD_NAPI_POLL_QUOTA 64 static void -bnad_napi_init(struct bnad *bnad, u32 rx_id) +bnad_napi_add(struct bnad *bnad, u32 rx_id) { struct bnad_rx_ctrl *rx_ctrl; int i; @@ -1718,34 +1911,18 @@ bnad_napi_init(struct bnad *bnad, u32 rx_id) } static void -bnad_napi_enable(struct bnad *bnad, u32 rx_id) -{ - struct bnad_rx_ctrl *rx_ctrl; - int i; - - /* Initialize & enable NAPI */ - for (i = 0; i < bnad->num_rxp_per_rx; i++) { - rx_ctrl = &bnad->rx_info[rx_id].rx_ctrl[i]; - - napi_enable(&rx_ctrl->napi); - } -} - -static void -bnad_napi_disable(struct bnad *bnad, u32 rx_id) +bnad_napi_delete(struct bnad *bnad, u32 rx_id) { int i; /* First disable and then clean up */ - for (i = 0; i < bnad->num_rxp_per_rx; i++) { - napi_disable(&bnad->rx_info[rx_id].rx_ctrl[i].napi); + for (i = 0; i < bnad->num_rxp_per_rx; i++) netif_napi_del(&bnad->rx_info[rx_id].rx_ctrl[i].napi); - } } /* Should be held with conf_lock held */ void -bnad_cleanup_tx(struct bnad *bnad, u32 tx_id) +bnad_destroy_tx(struct bnad *bnad, u32 tx_id) { struct bnad_tx_info *tx_info = &bnad->tx_info[tx_id]; struct bna_res_info *res_info = &bnad->tx_res_info[tx_id].res_info[0]; @@ -1764,9 +1941,6 @@ bnad_cleanup_tx(struct bnad *bnad, u32 tx_id) bnad_tx_msix_unregister(bnad, tx_info, bnad->num_txq_per_tx); - if (0 == tx_id) - tasklet_kill(&bnad->tx_free_tasklet); - spin_lock_irqsave(&bnad->bna_lock, flags); bna_tx_destroy(tx_info->tx); spin_unlock_irqrestore(&bnad->bna_lock, flags); @@ -1813,10 +1987,9 @@ bnad_setup_tx(struct bnad *bnad, u32 tx_id) spin_unlock_irqrestore(&bnad->bna_lock, flags); /* Fill Unmap Q memory requirements */ - BNAD_FILL_UNMAPQ_MEM_REQ( - &res_info[BNA_TX_RES_MEM_T_UNMAPQ], - bnad->num_txq_per_tx, - BNAD_TX_UNMAPQ_DEPTH); + BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_TX_RES_MEM_T_UNMAPQ], + bnad->num_txq_per_tx, (sizeof(struct bnad_tx_unmap) * + bnad->txq_depth)); /* Allocate resources */ err = bnad_tx_res_alloc(bnad, res_info, tx_id); @@ -1828,16 +2001,21 @@ bnad_setup_tx(struct bnad *bnad, u32 tx_id) tx = bna_tx_create(&bnad->bna, bnad, tx_config, &tx_cbfn, res_info, tx_info); spin_unlock_irqrestore(&bnad->bna_lock, flags); - if (!tx) + if (!tx) { + err = -ENOMEM; goto err_return; + } tx_info->tx = tx; + INIT_DELAYED_WORK(&tx_info->tx_cleanup_work, + (work_func_t)bnad_tx_cleanup); + /* Register ISR for the Tx object */ if (intr_info->intr_type == BNA_INTR_T_MSIX) { err = bnad_tx_msix_register(bnad, tx_info, tx_id, bnad->num_txq_per_tx); if (err) - goto err_return; + goto cleanup_tx; } spin_lock_irqsave(&bnad->bna_lock, flags); @@ -1846,6 +2024,12 @@ bnad_setup_tx(struct bnad *bnad, u32 tx_id) return 0; +cleanup_tx: + spin_lock_irqsave(&bnad->bna_lock, flags); + bna_tx_destroy(tx_info->tx); + spin_unlock_irqrestore(&bnad->bna_lock, flags); + tx_info->tx = NULL; + tx_info->tx_id = 0; err_return: bnad_tx_res_free(bnad, res_info); return err; @@ -1856,6 +2040,7 @@ err_return: static void bnad_init_rx_config(struct bnad *bnad, struct bna_rx_config *rx_config) { + memset(rx_config, 0, sizeof(*rx_config)); rx_config->rx_type = BNA_RX_T_REGULAR; rx_config->num_paths = bnad->num_rxp_per_rx; rx_config->coalescing_timeo = bnad->rx_coalescing_timeo; @@ -1876,12 +2061,43 @@ bnad_init_rx_config(struct bnad *bnad, struct bna_rx_config *rx_config) memset(&rx_config->rss_config, 0, sizeof(rx_config->rss_config)); } + + rx_config->frame_size = BNAD_FRAME_SIZE(bnad->netdev->mtu); + rx_config->q0_multi_buf = BNA_STATUS_T_DISABLED; + + /* BNA_RXP_SINGLE - one data-buffer queue + * BNA_RXP_SLR - one small-buffer and one large-buffer queues + * BNA_RXP_HDS - one header-buffer and one data-buffer queues + */ + /* TODO: configurable param for queue type */ rx_config->rxp_type = BNA_RXP_SLR; - rx_config->q_depth = bnad->rxq_depth; - rx_config->small_buff_size = BFI_SMALL_RXBUF_SIZE; + if (BNAD_PCI_DEV_IS_CAT2(bnad) && + rx_config->frame_size > 4096) { + /* though size_routing_enable is set in SLR, + * small packets may get routed to same rxq. + * set buf_size to 2048 instead of PAGE_SIZE. + */ + rx_config->q0_buf_size = 2048; + /* this should be in multiples of 2 */ + rx_config->q0_num_vecs = 4; + rx_config->q0_depth = bnad->rxq_depth * rx_config->q0_num_vecs; + rx_config->q0_multi_buf = BNA_STATUS_T_ENABLED; + } else { + rx_config->q0_buf_size = rx_config->frame_size; + rx_config->q0_num_vecs = 1; + rx_config->q0_depth = bnad->rxq_depth; + } + + /* initialize for q1 for BNA_RXP_SLR/BNA_RXP_HDS */ + if (rx_config->rxp_type == BNA_RXP_SLR) { + rx_config->q1_depth = bnad->rxq_depth; + rx_config->q1_buf_size = BFI_SMALL_RXBUF_SIZE; + } - rx_config->vlan_strip_status = BNA_STATUS_T_ENABLED; + rx_config->vlan_strip_status = + (bnad->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) ? + BNA_STATUS_T_ENABLED : BNA_STATUS_T_DISABLED; } static void @@ -1895,8 +2111,51 @@ bnad_rx_ctrl_init(struct bnad *bnad, u32 rx_id) } /* Called with mutex_lock(&bnad->conf_mutex) held */ +static u32 +bnad_reinit_rx(struct bnad *bnad) +{ + struct net_device *netdev = bnad->netdev; + u32 err = 0, current_err = 0; + u32 rx_id = 0, count = 0; + unsigned long flags; + + /* destroy and create new rx objects */ + for (rx_id = 0; rx_id < bnad->num_rx; rx_id++) { + if (!bnad->rx_info[rx_id].rx) + continue; + bnad_destroy_rx(bnad, rx_id); + } + + spin_lock_irqsave(&bnad->bna_lock, flags); + bna_enet_mtu_set(&bnad->bna.enet, + BNAD_FRAME_SIZE(bnad->netdev->mtu), NULL); + spin_unlock_irqrestore(&bnad->bna_lock, flags); + + for (rx_id = 0; rx_id < bnad->num_rx; rx_id++) { + count++; + current_err = bnad_setup_rx(bnad, rx_id); + if (current_err && !err) { + err = current_err; + pr_err("RXQ:%u setup failed\n", rx_id); + } + } + + /* restore rx configuration */ + if (bnad->rx_info[0].rx && !err) { + bnad_restore_vlans(bnad, 0); + bnad_enable_default_bcast(bnad); + spin_lock_irqsave(&bnad->bna_lock, flags); + bnad_mac_addr_set_locked(bnad, netdev->dev_addr); + spin_unlock_irqrestore(&bnad->bna_lock, flags); + bnad_set_rx_mode(netdev); + } + + return count; +} + +/* Called with bnad_conf_lock() held */ void -bnad_cleanup_rx(struct bnad *bnad, u32 rx_id) +bnad_destroy_rx(struct bnad *bnad, u32 rx_id) { struct bnad_rx_info *rx_info = &bnad->rx_info[rx_id]; struct bna_rx_config *rx_config = &bnad->rx_config[rx_id]; @@ -1928,7 +2187,7 @@ bnad_cleanup_rx(struct bnad *bnad, u32 rx_id) if (rx_info->rx_ctrl[0].ccb->intr_type == BNA_INTR_T_MSIX) bnad_rx_msix_unregister(bnad, rx_info, rx_config->num_paths); - bnad_napi_disable(bnad, rx_id); + bnad_napi_delete(bnad, rx_id); spin_lock_irqsave(&bnad->bna_lock, flags); bna_rx_destroy(rx_info->rx); @@ -1951,8 +2210,8 @@ bnad_setup_rx(struct bnad *bnad, u32 rx_id) &res_info[BNA_RX_RES_T_INTR].res_u.intr_info; struct bna_rx_config *rx_config = &bnad->rx_config[rx_id]; static const struct bna_rx_event_cbfn rx_cbfn = { - .rcb_setup_cbfn = bnad_cb_rcb_setup, - .rcb_destroy_cbfn = bnad_cb_rcb_destroy, + .rcb_setup_cbfn = NULL, + .rcb_destroy_cbfn = NULL, .ccb_setup_cbfn = bnad_cb_ccb_setup, .ccb_destroy_cbfn = bnad_cb_ccb_destroy, .rx_stall_cbfn = bnad_cb_rx_stall, @@ -1973,12 +2232,19 @@ bnad_setup_rx(struct bnad *bnad, u32 rx_id) spin_unlock_irqrestore(&bnad->bna_lock, flags); /* Fill Unmap Q memory requirements */ - BNAD_FILL_UNMAPQ_MEM_REQ( - &res_info[BNA_RX_RES_MEM_T_UNMAPQ], - rx_config->num_paths + - ((rx_config->rxp_type == BNA_RXP_SINGLE) ? 0 : - rx_config->num_paths), BNAD_RX_UNMAPQ_DEPTH); - + BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_RX_RES_MEM_T_UNMAPDQ], + rx_config->num_paths, + (rx_config->q0_depth * + sizeof(struct bnad_rx_unmap)) + + sizeof(struct bnad_rx_unmap_q)); + + if (rx_config->rxp_type != BNA_RXP_SINGLE) { + BNAD_FILL_UNMAPQ_MEM_REQ(&res_info[BNA_RX_RES_MEM_T_UNMAPHQ], + rx_config->num_paths, + (rx_config->q1_depth * + sizeof(struct bnad_rx_unmap) + + sizeof(struct bnad_rx_unmap_q))); + } /* Allocate resource */ err = bnad_rx_res_alloc(bnad, res_info, rx_id); if (err) @@ -1998,11 +2264,14 @@ bnad_setup_rx(struct bnad *bnad, u32 rx_id) rx_info->rx = rx; spin_unlock_irqrestore(&bnad->bna_lock, flags); + INIT_WORK(&rx_info->rx_cleanup_work, + (work_func_t)(bnad_rx_cleanup)); + /* * Init NAPI, so that state is set to NAPI_STATE_SCHED, * so that IRQ handler cannot schedule NAPI at this point. */ - bnad_napi_init(bnad, rx_id); + bnad_napi_add(bnad, rx_id); /* Register ISR for the Rx object */ if (intr_info->intr_type == BNA_INTR_T_MSIX) { @@ -2028,13 +2297,10 @@ bnad_setup_rx(struct bnad *bnad, u32 rx_id) bna_rx_enable(rx); spin_unlock_irqrestore(&bnad->bna_lock, flags); - /* Enable scheduling of NAPI */ - bnad_napi_enable(bnad, rx_id); - return 0; err_return: - bnad_cleanup_rx(bnad, rx_id); + bnad_destroy_rx(bnad, rx_id); return err; } @@ -2230,12 +2496,10 @@ bnad_tso_prepare(struct bnad *bnad, struct sk_buff *skb) { int err; - if (skb_header_cloned(skb)) { - err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); - if (err) { - BNAD_UPDATE_CTR(bnad, tso_err); - return err; - } + err = skb_cow_head(skb, 0); + if (err < 0) { + BNAD_UPDATE_CTR(bnad, tso_err); + return err; } /* @@ -2403,9 +2667,11 @@ bnad_enable_msix(struct bnad *bnad) for (i = 0; i < bnad->msix_num; i++) bnad->msix_table[i].entry = i; - ret = pci_enable_msix(bnad->pcidev, bnad->msix_table, bnad->msix_num); - if (ret > 0) { - /* Not enough MSI-X vectors. */ + ret = pci_enable_msix_range(bnad->pcidev, bnad->msix_table, + 1, bnad->msix_num); + if (ret < 0) { + goto intx_mode; + } else if (ret < bnad->msix_num) { pr_warn("BNA: %d MSI-X vectors allocated < %d requested\n", ret, bnad->msix_num); @@ -2418,18 +2684,11 @@ bnad_enable_msix(struct bnad *bnad) bnad->msix_num = BNAD_NUM_TXQ + BNAD_NUM_RXP + BNAD_MAILBOX_MSIX_VECTORS; - if (bnad->msix_num > ret) + if (bnad->msix_num > ret) { + pci_disable_msix(bnad->pcidev); goto intx_mode; - - /* Try once more with adjusted numbers */ - /* If this fails, fall back to INTx */ - ret = pci_enable_msix(bnad->pcidev, bnad->msix_table, - bnad->msix_num); - if (ret) - goto intx_mode; - - } else if (ret < 0) - goto intx_mode; + } + } pci_intx(bnad->pcidev, 0); @@ -2473,7 +2732,6 @@ bnad_open(struct net_device *netdev) int err; struct bnad *bnad = netdev_priv(netdev); struct bna_pause_config pause_config; - int mtu; unsigned long flags; mutex_lock(&bnad->conf_mutex); @@ -2492,10 +2750,9 @@ bnad_open(struct net_device *netdev) pause_config.tx_pause = 0; pause_config.rx_pause = 0; - mtu = ETH_HLEN + VLAN_HLEN + bnad->netdev->mtu + ETH_FCS_LEN; - spin_lock_irqsave(&bnad->bna_lock, flags); - bna_enet_mtu_set(&bnad->bna.enet, mtu, NULL); + bna_enet_mtu_set(&bnad->bna.enet, + BNAD_FRAME_SIZE(bnad->netdev->mtu), NULL); bna_enet_pause_config(&bnad->bna.enet, &pause_config, NULL); bna_enet_enable(&bnad->bna.enet); spin_unlock_irqrestore(&bnad->bna_lock, flags); @@ -2519,7 +2776,7 @@ bnad_open(struct net_device *netdev) return 0; cleanup_tx: - bnad_cleanup_tx(bnad, 0); + bnad_destroy_tx(bnad, 0); err_return: mutex_unlock(&bnad->conf_mutex); @@ -2546,8 +2803,8 @@ bnad_stop(struct net_device *netdev) wait_for_completion(&bnad->bnad_completions.enet_comp); - bnad_cleanup_tx(bnad, 0); - bnad_cleanup_rx(bnad, 0); + bnad_destroy_tx(bnad, 0); + bnad_destroy_rx(bnad, 0); /* Synchronize mailbox IRQ */ bnad_mbox_irq_sync(bnad); @@ -2558,166 +2815,71 @@ bnad_stop(struct net_device *netdev) } /* TX */ -/* - * bnad_start_xmit : Netdev entry point for Transmit - * Called under lock held by net_device - */ -static netdev_tx_t -bnad_start_xmit(struct sk_buff *skb, struct net_device *netdev) +/* Returns 0 for success */ +static int +bnad_txq_wi_prepare(struct bnad *bnad, struct bna_tcb *tcb, + struct sk_buff *skb, struct bna_txq_entry *txqent) { - struct bnad *bnad = netdev_priv(netdev); - u32 txq_id = 0; - struct bna_tcb *tcb = bnad->tx_info[0].tcb[txq_id]; - - u16 txq_prod, vlan_tag = 0; - u32 unmap_prod, wis, wis_used, wi_range; - u32 vectors, vect_id, i, acked; - int err; - unsigned int len; - u32 gso_size; - - struct bnad_unmap_q *unmap_q = tcb->unmap_q; - dma_addr_t dma_addr; - struct bna_txq_entry *txqent; - u16 flags; - - if (unlikely(skb->len <= ETH_HLEN)) { - dev_kfree_skb(skb); - BNAD_UPDATE_CTR(bnad, tx_skb_too_short); - return NETDEV_TX_OK; - } - if (unlikely(skb_headlen(skb) > BFI_TX_MAX_DATA_PER_VECTOR)) { - dev_kfree_skb(skb); - BNAD_UPDATE_CTR(bnad, tx_skb_headlen_too_long); - return NETDEV_TX_OK; - } - if (unlikely(skb_headlen(skb) == 0)) { - dev_kfree_skb(skb); - BNAD_UPDATE_CTR(bnad, tx_skb_headlen_zero); - return NETDEV_TX_OK; - } - - /* - * Takes care of the Tx that is scheduled between clearing the flag - * and the netif_tx_stop_all_queues() call. - */ - if (unlikely(!test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) { - dev_kfree_skb(skb); - BNAD_UPDATE_CTR(bnad, tx_skb_stopping); - return NETDEV_TX_OK; - } - - vectors = 1 + skb_shinfo(skb)->nr_frags; - if (unlikely(vectors > BFI_TX_MAX_VECTORS_PER_PKT)) { - dev_kfree_skb(skb); - BNAD_UPDATE_CTR(bnad, tx_skb_max_vectors); - return NETDEV_TX_OK; - } - wis = BNA_TXQ_WI_NEEDED(vectors); /* 4 vectors per work item */ - acked = 0; - if (unlikely(wis > BNA_QE_FREE_CNT(tcb, tcb->q_depth) || - vectors > BNA_QE_FREE_CNT(unmap_q, unmap_q->q_depth))) { - if ((u16) (*tcb->hw_consumer_index) != - tcb->consumer_index && - !test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) { - acked = bnad_free_txbufs(bnad, tcb); - if (likely(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) - bna_ib_ack(tcb->i_dbell, acked); - smp_mb__before_clear_bit(); - clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); - } else { - netif_stop_queue(netdev); - BNAD_UPDATE_CTR(bnad, netif_queue_stop); - } - - smp_mb(); - /* - * Check again to deal with race condition between - * netif_stop_queue here, and netif_wake_queue in - * interrupt handler which is not inside netif tx lock. - */ - if (likely - (wis > BNA_QE_FREE_CNT(tcb, tcb->q_depth) || - vectors > BNA_QE_FREE_CNT(unmap_q, unmap_q->q_depth))) { - BNAD_UPDATE_CTR(bnad, netif_queue_stop); - return NETDEV_TX_BUSY; - } else { - netif_wake_queue(netdev); - BNAD_UPDATE_CTR(bnad, netif_queue_wakeup); - } - } - - unmap_prod = unmap_q->producer_index; - flags = 0; - - txq_prod = tcb->producer_index; - BNA_TXQ_QPGE_PTR_GET(txq_prod, tcb->sw_qpt, txqent, wi_range); - txqent->hdr.wi.reserved = 0; - txqent->hdr.wi.num_vectors = vectors; + u16 flags = 0; + u32 gso_size; + u16 vlan_tag = 0; if (vlan_tx_tag_present(skb)) { - vlan_tag = (u16) vlan_tx_tag_get(skb); + vlan_tag = (u16)vlan_tx_tag_get(skb); flags |= (BNA_TXQ_WI_CF_INS_PRIO | BNA_TXQ_WI_CF_INS_VLAN); } if (test_bit(BNAD_RF_CEE_RUNNING, &bnad->run_flags)) { - vlan_tag = - (tcb->priority & 0x7) << 13 | (vlan_tag & 0x1fff); + vlan_tag = ((tcb->priority & 0x7) << VLAN_PRIO_SHIFT) + | (vlan_tag & 0x1fff); flags |= (BNA_TXQ_WI_CF_INS_PRIO | BNA_TXQ_WI_CF_INS_VLAN); } - txqent->hdr.wi.vlan_tag = htons(vlan_tag); if (skb_is_gso(skb)) { gso_size = skb_shinfo(skb)->gso_size; - - if (unlikely(gso_size > netdev->mtu)) { - dev_kfree_skb(skb); + if (unlikely(gso_size > bnad->netdev->mtu)) { BNAD_UPDATE_CTR(bnad, tx_skb_mss_too_long); - return NETDEV_TX_OK; + return -EINVAL; } if (unlikely((gso_size + skb_transport_offset(skb) + - tcp_hdrlen(skb)) >= skb->len)) { - txqent->hdr.wi.opcode = - __constant_htons(BNA_TXQ_WI_SEND); + tcp_hdrlen(skb)) >= skb->len)) { + txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND); txqent->hdr.wi.lso_mss = 0; BNAD_UPDATE_CTR(bnad, tx_skb_tso_too_short); } else { - txqent->hdr.wi.opcode = - __constant_htons(BNA_TXQ_WI_SEND_LSO); + txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND_LSO); txqent->hdr.wi.lso_mss = htons(gso_size); } - err = bnad_tso_prepare(bnad, skb); - if (unlikely(err)) { - dev_kfree_skb(skb); + if (bnad_tso_prepare(bnad, skb)) { BNAD_UPDATE_CTR(bnad, tx_skb_tso_prepare); - return NETDEV_TX_OK; + return -EINVAL; } + flags |= (BNA_TXQ_WI_CF_IP_CKSUM | BNA_TXQ_WI_CF_TCP_CKSUM); txqent->hdr.wi.l4_hdr_size_n_offset = - htons(BNA_TXQ_WI_L4_HDR_N_OFFSET - (tcp_hdrlen(skb) >> 2, - skb_transport_offset(skb))); - } else { - txqent->hdr.wi.opcode = __constant_htons(BNA_TXQ_WI_SEND); + htons(BNA_TXQ_WI_L4_HDR_N_OFFSET( + tcp_hdrlen(skb) >> 2, skb_transport_offset(skb))); + } else { + txqent->hdr.wi.opcode = htons(BNA_TXQ_WI_SEND); txqent->hdr.wi.lso_mss = 0; - if (unlikely(skb->len > (netdev->mtu + ETH_HLEN))) { - dev_kfree_skb(skb); + if (unlikely(skb->len > (bnad->netdev->mtu + ETH_HLEN))) { BNAD_UPDATE_CTR(bnad, tx_skb_non_tso_too_long); - return NETDEV_TX_OK; + return -EINVAL; } if (skb->ip_summed == CHECKSUM_PARTIAL) { u8 proto = 0; - if (skb->protocol == __constant_htons(ETH_P_IP)) + if (skb->protocol == htons(ETH_P_IP)) proto = ip_hdr(skb)->protocol; - else if (skb->protocol == - __constant_htons(ETH_P_IPV6)) { +#ifdef NETIF_F_IPV6_CSUM + else if (skb->protocol == htons(ETH_P_IPV6)) { /* nexthdr may not be TCP immediately. */ proto = ipv6_hdr(skb)->nexthdr; } +#endif if (proto == IPPROTO_TCP) { flags |= BNA_TXQ_WI_CF_TCP_CKSUM; txqent->hdr.wi.l4_hdr_size_n_offset = @@ -2727,12 +2889,11 @@ bnad_start_xmit(struct sk_buff *skb, struct net_device *netdev) BNAD_UPDATE_CTR(bnad, tcpcsum_offload); if (unlikely(skb_headlen(skb) < - skb_transport_offset(skb) + tcp_hdrlen(skb))) { - dev_kfree_skb(skb); + skb_transport_offset(skb) + + tcp_hdrlen(skb))) { BNAD_UPDATE_CTR(bnad, tx_skb_tcp_hdr); - return NETDEV_TX_OK; + return -EINVAL; } - } else if (proto == IPPROTO_UDP) { flags |= BNA_TXQ_WI_CF_UDP_CKSUM; txqent->hdr.wi.l4_hdr_size_n_offset = @@ -2741,111 +2902,196 @@ bnad_start_xmit(struct sk_buff *skb, struct net_device *netdev) BNAD_UPDATE_CTR(bnad, udpcsum_offload); if (unlikely(skb_headlen(skb) < - skb_transport_offset(skb) + + skb_transport_offset(skb) + sizeof(struct udphdr))) { - dev_kfree_skb(skb); BNAD_UPDATE_CTR(bnad, tx_skb_udp_hdr); - return NETDEV_TX_OK; + return -EINVAL; } } else { - dev_kfree_skb(skb); + BNAD_UPDATE_CTR(bnad, tx_skb_csum_err); - return NETDEV_TX_OK; + return -EINVAL; } - } else { + } else txqent->hdr.wi.l4_hdr_size_n_offset = 0; - } } txqent->hdr.wi.flags = htons(flags); - txqent->hdr.wi.frame_length = htonl(skb->len); - unmap_q->unmap_array[unmap_prod].skb = skb; + return 0; +} + +/* + * bnad_start_xmit : Netdev entry point for Transmit + * Called under lock held by net_device + */ +static netdev_tx_t +bnad_start_xmit(struct sk_buff *skb, struct net_device *netdev) +{ + struct bnad *bnad = netdev_priv(netdev); + u32 txq_id = 0; + struct bna_tcb *tcb = NULL; + struct bnad_tx_unmap *unmap_q, *unmap, *head_unmap; + u32 prod, q_depth, vect_id; + u32 wis, vectors, len; + int i; + dma_addr_t dma_addr; + struct bna_txq_entry *txqent; + len = skb_headlen(skb); - txqent->vector[0].length = htons(len); - dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data, - skb_headlen(skb), DMA_TO_DEVICE); - dma_unmap_addr_set(&unmap_q->unmap_array[unmap_prod], dma_addr, - dma_addr); - BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[0].host_addr); - BNA_QE_INDX_ADD(unmap_prod, 1, unmap_q->q_depth); + /* Sanity checks for the skb */ + + if (unlikely(skb->len <= ETH_HLEN)) { + dev_kfree_skb_any(skb); + BNAD_UPDATE_CTR(bnad, tx_skb_too_short); + return NETDEV_TX_OK; + } + if (unlikely(len > BFI_TX_MAX_DATA_PER_VECTOR)) { + dev_kfree_skb_any(skb); + BNAD_UPDATE_CTR(bnad, tx_skb_headlen_zero); + return NETDEV_TX_OK; + } + if (unlikely(len == 0)) { + dev_kfree_skb_any(skb); + BNAD_UPDATE_CTR(bnad, tx_skb_headlen_zero); + return NETDEV_TX_OK; + } + + tcb = bnad->tx_info[0].tcb[txq_id]; + + /* + * Takes care of the Tx that is scheduled between clearing the flag + * and the netif_tx_stop_all_queues() call. + */ + if (unlikely(!tcb || !test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) { + dev_kfree_skb_any(skb); + BNAD_UPDATE_CTR(bnad, tx_skb_stopping); + return NETDEV_TX_OK; + } + + q_depth = tcb->q_depth; + prod = tcb->producer_index; + unmap_q = tcb->unmap_q; + + vectors = 1 + skb_shinfo(skb)->nr_frags; + wis = BNA_TXQ_WI_NEEDED(vectors); /* 4 vectors per work item */ + + if (unlikely(vectors > BFI_TX_MAX_VECTORS_PER_PKT)) { + dev_kfree_skb_any(skb); + BNAD_UPDATE_CTR(bnad, tx_skb_max_vectors); + return NETDEV_TX_OK; + } + + /* Check for available TxQ resources */ + if (unlikely(wis > BNA_QE_FREE_CNT(tcb, q_depth))) { + if ((*tcb->hw_consumer_index != tcb->consumer_index) && + !test_and_set_bit(BNAD_TXQ_FREE_SENT, &tcb->flags)) { + u32 sent; + sent = bnad_txcmpl_process(bnad, tcb); + if (likely(test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) + bna_ib_ack(tcb->i_dbell, sent); + smp_mb__before_atomic(); + clear_bit(BNAD_TXQ_FREE_SENT, &tcb->flags); + } else { + netif_stop_queue(netdev); + BNAD_UPDATE_CTR(bnad, netif_queue_stop); + } + + smp_mb(); + /* + * Check again to deal with race condition between + * netif_stop_queue here, and netif_wake_queue in + * interrupt handler which is not inside netif tx lock. + */ + if (likely(wis > BNA_QE_FREE_CNT(tcb, q_depth))) { + BNAD_UPDATE_CTR(bnad, netif_queue_stop); + return NETDEV_TX_BUSY; + } else { + netif_wake_queue(netdev); + BNAD_UPDATE_CTR(bnad, netif_queue_wakeup); + } + } - vect_id = 0; - wis_used = 1; + txqent = &((struct bna_txq_entry *)tcb->sw_q)[prod]; + head_unmap = &unmap_q[prod]; + + /* Program the opcode, flags, frame_len, num_vectors in WI */ + if (bnad_txq_wi_prepare(bnad, tcb, skb, txqent)) { + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; + } + txqent->hdr.wi.reserved = 0; + txqent->hdr.wi.num_vectors = vectors; + + head_unmap->skb = skb; + head_unmap->nvecs = 0; + + /* Program the vectors */ + unmap = head_unmap; + dma_addr = dma_map_single(&bnad->pcidev->dev, skb->data, + len, DMA_TO_DEVICE); + BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[0].host_addr); + txqent->vector[0].length = htons(len); + dma_unmap_addr_set(&unmap->vectors[0], dma_addr, dma_addr); + head_unmap->nvecs++; - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + for (i = 0, vect_id = 0; i < vectors - 1; i++) { const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; - u16 size = skb_frag_size(frag); + u32 size = skb_frag_size(frag); if (unlikely(size == 0)) { - unmap_prod = unmap_q->producer_index; - - unmap_prod = bnad_pci_unmap_skb(&bnad->pcidev->dev, - unmap_q->unmap_array, - unmap_prod, unmap_q->q_depth, skb, - i); - dev_kfree_skb(skb); + /* Undo the changes starting at tcb->producer_index */ + bnad_tx_buff_unmap(bnad, unmap_q, q_depth, + tcb->producer_index); + dev_kfree_skb_any(skb); BNAD_UPDATE_CTR(bnad, tx_skb_frag_zero); return NETDEV_TX_OK; } len += size; - if (++vect_id == BFI_TX_MAX_VECTORS_PER_WI) { + vect_id++; + if (vect_id == BFI_TX_MAX_VECTORS_PER_WI) { vect_id = 0; - if (--wi_range) - txqent++; - else { - BNA_QE_INDX_ADD(txq_prod, wis_used, - tcb->q_depth); - wis_used = 0; - BNA_TXQ_QPGE_PTR_GET(txq_prod, tcb->sw_qpt, - txqent, wi_range); - } - wis_used++; - txqent->hdr.wi_ext.opcode = - __constant_htons(BNA_TXQ_WI_EXTENSION); + BNA_QE_INDX_INC(prod, q_depth); + txqent = &((struct bna_txq_entry *)tcb->sw_q)[prod]; + txqent->hdr.wi_ext.opcode = htons(BNA_TXQ_WI_EXTENSION); + unmap = &unmap_q[prod]; } - BUG_ON(!(size <= BFI_TX_MAX_DATA_PER_VECTOR)); - txqent->vector[vect_id].length = htons(size); dma_addr = skb_frag_dma_map(&bnad->pcidev->dev, frag, 0, size, DMA_TO_DEVICE); - dma_unmap_addr_set(&unmap_q->unmap_array[unmap_prod], dma_addr, - dma_addr); + dma_unmap_len_set(&unmap->vectors[vect_id], dma_len, size); BNA_SET_DMA_ADDR(dma_addr, &txqent->vector[vect_id].host_addr); - BNA_QE_INDX_ADD(unmap_prod, 1, unmap_q->q_depth); + txqent->vector[vect_id].length = htons(size); + dma_unmap_addr_set(&unmap->vectors[vect_id], dma_addr, + dma_addr); + head_unmap->nvecs++; } if (unlikely(len != skb->len)) { - unmap_prod = unmap_q->producer_index; - - unmap_prod = bnad_pci_unmap_skb(&bnad->pcidev->dev, - unmap_q->unmap_array, unmap_prod, - unmap_q->q_depth, skb, - skb_shinfo(skb)->nr_frags); - dev_kfree_skb(skb); + /* Undo the changes starting at tcb->producer_index */ + bnad_tx_buff_unmap(bnad, unmap_q, q_depth, tcb->producer_index); + dev_kfree_skb_any(skb); BNAD_UPDATE_CTR(bnad, tx_skb_len_mismatch); return NETDEV_TX_OK; } - unmap_q->producer_index = unmap_prod; - BNA_QE_INDX_ADD(txq_prod, wis_used, tcb->q_depth); - tcb->producer_index = txq_prod; + BNA_QE_INDX_INC(prod, q_depth); + tcb->producer_index = prod; smp_mb(); if (unlikely(!test_bit(BNAD_TXQ_TX_STARTED, &tcb->flags))) return NETDEV_TX_OK; + skb_tx_timestamp(skb); + bna_txq_prod_indx_doorbell(tcb); smp_mb(); - if ((u16) (*tcb->hw_consumer_index) != tcb->consumer_index) - tasklet_schedule(&bnad->tx_free_tasklet); - return NETDEV_TX_OK; } @@ -2869,73 +3115,128 @@ bnad_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) return stats; } +static void +bnad_set_rx_ucast_fltr(struct bnad *bnad) +{ + struct net_device *netdev = bnad->netdev; + int uc_count = netdev_uc_count(netdev); + enum bna_cb_status ret; + u8 *mac_list; + struct netdev_hw_addr *ha; + int entry; + + if (netdev_uc_empty(bnad->netdev)) { + bna_rx_ucast_listset(bnad->rx_info[0].rx, 0, NULL, NULL); + return; + } + + if (uc_count > bna_attr(&bnad->bna)->num_ucmac) + goto mode_default; + + mac_list = kzalloc(uc_count * ETH_ALEN, GFP_ATOMIC); + if (mac_list == NULL) + goto mode_default; + + entry = 0; + netdev_for_each_uc_addr(ha, netdev) { + memcpy(&mac_list[entry * ETH_ALEN], + &ha->addr[0], ETH_ALEN); + entry++; + } + + ret = bna_rx_ucast_listset(bnad->rx_info[0].rx, entry, + mac_list, NULL); + kfree(mac_list); + + if (ret != BNA_CB_SUCCESS) + goto mode_default; + + return; + + /* ucast packets not in UCAM are routed to default function */ +mode_default: + bnad->cfg_flags |= BNAD_CF_DEFAULT; + bna_rx_ucast_listset(bnad->rx_info[0].rx, 0, NULL, NULL); +} + +static void +bnad_set_rx_mcast_fltr(struct bnad *bnad) +{ + struct net_device *netdev = bnad->netdev; + int mc_count = netdev_mc_count(netdev); + enum bna_cb_status ret; + u8 *mac_list; + + if (netdev->flags & IFF_ALLMULTI) + goto mode_allmulti; + + if (netdev_mc_empty(netdev)) + return; + + if (mc_count > bna_attr(&bnad->bna)->num_mcmac) + goto mode_allmulti; + + mac_list = kzalloc((mc_count + 1) * ETH_ALEN, GFP_ATOMIC); + + if (mac_list == NULL) + goto mode_allmulti; + + memcpy(&mac_list[0], &bnad_bcast_addr[0], ETH_ALEN); + + /* copy rest of the MCAST addresses */ + bnad_netdev_mc_list_get(netdev, mac_list); + ret = bna_rx_mcast_listset(bnad->rx_info[0].rx, mc_count + 1, + mac_list, NULL); + kfree(mac_list); + + if (ret != BNA_CB_SUCCESS) + goto mode_allmulti; + + return; + +mode_allmulti: + bnad->cfg_flags |= BNAD_CF_ALLMULTI; + bna_rx_mcast_delall(bnad->rx_info[0].rx, NULL); +} + void bnad_set_rx_mode(struct net_device *netdev) { struct bnad *bnad = netdev_priv(netdev); - u32 new_mask, valid_mask; + enum bna_rxmode new_mode, mode_mask; unsigned long flags; spin_lock_irqsave(&bnad->bna_lock, flags); - new_mask = valid_mask = 0; - - if (netdev->flags & IFF_PROMISC) { - if (!(bnad->cfg_flags & BNAD_CF_PROMISC)) { - new_mask = BNAD_RXMODE_PROMISC_DEFAULT; - valid_mask = BNAD_RXMODE_PROMISC_DEFAULT; - bnad->cfg_flags |= BNAD_CF_PROMISC; - } - } else { - if (bnad->cfg_flags & BNAD_CF_PROMISC) { - new_mask = ~BNAD_RXMODE_PROMISC_DEFAULT; - valid_mask = BNAD_RXMODE_PROMISC_DEFAULT; - bnad->cfg_flags &= ~BNAD_CF_PROMISC; - } - } - - if (netdev->flags & IFF_ALLMULTI) { - if (!(bnad->cfg_flags & BNAD_CF_ALLMULTI)) { - new_mask |= BNA_RXMODE_ALLMULTI; - valid_mask |= BNA_RXMODE_ALLMULTI; - bnad->cfg_flags |= BNAD_CF_ALLMULTI; - } - } else { - if (bnad->cfg_flags & BNAD_CF_ALLMULTI) { - new_mask &= ~BNA_RXMODE_ALLMULTI; - valid_mask |= BNA_RXMODE_ALLMULTI; - bnad->cfg_flags &= ~BNAD_CF_ALLMULTI; - } + if (bnad->rx_info[0].rx == NULL) { + spin_unlock_irqrestore(&bnad->bna_lock, flags); + return; } - if (bnad->rx_info[0].rx == NULL) - goto unlock; + /* clear bnad flags to update it with new settings */ + bnad->cfg_flags &= ~(BNAD_CF_PROMISC | BNAD_CF_DEFAULT | + BNAD_CF_ALLMULTI); - bna_rx_mode_set(bnad->rx_info[0].rx, new_mask, valid_mask, NULL); - - if (!netdev_mc_empty(netdev)) { - u8 *mcaddr_list; - int mc_count = netdev_mc_count(netdev); + new_mode = 0; + if (netdev->flags & IFF_PROMISC) { + new_mode |= BNAD_RXMODE_PROMISC_DEFAULT; + bnad->cfg_flags |= BNAD_CF_PROMISC; + } else { + bnad_set_rx_mcast_fltr(bnad); - /* Index 0 holds the broadcast address */ - mcaddr_list = - kzalloc((mc_count + 1) * ETH_ALEN, - GFP_ATOMIC); - if (!mcaddr_list) - goto unlock; + if (bnad->cfg_flags & BNAD_CF_ALLMULTI) + new_mode |= BNA_RXMODE_ALLMULTI; - memcpy(&mcaddr_list[0], &bnad_bcast_addr[0], ETH_ALEN); + bnad_set_rx_ucast_fltr(bnad); - /* Copy rest of the MC addresses */ - bnad_netdev_mc_list_get(netdev, mcaddr_list); + if (bnad->cfg_flags & BNAD_CF_DEFAULT) + new_mode |= BNA_RXMODE_DEFAULT; + } - bna_rx_mcast_listset(bnad->rx_info[0].rx, mc_count + 1, - mcaddr_list, NULL); + mode_mask = BNA_RXMODE_PROMISC | BNA_RXMODE_DEFAULT | + BNA_RXMODE_ALLMULTI; + bna_rx_mode_set(bnad->rx_info[0].rx, new_mode, mode_mask, NULL); - /* Should we enable BNAD_CF_ALLMULTI for err != 0 ? */ - kfree(mcaddr_list); - } -unlock: spin_unlock_irqrestore(&bnad->bna_lock, flags); } @@ -2965,14 +3266,14 @@ bnad_set_mac_address(struct net_device *netdev, void *mac_addr) } static int -bnad_mtu_set(struct bnad *bnad, int mtu) +bnad_mtu_set(struct bnad *bnad, int frame_size) { unsigned long flags; init_completion(&bnad->bnad_completions.mtu_comp); spin_lock_irqsave(&bnad->bna_lock, flags); - bna_enet_mtu_set(&bnad->bna.enet, mtu, bnad_cb_enet_mtu_set); + bna_enet_mtu_set(&bnad->bna.enet, frame_size, bnad_cb_enet_mtu_set); spin_unlock_irqrestore(&bnad->bna_lock, flags); wait_for_completion(&bnad->bnad_completions.mtu_comp); @@ -2983,18 +3284,34 @@ bnad_mtu_set(struct bnad *bnad, int mtu) static int bnad_change_mtu(struct net_device *netdev, int new_mtu) { - int err, mtu = netdev->mtu; + int err, mtu; struct bnad *bnad = netdev_priv(netdev); + u32 rx_count = 0, frame, new_frame; if (new_mtu + ETH_HLEN < ETH_ZLEN || new_mtu > BNAD_JUMBO_MTU) return -EINVAL; mutex_lock(&bnad->conf_mutex); + mtu = netdev->mtu; netdev->mtu = new_mtu; - mtu = ETH_HLEN + VLAN_HLEN + new_mtu + ETH_FCS_LEN; - err = bnad_mtu_set(bnad, mtu); + frame = BNAD_FRAME_SIZE(mtu); + new_frame = BNAD_FRAME_SIZE(new_mtu); + + /* check if multi-buffer needs to be enabled */ + if (BNAD_PCI_DEV_IS_CAT2(bnad) && + netif_running(bnad->netdev)) { + /* only when transition is over 4K */ + if ((frame <= 4096 && new_frame > 4096) || + (frame > 4096 && new_frame <= 4096)) + rx_count = bnad_reinit_rx(bnad); + } + + /* rx_count > 0 - new rx created + * - Linux set err = 0 and return + */ + err = bnad_mtu_set(bnad, new_frame); if (err) err = -EBUSY; @@ -3003,8 +3320,7 @@ bnad_change_mtu(struct net_device *netdev, int new_mtu) } static int -bnad_vlan_rx_add_vid(struct net_device *netdev, - unsigned short vid) +bnad_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) { struct bnad *bnad = netdev_priv(netdev); unsigned long flags; @@ -3025,8 +3341,7 @@ bnad_vlan_rx_add_vid(struct net_device *netdev, } static int -bnad_vlan_rx_kill_vid(struct net_device *netdev, - unsigned short vid) +bnad_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) { struct bnad *bnad = netdev_priv(netdev); unsigned long flags; @@ -3046,6 +3361,27 @@ bnad_vlan_rx_kill_vid(struct net_device *netdev, return 0; } +static int bnad_set_features(struct net_device *dev, netdev_features_t features) +{ + struct bnad *bnad = netdev_priv(dev); + netdev_features_t changed = features ^ dev->features; + + if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(dev)) { + unsigned long flags; + + spin_lock_irqsave(&bnad->bna_lock, flags); + + if (features & NETIF_F_HW_VLAN_CTAG_RX) + bna_rx_vlan_strip_enable(bnad->rx_info[0].rx); + else + bna_rx_vlan_strip_disable(bnad->rx_info[0].rx); + + spin_unlock_irqrestore(&bnad->bna_lock, flags); + } + + return 0; +} + #ifdef CONFIG_NET_POLL_CONTROLLER static void bnad_netpoll(struct net_device *netdev) @@ -3093,6 +3429,7 @@ static const struct net_device_ops bnad_netdev_ops = { .ndo_change_mtu = bnad_change_mtu, .ndo_vlan_rx_add_vid = bnad_vlan_rx_add_vid, .ndo_vlan_rx_kill_vid = bnad_vlan_rx_kill_vid, + .ndo_set_features = bnad_set_features, #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = bnad_netpoll #endif @@ -3105,14 +3442,14 @@ bnad_netdev_init(struct bnad *bnad, bool using_dac) netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | - NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_VLAN_TX; + NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_CTAG_RX; netdev->vlan_features = NETIF_F_SG | NETIF_F_HIGHDMA | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_TSO | NETIF_F_TSO6; - netdev->features |= netdev->hw_features | - NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_FILTER; + netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; if (using_dac) netdev->features |= NETIF_F_HIGHDMA; @@ -3127,8 +3464,8 @@ bnad_netdev_init(struct bnad *bnad, bool using_dac) /* * 1. Initialize the bnad structure * 2. Setup netdev pointer in pci_dev - * 3. Initialze Tx free tasklet - * 4. Initialize no. of TxQ & CQs & MSIX vectors + * 3. Initialize no. of TxQ & CQs & MSIX vectors + * 4. Initialize work queue. */ static int bnad_init(struct bnad *bnad, @@ -3146,7 +3483,6 @@ bnad_init(struct bnad *bnad, bnad->bar0 = ioremap_nocache(bnad->mmio_start, bnad->mmio_len); if (!bnad->bar0) { dev_err(&pdev->dev, "ioremap for bar0 failed\n"); - pci_set_drvdata(pdev, NULL); return -ENOMEM; } pr_info("bar0 mapped to %p, len %llu\n", bnad->bar0, @@ -3171,8 +3507,12 @@ bnad_init(struct bnad *bnad, bnad->tx_coalescing_timeo = BFI_TX_COALESCING_TIMEO; bnad->rx_coalescing_timeo = BFI_RX_COALESCING_TIMEO; - tasklet_init(&bnad->tx_free_tasklet, bnad_tx_free_tasklet, - (unsigned long)bnad); + sprintf(bnad->wq_name, "%s_wq_%d", BNAD_NAME, bnad->id); + bnad->work_q = create_singlethread_workqueue(bnad->wq_name); + if (!bnad->work_q) { + iounmap(bnad->bar0); + return -ENOMEM; + } return 0; } @@ -3185,9 +3525,14 @@ bnad_init(struct bnad *bnad, static void bnad_uninit(struct bnad *bnad) { + if (bnad->work_q) { + flush_workqueue(bnad->work_q); + destroy_workqueue(bnad->work_q); + bnad->work_q = NULL; + } + if (bnad->bar0) iounmap(bnad->bar0); - pci_set_drvdata(bnad->pcidev, NULL); } /* @@ -3224,17 +3569,12 @@ bnad_pci_init(struct bnad *bnad, err = pci_request_regions(pdev, BNAD_NAME); if (err) goto disable_device; - if (!dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)) && - !dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64))) { + if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { *using_dac = true; } else { - err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); - if (err) { - err = dma_set_coherent_mask(&pdev->dev, - DMA_BIT_MASK(32)); - if (err) - goto release_regions; - } + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (err) + goto release_regions; *using_dac = false; } pci_set_master(pdev); @@ -3255,7 +3595,7 @@ bnad_pci_uninit(struct pci_dev *pdev) pci_disable_device(pdev); } -static int __devinit +static int bnad_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pcidev_id) { @@ -3284,7 +3624,6 @@ bnad_pci_probe(struct pci_dev *pdev, */ netdev = alloc_etherdev(sizeof(struct bnad)); if (!netdev) { - dev_err(&pdev->dev, "netdev allocation failed\n"); err = -ENOMEM; return err; } @@ -3298,6 +3637,7 @@ bnad_pci_probe(struct pci_dev *pdev, * Output : using_dac = 1 for 64 bit DMA * = 0 for 32 bit DMA */ + using_dac = false; err = bnad_pci_init(bnad, pdev, &using_dac); if (err) goto unlock_mutex; @@ -3305,7 +3645,6 @@ bnad_pci_probe(struct pci_dev *pdev, /* * Initialize bnad structure * Setup relation between pci_dev & netdev - * Init Tx free tasklet */ err = bnad_init(bnad, pdev, netdev); if (err) @@ -3350,7 +3689,6 @@ bnad_pci_probe(struct pci_dev *pdev, if (err) goto res_free; - /* Set up timers */ setup_timer(&bnad->bna.ioceth.ioc.ioc_timer, bnad_ioc_timeout, ((unsigned long)bnad)); @@ -3456,7 +3794,7 @@ unlock_mutex: return err; } -static void __devexit +static void bnad_pci_remove(struct pci_dev *pdev) { struct net_device *netdev = pci_get_drvdata(pdev); @@ -3520,7 +3858,7 @@ static struct pci_driver bnad_pci_driver = { .name = BNAD_NAME, .id_table = bnad_pci_id_table, .probe = bnad_pci_probe, - .remove = __devexit_p(bnad_pci_remove), + .remove = bnad_pci_remove, }; static int __init @@ -3547,9 +3885,7 @@ static void __exit bnad_module_exit(void) { pci_unregister_driver(&bnad_pci_driver); - - if (bfi_fw) - release_firmware(bfi_fw); + release_firmware(bfi_fw); } module_init(bnad_module_init); |