aboutsummaryrefslogtreecommitdiff
path: root/drivers/infiniband/hw/cxgb3/iwch_cm.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/infiniband/hw/cxgb3/iwch_cm.c')
-rw-r--r--drivers/infiniband/hw/cxgb3/iwch_cm.c873
1 files changed, 514 insertions, 359 deletions
diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c
index b2faff5abce..cb78b1e9bcd 100644
--- a/drivers/infiniband/hw/cxgb3/iwch_cm.c
+++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c
@@ -31,10 +31,12 @@
*/
#include <linux/module.h>
#include <linux/list.h>
+#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/skbuff.h>
#include <linux/timer.h>
#include <linux/notifier.h>
+#include <linux/inetdevice.h>
#include <net/neighbour.h>
#include <net/netevent.h>
@@ -62,46 +64,47 @@ static char *states[] = {
NULL,
};
-static int ep_timeout_secs = 10;
-module_param(ep_timeout_secs, int, 0444);
+int peer2peer = 0;
+module_param(peer2peer, int, 0644);
+MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)");
+
+static int ep_timeout_secs = 60;
+module_param(ep_timeout_secs, int, 0644);
MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout "
- "in seconds (default=10)");
+ "in seconds (default=60)");
static int mpa_rev = 1;
-module_param(mpa_rev, int, 0444);
+module_param(mpa_rev, int, 0644);
MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, "
"1 is spec compliant. (default=1)");
static int markers_enabled = 0;
-module_param(markers_enabled, int, 0444);
+module_param(markers_enabled, int, 0644);
MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)");
static int crc_enabled = 1;
-module_param(crc_enabled, int, 0444);
+module_param(crc_enabled, int, 0644);
MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)");
static int rcv_win = 256 * 1024;
-module_param(rcv_win, int, 0444);
+module_param(rcv_win, int, 0644);
MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256)");
static int snd_win = 32 * 1024;
-module_param(snd_win, int, 0444);
+module_param(snd_win, int, 0644);
MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=32KB)");
static unsigned int nocong = 0;
-module_param(nocong, uint, 0444);
+module_param(nocong, uint, 0644);
MODULE_PARM_DESC(nocong, "Turn off congestion control (default=0)");
static unsigned int cong_flavor = 1;
-module_param(cong_flavor, uint, 0444);
+module_param(cong_flavor, uint, 0644);
MODULE_PARM_DESC(cong_flavor, "TCP Congestion control flavor (default=1)");
-static void process_work(struct work_struct *work);
static struct workqueue_struct *workq;
-static DECLARE_WORK(skb_work, process_work);
static struct sk_buff_head rxq;
-static cxgb3_cpl_handler_func work_handlers[NUM_CPL_CMDS];
static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp);
static void ep_timeout(unsigned long arg);
@@ -109,9 +112,9 @@ static void connect_reply_upcall(struct iwch_ep *ep, int status);
static void start_ep_timer(struct iwch_ep *ep)
{
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
if (timer_pending(&ep->timer)) {
- PDBG("%s stopped / restarted timer ep %p\n", __FUNCTION__, ep);
+ PDBG("%s stopped / restarted timer ep %p\n", __func__, ep);
del_timer_sync(&ep->timer);
} else
get_ep(&ep->com);
@@ -123,11 +126,48 @@ static void start_ep_timer(struct iwch_ep *ep)
static void stop_ep_timer(struct iwch_ep *ep)
{
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
+ if (!timer_pending(&ep->timer)) {
+ WARN(1, "%s timer stopped when its not running! ep %p state %u\n",
+ __func__, ep, ep->com.state);
+ return;
+ }
del_timer_sync(&ep->timer);
put_ep(&ep->com);
}
+static int iwch_l2t_send(struct t3cdev *tdev, struct sk_buff *skb, struct l2t_entry *l2e)
+{
+ int error = 0;
+ struct cxio_rdev *rdev;
+
+ rdev = (struct cxio_rdev *)tdev->ulp;
+ if (cxio_fatal_error(rdev)) {
+ kfree_skb(skb);
+ return -EIO;
+ }
+ error = l2t_send(tdev, skb, l2e);
+ if (error < 0)
+ kfree_skb(skb);
+ return error;
+}
+
+int iwch_cxgb3_ofld_send(struct t3cdev *tdev, struct sk_buff *skb)
+{
+ int error = 0;
+ struct cxio_rdev *rdev;
+
+ rdev = (struct cxio_rdev *)tdev->ulp;
+ if (cxio_fatal_error(rdev)) {
+ kfree_skb(skb);
+ return -EIO;
+ }
+ error = cxgb3_ofld_send(tdev, skb);
+ if (error < 0)
+ kfree_skb(skb);
+ return error;
+}
+
static void release_tid(struct t3cdev *tdev, u32 hwtid, struct sk_buff *skb)
{
struct cpl_tid_release *req;
@@ -139,7 +179,7 @@ static void release_tid(struct t3cdev *tdev, u32 hwtid, struct sk_buff *skb)
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid));
skb->priority = CPL_PRIORITY_SETUP;
- tdev->send(tdev, skb);
+ iwch_cxgb3_ofld_send(tdev, skb);
return;
}
@@ -161,8 +201,7 @@ int iwch_quiesce_tid(struct iwch_ep *ep)
req->val = cpu_to_be64(1 << S_TCB_RX_QUIESCE);
skb->priority = CPL_PRIORITY_DATA;
- ep->com.tdev->send(ep->com.tdev, skb);
- return 0;
+ return iwch_cxgb3_ofld_send(ep->com.tdev, skb);
}
int iwch_resume_tid(struct iwch_ep *ep)
@@ -183,13 +222,12 @@ int iwch_resume_tid(struct iwch_ep *ep)
req->val = 0;
skb->priority = CPL_PRIORITY_DATA;
- ep->com.tdev->send(ep->com.tdev, skb);
- return 0;
+ return iwch_cxgb3_ofld_send(ep->com.tdev, skb);
}
static void set_emss(struct iwch_ep *ep, u16 opt)
{
- PDBG("%s ep %p opt %u\n", __FUNCTION__, ep, opt);
+ PDBG("%s ep %p opt %u\n", __func__, ep, opt);
ep->emss = T3C_DATA(ep->com.tdev)->mtus[G_TCPOPT_MSS(opt)] - 40;
if (G_TCPOPT_TSTAMP(opt))
ep->emss -= 12;
@@ -219,7 +257,7 @@ static void state_set(struct iwch_ep_common *epc, enum iwch_ep_state new)
unsigned long flags;
spin_lock_irqsave(&epc->lock, flags);
- PDBG("%s - %s -> %s\n", __FUNCTION__, states[epc->state], states[new]);
+ PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]);
__state_set(epc, new);
spin_unlock_irqrestore(&epc->lock, flags);
return;
@@ -229,57 +267,37 @@ static void *alloc_ep(int size, gfp_t gfp)
{
struct iwch_ep_common *epc;
- epc = kmalloc(size, gfp);
+ epc = kzalloc(size, gfp);
if (epc) {
- memset(epc, 0, size);
kref_init(&epc->kref);
spin_lock_init(&epc->lock);
init_waitqueue_head(&epc->waitq);
}
- PDBG("%s alloc ep %p\n", __FUNCTION__, epc);
+ PDBG("%s alloc ep %p\n", __func__, epc);
return epc;
}
void __free_ep(struct kref *kref)
{
- struct iwch_ep_common *epc;
- epc = container_of(kref, struct iwch_ep_common, kref);
- PDBG("%s ep %p state %s\n", __FUNCTION__, epc, states[state_read(epc)]);
- kfree(epc);
+ struct iwch_ep *ep;
+ ep = container_of(container_of(kref, struct iwch_ep_common, kref),
+ struct iwch_ep, com);
+ PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]);
+ if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) {
+ cxgb3_remove_tid(ep->com.tdev, (void *)ep, ep->hwtid);
+ dst_release(ep->dst);
+ l2t_release(ep->com.tdev, ep->l2t);
+ }
+ kfree(ep);
}
static void release_ep_resources(struct iwch_ep *ep)
{
- PDBG("%s ep %p tid %d\n", __FUNCTION__, ep, ep->hwtid);
- cxgb3_remove_tid(ep->com.tdev, (void *)ep, ep->hwtid);
- dst_release(ep->dst);
- l2t_release(L2DATA(ep->com.tdev), ep->l2t);
- if (ep->com.tdev->type == T3B)
- release_tid(ep->com.tdev, ep->hwtid, NULL);
+ PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid);
+ set_bit(RELEASE_RESOURCES, &ep->com.flags);
put_ep(&ep->com);
}
-static void process_work(struct work_struct *work)
-{
- struct sk_buff *skb = NULL;
- void *ep;
- struct t3cdev *tdev;
- int ret;
-
- while ((skb = skb_dequeue(&rxq))) {
- ep = *((void **) (skb->cb));
- tdev = *((struct t3cdev **) (skb->cb + sizeof(void *)));
- ret = work_handlers[G_OPCODE(ntohl((__force __be32)skb->csum))](tdev, skb, ep);
- if (ret & CPL_RET_BUF_DONE)
- kfree_skb(skb);
-
- /*
- * ep was referenced in sched(), and is freed here.
- */
- put_ep((struct iwch_ep_common *)ep);
- }
-}
-
static int status2errno(int status)
{
switch (status) {
@@ -319,23 +337,12 @@ static struct rtable *find_route(struct t3cdev *dev, __be32 local_ip,
__be16 peer_port, u8 tos)
{
struct rtable *rt;
- struct flowi fl = {
- .oif = 0,
- .nl_u = {
- .ip4_u = {
- .daddr = peer_ip,
- .saddr = local_ip,
- .tos = tos}
- },
- .proto = IPPROTO_TCP,
- .uli_u = {
- .ports = {
- .sport = local_port,
- .dport = peer_port}
- }
- };
-
- if (ip_route_output_flow(&rt, &fl, NULL, 0))
+ struct flowi4 fl4;
+
+ rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip,
+ peer_port, local_port, IPPROTO_TCP,
+ tos, 0);
+ if (IS_ERR(rt))
return NULL;
return rt;
}
@@ -351,7 +358,7 @@ static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
static void arp_failure_discard(struct t3cdev *dev, struct sk_buff *skb)
{
- PDBG("%s t3cdev %p\n", __FUNCTION__, dev);
+ PDBG("%s t3cdev %p\n", __func__, dev);
kfree_skb(skb);
}
@@ -372,9 +379,9 @@ static void abort_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
{
struct cpl_abort_req *req = cplhdr(skb);
- PDBG("%s t3cdev %p\n", __FUNCTION__, dev);
+ PDBG("%s t3cdev %p\n", __func__, dev);
req->cmd = CPL_ABORT_NO_RST;
- cxgb3_ofld_send(dev, skb);
+ iwch_cxgb3_ofld_send(dev, skb);
}
static int send_halfclose(struct iwch_ep *ep, gfp_t gfp)
@@ -382,10 +389,10 @@ static int send_halfclose(struct iwch_ep *ep, gfp_t gfp)
struct cpl_close_con_req *req;
struct sk_buff *skb;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
skb = get_skb(NULL, sizeof(*req), gfp);
if (!skb) {
- printk(KERN_ERR MOD "%s - failed to alloc skb\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
return -ENOMEM;
}
skb->priority = CPL_PRIORITY_DATA;
@@ -394,30 +401,29 @@ static int send_halfclose(struct iwch_ep *ep, gfp_t gfp)
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, ep->hwtid));
- l2t_send(ep->com.tdev, skb, ep->l2t);
- return 0;
+ return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
}
static int send_abort(struct iwch_ep *ep, struct sk_buff *skb, gfp_t gfp)
{
struct cpl_abort_req *req;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
skb = get_skb(skb, sizeof(*req), gfp);
if (!skb) {
printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
- __FUNCTION__);
+ __func__);
return -ENOMEM;
}
skb->priority = CPL_PRIORITY_DATA;
set_arp_failure_handler(skb, abort_arp_failure);
req = (struct cpl_abort_req *) skb_put(skb, sizeof(*req));
+ memset(req, 0, sizeof(*req));
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
req->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid));
req->cmd = CPL_ABORT_SEND_RST;
- l2t_send(ep->com.tdev, skb, ep->l2t);
- return 0;
+ return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
}
static int send_connect(struct iwch_ep *ep)
@@ -428,12 +434,12 @@ static int send_connect(struct iwch_ep *ep)
unsigned int mtu_idx;
int wscale;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
if (!skb) {
printk(KERN_ERR MOD "%s - failed to alloc skb.\n",
- __FUNCTION__);
+ __func__);
return -ENOMEM;
}
mtu_idx = find_best_mtu(T3C_DATA(ep->com.tdev), dst_mtu(ep->dst));
@@ -446,7 +452,8 @@ static int send_connect(struct iwch_ep *ep)
V_MSS_IDX(mtu_idx) |
V_L2T_IDX(ep->l2t->idx) | V_TX_CHANNEL(ep->l2t->smt_idx);
opt0l = V_TOS((ep->tos >> 2) & M_TOS) | V_RCV_BUFSIZ(rcv_win>>10);
- opt2 = V_FLAVORS_VALID(1) | V_CONG_CONTROL_FLAVOR(cong_flavor);
+ opt2 = F_RX_COALESCE_VALID | V_RX_COALESCE(0) | V_FLAVORS_VALID(1) |
+ V_CONG_CONTROL_FLAVOR(cong_flavor);
skb->priority = CPL_PRIORITY_SETUP;
set_arp_failure_handler(skb, act_open_req_arp_failure);
@@ -461,8 +468,7 @@ static int send_connect(struct iwch_ep *ep)
req->opt0l = htonl(opt0l);
req->params = 0;
req->opt2 = htonl(opt2);
- l2t_send(ep->com.tdev, skb, ep->l2t);
- return 0;
+ return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
}
static void send_mpa_req(struct iwch_ep *ep, struct sk_buff *skb)
@@ -472,7 +478,7 @@ static void send_mpa_req(struct iwch_ep *ep, struct sk_buff *skb)
struct mpa_message *mpa;
int len;
- PDBG("%s ep %p pd_len %d\n", __FUNCTION__, ep, ep->plen);
+ PDBG("%s ep %p pd_len %d\n", __func__, ep, ep->plen);
BUG_ON(skb_cloned(skb));
@@ -510,16 +516,16 @@ static void send_mpa_req(struct iwch_ep *ep, struct sk_buff *skb)
skb_reset_transport_header(skb);
len = skb->len;
req = (struct tx_data_wr *) skb_push(skb, sizeof(*req));
- req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
+ req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);
req->wr_lo = htonl(V_WR_TID(ep->hwtid));
req->len = htonl(len);
req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) |
V_TX_SNDBUF(snd_win>>15));
- req->flags = htonl(F_TX_IMM_ACK|F_TX_INIT);
+ req->flags = htonl(F_TX_INIT);
req->sndseq = htonl(ep->snd_seq);
BUG_ON(ep->mpa_skb);
ep->mpa_skb = skb;
- l2t_send(ep->com.tdev, skb, ep->l2t);
+ iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
start_ep_timer(ep);
state_set(&ep->com, MPA_REQ_SENT);
return;
@@ -532,13 +538,13 @@ static int send_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen)
struct mpa_message *mpa;
struct sk_buff *skb;
- PDBG("%s ep %p plen %d\n", __FUNCTION__, ep, plen);
+ PDBG("%s ep %p plen %d\n", __func__, ep, plen);
mpalen = sizeof(*mpa) + plen;
skb = get_skb(NULL, mpalen + sizeof(*req), GFP_KERNEL);
if (!skb) {
- printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
return -ENOMEM;
}
skb_reserve(skb, sizeof(*req));
@@ -561,17 +567,16 @@ static int send_mpa_reject(struct iwch_ep *ep, const void *pdata, u8 plen)
set_arp_failure_handler(skb, arp_failure_discard);
skb_reset_transport_header(skb);
req = (struct tx_data_wr *) skb_push(skb, sizeof(*req));
- req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
+ req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);
req->wr_lo = htonl(V_WR_TID(ep->hwtid));
req->len = htonl(mpalen);
req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) |
V_TX_SNDBUF(snd_win>>15));
- req->flags = htonl(F_TX_IMM_ACK|F_TX_INIT);
+ req->flags = htonl(F_TX_INIT);
req->sndseq = htonl(ep->snd_seq);
BUG_ON(ep->mpa_skb);
ep->mpa_skb = skb;
- l2t_send(ep->com.tdev, skb, ep->l2t);
- return 0;
+ return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
}
static int send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen)
@@ -582,13 +587,13 @@ static int send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen)
int len;
struct sk_buff *skb;
- PDBG("%s ep %p plen %d\n", __FUNCTION__, ep, plen);
+ PDBG("%s ep %p plen %d\n", __func__, ep, plen);
mpalen = sizeof(*mpa) + plen;
skb = get_skb(NULL, mpalen + sizeof(*req), GFP_KERNEL);
if (!skb) {
- printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__);
return -ENOMEM;
}
skb->priority = CPL_PRIORITY_DATA;
@@ -613,17 +618,16 @@ static int send_mpa_reply(struct iwch_ep *ep, const void *pdata, u8 plen)
skb_reset_transport_header(skb);
len = skb->len;
req = (struct tx_data_wr *) skb_push(skb, sizeof(*req));
- req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
+ req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)|F_WR_COMPL);
req->wr_lo = htonl(V_WR_TID(ep->hwtid));
req->len = htonl(len);
req->param = htonl(V_TX_PORT(ep->l2t->smt_idx) |
V_TX_SNDBUF(snd_win>>15));
- req->flags = htonl(F_TX_MORE | F_TX_IMM_ACK | F_TX_INIT);
+ req->flags = htonl(F_TX_INIT);
req->sndseq = htonl(ep->snd_seq);
ep->mpa_skb = skb;
state_set(&ep->com, MPA_REP_SENT);
- l2t_send(ep->com.tdev, skb, ep->l2t);
- return 0;
+ return iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
}
static int act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
@@ -632,7 +636,7 @@ static int act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
struct cpl_act_establish *req = cplhdr(skb);
unsigned int tid = GET_TID(req);
- PDBG("%s ep %p tid %d\n", __FUNCTION__, ep, tid);
+ PDBG("%s ep %p tid %d\n", __func__, ep, tid);
dst_confirm(ep->dst);
@@ -641,6 +645,7 @@ static int act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
cxgb3_insert_tid(ep->com.tdev, &t3c_client, ep, tid);
ep->snd_seq = ntohl(req->snd_isn);
+ ep->rcv_seq = ntohl(req->rcv_isn);
set_emss(ep, ntohs(req->tcp_opt));
@@ -664,7 +669,7 @@ static void close_complete_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CLOSE;
if (ep->com.cm_id) {
@@ -681,7 +686,7 @@ static void peer_close_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_DISCONNECT;
if (ep->com.cm_id) {
@@ -695,7 +700,7 @@ static void peer_abort_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CLOSE;
event.status = -ECONNRESET;
@@ -713,19 +718,21 @@ static void connect_reply_upcall(struct iwch_ep *ep, int status)
{
struct iw_cm_event event;
- PDBG("%s ep %p status %d\n", __FUNCTION__, ep, status);
+ PDBG("%s ep %p status %d\n", __func__, ep, status);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CONNECT_REPLY;
event.status = status;
- event.local_addr = ep->com.local_addr;
- event.remote_addr = ep->com.remote_addr;
+ memcpy(&event.local_addr, &ep->com.local_addr,
+ sizeof(ep->com.local_addr));
+ memcpy(&event.remote_addr, &ep->com.remote_addr,
+ sizeof(ep->com.remote_addr));
if ((status == 0) || (status == -ECONNREFUSED)) {
event.private_data_len = ep->plen;
event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
}
if (ep->com.cm_id) {
- PDBG("%s ep %p tid %d status %d\n", __FUNCTION__, ep,
+ PDBG("%s ep %p tid %d status %d\n", __func__, ep,
ep->hwtid, status);
ep->com.cm_id->event_handler(ep->com.cm_id, &event);
}
@@ -740,18 +747,27 @@ static void connect_request_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
- PDBG("%s ep %p tid %d\n", __FUNCTION__, ep, ep->hwtid);
+ PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_CONNECT_REQUEST;
- event.local_addr = ep->com.local_addr;
- event.remote_addr = ep->com.remote_addr;
+ memcpy(&event.local_addr, &ep->com.local_addr,
+ sizeof(ep->com.local_addr));
+ memcpy(&event.remote_addr, &ep->com.remote_addr,
+ sizeof(ep->com.local_addr));
event.private_data_len = ep->plen;
event.private_data = ep->mpa_pkt + sizeof(struct mpa_message);
event.provider_data = ep;
- if (state_read(&ep->parent_ep->com) != DEAD)
+ /*
+ * Until ird/ord negotiation via MPAv2 support is added, send max
+ * supported values
+ */
+ event.ird = event.ord = 8;
+ if (state_read(&ep->parent_ep->com) != DEAD) {
+ get_ep(&ep->com);
ep->parent_ep->com.cm_id->event_handler(
ep->parent_ep->com.cm_id,
&event);
+ }
put_ep(&ep->parent_ep->com);
ep->parent_ep = NULL;
}
@@ -760,11 +776,16 @@ static void established_upcall(struct iwch_ep *ep)
{
struct iw_cm_event event;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
memset(&event, 0, sizeof(event));
event.event = IW_CM_EVENT_ESTABLISHED;
+ /*
+ * Until ird/ord negotiation via MPAv2 support is added, send max
+ * supported values
+ */
+ event.ird = event.ord = 8;
if (ep->com.cm_id) {
- PDBG("%s ep %p tid %d\n", __FUNCTION__, ep, ep->hwtid);
+ PDBG("%s ep %p tid %d\n", __func__, ep, ep->hwtid);
ep->com.cm_id->event_handler(ep->com.cm_id, &event);
}
}
@@ -774,7 +795,7 @@ static int update_rx_credits(struct iwch_ep *ep, u32 credits)
struct cpl_rx_data_ack *req;
struct sk_buff *skb;
- PDBG("%s ep %p credits %u\n", __FUNCTION__, ep, credits);
+ PDBG("%s ep %p credits %u\n", __func__, ep, credits);
skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
if (!skb) {
printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n");
@@ -786,7 +807,7 @@ static int update_rx_credits(struct iwch_ep *ep, u32 credits)
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, ep->hwtid));
req->credit_dack = htonl(V_RX_CREDITS(credits) | V_RX_FORCE_ACK(1));
skb->priority = CPL_PRIORITY_ACK;
- ep->com.tdev->send(ep->com.tdev, skb);
+ iwch_cxgb3_ofld_send(ep->com.tdev, skb);
return credits;
}
@@ -798,7 +819,7 @@ static void process_mpa_reply(struct iwch_ep *ep, struct sk_buff *skb)
enum iwch_qp_attr_mask mask;
int err;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
/*
* Stop mpa timer. If it expired, then the state has
@@ -880,12 +901,13 @@ static void process_mpa_reply(struct iwch_ep *ep, struct sk_buff *skb)
* the MPA header is valid.
*/
state_set(&ep->com, FPDU_MODE);
+ ep->mpa_attr.initiator = 1;
ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
ep->mpa_attr.recv_marker_enabled = markers_enabled;
ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
ep->mpa_attr.version = mpa_rev;
PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
- "xmit_marker_enabled=%d, version=%d\n", __FUNCTION__,
+ "xmit_marker_enabled=%d, version=%d\n", __func__,
ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
@@ -902,8 +924,14 @@ static void process_mpa_reply(struct iwch_ep *ep, struct sk_buff *skb)
/* bind QP and TID with INIT_WR */
err = iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp, mask, &attrs, 1);
- if (!err)
- goto out;
+ if (err)
+ goto err;
+
+ if (peer2peer && iwch_rqes_posted(ep->com.qp) == 0) {
+ iwch_post_zb_read(ep);
+ }
+
+ goto out;
err:
abort_connection(ep, skb, GFP_KERNEL);
out:
@@ -916,7 +944,7 @@ static void process_mpa_request(struct iwch_ep *ep, struct sk_buff *skb)
struct mpa_message *mpa;
u16 plen;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
/*
* Stop mpa timer. If it expired, then the state has
@@ -936,7 +964,7 @@ static void process_mpa_request(struct iwch_ep *ep, struct sk_buff *skb)
return;
}
- PDBG("%s enter (%s line %u)\n", __FUNCTION__, __FILE__, __LINE__);
+ PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);
/*
* Copy the new data into our accumulation buffer.
@@ -951,7 +979,7 @@ static void process_mpa_request(struct iwch_ep *ep, struct sk_buff *skb)
*/
if (ep->mpa_pkt_len < sizeof(*mpa))
return;
- PDBG("%s enter (%s line %u)\n", __FUNCTION__, __FILE__, __LINE__);
+ PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__);
mpa = (struct mpa_message *) ep->mpa_pkt;
/*
@@ -996,12 +1024,13 @@ static void process_mpa_request(struct iwch_ep *ep, struct sk_buff *skb)
* If we get here we have accumulated the entire mpa
* start reply message including private data.
*/
+ ep->mpa_attr.initiator = 0;
ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0;
ep->mpa_attr.recv_marker_enabled = markers_enabled;
ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0;
ep->mpa_attr.version = mpa_rev;
PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, "
- "xmit_marker_enabled=%d, version=%d\n", __FUNCTION__,
+ "xmit_marker_enabled=%d, version=%d\n", __func__,
ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled,
ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version);
@@ -1018,11 +1047,14 @@ static int rx_data(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
struct cpl_rx_data *hdr = cplhdr(skb);
unsigned int dlen = ntohs(hdr->len);
- PDBG("%s ep %p dlen %u\n", __FUNCTION__, ep, dlen);
+ PDBG("%s ep %p dlen %u\n", __func__, ep, dlen);
skb_pull(skb, sizeof(*hdr));
skb_trim(skb, dlen);
+ ep->rcv_seq += dlen;
+ BUG_ON(ep->rcv_seq != (ntohl(hdr->seq) + dlen));
+
switch (state_read(&ep->com)) {
case MPA_REQ_SENT:
process_mpa_reply(ep, skb);
@@ -1035,7 +1067,7 @@ static int rx_data(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
default:
printk(KERN_ERR MOD "%s Unexpected streaming data."
" ep %p state %d tid %d\n",
- __FUNCTION__, ep, state_read(&ep->com), ep->hwtid);
+ __func__, ep, state_read(&ep->com), ep->hwtid);
/*
* The ep will timeout and inform the ULP of the failure.
@@ -1060,84 +1092,107 @@ static int tx_ack(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
struct iwch_ep *ep = ctx;
struct cpl_wr_ack *hdr = cplhdr(skb);
unsigned int credits = ntohs(hdr->credits);
- enum iwch_qp_attr_mask mask;
+ unsigned long flags;
+ int post_zb = 0;
- PDBG("%s ep %p credits %u\n", __FUNCTION__, ep, credits);
+ PDBG("%s ep %p credits %u\n", __func__, ep, credits);
- if (credits == 0)
+ if (credits == 0) {
+ PDBG("%s 0 credit ack ep %p state %u\n",
+ __func__, ep, state_read(&ep->com));
return CPL_RET_BUF_DONE;
+ }
+
+ spin_lock_irqsave(&ep->com.lock, flags);
BUG_ON(credits != 1);
- BUG_ON(ep->mpa_skb == NULL);
- kfree_skb(ep->mpa_skb);
- ep->mpa_skb = NULL;
dst_confirm(ep->dst);
- if (state_read(&ep->com) == MPA_REP_SENT) {
- struct iwch_qp_attributes attrs;
-
- /* bind QP to EP and move to RTS */
- attrs.mpa_attr = ep->mpa_attr;
- attrs.max_ird = ep->ord;
- attrs.max_ord = ep->ord;
- attrs.llp_stream_handle = ep;
- attrs.next_state = IWCH_QP_STATE_RTS;
-
- /* bind QP and TID with INIT_WR */
- mask = IWCH_QP_ATTR_NEXT_STATE |
- IWCH_QP_ATTR_LLP_STREAM_HANDLE |
- IWCH_QP_ATTR_MPA_ATTR |
- IWCH_QP_ATTR_MAX_IRD |
- IWCH_QP_ATTR_MAX_ORD;
-
- ep->com.rpl_err = iwch_modify_qp(ep->com.qp->rhp,
- ep->com.qp, mask, &attrs, 1);
-
- if (!ep->com.rpl_err) {
- state_set(&ep->com, FPDU_MODE);
- established_upcall(ep);
+ if (!ep->mpa_skb) {
+ PDBG("%s rdma_init wr_ack ep %p state %u\n",
+ __func__, ep, ep->com.state);
+ if (ep->mpa_attr.initiator) {
+ PDBG("%s initiator ep %p state %u\n",
+ __func__, ep, ep->com.state);
+ if (peer2peer && ep->com.state == FPDU_MODE)
+ post_zb = 1;
+ } else {
+ PDBG("%s responder ep %p state %u\n",
+ __func__, ep, ep->com.state);
+ if (ep->com.state == MPA_REQ_RCVD) {
+ ep->com.rpl_done = 1;
+ wake_up(&ep->com.waitq);
+ }
}
-
- ep->com.rpl_done = 1;
- PDBG("waking up ep %p\n", ep);
- wake_up(&ep->com.waitq);
+ } else {
+ PDBG("%s lsm ack ep %p state %u freeing skb\n",
+ __func__, ep, ep->com.state);
+ kfree_skb(ep->mpa_skb);
+ ep->mpa_skb = NULL;
}
+ spin_unlock_irqrestore(&ep->com.lock, flags);
+ if (post_zb)
+ iwch_post_zb_read(ep);
return CPL_RET_BUF_DONE;
}
static int abort_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
{
struct iwch_ep *ep = ctx;
+ unsigned long flags;
+ int release = 0;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
+ BUG_ON(!ep);
/*
* We get 2 abort replies from the HW. The first one must
* be ignored except for scribbling that we need one more.
*/
- if (!(ep->flags & ABORT_REQ_IN_PROGRESS)) {
- ep->flags |= ABORT_REQ_IN_PROGRESS;
+ if (!test_and_set_bit(ABORT_REQ_IN_PROGRESS, &ep->com.flags)) {
return CPL_RET_BUF_DONE;
}
- close_complete_upcall(ep);
- state_set(&ep->com, DEAD);
- release_ep_resources(ep);
+ spin_lock_irqsave(&ep->com.lock, flags);
+ switch (ep->com.state) {
+ case ABORTING:
+ close_complete_upcall(ep);
+ __state_set(&ep->com, DEAD);
+ release = 1;
+ break;
+ default:
+ printk(KERN_ERR "%s ep %p state %d\n",
+ __func__, ep, ep->com.state);
+ break;
+ }
+ spin_unlock_irqrestore(&ep->com.lock, flags);
+
+ if (release)
+ release_ep_resources(ep);
return CPL_RET_BUF_DONE;
}
+/*
+ * Return whether a failed active open has allocated a TID
+ */
+static inline int act_open_has_tid(int status)
+{
+ return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST &&
+ status != CPL_ERR_ARP_MISS;
+}
+
static int act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
{
struct iwch_ep *ep = ctx;
struct cpl_act_open_rpl *rpl = cplhdr(skb);
- PDBG("%s ep %p status %u errno %d\n", __FUNCTION__, ep, rpl->status,
+ PDBG("%s ep %p status %u errno %d\n", __func__, ep, rpl->status,
status2errno(rpl->status));
connect_reply_upcall(ep, status2errno(rpl->status));
state_set(&ep->com, DEAD);
- if (ep->com.tdev->type == T3B)
+ if (ep->com.tdev->type != T3A && act_open_has_tid(rpl->status))
release_tid(ep->com.tdev, GET_TID(rpl), NULL);
cxgb3_free_atid(ep->com.tdev, ep->atid);
dst_release(ep->dst);
- l2t_release(L2DATA(ep->com.tdev), ep->l2t);
+ l2t_release(ep->com.tdev, ep->l2t);
put_ep(&ep->com);
return CPL_RET_BUF_DONE;
}
@@ -1147,7 +1202,7 @@ static int listen_start(struct iwch_listen_ep *ep)
struct sk_buff *skb;
struct cpl_pass_open_req *req;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
if (!skb) {
printk(KERN_ERR MOD "t3c_listen_start failed to alloc skb!\n");
@@ -1167,8 +1222,7 @@ static int listen_start(struct iwch_listen_ep *ep)
req->opt1 = htonl(V_CONN_POLICY(CPL_CONN_POLICY_ASK));
skb->priority = 1;
- ep->com.tdev->send(ep->com.tdev, skb);
- return 0;
+ return iwch_cxgb3_ofld_send(ep->com.tdev, skb);
}
static int pass_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
@@ -1176,7 +1230,7 @@ static int pass_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
struct iwch_listen_ep *ep = ctx;
struct cpl_pass_open_rpl *rpl = cplhdr(skb);
- PDBG("%s ep %p status %d error %d\n", __FUNCTION__, ep,
+ PDBG("%s ep %p status %d error %d\n", __func__, ep,
rpl->status, status2errno(rpl->status));
ep->com.rpl_err = status2errno(rpl->status);
ep->com.rpl_done = 1;
@@ -1190,10 +1244,10 @@ static int listen_stop(struct iwch_listen_ep *ep)
struct sk_buff *skb;
struct cpl_close_listserv_req *req;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
skb = get_skb(NULL, sizeof(*req), GFP_KERNEL);
if (!skb) {
- printk(KERN_ERR MOD "%s - failed to alloc skb\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__);
return -ENOMEM;
}
req = (struct cpl_close_listserv_req *) skb_put(skb, sizeof(*req));
@@ -1201,8 +1255,7 @@ static int listen_stop(struct iwch_listen_ep *ep)
req->cpu_idx = 0;
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, ep->stid));
skb->priority = 1;
- ep->com.tdev->send(ep->com.tdev, skb);
- return 0;
+ return iwch_cxgb3_ofld_send(ep->com.tdev, skb);
}
static int close_listsrv_rpl(struct t3cdev *tdev, struct sk_buff *skb,
@@ -1211,7 +1264,7 @@ static int close_listsrv_rpl(struct t3cdev *tdev, struct sk_buff *skb,
struct iwch_listen_ep *ep = ctx;
struct cpl_close_listserv_rpl *rpl = cplhdr(skb);
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
ep->com.rpl_err = status2errno(rpl->status);
ep->com.rpl_done = 1;
wake_up(&ep->com.waitq);
@@ -1225,7 +1278,7 @@ static void accept_cr(struct iwch_ep *ep, __be32 peer_ip, struct sk_buff *skb)
u32 opt0h, opt0l, opt2;
int wscale;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
BUG_ON(skb_cloned(skb));
skb_trim(skb, sizeof(*rpl));
skb_get(skb);
@@ -1239,7 +1292,8 @@ static void accept_cr(struct iwch_ep *ep, __be32 peer_ip, struct sk_buff *skb)
V_MSS_IDX(mtu_idx) |
V_L2T_IDX(ep->l2t->idx) | V_TX_CHANNEL(ep->l2t->smt_idx);
opt0l = V_TOS((ep->tos >> 2) & M_TOS) | V_RCV_BUFSIZ(rcv_win>>10);
- opt2 = V_FLAVORS_VALID(1) | V_CONG_CONTROL_FLAVOR(cong_flavor);
+ opt2 = F_RX_COALESCE_VALID | V_RX_COALESCE(0) | V_FLAVORS_VALID(1) |
+ V_CONG_CONTROL_FLAVOR(cong_flavor);
rpl = cplhdr(skb);
rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
@@ -1250,7 +1304,7 @@ static void accept_cr(struct iwch_ep *ep, __be32 peer_ip, struct sk_buff *skb)
rpl->opt2 = htonl(opt2);
rpl->rsvd = rpl->opt2; /* workaround for HW bug */
skb->priority = CPL_PRIORITY_SETUP;
- l2t_send(ep->com.tdev, skb, ep->l2t);
+ iwch_l2t_send(ep->com.tdev, skb, ep->l2t);
return;
}
@@ -1258,13 +1312,13 @@ static void accept_cr(struct iwch_ep *ep, __be32 peer_ip, struct sk_buff *skb)
static void reject_cr(struct t3cdev *tdev, u32 hwtid, __be32 peer_ip,
struct sk_buff *skb)
{
- PDBG("%s t3cdev %p tid %u peer_ip %x\n", __FUNCTION__, tdev, hwtid,
+ PDBG("%s t3cdev %p tid %u peer_ip %x\n", __func__, tdev, hwtid,
peer_ip);
BUG_ON(skb_cloned(skb));
skb_trim(skb, sizeof(struct cpl_tid_release));
skb_get(skb);
- if (tdev->type == T3B)
+ if (tdev->type != T3A)
release_tid(tdev, hwtid, skb);
else {
struct cpl_pass_accept_rpl *rpl;
@@ -1279,7 +1333,7 @@ static void reject_cr(struct t3cdev *tdev, u32 hwtid, __be32 peer_ip,
rpl->opt0l_status = htonl(CPL_PASS_OPEN_REJECT);
rpl->opt2 = 0;
rpl->rsvd = rpl->opt2;
- tdev->send(tdev, skb);
+ iwch_cxgb3_ofld_send(tdev, skb);
}
}
@@ -1293,11 +1347,11 @@ static int pass_accept_req(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
struct rtable *rt;
struct iff_mac tim;
- PDBG("%s parent ep %p tid %u\n", __FUNCTION__, parent_ep, hwtid);
+ PDBG("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid);
if (state_read(&parent_ep->com) != LISTEN) {
printk(KERN_ERR "%s - listening ep not in LISTEN\n",
- __FUNCTION__);
+ __func__);
goto reject;
}
@@ -1307,15 +1361,8 @@ static int pass_accept_req(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
tim.mac_addr = req->dst_mac;
tim.vlan_tag = ntohs(req->vlan_tag);
if (tdev->ctl(tdev, GET_IFF_FROM_MAC, &tim) < 0 || !tim.dev) {
- printk(KERN_ERR
- "%s bad dst mac %02x %02x %02x %02x %02x %02x\n",
- __FUNCTION__,
- req->dst_mac[0],
- req->dst_mac[1],
- req->dst_mac[2],
- req->dst_mac[3],
- req->dst_mac[4],
- req->dst_mac[5]);
+ printk(KERN_ERR "%s bad dst mac %pM\n",
+ __func__, req->dst_mac);
goto reject;
}
@@ -1327,22 +1374,22 @@ static int pass_accept_req(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
req->peer_port, G_PASS_OPEN_TOS(ntohl(req->tos_tid)));
if (!rt) {
printk(KERN_ERR MOD "%s - failed to find dst entry!\n",
- __FUNCTION__);
+ __func__);
goto reject;
}
- dst = &rt->u.dst;
- l2t = t3_l2t_get(tdev, dst->neighbour, dst->neighbour->dev);
+ dst = &rt->dst;
+ l2t = t3_l2t_get(tdev, dst, NULL, &req->peer_ip);
if (!l2t) {
printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n",
- __FUNCTION__);
+ __func__);
dst_release(dst);
goto reject;
}
child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL);
if (!child_ep) {
printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n",
- __FUNCTION__);
- l2t_release(L2DATA(tdev), l2t);
+ __func__);
+ l2t_release(tdev, l2t);
dst_release(dst);
goto reject;
}
@@ -1376,8 +1423,9 @@ static int pass_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
struct iwch_ep *ep = ctx;
struct cpl_pass_establish *req = cplhdr(skb);
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
ep->snd_seq = ntohl(req->snd_isn);
+ ep->rcv_seq = ntohl(req->rcv_isn);
set_emss(ep, ntohs(req->tcp_opt));
@@ -1396,7 +1444,7 @@ static int peer_close(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
int disconnect = 1;
int release = 0;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
dst_confirm(ep->dst);
spin_lock_irqsave(&ep->com.lock, flags);
@@ -1413,10 +1461,14 @@ static int peer_close(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
/*
* We're gonna mark this puppy DEAD, but keep
* the reference on it until the ULP accepts or
- * rejects the CR.
+ * rejects the CR. Also wake up anyone waiting
+ * in rdma connection migration (see iwch_accept_cr()).
*/
__state_set(&ep->com, CLOSING);
- get_ep(&ep->com);
+ ep->com.rpl_done = 1;
+ ep->com.rpl_err = -ECONNRESET;
+ PDBG("waking up ep %p\n", ep);
+ wake_up(&ep->com.waitq);
break;
case MPA_REP_SENT:
__state_set(&ep->com, CLOSING);
@@ -1483,27 +1535,27 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
struct sk_buff *rpl_skb;
struct iwch_qp_attributes attrs;
int ret;
- int state;
+ int release = 0;
+ unsigned long flags;
+
+ if (is_neg_adv_abort(req->status)) {
+ PDBG("%s neg_adv_abort ep %p tid %d\n", __func__, ep,
+ ep->hwtid);
+ t3_l2t_send_event(ep->com.tdev, ep->l2t);
+ return CPL_RET_BUF_DONE;
+ }
/*
* We get 2 peer aborts from the HW. The first one must
* be ignored except for scribbling that we need one more.
*/
- if (!(ep->flags & PEER_ABORT_IN_PROGRESS)) {
- ep->flags |= PEER_ABORT_IN_PROGRESS;
- return CPL_RET_BUF_DONE;
- }
-
- if (is_neg_adv_abort(req->status)) {
- PDBG("%s neg_adv_abort ep %p tid %d\n", __FUNCTION__, ep,
- ep->hwtid);
- t3_l2t_send_event(ep->com.tdev, ep->l2t);
+ if (!test_and_set_bit(PEER_ABORT_IN_PROGRESS, &ep->com.flags)) {
return CPL_RET_BUF_DONE;
}
- state = state_read(&ep->com);
- PDBG("%s ep %p state %u\n", __FUNCTION__, ep, state);
- switch (state) {
+ spin_lock_irqsave(&ep->com.lock, flags);
+ PDBG("%s ep %p state %u\n", __func__, ep, ep->com.state);
+ switch (ep->com.state) {
case CONNECTING:
break;
case MPA_REQ_WAIT:
@@ -1524,9 +1576,13 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
/*
* We're gonna mark this puppy DEAD, but keep
* the reference on it until the ULP accepts or
- * rejects the CR.
+ * rejects the CR. Also wake up anyone waiting
+ * in rdma connection migration (see iwch_accept_cr()).
*/
- get_ep(&ep->com);
+ ep->com.rpl_done = 1;
+ ep->com.rpl_err = -ECONNRESET;
+ PDBG("waking up ep %p\n", ep);
+ wake_up(&ep->com.waitq);
break;
case MORIBUND:
case CLOSING:
@@ -1541,29 +1597,33 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
if (ret)
printk(KERN_ERR MOD
"%s - qp <- error failed!\n",
- __FUNCTION__);
+ __func__);
}
peer_abort_upcall(ep);
break;
case ABORTING:
break;
case DEAD:
- PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __FUNCTION__);
+ PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__);
+ spin_unlock_irqrestore(&ep->com.lock, flags);
return CPL_RET_BUF_DONE;
default:
BUG_ON(1);
break;
}
dst_confirm(ep->dst);
+ if (ep->com.state != ABORTING) {
+ __state_set(&ep->com, DEAD);
+ release = 1;
+ }
+ spin_unlock_irqrestore(&ep->com.lock, flags);
rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL);
if (!rpl_skb) {
printk(KERN_ERR MOD "%s - cannot allocate skb!\n",
- __FUNCTION__);
- dst_release(ep->dst);
- l2t_release(L2DATA(ep->com.tdev), ep->l2t);
- put_ep(&ep->com);
- return CPL_RET_BUF_DONE;
+ __func__);
+ release = 1;
+ goto out;
}
rpl_skb->priority = CPL_PRIORITY_DATA;
rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl));
@@ -1571,11 +1631,10 @@ static int peer_abort(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
rpl->wr.wr_lo = htonl(V_WR_TID(ep->hwtid));
OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid));
rpl->cmd = CPL_ABORT_NO_RST;
- ep->com.tdev->send(ep->com.tdev, rpl_skb);
- if (state != ABORTING) {
- state_set(&ep->com, DEAD);
+ iwch_cxgb3_ofld_send(ep->com.tdev, rpl_skb);
+out:
+ if (release)
release_ep_resources(ep);
- }
return CPL_RET_BUF_DONE;
}
@@ -1586,7 +1645,7 @@ static int close_con_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
unsigned long flags;
int release = 0;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
BUG_ON(!ep);
/* The cm_id may be null if we failed to connect */
@@ -1609,8 +1668,8 @@ static int close_con_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
release = 1;
break;
case ABORTING:
- break;
case DEAD:
+ break;
default:
BUG_ON(1);
break;
@@ -1625,7 +1684,7 @@ static int close_con_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
* T3A does 3 things when a TERM is received:
* 1) send up a CPL_RDMA_TERMINATE message with the TERM packet
* 2) generate an async event on the QP with the TERMINATE opcode
- * 3) post a TERMINATE opcde cqe into the associated CQ.
+ * 3) post a TERMINATE opcode cqe into the associated CQ.
*
* For (1), we save the message in the qp for later consumer consumption.
* For (2), we move the QP into TERMINATE, post a QP event and disconnect.
@@ -1637,9 +1696,12 @@ static int terminate(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
{
struct iwch_ep *ep = ctx;
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ if (state_read(&ep->com) != FPDU_MODE)
+ return CPL_RET_BUF_DONE;
+
+ PDBG("%s ep %p\n", __func__, ep);
skb_pull(skb, sizeof(struct cpl_rdma_terminate));
- PDBG("%s saving %d bytes of term msg\n", __FUNCTION__, skb->len);
+ PDBG("%s saving %d bytes of term msg\n", __func__, skb->len);
skb_copy_from_linear_data(skb, ep->com.qp->attr.terminate_buffer,
skb->len);
ep->com.qp->attr.terminate_msg_len = skb->len;
@@ -1652,13 +1714,13 @@ static int ec_status(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
struct cpl_rdma_ec_status *rep = cplhdr(skb);
struct iwch_ep *ep = ctx;
- PDBG("%s ep %p tid %u status %d\n", __FUNCTION__, ep, ep->hwtid,
+ PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid,
rep->status);
if (rep->status) {
struct iwch_qp_attributes attrs;
printk(KERN_ERR MOD "%s BAD CLOSE - Aborting tid %u\n",
- __FUNCTION__, ep->hwtid);
+ __func__, ep->hwtid);
stop_ep_timer(ep);
attrs.next_state = IWCH_QP_STATE_ERROR;
iwch_modify_qp(ep->com.qp->rhp,
@@ -1674,15 +1736,18 @@ static void ep_timeout(unsigned long arg)
struct iwch_ep *ep = (struct iwch_ep *)arg;
struct iwch_qp_attributes attrs;
unsigned long flags;
+ int abort = 1;
spin_lock_irqsave(&ep->com.lock, flags);
- PDBG("%s ep %p tid %u state %d\n", __FUNCTION__, ep, ep->hwtid,
+ PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid,
ep->com.state);
switch (ep->com.state) {
case MPA_REQ_SENT:
+ __state_set(&ep->com, ABORTING);
connect_reply_upcall(ep, -ETIMEDOUT);
break;
case MPA_REQ_WAIT:
+ __state_set(&ep->com, ABORTING);
break;
case CLOSING:
case MORIBUND:
@@ -1692,13 +1757,16 @@ static void ep_timeout(unsigned long arg)
ep->com.qp, IWCH_QP_ATTR_NEXT_STATE,
&attrs, 1);
}
+ __state_set(&ep->com, ABORTING);
break;
default:
- BUG();
+ WARN(1, "%s unexpected state ep %p state %u\n",
+ __func__, ep, ep->com.state);
+ abort = 0;
}
- __state_set(&ep->com, CLOSING);
spin_unlock_irqrestore(&ep->com.lock, flags);
- abort_connection(ep, NULL, GFP_ATOMIC);
+ if (abort)
+ abort_connection(ep, NULL, GFP_ATOMIC);
put_ep(&ep->com);
}
@@ -1706,7 +1774,7 @@ int iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
{
int err;
struct iwch_ep *ep = to_ep(cm_id);
- PDBG("%s ep %p tid %u\n", __FUNCTION__, ep, ep->hwtid);
+ PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
if (state_read(&ep->com) == DEAD) {
put_ep(&ep->com);
@@ -1719,6 +1787,7 @@ int iwch_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
err = send_mpa_reject(ep, pdata, pdata_len);
err = iwch_ep_disconnect(ep, 0, GFP_KERNEL);
}
+ put_ep(&ep->com);
return 0;
}
@@ -1731,10 +1800,10 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
struct iwch_dev *h = to_iwch_dev(cm_id->device);
struct iwch_qp *qp = get_qhp(h, conn_param->qpn);
- PDBG("%s ep %p tid %u\n", __FUNCTION__, ep, ep->hwtid);
+ PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid);
if (state_read(&ep->com) == DEAD) {
- put_ep(&ep->com);
- return -ECONNRESET;
+ err = -ECONNRESET;
+ goto err;
}
BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD);
@@ -1743,33 +1812,25 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
if ((conn_param->ord > qp->rhp->attr.max_rdma_read_qp_depth) ||
(conn_param->ird > qp->rhp->attr.max_rdma_reads_per_qp)) {
abort_connection(ep, NULL, GFP_KERNEL);
- return -EINVAL;
+ err = -EINVAL;
+ goto err;
}
cm_id->add_ref(cm_id);
ep->com.cm_id = cm_id;
ep->com.qp = qp;
- ep->com.rpl_done = 0;
- ep->com.rpl_err = 0;
ep->ird = conn_param->ird;
ep->ord = conn_param->ord;
- PDBG("%s %d ird %d ord %d\n", __FUNCTION__, __LINE__, ep->ird, ep->ord);
- get_ep(&ep->com);
- err = send_mpa_reply(ep, conn_param->private_data,
- conn_param->private_data_len);
- if (err) {
- ep->com.cm_id = NULL;
- ep->com.qp = NULL;
- cm_id->rem_ref(cm_id);
- abort_connection(ep, NULL, GFP_KERNEL);
- put_ep(&ep->com);
- return err;
- }
+
+ if (peer2peer && ep->ird == 0)
+ ep->ird = 1;
+
+ PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord);
/* bind QP to EP and move to RTS */
attrs.mpa_attr = ep->mpa_attr;
- attrs.max_ird = ep->ord;
+ attrs.max_ird = ep->ird;
attrs.max_ord = ep->ord;
attrs.llp_stream_handle = ep;
attrs.next_state = IWCH_QP_STATE_RTS;
@@ -1783,30 +1844,70 @@ int iwch_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
err = iwch_modify_qp(ep->com.qp->rhp,
ep->com.qp, mask, &attrs, 1);
+ if (err)
+ goto err1;
- if (err) {
- ep->com.cm_id = NULL;
- ep->com.qp = NULL;
- cm_id->rem_ref(cm_id);
- abort_connection(ep, NULL, GFP_KERNEL);
- } else {
- state_set(&ep->com, FPDU_MODE);
- established_upcall(ep);
+ /* if needed, wait for wr_ack */
+ if (iwch_rqes_posted(qp)) {
+ wait_event(ep->com.waitq, ep->com.rpl_done);
+ err = ep->com.rpl_err;
+ if (err)
+ goto err1;
}
+
+ err = send_mpa_reply(ep, conn_param->private_data,
+ conn_param->private_data_len);
+ if (err)
+ goto err1;
+
+
+ state_set(&ep->com, FPDU_MODE);
+ established_upcall(ep);
+ put_ep(&ep->com);
+ return 0;
+err1:
+ ep->com.cm_id = NULL;
+ ep->com.qp = NULL;
+ cm_id->rem_ref(cm_id);
+err:
put_ep(&ep->com);
return err;
}
+static int is_loopback_dst(struct iw_cm_id *cm_id)
+{
+ struct net_device *dev;
+ struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr;
+
+ dev = ip_dev_find(&init_net, raddr->sin_addr.s_addr);
+ if (!dev)
+ return 0;
+ dev_put(dev);
+ return 1;
+}
+
int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
{
- int err = 0;
struct iwch_dev *h = to_iwch_dev(cm_id->device);
struct iwch_ep *ep;
struct rtable *rt;
+ int err = 0;
+ struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr;
+ struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr;
+
+ if (cm_id->remote_addr.ss_family != PF_INET) {
+ err = -ENOSYS;
+ goto out;
+ }
+
+ if (is_loopback_dst(cm_id)) {
+ err = -ENOSYS;
+ goto out;
+ }
ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
if (!ep) {
- printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__);
err = -ENOMEM;
goto out;
}
@@ -1817,13 +1918,17 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
conn_param->private_data, ep->plen);
ep->ird = conn_param->ird;
ep->ord = conn_param->ord;
+
+ if (peer2peer && ep->ord == 0)
+ ep->ord = 1;
+
ep->com.tdev = h->rdev.t3cdev_p;
cm_id->add_ref(cm_id);
ep->com.cm_id = cm_id;
ep->com.qp = get_qhp(h, conn_param->qpn);
BUG_ON(!ep->com.qp);
- PDBG("%s qpn 0x%x qp %p cm_id %p\n", __FUNCTION__, conn_param->qpn,
+ PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn,
ep->com.qp, cm_id);
/*
@@ -1831,49 +1936,48 @@ int iwch_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
*/
ep->atid = cxgb3_alloc_atid(h->rdev.t3cdev_p, &t3c_client, ep);
if (ep->atid == -1) {
- printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__);
err = -ENOMEM;
goto fail2;
}
/* find a route */
- rt = find_route(h->rdev.t3cdev_p,
- cm_id->local_addr.sin_addr.s_addr,
- cm_id->remote_addr.sin_addr.s_addr,
- cm_id->local_addr.sin_port,
- cm_id->remote_addr.sin_port, IPTOS_LOWDELAY);
+ rt = find_route(h->rdev.t3cdev_p, laddr->sin_addr.s_addr,
+ raddr->sin_addr.s_addr, laddr->sin_port,
+ raddr->sin_port, IPTOS_LOWDELAY);
if (!rt) {
- printk(KERN_ERR MOD "%s - cannot find route.\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - cannot find route.\n", __func__);
err = -EHOSTUNREACH;
goto fail3;
}
- ep->dst = &rt->u.dst;
-
- /* get a l2t entry */
- ep->l2t = t3_l2t_get(ep->com.tdev, ep->dst->neighbour,
- ep->dst->neighbour->dev);
+ ep->dst = &rt->dst;
+ ep->l2t = t3_l2t_get(ep->com.tdev, ep->dst, NULL,
+ &raddr->sin_addr.s_addr);
if (!ep->l2t) {
- printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__);
err = -ENOMEM;
goto fail4;
}
state_set(&ep->com, CONNECTING);
ep->tos = IPTOS_LOWDELAY;
- ep->com.local_addr = cm_id->local_addr;
- ep->com.remote_addr = cm_id->remote_addr;
+ memcpy(&ep->com.local_addr, &cm_id->local_addr,
+ sizeof(ep->com.local_addr));
+ memcpy(&ep->com.remote_addr, &cm_id->remote_addr,
+ sizeof(ep->com.remote_addr));
/* send connect request to rnic */
err = send_connect(ep);
if (!err)
goto out;
- l2t_release(L2DATA(h->rdev.t3cdev_p), ep->l2t);
+ l2t_release(h->rdev.t3cdev_p, ep->l2t);
fail4:
dst_release(ep->dst);
fail3:
cxgb3_free_atid(ep->com.tdev, ep->atid);
fail2:
+ cm_id->rem_ref(cm_id);
put_ep(&ep->com);
out:
return err;
@@ -1888,25 +1992,31 @@ int iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
might_sleep();
+ if (cm_id->local_addr.ss_family != PF_INET) {
+ err = -ENOSYS;
+ goto fail1;
+ }
+
ep = alloc_ep(sizeof(*ep), GFP_KERNEL);
if (!ep) {
- printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__);
err = -ENOMEM;
goto fail1;
}
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
ep->com.tdev = h->rdev.t3cdev_p;
cm_id->add_ref(cm_id);
ep->com.cm_id = cm_id;
ep->backlog = backlog;
- ep->com.local_addr = cm_id->local_addr;
+ memcpy(&ep->com.local_addr, &cm_id->local_addr,
+ sizeof(ep->com.local_addr));
/*
* Allocate a server TID.
*/
ep->stid = cxgb3_alloc_stid(h->rdev.t3cdev_p, &t3c_client, ep);
if (ep->stid == -1) {
- printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __FUNCTION__);
+ printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__);
err = -ENOMEM;
goto fail2;
}
@@ -1926,6 +2036,7 @@ int iwch_create_listen(struct iw_cm_id *cm_id, int backlog)
fail3:
cxgb3_free_stid(ep->com.tdev, ep->stid);
fail2:
+ cm_id->rem_ref(cm_id);
put_ep(&ep->com);
fail1:
out:
@@ -1937,15 +2048,18 @@ int iwch_destroy_listen(struct iw_cm_id *cm_id)
int err;
struct iwch_listen_ep *ep = to_listen_ep(cm_id);
- PDBG("%s ep %p\n", __FUNCTION__, ep);
+ PDBG("%s ep %p\n", __func__, ep);
might_sleep();
state_set(&ep->com, DEAD);
ep->com.rpl_done = 0;
ep->com.rpl_err = 0;
err = listen_stop(ep);
+ if (err)
+ goto done;
wait_event(ep->com.waitq, ep->com.rpl_done);
cxgb3_free_stid(ep->com.tdev, ep->stid);
+done:
err = ep->com.rpl_err;
cm_id->rem_ref(cm_id);
put_ep(&ep->com);
@@ -1957,53 +2071,69 @@ int iwch_ep_disconnect(struct iwch_ep *ep, int abrupt, gfp_t gfp)
int ret=0;
unsigned long flags;
int close = 0;
+ int fatal = 0;
+ struct t3cdev *tdev;
+ struct cxio_rdev *rdev;
spin_lock_irqsave(&ep->com.lock, flags);
- PDBG("%s ep %p state %s, abrupt %d\n", __FUNCTION__, ep,
+ PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep,
states[ep->com.state], abrupt);
- if (ep->com.state == DEAD) {
- PDBG("%s already dead ep %p\n", __FUNCTION__, ep);
- goto out;
- }
-
- if (abrupt) {
- if (ep->com.state != ABORTING) {
- ep->com.state = ABORTING;
- close = 1;
- }
- goto out;
+ tdev = (struct t3cdev *)ep->com.tdev;
+ rdev = (struct cxio_rdev *)tdev->ulp;
+ if (cxio_fatal_error(rdev)) {
+ fatal = 1;
+ close_complete_upcall(ep);
+ ep->com.state = DEAD;
}
-
switch (ep->com.state) {
case MPA_REQ_WAIT:
case MPA_REQ_SENT:
case MPA_REQ_RCVD:
case MPA_REP_SENT:
case FPDU_MODE:
- start_ep_timer(ep);
- ep->com.state = CLOSING;
close = 1;
+ if (abrupt)
+ ep->com.state = ABORTING;
+ else {
+ ep->com.state = CLOSING;
+ start_ep_timer(ep);
+ }
+ set_bit(CLOSE_SENT, &ep->com.flags);
break;
case CLOSING:
- ep->com.state = MORIBUND;
- close = 1;
+ if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) {
+ close = 1;
+ if (abrupt) {
+ stop_ep_timer(ep);
+ ep->com.state = ABORTING;
+ } else
+ ep->com.state = MORIBUND;
+ }
break;
case MORIBUND:
+ case ABORTING:
+ case DEAD:
+ PDBG("%s ignoring disconnect ep %p state %u\n",
+ __func__, ep, ep->com.state);
break;
default:
BUG();
break;
}
-out:
+
spin_unlock_irqrestore(&ep->com.lock, flags);
if (close) {
if (abrupt)
ret = send_abort(ep, NULL, gfp);
else
ret = send_halfclose(ep, gfp);
+ if (ret)
+ fatal = 1;
}
+ if (fatal)
+ release_ep_resources(ep);
return ret;
}
@@ -2015,10 +2145,10 @@ int iwch_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
if (ep->dst != old)
return 0;
- PDBG("%s ep %p redirect to dst %p l2t %p\n", __FUNCTION__, ep, new,
+ PDBG("%s ep %p redirect to dst %p l2t %p\n", __func__, ep, new,
l2t);
dst_hold(new);
- l2t_release(L2DATA(ep->com.tdev), ep->l2t);
+ l2t_release(ep->com.tdev, ep->l2t);
ep->l2t = l2t;
dst_release(old);
ep->dst = new;
@@ -2027,7 +2157,49 @@ int iwch_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
/*
* All the CM events are handled on a work queue to have a safe context.
+ * These are the real handlers that are called from the work queue.
*/
+static const cxgb3_cpl_handler_func work_handlers[NUM_CPL_CMDS] = {
+ [CPL_ACT_ESTABLISH] = act_establish,
+ [CPL_ACT_OPEN_RPL] = act_open_rpl,
+ [CPL_RX_DATA] = rx_data,
+ [CPL_TX_DMA_ACK] = tx_ack,
+ [CPL_ABORT_RPL_RSS] = abort_rpl,
+ [CPL_ABORT_RPL] = abort_rpl,
+ [CPL_PASS_OPEN_RPL] = pass_open_rpl,
+ [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl,
+ [CPL_PASS_ACCEPT_REQ] = pass_accept_req,
+ [CPL_PASS_ESTABLISH] = pass_establish,
+ [CPL_PEER_CLOSE] = peer_close,
+ [CPL_ABORT_REQ_RSS] = peer_abort,
+ [CPL_CLOSE_CON_RPL] = close_con_rpl,
+ [CPL_RDMA_TERMINATE] = terminate,
+ [CPL_RDMA_EC_STATUS] = ec_status,
+};
+
+static void process_work(struct work_struct *work)
+{
+ struct sk_buff *skb = NULL;
+ void *ep;
+ struct t3cdev *tdev;
+ int ret;
+
+ while ((skb = skb_dequeue(&rxq))) {
+ ep = *((void **) (skb->cb));
+ tdev = *((struct t3cdev **) (skb->cb + sizeof(void *)));
+ ret = work_handlers[G_OPCODE(ntohl((__force __be32)skb->csum))](tdev, skb, ep);
+ if (ret & CPL_RET_BUF_DONE)
+ kfree_skb(skb);
+
+ /*
+ * ep was referenced in sched(), and is freed here.
+ */
+ put_ep((struct iwch_ep_common *)ep);
+ }
+}
+
+static DECLARE_WORK(skb_work, process_work);
+
static int sched(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
{
struct iwch_ep_common *epc = ctx;
@@ -2059,6 +2231,29 @@ static int set_tcb_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx)
return CPL_RET_BUF_DONE;
}
+/*
+ * All upcalls from the T3 Core go to sched() to schedule the
+ * processing on a work queue.
+ */
+cxgb3_cpl_handler_func t3c_handlers[NUM_CPL_CMDS] = {
+ [CPL_ACT_ESTABLISH] = sched,
+ [CPL_ACT_OPEN_RPL] = sched,
+ [CPL_RX_DATA] = sched,
+ [CPL_TX_DMA_ACK] = sched,
+ [CPL_ABORT_RPL_RSS] = sched,
+ [CPL_ABORT_RPL] = sched,
+ [CPL_PASS_OPEN_RPL] = sched,
+ [CPL_CLOSE_LISTSRV_RPL] = sched,
+ [CPL_PASS_ACCEPT_REQ] = sched,
+ [CPL_PASS_ESTABLISH] = sched,
+ [CPL_PEER_CLOSE] = sched,
+ [CPL_CLOSE_CON_RPL] = sched,
+ [CPL_ABORT_REQ_RSS] = sched,
+ [CPL_RDMA_TERMINATE] = sched,
+ [CPL_RDMA_EC_STATUS] = sched,
+ [CPL_SET_TCB_RPL] = set_tcb_rpl,
+};
+
int __init iwch_cm_init(void)
{
skb_queue_head_init(&rxq);
@@ -2067,46 +2262,6 @@ int __init iwch_cm_init(void)
if (!workq)
return -ENOMEM;
- /*
- * All upcalls from the T3 Core go to sched() to
- * schedule the processing on a work queue.
- */
- t3c_handlers[CPL_ACT_ESTABLISH] = sched;
- t3c_handlers[CPL_ACT_OPEN_RPL] = sched;
- t3c_handlers[CPL_RX_DATA] = sched;
- t3c_handlers[CPL_TX_DMA_ACK] = sched;
- t3c_handlers[CPL_ABORT_RPL_RSS] = sched;
- t3c_handlers[CPL_ABORT_RPL] = sched;
- t3c_handlers[CPL_PASS_OPEN_RPL] = sched;
- t3c_handlers[CPL_CLOSE_LISTSRV_RPL] = sched;
- t3c_handlers[CPL_PASS_ACCEPT_REQ] = sched;
- t3c_handlers[CPL_PASS_ESTABLISH] = sched;
- t3c_handlers[CPL_PEER_CLOSE] = sched;
- t3c_handlers[CPL_CLOSE_CON_RPL] = sched;
- t3c_handlers[CPL_ABORT_REQ_RSS] = sched;
- t3c_handlers[CPL_RDMA_TERMINATE] = sched;
- t3c_handlers[CPL_RDMA_EC_STATUS] = sched;
- t3c_handlers[CPL_SET_TCB_RPL] = set_tcb_rpl;
-
- /*
- * These are the real handlers that are called from a
- * work queue.
- */
- work_handlers[CPL_ACT_ESTABLISH] = act_establish;
- work_handlers[CPL_ACT_OPEN_RPL] = act_open_rpl;
- work_handlers[CPL_RX_DATA] = rx_data;
- work_handlers[CPL_TX_DMA_ACK] = tx_ack;
- work_handlers[CPL_ABORT_RPL_RSS] = abort_rpl;
- work_handlers[CPL_ABORT_RPL] = abort_rpl;
- work_handlers[CPL_PASS_OPEN_RPL] = pass_open_rpl;
- work_handlers[CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl;
- work_handlers[CPL_PASS_ACCEPT_REQ] = pass_accept_req;
- work_handlers[CPL_PASS_ESTABLISH] = pass_establish;
- work_handlers[CPL_PEER_CLOSE] = peer_close;
- work_handlers[CPL_ABORT_REQ_RSS] = peer_abort;
- work_handlers[CPL_CLOSE_CON_RPL] = close_con_rpl;
- work_handlers[CPL_RDMA_TERMINATE] = terminate;
- work_handlers[CPL_RDMA_EC_STATUS] = ec_status;
return 0;
}
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-12-21 00:59:44 +0000