[IB] Fix MAD layer DMA mappings to avoid touching data buffer once mapped

The MAD layer was violating the DMA API by touching data buffers used
for sends after the DMA mapping was done.  This causes problems on
non-cache-coherent architectures, because the device doing DMA won't
see updates to the payload buffers that exist only in the CPU cache.

Fix this by having all MAD consumers use ib_create_send_mad() to
allocate their send buffers, and moving the DMA mapping into the MAD
layer so it can be done just before calling send (and after any
modifications of the send buffer by the MAD layer).

Tested on a non-cache-coherent PowerPC 440SPe system.

Signed-off-by: Sean Hefty <sean.hefty@intel.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

12 files changed, 441 insertions, 809 deletions

diff --git a/drivers/infiniband/core/agent.c b/drivers/infiniband/core/agent.c
index 5ac86f566dc..0c3c6952faa 100644
--- a/drivers/infiniband/core/agent.c
+++ b/drivers/infiniband/core/agent.c
@@ -37,58 +37,41 @@
  * $Id: agent.c 1389 2004-12-27 22:56:47Z roland $
  */
 
-#include <linux/dma-mapping.h>
-
-#include <asm/bug.h>
+#include "agent.h"
+#include "smi.h"
 
-#include <rdma/ib_smi.h>
+#define SPFX "ib_agent: "
 
-#include "smi.h"
-#include "agent_priv.h"
-#include "mad_priv.h"
-#include "agent.h"
+struct ib_agent_port_private {
+	struct list_head port_list;
+	struct ib_mad_agent *agent[2];
+};
 
-spinlock_t ib_agent_port_list_lock;
+static DEFINE_SPINLOCK(ib_agent_port_list_lock);
 static LIST_HEAD(ib_agent_port_list);
 
-/*
- * Caller must hold ib_agent_port_list_lock
- */
-static inline struct ib_agent_port_private *
-__ib_get_agent_port(struct ib_device *device, int port_num,
-		    struct ib_mad_agent *mad_agent)
+static struct ib_agent_port_private *
+__ib_get_agent_port(struct ib_device *device, int port_num)
 {
 	struct ib_agent_port_private *entry;
 
-	BUG_ON(!(!!device ^ !!mad_agent));  /* Exactly one MUST be (!NULL) */
-
-	if (device) {
-		list_for_each_entry(entry, &ib_agent_port_list, port_list) {
-			if (entry->smp_agent->device == device &&
-			    entry->port_num == port_num)
-				return entry;
-		}
-	} else {
-		list_for_each_entry(entry, &ib_agent_port_list, port_list) {
-			if ((entry->smp_agent == mad_agent) ||
-			    (entry->perf_mgmt_agent == mad_agent))
-				return entry;
-		}
+	list_for_each_entry(entry, &ib_agent_port_list, port_list) {
+		if (entry->agent[0]->device == device &&
+		    entry->agent[0]->port_num == port_num)
+			return entry;
 	}
 	return NULL;
 }
 
-static inline struct ib_agent_port_private *
-ib_get_agent_port(struct ib_device *device, int port_num,
-		  struct ib_mad_agent *mad_agent)
+static struct ib_agent_port_private *
+ib_get_agent_port(struct ib_device *device, int port_num)
 {
 	struct ib_agent_port_private *entry;
 	unsigned long flags;
 
 	spin_lock_irqsave(&ib_agent_port_list_lock, flags);
-	entry = __ib_get_agent_port(device, port_num, mad_agent);
+	entry = __ib_get_agent_port(device, port_num);
 	spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
-
 	return entry;
 }
 
@@ -100,192 +83,76 @@ int smi_check_local_dr_smp(struct ib_smp *smp,
 
 	if (smp->mgmt_class != IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
 		return 1;
-	port_priv = ib_get_agent_port(device, port_num, NULL);
+
+	port_priv = ib_get_agent_port(device, port_num);
 	if (!port_priv) {
 		printk(KERN_DEBUG SPFX "smi_check_local_dr_smp %s port %d "
-		       "not open\n",
-		       device->name, port_num);
+		       "not open\n", device->name, port_num);
 		return 1;
 	}
 
-	return smi_check_local_smp(port_priv->smp_agent, smp);
+	return smi_check_local_smp(port_priv->agent[0], smp);
 }
 
-static int agent_mad_send(struct ib_mad_agent *mad_agent,
-			  struct ib_agent_port_private *port_priv,
-			  struct ib_mad_private *mad_priv,
-			  struct ib_grh *grh,
-			  struct ib_wc *wc)
+int agent_send_response(struct ib_mad *mad, struct ib_grh *grh,
+			struct ib_wc *wc, struct ib_device *device,
+			int port_num, int qpn)
 {
-	struct ib_agent_send_wr *agent_send_wr;
-	struct ib_sge gather_list;
-	struct ib_send_wr send_wr;
-	struct ib_send_wr *bad_send_wr;
-	struct ib_ah_attr ah_attr;
-	unsigned long flags;
-	int ret = 1;
-
-	agent_send_wr = kmalloc(sizeof(*agent_send_wr), GFP_KERNEL);
-	if (!agent_send_wr)
-		goto out;
-	agent_send_wr->mad = mad_priv;
-
-	gather_list.addr = dma_map_single(mad_agent->device->dma_device,
-					  &mad_priv->mad,
-					  sizeof(mad_priv->mad),
-					  DMA_TO_DEVICE);
-	gather_list.length = sizeof(mad_priv->mad);
-	gather_list.lkey = mad_agent->mr->lkey;
-
-	send_wr.next = NULL;
-	send_wr.opcode = IB_WR_SEND;
-	send_wr.sg_list = &gather_list;
-	send_wr.num_sge = 1;
-	send_wr.wr.ud.remote_qpn = wc->src_qp; /* DQPN */
-	send_wr.wr.ud.timeout_ms = 0;
-	send_wr.send_flags = IB_SEND_SIGNALED | IB_SEND_SOLICITED;
+	struct ib_agent_port_private *port_priv;
+	struct ib_mad_agent *agent;
+	struct ib_mad_send_buf *send_buf;
+	struct ib_ah *ah;
+	int ret;
 
-	ah_attr.dlid = wc->slid;
-	ah_attr.port_num = mad_agent->port_num;
-	ah_attr.src_path_bits = wc->dlid_path_bits;
-	ah_attr.sl = wc->sl;
-	ah_attr.static_rate = 0;
-	ah_attr.ah_flags = 0; /* No GRH */
-	if (mad_priv->mad.mad.mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT) {
-		if (wc->wc_flags & IB_WC_GRH) {
-			ah_attr.ah_flags = IB_AH_GRH;
-			/* Should sgid be looked up ? */
-			ah_attr.grh.sgid_index = 0;
-			ah_attr.grh.hop_limit = grh->hop_limit;
-			ah_attr.grh.flow_label = be32_to_cpu(
-				grh->version_tclass_flow)  & 0xfffff;
-			ah_attr.grh.traffic_class = (be32_to_cpu(
-				grh->version_tclass_flow) >> 20) & 0xff;
-			memcpy(ah_attr.grh.dgid.raw,
-			       grh->sgid.raw,
-			       sizeof(ah_attr.grh.dgid));
-		}
+	port_priv = ib_get_agent_port(device, port_num);
+	if (!port_priv) {
+		printk(KERN_ERR SPFX "Unable to find port agent\n");
+		return -ENODEV;
 	}
 
-	agent_send_wr->ah = ib_create_ah(mad_agent->qp->pd, &ah_attr);
-	if (IS_ERR(agent_send_wr->ah)) {
-		printk(KERN_ERR SPFX "No memory for address handle\n");
-		kfree(agent_send_wr);
-		goto out;
+	agent = port_priv->agent[qpn];
+	ah = ib_create_ah_from_wc(agent->qp->pd, wc, grh, port_num);
+	if (IS_ERR(ah)) {
+		ret = PTR_ERR(ah);
+		printk(KERN_ERR SPFX "ib_create_ah_from_wc error:%d\n", ret);
+		return ret;
 	}
 
-	send_wr.wr.ud.ah = agent_send_wr->ah;
-	if (mad_priv->mad.mad.mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT) {
-		send_wr.wr.ud.pkey_index = wc->pkey_index;
-		send_wr.wr.ud.remote_qkey = IB_QP1_QKEY;
-	} else { 	/* for SMPs */
-		send_wr.wr.ud.pkey_index = 0;
-		send_wr.wr.ud.remote_qkey = 0;
+	send_buf = ib_create_send_mad(agent, wc->src_qp, wc->pkey_index, 0,
+				      IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
+				      GFP_KERNEL);
+	if (IS_ERR(send_buf)) {
+		ret = PTR_ERR(send_buf);
+		printk(KERN_ERR SPFX "ib_create_send_mad error:%d\n", ret);
+		goto err1;
 	}
-	send_wr.wr.ud.mad_hdr = &mad_priv->mad.mad.mad_hdr;
-	send_wr.wr_id = (unsigned long)agent_send_wr;
 
-	pci_unmap_addr_set(agent_send_wr, mapping, gather_list.addr);
-
-	/* Send */
-	spin_lock_irqsave(&port_priv->send_list_lock, flags);
-	if (ib_post_send_mad(mad_agent, &send_wr, &bad_send_wr)) {
-		spin_unlock_irqrestore(&port_priv->send_list_lock, flags);
-		dma_unmap_single(mad_agent->device->dma_device,
-				 pci_unmap_addr(agent_send_wr, mapping),
-				 sizeof(mad_priv->mad),
-				 DMA_TO_DEVICE);
-		ib_destroy_ah(agent_send_wr->ah);
-		kfree(agent_send_wr);
-	} else {
-		list_add_tail(&agent_send_wr->send_list,
-			      &port_priv->send_posted_list);
-		spin_unlock_irqrestore(&port_priv->send_list_lock, flags);
-		ret = 0;
+	memcpy(send_buf->mad, mad, sizeof *mad);
+	send_buf->ah = ah;
+	if ((ret = ib_post_send_mad(send_buf, NULL))) {
+		printk(KERN_ERR SPFX "ib_post_send_mad error:%d\n", ret);
+		goto err2;
 	}
-
-out:
+	return 0;
+err2:
+	ib_free_send_mad(send_buf);
+err1:
+	ib_destroy_ah(ah);
 	return ret;
 }
 
-int agent_send(struct ib_mad_private *mad,
-	       struct ib_grh *grh,
-	       struct ib_wc *wc,
-	       struct ib_device *device,
-	       int port_num)
-{
-	struct ib_agent_port_private *port_priv;
-	struct ib_mad_agent *mad_agent;
-
-	port_priv = ib_get_agent_port(device, port_num, NULL);
-	if (!port_priv) {
-		printk(KERN_DEBUG SPFX "agent_send %s port %d not open\n",
-		       device->name, port_num);
-		return 1;
-	}
-
-	/* Get mad agent based on mgmt_class in MAD */
-	switch (mad->mad.mad.mad_hdr.mgmt_class) {
-		case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE:
-		case IB_MGMT_CLASS_SUBN_LID_ROUTED:
-			mad_agent = port_priv->smp_agent;
-			break;
-		case IB_MGMT_CLASS_PERF_MGMT:
-			mad_agent = port_priv->perf_mgmt_agent;
-			break;
-		default:
-			return 1;
-	}
-
-	return agent_mad_send(mad_agent, port_priv, mad, grh, wc);
-}
-
 static void agent_send_handler(struct ib_mad_agent *mad_agent,
 			       struct ib_mad_send_wc *mad_send_wc)
 {
-	struct ib_agent_port_private	*port_priv;
-	struct ib_agent_send_wr		*agent_send_wr;
-	unsigned long			flags;
-
-	/* Find matching MAD agent */
-	port_priv = ib_get_agent_port(NULL, 0, mad_agent);
-	if (!port_priv) {
-		printk(KERN_ERR SPFX "agent_send_handler: no matching MAD "
-		       "agent %p\n", mad_agent);
-		return;
-	}
-
-	agent_send_wr = (struct ib_agent_send_wr *)(unsigned long)mad_send_wc->wr_id;
-	spin_lock_irqsave(&port_priv->send_list_lock, flags);
-	/* Remove completed send from posted send MAD list */
-	list_del(&agent_send_wr->send_list);
-	spin_unlock_irqrestore(&port_priv->send_list_lock, flags);
-
-	dma_unmap_single(mad_agent->device->dma_device,
-			 pci_unmap_addr(agent_send_wr, mapping),
-			 sizeof(agent_send_wr->mad->mad),
-			 DMA_TO_DEVICE);
-
-	ib_destroy_ah(agent_send_wr->ah);
-
-	/* Release allocated memory */
-	kmem_cache_free(ib_mad_cache, agent_send_wr->mad);
-	kfree(agent_send_wr);
+	ib_destroy_ah(mad_send_wc->send_buf->ah);
+	ib_free_send_mad(mad_send_wc->send_buf);
 }
 
 int ib_agent_port_open(struct ib_device *device, int port_num)
 {
-	int ret;
 	struct ib_agent_port_private *port_priv;
 	unsigned long flags;
-
-	/* First, check if port already open for SMI */
-	port_priv = ib_get_agent_port(device, port_num, NULL);
-	if (port_priv) {
-		printk(KERN_DEBUG SPFX "%s port %d already open\n",
-		       device->name, port_num);
-		return 0;
-	}
+	int ret;
 
 	/* Create new device info */
 	port_priv = kmalloc(sizeof *port_priv, GFP_KERNEL);
@@ -294,32 +161,25 @@ int ib_agent_port_open(struct ib_device *device, int port_num)
 		ret = -ENOMEM;
 		goto error1;
 	}
-
 	memset(port_priv, 0, sizeof *port_priv);
-	port_priv->port_num = port_num;
-	spin_lock_init(&port_priv->send_list_lock);
-	INIT_LIST_HEAD(&port_priv->send_posted_list);
 
-	/* Obtain send only MAD agent for SM class (SMI QP) */
-	port_priv->smp_agent = ib_register_mad_agent(device, port_num,
-						     IB_QPT_SMI,
-						     NULL, 0,
+	/* Obtain send only MAD agent for SMI QP */
+	port_priv->agent[0] = ib_register_mad_agent(device, port_num,
+						    IB_QPT_SMI, NULL, 0,
 						    &agent_send_handler,
-						     NULL, NULL);
-
-	if (IS_ERR(port_priv->smp_agent)) {
-		ret = PTR_ERR(port_priv->smp_agent);
+						    NULL, NULL);
+	if (IS_ERR(port_priv->agent[0])) {
+		ret = PTR_ERR(port_priv->agent[0]);
 		goto error2;
 	}
 
-	/* Obtain send only MAD agent for PerfMgmt class (GSI QP) */
-	port_priv->perf_mgmt_agent = ib_register_mad_agent(device, port_num,
-							   IB_QPT_GSI,
-							   NULL, 0,
-							  &agent_send_handler,
-							   NULL, NULL);
-	if (IS_ERR(port_priv->perf_mgmt_agent)) {
-		ret = PTR_ERR(port_priv->perf_mgmt_agent);
+	/* Obtain send only MAD agent for GSI QP */
+	port_priv->agent[1] = ib_register_mad_agent(device, port_num,
+						    IB_QPT_GSI, NULL, 0,
+						    &agent_send_handler,
+						    NULL, NULL);
+	if (IS_ERR(port_priv->agent[1])) {
+		ret = PTR_ERR(port_priv->agent[1]);
 		goto error3;
 	}
 
@@ -330,7 +190,7 @@ int ib_agent_port_open(struct ib_device *device, int port_num)
 	return 0;
 
 error3:
-	ib_unregister_mad_agent(port_priv->smp_agent);
+	ib_unregister_mad_agent(port_priv->agent[0]);
 error2:
 	kfree(port_priv);
 error1:
@@ -343,7 +203,7 @@ int ib_agent_port_close(struct ib_device *device, int port_num)
 	unsigned long flags;
 
 	spin_lock_irqsave(&ib_agent_port_list_lock, flags);
-	port_priv = __ib_get_agent_port(device, port_num, NULL);
+	port_priv = __ib_get_agent_port(device, port_num);
 	if (port_priv == NULL) {
 		spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
 		printk(KERN_ERR SPFX "Port %d not found\n", port_num);
@@ -352,9 +212,8 @@ int ib_agent_port_close(struct ib_device *device, int port_num)
 	list_del(&port_priv->port_list);
 	spin_unlock_irqrestore(&ib_agent_port_list_lock, flags);
 
-	ib_unregister_mad_agent(port_priv->perf_mgmt_agent);
-	ib_unregister_mad_agent(port_priv->smp_agent);
+	ib_unregister_mad_agent(port_priv->agent[1]);
+	ib_unregister_mad_agent(port_priv->agent[0]);
 	kfree(port_priv);
-
 	return 0;
 }
diff --git a/drivers/infiniband/core/agent.h b/drivers/infiniband/core/agent.h
index d9426842254..c5f3cfec942 100644
--- a/drivers/infiniband/core/agent.h
+++ b/drivers/infiniband/core/agent.h
@@ -39,17 +39,14 @@
 #ifndef __AGENT_H_
 #define __AGENT_H_
 
-extern spinlock_t ib_agent_port_list_lock;
+#include <rdma/ib_mad.h>
 
-extern int ib_agent_port_open(struct ib_device *device,
-			      int port_num);
+extern int ib_agent_port_open(struct ib_device *device, int port_num);
 
 extern int ib_agent_port_close(struct ib_device *device, int port_num);
 
-extern int agent_send(struct ib_mad_private *mad,
-		      struct ib_grh *grh,
-		      struct ib_wc *wc,
-		      struct ib_device *device,
-		      int port_num);
+extern int agent_send_response(struct ib_mad *mad, struct ib_grh *grh,
+			       struct ib_wc *wc, struct ib_device *device,
+			       int port_num, int qpn);
 
 #endif	/* __AGENT_H_ */
diff --git a/drivers/infiniband/core/agent_priv.h b/drivers/infiniband/core/agent_priv.h
deleted file mode 100644
index 2ec6d7f1b7d..00000000000
--- a/drivers/infiniband/core/agent_priv.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright (c) 2004, 2005 Mellanox Technologies Ltd.  All rights reserved.
- * Copyright (c) 2004, 2005 Infinicon Corporation.  All rights reserved.
- * Copyright (c) 2004, 2005 Intel Corporation.  All rights reserved.
- * Copyright (c) 2004, 2005 Topspin Corporation.  All rights reserved.
- * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- * $Id: agent_priv.h 1640 2005-01-24 22:39:02Z halr $
- */
-
-#ifndef __IB_AGENT_PRIV_H__
-#define __IB_AGENT_PRIV_H__
-
-#include <linux/pci.h>
-
-#define SPFX "ib_agent: "
-
-struct ib_agent_send_wr {
-	struct list_head send_list;
-	struct ib_ah *ah;
-	struct ib_mad_private *mad;
-	DECLARE_PCI_UNMAP_ADDR(mapping)
-};
-
-struct ib_agent_port_private {
-	struct list_head port_list;
-	struct list_head send_posted_list;
-	spinlock_t send_list_lock;
-	int port_num;
-	struct ib_mad_agent *smp_agent;	      /* SM class */
-	struct ib_mad_agent *perf_mgmt_agent; /* PerfMgmt class */
-};
-
-#endif	/* __IB_AGENT_PRIV_H__ */
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 389fecbaf66..580c3a2bb10 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -176,8 +176,7 @@ static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
 
 	m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn, 
 			       cm_id_priv->av.pkey_index,
-			       ah, 0, sizeof(struct ib_mad_hdr),
-			       sizeof(struct ib_mad)-sizeof(struct ib_mad_hdr),
+			       0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 			       GFP_ATOMIC);
 	if (IS_ERR(m)) {
 		ib_destroy_ah(ah);
@@ -185,7 +184,8 @@ static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
 	}
 
 	/* Timeout set by caller if response is expected. */
-	m->send_wr.wr.ud.retries = cm_id_priv->max_cm_retries;
+	m->ah = ah;
+	m->retries = cm_id_priv->max_cm_retries;
 
 	atomic_inc(&cm_id_priv->refcount);
 	m->context[0] = cm_id_priv;
@@ -206,20 +206,20 @@ static int cm_alloc_response_msg(struct cm_port *port,
 		return PTR_ERR(ah);
 
 	m = ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
-			       ah, 0, sizeof(struct ib_mad_hdr),
-			       sizeof(struct ib_mad)-sizeof(struct ib_mad_hdr),
+			       0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 			       GFP_ATOMIC);
 	if (IS_ERR(m)) {
 		ib_destroy_ah(ah);
 		return PTR_ERR(m);
 	}
+	m->ah = ah;
 	*msg = m;
 	return 0;
 }
 
 static void cm_free_msg(struct ib_mad_send_buf *msg)
 {
-	ib_destroy_ah(msg->send_wr.wr.ud.ah);
+	ib_destroy_ah(msg->ah);
 	if (msg->context[0])
 		cm_deref_id(msg->context[0]);
 	ib_free_send_mad(msg);
@@ -678,8 +678,7 @@ retest:
 		break;
 	case IB_CM_SIDR_REQ_SENT:
 		cm_id->state = IB_CM_IDLE;
-		ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-			      (unsigned long) cm_id_priv->msg);
+		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		break;
 	case IB_CM_SIDR_REQ_RCVD:
@@ -690,8 +689,7 @@ retest:
 	case IB_CM_MRA_REQ_RCVD:
 	case IB_CM_REP_SENT:
 	case IB_CM_MRA_REP_RCVD:
-		ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-			      (unsigned long) cm_id_priv->msg);
+		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 		/* Fall through */
 	case IB_CM_REQ_RCVD:
 	case IB_CM_MRA_REQ_SENT:
@@ -708,8 +706,7 @@ retest:
 		ib_send_cm_dreq(cm_id, NULL, 0);
 		goto retest;
 	case IB_CM_DREQ_SENT:
-		ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-			      (unsigned long) cm_id_priv->msg);
+		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 		cm_enter_timewait(cm_id_priv);
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		break;
@@ -883,7 +880,6 @@ int ib_send_cm_req(struct ib_cm_id *cm_id,
 		   struct ib_cm_req_param *param)
 {
 	struct cm_id_private *cm_id_priv;
-	struct ib_send_wr *bad_send_wr;
 	struct cm_req_msg *req_msg;
 	unsigned long flags;
 	int ret;
@@ -936,7 +932,7 @@ int ib_send_cm_req(struct ib_cm_id *cm_id,
 	req_msg = (struct cm_req_msg *) cm_id_priv->msg->mad;
 	cm_format_req(req_msg, cm_id_priv, param);
 	cm_id_priv->tid = req_msg->hdr.tid;
-	cm_id_priv->msg->send_wr.wr.ud.timeout_ms = cm_id_priv->timeout_ms;
+	cm_id_priv->msg->timeout_ms = cm_id_priv->timeout_ms;
 	cm_id_priv->msg->context[1] = (void *) (unsigned long) IB_CM_REQ_SENT;
 
 	cm_id_priv->local_qpn = cm_req_get_local_qpn(req_msg);
@@ -945,8 +941,7 @@ int ib_send_cm_req(struct ib_cm_id *cm_id,
 				cm_req_get_primary_local_ack_timeout(req_msg);
 
 	spin_lock_irqsave(&cm_id_priv->lock, flags);
-	ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-				&cm_id_priv->msg->send_wr, &bad_send_wr);
+	ret = ib_post_send_mad(cm_id_priv->msg, NULL);
 	if (ret) {
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		goto error2;
@@ -969,7 +964,6 @@ static int cm_issue_rej(struct cm_port *port,
 			void *ari, u8 ari_length)
 {
 	struct ib_mad_send_buf *msg = NULL;
-	struct ib_send_wr *bad_send_wr;
 	struct cm_rej_msg *rej_msg, *rcv_msg;
 	int ret;
 
@@ -992,7 +986,7 @@ static int cm_issue_rej(struct cm_port *port,
 		memcpy(rej_msg->ari, ari, ari_length);
 	}
 
-	ret = ib_post_send_mad(port->mad_agent, &msg->send_wr, &bad_send_wr);
+	ret = ib_post_send_mad(msg, NULL);
 	if (ret)
 		cm_free_msg(msg);
 
@@ -1172,7 +1166,6 @@ static void cm_dup_req_handler(struct cm_work *work,
 			       struct cm_id_private *cm_id_priv)
 {
 	struct ib_mad_send_buf *msg = NULL;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	int ret;
 
@@ -1201,8 +1194,7 @@ static void cm_dup_req_handler(struct cm_work *work,
 	}
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-	ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent, &msg->send_wr,
-			       &bad_send_wr);
+	ret = ib_post_send_mad(msg, NULL);
 	if (ret)
 		goto free;
 	return;
@@ -1367,7 +1359,6 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id,
 	struct cm_id_private *cm_id_priv;
 	struct ib_mad_send_buf *msg;
 	struct cm_rep_msg *rep_msg;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	int ret;
 
@@ -1389,11 +1380,10 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id,
 
 	rep_msg = (struct cm_rep_msg *) msg->mad;
 	cm_format_rep(rep_msg, cm_id_priv, param);
-	msg->send_wr.wr.ud.timeout_ms = cm_id_priv->timeout_ms;
+	msg->timeout_ms = cm_id_priv->timeout_ms;
 	msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
 
-	ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-			       &msg->send_wr, &bad_send_wr);
+	ret = ib_post_send_mad(msg, NULL);
 	if (ret) {
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		cm_free_msg(msg);
@@ -1431,7 +1421,6 @@ int ib_send_cm_rtu(struct ib_cm_id *cm_id,
 {
 	struct cm_id_private *cm_id_priv;
 	struct ib_mad_send_buf *msg;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	void *data;
 	int ret;
@@ -1458,8 +1447,7 @@ int ib_send_cm_rtu(struct ib_cm_id *cm_id,
 	cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
 		      private_data, private_data_len);
 
-	ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-			       &msg->send_wr, &bad_send_wr);
+	ret = ib_post_send_mad(msg, NULL);
 	if (ret) {
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		cm_free_msg(msg);
@@ -1504,7 +1492,6 @@ static void cm_dup_rep_handler(struct cm_work *work)
 	struct cm_id_private *cm_id_priv;
 	struct cm_rep_msg *rep_msg;
 	struct ib_mad_send_buf *msg = NULL;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	int ret;
 
@@ -1532,8 +1519,7 @@ static void cm_dup_rep_handler(struct cm_work *work)
 		goto unlock;
 	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-	ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent, &msg->send_wr,
-			       &bad_send_wr);
+	ret = ib_post_send_mad(msg, NULL);
 	if (ret)
 		goto free;
 	goto deref;
@@ -1601,8 +1587,7 @@ static int cm_rep_handler(struct cm_work *work)
 
 	/* todo: handle peer_to_peer */
 
-	ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-		      (unsigned long) cm_id_priv->msg);
+	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 	ret = atomic_inc_and_test(&cm_id_priv->work_count);
 	if (!ret)
 		list_add_tail(&work->list, &cm_id_priv->work_list);
@@ -1636,8 +1621,7 @@ static int cm_establish_handler(struct cm_work *work)
 		goto out;
 	}
 
-	ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-		      (unsigned long) cm_id_priv->msg);
+	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 	ret = atomic_inc_and_test(&cm_id_priv->work_count);
 	if (!ret)
 		list_add_tail(&work->list, &cm_id_priv->work_list);
@@ -1676,8 +1660,7 @@ static int cm_rtu_handler(struct cm_work *work)
 	}
 	cm_id_priv->id.state = IB_CM_ESTABLISHED;
 
-	ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-		      (unsigned long) cm_id_priv->msg);
+	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 	ret = atomic_inc_and_test(&cm_id_priv->work_count);
 	if (!ret)
 		list_add_tail(&work->list, &cm_id_priv->work_list);
@@ -1714,7 +1697,6 @@ int ib_send_cm_dreq(struct ib_cm_id *cm_id,
 {
 	struct cm_id_private *cm_id_priv;
 	struct ib_mad_send_buf *msg;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	int ret;
 
@@ -1736,11 +1718,10 @@ int ib_send_cm_dreq(struct ib_cm_id *cm_id,
 
 	cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
 		       private_data, private_data_len);
-	msg->send_wr.wr.ud.timeout_ms = cm_id_priv->timeout_ms;
+	msg->timeout_ms = cm_id_priv->timeout_ms;
 	msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
 
-	ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-			       &msg->send_wr, &bad_send_wr);
+	ret = ib_post_send_mad(msg, NULL);
 	if (ret) {
 		cm_enter_timewait(cm_id_priv);
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
@@ -1774,7 +1755,6 @@ int ib_send_cm_drep(struct ib_cm_id *cm_id,
 {
 	struct cm_id_private *cm_id_priv;
 	struct ib_mad_send_buf *msg;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	void *data;
 	int ret;
@@ -1804,8 +1784,7 @@ int ib_send_cm_drep(struct ib_cm_id *cm_id,
 	cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
 		       private_data, private_data_len);
 
-	ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent, &msg->send_wr,
-			       &bad_send_wr);
+	ret = ib_post_send_mad(msg, NULL);
 	if (ret) {
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		cm_free_msg(msg);
@@ -1822,7 +1801,6 @@ static int cm_dreq_handler(struct cm_work *work)
 	struct cm_id_private *cm_id_priv;
 	struct cm_dreq_msg *dreq_msg;
 	struct ib_mad_send_buf *msg = NULL;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	int ret;
 
@@ -1841,8 +1819,7 @@ static int cm_dreq_handler(struct cm_work *work)
 	switch (cm_id_priv->id.state) {
 	case IB_CM_REP_SENT:
 	case IB_CM_DREQ_SENT:
-		ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-			      (unsigned long) cm_id_priv->msg);
+		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 		break;
 	case IB_CM_ESTABLISHED:
 	case IB_CM_MRA_REP_RCVD:
@@ -1856,8 +1833,7 @@ static int cm_dreq_handler(struct cm_work *work)
 			       cm_id_priv->private_data_len);
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-		if (ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-				     &msg->send_wr, &bad_send_wr))
+		if (ib_post_send_mad(msg, NULL))
 			cm_free_msg(msg);
 		goto deref;
 	default:
@@ -1904,8 +1880,7 @@ static int cm_drep_handler(struct cm_work *work)
 	}
 	cm_enter_timewait(cm_id_priv);
 
-	ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-		      (unsigned long) cm_id_priv->msg);
+	ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 	ret = atomic_inc_and_test(&cm_id_priv->work_count);
 	if (!ret)
 		list_add_tail(&work->list, &cm_id_priv->work_list);
@@ -1930,7 +1905,6 @@ int ib_send_cm_rej(struct ib_cm_id *cm_id,
 {
 	struct cm_id_private *cm_id_priv;
 	struct ib_mad_send_buf *msg;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	int ret;
 
@@ -1974,8 +1948,7 @@ int ib_send_cm_rej(struct ib_cm_id *cm_id,
 	if (ret)
 		goto out;
 
-	ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-			       &msg->send_wr, &bad_send_wr);
+	ret = ib_post_send_mad(msg, NULL);
 	if (ret)
 		cm_free_msg(msg);
 
@@ -2051,8 +2024,7 @@ static int cm_rej_handler(struct cm_work *work)
 	case IB_CM_MRA_REQ_RCVD:
 	case IB_CM_REP_SENT:
 	case IB_CM_MRA_REP_RCVD:
-		ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-			      (unsigned long) cm_id_priv->msg);
+		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 		/* fall through */
 	case IB_CM_REQ_RCVD:
 	case IB_CM_MRA_REQ_SENT:
@@ -2062,8 +2034,7 @@ static int cm_rej_handler(struct cm_work *work)
 			cm_reset_to_idle(cm_id_priv);
 		break;
 	case IB_CM_DREQ_SENT:
-		ib_cancel_mad(cm_id_priv->av.port->mad_agent,
-			      (unsigned long) cm_id_priv->msg);
+		ib_cancel_mad(cm_id_priv->av.port->mad_agent, cm_id_priv->msg);
 		/* fall through */
 	case IB_CM_REP_RCVD:
 	case IB_CM_MRA_REP_SENT:
@@ -2098,7 +2069,6 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id,
 {
 	struct cm_id_private *cm_id_priv;
 	struct ib_mad_send_buf *msg;
-	struct ib_send_wr *bad_send_wr;
 	void *data;
 	unsigned long flags;
 	int ret;
@@ -2122,8 +2092,7 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id,
 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
 			      CM_MSG_RESPONSE_REQ, service_timeout,
 			      private_data, private_data_len);
-		ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-				       &msg->send_wr, &bad_send_wr);
+		ret = ib_post_send_mad(msg, NULL);
 		if (ret)
 			goto error2;
 		cm_id->state = IB_CM_MRA_REQ_SENT;
@@ -2136,8 +2105,7 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id,
 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
 			      CM_MSG_RESPONSE_REP, service_timeout,
 			      private_data, private_data_len);
-		ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-				       &msg->send_wr, &bad_send_wr);
+		ret = ib_post_send_mad(msg, NULL);
 		if (ret)
 			goto error2;
 		cm_id->state = IB_CM_MRA_REP_SENT;
@@ -2150,8 +2118,7 @@ int ib_send_cm_mra(struct ib_cm_id *cm_id,
 		cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
 			      CM_MSG_RESPONSE_OTHER, service_timeout,
 			      private_data, private_data_len);
-		ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-				       &msg->send_wr, &bad_send_wr);
+		ret = ib_post_send_mad(msg, NULL);
 		if (ret)
 			goto error2;
 		cm_id->lap_state = IB_CM_MRA_LAP_SENT;
@@ -2213,14 +2180,14 @@ static int cm_mra_handler(struct cm_work *work)
 	case IB_CM_REQ_SENT:
 		if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REQ ||
 		    ib_modify_mad(cm_id_priv->av.port->mad_agent,
-				  (unsigned long) cm_id_priv->msg, timeout))
+				  cm_id_priv->msg, timeout))
 			goto out;
 		cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
 		break;
 	case IB_CM_REP_SENT:
 		if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_REP ||
 		    ib_modify_mad(cm_id_priv->av.port->mad_agent,
-				  (unsigned long) cm_id_priv->msg, timeout))
+				  cm_id_priv->msg, timeout))
 			goto out;
 		cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
 		break;
@@ -2228,7 +2195,7 @@ static int cm_mra_handler(struct cm_work *work)
 		if (cm_mra_get_msg_mraed(mra_msg) != CM_MSG_RESPONSE_OTHER ||
 		    cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
 		    ib_modify_mad(cm_id_priv->av.port->mad_agent,
-				  (unsigned long) cm_id_priv->msg, timeout))
+				  cm_id_priv->msg, timeout))
 			goto out;
 		cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
 		break;
@@ -2291,7 +2258,6 @@ int ib_send_cm_lap(struct ib_cm_id *cm_id,
 {
 	struct cm_id_private *cm_id_priv;
 	struct ib_mad_send_buf *msg;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	int ret;
 
@@ -2312,11 +2278,10 @@ int ib_send_cm_lap(struct ib_cm_id *cm_id,
 
 	cm_format_lap((struct cm_lap_msg *) msg->mad, cm_id_priv,
 		      alternate_path, private_data, private_data_len);
-	msg->send_wr.wr.ud.timeout_ms = cm_id_priv->timeout_ms;
+	msg->timeout_ms = cm_id_priv->timeout_ms;
 	msg->context[1] = (void *) (unsigned long) IB_CM_ESTABLISHED;
 
-	ret = ib_post_send_mad(cm_id_priv->av.port->mad_agent,
-			       &msg->send_wr, &bad_send_wr);
+	ret = ib_post_send_mad(msg, NULL);
 	if (ret) {
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 		cm_free_msg(msg);
@@ -2360,7 +2325,6 @@ static int cm_lap_handler(struct cm_work *work)
 	struct cm_lap_msg *lap_msg;
 	struct ib_cm_lap_event_param *param;
 	struct ib_mad_send_buf *msg = NULL;
-	struct ib_send_wr *bad_send_wr;
 	unsigned long flags;
 	int ret;
 
@@ -2394,8 +2358,7 @@ static int cm_lap_handler(struct cm_work *work)
 			      cm_id_priv->private_data_len);
 		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
 
-		if (ib_post_send_mad(cm_id_priv