Merge /scratch/Ksrc/linux-git/

1 files changed, 343 insertions, 257 deletions

diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index 23628c622a5..b97e210ce9c 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -1,5 +1,7 @@
 /*
  * Copyright (c) 2004, 2005 Voltaire, Inc. All rights reserved.
+ * Copyright (c) 2005 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -29,12 +31,12 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  *
- * $Id: mad.c 1389 2004-12-27 22:56:47Z roland $
+ * $Id: mad.c 2817 2005-07-07 11:29:26Z halr $
  */
-
 #include <linux/dma-mapping.h>
 
 #include "mad_priv.h"
+#include "mad_rmpp.h"
 #include "smi.h"
 #include "agent.h"
 
@@ -45,6 +47,7 @@ MODULE_AUTHOR("Sean Hefty");
 
 
 kmem_cache_t *ib_mad_cache;
+
 static struct list_head ib_mad_port_list;
 static u32 ib_mad_client_id = 0;
 
@@ -58,16 +61,12 @@ static int method_in_use(struct ib_mad_mgmt_method_table **method,
 static void remove_mad_reg_req(struct ib_mad_agent_private *priv);
 static struct ib_mad_agent_private *find_mad_agent(
 					struct ib_mad_port_private *port_priv,
-					struct ib_mad *mad, int solicited);
+					struct ib_mad *mad);
 static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
 				    struct ib_mad_private *mad);
 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv);
-static void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
-				    struct ib_mad_send_wc *mad_send_wc);
 static void timeout_sends(void *data);
-static void cancel_sends(void *data);
 static void local_completions(void *data);
-static int solicited_mad(struct ib_mad *mad);
 static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
 			      struct ib_mad_agent_private *agent_priv,
 			      u8 mgmt_class);
@@ -197,8 +196,8 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
 	if (qpn == -1)
 		goto error1;
 
-	if (rmpp_version)
-		goto error1;	/* XXX: until RMPP implemented */
+	if (rmpp_version && rmpp_version != IB_MGMT_RMPP_VERSION)
+		goto error1;
 
 	/* Validate MAD registration request if supplied */
 	if (mad_reg_req) {
@@ -261,22 +260,29 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
 		ret = ERR_PTR(-ENOMEM);
 		goto error1;
 	}
+	memset(mad_agent_priv, 0, sizeof *mad_agent_priv);
+
+	mad_agent_priv->agent.mr = ib_get_dma_mr(port_priv->qp_info[qpn].qp->pd,
+						 IB_ACCESS_LOCAL_WRITE);
+	if (IS_ERR(mad_agent_priv->agent.mr)) {
+		ret = ERR_PTR(-ENOMEM);
+		goto error2;
+	}
 
 	if (mad_reg_req) {
 		reg_req = kmalloc(sizeof *reg_req, GFP_KERNEL);
 		if (!reg_req) {
 			ret = ERR_PTR(-ENOMEM);
-			goto error2;
+			goto error3;
 		}
 		/* Make a copy of the MAD registration request */
 		memcpy(reg_req, mad_reg_req, sizeof *reg_req);
 	}
 
 	/* Now, fill in the various structures */
-	memset(mad_agent_priv, 0, sizeof *mad_agent_priv);
 	mad_agent_priv->qp_info = &port_priv->qp_info[qpn];
 	mad_agent_priv->reg_req = reg_req;
-	mad_agent_priv->rmpp_version = rmpp_version;
+	mad_agent_priv->agent.rmpp_version = rmpp_version;
 	mad_agent_priv->agent.device = device;
 	mad_agent_priv->agent.recv_handler = recv_handler;
 	mad_agent_priv->agent.send_handler = send_handler;
@@ -301,7 +307,7 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
 				if (method) {
 					if (method_in_use(&method,
 							   mad_reg_req))
-						goto error3;
+						goto error4;
 				}
 			}
 			ret2 = add_nonoui_reg_req(mad_reg_req, mad_agent_priv,
@@ -317,14 +323,14 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
 					if (is_vendor_method_in_use(
 							vendor_class,
 							mad_reg_req))
-						goto error3;
+						goto error4;
 				}
 			}
 			ret2 = add_oui_reg_req(mad_reg_req, mad_agent_priv);
 		}
 		if (ret2) {
 			ret = ERR_PTR(ret2);
-			goto error3;
+			goto error4;
 		}
 	}
 
@@ -335,22 +341,24 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
 	spin_lock_init(&mad_agent_priv->lock);
 	INIT_LIST_HEAD(&mad_agent_priv->send_list);
 	INIT_LIST_HEAD(&mad_agent_priv->wait_list);
+	INIT_LIST_HEAD(&mad_agent_priv->done_list);
+	INIT_LIST_HEAD(&mad_agent_priv->rmpp_list);
 	INIT_WORK(&mad_agent_priv->timed_work, timeout_sends, mad_agent_priv);
 	INIT_LIST_HEAD(&mad_agent_priv->local_list);
 	INIT_WORK(&mad_agent_priv->local_work, local_completions,
 		   mad_agent_priv);
-	INIT_LIST_HEAD(&mad_agent_priv->canceled_list);
-	INIT_WORK(&mad_agent_priv->canceled_work, cancel_sends, mad_agent_priv);
 	atomic_set(&mad_agent_priv->refcount, 1);
 	init_waitqueue_head(&mad_agent_priv->wait);
 
 	return &mad_agent_priv->agent;
 
-error3:
+error4:
 	spin_unlock_irqrestore(&port_priv->reg_lock, flags);
 	kfree(reg_req);
-error2:
+error3:
 	kfree(mad_agent_priv);
+error2:
+	ib_dereg_mr(mad_agent_priv->agent.mr);
 error1:
 	return ret;
 }
@@ -487,18 +495,16 @@ static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 	 * MADs, preventing us from queuing additional work
 	 */
 	cancel_mads(mad_agent_priv);
-
 	port_priv = mad_agent_priv->qp_info->port_priv;
-
 	cancel_delayed_work(&mad_agent_priv->timed_work);
-	flush_workqueue(port_priv->wq);
 
 	spin_lock_irqsave(&port_priv->reg_lock, flags);
 	remove_mad_reg_req(mad_agent_priv);
 	list_del(&mad_agent_priv->agent_list);
 	spin_unlock_irqrestore(&port_priv->reg_lock, flags);
 
-	/* XXX: Cleanup pending RMPP receives for this agent */
+	flush_workqueue(port_priv->wq);
+	ib_cancel_rmpp_recvs(mad_agent_priv);
 
 	atomic_dec(&mad_agent_priv->refcount);
 	wait_event(mad_agent_priv->wait,
@@ -506,6 +512,7 @@ static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 
 	if (mad_agent_priv->reg_req)
 		kfree(mad_agent_priv->reg_req);
+	ib_dereg_mr(mad_agent_priv->agent.mr);
 	kfree(mad_agent_priv);
 }
 
@@ -551,6 +558,13 @@ int ib_unregister_mad_agent(struct ib_mad_agent *mad_agent)
 }
 EXPORT_SYMBOL(ib_unregister_mad_agent);
 
+static inline int response_mad(struct ib_mad *mad)
+{
+	/* Trap represses are responses although response bit is reset */
+	return ((mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) ||
+		(mad->mad_hdr.method & IB_MGMT_METHOD_RESP));
+}
+
 static void dequeue_mad(struct ib_mad_list_head *mad_list)
 {
 	struct ib_mad_queue *mad_queue;
@@ -643,7 +657,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 				  struct ib_smp *smp,
 				  struct ib_send_wr *send_wr)
 {
-	int ret, solicited;
+	int ret;
 	unsigned long flags;
 	struct ib_mad_local_private *local;
 	struct ib_mad_private *mad_priv;
@@ -689,11 +703,7 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	switch (ret)
 	{
 	case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY:
-		/*
-		 * See if response is solicited and
-		 * there is a recv handler
-		 */
-		if (solicited_mad(&mad_priv->mad.mad) &&
+		if (response_mad(&mad_priv->mad.mad) &&
 		    mad_agent_priv->agent.recv_handler) {
 			local->mad_priv = mad_priv;
 			local->recv_mad_agent = mad_agent_priv;
@@ -710,15 +720,13 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 		break;
 	case IB_MAD_RESULT_SUCCESS:
 		/* Treat like an incoming receive MAD */
-		solicited = solicited_mad(&mad_priv->mad.mad);
 		port_priv = ib_get_mad_port(mad_agent_priv->agent.device,
 					    mad_agent_priv->agent.port_num);
 		if (port_priv) {
 			mad_priv->mad.mad.mad_hdr.tid =
 				((struct ib_mad *)smp)->mad_hdr.tid;
 			recv_mad_agent = find_mad_agent(port_priv,
-						       &mad_priv->mad.mad,
-							solicited);
+						        &mad_priv->mad.mad);
 		}
 		if (!port_priv || !recv_mad_agent) {
 			kmem_cache_free(ib_mad_cache, mad_priv);
@@ -750,43 +758,133 @@ static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 	list_add_tail(&local->completion_list, &mad_agent_priv->local_list);
 	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 	queue_work(mad_agent_priv->qp_info->port_priv->wq,
-		  &mad_agent_priv->local_work);
+		   &mad_agent_priv->local_work);
 	ret = 1;
 out:
 	return ret;
 }
 
-static int ib_send_mad(struct ib_mad_agent_private *mad_agent_priv,
-		       struct ib_mad_send_wr_private *mad_send_wr)
+static int get_buf_length(int hdr_len, int data_len)
+{
+	int seg_size, pad;
+
+	seg_size = sizeof(struct ib_mad) - hdr_len;
+	if (data_len && seg_size) {
+		pad = seg_size - data_len % seg_size;
+		if (pad == seg_size)
+			pad = 0;
+	} else
+		pad = seg_size;
+	return hdr_len + data_len + pad;
+}
+
+struct ib_mad_send_buf * ib_create_send_mad(struct ib_mad_agent *mad_agent,
+					    u32 remote_qpn, u16 pkey_index,
+					    struct ib_ah *ah, int rmpp_active,
+					    int hdr_len, int data_len,
+					    unsigned int __nocast gfp_mask)
+{
+	struct ib_mad_agent_private *mad_agent_priv;
+	struct ib_mad_send_buf *send_buf;
+	int buf_size;
+	void *buf;
+
+	mad_agent_priv = container_of(mad_agent,
+				      struct ib_mad_agent_private, agent);
+	buf_size = get_buf_length(hdr_len, data_len);
+
+	if ((!mad_agent->rmpp_version &&
+	     (rmpp_active || buf_size > sizeof(struct ib_mad))) ||
+	    (!rmpp_active && buf_size > sizeof(struct ib_mad)))
+		return ERR_PTR(-EINVAL);
+
+	buf = kmalloc(sizeof *send_buf + buf_size, gfp_mask);
+	if (!buf)
+		return ERR_PTR(-ENOMEM);
+	memset(buf, 0, sizeof *send_buf + buf_size);
+
+	send_buf = buf + buf_size;
+	send_buf->mad = buf;
+
+	send_buf->sge.addr = dma_map_single(mad_agent->device->dma_device,
+					    buf, buf_size, DMA_TO_DEVICE);
+	pci_unmap_addr_set(send_buf, mapping, send_buf->sge.addr);
+	send_buf->sge.length = buf_size;
+	send_buf->sge.lkey = mad_agent->mr->lkey;
+
+	send_buf->send_wr.wr_id = (unsigned long) send_buf;
+	send_buf->send_wr.sg_list = &send_buf->sge;
+	send_buf->send_wr.num_sge = 1;
+	send_buf->send_wr.opcode = IB_WR_SEND;
+	send_buf->send_wr.send_flags = IB_SEND_SIGNALED;
+	send_buf->send_wr.wr.ud.ah = ah;
+	send_buf->send_wr.wr.ud.mad_hdr = &send_buf->mad->mad_hdr;
+	send_buf->send_wr.wr.ud.remote_qpn = remote_qpn;
+	send_buf->send_wr.wr.ud.remote_qkey = IB_QP_SET_QKEY;
+	send_buf->send_wr.wr.ud.pkey_index = pkey_index;
+
+	if (rmpp_active) {
+		struct ib_rmpp_mad *rmpp_mad;
+		rmpp_mad = (struct ib_rmpp_mad *)send_buf->mad;
+		rmpp_mad->rmpp_hdr.paylen_newwin = cpu_to_be32(hdr_len -
+			offsetof(struct ib_rmpp_mad, data) + data_len);
+		rmpp_mad->rmpp_hdr.rmpp_version = mad_agent->rmpp_version;
+		rmpp_mad->rmpp_hdr.rmpp_type = IB_MGMT_RMPP_TYPE_DATA;
+		ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr,
+				  IB_MGMT_RMPP_FLAG_ACTIVE);
+	}
+
+	send_buf->mad_agent = mad_agent;
+	atomic_inc(&mad_agent_priv->refcount);
+	return send_buf;
+}
+EXPORT_SYMBOL(ib_create_send_mad);
+
+void ib_free_send_mad(struct ib_mad_send_buf *send_buf)
+{
+	struct ib_mad_agent_private *mad_agent_priv;
+
+	mad_agent_priv = container_of(send_buf->mad_agent,
+				      struct ib_mad_agent_private, agent);
+
+	dma_unmap_single(send_buf->mad_agent->device->dma_device,
+			 pci_unmap_addr(send_buf, mapping),
+			 send_buf->sge.length, DMA_TO_DEVICE);
+	kfree(send_buf->mad);
+
+	if (atomic_dec_and_test(&mad_agent_priv->refcount))
+		wake_up(&mad_agent_priv->wait);
+}
+EXPORT_SYMBOL(ib_free_send_mad);
+
+int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)
 {
 	struct ib_mad_qp_info *qp_info;
 	struct ib_send_wr *bad_send_wr;
+	struct list_head *list;
 	unsigned long flags;
 	int ret;
 
-	/* Replace user's WR ID with our own to find WR upon completion */
-	qp_info = mad_agent_priv->qp_info;
-	mad_send_wr->wr_id = mad_send_wr->send_wr.wr_id;
+	/* Set WR ID to find mad_send_wr upon completion */
+	qp_info = mad_send_wr->mad_agent_priv->qp_info;
 	mad_send_wr->send_wr.wr_id = (unsigned long)&mad_send_wr->mad_list;
 	mad_send_wr->mad_list.mad_queue = &qp_info->send_queue;
 
 	spin_lock_irqsave(&qp_info->send_queue.lock, flags);
-	if (qp_info->send_queue.count++ < qp_info->send_queue.max_active) {
-		list_add_tail(&mad_send_wr->mad_list.list,
-			      &qp_info->send_queue.list);
-		spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
-		ret = ib_post_send(mad_agent_priv->agent.qp,
+	if (qp_info->send_queue.count < qp_info->send_queue.max_active) {
+		ret = ib_post_send(mad_send_wr->mad_agent_priv->agent.qp,
 				   &mad_send_wr->send_wr, &bad_send_wr);
-		if (ret) {
-			printk(KERN_ERR PFX "ib_post_send failed: %d\n", ret);
-			dequeue_mad(&mad_send_wr->mad_list);
-		}
+		list = &qp_info->send_queue.list;
 	} else {
-		list_add_tail(&mad_send_wr->mad_list.list,
-			      &qp_info->overflow_list);
-		spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
 		ret = 0;
+		list = &qp_info->overflow_list;
 	}
+
+	if (!ret) {
+		qp_info->send_queue.count++;
+		list_add_tail(&mad_send_wr->mad_list.list, list);
+	}
+	spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
 	return ret;
 }
 
@@ -860,18 +958,19 @@ int ib_post_send_mad(struct ib_mad_agent *mad_agent,
 			ret = -ENOMEM;
 			goto error2;
 		}
+		memset(mad_send_wr, 0, sizeof *mad_send_wr);
 
 		mad_send_wr->send_wr = *send_wr;
 		mad_send_wr->send_wr.sg_list = mad_send_wr->sg_list;
 		memcpy(mad_send_wr->sg_list, send_wr->sg_list,
 		       sizeof *send_wr->sg_list * send_wr->num_sge);
-		mad_send_wr->send_wr.next = NULL;
+		mad_send_wr->wr_id = send_wr->wr_id;
 		mad_send_wr->tid = send_wr->wr.ud.mad_hdr->tid;
-		mad_send_wr->agent = mad_agent;
+		mad_send_wr->mad_agent_priv = mad_agent_priv;
 		/* Timeout will be updated after send completes */
 		mad_send_wr->timeout = msecs_to_jiffies(send_wr->wr.
 							ud.timeout_ms);
-		mad_send_wr->retry = 0;
+		mad_send_wr->retries = mad_send_wr->send_wr.wr.ud.retries;
 		/* One reference for each work request to QP + response */
 		mad_send_wr->refcount = 1 + (mad_send_wr->timeout > 0);
 		mad_send_wr->status = IB_WC_SUCCESS;
@@ -883,8 +982,13 @@ int ib_post_send_mad(struct ib_mad_agent *mad_agent,
 			      &mad_agent_priv->send_list);
 		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 
-		ret = ib_send_mad(mad_agent_priv, mad_send_wr);
-		if (ret) {
+		if (mad_agent_priv->agent.rmpp_version) {
+			ret = ib_send_rmpp_mad(mad_send_wr);
+			if (ret >= 0 && ret != IB_RMPP_RESULT_CONSUMED)
+				ret = ib_send_mad(mad_send_wr);
+		} else
+			ret = ib_send_mad(mad_send_wr);
+		if (ret < 0) {
 			/* Fail send request */
 			spin_lock_irqsave(&mad_agent_priv->lock, flags);
 			list_del(&mad_send_wr->agent_list);
@@ -910,41 +1014,28 @@ EXPORT_SYMBOL(ib_post_send_mad);
  */
 void ib_free_recv_mad(struct ib_mad_recv_wc *mad_recv_wc)
 {
-	struct ib_mad_recv_buf *entry;
+	struct ib_mad_recv_buf *mad_recv_buf, *temp_recv_buf;
 	struct ib_mad_private_header *mad_priv_hdr;
 	struct ib_mad_private *priv;
+	struct list_head free_list;
 
-	mad_priv_hdr = container_of(mad_recv_wc,
-				    struct ib_mad_private_header,
-				    recv_wc);
-	priv = container_of(mad_priv_hdr, struct ib_mad_private, header);
+	INIT_LIST_HEAD(&free_list);
+	list_splice_init(&mad_recv_wc->rmpp_list, &free_list);
 
-	/*
-	 * Walk receive buffer list associated with this WC
-	 * No need to remove them from list of receive buffers
-	 */
-	list_for_each_entry(entry, &mad_recv_wc->recv_buf.list, list) {
-		/* Free previous receive buffer */
-		kmem_cache_free(ib_mad_cache, priv);
+	list_for_each_entry_safe(mad_recv_buf, temp_recv_buf,
+					&free_list, list) {
+		mad_recv_wc = container_of(mad_recv_buf, struct ib_mad_recv_wc,
+					   recv_buf);
 		mad_priv_hdr = container_of(mad_recv_wc,
 					    struct ib_mad_private_header,
 					    recv_wc);
 		priv = container_of(mad_priv_hdr, struct ib_mad_private,
 				    header);
+		kmem_cache_free(ib_mad_cache, priv);
 	}
-
-	/* Free last buffer */
-	kmem_cache_free(ib_mad_cache, priv);
 }
 EXPORT_SYMBOL(ib_free_recv_mad);
 
-void ib_coalesce_recv_mad(struct ib_mad_recv_wc *mad_recv_wc,
-			  void *buf)
-{
-	printk(KERN_ERR PFX "ib_coalesce_recv_mad() not implemented yet\n");
-}
-EXPORT_SYMBOL(ib_coalesce_recv_mad);
-
 struct ib_mad_agent *ib_redirect_mad_qp(struct ib_qp *qp,
 					u8 rmpp_version,
 					ib_mad_send_handler send_handler,
@@ -1338,42 +1429,15 @@ out:
 	return;
 }
 
-static int response_mad(struct ib_mad *mad)
-{
-	/* Trap represses are responses although response bit is reset */
-	return ((mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) ||
-		(mad->mad_hdr.method & IB_MGMT_METHOD_RESP));
-}
-
-static int solicited_mad(struct ib_mad *mad)
-{
-	/* CM MADs are never solicited */
-	if (mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_CM) {
-		return 0;
-	}
-
-	/* XXX: Determine whether MAD is using RMPP */
-
-	/* Not using RMPP */
-	/* Is this MAD a response to a previous MAD ? */
-	return response_mad(mad);
-}
-
 static struct ib_mad_agent_private *
 find_mad_agent(struct ib_mad_port_private *port_priv,
-	       struct ib_mad *mad,
-	       int solicited)
+	       struct ib_mad *mad)
 {
 	struct ib_mad_agent_private *mad_agent = NULL;
 	unsigned long flags;
 
 	spin_lock_irqsave(&port_priv->reg_lock, flags);
-
-	/*
-	 * Whether MAD was solicited determines type of routing to
-	 * MAD client.
-	 */
-	if (solicited) {
+	if (response_mad(mad)) {
 		u32 hi_tid;
 		struct ib_mad_agent_private *entry;
 
@@ -1477,21 +1541,20 @@ out:
 	return valid;
 }
 
-/*
- * Return start of fully reassembled MAD, or NULL, if MAD isn't assembled yet
- */
-static struct ib_mad_private *
-reassemble_recv(struct ib_mad_agent_private *mad_agent_priv,
-		struct ib_mad_private *recv)
+static int is_data_mad(struct ib_mad_agent_private *mad_agent_priv,
+		       struct ib_mad_hdr *mad_hdr)
 {
-	/* Until we have RMPP, all receives are reassembled!... */
-	INIT_LIST_HEAD(&recv->header.recv_wc.recv_buf.list);
-	return recv;
+	struct ib_rmpp_mad *rmpp_mad;
+
+	rmpp_mad = (struct ib_rmpp_mad *)mad_hdr;
+	return !mad_agent_priv->agent.rmpp_version ||
+		!(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
+				    IB_MGMT_RMPP_FLAG_ACTIVE) ||
+		(rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA);
 }
 
-static struct ib_mad_send_wr_private*
-find_send_req(struct ib_mad_agent_private *mad_agent_priv,
-	      u64 tid)
+struct ib_mad_send_wr_private*
+ib_find_send_mad(struct ib_mad_agent_private *mad_agent_priv, u64 tid)
 {
 	struct ib_mad_send_wr_private *mad_send_wr;
 
@@ -1507,7 +1570,9 @@ find_send_req(struct ib_mad_agent_private *mad_agent_priv,
 	 */
 	list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,
 			    agent_list) {
-		if (mad_send_wr->tid == tid && mad_send_wr->timeout) {
+		if (is_data_mad(mad_agent_priv,
+				mad_send_wr->send_wr.wr.ud.mad_hdr) &&
+		    mad_send_wr->tid == tid && mad_send_wr->timeout) {
 			/* Verify request has not been canceled */
 			return (mad_send_wr->status == IB_WC_SUCCESS) ?
 				mad_send_wr : NULL;
@@ -1516,43 +1581,55 @@ find_send_req(struct ib_mad_agent_private *mad_agent_priv,
 	return NULL;
 }
 
+void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)
+{
+	mad_send_wr->timeout = 0;
+	if (mad_send_wr->refcount == 1) {
+		list_del(&mad_send_wr->agent_list);
+		list_add_tail(&mad_send_wr->agent_list,
+			      &mad_send_wr->mad_agent_priv->done_list);
+	}
+}
+
 static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
-				 struct ib_mad_private *recv,
-				 int solicited)
+				 struct ib_mad_recv_wc *mad_recv_wc)
 {
 	struct ib_mad_send_wr_private *mad_send_wr;
 	struct ib_mad_send_wc mad_send_wc;
 	unsigned long flags;
-
-	/* Fully reassemble receive before processing */
-	recv = reassemble_recv(mad_agent_priv, recv);
-	if (!recv) {
-		if (atomic_dec_and_test(&mad_agent_priv->refcount))
-			wake_up(&mad_agent_priv->wait);
-		return;
+	u64 tid;
+
+	INIT_LIST_HEAD(&mad_recv_wc->rmpp_list);
+	list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list);
+	if (mad_agent_priv->agent.rmpp_version) {
+		mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv,
+						      mad_recv_wc);
+		if (!mad_recv_wc) {
+			if (atomic_dec_and_test(&mad_agent_priv->refcount))
+				wake_up(&mad_agent_priv->wait);
+			return;
+		}
 	}
 
 	/* Complete corresponding request */
-	if (solicited) {
+	if (response_mad(mad_recv_wc->recv_buf.mad)) {
+		tid = mad_recv_wc->recv_buf.mad->mad_hdr.tid;
 		spin_lock_irqsave(&mad_agent_priv->lock, flags);
-		mad_send_wr = find_send_req(mad_agent_priv,
-					    recv->mad.mad.mad_hdr.tid);
+		mad_send_wr = ib_find_send_mad(mad_agent_priv, tid);
 		if (!mad_send_wr) {
 			spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
-			ib_free_recv_mad(&recv->header.recv_wc);
+			ib_free_recv_mad(mad_recv_wc);
 			if (atomic_dec_and_test(&mad_agent_priv->refcount))
 				wake_up(&mad_agent_priv->wait);
 			return;
 		}
-		/* Timeout = 0 means that we won't wait for a response */
-		mad_send_wr->timeout = 0;
+		ib_mark_mad_done(mad_send_wr);
 		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 
 		/* Defined behavior is to complete response before request */
-		recv->header.recv_wc.wc->wr_id = mad_send_wr->wr_id;
-		mad_agent_priv->agent.recv_handler(
-						&mad_agent_priv->agent,
-						&recv->header.recv_wc);
+		mad_recv_wc->wc->wr_id = mad_send_wr->wr_id;
+		mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent,
+						   mad_recv_wc);
 		atomic_dec(&mad_agent_priv->refcount);
 
 		mad_send_wc.status = IB_WC_SUCCESS;
@@ -1560,9 +1637,8 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
 		mad_send_wc.wr_id = mad_send_wr->wr_id;
 		ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
 	} else {
-		mad_agent_priv->agent.recv_handler(
-						&mad_agent_priv->agent,
-						&recv->header.recv_wc);
+		mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent,
+						   mad_recv_wc);
 		if (atomic_dec_and_test(&mad_agent_priv->refcount))
 			wake_up(&mad_agent_priv->wait);
 	}
@@ -1576,7 +1652,6 @@ static void ib_mad_recv_done_handler(struct ib_mad_port_private *port_priv,
 	struct ib_mad_private *recv, *response;
 	struct ib_mad_list_head *mad_list;
 	struct ib_mad_agent_private *mad_agent;
-	int solicited;
 
 	response = kmem_cache_alloc(ib_mad_cache, GFP_KERNEL);
 	if (!response)
@@ -1662,11 +1737,9 @@ local:
 		}
 	}
 
-	/* Determine corresponding MAD agent for incoming receive MAD */
-	solicited = solicited_mad(&recv->mad.mad);
-	mad_agent = find_mad_agent(port_priv, &recv->mad.mad, solicited);
+	mad_agent = find_mad_agent(port_priv, &recv->mad.mad);
 	if (mad_agent) {
-		ib_mad_complete_recv(mad_agent, recv, solicited);
+		ib_mad_complete_recv(mad_agent, &recv->header.recv_wc);
 		/*
 		 * recv is freed up in error cases in ib_mad_complete_recv
 		 * or via recv_handler in ib_mad_complete_recv()
@@ -1710,26 +1783,31 @@ static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)
 	}
 }
 
-static void wait_for_response(struct ib_mad_agent_private *mad_agent_priv,
-			      struct ib_mad_send_wr_private *mad_send_wr )
+static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr)
 {
+	struct ib_mad_agent_private *mad_agent_priv;
 	struct ib_mad_send_wr_private *temp_mad_send_wr;
 	struct list_head *list_item;
 	unsigned long delay;
 
+	mad_agent_priv = mad_send_wr->mad_agent_priv;
 	list_del(&mad_send_wr->agent_list);
 
 	delay = mad_send_wr->timeout;
 	mad_send_wr->timeout += jiffies;
 
-	list_for_each_prev(list_item, &mad_agent_priv->wait_list) {
-		temp_mad_send_wr = list_entry(list_item,
-					      struct ib_mad_send_wr_private,
-					      agent_list);
-		if (time_after(mad_send_wr->timeout,
-			       temp_mad_send_wr->timeout))
-			break;
+	if (delay) {
+		list_for_each_prev(list_item, &mad_agent_priv->wait_list) {
+			temp_mad_send_wr = list_entry(list_item,
+						struct ib_mad_send_wr_private,
+						agent_list);
+			if (time_after(mad_send_wr->timeout,
+				       temp_mad_send_wr->timeout))
+				break;
+		}
 	}
+	else
+		list_item = &mad_agent_priv->wait_list;
 	list_add(&mad_send_wr->agent_list, list_item);
 
 	/* Reschedule a work item if we have a shorter timeout */
@@ -1740,19 +1818,32 @@ static void wait_for_response(struct ib_mad_agent_private *mad_agent_priv,
 	}
 }
 
+void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr,
+			  int timeout_ms)
+{
+	mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
+	wait_for_response(mad_send_wr);
+}
+
 /*
  * Process a send work completion
  */
-static void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
-				    struct ib_mad_send_wc *mad_send_wc)
+void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
+			     struct ib_mad_send_wc *mad_send_wc)
 {
 	struct ib_mad_agent_private	*mad_agent_priv;
 	unsigned long			flags;
+	int				ret;
 
-	mad_agent_priv = container_of(mad_send_wr->agent,
-				      struct ib_mad_agent_private, agent);
-
+	mad_agent_priv = mad_send_wr->mad_agent_priv;
 	spin_lock_irqsave(&mad_agent_priv->lock, flags);
+	if (mad_agent_priv->agent.rmpp_version) {
+		ret = ib_process_rmpp_send_wc(mad_send_wr, mad_send_wc);
+		if (ret == IB_RMPP_RESULT_CONSUMED)
+			goto done;
+	} else
+		ret = IB_RMPP_RESULT_UNHANDLED;
+
 	if (mad_send_wc->status != IB_WC_SUCCESS &&
 	    mad_send_wr->status == IB_WC_SUCCESS) {
 		mad_send_wr->status = mad_send_wc->status;
@@ -1762,10 +1853,9 @@ static void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
 	if (--mad_send_wr->refcount > 0) {
 		if (mad_send_wr->refcount == 1 && mad_send_wr->timeout &&
 		    mad_send_wr->status == IB_WC_SUCCESS) {
-			wait_for_response(mad_agent_priv, mad_send_wr);
+			wait_for_response(mad_send_wr);
 		}
-		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
-		return;
+		goto done;
 	}
 
 	/* Remove send from MAD agent and notify client of completion */
@@ -1775,14 +1865,18 @@ static void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
 
 	if (mad_send_wr->status != IB_WC_SUCCESS )
 		mad_send_wc->status = mad_send_wr->status;
-	mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
-					    mad_send_wc);
+	if (ret != IB_RMPP_RESULT_INTERNAL)
+		mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
+						   mad_send_wc);
 
 	/* Release reference on agent taken when sending */
 	if (atomic_dec_and_test(&mad_agent_priv->refcount))
 		wake_up(&mad_agent_priv->wait);
 
 	kfree(mad_send_wr);
+	return;
+done:
+	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 }
 
 static void ib_mad_send_done_handler(struct ib_mad_port_private *port_priv,
@@ -1961,6 +2055,8 @@ static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
 
 	/* Empty wait list to prevent receives from finding a request */
 	list_splice_init(&mad_agent_priv->wait_list, &cancel_list);
+	/* Empty local completion list as well */
+	list_splice_init(&mad_agent_priv->local_list, &cancel_list);
 	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 
 	/* Report all cancelled requests */
@@ -1980,8 +2076,7 @@ static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
 }
 
 static struct ib_mad_send_wr_private*
-find_send_by_wr_id(struct ib_mad_agent_private *mad_agent_priv,
-		   u64 wr_id)
+find_send_by_wr_id(struct ib_mad_agent_private *mad_agent_priv, u64 wr_id)
 {
 	struct ib_mad_send_wr_private *mad_send_wr;
 
@@ -1993,79 +2088,50 @@ find_send_by_wr_id(struct ib_mad_agent_private *mad_agent_priv,
 
 	list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,
 			    agent_list) {
-		if (mad_send_wr->wr_id == wr_id)
+		if (is_data_mad(mad_agent_priv,
+				mad_send_wr->send_wr.wr.ud.mad_hdr) &&
+		    mad_send_wr->wr_id == wr_id)
 			return mad_send_wr;
 	}
 	return NULL;
 }
 
-void cancel_sends(void *data)
-{
-	struct ib_mad_agent_private *mad_agent_priv;
-	struct ib_mad_send_wr_private *mad_send_wr;
-	struct ib_mad_send_wc mad_send_wc;
-	unsigned long flags;
-
-	mad_agent_priv = data;
-
-	mad_send_wc.status = IB_WC_WR_FLUSH_ERR;
-	mad_send_wc.vendor_err = 0;
-
-	spin_lock_irqsave(&mad_agent_priv->lock, flags);
-	while (!list_empty(&mad_agent_priv->canceled_list)) {
-		mad_send_wr = list_entry(mad_agent_priv->canceled_list.next,
-					 struct ib_mad_send_wr_private,
-					 agent_list);
-
-		list_del(&mad_send_wr->agent_list);
-		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
-
-		mad_send_wc.wr_id = mad_send_wr->wr_id;
-		mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
-						   &mad_send_wc);
-
-		kfree(mad_send_wr);
-		if (atomic_dec_and_test(&mad_agent_priv->refcount))
-			wake_up(&mad_agent_priv->wait);
-		spin_lock_irqsave(&mad_agent_priv->lock, flags);
-	}
-	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
-}
-
-void ib_cancel_mad(struct ib_mad_agent *mad_agent,
-		  u64 wr_id)
+int ib_modify_mad(struct ib_mad_agent *mad_agent, u64 wr_id, u32 timeout_ms)
 {
 	struct ib_mad_agent_private *mad_agent_priv;
 	struct ib_mad_send_wr_private *mad_send_wr;
 	unsigned long flags;
+	int active;
 
 	mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private,
 				      agent);
 	spin_lock_irqsave(&mad_agent_priv->lock, flags);
 	mad_send_wr = find_send_by_wr_id(mad_agent_priv, wr_id);
-	if (!mad_send_wr) {
+	if (!mad_send_wr || mad_send_wr->status != IB_WC_SUCCESS) {
 		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
-		goto out;
+		return -EINVAL;
 	}
 
-	if (mad_send_wr->status == IB_WC_SUCCESS)
-		mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
-
-	if (mad_send_wr->refcount != 0) {
+	active = (!mad_send_wr->timeout || mad_send_wr->refcount > 1);
+	if (!timeout_ms) {
 		mad_send_wr->status = IB_WC_WR_FLUSH_ERR;
-		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
-		goto out;
+		mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
 	}
 
-	list_del(&mad_send_wr->agent_list);
-	list_add_tail(&mad_send_wr->agent_list, &mad_agent_priv->canceled_list);
-	adjust_timeout(mad_agent_priv);
+	mad_send_wr->send_wr.wr.ud.timeout_ms = timeout_ms;
+	if (active)
+		mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
+	else
+		ib_reset_mad_timeout(mad_send_wr, timeout_ms);
+
 	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
+	return 0;
+}
+EXPORT_SYMBOL(ib_modify_mad);
 
-	queue_work(mad_agent_priv->qp_info->port_priv->wq,
-		   &mad_agent_priv->canceled_work);
-out:
-	return;
+void ib_cancel_mad(struct ib_mad_agent *mad_agent, u64 wr_id)
+{
+	ib_modify_mad(mad_agent, wr_id, 0);
 }
 EXPORT_SYMBOL(ib_cancel_mad);
 
@@ -2075,6 +2141,7 @@ static void local_completions(void *data)
 	struct ib_mad_local_private *local;
 	struct ib_mad_agent_private *recv_mad_agent;
 	unsigned long flags;
+	int recv = 0;
 	struct ib_wc wc;
 	struct ib_mad_send_wc mad_send_wc;
 
@@ -2090,10 +2157,10 @@ static void local_completions(void *data)
 			recv_mad_agent = local->recv_mad_agent;
 			if (!recv_mad_agent) {
 				printk(KERN_ERR PFX "No receive MAD agent for local completion\n");
-				kmem_cache_free(ib_mad_cache, local->mad_priv);
 				goto local_send_completion;
 			}
 
+			recv = 1;
 			/*
 			 * Defined behavior is to complete response
 			 * before request
@@ -2105,7 +2172,9 @@ static void local_completions(void *data)
 			local->mad_priv->header.recv_wc.wc = &wc;
 			local->mad_priv->header.recv_wc.mad_len =
 						sizeof(struct ib_mad);
-			INIT_LIST_HEAD(&local->mad_priv->header.recv_wc.recv_buf.list);
+			INIT_LIST_HEAD(&local->mad_priv->header.recv_wc.rmpp_list);
+			list_add(&local->mad_priv->header.recv_wc.recv_buf.list,
+				 &local->mad_priv->header.recv_wc.rmpp_list);
 			local->mad_priv->header.recv_wc.recv_buf.grh = NULL;
 			local->mad_priv->header.recv_wc.recv_buf.mad =
 						&local->mad_priv->mad.mad;
@@ -2136,11 +2205,47 @@ local_send_completion:
 		spin_lock_irqsave(&mad_agent_priv->lock, flags);
 		list_del(&local->completion_list);
 		atomic_dec(&mad_agent_priv->refcount);
+		if (!recv)
+			kmem_cache_free(ib_mad_cache, local->mad_priv);
 		kfree(local);
 	}
 	spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 }
 
+static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)
+{
+	int ret;
+
+	if (!mad_send_wr->retries--)
+		return -ETIMEDOUT;
+
+	mad_send_wr->timeout = msecs_to_jiffies(mad_send_wr->send_wr.
+						wr.ud.timeout_ms);
+
+	if (mad_send_wr->mad_agent_priv->agent.rmpp_version) {
+		ret = ib_retry_rmpp(mad_send_wr);
+		switch (ret) {
+		case IB_RMPP_RESULT_UNHANDLED:
+			ret = ib_send_mad(mad_send_wr);
+			break;
+		case IB_RMPP_RESULT_CONSUMED:
+			ret = 0;
+			break;
+		default:
+			ret = -ECOMM;
+			break;
+		}
+	} else
+		ret = ib_send_mad(mad_send_wr);
+
+	if (!ret) {
+		mad_send_wr->refcount++;
+		list_add_tail(&mad_send_wr->agent_list,
+			      &mad_send_wr->mad_agent_priv->send_list);
+	}
+	return ret;
+}
+
 static void timeout_sends(void *data)
 {
 	struct ib_mad_agent_private *mad_agent_priv;
@@ -2149,8 +2254,6 @@ static void timeout_sends(void *data)
 	unsigned long flags, delay;
 
 	mad_agent_priv = (struct ib_mad_agent_private *)data;
-
-	mad_send_wc.status = IB_WC_RESP_TIMEOUT_ERR;
 	mad_send_wc.vendor_err = 0;
 
 	spin_lock_irqsave(&mad_agent_priv->lock, flags);
@@ -2170,8 +2273,16 @@ static void timeout_sends(void *data)
 		}
 
 		list_del(&mad_send_wr->agent_list);
+		if (mad_send_wr->status == IB_WC_SUCCESS &&
+		    !retry_send(mad_send_wr))
+			continue;
+
 		spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 
+		if (mad_send_wr->status == IB_WC_SUCCESS)
+			mad_send_wc.status = IB_WC_RESP_TIMEOUT_ERR;
+		else
+			mad_send_wc.status = mad_send_wr->status;
 		mad_send_wc.wr_id = mad_send_wr->wr_id;
 		mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
 						   &mad_send_wc);
@@ -2447,14 +2558,6 @@ static int ib_mad_port_open(struct ib_device *device,
 	unsigned long flags;
 	char name[sizeof "ib_mad123"];
 
-	/* First, check if port already open at MAD layer */
-	port_priv = ib_get_mad_port(device, port_num);
-	if (port_priv) {
-		printk(KERN_DEBUG PFX "%s port %d already open\n",
-		       device->name, port_num);
-		return 0;
-	}
-
 	/* Create new device info */
 	port_priv = kmalloc(sizeof *port_priv, GFP_KERNEL);
 	if (!port_priv) {
@@ -2579,7 +2682,7 @@ static int ib_mad_port_close(struct ib_device *device, int port_num)
 
 static void ib_mad_init_device(struct ib_device *device)
 {
-	int ret, num_ports, cur_port, i, ret2;
+	int num_ports, cur_port, i;
 
 	if (device->node_type == IB_NODE_SWITCH) {
 		num_ports = 1;
@@ -2589,47 +2692,37 @@ static void ib_mad_init_device(struct ib_device *device)
 		cur_port = 1;
 	}
 	for (i = 0; i < num_ports; i++, cur_port++) {
-		ret = ib_mad_port_open(device, cur_port);
-		if (ret) {
+		if (ib_mad_port_open(device, cur_port)) {
 			printk(KERN_ERR PFX "Couldn't open %s port %d\n",
 			       device->name, cur_port);
 			goto error_device_open;
 		}
-		ret = ib_agent_port_open(device, cur_port);
-		if (ret) {
+		if (ib_agent_port_open(device, cur_port)) {
 			printk(KERN_ERR PFX "Couldn't open %s port %d "
 			       "for agents\n",
 			       device->name, cur_port);
 			goto error_device_open;
 		}
 	}
-
-	goto error_device_query;
+	return;
 
 error_device_open:
 	while (i > 0) {
 		cur_port--;
-		ret2 = ib_agent_port_close(device, cur_port);
-		if (ret2) {
+		if (ib_agent_port_close(device, cur_port))
 			printk(KERN_ERR PFX "Couldn't close %s port %d "
 			       "for agents\n",
 			       device->name, cur_port);
-		}
-		ret2 = ib_mad_port_close(device, cur_port);
-		if (ret2) {
+		if (ib_mad_port_close(device, cur_port))
 			printk(KERN_ERR PFX "Couldn't close %s port %d\n",
 			       device->name, cur_port);
-		}
 		i--;
 	}
-
-error_device_query:
-	return;
 }
 
 static void ib_mad_remove_device(struct ib_device *device)
 {
-	int ret = 0, i, num_ports, cur_port, ret2;
+	int i, num_ports, cur_port;
 
 	if (device->node_type == IB_NODE_SWITCH) {
 		num_ports = 1;
@@ -2639,21 +2732,13 @@ static void ib_mad_remove_device(struct ib_device *device)
 		cur_port = 1;
 	}
 	for (i = 0; i < num_ports; i++, cur_port++) {
-		ret2 = ib_agent_port_close(device, cur_port);
-		if (ret2) {
+		if (ib_agent_port_close(device, cur_port))
 			printk(KERN_ERR PFX "Couldn't close %s port %d "
 			       "for agents\n",
 			       device->name, cur_port);
-			if (!ret)
-				ret = ret2;
-		}
-		ret2 = ib_mad_port_close(device, cur_port);
-		if (ret2) {
+		if (ib_mad_port_close(device, cur_port))
 			printk(KERN_ERR PFX "Couldn't close %s port %d\n",
 			       device->name, cur_port);
-			if (!ret)
-				ret = ret2;
-		}
 	}
 }
 
@@ -2709,3 +2794,4 @@ static void __exit ib_mad_cleanup_module(void)
 
 module_init(ib_mad_init_module);
 module_exit(ib_mad_cleanup_module);
+