7 files changed, 3069 insertions, 2303 deletions

diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile
index 81803a16f98..3ce80ab4824 100644
--- a/drivers/scsi/Makefile
+++ b/drivers/scsi/Makefile
@@ -33,7 +33,7 @@ obj-$(CONFIG_SCSI_FC_ATTRS) 	+= scsi_transport_fc.o
 obj-$(CONFIG_SCSI_ISCSI_ATTRS)	+= scsi_transport_iscsi.o
 obj-$(CONFIG_SCSI_SAS_ATTRS)	+= scsi_transport_sas.o
 
-obj-$(CONFIG_ISCSI_TCP) 	+= iscsi_tcp.o
+obj-$(CONFIG_ISCSI_TCP) 	+= libiscsi.o	iscsi_tcp.o
 obj-$(CONFIG_SCSI_AMIGA7XX)	+= amiga7xx.o	53c7xx.o
 obj-$(CONFIG_A3000_SCSI)	+= a3000.o	wd33c93.o
 obj-$(CONFIG_A2091_SCSI)	+= a2091.o	wd33c93.o
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 2068b66822b..fe00a3f6d20 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -3,7 +3,8 @@
  *
  * Copyright (C) 2004 Dmitry Yusupov
  * Copyright (C) 2004 Alex Aizman
- * Copyright (C) 2005 Mike Christie
+ * Copyright (C) 2005 - 2006 Mike Christie
+ * Copyright (C) 2006 Red Hat, Inc.  All rights reserved.
  * maintained by open-iscsi@googlegroups.com
  *
  * This program is free software; you can redistribute it and/or modify
@@ -36,10 +37,6 @@
 #include <linux/mutex.h>
 #include <net/tcp.h>
 #include <scsi/scsi_cmnd.h>
-#include <scsi/scsi_device.h>
-#include <scsi/scsi_eh.h>
-#include <scsi/scsi_request.h>
-#include <scsi/scsi_tcq.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_transport_iscsi.h>
@@ -52,21 +49,14 @@ MODULE_DESCRIPTION("iSCSI/TCP data-path");
 MODULE_LICENSE("GPL");
 MODULE_VERSION("0:4.445");
 /* #define DEBUG_TCP */
-/* #define DEBUG_SCSI */
 #define DEBUG_ASSERT
 
 #ifdef DEBUG_TCP
-#define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt)
+#define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
 #else
 #define debug_tcp(fmt...)
 #endif
 
-#ifdef DEBUG_SCSI
-#define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
-#else
-#define debug_scsi(fmt...)
-#endif
-
 #ifndef DEBUG_ASSERT
 #ifdef BUG_ON
 #undef BUG_ON
@@ -74,8 +64,6 @@ MODULE_VERSION("0:4.445");
 #define BUG_ON(expr)
 #endif
 
-#define INVALID_SN_DELTA	0xffff
-
 static unsigned int iscsi_max_lun = 512;
 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
 
@@ -130,68 +118,39 @@ static inline void
 iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
 		 u8* crc)
 {
-	crypto_digest_digest(conn->tx_tfm, &buf->sg, 1, crc);
-	buf->sg.length += sizeof(uint32_t);
-}
+	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 
-static void
-iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
-{
-	struct iscsi_session *session = conn->session;
-	unsigned long flags;
-
-	spin_lock_irqsave(&session->lock, flags);
-	if (session->conn_cnt == 1 || session->leadconn == conn)
-		session->state = ISCSI_STATE_FAILED;
-	spin_unlock_irqrestore(&session->lock, flags);
-	set_bit(SUSPEND_BIT, &conn->suspend_tx);
-	set_bit(SUSPEND_BIT, &conn->suspend_rx);
-	iscsi_conn_error(conn->cls_conn, err);
+	crypto_digest_digest(tcp_conn->tx_tfm, &buf->sg, 1, crc);
+	buf->sg.length += sizeof(uint32_t);
 }
 
 static inline int
-iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
+iscsi_hdr_extract(struct iscsi_tcp_conn *tcp_conn)
 {
-	uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
-	uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
-
-	if (max_cmdsn < exp_cmdsn -1 &&
-	    max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
-		return ISCSI_ERR_MAX_CMDSN;
-	if (max_cmdsn > session->max_cmdsn ||
-	    max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
-		session->max_cmdsn = max_cmdsn;
-	if (exp_cmdsn > session->exp_cmdsn ||
-	    exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
-		session->exp_cmdsn = exp_cmdsn;
+	struct sk_buff *skb = tcp_conn->in.skb;
 
-	return 0;
-}
-
-static inline int
-iscsi_hdr_extract(struct iscsi_conn *conn)
-{
-	struct sk_buff *skb = conn->in.skb;
+	tcp_conn->in.zero_copy_hdr = 0;
 
-	if (conn->in.copy >= conn->hdr_size &&
-	    conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
+	if (tcp_conn->in.copy >= tcp_conn->hdr_size &&
+	    tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
 		/*
 		 * Zero-copy PDU Header: using connection context
 		 * to store header pointer.
 		 */
 		if (skb_shinfo(skb)->frag_list == NULL &&
-		    !skb_shinfo(skb)->nr_frags)
-			conn->in.hdr = (struct iscsi_hdr *)
-				((char*)skb->data + conn->in.offset);
-		else {
+		    !skb_shinfo(skb)->nr_frags) {
+			tcp_conn->in.hdr = (struct iscsi_hdr *)
+				((char*)skb->data + tcp_conn->in.offset);
+			tcp_conn->in.zero_copy_hdr = 1;
+		} else {
 			/* ignoring return code since we checked
 			 * in.copy before */
-			skb_copy_bits(skb, conn->in.offset,
-				&conn->hdr, conn->hdr_size);
-			conn->in.hdr = &conn->hdr;
+			skb_copy_bits(skb, tcp_conn->in.offset,
+				&tcp_conn->hdr, tcp_conn->hdr_size);
+			tcp_conn->in.hdr = &tcp_conn->hdr;
 		}
-		conn->in.offset += conn->hdr_size;
-		conn->in.copy -= conn->hdr_size;
+		tcp_conn->in.offset += tcp_conn->hdr_size;
+		tcp_conn->in.copy -= tcp_conn->hdr_size;
 	} else {
 		int hdr_remains;
 		int copylen;
@@ -201,118 +160,61 @@ iscsi_hdr_extract(struct iscsi_conn *conn)
 		 * copying it... This'll happen quite rarely.
 		 */
 
-		if (conn->in_progress == IN_PROGRESS_WAIT_HEADER)
-			conn->in.hdr_offset = 0;
+		if (tcp_conn->in_progress == IN_PROGRESS_WAIT_HEADER)
+			tcp_conn->in.hdr_offset = 0;
 
-		hdr_remains = conn->hdr_size - conn->in.hdr_offset;
+		hdr_remains = tcp_conn->hdr_size - tcp_conn->in.hdr_offset;
 		BUG_ON(hdr_remains <= 0);
 
-		copylen = min(conn->in.copy, hdr_remains);
-		skb_copy_bits(skb, conn->in.offset,
-			(char*)&conn->hdr + conn->in.hdr_offset, copylen);
+		copylen = min(tcp_conn->in.copy, hdr_remains);
+		skb_copy_bits(skb, tcp_conn->in.offset,
+			(char*)&tcp_conn->hdr + tcp_conn->in.hdr_offset,
+			copylen);
 
 		debug_tcp("PDU gather offset %d bytes %d in.offset %d "
-		       "in.copy %d\n", conn->in.hdr_offset, copylen,
-		       conn->in.offset, conn->in.copy);
+		       "in.copy %d\n", tcp_conn->in.hdr_offset, copylen,
+		       tcp_conn->in.offset, tcp_conn->in.copy);
 
-		conn->in.offset += copylen;
-		conn->in.copy -= copylen;
+		tcp_conn->in.offset += copylen;
+		tcp_conn->in.copy -= copylen;
 		if (copylen < hdr_remains)  {
-			conn->in_progress = IN_PROGRESS_HEADER_GATHER;
-			conn->in.hdr_offset += copylen;
+			tcp_conn->in_progress = IN_PROGRESS_HEADER_GATHER;
+			tcp_conn->in.hdr_offset += copylen;
 		        return -EAGAIN;
 		}
-		conn->in.hdr = &conn->hdr;
-		conn->discontiguous_hdr_cnt++;
-	        conn->in_progress = IN_PROGRESS_WAIT_HEADER;
+		tcp_conn->in.hdr = &tcp_conn->hdr;
+		tcp_conn->discontiguous_hdr_cnt++;
+	        tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
 	}
 
 	return 0;
 }
 
-static inline void
-iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+/*
+ * must be called with session lock
+ */
+static void
+__iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 {
-	struct scsi_cmnd *sc = ctask->sc;
-	struct iscsi_session *session = conn->session;
+	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
+	struct scsi_cmnd *sc;
 
-	spin_lock(&session->lock);
-	if (unlikely(!sc)) {
-		spin_unlock(&session->lock);
+	sc = ctask->sc;
+	if (unlikely(!sc))
 		return;
-	}
+
 	if (sc->sc_data_direction == DMA_TO_DEVICE) {
 		struct iscsi_data_task *dtask, *n;
+
 		/* WRITE: cleanup Data-Out's if any */
-		list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
+		list_for_each_entry_safe(dtask, n, &tcp_ctask->dataqueue,
+					 item) {
 			list_del(&dtask->item);
-			mempool_free(dtask, ctask->datapool);
+			mempool_free(dtask, tcp_ctask->datapool);
 		}
 	}
-	ctask->xmstate = XMSTATE_IDLE;
-	ctask->r2t = NULL;
-	ctask->sc = NULL;
-	__kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
-	spin_unlock(&session->lock);
-}
-
-/**
- * iscsi_cmd_rsp - SCSI Command Response processing
- * @conn: iscsi connection
- * @ctask: scsi command task
- **/
-static int
-iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
-	int rc;
-	struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
-	struct iscsi_session *session = conn->session;
-	struct scsi_cmnd *sc = ctask->sc;
-
-	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
-	if (rc) {
-		sc->result = (DID_ERROR << 16);
-		goto out;
-	}
-
-	conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
-
-	sc->result = (DID_OK << 16) | rhdr->cmd_status;
-
-	if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
-		sc->result = (DID_ERROR << 16);
-		goto out;
-	}
-
-	if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
-		int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
-
-		memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
-		debug_scsi("copied %d bytes of sense\n", sensecopy);
-	}
-
-	if (sc->sc_data_direction == DMA_TO_DEVICE)
-		goto out;
-
-	if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
-		int res_count = be32_to_cpu(rhdr->residual_count);
-
-		if (res_count > 0 && res_count <= sc->request_bufflen)
-			sc->resid = res_count;
-		else
-			sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
-	} else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
-		sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
-	else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
-		sc->resid = be32_to_cpu(rhdr->residual_count);
-
-out:
-	debug_scsi("done [sc %lx res %d itt 0x%x]\n",
-		   (long)sc, sc->result, ctask->itt);
-	conn->scsirsp_pdus_cnt++;
-	iscsi_ctask_cleanup(conn, ctask);
-	sc->scsi_done(sc);
-	return rc;
+	tcp_ctask->xmstate = XMSTATE_IDLE;
+	tcp_ctask->r2t = NULL;
 }
 
 /**
@@ -324,7 +226,9 @@ static int
 iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 {
 	int rc;
-	struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr;
+	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
+	struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
 	struct iscsi_session *session = conn->session;
 	int datasn = be32_to_cpu(rhdr->datasn);
 
@@ -334,9 +238,9 @@ iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 	/*
 	 * setup Data-In byte counter (gets decremented..)
 	 */
-	ctask->data_count = conn->in.datalen;
+	ctask->data_count = tcp_conn->in.datalen;
 
-	if (conn->in.datalen == 0)
+	if (tcp_conn->in.datalen == 0)
 		return 0;
 
 	if (ctask->datasn != datasn)
@@ -344,8 +248,8 @@ iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 
 	ctask->datasn++;
 
-	ctask->data_offset = be32_to_cpu(rhdr->offset);
-	if (ctask->data_offset + conn->in.datalen > ctask->total_length)
+	tcp_ctask->data_offset = be32_to_cpu(rhdr->offset);
+	if (tcp_ctask->data_offset + tcp_conn->in.datalen > ctask->total_length)
 		return ISCSI_ERR_DATA_OFFSET;
 
 	if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
@@ -392,17 +296,19 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 	struct iscsi_data *hdr;
 	struct iscsi_data_task *dtask;
 	struct scsi_cmnd *sc = ctask->sc;
+	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
 
-	dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
+	dtask = mempool_alloc(tcp_ctask->datapool, GFP_ATOMIC);
 	BUG_ON(!dtask);
+	INIT_LIST_HEAD(&dtask->item);
 	hdr = &dtask->hdr;
 	memset(hdr, 0, sizeof(struct iscsi_data));
 	hdr->ttt = r2t->ttt;
 	hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
 	r2t->solicit_datasn++;
 	hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
-	memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
-	hdr->itt = ctask->hdr.itt;
+	memcpy(hdr->lun, ctask->hdr->lun, sizeof(hdr->lun));
+	hdr->itt = ctask->hdr->itt;
 	hdr->exp_statsn = r2t->exp_statsn;
 	hdr->offset = cpu_to_be32(r2t->data_offset);
 	if (r2t->data_length > conn->max_xmit_dlength) {
@@ -451,11 +357,11 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 		}
 		BUG_ON(r2t->sg == NULL);
 	} else
-		iscsi_buf_init_iov(&ctask->sendbuf,
+		iscsi_buf_init_iov(&tcp_ctask->sendbuf,
 			    (char*)sc->request_buffer + r2t->data_offset,
 			    r2t->data_count);
 
-	list_add(&dtask->item, &ctask->dataqueue);
+	list_add(&dtask->item, &tcp_ctask->dataqueue);
 }
 
 /**
@@ -468,17 +374,16 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 {
 	struct iscsi_r2t_info *r2t;
 	struct iscsi_session *session = conn->session;
-	struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr;
+	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
+	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+	struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
 	int r2tsn = be32_to_cpu(rhdr->r2tsn);
 	int rc;
 
-	if (conn->in.ahslen)
-		return ISCSI_ERR_AHSLEN;
-
-	if (conn->in.datalen)
+	if (tcp_conn->in.datalen)
 		return ISCSI_ERR_DATALEN;
 
-	if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn)
+	if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn)
 		return ISCSI_ERR_R2TSN;
 
 	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
@@ -496,7 +401,7 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 		spin_unlock(&session->lock);
 		return 0;
 	}
-	rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
+	rc = __kfifo_get(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
 	BUG_ON(!rc);
 
 	r2t->exp_statsn = rhdr->statsn;
@@ -518,10 +423,10 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 
 	iscsi_solicit_data_init(conn, ctask, r2t);
 
-	ctask->exp_r2tsn = r2tsn + 1;
-	ctask->xmstate |= XMSTATE_SOL_HDR;
-	__kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*));
-	__kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*));
+	tcp_ctask->exp_r2tsn = r2tsn + 1;
+	tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
+	__kfifo_put(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*));
+	__kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
 
 	scsi_queue_work(session->host, &conn->xmitwork);
 	conn->r2t_pdus_cnt++;
@@ -531,258 +436,136 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 }
 
 static int
-iscsi_hdr_recv(struct iscsi_conn *conn)
+iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
 {
-	int rc = 0;
+	int rc = 0, opcode, ahslen;
 	struct iscsi_hdr *hdr;
-	struct iscsi_cmd_task *ctask;
 	struct iscsi_session *session = conn->session;
-	uint32_t cdgst, rdgst = 0;
+	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+	uint32_t cdgst, rdgst = 0, itt;
 
-	hdr = conn->in.hdr;
+	hdr = tcp_conn->in.hdr;
 
 	/* verify PDU length */
-	conn->in.datalen = ntoh24(hdr->dlength);
-	if (conn->in.datalen > conn->max_recv_dlength) {
+	tcp_conn->in.datalen = ntoh24(hdr->dlength);
+	if (tcp_conn->in.datalen > conn->max_recv_dlength) {
 		printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
-		       conn->in.datalen, conn->max_recv_dlength);
+		       tcp_conn->in.datalen, conn->max_recv_dlength);
 		return ISCSI_ERR_DATALEN;
 	}
-	conn->data_copied = 0;
+	tcp_conn->data_copied = 0;
 
 	/* read AHS */
-	conn->in.ahslen = hdr->hlength * 4;
-	conn->in.offset += conn->in.ahslen;
-	conn->in.copy -= conn->in.ahslen;
-	if (conn->in.copy < 0) {
+	ahslen = hdr->hlength << 2;
+	tcp_conn->in.offset += ahslen;
+	tcp_conn->in.copy -= ahslen;
+	if (tcp_conn->in.copy < 0) {
 		printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
-		       "%d bytes\n", conn->in.ahslen);
+		       "%d bytes\n", ahslen);
 		return ISCSI_ERR_AHSLEN;
 	}
 
 	/* calculate read padding */
-	conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1);
-	if (conn->in.padding) {
-		conn->in.padding = ISCSI_PAD_LEN - conn->in.padding;
-		debug_scsi("read padding %d bytes\n", conn->in.padding);
+	tcp_conn->in.padding = tcp_conn->in.datalen & (ISCSI_PAD_LEN-1);
+	if (tcp_conn->in.padding) {
+		tcp_conn->in.padding = ISCSI_PAD_LEN - tcp_conn->in.padding;
+		debug_scsi("read padding %d bytes\n", tcp_conn->in.padding);
 	}
 
 	if (conn->hdrdgst_en) {
 		struct scatterlist sg;
 
 		sg_init_one(&sg, (u8 *)hdr,
-			    sizeof(struct iscsi_hdr) + conn->in.ahslen);
-		crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
+			    sizeof(struct iscsi_hdr) + ahslen);
+		crypto_digest_digest(tcp_conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
 		rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
-				     conn->in.ahslen);
+				     ahslen);
 		if (cdgst != rdgst) {
-			printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error "
-			       "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst,
-			       cdgst);
+			printk(KERN_ERR "iscsi_tcp: hdrdgst error "
+			       "recv 0x%x calc 0x%x\n", rdgst, cdgst);
 			return ISCSI_ERR_HDR_DGST;
 		}
 	}
 
-	/* save opcode for later */
-	conn->in.opcode = hdr->opcode & ISCSI_OPCODE_MASK;
-
+	opcode = hdr->opcode & ISCSI_OPCODE_MASK;
 	/* verify itt (itt encoding: age+cid+itt) */
-	if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
-		if ((hdr->itt & AGE_MASK) !=
-				(session->age << AGE_SHIFT)) {
-			printk(KERN_ERR "iscsi_tcp: received itt %x expected "
-				"session age (%x)\n", hdr->itt,
-				session->age & AGE_MASK);
-			return ISCSI_ERR_BAD_ITT;
-		}
-
-		if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
-			printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
-				"CID (%x)\n", hdr->itt, conn->id);
-			return ISCSI_ERR_BAD_ITT;
-		}
-		conn->in.itt = hdr->itt & ITT_MASK;
-	} else
-		conn->in.itt = hdr->itt;
+	rc = iscsi_verify_itt(conn, hdr, &itt);
+	if (rc == ISCSI_ERR_NO_SCSI_CMD) {
+		tcp_conn->in.datalen = 0; /* force drop */
+		return 0;
+	} else if (rc)
+		return rc;
 
 	debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
-		  hdr->opcode, conn->in.offset, conn->in.copy,
-		  conn->in.ahslen, conn->in.datalen);
-
-	if (conn->in.itt < session->cmds_max) {
-		ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
-
-		if (!ctask->sc) {
-			printk(KERN_INFO "iscsi_tcp: dropping ctask with "
-			       "itt 0x%x\n", ctask->itt);
-			conn->in.datalen = 0; /* force drop */
-			return 0;
-		}
-
-		if (ctask->sc->SCp.phase != session->age) {
-			printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
-				"expected %d\n", ctask->sc->SCp.phase,
-				session->age);
-			return ISCSI_ERR_SESSION_FAILED;
-		}
-
-		conn->in.ctask = ctask;
-
-		debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
-			   hdr->opcode, conn->id, (long)ctask->sc,
-			   ctask->itt, conn->in.datalen);
-
-		switch(conn->in.opcode) {
-		case ISCSI_OP_SCSI_CMD_RSP:
-			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
-			if (!conn->in.datalen)
-				rc = iscsi_cmd_rsp(conn, ctask);
-			else
-				/*
-				 * got sense or response data; copying PDU
-				 * Header to the connection's header
-				 * placeholder
-				 */
-				memcpy(&conn->hdr, hdr,
-				       sizeof(struct iscsi_hdr));
-			break;
-		case ISCSI_OP_SCSI_DATA_IN:
-			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
-			/* save flags for non-exceptional status */
-			conn->in.flags = hdr->flags;
-			/* save cmd_status for sense data */
-			conn->in.cmd_status =
-				((struct iscsi_data_rsp*)hdr)->cmd_status;
-			rc = iscsi_data_rsp(conn, ctask);
-			break;
-		case ISCSI_OP_R2T:
-			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
-			if (ctask->sc->sc_data_direction == DMA_TO_DEVICE)
-				rc = iscsi_r2t_rsp(conn, ctask);
-			else
-				rc = ISCSI_ERR_PROTO;
-			break;
-		default:
-			rc = ISCSI_ERR_BAD_OPCODE;
-			break;
-		}
-	} else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
-		   conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
-					session->mgmtpool_max) {
-		struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
-					session->mgmt_cmds[conn->in.itt -
-						ISCSI_MGMT_ITT_OFFSET];
-
-		debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
-			   conn->in.opcode, conn->id, mtask->itt,
-			   conn->in.datalen);
-
-		switch(conn->in.opcode) {
-		case ISCSI_OP_LOGIN_RSP:
-		case ISCSI_OP_TEXT_RSP:
-		case ISCSI_OP_LOGOUT_RSP:
-			rc = iscsi_check_assign_cmdsn(session,
-						 (struct iscsi_nopin*)hdr);
-			if (rc)
-				break;
-
-			if (!conn->in.datalen) {
-				rc = iscsi_recv_pdu(conn->cls_conn, hdr,
-						    NULL, 0);
-				if (conn->login_mtask != mtask) {
-					spin_lock(&session->lock);
-					__kfifo_put(session->mgmtpool.queue,
-					    (void*)&mtask, sizeof(void*));
-					spin_unlock(&session->lock);
-				}
-			}
-			break;
-		case ISCSI_OP_SCSI_TMFUNC_RSP:
-			rc = iscsi_check_assign_cmdsn(session,
-						 (struct iscsi_nopin*)hdr);
-			if (rc)
-				break;
+		  opcode, tcp_conn->in.offset, tcp_conn->in.copy,
+		  ahslen, tcp_conn->in.datalen);
 
-			if (conn->in.datalen || conn->in.ahslen) {
-				rc = ISCSI_ERR_PROTO;
-				break;
-			}
-			conn->tmfrsp_pdus_cnt++;
-			spin_lock(&session->lock);
-			if (conn->tmabort_state == TMABORT_INITIAL) {
-				__kfifo_put(session->mgmtpool.queue,
-						(void*)&mtask, sizeof(void*));
-				conn->tmabort_state =
-					((struct iscsi_tm_rsp *)hdr)->
-					response == ISCSI_TMF_RSP_COMPLETE ?
-						TMABORT_SUCCESS:TMABORT_FAILED;
-				/* unblock eh_abort() */
-				wake_up(&conn->ehwait);
-			}
-			spin_unlock(&session->lock);
-			break;
-		case ISCSI_OP_NOOP_IN:
-			if (hdr->ttt != ISCSI_RESERVED_TAG) {
-				rc = ISCSI_ERR_PROTO;
-				break;
-			}
-			rc = iscsi_check_assign_cmdsn(session,
-						(struct iscsi_nopin*)hdr);
-			if (rc)
-				break;
-			conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
-
-			if (!conn->in.datalen) {
-				struct iscsi_mgmt_task *mtask;
-
-				rc = iscsi_recv_pdu(conn->cls_conn, hdr,
-						    NULL, 0);
-				mtask = (struct iscsi_mgmt_task *)
-					session->mgmt_cmds[conn->in.itt -
-							ISCSI_MGMT_ITT_OFFSET];
-				if (conn->login_mtask != mtask) {
-					spin_lock(&session->lock);
-					__kfifo_put(session->mgmtpool.queue,
-						  (void*)&mtask, sizeof(void*));
-					spin_unlock(&session->lock);
-				}
-			}
-			break;
-		default:
-			rc = ISCSI_ERR_BAD_OPCODE;
-			break;
-		}
-	} else if (conn->in.itt == ISCSI_RESERVED_TAG) {
-		switch(conn->in.opcode) {
-		case ISCSI_OP_NOOP_IN:
-			if (!conn->in.datalen) {
-				rc = iscsi_check_assign_cmdsn(session,
-						 (struct iscsi_nopin*)hdr);
-				if (!rc && hdr->ttt != ISCSI_RESERVED_TAG)
-					rc = iscsi_recv_pdu(conn->cls_conn,
-							    hdr, NULL, 0);
-			} else
-				rc = ISCSI_ERR_PROTO;
-			break;
-		case ISCSI_OP_REJECT:
-			/* we need sth like iscsi_reject_rsp()*/
-		case ISCSI_OP_ASYNC_EVENT:
-			/* we need sth like iscsi_async_event_rsp() */
-			rc = ISCSI_ERR_BAD_OPCODE;
-			break;
-		default:
-			rc = ISCSI_ERR_BAD_OPCODE;
-			break;
-		}
-	} else
-		rc = ISCSI_ERR_BAD_ITT;
+	switch(opcode) {
+	case ISCSI_OP_SCSI_DATA_IN:
+		tcp_conn->in.ctask = session->cmds[itt];
+		rc = iscsi_data_rsp(conn, tcp_conn->in.ctask);
+		/* fall through */
+	case ISCSI_OP_SCSI_CMD_RSP:
+		tcp_conn->in.ctask = session->cmds[itt];
+		if (tcp_conn->in.datalen)
+			goto copy_hdr;
+
+		spin_lock(&session->lock);
+		__iscsi_ctask_cleanup(conn, tcp_conn->in.ctask);
+		rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
+		spin_unlock(&session->lock);
+		break;
+	case ISCSI_OP_R2T:
+		tcp_conn->in.ctask = session->cmds[itt];
+		if (ahslen)
+			rc = ISCSI_ERR_AHSLEN;
+		else if (tcp_conn->in.ctask->sc->sc_data_direction ==
+								DMA_TO_DEVICE)
+			rc = iscsi_r2t_rsp(conn, tcp_conn->in.ctask);
+		else
+			rc = ISCSI_ERR_PROTO;
+		break;
+	case ISCSI_OP_LOGIN_RSP:
+	case ISCSI_OP_TEXT_RSP:
+	case ISCSI_OP_LOGOUT_RSP:
+	case ISCSI_OP_NOOP_IN:
+	case ISCSI_OP_REJECT:
+	case ISCSI_OP_ASYNC_EVENT:
+		if (tcp_conn->in.datalen)
+			goto copy_hdr;
+	/* fall through */
+	case ISCSI_OP_SCSI_TMFUNC_RSP:
+		rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
+		break;
+	default:
+		rc = ISCSI_ERR_BAD_OPCODE;
+		break;
+	}
 
 	return rc;
+
+copy_hdr:
+	/*
+	 * if we did zero copy for the header but we will need multiple
+	 * skbs to complete the command then we have to copy the header
+	 * for later use
+	 */
+	if (tcp_conn->in.zero_copy_hdr && tcp_conn->in.copy <
+	   (tcp_conn->in.datalen + tcp_conn->in.padding +
+	    (conn->datadgst_en ? 4 : 0))) {
+		debug_tcp("Copying header for later use. in.copy %d in.datalen"
+			  " %d\n", tcp_conn->in.copy, tcp_conn->in.datalen);
+		memcpy(&tcp_conn->hdr, tcp_conn->in.hdr,
+		       sizeof(struct iscsi_hdr));
+		tcp_conn->in.hdr = &tcp_conn->hdr;
+		tcp_conn->in.zero_copy_hdr = 0;
+	}
+	return 0;
 }
 
 /**
  * iscsi_ctask_copy - copy skb bits to the destanation cmd task
- * @conn: iscsi connection
+ * @conn: iscsi tcp connection
  * @ctask: scsi command task
  * @buf: buffer to copy to
  * @buf_size: size of buffer
@@ -804,110 +587,113 @@ iscsi_hdr_recv(struct iscsi_conn *conn)
  *	buf_left		left to copy from in progress buffer
  **/
 static inline int
-iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
+iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
 		void *buf, int buf_size, int offset)
 {
-	int buf_left = buf_size - (conn->data_copied + offset);
-	int size = min(conn->in.copy, buf_left);
+	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
+	int buf_left = buf_size - (tcp_conn->data_copied + offset);
+	int size = min(tcp_conn->in.copy, buf_left);
 	int rc;
 
 	size = min(size, ctask->data_count);
 
 	debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
-	       size, conn->in.offset, conn->in.copied);
+	       size, tcp_conn->in.offset, tcp_conn->in.copied);
 
 	BUG_ON(size <= 0);
-	BUG_ON(ctask->sent + size > ctask->total_length);
+	BUG_ON(tcp_ctask->sent + size > ctask->total_length);
 
-	rc = skb_copy_bits(conn->in.skb, conn->in.offset,
-			   (char*)buf + (offset + conn->data_copied), size);
+	rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
+			   (char*)buf + (offset + tcp_conn->data_copied), size);
 	/* must fit into skb->len */
 	BUG_ON(rc);
 
-	conn->in.offset += size;
-	conn->in.copy -= size;
-	conn->in.copied += size;
-	conn->data_copied += size;
-	ctask->sent += size;
+	tcp_conn->in.offset += size;
+	tcp_conn->in.copy -= size;
+	tcp_conn->in.copied += size;
+	tcp_conn->data_copied += size;
+	tcp_ctask->sent += size;
 	ctask->data_count -= size;
 
-	BUG_ON(conn->in.copy < 0);
+	BUG_ON(tcp_conn->in.copy < 0);
 	BUG_ON(ctask->data_count < 0);
 
-	if (buf_size != (conn->data_copied + offset)) {
+	if (buf_size != (tcp_conn->data_copied + offset)) {
 		if (!ctask->data_count) {
-			BUG_ON(buf_size - conn->data_copied < 0);
+			BUG_ON(buf_size - tcp_conn->data_copied < 0);
 			/* done with this PDU */
-			return buf_size - conn->data_copied;
+			return buf_size - tcp_conn->data_copied;
 		}
 		return -EAGAIN;
 	}
 
 	/* done with this buffer or with both - PDU and buffer */
-	conn->data_copied = 0;
+	tcp_conn->data_copied = 0;
 	return 0;
 }
 
 /**
  * iscsi_tcp_copy - copy skb bits to the destanation buffer
- * @conn: iscsi connection
- * @buf: buffer to copy to
- * @buf_size: number of bytes to copy
+ * @conn: iscsi tcp connection
  *
  * Notes:
  *	The function calls skb_copy_bits() and updates per-connection
  *	byte counters.
  **/
 static inline int
-iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size)
+iscsi_tcp_copy(struct iscsi_tcp_conn *tcp_conn)
 {
-	int buf_left = buf_size - conn->data_copied;
-	int size = min(conn->in.copy, buf_left);
+	void *buf = tcp_conn->data;
+	int buf_size = tcp_conn->in.datalen;
+	int buf_left = buf_size - tcp_conn->data_copied;
+	int size = min(tcp_conn->in.copy, buf_left);
 	int rc;
 
 	debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
-	       size, conn->in.offset, conn->data_copied);
+	       size, tcp_conn->in.offset, tcp_conn->data_copied);
 	BUG_ON(size <= 0);
 
-	rc = skb_copy_bits(conn->in.skb, conn->in.offset,
-			   (char*)buf + conn->data_copied, size);
+	rc = skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
+			   (char*)buf + tcp_conn->data_copied, size);
 	BUG_ON(rc);
 
-	conn->in.offset += size;
-	conn->in.copy -= size;
-	conn->in.copied += size;
-	conn->data_copied += size;
+	tcp_conn->in.offset += size;
+	tcp_conn->in.copy -= size;
+	tcp_conn->in.copied += size;
+	tcp_conn->data_copied += size;
 
-	if (buf_size != conn->data_copied)
+	if (buf_size != tcp_conn->data_copied)
 		return -EAGAIN;
 
 	return 0;
 }
 
 static inline void
-partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg,
-			 int offset, int length)
+partial_sg_digest_update(struct iscsi_tcp_conn *tcp_conn,
+			 struct scatterlist *sg, int offset, int length)
 {
 	struct scatterlist temp;
 
 	memcpy(&temp, sg, sizeof(struct scatterlist));
 	temp.offset = offset;
 	temp.length = length;
-	crypto_digest_update(conn->data_rx_tfm, &temp, 1);
+	crypto_digest_update(tcp_conn->data_rx_tfm, &temp, 1);
 }
 
 static void
-iscsi_recv_digest_update(struct iscsi_conn *conn, char* buf, int len)
+iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
 {
 	struct scatterlist tmp;
 
 	sg_init_one(&tmp, buf, len);
-	crypto_digest_update(conn->data_rx_tfm, &tmp, 1);
+	crypto_digest_update(tcp_conn->data_rx_tfm, &tmp, 1);
 }
 
 static int iscsi_scsi_data_in(struct iscsi_conn *conn)
 {
-	struct iscsi_cmd_task *ctask = conn->in.ctask;
+	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+	struct iscsi_cmd_task *ctask = tcp_conn->in.ctask;
+	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
 	struct scsi_cmnd *sc = ctask->sc;
 	struct scatterlist *sg;
 	int i, offset, rc = 0;
@@ -919,31 +705,33 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
 	 */
 	if (!sc->use_sg) {
 		i = ctask->data_count;
-		rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer,
-				      sc->request_bufflen, ctask->data_offset);
+		rc = iscsi_ctask_copy(tcp_conn, ctask, sc->request_buffer,
+				      sc->request_bufflen,
+				      tcp_ctask->data_offset);
 		if (rc == -EAGAIN)
 			return rc;
 		if (conn->datadgst_en)
-			iscsi_recv_digest_update(conn, sc->request_buffer, i);
+			iscsi_recv_digest_update(tcp_conn, sc->request_buffer,
+						 i);
 		rc = 0;
 		goto done;
 	}
 
-	offset = ctask->data_offset;
+	offset = tcp_ctask->data_offset;
 	sg = sc->request_buffer;
 
-	if (ctask->data_offset)
-		for (i = 0; i < ctask->sg_count; i++)
+	if (tcp_ctask->data_offset)
+		for (i = 0; i < tcp_ctask->sg_count; i++)
 			offset -= sg[i].length;
 	/* we've passed through partial sg*/
 	if (offset < 0)
 		offset = 0;
 
-	for (i = ctask->sg_count; i < sc->use_sg; i++) {
+	for (i = tcp_ctask->sg_count; i < sc->use_sg; i++) {
 		char *dest;
 
 		dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
-		rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset,
+		rc = iscsi_ctask_copy(tcp_conn, ctask, dest + sg[i].offset,
 				      sg[i].length, offset);
 		kunmap_atomic(dest, KM_SOFTIRQ0);
 		if (rc == -EAGAIN)
@@ -952,15 +740,17 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
 		if (!rc) {
 			if (conn->datadgst_en) {
 				if (!offset)
-					crypto_digest_update(conn->data_rx_tfm,
-							     &sg[i], 1);
+					crypto_digest_update(
+							tcp_conn->data_rx_tfm,
+							&sg[i], 1);
 				else
-					partial_sg_digest_update(conn, &sg[i],
+					partial_sg_digest_update(tcp_conn,
+							&sg[i],
 							sg[i].offset + offset,
 							sg[i].length - offset);
 			}
 			offset = 0;
-			ctask->sg_count++;
+			tcp_ctask->sg_count++;
 		}
 
 		if (!ctask->data_count) {
@@ -968,25 +758,26 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
 				/*
 				 * data-in is complete, but buffer not...
 				 */
-				partial_sg_digest_update(conn, &sg[i],
+				partial_sg_digest_update(tcp_conn, &sg[i],
 						sg[i].offset, sg[i].length-rc);
 			rc = 0;
 			break;
 		}
 
-		if (!conn->in.copy)
+		if (!tcp_conn->in.copy)
 			return -EAGAIN;
 	}
 	BUG_ON(ctask->data_count);
 
 done:
 	/* check for non-exceptional status */
-	if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) {
+	if (tcp_conn->in.hdr->flags & ISCSI_FLAG_DATA_STATUS) {
 		debug_scsi("done [sc %lx res %d itt 0x%x]\n",
 			   (long)sc, sc->result, ctask->itt);
-		conn->scsirsp_pdus_cnt++;
-		iscsi_ctask_cleanup(conn, ctask);
-		sc->scsi_done(sc);
+		spin_lock(&conn->session->lock);
+		__iscsi_ctask_cleanup(conn, ctask);
+		__iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
+		spin_unlock(&conn->session->lock);