[SCSI] open-iscsi/linux-iscsi-5 Initiator: Initiator code

drivers/scsi/iscsi_tcp.c, iscsi data path.

Signed-off-by: Alex Aizman <itn780@yahoo.com>
Signed-off-by: Dmitry Yusupov <dmitry_yus@yahoo.com>
Signed-off-by: Mike Christie <michaelc@cs.wisc.edu>
Signed-off-by: James Bottomley <James.Bottomley@SteelEye.com>

1 files changed, 3627 insertions, 0 deletions

diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
new file mode 100644
index 00000000000..810e5e59658
--- /dev/null
+++ b/drivers/scsi/iscsi_tcp.c
@@ -0,0 +1,3627 @@
+/*
+ * iSCSI Initiator over TCP/IP Data-Path
+ *
+ * Copyright (C) 2004 Dmitry Yusupov
+ * Copyright (C) 2004 Alex Aizman
+ * Copyright (C) 2005 Mike Christie
+ * maintained by open-iscsi@googlegroups.com
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published
+ * by the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * See the file COPYING included with this distribution for more details.
+ *
+ * Credits:
+ *	Christoph Hellwig
+ *	FUJITA Tomonori
+ *	Arne Redlich
+ *	Zhenyu Wang
+ */
+
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/inet.h>
+#include <linux/blkdev.h>
+#include <linux/crypto.h>
+#include <linux/delay.h>
+#include <linux/kfifo.h>
+#include <linux/scatterlist.h>
+#include <net/tcp.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_eh.h>
+#include <scsi/scsi_request.h>
+#include <scsi/scsi_tcq.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi.h>
+#include <scsi/scsi_transport_iscsi.h>
+
+#include "iscsi_tcp.h"
+
+MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
+	      "Alex Aizman <itn780@yahoo.com>");
+MODULE_DESCRIPTION("iSCSI/TCP data-path");
+MODULE_LICENSE("GPL");
+
+/* #define DEBUG_TCP */
+/* #define DEBUG_SCSI */
+#define DEBUG_ASSERT
+
+#ifdef DEBUG_TCP
+#define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt)
+#else
+#define debug_tcp(fmt...)
+#endif
+
+#ifdef DEBUG_SCSI
+#define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
+#else
+#define debug_scsi(fmt...)
+#endif
+
+#ifndef DEBUG_ASSERT
+#ifdef BUG_ON
+#undef BUG_ON
+#endif
+#define BUG_ON(expr)
+#endif
+
+#define INVALID_SN_DELTA	0xffff
+
+static unsigned int iscsi_max_lun = 512;
+module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
+
+/* global data */
+static kmem_cache_t *taskcache;
+
+static inline void
+iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
+{
+	sg_init_one(&ibuf->sg, (u8 *)vbuf, size);
+	ibuf->sent = 0;
+}
+
+static inline void
+iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
+{
+	ibuf->sg.page = (void*)vbuf;
+	ibuf->sg.offset = (unsigned int)-1;
+	ibuf->sg.length = size;
+	ibuf->sent = 0;
+}
+
+static inline void*
+iscsi_buf_iov_base(struct iscsi_buf *ibuf)
+{
+	return (char*)ibuf->sg.page + ibuf->sent;
+}
+
+static inline void
+iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
+{
+	/*
+	 * Fastpath: sg element fits into single page
+	 */
+	if (sg->length + sg->offset <= PAGE_SIZE && page_count(sg->page) >= 2) {
+		ibuf->sg.page = sg->page;
+		ibuf->sg.offset = sg->offset;
+		ibuf->sg.length = sg->length;
+	} else
+		iscsi_buf_init_iov(ibuf, page_address(sg->page), sg->length);
+	ibuf->sent = 0;
+}
+
+static inline int
+iscsi_buf_left(struct iscsi_buf *ibuf)
+{
+	int rc;
+
+	rc = ibuf->sg.length - ibuf->sent;
+	BUG_ON(rc < 0);
+	return rc;
+}
+
+static inline void
+iscsi_buf_init_hdr(struct iscsi_conn *conn, struct iscsi_buf *ibuf,
+		   char *vbuf, u8 *crc)
+{
+	iscsi_buf_init_virt(ibuf, vbuf, sizeof(struct iscsi_hdr));
+	if (conn->hdrdgst_en) {
+		crypto_digest_digest(conn->tx_tfm, &ibuf->sg, 1, crc);
+		ibuf->sg.length += sizeof(uint32_t);
+	}
+}
+
+static void
+iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
+{
+	struct iscsi_session *session = conn->session;
+	unsigned long flags;
+
+	spin_lock_irqsave(&session->lock, flags);
+	if (session->conn_cnt == 1 || session->leadconn == conn)
+		session->state = ISCSI_STATE_FAILED;
+	spin_unlock_irqrestore(&session->lock, flags);
+	set_bit(SUSPEND_BIT, &conn->suspend_tx);
+	set_bit(SUSPEND_BIT, &conn->suspend_rx);
+	iscsi_conn_error(iscsi_handle(conn), err);
+}
+
+static inline int
+iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
+{
+	uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
+	uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
+
+	if (max_cmdsn < exp_cmdsn -1 &&
+	    max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
+		return ISCSI_ERR_MAX_CMDSN;
+	if (max_cmdsn > session->max_cmdsn ||
+	    max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
+		session->max_cmdsn = max_cmdsn;
+	if (exp_cmdsn > session->exp_cmdsn ||
+	    exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
+		session->exp_cmdsn = exp_cmdsn;
+
+	return 0;
+}
+
+static inline int
+iscsi_hdr_extract(struct iscsi_conn *conn)
+{
+	struct sk_buff *skb = conn->in.skb;
+
+	if (conn->in.copy >= conn->hdr_size &&
+	    conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
+		/*
+		 * Zero-copy PDU Header: using connection context
+		 * to store header pointer.
+		 */
+		if (skb_shinfo(skb)->frag_list == NULL &&
+		    !skb_shinfo(skb)->nr_frags)
+			conn->in.hdr = (struct iscsi_hdr *)
+				((char*)skb->data + conn->in.offset);
+		else {
+			/* ignoring return code since we checked
+			 * in.copy before */
+			skb_copy_bits(skb, conn->in.offset,
+				&conn->hdr, conn->hdr_size);
+			conn->in.hdr = &conn->hdr;
+		}
+		conn->in.offset += conn->hdr_size;
+		conn->in.copy -= conn->hdr_size;
+	} else {
+		int hdr_remains;
+		int copylen;
+
+		/*
+		 * PDU header scattered across SKB's,
+		 * copying it... This'll happen quite rarely.
+		 */
+
+		if (conn->in_progress == IN_PROGRESS_WAIT_HEADER)
+			conn->in.hdr_offset = 0;
+
+		hdr_remains = conn->hdr_size - conn->in.hdr_offset;
+		BUG_ON(hdr_remains <= 0);
+
+		copylen = min(conn->in.copy, hdr_remains);
+		skb_copy_bits(skb, conn->in.offset,
+			(char*)&conn->hdr + conn->in.hdr_offset, copylen);
+
+		debug_tcp("PDU gather offset %d bytes %d in.offset %d "
+		       "in.copy %d\n", conn->in.hdr_offset, copylen,
+		       conn->in.offset, conn->in.copy);
+
+		conn->in.offset += copylen;
+		conn->in.copy -= copylen;
+		if (copylen < hdr_remains)  {
+			conn->in_progress = IN_PROGRESS_HEADER_GATHER;
+			conn->in.hdr_offset += copylen;
+		        return -EAGAIN;
+		}
+		conn->in.hdr = &conn->hdr;
+		conn->discontiguous_hdr_cnt++;
+	        conn->in_progress = IN_PROGRESS_WAIT_HEADER;
+	}
+
+	return 0;
+}
+
+static inline void
+iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+{
+	struct scsi_cmnd *sc = ctask->sc;
+	struct iscsi_session *session = conn->session;
+
+	spin_lock(&session->lock);
+	if (unlikely(!sc)) {
+		spin_unlock(&session->lock);
+		return;
+	}
+	if (sc->sc_data_direction == DMA_TO_DEVICE) {
+		struct iscsi_data_task *dtask, *n;
+		/* WRITE: cleanup Data-Out's if any */
+		spin_lock(&conn->lock);
+		list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
+			list_del(&dtask->item);
+			mempool_free(dtask, ctask->datapool);
+		}
+		spin_unlock(&conn->lock);
+	}
+	ctask->xmstate = XMSTATE_IDLE;
+	ctask->r2t = NULL;
+	ctask->sc = NULL;
+	__kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
+	spin_unlock(&session->lock);
+}
+
+/**
+ * iscsi_cmd_rsp - SCSI Command Response processing
+ * @conn: iscsi connection
+ * @ctask: scsi command task
+ **/
+static int
+iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+{
+	int rc;
+	struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
+	struct iscsi_session *session = conn->session;
+	struct scsi_cmnd *sc = ctask->sc;
+
+	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
+	if (rc) {
+		sc->result = (DID_ERROR << 16);
+		goto out;
+	}
+
+	conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
+
+	sc->result = (DID_OK << 16) | rhdr->cmd_status;
+
+	if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
+		sc->result = (DID_ERROR << 16);
+		goto out;
+	}
+
+	if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
+		int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
+
+		memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
+		debug_scsi("copied %d bytes of sense\n", sensecopy);
+	}
+
+	if (sc->sc_data_direction == DMA_TO_DEVICE)
+		goto out;
+
+	if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
+		int res_count = be32_to_cpu(rhdr->residual_count);
+
+		if (res_count > 0 && res_count <= sc->request_bufflen)
+			sc->resid = res_count;
+		else
+			sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
+	} else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
+		sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
+	else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
+		sc->resid = be32_to_cpu(rhdr->residual_count);
+
+out:
+	debug_scsi("done [sc %lx res %d itt 0x%x]\n",
+		   (long)sc, sc->result, ctask->itt);
+	conn->scsirsp_pdus_cnt++;
+	iscsi_ctask_cleanup(conn, ctask);
+	sc->scsi_done(sc);
+	return rc;
+}
+
+/**
+ * iscsi_data_rsp - SCSI Data-In Response processing
+ * @conn: iscsi connection
+ * @ctask: scsi command task
+ **/
+static int
+iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+{
+	int rc;
+	struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr;
+	struct iscsi_session *session = conn->session;
+	int datasn = be32_to_cpu(rhdr->datasn);
+
+	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
+	if (rc)
+		return rc;
+	/*
+	 * setup Data-In byte counter (gets decremented..)
+	 */
+	ctask->data_count = conn->in.datalen;
+
+	if (conn->in.datalen == 0)
+		return 0;
+
+	if (ctask->datasn != datasn)
+		return ISCSI_ERR_DATASN;
+
+	ctask->datasn++;
+
+	ctask->data_offset = be32_to_cpu(rhdr->offset);
+	if (ctask->data_offset + conn->in.datalen > ctask->total_length)
+		return ISCSI_ERR_DATA_OFFSET;
+
+	if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
+		struct scsi_cmnd *sc = ctask->sc;
+
+		conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
+		if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
+			int res_count = be32_to_cpu(rhdr->residual_count);
+
+			if (res_count > 0 &&
+			    res_count <= sc->request_bufflen) {
+				sc->resid = res_count;
+				sc->result = (DID_OK << 16) | rhdr->cmd_status;
+			} else
+				sc->result = (DID_BAD_TARGET << 16) |
+					rhdr->cmd_status;
+		} else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
+			sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
+		else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW) {
+			sc->resid = be32_to_cpu(rhdr->residual_count);
+			sc->result = (DID_OK << 16) | rhdr->cmd_status;
+		} else
+			sc->result = (DID_OK << 16) | rhdr->cmd_status;
+	}
+
+	conn->datain_pdus_cnt++;
+	return 0;
+}
+
+/**
+ * iscsi_solicit_data_init - initialize first Data-Out
+ * @conn: iscsi connection
+ * @ctask: scsi command task
+ * @r2t: R2T info
+ *
+ * Notes:
+ *	Initialize first Data-Out within this R2T sequence and finds
+ *	proper data_offset within this SCSI command.
+ *
+ *	This function is called with connection lock taken.
+ **/
+static void
+iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
+			struct iscsi_r2t_info *r2t)
+{
+	struct iscsi_data *hdr;
+	struct iscsi_data_task *dtask;
+	struct scsi_cmnd *sc = ctask->sc;
+
+	dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
+	BUG_ON(!dtask);
+	hdr = &dtask->hdr;
+	memset(hdr, 0, sizeof(struct iscsi_data));
+	hdr->ttt = r2t->ttt;
+	hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
+	r2t->solicit_datasn++;
+	hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
+	memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
+	hdr->itt = ctask->hdr.itt;
+	hdr->exp_statsn = r2t->exp_statsn;
+	hdr->offset = cpu_to_be32(r2t->data_offset);
+	if (r2t->data_length > conn->max_xmit_dlength) {
+		hton24(hdr->dlength, conn->max_xmit_dlength);
+		r2t->data_count = conn->max_xmit_dlength;
+		hdr->flags = 0;
+	} else {
+		hton24(hdr->dlength, r2t->data_length);
+		r2t->data_count = r2t->data_length;
+		hdr->flags = ISCSI_FLAG_CMD_FINAL;
+	}
+	conn->dataout_pdus_cnt++;
+
+	r2t->sent = 0;
+
+	iscsi_buf_init_hdr(conn, &r2t->headbuf, (char*)hdr,
+			   (u8 *)dtask->hdrext);
+
+	r2t->dtask = dtask;
+
+	if (sc->use_sg) {
+		int i, sg_count = 0;
+		struct scatterlist *sg = sc->request_buffer;
+
+		r2t->sg = NULL;
+		for (i = 0; i < sc->use_sg; i++, sg += 1) {
+			/* FIXME: prefetch ? */
+			if (sg_count + sg->length > r2t->data_offset) {
+				int page_offset;
+
+				/* sg page found! */
+
+				/* offset within this page */
+				page_offset = r2t->data_offset - sg_count;
+
+				/* fill in this buffer */
+				iscsi_buf_init_sg(&r2t->sendbuf, sg);
+				r2t->sendbuf.sg.offset += page_offset;
+				r2t->sendbuf.sg.length -= page_offset;
+
+				/* xmit logic will continue with next one */
+				r2t->sg = sg + 1;
+				break;
+			}
+			sg_count += sg->length;
+		}
+		BUG_ON(r2t->sg == NULL);
+	} else
+		iscsi_buf_init_iov(&ctask->sendbuf,
+			    (char*)sc->request_buffer + r2t->data_offset,
+			    r2t->data_count);
+
+	list_add(&dtask->item, &ctask->dataqueue);
+}
+
+/**
+ * iscsi_r2t_rsp - iSCSI R2T Response processing
+ * @conn: iscsi connection
+ * @ctask: scsi command task
+ **/
+static int
+iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+{
+	struct iscsi_r2t_info *r2t;
+	struct iscsi_session *session = conn->session;
+	struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr;
+	int r2tsn = be32_to_cpu(rhdr->r2tsn);
+	int rc;
+
+	if (conn->in.ahslen)
+		return ISCSI_ERR_AHSLEN;
+
+	if (conn->in.datalen)
+		return ISCSI_ERR_DATALEN;
+
+	if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn)
+		return ISCSI_ERR_R2TSN;
+
+	rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
+	if (rc)
+		return rc;
+
+	/* FIXME: use R2TSN to detect missing R2T */
+
+	/* fill-in new R2T associated with the task */
+	spin_lock(&session->lock);
+	if (!ctask->sc || ctask->mtask ||
+	     session->state != ISCSI_STATE_LOGGED_IN) {
+		printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
+		       "recovery...\n", ctask->itt);
+		spin_unlock(&session->lock);
+		return 0;
+	}
+	rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
+	BUG_ON(!rc);
+
+	r2t->exp_statsn = rhdr->statsn;
+	r2t->data_length = be32_to_cpu(rhdr->data_length);
+	if (r2t->data_length == 0 ||
+	    r2t->data_length > session->max_burst) {
+		spin_unlock(&session->lock);
+		return ISCSI_ERR_DATALEN;
+	}
+
+	r2t->data_offset = be32_to_cpu(rhdr->data_offset);
+	if (r2t->data_offset + r2t->data_length > ctask->total_length) {
+		spin_unlock(&session->lock);
+		return ISCSI_ERR_DATALEN;
+	}
+
+	r2t->ttt = rhdr->ttt; /* no flip */
+	r2t->solicit_datasn = 0;
+
+	iscsi_solicit_data_init(conn, ctask, r2t);
+
+	ctask->exp_r2tsn = r2tsn + 1;
+	ctask->xmstate |= XMSTATE_SOL_HDR;
+	__kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*));
+	__kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*));
+
+	schedule_work(&conn->xmitwork);
+	conn->r2t_pdus_cnt++;
+	spin_unlock(&session->lock);
+
+	return 0;
+}
+
+static int
+iscsi_hdr_recv(struct iscsi_conn *conn)
+{
+	int rc = 0;
+	struct iscsi_hdr *hdr;
+	struct iscsi_cmd_task *ctask;
+	struct iscsi_session *session = conn->session;
+	uint32_t cdgst, rdgst = 0;
+
+	hdr = conn->in.hdr;
+
+	/* verify PDU length */
+	conn->in.datalen = ntoh24(hdr->dlength);
+	if (conn->in.datalen > conn->max_recv_dlength) {
+		printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
+		       conn->in.datalen, conn->max_recv_dlength);
+		return ISCSI_ERR_DATALEN;
+	}
+	conn->data_copied = 0;
+
+	/* read AHS */
+	conn->in.ahslen = hdr->hlength*(4*sizeof(__u16));
+	conn->in.offset += conn->in.ahslen;
+	conn->in.copy -= conn->in.ahslen;
+	if (conn->in.copy < 0) {
+		printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
+		       "%d bytes\n", conn->in.ahslen);
+		return ISCSI_ERR_AHSLEN;
+	}
+
+	/* calculate read padding */
+	conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1);
+	if (conn->in.padding) {
+		conn->in.padding = ISCSI_PAD_LEN - conn->in.padding;
+		debug_scsi("read padding %d bytes\n", conn->in.padding);
+	}
+
+	if (conn->hdrdgst_en) {
+		struct scatterlist sg;
+
+		sg_init_one(&sg, (u8 *)hdr,
+			    sizeof(struct iscsi_hdr) + conn->in.ahslen);
+		crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
+		rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
+				     conn->in.ahslen);
+	}
+
+	/* save opcode for later */
+	conn->in.opcode = hdr->opcode;
+
+	/* verify itt (itt encoding: age+cid+itt) */
+	if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
+		if ((hdr->itt & AGE_MASK) !=
+				(session->age << AGE_SHIFT)) {
+			printk(KERN_ERR "iscsi_tcp: received itt %x expected "
+				"session age (%x)\n", hdr->itt,
+				session->age & AGE_MASK);
+			return ISCSI_ERR_BAD_ITT;
+		}
+
+		if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
+			printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
+				"CID (%x)\n", hdr->itt, conn->id);
+			return ISCSI_ERR_BAD_ITT;
+		}
+		conn->in.itt = hdr->itt & ITT_MASK;
+	} else
+		conn->in.itt = hdr->itt;
+
+	debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
+		  hdr->opcode, conn->in.offset, conn->in.copy,
+		  conn->in.ahslen, conn->in.datalen);
+
+	if (conn->in.itt < session->cmds_max) {
+		if (conn->hdrdgst_en && cdgst != rdgst) {
+			printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error "
+			       "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst,
+			       cdgst);
+			return ISCSI_ERR_HDR_DGST;
+		}
+
+		ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
+
+		if (!ctask->sc) {
+			printk(KERN_INFO "iscsi_tcp: dropping ctask with "
+			       "itt 0x%x\n", ctask->itt);
+			conn->in.datalen = 0; /* force drop */
+			return 0;
+		}
+
+		if (ctask->sc->SCp.phase != session->age) {
+			printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
+				"expected %d\n", ctask->sc->SCp.phase,
+				session->age);
+			return ISCSI_ERR_SESSION_FAILED;
+		}
+
+		conn->in.ctask = ctask;
+
+		debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
+			   hdr->opcode, conn->id, (long)ctask->sc,
+			   ctask->itt, conn->in.datalen);
+
+		switch(conn->in.opcode) {
+		case ISCSI_OP_SCSI_CMD_RSP:
+			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
+			if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE)
+				rc = iscsi_cmd_rsp(conn, ctask);
+			else if (!conn->in.datalen)
+				rc = iscsi_cmd_rsp(conn, ctask);
+			else
+				/*
+				 * got sense or response data; copying PDU
+				 * Header to the connection's header
+				 * placeholder
+				 */
+				memcpy(&conn->hdr, hdr,
+				       sizeof(struct iscsi_hdr));
+			break;
+		case ISCSI_OP_SCSI_DATA_IN:
+			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
+			/* save flags for non-exceptional status */
+			conn->in.flags = hdr->flags;
+			/* save cmd_status for sense data */
+			conn->in.cmd_status =
+				((struct iscsi_data_rsp*)hdr)->cmd_status;
+			rc = iscsi_data_rsp(conn, ctask);
+			break;
+		case ISCSI_OP_R2T:
+			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
+			if (ctask->hdr.flags & ISCSI_FLAG_CMD_WRITE &&
+			    ctask->sc->sc_data_direction == DMA_TO_DEVICE)
+				rc = iscsi_r2t_rsp(conn, ctask);
+			else
+				rc = ISCSI_ERR_PROTO;
+			break;
+		case ISCSI_OP_NOOP_IN:
+		case ISCSI_OP_TEXT_RSP:
+		case ISCSI_OP_LOGOUT_RSP:
+		case ISCSI_OP_ASYNC_EVENT:
+		case ISCSI_OP_REJECT:
+			rc = iscsi_check_assign_cmdsn(session,
+						 (struct iscsi_nopin*)hdr);
+			if (rc)
+				break;
+
+			/* update ExpStatSN */
+			conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
+			if (!conn->in.datalen) {
+				struct iscsi_mgmt_task *mtask;
+
+				rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
+						    NULL, 0);
+				mtask = (struct iscsi_mgmt_task *)
+					session->mgmt_cmds[conn->in.itt -
+						ISCSI_MGMT_ITT_OFFSET];
+				if (conn->login_mtask != mtask) {
+					spin_lock(&session->lock);
+					__kfifo_put(session->mgmtpool.queue,
+					    (void*)&mtask, sizeof(void*));
+					spin_unlock(&session->lock);
+				}
+			}
+			break;
+		default:
+			rc = ISCSI_ERR_BAD_OPCODE;
+			break;
+		}
+	} else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
+		   conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
+					session->mgmtpool_max) {
+		struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
+					session->mgmt_cmds[conn->in.itt -
+						ISCSI_MGMT_ITT_OFFSET];
+
+		debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
+			   conn->in.opcode, conn->id, mtask->itt,
+			   conn->in.datalen);
+
+		switch(conn->in.opcode) {
+		case ISCSI_OP_LOGIN_RSP:
+		case ISCSI_OP_TEXT_RSP:
+			rc = iscsi_check_assign_cmdsn(session,
+						 (struct iscsi_nopin*)hdr);
+			if (rc)
+				break;
+
+			if (!conn->in.datalen) {
+				rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
+						    NULL, 0);
+				if (conn->login_mtask != mtask) {
+					spin_lock(&session->lock);
+					__kfifo_put(session->mgmtpool.queue,
+					    (void*)&mtask, sizeof(void*));
+					spin_unlock(&session->lock);
+				}
+			}
+			break;
+		case ISCSI_OP_SCSI_TMFUNC_RSP:
+			rc = iscsi_check_assign_cmdsn(session,
+						 (struct iscsi_nopin*)hdr);
+			if (rc)
+				break;
+
+			if (conn->in.datalen || conn->in.ahslen) {
+				rc = ISCSI_ERR_PROTO;
+				break;
+			}
+			conn->tmfrsp_pdus_cnt++;
+			spin_lock(&session->lock);
+			if (conn->tmabort_state == TMABORT_INITIAL) {
+				__kfifo_put(session->mgmtpool.queue,
+						(void*)&mtask, sizeof(void*));
+				conn->tmabort_state =
+					((struct iscsi_tm_rsp *)hdr)->
+					response == SCSI_TCP_TM_RESP_COMPLETE ?
+						TMABORT_SUCCESS:TMABORT_FAILED;
+				/* unblock eh_abort() */
+				wake_up(&conn->ehwait);
+			}
+			spin_unlock(&session->lock);
+			break;
+		default:
+			rc = ISCSI_ERR_BAD_OPCODE;
+			break;
+		}
+	} else if (conn->in.itt == ISCSI_RESERVED_TAG) {
+		if (conn->in.opcode == ISCSI_OP_NOOP_IN && !conn->in.datalen) {
+			rc = iscsi_check_assign_cmdsn(session,
+						 (struct iscsi_nopin*)hdr);
+			if (!rc)
+				rc = iscsi_recv_pdu(iscsi_handle(conn),
+						    hdr, NULL, 0);
+		}
+		else
+			rc = ISCSI_ERR_BAD_OPCODE;
+	} else
+		rc = ISCSI_ERR_BAD_ITT;
+
+	return rc;
+}
+
+/**
+ * iscsi_ctask_copy - copy skb bits to the destanation cmd task
+ * @conn: iscsi connection
+ * @ctask: scsi command task
+ * @buf: buffer to copy to
+ * @buf_size: size of buffer
+ * @offset: offset within the buffer
+ *
+ * Notes:
+ *	The function calls skb_copy_bits() and updates per-connection and
+ *	per-cmd byte counters.
+ *
+ *	Read counters (in bytes):
+ *
+ *	conn->in.offset		offset within in progress SKB
+ *	conn->in.copy		left to copy from in progress SKB
+ *				including padding
+ *	conn->in.copied		copied already from in progress SKB
+ *	conn->data_copied	copied already from in progress buffer
+ *	ctask->sent		total bytes sent up to the MidLayer
+ *	ctask->data_count	left to copy from in progress Data-In
+ *	buf_left		left to copy from in progress buffer
+ **/
+static inline int
+iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
+		void *buf, int buf_size, int offset)
+{
+	int buf_left = buf_size - (conn->data_copied + offset);
+	int size = min(conn->in.copy, buf_left);
+	int rc;
+
+	size = min(size, ctask->data_count);
+
+	debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
+	       size, conn->in.offset, conn->in.copied);
+
+	BUG_ON(size <= 0);
+	BUG_ON(ctask->sent + size > ctask->total_length);
+
+	rc = skb_copy_bits(conn->in.skb, conn->in.offset,
+			   (char*)buf + (offset + conn->data_copied), size);
+	/* must fit into skb->len */
+	BUG_ON(rc);
+
+	conn->in.offset += size;
+	conn->in.copy -= size;
+	conn->in.copied += size;
+	conn->data_copied += size;
+	ctask->sent += size;
+	ctask->data_count -= size;
+
+	BUG_ON(conn->in.copy < 0);
+	BUG_ON(ctask->data_count < 0);
+
+	if (buf_size != (conn->data_copied + offset)) {
+		if (!ctask->data_count) {
+			BUG_ON(buf_size - conn->data_copied < 0);
+			/* done with this PDU */
+			return buf_size - conn->data_copied;
+		}
+		return -EAGAIN;
+	}
+
+	/* done with this buffer or with both - PDU and buffer */
+	conn->data_copied = 0;
+	return 0;
+}
+
+/**
+ * iscsi_tcp_copy - copy skb bits to the destanation buffer
+ * @conn: iscsi connection
+ * @buf: buffer to copy to
+ * @buf_size: number of bytes to copy
+ *
+ * Notes:
+ *	The function calls skb_copy_bits() and updates per-connection
+ *	byte counters.
+ **/
+static inline int
+iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size)
+{
+	int buf_left = buf_size - conn->data_copied;
+	int size = min(conn->in.copy, buf_left);
+	int rc;
+
+	debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
+	       size, conn->in.offset, conn->data_copied);
+	BUG_ON(size <= 0);
+
+	rc = skb_copy_bits(conn->in.skb, conn->in.offset,
+			   (char*)buf + conn->data_copied, size);
+	BUG_ON(rc);
+
+	conn->in.offset += size;
+	conn->in.copy -= size;
+	conn->in.copied += size;
+	conn->data_copied += size;
+
+	if (buf_size != conn->data_copied)
+		return -EAGAIN;
+
+	return 0;
+}
+
+static inline void
+partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg,
+			 int offset, int length)
+{
+	struct scatterlist temp;
+
+	memcpy(&temp, sg, sizeof(struct scatterlist));
+	temp.offset = offset;
+	temp.length = length;
+	crypto_digest_update(conn->data_rx_tfm, &temp, 1);
+}
+
+static int iscsi_scsi_data_in(struct iscsi_conn *conn)
+{
+	struct iscsi_cmd_task *ctask = conn->in.ctask;
+	struct scsi_cmnd *sc = ctask->sc;
+	struct scatterlist tmp, *sg;
+	int i, offset, rc = 0;
+
+	BUG_ON((void*)ctask != sc->SCp.ptr);
+
+	/*
+	 * copying Data-In into the Scsi_Cmnd
+	 */
+	if (!sc->use_sg) {
+		i = ctask->data_count;
+		rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer,
+				      sc->request_bufflen, ctask->data_offset);
+		if (rc == -EAGAIN)
+			return rc;
+		if (conn->datadgst_en) {
+			sg_init_one(&tmp, sc->request_buffer, i);
+			crypto_digest_update(conn->data_rx_tfm, &tmp, 1);
+		}
+		rc = 0;
+		goto done;
+	}
+
+	offset = ctask->data_offset;
+	sg = sc->request_buffer;
+
+	if (ctask->data_offset)
+		for (i = 0; i < ctask->sg_count; i++)
+			offset -= sg[i].length;
+	/* we've passed through partial sg*/
+	if (offset < 0)
+		offset = 0;
+
+	for (i = ctask->sg_count; i < sc->use_sg; i++) {
+		char *dest;
+
+		dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
+		rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset,
+				      sg[i].length, offset);
+		kunmap_atomic(dest, KM_SOFTIRQ0);
+		if (rc == -EAGAIN)
+			/* continue with the next SKB/PDU */
+			return rc;
+		if (!rc) {
+			if (conn->datadgst_en) {
+				if (!offset)
+					crypto_digest_update(conn->data_rx_tfm,
+							     &sg[i], 1);
+				else
+					partial_sg_digest_update(conn, &sg[i],
+							sg[i].offset + offset,
+							sg[i].length - offset);
+			}
+			offset = 0;
+			ctask->sg_count++;
+		}
+
+		if (!ctask->data_count) {
+			if (rc && conn->datadgst_en)
+				/*
+				 * data-in is complete, but buffer not...
+				 */
+				partial_sg_digest_update(conn, &sg[i],
+						sg[i].offset, sg[i].length-rc);
+			rc = 0;
+			break;
+		}
+
+		if (!conn->in.copy)
+			return -EAGAIN;
+	}
+	BUG_ON(ctask->data_count);
+
+done:
+	/* check for non-exceptional status */
+	if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) {
+		debug_scsi("done [sc %lx res %d itt 0x%x]\n",
+			   (long)sc, sc->result, ctask->itt);
+		conn->scsirsp_pdus_cnt++;
+		iscsi_ctask_cleanup(conn, ctask);
+		sc->scsi_done(sc);
+	}
+
+	return rc;
+}
+
+static int
+iscsi_data_recv(struct iscsi_conn *conn)
+{
+	struct iscsi_session *session = conn->session;
+	int rc = 0;
+
+	switch(conn->in.opcode) {
+	case ISCSI_OP_SCSI_DATA_IN:
+		rc = iscsi_scsi_data_in(conn);
+		break;
+	case ISCSI_OP_SCSI_CMD_RSP: {
+		/*
+		 * SCSI Sense Data:
+		 * copying the entire Data Segment.
+		 */
+		if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
+			rc = -EAGAIN;
+			goto exit;
+		}
+
+		/*
+		 * check for sense
+		 */
+		conn->in.hdr = &conn->hdr;
+		conn->senselen = (conn->data[0] << 8) | conn->data[1];
+		rc = iscsi_cmd_rsp(conn, conn->in.ctask);
+	}
+	break;
+	case ISCSI_OP_TEXT_RSP:
+	case ISCSI_OP_LOGIN_RSP:
+	case ISCSI_OP_NOOP_IN: {
+		struct iscsi_mgmt_task *mtask = NULL;
+
+		if (conn->in.itt != ISCSI_RESERVED_TAG)
+			mtask = (struct iscsi_mgmt_task *)
+				session->mgmt_cmds[conn->in.itt -
+					ISCSI_MGMT_ITT_OFFSET];
+
+		/*
+		 * Collect data segment to the connection's data
+		 * placeholder
+		 */
+		if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
+			rc = -EAGAIN;
+			goto exit;
+		}
+
+		rc = iscsi_recv_pdu(iscsi_handle(conn), conn->in.hdr,
+				    conn->data, conn->in.datalen);
+
+		if (mtask && conn->login_mtask != mtask) {
+			spin_lock(&session->lock);
+			__kfifo_put(session->mgmtpool.queue, (void*)&mtask,
+				    sizeof(void*));
+			spin_unlock(&session->lock);
+		}
+	}
+	break;
+	default:
+		BUG_ON(1);
+	}
+exit:
+	return rc;
+}
+
+/**
+ * iscsi_tcp_data_recv - TCP receive in sendfile fashion
+ * @rd_desc: read descriptor
+ * @skb: socket buffer
+ * @offset: offset in skb
+ * @len: skb->len - offset
+ **/
+static int
+iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
+		unsigned int offset, size_t len)
+{
+	int rc;
+	struct iscsi_conn *conn = rd_desc->arg.data;
+	int start = skb_headlen(skb);
+	int processed;
+	char pad[ISCSI_PAD_LEN];
+	struct scatterlist sg;
+
+	/*
+	 * Save current SKB and its offset in the corresponding
+	 * connection context.
+	 */
+	conn->in.copy = start - offset;
+	conn->in.offset = offset;
+	conn->in.skb = skb;
+	conn->in.len = conn->in.copy;
+	BUG_ON(conn->in.copy <= 0);
+	debug_tcp("in %d bytes\n", conn->in.copy);
+
+more:
+	conn->in.copied = 0;
+	rc = 0;
+
+	if (unlikely(conn->suspend_rx)) {
+		debug_tcp("conn %d Rx suspended!\n", conn->id);
+		return 0;
+	}
+
+	if (conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
+	    conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
+		rc = iscsi_hdr_extract(conn);
+		if (rc) {
+		       if (rc == -EAGAIN)
+				goto nomore;
+		       else {
+				iscsi_conn_failure(conn, rc);
+				return 0;
+		       }
+		}
+
+		/*
+		 * Verify and process incoming PDU header.
+		 */
+		rc = iscsi_hdr_recv(conn);
+		if (!rc && conn->in.datalen) {
+			if (conn->datadgst_en &&
+			    conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
+				BUG_ON(!conn->data_rx_tfm);
+				crypto_digest_init(conn->data_rx_tfm);
+			}
+			conn->in_progress = IN_PROGRESS_DATA_RECV;
+		} else if (rc) {
+			iscsi_conn_failure(conn, rc);
+			return 0;
+		}
+	}
+
+	if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
+		debug_tcp("extra data_recv offset %d copy %d\n",
+			  conn->in.offset, conn->in.copy);
+		if (conn->in.opcode == ISCSI_OP_SCSI_DATA_IN) {
+			uint32_t recv_digest;
+			skb_copy_bits(conn->in.skb, conn->in.offset,
+				      &recv_digest, 4);
+			conn->in.offset += 4;
+			conn->in.copy -= 4;
+			if (recv_digest != conn->in.datadgst) {
+				debug_tcp("iscsi_tcp: data digest error!"
+					  "0x%x != 0x%x\n", recv_digest,
+					  conn->in.datadgst);
+				iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
+				return 0;
+			} else {
+				debug_tcp("iscsi_tcp: data digest match!"
+					  "0x%x == 0x%x\n", recv_digest,
+					  conn->in.datadgst);
+				conn->in_prog