20 files changed, 6157 insertions, 154 deletions

diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index e54185223c8..c64c3807f51 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -91,6 +91,8 @@ config CRYPTO_SHA1_S390
 	  This is the s390 hardware accelerated implementation of the
 	  SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2).
 
+	  It is available as of z990.
+
 config CRYPTO_SHA256_S390
 	tristate "SHA256 digest algorithm"
 	depends on S390
@@ -99,8 +101,7 @@ config CRYPTO_SHA256_S390
 	  This is the s390 hardware accelerated implementation of the
 	  SHA256 secure hash standard (DFIPS 180-2).
 
-	  This version of SHA implements a 256 bit hash with 128 bits of
-	  security against collision attacks.
+	  It is available as of z9.
 
 config CRYPTO_SHA512_S390
 	tristate "SHA384 and SHA512 digest algorithm"
@@ -110,10 +111,7 @@ config CRYPTO_SHA512_S390
 	  This is the s390 hardware accelerated implementation of the
 	  SHA512 secure hash standard.
 
-	  This version of SHA implements a 512 bit hash with 256 bits of
-	  security against collision attacks. The code also includes SHA-384,
-	  a 384 bit hash with 192 bits of security against collision attacks.
-
+	  It is available as of z10.
 
 config CRYPTO_DES_S390
 	tristate "DES and Triple DES cipher algorithms"
@@ -121,9 +119,12 @@ config CRYPTO_DES_S390
 	select CRYPTO_ALGAPI
 	select CRYPTO_BLKCIPHER
 	help
-	  This us the s390 hardware accelerated implementation of the
+	  This is the s390 hardware accelerated implementation of the
 	  DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3).
 
+	  As of z990 the ECB and CBC mode are hardware accelerated.
+	  As of z196 the CTR mode is hardware accelerated.
+
 config CRYPTO_AES_S390
 	tristate "AES cipher algorithms"
 	depends on S390
@@ -131,20 +132,15 @@ config CRYPTO_AES_S390
 	select CRYPTO_BLKCIPHER
 	help
 	  This is the s390 hardware accelerated implementation of the
-	  AES cipher algorithms (FIPS-197). AES uses the Rijndael
-	  algorithm.
-
-	  Rijndael appears to be consistently a very good performer in
-	  both hardware and software across a wide range of computing
-	  environments regardless of its use in feedback or non-feedback
-	  modes. Its key setup time is excellent, and its key agility is
-	  good. Rijndael's very low memory requirements make it very well
-	  suited for restricted-space environments, in which it also
-	  demonstrates excellent performance. Rijndael's operations are
-	  among the easiest to defend against power and timing attacks.
+	  AES cipher algorithms (FIPS-197).
 
-	  On s390 the System z9-109 currently only supports the key size
-	  of 128 bit.
+	  As of z9 the ECB and CBC modes are hardware accelerated
+	  for 128 bit keys.
+	  As of z10 the ECB and CBC modes are hardware accelerated
+	  for all AES key sizes.
+	  As of z196 the CTR mode is hardware accelerated for all AES
+	  key sizes and XTS mode is hardware accelerated for 256 and
+	  512 bit keys.
 
 config S390_PRNG
 	tristate "Pseudo random number generator device driver"
@@ -154,8 +150,20 @@ config S390_PRNG
 	  Select this option if you want to use the s390 pseudo random number
 	  generator. The PRNG is part of the cryptographic processor functions
 	  and uses triple-DES to generate secure random numbers like the
-	  ANSI X9.17 standard. The PRNG is usable via the char device
-	  /dev/prandom.
+	  ANSI X9.17 standard. User-space programs access the
+	  pseudo-random-number device through the char device /dev/prandom.
+
+	  It is available as of z9.
+
+config CRYPTO_GHASH_S390
+	tristate "GHASH digest algorithm"
+	depends on S390
+	select CRYPTO_HASH
+	help
+	  This is the s390 hardware accelerated implementation of the
+	  GHASH message digest algorithm for GCM (Galois/Counter Mode).
+
+	  It is available as of z196.
 
 config CRYPTO_DEV_MV_CESA
 	tristate "Marvell's Cryptographic Engine"
@@ -200,6 +208,8 @@ config CRYPTO_DEV_HIFN_795X_RNG
 	  Select this option if you want to enable the random number generator
 	  on the HIFN 795x crypto adapters.
 
+source drivers/crypto/caam/Kconfig
+
 config CRYPTO_DEV_TALITOS
 	tristate "Talitos Freescale Security Engine (SEC)"
 	select CRYPTO_ALGAPI
@@ -269,4 +279,15 @@ config CRYPTO_DEV_PICOXCELL
 
 	  Saying m here will build a module named pipcoxcell_crypto.
 
+config CRYPTO_DEV_S5P
+	tristate "Support for Samsung S5PV210 crypto accelerator"
+	depends on ARCH_S5PV210
+	select CRYPTO_AES
+	select CRYPTO_ALGAPI
+	select CRYPTO_BLKCIPHER
+	help
+	  This option allows you to have support for S5P crypto acceleration.
+	  Select this to offload Samsung S5PV210 or S5PC110 from AES
+	  algorithms execution.
+
 endif # CRYPTO_HW
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 5203e34248d..53ea5015531 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -6,8 +6,10 @@ n2_crypto-y := n2_core.o n2_asm.o
 obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
 obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o
 obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
 obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
 obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/
 obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o
 obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o
 obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o
+obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig
new file mode 100644
index 00000000000..2d876bb98ff
--- /dev/null
+++ b/drivers/crypto/caam/Kconfig
@@ -0,0 +1,72 @@
+config CRYPTO_DEV_FSL_CAAM
+	tristate "Freescale CAAM-Multicore driver backend"
+	depends on FSL_SOC
+	help
+	  Enables the driver module for Freescale's Cryptographic Accelerator
+	  and Assurance Module (CAAM), also known as the SEC version 4 (SEC4).
+	  This module adds a job ring operation interface, and configures h/w
+	  to operate as a DPAA component automatically, depending
+	  on h/w feature availability.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called caam.
+
+config CRYPTO_DEV_FSL_CAAM_RINGSIZE
+	int "Job Ring size"
+	depends on CRYPTO_DEV_FSL_CAAM
+	range 2 9
+	default "9"
+	help
+	  Select size of Job Rings as a power of 2, within the
+	  range 2-9 (ring size 4-512).
+	  Examples:
+		2 => 4
+		3 => 8
+		4 => 16
+		5 => 32
+		6 => 64
+		7 => 128
+		8 => 256
+		9 => 512
+
+config CRYPTO_DEV_FSL_CAAM_INTC
+	bool "Job Ring interrupt coalescing"
+	depends on CRYPTO_DEV_FSL_CAAM
+	default y
+	help
+	  Enable the Job Ring's interrupt coalescing feature.
+
+config CRYPTO_DEV_FSL_CAAM_INTC_COUNT_THLD
+	int "Job Ring interrupt coalescing count threshold"
+	depends on CRYPTO_DEV_FSL_CAAM_INTC
+	range 1 255
+	default 255
+	help
+	  Select number of descriptor completions to queue before
+	  raising an interrupt, in the range 1-255. Note that a selection
+	  of 1 functionally defeats the coalescing feature, and a selection
+	  equal or greater than the job ring size will force timeouts.
+
+config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
+	int "Job Ring interrupt coalescing timer threshold"
+	depends on CRYPTO_DEV_FSL_CAAM_INTC
+	range 1 65535
+	default 2048
+	help
+	  Select number of bus clocks/64 to timeout in the case that one or
+	  more descriptor completions are queued without reaching the count
+	  threshold. Range is 1-65535.
+
+config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
+	tristate "Register algorithm implementations with the Crypto API"
+	depends on CRYPTO_DEV_FSL_CAAM
+	default y
+	select CRYPTO_ALGAPI
+	select CRYPTO_AUTHENC
+	help
+	  Selecting this will offload crypto for users of the
+	  scatterlist crypto API (such as the linux native IPSec
+	  stack) to the SEC4 via job ring.
+
+	  To compile this as a module, choose M here: the module
+	  will be called caamalg.
diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile
new file mode 100644
index 00000000000..ef39011b450
--- /dev/null
+++ b/drivers/crypto/caam/Makefile
@@ -0,0 +1,8 @@
+#
+# Makefile for the CAAM backend and dependent components
+#
+
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
+
+caam-objs := ctrl.o jr.o error.o
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
new file mode 100644
index 00000000000..d0e65d6ddc7
--- /dev/null
+++ b/drivers/crypto/caam/caamalg.c
@@ -0,0 +1,1268 @@
+/*
+ * caam - Freescale FSL CAAM support for crypto API
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Based on talitos crypto API driver.
+ *
+ * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
+ *
+ * ---------------                     ---------------
+ * | JobDesc #1  |-------------------->|  ShareDesc  |
+ * | *(packet 1) |                     |   (PDB)     |
+ * ---------------      |------------->|  (hashKey)  |
+ *       .              |              | (cipherKey) |
+ *       .              |    |-------->| (operation) |
+ * ---------------      |    |         ---------------
+ * | JobDesc #2  |------|    |
+ * | *(packet 2) |           |
+ * ---------------           |
+ *       .                   |
+ *       .                   |
+ * ---------------           |
+ * | JobDesc #3  |------------
+ * | *(packet 3) |
+ * ---------------
+ *
+ * The SharedDesc never changes for a connection unless rekeyed, but
+ * each packet will likely be in a different place. So all we need
+ * to know to process the packet is where the input is, where the
+ * output goes, and what context we want to process with. Context is
+ * in the SharedDesc, packet references in the JobDesc.
+ *
+ * So, a job desc looks like:
+ *
+ * ---------------------
+ * | Header            |
+ * | ShareDesc Pointer |
+ * | SEQ_OUT_PTR       |
+ * | (output buffer)   |
+ * | SEQ_IN_PTR        |
+ * | (input buffer)    |
+ * | LOAD (to DECO)    |
+ * ---------------------
+ */
+
+#include "compat.h"
+
+#include "regs.h"
+#include "intern.h"
+#include "desc_constr.h"
+#include "jr.h"
+#include "error.h"
+
+/*
+ * crypto alg
+ */
+#define CAAM_CRA_PRIORITY		3000
+/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
+#define CAAM_MAX_KEY_SIZE		(AES_MAX_KEY_SIZE + \
+					 SHA512_DIGEST_SIZE * 2)
+/* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
+#define CAAM_MAX_IV_LENGTH		16
+
+/* length of descriptors text */
+#define DESC_AEAD_SHARED_TEXT_LEN	4
+#define DESC_AEAD_ENCRYPT_TEXT_LEN 	21
+#define DESC_AEAD_DECRYPT_TEXT_LEN 	24
+#define DESC_AEAD_GIVENCRYPT_TEXT_LEN 	27
+
+#ifdef DEBUG
+/* for print_hex_dumps with line references */
+#define xstr(s) str(s)
+#define str(s) #s
+#define debug(format, arg...) printk(format, arg)
+#else
+#define debug(format, arg...)
+#endif
+
+/*
+ * per-session context
+ */
+struct caam_ctx {
+	struct device *jrdev;
+	u32 *sh_desc;
+	dma_addr_t shared_desc_phys;
+	u32 class1_alg_type;
+	u32 class2_alg_type;
+	u32 alg_op;
+	u8 *key;
+	dma_addr_t key_phys;
+	unsigned int enckeylen;
+	unsigned int split_key_len;
+	unsigned int split_key_pad_len;
+	unsigned int authsize;
+};
+
+static int aead_authenc_setauthsize(struct crypto_aead *authenc,
+				    unsigned int authsize)
+{
+	struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+
+	ctx->authsize = authsize;
+
+	return 0;
+}
+
+struct split_key_result {
+	struct completion completion;
+	int err;
+};
+
+static void split_key_done(struct device *dev, u32 *desc, u32 err,
+			   void *context)
+{
+	struct split_key_result *res = context;
+
+#ifdef DEBUG
+	dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
+#endif
+	if (err) {
+		char tmp[CAAM_ERROR_STR_MAX];
+
+		dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
+	}
+
+	res->err = err;
+
+	complete(&res->completion);
+}
+
+/*
+get a split ipad/opad key
+
+Split key generation-----------------------------------------------
+
+[00] 0xb0810008    jobdesc: stidx=1 share=never len=8
+[01] 0x04000014        key: class2->keyreg len=20
+			@0xffe01000
+[03] 0x84410014  operation: cls2-op sha1 hmac init dec
+[04] 0x24940000     fifold: class2 msgdata-last2 len=0 imm
+[05] 0xa4000001       jump: class2 local all ->1 [06]
+[06] 0x64260028    fifostr: class2 mdsplit-jdk len=40
+			@0xffe04000
+*/
+static u32 gen_split_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen)
+{
+	struct device *jrdev = ctx->jrdev;
+	u32 *desc;
+	struct split_key_result result;
+	dma_addr_t dma_addr_in, dma_addr_out;
+	int ret = 0;
+
+	desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
+
+	init_job_desc(desc, 0);
+
+	dma_addr_in = dma_map_single(jrdev, (void *)key_in, authkeylen,
+				     DMA_TO_DEVICE);
+	if (dma_mapping_error(jrdev, dma_addr_in)) {
+		dev_err(jrdev, "unable to map key input memory\n");
+		kfree(desc);
+		return -ENOMEM;
+	}
+	append_key(desc, dma_addr_in, authkeylen, CLASS_2 |
+		       KEY_DEST_CLASS_REG);
+
+	/* Sets MDHA up into an HMAC-INIT */
+	append_operation(desc, ctx->alg_op | OP_ALG_DECRYPT |
+			     OP_ALG_AS_INIT);
+
+	/*
+	 * do a FIFO_LOAD of zero, this will trigger the internal key expansion
+	   into both pads inside MDHA
+	 */
+	append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB |
+				FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2);
+
+	/*
+	 * FIFO_STORE with the explicit split-key content store
+	 * (0x26 output type)
+	 */
+	dma_addr_out = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len,
+				      DMA_FROM_DEVICE);
+	if (dma_mapping_error(jrdev, dma_addr_out)) {
+		dev_err(jrdev, "unable to map key output memory\n");
+		kfree(desc);
+		return -ENOMEM;
+	}
+	append_fifo_store(desc, dma_addr_out, ctx->split_key_len,
+			  LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, key_in, authkeylen, 1);
+	print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+
+	result.err = 0;
+	init_completion(&result.completion);
+
+	ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
+	if (!ret) {
+		/* in progress */
+		wait_for_completion_interruptible(&result.completion);
+		ret = result.err;
+#ifdef DEBUG
+		print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+			       DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
+			       ctx->split_key_pad_len, 1);
+#endif
+	}
+
+	dma_unmap_single(jrdev, dma_addr_out, ctx->split_key_pad_len,
+			 DMA_FROM_DEVICE);
+	dma_unmap_single(jrdev, dma_addr_in, authkeylen, DMA_TO_DEVICE);
+
+	kfree(desc);
+
+	return ret;
+}
+
+static int build_sh_desc_ipsec(struct caam_ctx *ctx)
+{
+	struct device *jrdev = ctx->jrdev;
+	u32 *sh_desc;
+	u32 *jump_cmd;
+	bool keys_fit_inline = 0;
+
+	/*
+	 * largest Job Descriptor and its Shared Descriptor
+	 * must both fit into the 64-word Descriptor h/w Buffer
+	 */
+	if ((DESC_AEAD_GIVENCRYPT_TEXT_LEN +
+	     DESC_AEAD_SHARED_TEXT_LEN) * CAAM_CMD_SZ +
+	    ctx->split_key_pad_len + ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
+		keys_fit_inline = 1;
+
+	/* build shared descriptor for this session */
+	sh_desc = kmalloc(CAAM_CMD_SZ * DESC_AEAD_SHARED_TEXT_LEN +
+			  keys_fit_inline ?
+			  ctx->split_key_pad_len + ctx->enckeylen :
+			  CAAM_PTR_SZ * 2, GFP_DMA | GFP_KERNEL);
+	if (!sh_desc) {
+		dev_err(jrdev, "could not allocate shared descriptor\n");
+		return -ENOMEM;
+	}
+
+	init_sh_desc(sh_desc, HDR_SAVECTX | HDR_SHARE_SERIAL);
+
+	jump_cmd = append_jump(sh_desc, CLASS_BOTH | JUMP_TEST_ALL |
+			       JUMP_COND_SHRD | JUMP_COND_SELF);
+
+	/*
+	 * process keys, starting with class 2/authentication.
+	 */
+	if (keys_fit_inline) {
+		append_key_as_imm(sh_desc, ctx->key, ctx->split_key_pad_len,
+				  ctx->split_key_len,
+				  CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
+
+		append_key_as_imm(sh_desc, (void *)ctx->key +
+				  ctx->split_key_pad_len, ctx->enckeylen,
+				  ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
+	} else {
+		append_key(sh_desc, ctx->key_phys, ctx->split_key_len, CLASS_2 |
+			   KEY_DEST_MDHA_SPLIT | KEY_ENC);
+		append_key(sh_desc, ctx->key_phys + ctx->split_key_pad_len,
+			   ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
+	}
+
+	/* update jump cmd now that we are at the jump target */
+	set_jump_tgt_here(sh_desc, jump_cmd);
+
+	ctx->shared_desc_phys = dma_map_single(jrdev, sh_desc,
+					       desc_bytes(sh_desc),
+					       DMA_TO_DEVICE);
+	if (dma_mapping_error(jrdev, ctx->shared_desc_phys)) {
+		dev_err(jrdev, "unable to map shared descriptor\n");
+		kfree(sh_desc);
+		return -ENOMEM;
+	}
+
+	ctx->sh_desc = sh_desc;
+
+	return 0;
+}
+
+static int aead_authenc_setkey(struct crypto_aead *aead,
+			       const u8 *key, unsigned int keylen)
+{
+	/* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
+	static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
+	struct caam_ctx *ctx = crypto_aead_ctx(aead);
+	struct device *jrdev = ctx->jrdev;
+	struct rtattr *rta = (void *)key;
+	struct crypto_authenc_key_param *param;
+	unsigned int authkeylen;
+	unsigned int enckeylen;
+	int ret = 0;
+
+	param = RTA_DATA(rta);
+	enckeylen = be32_to_cpu(param->enckeylen);
+
+	key += RTA_ALIGN(rta->rta_len);
+	keylen -= RTA_ALIGN(rta->rta_len);
+
+	if (keylen < enckeylen)
+		goto badkey;
+
+	authkeylen = keylen - enckeylen;
+
+	if (keylen > CAAM_MAX_KEY_SIZE)
+		goto badkey;
+
+	/* Pick class 2 key length from algorithm submask */
+	ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
+				      OP_ALG_ALGSEL_SHIFT] * 2;
+	ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
+
+#ifdef DEBUG
+	printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
+	       keylen, enckeylen, authkeylen);
+	printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
+	       ctx->split_key_len, ctx->split_key_pad_len);
+	print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+#endif
+	ctx->key = kmalloc(ctx->split_key_pad_len + enckeylen,
+			   GFP_KERNEL | GFP_DMA);
+	if (!ctx->key) {
+		dev_err(jrdev, "could not allocate key output memory\n");
+		return -ENOMEM;
+	}
+
+	ret = gen_split_key(ctx, key, authkeylen);
+	if (ret) {
+		kfree(ctx->key);
+		goto badkey;
+	}
+
+	/* postpend encryption key to auth split key */
+	memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen);
+
+	ctx->key_phys = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
+				       enckeylen, DMA_TO_DEVICE);
+	if (dma_mapping_error(jrdev, ctx->key_phys)) {
+		dev_err(jrdev, "unable to map key i/o memory\n");
+		kfree(ctx->key);
+		return -ENOMEM;
+	}
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
+		       ctx->split_key_pad_len + enckeylen, 1);
+#endif
+
+	ctx->enckeylen = enckeylen;
+
+	ret = build_sh_desc_ipsec(ctx);
+	if (ret) {
+		dma_unmap_single(jrdev, ctx->key_phys, ctx->split_key_pad_len +
+				 enckeylen, DMA_TO_DEVICE);
+		kfree(ctx->key);
+	}
+
+	return ret;
+badkey:
+	crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+	return -EINVAL;
+}
+
+struct link_tbl_entry {
+	u64 ptr;
+	u32 len;
+	u8 reserved;
+	u8 buf_pool_id;
+	u16 offset;
+};
+
+/*
+ * ipsec_esp_edesc - s/w-extended ipsec_esp descriptor
+ * @src_nents: number of segments in input scatterlist
+ * @dst_nents: number of segments in output scatterlist
+ * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
+ * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
+ * @link_tbl_bytes: length of dma mapped link_tbl space
+ * @link_tbl_dma: bus physical mapped address of h/w link table
+ * @hw_desc: the h/w job descriptor followed by any referenced link tables
+ */
+struct ipsec_esp_edesc {
+	int assoc_nents;
+	int src_nents;
+	int dst_nents;
+	int link_tbl_bytes;
+	dma_addr_t link_tbl_dma;
+	struct link_tbl_entry *link_tbl;
+	u32 hw_desc[0];
+};
+
+static void ipsec_esp_unmap(struct device *dev,
+			    struct ipsec_esp_edesc *edesc,
+			    struct aead_request *areq)
+{
+	dma_unmap_sg(dev, areq->assoc, edesc->assoc_nents, DMA_TO_DEVICE);
+
+	if (unlikely(areq->dst != areq->src)) {
+		dma_unmap_sg(dev, areq->src, edesc->src_nents,
+			     DMA_TO_DEVICE);
+		dma_unmap_sg(dev, areq->dst, edesc->dst_nents,
+			     DMA_FROM_DEVICE);
+	} else {
+		dma_unmap_sg(dev, areq->src, edesc->src_nents,
+			     DMA_BIDIRECTIONAL);
+	}
+
+	if (edesc->link_tbl_bytes)
+		dma_unmap_single(dev, edesc->link_tbl_dma,
+				 edesc->link_tbl_bytes,
+				 DMA_TO_DEVICE);
+}
+
+/*
+ * ipsec_esp descriptor callbacks
+ */
+static void ipsec_esp_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
+				   void *context)
+{
+	struct aead_request *areq = context;
+	struct ipsec_esp_edesc *edesc;
+#ifdef DEBUG
+	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
+	int ivsize = crypto_aead_ivsize(aead);
+	struct caam_ctx *ctx = crypto_aead_ctx(aead);
+
+	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
+#endif
+	edesc = (struct ipsec_esp_edesc *)((char *)desc -
+		 offsetof(struct ipsec_esp_edesc, hw_desc));
+
+	if (err) {
+		char tmp[CAAM_ERROR_STR_MAX];
+
+		dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
+	}
+
+	ipsec_esp_unmap(jrdev, edesc, areq);
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR, "assoc  @"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc),
+		       areq->assoclen , 1);
+	print_hex_dump(KERN_ERR, "dstiv  @"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize,
+		       edesc->src_nents ? 100 : ivsize, 1);
+	print_hex_dump(KERN_ERR, "dst    @"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src),
+		       edesc->src_nents ? 100 : areq->cryptlen +
+		       ctx->authsize + 4, 1);
+#endif
+
+	kfree(edesc);
+
+	aead_request_complete(areq, err);
+}
+
+static void ipsec_esp_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
+				   void *context)
+{
+	struct aead_request *areq = context;
+	struct ipsec_esp_edesc *edesc;
+#ifdef DEBUG
+	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
+	struct caam_ctx *ctx = crypto_aead_ctx(aead);
+
+	dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
+#endif
+	edesc = (struct ipsec_esp_edesc *)((char *)desc -
+		 offsetof(struct ipsec_esp_edesc, hw_desc));
+
+	if (err) {
+		char tmp[CAAM_ERROR_STR_MAX];
+
+		dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
+	}
+
+	ipsec_esp_unmap(jrdev, edesc, areq);
+
+	/*
+	 * verify hw auth check passed else return -EBADMSG
+	 */
+	if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
+		err = -EBADMSG;
+
+#ifdef DEBUG
+	print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4,
+		       ((char *)sg_virt(areq->assoc) - sizeof(struct iphdr)),
+		       sizeof(struct iphdr) + areq->assoclen +
+		       ((areq->cryptlen > 1500) ? 1500 : areq->cryptlen) +
+		       ctx->authsize + 36, 1);
+	if (!err && edesc->link_tbl_bytes) {
+		struct scatterlist *sg = sg_last(areq->src, edesc->src_nents);
+		print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ",
+			       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
+			sg->length + ctx->authsize + 16, 1);
+	}
+#endif
+	kfree(edesc);
+
+	aead_request_complete(areq, err);
+}
+
+/*
+ * convert scatterlist to h/w link table format
+ * scatterlist must have been previously dma mapped
+ */
+static void sg_to_link_tbl(struct scatterlist *sg, int sg_count,
+			   struct link_tbl_entry *link_tbl_ptr, u32 offset)
+{
+	while (sg_count) {
+		link_tbl_ptr->ptr = sg_dma_address(sg);
+		link_tbl_ptr->len = sg_dma_len(sg);
+		link_tbl_ptr->reserved = 0;
+		link_tbl_ptr->buf_pool_id = 0;
+		link_tbl_ptr->offset = offset;
+		link_tbl_ptr++;
+		sg = sg_next(sg);
+		sg_count--;
+	}
+
+	/* set Final bit (marks end of link table) */
+	link_tbl_ptr--;
+	link_tbl_ptr->len |= 0x40000000;
+}
+
+/*
+ * fill in and submit ipsec_esp job descriptor
+ */
+static int ipsec_esp(struct ipsec_esp_edesc *edesc, struct aead_request *areq,
+		     u32 encrypt,
+		     void (*callback) (struct device *dev, u32 *desc,
+				       u32 err, void *context))
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
+	struct caam_ctx *ctx = crypto_aead_ctx(aead);
+	struct device *jrdev = ctx->jrdev;
+	u32 *desc = edesc->hw_desc, options;
+	int ret, sg_count, assoc_sg_count;
+	int ivsize = crypto_aead_ivsize(aead);
+	int authsize = ctx->authsize;
+	dma_addr_t ptr, dst_dma, src_dma;
+#ifdef DEBUG
+	u32 *sh_desc = ctx->sh_desc;
+
+	debug("assoclen %d cryptlen %d authsize %d\n",
+	      areq->assoclen, areq->cryptlen, authsize);
+	print_hex_dump(KERN_ERR, "assoc  @"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->assoc),
+		       areq->assoclen , 1);
+	print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src) - ivsize,
+		       edesc->src_nents ? 100 : ivsize, 1);
+	print_hex_dump(KERN_ERR, "src    @"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(areq->src),
+			edesc->src_nents ? 100 : areq->cryptlen + authsize, 1);
+	print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
+		       desc_bytes(sh_desc), 1);
+#endif
+	assoc_sg_count = dma_map_sg(jrdev, areq->assoc, edesc->assoc_nents ?: 1,
+				    DMA_TO_DEVICE);
+	if (areq->src == areq->dst)
+		sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1,
+				      DMA_BIDIRECTIONAL);
+	else
+		sg_count = dma_map_sg(jrdev, areq->src, edesc->src_nents ? : 1,
+				      DMA_TO_DEVICE);
+
+	/* start auth operation */
+	append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL |
+			 (encrypt ? : OP_ALG_ICV_ON));
+
+	/* Load FIFO with data for Class 2 CHA */
+	options = FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG;
+	if (!edesc->assoc_nents) {
+		ptr = sg_dma_address(areq->assoc);
+	} else {
+		sg_to_link_tbl(areq->assoc, edesc->assoc_nents,
+			       edesc->link_tbl, 0);
+		ptr = edesc->link_tbl_dma;
+		options |= LDST_SGF;
+	}
+	append_fifo_load(desc, ptr, areq->assoclen, options);
+
+	/* copy iv from cipher/class1 input context to class2 infifo */
+	append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize);
+
+	if (!encrypt) {
+		u32 *jump_cmd, *uncond_jump_cmd;
+
+		/* JUMP if shared */
+		jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
+
+		/* start class 1 (cipher) operation, non-shared version */
+		append_operation(desc, ctx->class1_alg_type |
+				 OP_ALG_AS_INITFINAL);
+
+		uncond_jump_cmd = append_jump(desc, 0);
+
+		set_jump_tgt_here(desc, jump_cmd);
+
+		/* start class 1 (cipher) operation, shared version */
+		append_operation(desc, ctx->class1_alg_type |
+				 OP_ALG_AS_INITFINAL | OP_ALG_AAI_DK);
+		set_jump_tgt_here(desc, uncond_jump_cmd);
+	} else
+		append_operation(desc, ctx->class1_alg_type |
+				 OP_ALG_AS_INITFINAL | encrypt);
+
+	/* load payload & instruct to class2 to snoop class 1 if encrypting */
+	options = 0;
+	if (!edesc->src_nents) {
+		src_dma = sg_dma_address(areq->src);
+	} else {
+		sg_to_link_tbl(areq->src, edesc->src_nents, edesc->link_tbl +
+			       edesc->assoc_nents, 0);
+		src_dma = edesc->link_tbl_dma + edesc->assoc_nents *
+			  sizeof(struct link_tbl_entry);
+		options |= LDST_SGF;
+	}
+	append_seq_in_ptr(desc, src_dma, areq->cryptlen + authsize, options);
+	append_seq_fifo_load(desc, areq->cryptlen, FIFOLD_CLASS_BOTH |
+			     FIFOLD_TYPE_LASTBOTH |
+			     (encrypt ? FIFOLD_TYPE_MSG1OUT2
+				      : FIFOLD_TYPE_MSG));
+
+	/* specify destination */
+	if (areq->src == areq->dst) {
+		dst_dma = src_dma;
+	} else {
+		sg_count = dma_map_sg(jrdev, areq->dst, edesc->dst_nents ? : 1,
+				      DMA_FROM_DEVICE);
+		if (!edesc->dst_nents) {
+			dst_dma = sg_dma_address(areq->dst);
+			options = 0;
+		} else {
+			sg_to_link_tbl(areq->dst, edesc->dst_nents,
+				       edesc->link_tbl + edesc->assoc_nents +
+				       edesc->src_nents, 0);
+			dst_dma = edesc->link_tbl_dma + (edesc->assoc_nents +
+				  edesc->src_nents) *
+				  sizeof(struct link_tbl_entry);
+			options = LDST_SGF;
+		}
+	}
+	append_seq_out_ptr(desc, dst_dma, areq->cryptlen + authsize, options);
+	append_seq_fifo_store(desc, areq->cryptlen, FIFOST_TYPE_MESSAGE_DATA);
+
+	/* ICV */
+	if (encrypt)
+		append_seq_store(desc, authsize, LDST_CLASS_2_CCB |
+				 LDST_SRCDST_BYTE_CONTEXT);
+	else
+		append_seq_fifo_load(desc, authsize, FIFOLD_CLASS_CLASS2 |
+				     FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
+
+#ifdef DEBUG
+	debug("job_desc_len %d\n", desc_len(desc));
+	print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc) , 1);
+	print_hex_dump(KERN_ERR, "jdlinkt@"xstr(__LINE__)": ",
+		       DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl,
+			edesc->link_tbl_bytes, 1);
+#endif
+
+	ret = caam_jr_enqueue(jrdev, desc, callback, areq);
+	if (!ret)
+		ret = -EINPROGRESS;
+	else {
+		ipsec_esp_unmap(jrdev, edesc, areq);
+		kfree(edesc);
+	}
+
+	return ret;
+}
+
+/*
+ * derive number of elements in scatterlist
+ */
+static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
+{
+	struct scatterlist *sg = sg_list;
+	int sg_nents = 0;
+
+	*chained = 0;
+	while (nbytes > 0) {
+		sg_nents++;
+		nbytes -= sg->length;
+		if (!sg_is_last(sg) && (sg + 1)->length == 0)
+			*chained = 1;
+		sg = scatterwalk_sg_next(sg);
+	}
+
+	return sg_nents;
+}
+
+/*
+ * allocate and map the ipsec_esp extended descriptor
+ */
+static struct ipsec_esp_edesc *ipsec_esp_edesc_alloc(struct aead_request *areq,
+						     int desc_bytes)
+{
+	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
+	struct caam_ctx *ctx = crypto_aead_ctx(aead);
+	struct device *jrdev = ctx->jrdev;
+	gfp_t flags = areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
+		      GFP_ATOMIC;
+	int assoc_nents, src_nents, dst_nents = 0, chained, link_tbl_bytes;
+	struct ipsec_esp_edesc *edesc;
+
+	assoc_nents = sg_count(areq->assoc, areq->assoclen, &chained);
+	BUG_ON(chained);
+	if (likely(assoc_nents == 1))
+		assoc_nents = 0;
+
+	src_nents = sg_count(areq->src, areq->cryptlen + ctx->authsize,
+			     &chained);
+	BUG_ON(cha