Merge git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Pull crypto update from Herbert Xu:

 - XTS mode optimisation for twofish/cast6/camellia/aes on x86

 - AVX2/x86_64 implementation for blowfish/twofish/serpent/camellia

 - SSSE3/AVX/AVX2 optimisations for sha256/sha512

 - Added driver for SAHARA2 crypto accelerator

 - Fix for GMAC when used in non-IPsec secnarios

 - Added generic CMAC implementation (including IPsec glue)

 - IP update for crypto/atmel

 - Support for more than one device in hwrng/timeriomem

 - Added Broadcom BCM2835 RNG driver

 - Misc fixes

* git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (59 commits)
  crypto: caam - fix job ring cleanup code
  crypto: camellia - add AVX2/AES-NI/x86_64 assembler implementation of camellia cipher
  crypto: serpent - add AVX2/x86_64 assembler implementation of serpent cipher
  crypto: twofish - add AVX2/x86_64 assembler implementation of twofish cipher
  crypto: blowfish - add AVX2/x86_64 implementation of blowfish cipher
  crypto: tcrypt - add async cipher speed tests for blowfish
  crypto: testmgr - extend camellia test-vectors for camellia-aesni/avx2
  crypto: aesni_intel - fix Kconfig problem with CRYPTO_GLUE_HELPER_X86
  crypto: aesni_intel - add more optimized XTS mode for x86-64
  crypto: x86/camellia-aesni-avx - add more optimized XTS code
  crypto: cast6-avx: use new optimized XTS code
  crypto: x86/twofish-avx - use optimized XTS code
  crypto: x86 - add more optimized XTS-mode for serpent-avx
  xfrm: add rfc4494 AES-CMAC-96 support
  crypto: add CMAC support to CryptoAPI
  crypto: testmgr - add empty test vectors for null ciphers
  crypto: testmgr - add AES GMAC test vectors
  crypto: gcm - fix rfc4543 to handle async crypto correctly
  crypto: gcm - make GMAC work when dst and src are different
  hwrng: timeriomem - added devicetree hooks
  ...

29 files changed, 2580 insertions, 393 deletions

diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig
index c5a0262251b..2f9dbf7568f 100644
--- a/drivers/char/hw_random/Kconfig
+++ b/drivers/char/hw_random/Kconfig
@@ -86,6 +86,18 @@ config HW_RANDOM_BCM63XX
 
 	  If unusure, say Y.
 
+config HW_RANDOM_BCM2835
+	tristate "Broadcom BCM2835 Random Number Generator support"
+	depends on HW_RANDOM && ARCH_BCM2835
+	default HW_RANDOM
+	---help---
+	  This driver provides kernel-side support for the Random Number
+	  Generator hardware found on the Broadcom BCM2835 SoCs.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called bcm2835-rng
+
+	  If unsure, say Y.
 
 config HW_RANDOM_GEODE
 	tristate "AMD Geode HW Random Number Generator support"
diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile
index 1fd7eec9fbf..bed467c9300 100644
--- a/drivers/char/hw_random/Makefile
+++ b/drivers/char/hw_random/Makefile
@@ -26,3 +26,4 @@ obj-$(CONFIG_HW_RANDOM_PPC4XX) += ppc4xx-rng.o
 obj-$(CONFIG_HW_RANDOM_PSERIES) += pseries-rng.o
 obj-$(CONFIG_HW_RANDOM_EXYNOS)	+= exynos-rng.o
 obj-$(CONFIG_HW_RANDOM_TPM) += tpm-rng.o
+obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o
diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c
new file mode 100644
index 00000000000..eb7f14725eb
--- /dev/null
+++ b/drivers/char/hw_random/bcm2835-rng.c
@@ -0,0 +1,113 @@
+/**
+ * Copyright (c) 2010-2012 Broadcom. All rights reserved.
+ * Copyright (c) 2013 Lubomir Rintel
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License ("GPL")
+ * version 2, as published by the Free Software Foundation.
+ */
+
+#include <linux/hw_random.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+
+#define RNG_CTRL	0x0
+#define RNG_STATUS	0x4
+#define RNG_DATA	0x8
+
+/* enable rng */
+#define RNG_RBGEN	0x1
+
+/* the initial numbers generated are "less random" so will be discarded */
+#define RNG_WARMUP_COUNT 0x40000
+
+static int bcm2835_rng_read(struct hwrng *rng, void *buf, size_t max,
+			       bool wait)
+{
+	void __iomem *rng_base = (void __iomem *)rng->priv;
+
+	while ((__raw_readl(rng_base + RNG_STATUS) >> 24) == 0) {
+		if (!wait)
+			return 0;
+		cpu_relax();
+	}
+
+	*(u32 *)buf = __raw_readl(rng_base + RNG_DATA);
+	return sizeof(u32);
+}
+
+static struct hwrng bcm2835_rng_ops = {
+	.name	= "bcm2835",
+	.read	= bcm2835_rng_read,
+};
+
+static int bcm2835_rng_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	void __iomem *rng_base;
+	int err;
+
+	/* map peripheral */
+	rng_base = of_iomap(np, 0);
+	if (!rng_base) {
+		dev_err(dev, "failed to remap rng regs");
+		return -ENODEV;
+	}
+	bcm2835_rng_ops.priv = (unsigned long)rng_base;
+
+	/* register driver */
+	err = hwrng_register(&bcm2835_rng_ops);
+	if (err) {
+		dev_err(dev, "hwrng registration failed\n");
+		iounmap(rng_base);
+	} else {
+		dev_info(dev, "hwrng registered\n");
+
+		/* set warm-up count & enable */
+		__raw_writel(RNG_WARMUP_COUNT, rng_base + RNG_STATUS);
+		__raw_writel(RNG_RBGEN, rng_base + RNG_CTRL);
+	}
+	return err;
+}
+
+static int bcm2835_rng_remove(struct platform_device *pdev)
+{
+	void __iomem *rng_base = (void __iomem *)bcm2835_rng_ops.priv;
+
+	/* disable rng hardware */
+	__raw_writel(0, rng_base + RNG_CTRL);
+
+	/* unregister driver */
+	hwrng_unregister(&bcm2835_rng_ops);
+	iounmap(rng_base);
+
+	return 0;
+}
+
+static const struct of_device_id bcm2835_rng_of_match[] = {
+	{ .compatible = "brcm,bcm2835-rng", },
+	{},
+};
+MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match);
+
+static struct platform_driver bcm2835_rng_driver = {
+	.driver = {
+		.name = "bcm2835-rng",
+		.owner = THIS_MODULE,
+		.of_match_table = bcm2835_rng_of_match,
+	},
+	.probe		= bcm2835_rng_probe,
+	.remove		= bcm2835_rng_remove,
+};
+module_platform_driver(bcm2835_rng_driver);
+
+MODULE_AUTHOR("Lubomir Rintel <lkundrak@v3.sk>");
+MODULE_DESCRIPTION("BCM2835 Random Number Generator (RNG) driver");
+MODULE_LICENSE("GPLv2");
diff --git a/drivers/char/hw_random/exynos-rng.c b/drivers/char/hw_random/exynos-rng.c
index ac47631ab34..402ccfb625c 100644
--- a/drivers/char/hw_random/exynos-rng.c
+++ b/drivers/char/hw_random/exynos-rng.c
@@ -144,6 +144,7 @@ static int exynos_rng_remove(struct platform_device *pdev)
 	return 0;
 }
 
+#if defined(CONFIG_PM_SLEEP) || defined(CONFIG_PM_RUNTIME)
 static int exynos_rng_runtime_suspend(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
@@ -161,7 +162,7 @@ static int exynos_rng_runtime_resume(struct device *dev)
 
 	return clk_prepare_enable(exynos_rng->clk);
 }
-
+#endif
 
 static UNIVERSAL_DEV_PM_OPS(exynos_rng_pm_ops, exynos_rng_runtime_suspend,
 					exynos_rng_runtime_resume, NULL);
diff --git a/drivers/char/hw_random/mxc-rnga.c b/drivers/char/hw_random/mxc-rnga.c
index 895d0b8fb9a..4ca35e8a5d8 100644
--- a/drivers/char/hw_random/mxc-rnga.c
+++ b/drivers/char/hw_random/mxc-rnga.c
@@ -142,7 +142,7 @@ static void mxc_rnga_cleanup(struct hwrng *rng)
 static int __init mxc_rnga_probe(struct platform_device *pdev)
 {
 	int err = -ENODEV;
-	struct resource *res, *mem;
+	struct resource *res;
 	struct mxc_rng *mxc_rng;
 
 	mxc_rng = devm_kzalloc(&pdev->dev, sizeof(struct mxc_rng),
@@ -172,15 +172,9 @@ static int __init mxc_rnga_probe(struct platform_device *pdev)
 		goto err_region;
 	}
 
-	mem = request_mem_region(res->start, resource_size(res), pdev->name);
-	if (mem == NULL) {
-		err = -EBUSY;
-		goto err_region;
-	}
-
-	mxc_rng->mem = ioremap(res->start, resource_size(res));
-	if (!mxc_rng->mem) {
-		err = -ENOMEM;
+	mxc_rng->mem = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(mxc_rng->mem)) {
+		err = PTR_ERR(mxc_rng->mem);
 		goto err_ioremap;
 	}
 
@@ -195,8 +189,6 @@ static int __init mxc_rnga_probe(struct platform_device *pdev)
 	return 0;
 
 err_ioremap:
-	release_mem_region(res->start, resource_size(res));
-
 err_region:
 	clk_disable_unprepare(mxc_rng->clk);
 
@@ -206,15 +198,10 @@ out:
 
 static int __exit mxc_rnga_remove(struct platform_device *pdev)
 {
-	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	struct mxc_rng *mxc_rng = platform_get_drvdata(pdev);
 
 	hwrng_unregister(&mxc_rng->rng);
 
-	iounmap(mxc_rng->mem);
-
-	release_mem_region(res->start, resource_size(res));
-
 	clk_disable_unprepare(mxc_rng->clk);
 
 	return 0;
diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c
index 849db199c02..3e75737f5fe 100644
--- a/drivers/char/hw_random/timeriomem-rng.c
+++ b/drivers/char/hw_random/timeriomem-rng.c
@@ -23,127 +23,209 @@
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/platform_device.h>
+#include <linux/of.h>
 #include <linux/hw_random.h>
 #include <linux/io.h>
+#include <linux/slab.h>
 #include <linux/timeriomem-rng.h>
 #include <linux/jiffies.h>
 #include <linux/sched.h>
 #include <linux/timer.h>
 #include <linux/completion.h>
 
-static struct timeriomem_rng_data *timeriomem_rng_data;
+struct timeriomem_rng_private_data {
+	void __iomem		*io_base;
+	unsigned int		expires;
+	unsigned int		period;
+	unsigned int		present:1;
 
-static void timeriomem_rng_trigger(unsigned long);
-static DEFINE_TIMER(timeriomem_rng_timer, timeriomem_rng_trigger, 0, 0);
+	struct timer_list	timer;
+	struct completion	completion;
+
+	struct hwrng		timeriomem_rng_ops;
+};
+
+#define to_rng_priv(rng) \
+		((struct timeriomem_rng_private_data *)rng->priv)
 
 /*
  * have data return 1, however return 0 if we have nothing
  */
 static int timeriomem_rng_data_present(struct hwrng *rng, int wait)
 {
-	if (rng->priv == 0)
-		return 1;
+	struct timeriomem_rng_private_data *priv = to_rng_priv(rng);
 
-	if (!wait || timeriomem_rng_data->present)
-		return timeriomem_rng_data->present;
+	if (!wait || priv->present)
+		return priv->present;
 
-	wait_for_completion(&timeriomem_rng_data->completion);
+	wait_for_completion(&priv->completion);
 
 	return 1;
 }
 
 static int timeriomem_rng_data_read(struct hwrng *rng, u32 *data)
 {
+	struct timeriomem_rng_private_data *priv = to_rng_priv(rng);
 	unsigned long cur;
 	s32 delay;
 
-	*data = readl(timeriomem_rng_data->address);
+	*data = readl(priv->io_base);
 
-	if (rng->priv != 0) {
-		cur = jiffies;
+	cur = jiffies;
 
-		delay = cur - timeriomem_rng_timer.expires;
-		delay = rng->priv - (delay % rng->priv);
+	delay = cur - priv->expires;
+	delay = priv->period - (delay % priv->period);
 
-		timeriomem_rng_timer.expires = cur + delay;
-		timeriomem_rng_data->present = 0;
+	priv->expires = cur + delay;
+	priv->present = 0;
 
-		init_completion(&timeriomem_rng_data->completion);
-		add_timer(&timeriomem_rng_timer);
-	}
+	INIT_COMPLETION(priv->completion);
+	mod_timer(&priv->timer, priv->expires);
 
 	return 4;
 }
 
-static void timeriomem_rng_trigger(unsigned long dummy)
+static void timeriomem_rng_trigger(unsigned long data)
 {
-	timeriomem_rng_data->present = 1;
-	complete(&timeriomem_rng_data->completion);
-}
+	struct timeriomem_rng_private_data *priv
+			= (struct timeriomem_rng_private_data *)data;
 
-static struct hwrng timeriomem_rng_ops = {
-	.name		= "timeriomem",
-	.data_present	= timeriomem_rng_data_present,
-	.data_read	= timeriomem_rng_data_read,
-	.priv		= 0,
-};
+	priv->present = 1;
+	complete(&priv->completion);
+}
 
 static int timeriomem_rng_probe(struct platform_device *pdev)
 {
+	struct timeriomem_rng_data *pdata = pdev->dev.platform_data;
+	struct timeriomem_rng_private_data *priv;
 	struct resource *res;
-	int ret;
+	int err = 0;
+	int period;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!pdev->dev.of_node && !pdata) {
+		dev_err(&pdev->dev, "timeriomem_rng_data is missing\n");
+		return -EINVAL;
+	}
 
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res)
-		return -ENOENT;
+		return -ENXIO;
 
-	timeriomem_rng_data = pdev->dev.platform_data;
+	if (res->start % 4 != 0 || resource_size(res) != 4) {
+		dev_err(&pdev->dev,
+			"address must be four bytes wide and aligned\n");
+		return -EINVAL;
+	}
 
-	timeriomem_rng_data->address = ioremap(res->start, resource_size(res));
-	if (!timeriomem_rng_data->address)
-		return -EIO;
+	/* Allocate memory for the device structure (and zero it) */
+	priv = kzalloc(sizeof(struct timeriomem_rng_private_data), GFP_KERNEL);
+	if (!priv) {
+		dev_err(&pdev->dev, "failed to allocate device structure.\n");
+		return -ENOMEM;
+	}
+
+	platform_set_drvdata(pdev, priv);
+
+	if (pdev->dev.of_node) {
+		int i;
+
+		if (!of_property_read_u32(pdev->dev.of_node,
+						"period", &i))
+			period = i;
+		else {
+			dev_err(&pdev->dev, "missing period\n");
+			err = -EINVAL;
+			goto out_free;
+		}
+	} else
+		period = pdata->period;
+
+	priv->period = usecs_to_jiffies(period);
+	if (priv->period < 1) {
+		dev_err(&pdev->dev, "period is less than one jiffy\n");
+		err = -EINVAL;
+		goto out_free;
+	}
 
-	if (timeriomem_rng_data->period != 0
-		&& usecs_to_jiffies(timeriomem_rng_data->period) > 0) {
-		timeriomem_rng_timer.expires = jiffies;
+	priv->expires	= jiffies;
+	priv->present	= 1;
 
-		timeriomem_rng_ops.priv = usecs_to_jiffies(
-						timeriomem_rng_data->period);
+	init_completion(&priv->completion);
+	complete(&priv->completion);
+
+	setup_timer(&priv->timer, timeriomem_rng_trigger, (unsigned long)priv);
+
+	priv->timeriomem_rng_ops.name		= dev_name(&pdev->dev);
+	priv->timeriomem_rng_ops.data_present	= timeriomem_rng_data_present;
+	priv->timeriomem_rng_ops.data_read	= timeriomem_rng_data_read;
+	priv->timeriomem_rng_ops.priv		= (unsigned long)priv;
+
+	if (!request_mem_region(res->start, resource_size(res),
+				dev_name(&pdev->dev))) {
+		dev_err(&pdev->dev, "request_mem_region failed\n");
+		err = -EBUSY;
+		goto out_timer;
 	}
-	timeriomem_rng_data->present = 1;
 
-	ret = hwrng_register(&timeriomem_rng_ops);
-	if (ret)
-		goto failed;
+	priv->io_base = ioremap(res->start, resource_size(res));
+	if (priv->io_base == NULL) {
+		dev_err(&pdev->dev, "ioremap failed\n");
+		err = -EIO;
+		goto out_release_io;
+	}
+
+	err = hwrng_register(&priv->timeriomem_rng_ops);
+	if (err) {
+		dev_err(&pdev->dev, "problem registering\n");
+		goto out;
+	}
 
 	dev_info(&pdev->dev, "32bits from 0x%p @ %dus\n",
-			timeriomem_rng_data->address,
-			timeriomem_rng_data->period);
+			priv->io_base, period);
 
 	return 0;
 
-failed:
-	dev_err(&pdev->dev, "problem registering\n");
-	iounmap(timeriomem_rng_data->address);
-
-	return ret;
+out:
+	iounmap(priv->io_base);
+out_release_io:
+	release_mem_region(res->start, resource_size(res));
+out_timer:
+	del_timer_sync(&priv->timer);
+out_free:
+	platform_set_drvdata(pdev, NULL);
+	kfree(priv);
+	return err;
 }
 
 static int timeriomem_rng_remove(struct platform_device *pdev)
 {
-	del_timer_sync(&timeriomem_rng_timer);
-	hwrng_unregister(&timeriomem_rng_ops);
+	struct timeriomem_rng_private_data *priv = platform_get_drvdata(pdev);
+	struct resource *res;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 
-	iounmap(timeriomem_rng_data->address);
+	hwrng_unregister(&priv->timeriomem_rng_ops);
+
+	del_timer_sync(&priv->timer);
+	iounmap(priv->io_base);
+	release_mem_region(res->start, resource_size(res));
+	platform_set_drvdata(pdev, NULL);
+	kfree(priv);
 
 	return 0;
 }
 
+static const struct of_device_id timeriomem_rng_match[] = {
+	{ .compatible = "timeriomem_rng" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, timeriomem_rng_match);
+
 static struct platform_driver timeriomem_rng_driver = {
 	.driver = {
 		.name		= "timeriomem_rng",
 		.owner		= THIS_MODULE,
+		.of_match_table	= timeriomem_rng_match,
 	},
 	.probe		= timeriomem_rng_probe,
 	.remove		= timeriomem_rng_remove,
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 87ec4d027c2..dffb8552536 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -276,6 +276,16 @@ config CRYPTO_DEV_PICOXCELL
 
 	  Saying m here will build a module named pipcoxcell_crypto.
 
+config CRYPTO_DEV_SAHARA
+	tristate "Support for SAHARA crypto accelerator"
+	depends on ARCH_MXC && EXPERIMENTAL && OF
+	select CRYPTO_BLKCIPHER
+	select CRYPTO_AES
+	select CRYPTO_ECB
+	help
+	  This option enables support for the SAHARA HW crypto accelerator
+	  found in some Freescale i.MX chips.
+
 config CRYPTO_DEV_S5P
 	tristate "Support for Samsung S5PV210 crypto accelerator"
 	depends on ARCH_S5PV210
@@ -361,15 +371,17 @@ config CRYPTO_DEV_ATMEL_TDES
 	  will be called atmel-tdes.
 
 config CRYPTO_DEV_ATMEL_SHA
-	tristate "Support for Atmel SHA1/SHA256 hw accelerator"
+	tristate "Support for Atmel SHA hw accelerator"
 	depends on ARCH_AT91
 	select CRYPTO_SHA1
 	select CRYPTO_SHA256
+	select CRYPTO_SHA512
 	select CRYPTO_ALGAPI
 	help
-	  Some Atmel processors have SHA1/SHA256 hw accelerator.
+	  Some Atmel processors have SHA1/SHA224/SHA256/SHA384/SHA512
+	  hw accelerator.
 	  Select this if you want to use the Atmel module for
-	  SHA1/SHA256 algorithms.
+	  SHA1/SHA224/SHA256/SHA384/SHA512 algorithms.
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called atmel-sha.
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 880a47b0b02..38ce13d3b79 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/
 obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o
 obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o
 obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o
+obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o
 obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
 obj-$(CONFIG_CRYPTO_DEV_TEGRA_AES) += tegra-aes.o
 obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/
diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c
index 6f22ba51f96..c1efd910d97 100644
--- a/drivers/crypto/atmel-aes.c
+++ b/drivers/crypto/atmel-aes.c
@@ -38,7 +38,7 @@
 #include <crypto/aes.h>
 #include <crypto/hash.h>
 #include <crypto/internal/hash.h>
-#include <linux/platform_data/atmel-aes.h>
+#include <linux/platform_data/crypto-atmel.h>
 #include "atmel-aes-regs.h"
 
 #define CFB8_BLOCK_SIZE		1
@@ -47,7 +47,7 @@
 #define CFB64_BLOCK_SIZE	8
 
 /* AES flags */
-#define AES_FLAGS_MODE_MASK	0x01ff
+#define AES_FLAGS_MODE_MASK	0x03ff
 #define AES_FLAGS_ENCRYPT	BIT(0)
 #define AES_FLAGS_CBC		BIT(1)
 #define AES_FLAGS_CFB		BIT(2)
@@ -55,21 +55,26 @@
 #define AES_FLAGS_CFB16		BIT(4)
 #define AES_FLAGS_CFB32		BIT(5)
 #define AES_FLAGS_CFB64		BIT(6)
-#define AES_FLAGS_OFB		BIT(7)
-#define AES_FLAGS_CTR		BIT(8)
+#define AES_FLAGS_CFB128	BIT(7)
+#define AES_FLAGS_OFB		BIT(8)
+#define AES_FLAGS_CTR		BIT(9)
 
 #define AES_FLAGS_INIT		BIT(16)
 #define AES_FLAGS_DMA		BIT(17)
 #define AES_FLAGS_BUSY		BIT(18)
+#define AES_FLAGS_FAST		BIT(19)
 
-#define AES_FLAGS_DUALBUFF	BIT(24)
-
-#define ATMEL_AES_QUEUE_LENGTH	1
-#define ATMEL_AES_CACHE_SIZE	0
+#define ATMEL_AES_QUEUE_LENGTH	50
 
 #define ATMEL_AES_DMA_THRESHOLD		16
 
 
+struct atmel_aes_caps {
+	bool	has_dualbuff;
+	bool	has_cfb64;
+	u32		max_burst_size;
+};
+
 struct atmel_aes_dev;
 
 struct atmel_aes_ctx {
@@ -77,6 +82,8 @@ struct atmel_aes_ctx {
 
 	int		keylen;
 	u32		key[AES_KEYSIZE_256 / sizeof(u32)];
+
+	u16		block_size;
 };
 
 struct atmel_aes_reqctx {
@@ -112,20 +119,27 @@ struct atmel_aes_dev {
 
 	struct scatterlist	*in_sg;
 	unsigned int		nb_in_sg;
-
+	size_t				in_offset;
 	struct scatterlist	*out_sg;
 	unsigned int		nb_out_sg;
+	size_t				out_offset;
 
 	size_t	bufcnt;
+	size_t	buflen;
+	size_t	dma_size;
 
-	u8	buf_in[ATMEL_AES_DMA_THRESHOLD] __aligned(sizeof(u32));
-	int	dma_in;
+	void	*buf_in;
+	int		dma_in;
+	dma_addr_t	dma_addr_in;
 	struct atmel_aes_dma	dma_lch_in;
 
-	u8	buf_out[ATMEL_AES_DMA_THRESHOLD] __aligned(sizeof(u32));
-	int	dma_out;
+	void	*buf_out;
+	int		dma_out;
+	dma_addr_t	dma_addr_out;
 	struct atmel_aes_dma	dma_lch_out;
 
+	struct atmel_aes_caps	caps;
+
 	u32	hw_version;
 };
 
@@ -165,6 +179,37 @@ static int atmel_aes_sg_length(struct ablkcipher_request *req,
 	return sg_nb;
 }
 
+static int atmel_aes_sg_copy(struct scatterlist **sg, size_t *offset,
+			void *buf, size_t buflen, size_t total, int out)
+{
+	unsigned int count, off = 0;
+
+	while (buflen && total) {
+		count = min((*sg)->length - *offset, total);
+		count = min(count, buflen);
+
+		if (!count)
+			return off;
+
+		scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out);
+
+		off += count;
+		buflen -= count;
+		*offset += count;
+		total -= count;
+
+		if (*offset == (*sg)->length) {
+			*sg = sg_next(*sg);
+			if (*sg)
+				*offset = 0;
+			else
+				total = 0;
+		}
+	}
+
+	return off;
+}
+
 static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset)
 {
 	return readl_relaxed(dd->io_base + offset);
@@ -190,14 +235,6 @@ static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset,
 		atmel_aes_write(dd, offset, *value);
 }
 
-static void atmel_aes_dualbuff_test(struct atmel_aes_dev *dd)
-{
-	atmel_aes_write(dd, AES_MR, AES_MR_DUALBUFF);
-
-	if (atmel_aes_read(dd, AES_MR) & AES_MR_DUALBUFF)
-		dd->flags |= AES_FLAGS_DUALBUFF;
-}
-
 static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_ctx *ctx)
 {
 	struct atmel_aes_dev *aes_dd = NULL;
@@ -225,7 +262,7 @@ static int atmel_aes_hw_init(struct atmel_aes_dev *dd)
 
 	if (!(dd->flags & AES_FLAGS_INIT)) {
 		atmel_aes_write(dd, AES_CR, AES_CR_SWRST);
-		atmel_aes_dualbuff_test(dd);
+		atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET);
 		dd->flags |= AES_FLAGS_INIT;
 		dd->err = 0;
 	}
@@ -233,11 +270,19 @@ static int atmel_aes_hw_init(struct atmel_aes_dev *dd)
 	return 0;
 }
 
+static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd)
+{
+	return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff;
+}
+
 static void atmel_aes_hw_version_init(struct atmel_aes_dev *dd)
 {
 	atmel_aes_hw_init(dd);
 
-	dd->hw_version = atmel_aes_read(dd, AES_HW_VERSION);
+	dd->hw_version = atmel_aes_get_version(dd);
+
+	dev_info(dd->dev,
+			"version: 0x%x\n", dd->hw_version);
 
 	clk_disable_unprepare(dd->iclk);
 }
@@ -260,50 +305,77 @@ static void atmel_aes_dma_callback(void *data)
 	tasklet_schedule(&dd->done_task);
 }
 
-static int atmel_aes_crypt_dma(struct atmel_aes_dev *dd)
+static int atmel_aes_crypt_dma(struct atmel_aes_dev *dd,
+		dma_addr_t dma_addr_in, dma_addr_t dma_addr_out, int length)
 {
+	struct scatterlist sg[2];
 	struct dma_async_tx_descriptor	*in_desc, *out_desc;
-	int nb_dma_sg_in, nb_dma_sg_out;
 
-	dd->nb_in_sg = atmel_aes_sg_length(dd->req, dd->in_sg);
-	if (!dd->nb_in_sg)
-		goto exit_err;
+	dd->dma_size = length;
 
-	nb_dma_sg_in = dma_map_sg(dd->dev, dd->in_sg, dd->nb_in_sg,
-			DMA_TO_DEVICE);
-	if (!nb_dma_sg_in)
-		goto exit_err;
+	if (!(dd->flags & AES_FLAGS_FAST)) {
+		dma_sync_single_for_device(dd->dev, dma_addr_in, length,
+					   DMA_TO_DEVICE);
+	}
 
-	in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, dd->in_sg,
-				nb_dma_sg_in, DMA_MEM_TO_DEV,
-				DMA_PREP_INTERRUPT  |  DMA_CTRL_ACK);
+	if (dd->flags & AES_FLAGS_CFB8) {
+		dd->dma_lch_in.dma_conf.dst_addr_width =
+			DMA_SLAVE_BUSWIDTH_1_BYTE;
+		dd->dma_lch_out.dma_conf.src_addr_width =
+			DMA_SLAVE_BUSWIDTH_1_BYTE;
+	} else if (dd->flags & AES_FLAGS_CFB16) {
+		dd->dma_lch_in.dma_conf.dst_addr_width =
+			DMA_SLAVE_BUSWIDTH_2_BYTES;
+		dd->dma_lch_out.dma_conf.src_addr_width =
+			DMA_SLAVE_BUSWIDTH_2_BYTES;
+	} else {
+		dd->dma_lch_in.dma_conf.dst_addr_width =
+			DMA_SLAVE_BUSWIDTH_4_BYTES;
+		dd->dma_lch_out.dma_conf.src_addr_width =
+			DMA_SLAVE_BUSWIDTH_4_BYTES;
+	}
 
-	if (!in_desc)
-		goto unmap_in;
+	if (dd->flags & (AES_FLAGS_CFB8 | AES_FLAGS_CFB16 |
+			AES_FLAGS_CFB32 | AES_FLAGS_CFB64)) {
+		dd->dma_lch_in.dma_conf.src_maxburst = 1;
+		dd->dma_lch_in.dma_conf.dst_maxburst = 1;
+		dd->dma_lch_out.dma_conf.src_maxburst = 1;
+		dd->dma_lch_out.dma_conf.dst_maxburst = 1;
+	} else {
+		dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size;
+		dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size;
+		dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size;
+		dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size;
+	}
 
-	/* callback not needed */
+	dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);
+	dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf);
 
-	dd->nb_out_sg = atmel_aes_sg_length(dd->req, dd->out_sg);
-	if (!dd->nb_out_sg)
-		goto unmap_in;
+	dd->flags |= AES_FLAGS_DMA;
 
-	nb_dma_sg_out = dma_map_sg(dd->dev, dd->out_sg, dd->nb_out_sg,
-			DMA_FROM_DEVICE);
-	if (!nb_dma_sg_out)
-		goto unmap_out;
+	sg_init_table(&sg[0], 1);
+	sg_dma_address(&sg[0]) = dma_addr_in;
+	sg_dma_len(&sg[0]) = length;
 
-	out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, dd->out_sg,
-				nb_dma_sg_out, DMA_DEV_TO_MEM,
-				DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+	sg_init_table(&sg[1], 1);
+	sg_dma_address(&sg[1]) = dma_addr_out;
+	sg_dma_len(&sg[1]) = length;
+
+	in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0],
+				1, DMA_MEM_TO_DEV,
+				DMA_PREP_INTERRUPT  |  DMA_CTRL_ACK);
+	if (!in_desc)
+		return -EINVAL;
 
+	out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1],
+				1, DMA_DEV_TO_MEM,
+				DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 	if (!out_desc)
-		goto unmap_out;
+		return -EINVAL;
 
 	out_desc->callback = atmel_aes_dma_callback;
 	out_desc->callback_param = dd;
 
-	dd->total -= dd->req->nbytes;
-
 	dmaengine_submit(out_desc);
 	dma_async_issue_pending(dd->dma_lch_out.chan);
 
@@ -311,15 +383,6 @@ static int atmel_aes_crypt_dma(struct atmel_aes_dev *dd)
 	dma_async_issue_pending(dd->dma_lch_in.chan);
 
 	return 0;
-
-unmap_out:
-	dma_unmap_sg(dd->dev, dd->out_sg, dd->nb_out_sg,
-		DMA_FROM_DEVICE);
-unmap_in:
-	dma_unmap_sg(dd->dev, dd->in_sg, dd->nb_in_sg,
-		DMA_TO_DEVICE);
-exit_err: