diff options
Diffstat (limited to 'drivers/crypto')
49 files changed, 9131 insertions, 2887 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index f4fd837bcb8..02f177aeb16 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -262,6 +262,17 @@ config CRYPTO_DEV_OMAP_AES OMAP processors have AES module accelerator. Select this if you want to use the OMAP module for AES algorithms. +config CRYPTO_DEV_OMAP_DES + tristate "Support for OMAP DES3DES hw engine" + depends on ARCH_OMAP2PLUS + select CRYPTO_DES + select CRYPTO_BLKCIPHER2 + help + OMAP processors have DES/3DES module accelerator. Select this if you + want to use the OMAP module for DES and 3DES algorithms. Currently + the ECB and CBC modes of operation supported by the driver. Also + accesses made on unaligned boundaries are also supported. + config CRYPTO_DEV_PICOXCELL tristate "Support for picoXcell IPSEC and Layer2 crypto engines" depends on ARCH_PICOXCELL && HAVE_CLK @@ -289,41 +300,20 @@ config CRYPTO_DEV_SAHARA This option enables support for the SAHARA HW crypto accelerator found in some Freescale i.MX chips. -config CRYPTO_DEV_DCP - tristate "Support for the DCP engine" - depends on ARCH_MXS && OF - select CRYPTO_BLKCIPHER - select CRYPTO_AES - select CRYPTO_CBC - help - This options enables support for the hardware crypto-acceleration - capabilities of the DCP co-processor - config CRYPTO_DEV_S5P - tristate "Support for Samsung S5PV210 crypto accelerator" - depends on ARCH_S5PV210 + tristate "Support for Samsung S5PV210/Exynos crypto accelerator" + depends on ARCH_S5PV210 || ARCH_EXYNOS select CRYPTO_AES select CRYPTO_ALGAPI select CRYPTO_BLKCIPHER help This option allows you to have support for S5P crypto acceleration. - Select this to offload Samsung S5PV210 or S5PC110 from AES + Select this to offload Samsung S5PV210 or S5PC110, Exynos from AES algorithms execution. -config CRYPTO_DEV_TEGRA_AES - tristate "Support for TEGRA AES hw engine" - depends on ARCH_TEGRA - select CRYPTO_AES - help - TEGRA processors have AES module accelerator. Select this if you - want to use the TEGRA module for AES algorithms. - - To compile this driver as a module, choose M here: the module - will be called tegra-aes. - config CRYPTO_DEV_NX bool "Support for IBM Power7+ in-Nest cryptographic acceleration" - depends on PPC64 && IBMVIO + depends on PPC64 && IBMVIO && !CPU_LITTLE_ENDIAN default n help Support for Power7+ in-Nest cryptographic acceleration. @@ -399,4 +389,33 @@ config CRYPTO_DEV_ATMEL_SHA To compile this driver as a module, choose M here: the module will be called atmel-sha. +config CRYPTO_DEV_CCP + bool "Support for AMD Cryptographic Coprocessor" + depends on X86 && PCI + default n + help + The AMD Cryptographic Coprocessor provides hardware support + for encryption, hashing and related operations. + +if CRYPTO_DEV_CCP + source "drivers/crypto/ccp/Kconfig" +endif + +config CRYPTO_DEV_MXS_DCP + tristate "Support for Freescale MXS DCP" + depends on ARCH_MXS + select CRYPTO_SHA1 + select CRYPTO_SHA256 + select CRYPTO_CBC + select CRYPTO_ECB + select CRYPTO_AES + select CRYPTO_BLKCIPHER + select CRYPTO_ALGAPI + help + The Freescale i.MX23/i.MX28 has SHA1/SHA256 and AES128 CBC/ECB + co-processor on the die. + + To compile this driver as a module, choose M here: the module + will be called mxs-dcp. + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index b4946ddd255..482f090d16d 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -1,24 +1,25 @@ -obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o -obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o +obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o +obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ +obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o -obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o -n2_crypto-y := n2_core.o n2_asm.o obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o -obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o -obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/ obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o -obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ -obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o +obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o +obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o +obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o +n2_crypto-y := n2_core.o n2_asm.o +obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o +obj-$(CONFIG_CRYPTO_DEV_OMAP_DES) += omap-des.o +obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o +obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o +obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o -obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o -obj-$(CONFIG_CRYPTO_DEV_DCP) += dcp.o +obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o -obj-$(CONFIG_CRYPTO_DEV_TEGRA_AES) += tegra-aes.o +obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o +obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ -obj-$(CONFIG_CRYPTO_DEV_BFIN_CRC) += bfin_crc.o -obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ -obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o -obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o -obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index efaf6302405..37f9cc98ba1 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -724,7 +724,6 @@ static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) crypto4xx_destroy_pdr(core_dev->dev); crypto4xx_destroy_gdr(core_dev->dev); crypto4xx_destroy_sdr(core_dev->dev); - dev_set_drvdata(core_dev->device, NULL); iounmap(core_dev->dev->ce_base); kfree(core_dev->dev); kfree(core_dev); diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index c1efd910d97..a083474991a 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -30,6 +30,7 @@ #include <linux/irq.h> #include <linux/scatterlist.h> #include <linux/dma-mapping.h> +#include <linux/of_device.h> #include <linux/delay.h> #include <linux/crypto.h> #include <linux/cryptohash.h> @@ -39,6 +40,7 @@ #include <crypto/hash.h> #include <crypto/internal/hash.h> #include <linux/platform_data/crypto-atmel.h> +#include <dt-bindings/dma/at91.h> #include "atmel-aes-regs.h" #define CFB8_BLOCK_SIZE 1 @@ -714,6 +716,12 @@ static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) return -EINVAL; } ctx->block_size = CFB32_BLOCK_SIZE; + } else if (mode & AES_FLAGS_CFB64) { + if (!IS_ALIGNED(req->nbytes, CFB64_BLOCK_SIZE)) { + pr_err("request size is not exact amount of CFB64 blocks\n"); + return -EINVAL; + } + ctx->block_size = CFB64_BLOCK_SIZE; } else { if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { pr_err("request size is not exact amount of AES blocks\n"); @@ -747,59 +755,50 @@ static int atmel_aes_dma_init(struct atmel_aes_dev *dd, struct crypto_platform_data *pdata) { int err = -ENOMEM; - dma_cap_mask_t mask_in, mask_out; - - if (pdata && pdata->dma_slave->txdata.dma_dev && - pdata->dma_slave->rxdata.dma_dev) { - - /* Try to grab 2 DMA channels */ - dma_cap_zero(mask_in); - dma_cap_set(DMA_SLAVE, mask_in); - - dd->dma_lch_in.chan = dma_request_channel(mask_in, - atmel_aes_filter, &pdata->dma_slave->rxdata); - - if (!dd->dma_lch_in.chan) - goto err_dma_in; - - dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; - dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + - AES_IDATAR(0); - dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; - dd->dma_lch_in.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.device_fc = false; - - dma_cap_zero(mask_out); - dma_cap_set(DMA_SLAVE, mask_out); - dd->dma_lch_out.chan = dma_request_channel(mask_out, - atmel_aes_filter, &pdata->dma_slave->txdata); - - if (!dd->dma_lch_out.chan) - goto err_dma_out; + dma_cap_mask_t mask; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + /* Try to grab 2 DMA channels */ + dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask, + atmel_aes_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); + if (!dd->dma_lch_in.chan) + goto err_dma_in; + + dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + AES_IDATAR(0); + dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + dd->dma_lch_out.chan = dma_request_slave_channel_compat(mask, + atmel_aes_filter, &pdata->dma_slave->txdata, dd->dev, "rx"); + if (!dd->dma_lch_out.chan) + goto err_dma_out; + + dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; + dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + + AES_ODATAR(0); + dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; + dd->dma_lch_out.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.device_fc = false; - dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; - dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + - AES_ODATAR(0); - dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; - dd->dma_lch_out.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.device_fc = false; - - return 0; - } else { - return -ENODEV; - } + return 0; err_dma_out: dma_release_channel(dd->dma_lch_in.chan); err_dma_in: + dev_warn(dd->dev, "no DMA channel available\n"); return err; } @@ -1076,7 +1075,7 @@ static struct crypto_alg aes_algs[] = { .cra_driver_name = "atmel-cfb8-aes", .cra_priority = 100, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = CFB64_BLOCK_SIZE, + .cra_blocksize = CFB8_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), .cra_alignmask = 0x0, .cra_type = &crypto_ablkcipher_type, @@ -1261,6 +1260,47 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd) } } +#if defined(CONFIG_OF) +static const struct of_device_id atmel_aes_dt_ids[] = { + { .compatible = "atmel,at91sam9g46-aes" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids); + +static struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct crypto_platform_data *pdata; + + if (!np) { + dev_err(&pdev->dev, "device node not found\n"); + return ERR_PTR(-EINVAL); + } + + pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) { + dev_err(&pdev->dev, "could not allocate memory for pdata\n"); + return ERR_PTR(-ENOMEM); + } + + pdata->dma_slave = devm_kzalloc(&pdev->dev, + sizeof(*(pdata->dma_slave)), + GFP_KERNEL); + if (!pdata->dma_slave) { + dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); + devm_kfree(&pdev->dev, pdata); + return ERR_PTR(-ENOMEM); + } + + return pdata; +} +#else +static inline struct crypto_platform_data *atmel_aes_of_init(struct platform_device *pdev) +{ + return ERR_PTR(-EINVAL); +} +#endif + static int atmel_aes_probe(struct platform_device *pdev) { struct atmel_aes_dev *aes_dd; @@ -1272,6 +1312,14 @@ static int atmel_aes_probe(struct platform_device *pdev) pdata = pdev->dev.platform_data; if (!pdata) { + pdata = atmel_aes_of_init(pdev); + if (IS_ERR(pdata)) { + err = PTR_ERR(pdata); + goto aes_dd_err; + } + } + + if (!pdata->dma_slave) { err = -ENXIO; goto aes_dd_err; } @@ -1358,7 +1406,9 @@ static int atmel_aes_probe(struct platform_device *pdev) if (err) goto err_algs; - dev_info(dev, "Atmel AES\n"); + dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n", + dma_chan_name(aes_dd->dma_lch_in.chan), + dma_chan_name(aes_dd->dma_lch_out.chan)); return 0; @@ -1424,6 +1474,7 @@ static struct platform_driver atmel_aes_driver = { .driver = { .name = "atmel_aes", .owner = THIS_MODULE, + .of_match_table = of_match_ptr(atmel_aes_dt_ids), }, }; diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index eaed8bf183b..0618be06b9f 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -30,6 +30,7 @@ #include <linux/irq.h> #include <linux/scatterlist.h> #include <linux/dma-mapping.h> +#include <linux/of_device.h> #include <linux/delay.h> #include <linux/crypto.h> #include <linux/cryptohash.h> @@ -1263,32 +1264,29 @@ static int atmel_sha_dma_init(struct atmel_sha_dev *dd, int err = -ENOMEM; dma_cap_mask_t mask_in; - if (pdata && pdata->dma_slave->rxdata.dma_dev) { - /* Try to grab DMA channel */ - dma_cap_zero(mask_in); - dma_cap_set(DMA_SLAVE, mask_in); + /* Try to grab DMA channel */ + dma_cap_zero(mask_in); + dma_cap_set(DMA_SLAVE, mask_in); - dd->dma_lch_in.chan = dma_request_channel(mask_in, - atmel_sha_filter, &pdata->dma_slave->rxdata); - - if (!dd->dma_lch_in.chan) - return err; - - dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; - dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + - SHA_REG_DIN(0); - dd->dma_lch_in.dma_conf.src_maxburst = 1; - dd->dma_lch_in.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.dst_maxburst = 1; - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.device_fc = false; - - return 0; + dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask_in, + atmel_sha_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); + if (!dd->dma_lch_in.chan) { + dev_warn(dd->dev, "no DMA channel available\n"); + return err; } - return -ENODEV; + dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + SHA_REG_DIN(0); + dd->dma_lch_in.dma_conf.src_maxburst = 1; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + return 0; } static void atmel_sha_dma_cleanup(struct atmel_sha_dev *dd) @@ -1326,6 +1324,48 @@ static void atmel_sha_get_cap(struct atmel_sha_dev *dd) } } +#if defined(CONFIG_OF) +static const struct of_device_id atmel_sha_dt_ids[] = { + { .compatible = "atmel,at91sam9g46-sha" }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, atmel_sha_dt_ids); + +static struct crypto_platform_data *atmel_sha_of_init(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct crypto_platform_data *pdata; + + if (!np) { + dev_err(&pdev->dev, "device node not found\n"); + return ERR_PTR(-EINVAL); + } + + pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) { + dev_err(&pdev->dev, "could not allocate memory for pdata\n"); + return ERR_PTR(-ENOMEM); + } + + pdata->dma_slave = devm_kzalloc(&pdev->dev, + sizeof(*(pdata->dma_slave)), + GFP_KERNEL); + if (!pdata->dma_slave) { + dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); + devm_kfree(&pdev->dev, pdata); + return ERR_PTR(-ENOMEM); + } + + return pdata; +} +#else /* CONFIG_OF */ +static inline struct crypto_platform_data *atmel_sha_of_init(struct platform_device *dev) +{ + return ERR_PTR(-EINVAL); +} +#endif + static int atmel_sha_probe(struct platform_device *pdev) { struct atmel_sha_dev *sha_dd; @@ -1402,13 +1442,23 @@ static int atmel_sha_probe(struct platform_device *pdev) if (sha_dd->caps.has_dma) { pdata = pdev->dev.platform_data; if (!pdata) { - dev_err(&pdev->dev, "platform data not available\n"); + pdata = atmel_sha_of_init(pdev); + if (IS_ERR(pdata)) { + dev_err(&pdev->dev, "platform data not available\n"); + err = PTR_ERR(pdata); + goto err_pdata; + } + } + if (!pdata->dma_slave) { err = -ENXIO; goto err_pdata; } err = atmel_sha_dma_init(sha_dd, pdata); if (err) goto err_sha_dma; + + dev_info(dev, "using %s for DMA transfers\n", + dma_chan_name(sha_dd->dma_lch_in.chan)); } spin_lock(&atmel_sha.lock); @@ -1419,7 +1469,9 @@ static int atmel_sha_probe(struct platform_device *pdev) if (err) goto err_algs; - dev_info(dev, "Atmel SHA1/SHA256\n"); + dev_info(dev, "Atmel SHA1/SHA256%s%s\n", + sha_dd->caps.has_sha224 ? "/SHA224" : "", + sha_dd->caps.has_sha_384_512 ? "/SHA384/SHA512" : ""); return 0; @@ -1483,6 +1535,7 @@ static struct platform_driver atmel_sha_driver = { .driver = { .name = "atmel_sha", .owner = THIS_MODULE, + .of_match_table = of_match_ptr(atmel_sha_dt_ids), }, }; diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 4a99564a08e..6cde5b530c6 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -30,6 +30,7 @@ #include <linux/irq.h> #include <linux/scatterlist.h> #include <linux/dma-mapping.h> +#include <linux/of_device.h> #include <linux/delay.h> #include <linux/crypto.h> #include <linux/cryptohash.h> @@ -716,59 +717,50 @@ static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd, struct crypto_platform_data *pdata) { int err = -ENOMEM; - dma_cap_mask_t mask_in, mask_out; + dma_cap_mask_t mask; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + /* Try to grab 2 DMA channels */ + dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask, + atmel_tdes_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); + if (!dd->dma_lch_in.chan) + goto err_dma_in; + + dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; + dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + + TDES_IDATA1R; + dd->dma_lch_in.dma_conf.src_maxburst = 1; + dd->dma_lch_in.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.dst_maxburst = 1; + dd->dma_lch_in.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_in.dma_conf.device_fc = false; + + dd->dma_lch_out.chan = dma_request_slave_channel_compat(mask, + atmel_tdes_filter, &pdata->dma_slave->txdata, dd->dev, "rx"); + if (!dd->dma_lch_out.chan) + goto err_dma_out; + + dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; + dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + + TDES_ODATA1R; + dd->dma_lch_out.dma_conf.src_maxburst = 1; + dd->dma_lch_out.dma_conf.src_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.dst_maxburst = 1; + dd->dma_lch_out.dma_conf.dst_addr_width = + DMA_SLAVE_BUSWIDTH_4_BYTES; + dd->dma_lch_out.dma_conf.device_fc = false; - if (pdata && pdata->dma_slave->txdata.dma_dev && - pdata->dma_slave->rxdata.dma_dev) { - - /* Try to grab 2 DMA channels */ - dma_cap_zero(mask_in); - dma_cap_set(DMA_SLAVE, mask_in); - - dd->dma_lch_in.chan = dma_request_channel(mask_in, - atmel_tdes_filter, &pdata->dma_slave->rxdata); - - if (!dd->dma_lch_in.chan) - goto err_dma_in; - - dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; - dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + - TDES_IDATA1R; - dd->dma_lch_in.dma_conf.src_maxburst = 1; - dd->dma_lch_in.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.dst_maxburst = 1; - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.device_fc = false; - - dma_cap_zero(mask_out); - dma_cap_set(DMA_SLAVE, mask_out); - dd->dma_lch_out.chan = dma_request_channel(mask_out, - atmel_tdes_filter, &pdata->dma_slave->txdata); - - if (!dd->dma_lch_out.chan) - goto err_dma_out; - - dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; - dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + - TDES_ODATA1R; - dd->dma_lch_out.dma_conf.src_maxburst = 1; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.dst_maxburst = 1; - dd->dma_lch_out.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.device_fc = false; - - return 0; - } else { - return -ENODEV; - } + return 0; err_dma_out: dma_release_channel(dd->dma_lch_in.chan); err_dma_in: + dev_warn(dd->dev, "no DMA channel available\n"); return err; } @@ -1317,6 +1309,47 @@ static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd) } } +#if defined(CONFIG_OF) +static const struct of_device_id atmel_tdes_dt_ids[] = { + { .compatible = "atmel,at91sam9g46-tdes" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, atmel_tdes_dt_ids); + +static struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct crypto_platform_data *pdata; + + if (!np) { + dev_err(&pdev->dev, "device node not found\n"); + return ERR_PTR(-EINVAL); + } + + pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL); + if (!pdata) { + dev_err(&pdev->dev, "could not allocate memory for pdata\n"); + return ERR_PTR(-ENOMEM); + } + + pdata->dma_slave = devm_kzalloc(&pdev->dev, + sizeof(*(pdata->dma_slave)), + GFP_KERNEL); + if (!pdata->dma_slave) { + dev_err(&pdev->dev, "could not allocate memory for dma_slave\n"); + devm_kfree(&pdev->dev, pdata); + return ERR_PTR(-ENOMEM); + } + + return pdata; +} +#else /* CONFIG_OF */ +static inline struct crypto_platform_data *atmel_tdes_of_init(struct platform_device *pdev) +{ + return ERR_PTR(-EINVAL); +} +#endif + static int atmel_tdes_probe(struct platform_device *pdev) { struct atmel_tdes_dev *tdes_dd; @@ -1399,13 +1432,24 @@ static int atmel_tdes_probe(struct platform_device *pdev) if (tdes_dd->caps.has_dma) { pdata = pdev->dev.platform_data; if (!pdata) { - dev_err(&pdev->dev, "platform data not available\n"); + pdata = atmel_tdes_of_init(pdev); + if (IS_ERR(pdata)) { + dev_err(&pdev->dev, "platform data not available\n"); + err = PTR_ERR(pdata); + goto err_pdata; + } + } + if (!pdata->dma_slave) { err = -ENXIO; goto err_pdata; } err = atmel_tdes_dma_init(tdes_dd, pdata); if (err) goto err_tdes_dma; + + dev_info(dev, "using %s, %s for DMA transfers\n", + dma_chan_name(tdes_dd->dma_lch_in.chan), + dma_chan_name(tdes_dd->dma_lch_out.chan)); } spin_lock(&atmel_tdes.lock); @@ -1487,6 +1531,7 @@ static struct platform_driver atmel_tdes_driver = { .driver = { .name = "atmel_tdes", .owner = THIS_MODULE, + .of_match_table = of_match_ptr(atmel_tdes_dt_ids), }, }; diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index d797f31f5d8..b099e33cb07 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -29,10 +29,11 @@ #include <crypto/hash.h> #include <crypto/internal/hash.h> -#include <asm/blackfin.h> -#include <asm/bfin_crc.h> #include <asm/dma.h> #include <asm/portmux.h> +#include <asm/io.h> + +#include "bfin_crc.h" #define CRC_CCRYPTO_QUEUE_LENGTH 5 @@ -54,12 +55,13 @@ struct bfin_crypto_crc { int irq; int dma_ch; u32 poly; - volatile struct crc_register *regs; + struct crc_register *regs; struct ahash_request *req; /* current request in operation */ struct dma_desc_array *sg_cpu; /* virt addr of sg dma descriptors */ dma_addr_t sg_dma; /* phy addr of sg dma descriptors */ u8 *sg_mid_buf; + dma_addr_t sg_mid_dma; /* phy addr of sg mid buffer */ struct tasklet_struct done_task; struct crypto_queue queue; /* waiting requests */ @@ -132,14 +134,13 @@ static struct scatterlist *sg_get(struct scatterlist *sg_list, unsigned int nent static int bfin_crypto_crc_init_hw(struct bfin_crypto_crc *crc, u32 key) { - crc->regs->datacntrld = 0; - crc->regs->control = MODE_CALC_CRC << OPMODE_OFFSET; - crc->regs->curresult = key; + writel(0, &crc->regs->datacntrld); + writel(MODE_CALC_CRC << OPMODE_OFFSET, &crc->regs->control); + writel(key, &crc->regs->curresult); /* setup CRC interrupts */ - crc->regs->status = CMPERRI | DCNTEXPI; - crc->regs->intrenset = CMPERRI | DCNTEXPI; - SSYNC(); + writel(CMPERRI | DCNTEXPI, &crc->regs->status); + writel(CMPERRI | DCNTEXPI, &crc->regs->intrenset); return 0; } @@ -195,7 +196,6 @@ static void bfin_crypto_crc_config_dma(struct bfin_crypto_crc *crc) dma_map_sg(crc->dev, ctx->sg, ctx->sg_nents, DMA_TO_DEVICE); for_each_sg(ctx->sg, sg, ctx->sg_nents, j) { - dma_config = DMAFLOW_ARRAY | RESTART | NDSIZE_3 | DMAEN | PSIZE_32; dma_addr = sg_dma_address(sg); /* deduce extra bytes in last sg */ if (sg_is_last(sg)) @@ -208,12 +208,29 @@ static void bfin_crypto_crc_config_dma(struct bfin_crypto_crc *crc) bytes in current sg buffer. Move addr of current sg and deduce the length of current sg. */ - memcpy(crc->sg_mid_buf +((i-1) << 2) + mid_dma_count, - (void *)dma_addr, + memcpy(crc->sg_mid_buf +(i << 2) + mid_dma_count, + sg_virt(sg), CHKSUM_DIGEST_SIZE - mid_dma_count); dma_addr += CHKSUM_DIGEST_SIZE - mid_dma_count; dma_count -= CHKSUM_DIGEST_SIZE - mid_dma_count; + + dma_config = DMAFLOW_ARRAY | RESTART | NDSIZE_3 | + DMAEN | PSIZE_32 | WDSIZE_32; + + /* setup new dma descriptor for next middle dma */ + crc->sg_cpu[i].start_addr = crc->sg_mid_dma + (i << 2); + crc->sg_cpu[i].cfg = dma_config; + crc->sg_cpu[i].x_count = 1; + crc->sg_cpu[i].x_modify = CHKSUM_DIGEST_SIZE; + dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, " + "cfg:0x%lx, x_count:0x%lx, x_modify:0x%lx\n", + i, crc->sg_cpu[i].start_addr, + crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count, + crc->sg_cpu[i].x_modify); + i++; } + + dma_config = DMAFLOW_ARRAY | RESTART | NDSIZE_3 | DMAEN | PSIZE_32; /* chop current sg dma len to multiple of 32 bits */ mid_dma_count = dma_count % 4; dma_count &= ~0x3; @@ -244,24 +261,9 @@ static void bfin_crypto_crc_config_dma(struct bfin_crypto_crc *crc) if (mid_dma_count) { /* copy extra bytes to next middle dma buffer */ - dma_config = DMAFLOW_ARRAY | RESTART | NDSIZE_3 | - DMAEN | PSIZE_32 | WDSIZE_32; memcpy(crc->sg_mid_buf + (i << 2), - (void *)(dma_addr + (dma_count << 2)), + (u8*)sg_virt(sg) + (dma_count << 2), mid_dma_count); - /* setup new dma descriptor for next middle dma */ - crc->sg_cpu[i].start_addr = dma_map_single(crc->dev, - crc->sg_mid_buf + (i << 2), - CHKSUM_DIGEST_SIZE, DMA_TO_DEVICE); - crc->sg_cpu[i].cfg = dma_config; - crc->sg_cpu[i].x_count = 1; - crc->sg_cpu[i].x_modify = CHKSUM_DIGEST_SIZE; - dev_dbg(crc->dev, "%d: crc_dma: start_addr:0x%lx, " - "cfg:0x%lx, x_count:0x%lx, x_modify:0x%lx\n", - i, crc->sg_cpu[i].start_addr, - crc->sg_cpu[i].cfg, crc->sg_cpu[i].x_count, - crc->sg_cpu[i].x_modify); - i++; } } @@ -285,17 +287,12 @@ static void bfin_crypto_crc_config_dma(struct bfin_crypto_crc *crc) if (i == 0) return; - flush_dcache_range((unsigned int)crc->sg_cpu, - (unsigned int)crc->sg_cpu + - i * sizeof(struct dma_desc_array)); - /* Set the last descriptor to stop mode */ crc->sg_cpu[i - 1].cfg &= ~(DMAFLOW | NDSIZE); crc->sg_cpu[i - 1].cfg |= DI_EN; set_dma_curr_desc_addr(crc->dma_ch, (unsigned long *)crc->sg_dma); set_dma_x_count(crc->dma_ch, 0); set_dma_x_modify(crc->dma_ch, 0); - SSYNC(); set_dma_config(crc->dma_ch, dma_config); } @@ -309,6 +306,7 @@ static int bfin_crypto_crc_handle_queue(struct bfin_crypto_crc *crc, int nsg, i, j; unsigned int nextlen; unsigned long flags; + u32 reg; spin_lock_irqsave(&crc->lock, flags); if (req) @@ -408,14 +406,14 @@ finish_update: ctx->sg_buflen += CHKSUM_DIGEST_SIZE; /* set CRC data count before start DMA */ - crc->regs->datacnt = ctx->sg_buflen >> 2; + writel(ctx->sg_buflen >> 2, &crc->regs->datacnt); /* setup and enable CRC DMA */ bfin_crypto_crc_config_dma(crc); /* finally kick off CRC operation */ - crc->regs->control |= BLKEN; - SSYNC(); + reg = readl(&crc->regs->control); + writel(reg | BLKEN, &crc->regs->control); return -EINPROGRESS; } @@ -536,15 +534,17 @@ static void bfin_crypto_crc_done_task(unsigned long data) static irqreturn_t bfin_crypto_crc_handler(int irq, void *dev_id) { struct bfin_crypto_crc *crc = dev_id; + u32 reg; - if (crc->regs->status & DCNTEXP) { - crc->regs->status = DCNTEXP; - SSYNC(); + if (readl(&crc->regs->status) & DCNTEXP) { + writel(DCNTEXP, &crc->regs->status); /* prepare results */ - put_unaligned_le32(crc->regs->result, crc->req->result); + put_unaligned_le32(readl(&crc->regs->result), + crc->req->result); - crc->regs->control &= ~BLKEN; + reg = readl(&crc->regs->control); + writel(reg & ~BLKEN, &crc->regs->control); crc->busy = 0; if (crc->req->base.complete) @@ -568,7 +568,7 @@ static int bfin_crypto_crc_suspend(struct platform_device *pdev, pm_message_t st struct bfin_crypto_crc *crc = platform_get_drvdata(pdev); int i = 100000; - while ((crc->regs->control & BLKEN) && --i) + while ((readl(&crc->regs->control) & BLKEN) && --i) cpu_relax(); if (i == 0) @@ -594,7 +594,7 @@ static int bfin_crypto_crc_probe(struct platform_device *pdev) unsigned int timeout = 100000; int ret; - crc = kzalloc(sizeof(*crc), GFP_KERNEL); + crc = devm_kzalloc(dev, sizeof(*crc), GFP_KERNEL); if (!crc) { dev_err(&pdev->dev, "fail to malloc bfin_crypto_crc\n"); return -ENOMEM; @@ -610,42 +610,39 @@ static int bfin_crypto_crc_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (res == NULL) { dev_err(&pdev->dev, "Cannot get IORESOURCE_MEM\n"); - ret = -ENOENT; - goto out_error_free_mem; + return -ENOENT; } - crc->regs = ioremap(res->start, resource_size(res)); - if (!crc->regs) { + crc->regs = devm_ioremap_resource(dev, res); + if (IS_ERR((void *)crc->regs)) { dev_err(&pdev->dev, "Cannot map CRC IO\n"); - ret = -ENXIO; - goto out_error_free_mem; + return PTR_ERR((void *)crc->regs); } crc->irq = platform_get_irq(pdev, 0); if (crc->irq < 0) { dev_err(&pdev->dev, "No CRC DCNTEXP IRQ specified\n"); - ret = -ENOENT; - goto out_error_unmap; + return -ENOENT; } - ret = request_irq(crc->irq, bfin_crypto_crc_handler, IRQF_SHARED, dev_name(dev), crc); + ret = devm_request_irq(dev, crc->irq, bfin_crypto_crc_handler, + IRQF_SHARED, dev_name(dev), crc); if (ret) { dev_err(&pdev->dev, "Unable to request blackfin crc irq\n"); - goto out_error_unmap; + return ret; } res = platform_get_resource(pdev, IORESOURCE_DMA, 0); if (res == NULL) { dev_err(&pdev->dev, "No CRC DMA channel specified\n"); - ret = -ENOENT; - goto out_error_irq; + return -ENOENT; } crc->dma_ch = res->start; ret = request_dma(crc->dma_ch, dev_name(dev)); if (ret) { dev_err(&pdev->dev, "Unable to attach Blackfin CRC DMA channel\n"); - goto out_error_irq; + return ret; } crc->sg_cpu = dma_alloc_coherent(&pdev->dev, PAGE_SIZE, &crc->sg_dma, GFP_KERNEL); @@ -658,31 +655,32 @@ static int bfin_crypto_crc_probe(struct platform_device *pdev) * 1 last + 1 next dma descriptors */ crc->sg_mid_buf = (u8 *)(crc->sg_cpu + ((CRC_MAX_DMA_DESC + 1) << 1)); + crc->sg_mid_dma = crc->sg_dma + sizeof(struct dma_desc_array) + * ((CRC_MAX_DMA_DESC + 1) << 1); - crc->regs->control = 0; - SSYNC(); - crc->regs->poly = crc->poly = (u32)pdev->dev.platform_data; - SSYNC(); + writel(0, &crc->regs->control); + crc->poly = (u32)pdev->dev.platform_data; + writel(crc->poly, &crc->regs->poly); - while (!(crc->regs->status & LUTDONE) && (--timeout) > 0) + while (!(readl(&crc->regs->status) & LUTDONE) && (--timeout) > 0) cpu_relax(); if (timeout == 0) dev_info(&pdev->dev, "init crc poly timeout\n"); + platform_set_drvdata(pdev, crc); + spin_lock(&crc_list.lock); list_add(&crc->list, &crc_list.dev_list); spin_unlock(&crc_list.lock); - platform_set_drvdata(pdev, crc); - - ret = crypto_register_ahash(&algs); - if (ret) { - spin_lock(&crc_list.lock); - list_del(&crc->list); - spin_unlock(&crc_list.lock); - dev_err(&pdev->dev, "Cann't register crypto ahash device\n"); - goto out_error_dma; + if (list_is_singular(&crc_list.dev_list)) { + ret = crypto_register_ahash(&algs); + if (ret) { + dev_err(&pdev->dev, + "Can't register crypto ahash device\n"); + goto out_error_dma; + } } dev_info(&pdev->dev, "initialized\n"); @@ -693,12 +691,6 @@ out_error_dma: if (crc->sg_cpu) dma_free_coherent(&pdev->dev, PAGE_SIZE, crc->sg_cpu, crc->sg_dma); free_dma(crc->dma_ch); -out_error_irq: - free_irq(crc->irq, crc); -out_error_unmap: - iounmap((void *)crc->regs); -out_error_free_mem: - kfree(crc); return ret; } @@ -721,10 +713,6 @@ static int bfin_crypto_crc_remove(struct platform_device *pdev) crypto_unregister_ahash(&algs); tasklet_kill(&crc->done_task); free_dma(crc->dma_ch); - if (crc->irq > 0) - free_irq(crc->irq, crc); - iounmap((void *)crc->regs); - kfree(crc); return 0; } diff --git a/drivers/crypto/bfin_crc.h b/drivers/crypto/bfin_crc.h new file mode 100644 index 00000000000..75cef4dc85a --- /dev/null +++ b/drivers/crypto/bfin_crc.h @@ -0,0 +1,125 @@ +/* + * bfin_crc.h - interface to Blackfin CRC controllers + * + * Copyright 2012 Analog Devices Inc. + * + * Licensed under the GPL-2 or later. + */ + +#ifndef __BFIN_CRC_H__ +#define __BFIN_CRC_H__ + +/* Function driver which use hardware crc must initialize the structure */ +struct crc_info { + /* Input data address */ + unsigned char *in_addr; + /* Output data address */ + unsigned char *out_addr; + /* Input or output bytes */ + unsigned long datasize; + union { + /* CRC to compare with that of input buffer */ + unsigned long crc_compare; + /* Value to compare with input data */ + unsigned long val_verify; + /* Value to fill */ + unsigned long val_fill; + }; + /* Value to program the 32b CRC Polynomial */ + unsigned long crc_poly; + union { + /* CRC calculated from the input data */ + unsigned long crc_result; + /* First failed position to verify input data */ + unsigned long pos_verify; + }; + /* CRC mirror flags */ + unsigned int bitmirr:1; + unsigned int bytmirr:1; + unsigned int w16swp:1; + unsigned int fdsel:1; + unsigned int rsltmirr:1; + unsigned int polymirr:1; + unsigned int cmpmirr:1; +}; + +/* Userspace interface */ +#define CRC_IOC_MAGIC 'C' +#define CRC_IOC_CALC_CRC _IOWR('C', 0x01, unsigned int) +#define CRC_IOC_MEMCPY_CRC _IOWR('C', 0x02, unsigned int) +#define CRC_IOC_VERIFY_VAL _IOWR('C', 0x03, unsigned int) +#define CRC_IOC_FILL_VAL _IOWR('C', 0x04, unsigned int) + + +#ifdef __KERNEL__ + +#include <linux/types.h> +#include <linux/spinlock.h> +#include <linux/miscdevice.h> + +struct crc_register { + u32 control; + u32 datacnt; + u32 datacntrld; + u32 __pad_1[2]; + u32 compare; + u32 fillval; + u32 datafifo; + u32 intren; + u32 intrenset; + u32 intrenclr; + u32 poly; + u32 __pad_2[4]; + u32 status; + u32 datacntcap; + u32 __pad_3; + u32 result; + u32 curresult; + u32 __pad_4[3]; + u32 revid; +}; + +/* CRC_STATUS Masks */ +#define CMPERR 0x00000002 /* Compare error */ +#define DCNTEXP 0x00000010 /* datacnt register expired */ +#define IBR 0x00010000 /* Input buffer ready */ +#define OBR 0x00020000 /* Output buffer ready */ +#define IRR 0x00040000 /* Immediate result readt */ +#define LUTDONE 0x00080000 /* Look-up table generation done */ +#define FSTAT 0x00700000 /* FIFO status */ +#define MAX_FIFO 4 /* Max fifo size */ + +/* CRC_CONTROL Masks */ +#define BLKEN 0x00000001 /* Block enable */ +#define OPMODE 0x000000F0 /* Operation mode */ +#define OPMODE_OFFSET 4 /* Operation mode mask offset*/ +#define MODE_DMACPY_CRC 1 /* MTM CRC compute and compare */ +#define MODE_DATA_FILL 2 /* MTM data fill */ +#define MODE_CALC_CRC 3 /* MSM CRC compute and compare */ +#define MODE_DATA_VERIFY 4 /* MSM data verify */ +#define AUTOCLRZ 0x00000100 /* Auto clear to zero */ +#define AUTOCLRF 0x00000200 /* Auto clear to one */ +#define OBRSTALL 0x00001000 /* Stall on output buffer ready */ +#define IRRSTALL 0x00002000 /* Stall on immediate result ready */ +#define BITMIRR 0x00010000 /* Mirror bits within each byte of 32-bit input data */ +#define BITMIRR_OFFSET 16 /* Mirror bits offset */ +#define BYTMIRR 0x00020000 /* Mirror bytes of 32-bit input data */ +#define BYTMIRR_OFFSET 17 /* Mirror bytes offset */ +#define W16SWP 0x00040000 /* Mirror uppper and lower 16-bit word of 32-bit input data */ +#define W16SWP_OFFSET 18 /* Mirror 16-bit word offset */ +#define FDSEL 0x00080000 /* FIFO is written after input data is mirrored */ +#define FDSEL_OFFSET 19 /* Mirror FIFO offset */ +#define RSLTMIRR 0x00100000 /* CRC result registers are mirrored. */ +#define RSLTMIRR_OFFSET 20 /* Mirror CRC result offset. */ +#define POLYMIRR 0x00200000 /* CRC poly register is mirrored. */ +#define POLYMIRR_OFFSET 21 /* Mirror CRC poly offset. */ +#define CMPMIRR 0x00400000 /* CRC compare register is mirrored. */ +#define CMPMIRR_OFFSET 22 /* Mirror CRC compare offset. */ + +/* CRC_INTREN Masks */ +#define CMPERRI 0x02 /* CRC_ERROR_INTR */ +#define DCNTEXPI 0x10 /* CRC_STATUS_INTR */ + +#endif + +#endif diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 4cf5dec826e..c09ce1f040d 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -66,10 +66,14 @@ /* length of descriptors text */ #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) -#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ) -#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ) +#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) +#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ) #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ) +#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ) + #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ 20 * CAAM_CMD_SZ) @@ -104,27 +108,14 @@ static inline void append_dec_op1(u32 *desc, u32 type) } /* - * Wait for completion of class 1 key loading before allowing - * error propagation - */ -static inline void append_dec_shr_done(u32 *desc) -{ - u32 *jump_cmd; - - jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL); - set_jump_tgt_here(desc, jump_cmd); - append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); -} - -/* * For aead functions, read payload and write payload, * both of which are specified in req->src and req->dst */ static inline void aead_append_src_dst(u32 *desc, u32 msg_type) { + append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); - append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); } /* @@ -211,9 +202,197 @@ static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, append_key_aead(desc, ctx, keys_fit_inline); set_jump_tgt_here(desc, key_jump_cmd); +} + +static int aead_null_set_sh_desc(struct crypto_aead *aead) +{ + struct aead_tfm *tfm = &aead->base.crt_aead; + struct caam_ctx *ctx = crypto_aead_ctx(aead); + struct device *jrdev = ctx->jrdev; + bool keys_fit_inline = false; + u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd; + u32 *desc; + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + if (DESC_AEAD_NULL_ENC_LEN + DESC_JOB_IO_LEN + + ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + /* aead_encrypt shared descriptor */ + desc = ctx->sh_desc_enc; + + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (keys_fit_inline) + append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, + ctx->split_key_len, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + else + append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + set_jump_tgt_here(desc, key_jump_cmd); + + /* cryptlen = seqoutlen - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); + + /* + * NULL encryption; IV is zero + * assoclen = (assoclen + cryptlen) - cryptlen + */ + append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + KEY_VLF); + + /* Prepare to read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | + MOVE_DEST_MATH3 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | + MOVE_DEST_DESCBUF | + MOVE_WAITCOMP | + (0x8 << MOVE_LEN_SHIFT)); + + /* Class 2 operation */ + append_operation(desc, ctx->class2_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); + + /* Read and write cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | + MOVE_AUX_LS); + + /* Write ICV */ + append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | + LDST_SRCDST_BYTE_CONTEXT); + + ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "aead null enc shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + /* + * Job Descriptor and Shared Descriptors + * must all fit into the 64-word Descriptor h/w Buffer + */ + keys_fit_inline = false; + if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN + + ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) + keys_fit_inline = true; + + desc = ctx->sh_desc_dec; + + /* aead_decrypt shared descriptor */ + init_sh_desc(desc, HDR_SHARE_SERIAL); + + /* Skip if already shared */ + key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | + JUMP_COND_SHRD); + if (keys_fit_inline) + append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, + ctx->split_key_len, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + else + append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | + KEY_DEST_MDHA_SPLIT | KEY_ENC); + set_jump_tgt_here(desc, key_jump_cmd); - /* Propagate errors from shared to job descriptor */ - append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); + /* Class 2 operation */ + append_operation(desc, ctx->class2_alg_type | + OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); + + /* assoclen + cryptlen = seqinlen - ivsize - authsize */ + append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, + ctx->authsize + tfm->ivsize); + /* assoclen = (assoclen + cryptlen) - cryptlen */ + append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); + append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); + + /* read assoc before reading payload */ + append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | + KEY_VLF); + + /* Prepare to read and write cryptlen bytes */ + append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); + append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); + + /* + * MOVE_LEN opcode is not available in all SEC HW revisions, + * thus need to do some magic, i.e. self-patch the descriptor + * buffer. + */ + read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | + MOVE_DEST_MATH2 | + (0x6 << MOVE_LEN_SHIFT)); + write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | + MOVE_DEST_DESCBUF | + MOVE_WAITCOMP | + (0x8 << MOVE_LEN_SHIFT)); + + /* Read and write cryptlen bytes */ + aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); + + /* + * Insert a NOP here, since we need at least 4 instructions between + * code patching the descriptor buffer and the location being patched. + */ + jump_cmd = append_jump(desc, JUMP_TEST_ALL); + set_jump_tgt_here(desc, jump_cmd); + + set_move_tgt_here(desc, read_move_cmd); + set_move_tgt_here(desc, write_move_cmd); + append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); + append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | + MOVE_AUX_LS); + append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); + + /* Load ICV */ + append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | + FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); + + ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, + desc_bytes(desc), + DMA_TO_DEVICE); + if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { + dev_err(jrdev, "unable to map shared descriptor\n"); + return -ENOMEM; + } +#ifdef DEBUG + print_hex_dump(KERN_ERR, + "aead null dec shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +#endif + + return 0; } static int aead_set_sh_desc(struct crypto_aead *aead) @@ -222,13 +401,16 @@ static int aead_set_sh_desc(struct crypto_aead *aead) struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; bool keys_fit_inline = false; - u32 *key_jump_cmd, *jump_cmd; u32 geniv, moveiv; u32 *desc; - if (!ctx->enckeylen || !ctx->authsize) + if (!ctx->authsize) return 0; + /* NULL encryption / decryption */ + if (!ctx->enckeylen) + return aead_null_set_sh_desc(aead); + /* * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer @@ -253,7 +435,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* assoclen + cryptlen = seqinlen - ivsize */ append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize); - /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */ + /* assoclen = (assoclen + cryptlen) - cryptlen */ append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ); /* read assoc before reading payload */ @@ -291,35 +473,24 @@ static int aead_set_sh_desc(struct crypto_aead *aead) * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ + keys_fit_inline = false; if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN + ctx->split_key_pad_len + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) keys_fit_inline = true; - desc = ctx->sh_desc_dec; - /* aead_decrypt shared descriptor */ - init_sh_desc(desc, HDR_SHARE_SERIAL); - - /* Skip if already shared */ - key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | - JUMP_COND_SHRD); - - append_key_aead(desc, ctx, keys_fit_inline); + desc = ctx->sh_desc_dec; - /* Only propagate error immediately if shared */ - jump_cmd = append_jump(desc, JUMP_TEST_ALL); - set_jump_tgt_here(desc, key_jump_cmd); - append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); - set_jump_tgt_here(desc, jump_cmd); + init_sh_desc_key_aead(desc, ctx, keys_fit_inline); /* Class 2 operation */ append_operation(desc, ctx->class2_alg_type | OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); - /* assoclen + cryptlen = seqinlen - ivsize */ + /* assoclen + cryptlen = seqinlen - ivsize - authsize */ append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, - ctx->authsize + tfm->ivsize) + ctx->authsize + tfm->ivsize); /* assoclen = (assoclen + cryptlen) - cryptlen */ append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ); @@ -340,7 +511,6 @@ static int aead_set_sh_desc(struct crypto_aead *aead) /* Load ICV */ append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); - append_dec_shr_done(desc); ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), @@ -359,6 +529,7 @@ static int aead_set_sh_desc(struct crypto_aead *aead) * Job Descriptor and Shared Descriptors * must all fit into the 64-word Descriptor h/w Buffer */ + keys_fit_inline = false; if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN + ctx->split_key_pad_len + ctx->enckeylen <= CAAM_DESC_BYTES_MAX) @@ -467,24 +638,10 @@ static int aead_setkey(struct crypto_aead *aead, static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; - struct rtattr *rta = (void *)key; - struct crypto_authenc_key_param *param; - unsigned int authkeylen; - unsigned int enckeylen; + struct crypto_authenc_keys keys; int ret = 0; - param = RTA_DATA(rta); - enckeylen = be32_to_cpu(param->enckeylen); - - key += RTA_ALIGN(rta->rta_len); - keylen -= RTA_ALIGN(rta->rta_len); - - if (keylen < enckeylen) - goto badkey; - - authkeylen = keylen - enckeylen; - - if (keylen > CAAM_MAX_KEY_SIZE) + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; /* Pick class 2 key length from algorithm submask */ @@ -492,25 +649,29 @@ static int aead_setkey(struct crypto_aead *aead, OP_ALG_ALGSEL_SHIFT] * 2; ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); + if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) + goto badkey; + #ifdef DEBUG printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", - keylen, enckeylen, authkeylen); + keys.authkeylen + keys.enckeylen, keys.enckeylen, + keys.authkeylen); printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", ctx->split_key_len, ctx->split_key_pad_len); print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); #endif - ret = gen_split_aead_key(ctx, key, authkeylen); + ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); if (ret) { goto badkey; } /* postpend encryption key to auth split key */ - memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen); + memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + - enckeylen, DMA_TO_DEVICE); + keys.enckeylen, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, ctx->key_dma)) { dev_err(jrdev, "unable to map key i/o memory\n"); return -ENOMEM; @@ -518,15 +679,15 @@ static int aead_setkey(struct crypto_aead *aead, #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, - ctx->split_key_pad_len + enckeylen, 1); + ctx->split_key_pad_len + keys.enckeylen, 1); #endif - ctx->enckeylen = enckeylen; + ctx->enckeylen = keys.enckeylen; ret = aead_set_sh_desc(aead); if (ret) { dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + - enckeylen, DMA_TO_DEVICE); + keys.enckeylen, DMA_TO_DEVICE); } return ret; @@ -542,7 +703,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher; struct device *jrdev = ctx->jrdev; int ret = 0; - u32 *key_jump_cmd, *jump_cmd; + u32 *key_jump_cmd; u32 *desc; #ifdef DEBUG @@ -573,9 +734,6 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, set_jump_tgt_here(desc, key_jump_cmd); - /* Propagate errors from shared to job descriptor */ - append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); - /* Load iv */ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | tfm->ivsize); @@ -613,11 +771,7 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); - /* For aead, only propagate error immediately if shared */ - jump_cmd = append_jump(desc, JUMP_TEST_ALL); set_jump_tgt_here(desc, key_jump_cmd); - append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); - set_jump_tgt_here(desc, jump_cmd); /* load IV */ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | @@ -629,9 +783,6 @@ static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, /* Perform operation */ ablkcipher_append_src_dst(desc); - /* Wait for key to load before allowing propagating error */ - append_dec_shr_done(desc); - ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE); @@ -770,11 +921,8 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, edesc = (struct aead_edesc *)((char *)desc - offsetof(struct aead_edesc, hw_desc)); - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(jrdev, err); aead_unmap(jrdev, edesc, req); @@ -821,11 +969,8 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, req->cryptlen - ctx->authsize, 1); #endif - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(jrdev, err); aead_unmap(jrdev, edesc, req); @@ -870,11 +1015,8 @@ static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, edesc = (struct ablkcipher_edesc *)((char *)desc - offsetof(struct ablkcipher_edesc, hw_desc)); - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(jrdev, err); #ifdef DEBUG print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", @@ -905,11 +1047,8 @@ static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, edesc = (struct ablkcipher_edesc *)((char *)desc - offsetof(struct ablkcipher_edesc, hw_desc)); - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(jrdev, err); #ifdef DEBUG print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", @@ -1469,6 +1608,11 @@ static int aead_givencrypt(struct aead_givcrypt_request *areq) return ret; } +static int aead_null_givencrypt(struct aead_givcrypt_request *areq) +{ + return aead_encrypt(&areq->areq); +} + /* * allocate and map the ablkcipher extended descriptor for ablkcipher */ @@ -1658,6 +1802,124 @@ struct caam_alg_template { static struct caam_alg_template driver_algs[] = { /* single-pass ipsec_esp descriptor */ { + .name = "authenc(hmac(md5),ecb(cipher_null))", + .driver_name = "authenc-hmac-md5-ecb-cipher_null-caam", + .blocksize = NULL_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_null_givencrypt, + .geniv = "<built-in>", + .ivsize = NULL_IV_SIZE, + .maxauthsize = MD5_DIGEST_SIZE, + }, + .class1_alg_type = 0, + .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha1),ecb(cipher_null))", + .driver_name = "authenc-hmac-sha1-ecb-cipher_null-caam", + .blocksize = NULL_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_null_givencrypt, + .geniv = "<built-in>", + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + .class1_alg_type = 0, + .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha224),ecb(cipher_null))", + .driver_name = "authenc-hmac-sha224-ecb-cipher_null-caam", + .blocksize = NULL_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_null_givencrypt, + .geniv = "<built-in>", + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + }, + .class1_alg_type = 0, + .class2_alg_type = OP_ALG_ALGSEL_SHA224 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha256),ecb(cipher_null))", + .driver_name = "authenc-hmac-sha256-ecb-cipher_null-caam", + .blocksize = NULL_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_null_givencrypt, + .geniv = "<built-in>", + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + .class1_alg_type = 0, + .class2_alg_type = OP_ALG_ALGSEL_SHA256 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha384),ecb(cipher_null))", + .driver_name = "authenc-hmac-sha384-ecb-cipher_null-caam", + .blocksize = NULL_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_null_givencrypt, + .geniv = "<built-in>", + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + }, + .class1_alg_type = 0, + .class2_alg_type = OP_ALG_ALGSEL_SHA384 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, + }, + { + .name = "authenc(hmac(sha512),ecb(cipher_null))", + .driver_name = "authenc-hmac-sha512-ecb-cipher_null-caam", + .blocksize = NULL_BLOCK_SIZE, + .type = CRYPTO_ALG_TYPE_AEAD, + .template_aead = { + .setkey = aead_setkey, + .setauthsize = aead_setauthsize, + .encrypt = aead_encrypt, + .decrypt = aead_decrypt, + .givencrypt = aead_null_givencrypt, + .geniv = "<built-in>", + .ivsize = NULL_IV_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + }, + .class1_alg_type = 0, + .class2_alg_type = OP_ALG_ALGSEL_SHA512 | + OP_ALG_AAI_HMAC_PRECOMP, + .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, + }, + { .name = "authenc(hmac(md5),cbc(aes))", .driver_name = "authenc-hmac-md5-cbc-aes-caam", .blocksize = AES_BLOCK_SIZE, @@ -2109,6 +2371,11 @@ static void caam_cra_exit(struct crypto_tfm *tfm) dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, desc_bytes(ctx->sh_desc_givenc), DMA_TO_DEVICE); + if (ctx->key_dma && + !dma_mapping_error(ctx->jrdev, ctx->key_dma)) + dma_unmap_single(ctx->jrdev, ctx->key_dma, + ctx->enckeylen + ctx->split_key_pad_len, + DMA_TO_DEVICE); caam_jr_free(ctx->jrdev); } diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 0378328f47a..0d9284ef96a 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -545,7 +545,8 @@ static int ahash_setkey(struct crypto_ahash *ahash, DMA_TO_DEVICE); if (dma_mapping_error(jrdev, ctx->key_dma)) { dev_err(jrdev, "unable to map key i/o memory\n"); - return -ENOMEM; + ret = -ENOMEM; + goto map_err; } #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", @@ -559,6 +560,7 @@ static int ahash_setkey(struct crypto_ahash *ahash, DMA_TO_DEVICE); } +map_err: kfree(hashed_key); return ret; badkey: @@ -631,11 +633,8 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err, edesc = (struct ahash_edesc *)((char *)desc - offsetof(struct ahash_edesc, hw_desc)); - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(jrdev, err); ahash_unmap(jrdev, edesc, req, digestsize); kfree(edesc); @@ -669,11 +668,8 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, edesc = (struct ahash_edesc *)((char *)desc - offsetof(struct ahash_edesc, hw_desc)); - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(jrdev, err); ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); kfree(edesc); @@ -707,11 +703,8 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, edesc = (struct ahash_edesc *)((char *)desc - offsetof(struct ahash_edesc, hw_desc)); - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(jrdev, err); ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE); kfree(edesc); @@ -745,11 +738,8 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, edesc = (struct ahash_edesc *)((char *)desc - offsetof(struct ahash_edesc, hw_desc)); - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(jrdev, err); ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); kfree(edesc); diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index 28486b19fc3..8c07d3153f1 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -76,7 +76,7 @@ struct caam_rng_ctx { struct buf_data bufs[2]; }; -static struct caam_rng_ctx rng_ctx; +static struct caam_rng_ctx *rng_ctx; static inline void rng_unmap_buf(struct device *jrdev, struct buf_data *bd) { @@ -103,11 +103,8 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context) bd = (struct buf_data *)((char *)desc - offsetof(struct buf_data, hw_desc)); - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(jrdev, err); atomic_set(&bd->empty, BUF_NOT_EMPTY); complete(&bd->filled); @@ -137,7 +134,7 @@ static inline int submit_job(struct caam_rng_ctx *ctx, int to_current) static int caam_read(struct hwrng *rng, void *data, size_t max, bool wait) { - struct caam_rng_ctx *ctx = &rng_ctx; + struct caam_rng_ctx *ctx = rng_ctx; struct buf_data *bd = &ctx->bufs[ctx->current_buf]; int next_buf_idx, copied_idx; int err; @@ -237,12 +234,12 @@ static void caam_cleanup(struct hwrng *rng) struct buf_data *bd; for (i = 0; i < 2; i++) { - bd = &rng_ctx.bufs[i]; + bd = &rng_ctx->bufs[i]; if (atomic_read(&bd->empty) == BUF_PENDING) wait_for_completion(&bd->filled); } - rng_unmap_ctx(&rng_ctx); + rng_unmap_ctx(rng_ctx); } static void caam_init_buf(struct caam_rng_ctx *ctx, int buf_id) @@ -273,8 +270,9 @@ static struct hwrng caam_rng = { static void __exit caam_rng_exit(void) { - caam_jr_free(rng_ctx.jrdev); + caam_jr_free(rng_ctx->jrdev); hwrng_unregister(&caam_rng); + kfree(rng_ctx); } static int __init caam_rng_init(void) @@ -286,8 +284,10 @@ static int __init caam_rng_init(void) pr_err("Job Ring Device allocation for transform failed\n"); return PTR_ERR(dev); } - - caam_init_rng(&rng_ctx, dev); + rng_ctx = kmalloc(sizeof(struct caam_rng_ctx), GFP_DMA); + if (!rng_ctx) + return -ENOMEM; + caam_init_rng(rng_ctx, dev); dev_info(dev, "registering rng-caam\n"); return hwrng_register(&caam_rng); diff --git a/drivers/crypto/caam/compat.h b/drivers/crypto/caam/compat.h index 762aeff626a..f227922cea3 100644 --- a/drivers/crypto/caam/compat.h +++ b/drivers/crypto/caam/compat.h @@ -26,6 +26,7 @@ #include <net/xfrm.h> #include <crypto/algapi.h> +#include <crypto/null.h> #include <crypto/aes.h> #include <crypto/des.h> #include <crypto/sha.h> diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 63fb1af2c43..1c38f86bf63 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -14,7 +14,6 @@ #include "jr.h" #include "desc_constr.h" #include "error.h" -#include "ctrl.h" /* * Descriptor to instantiate RNG State Handle 0 in normal mode and @@ -352,32 +351,17 @@ static void kick_trng(struct platform_device *pdev, int ent_delay) /** * caam_get_era() - Return the ERA of the SEC on SoC, based - * on the SEC_VID register. - * Returns the ERA number (1..4) or -ENOTSUPP if the ERA is unknown. - * @caam_id - the value of the SEC_VID register + * on "sec-era" propery in the DTS. This property is updated by u-boot. **/ -int caam_get_era(u64 caam_id) +int caam_get_era(void) { - struct sec_vid *sec_vid = (struct sec_vid *)&caam_id; - static const struct { - u16 ip_id; - u8 maj_rev; - u8 era; - } caam_eras[] = { - {0x0A10, 1, 1}, - {0x0A10, 2, 2}, - {0x0A12, 1, 3}, - {0x0A14, 1, 3}, - {0x0A14, 2, 4}, - {0x0A16, 1, 4}, - {0x0A11, 1, 4} - }; - int i; - - for (i = 0; i < ARRAY_SIZE(caam_eras); i++) - if (caam_eras[i].ip_id == sec_vid->ip_id && - caam_eras[i].maj_rev == sec_vid->maj_rev) - return caam_eras[i].era; + struct device_node *caam_node; + for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") { + const uint32_t *prop = (uint32_t *)of_get_property(caam_node, + "fsl,sec-era", + NULL); + return prop ? *prop : -ENOTSUPP; + } return -ENOTSUPP; } @@ -443,13 +427,10 @@ static int caam_probe(struct platform_device *pdev) * for all, then go probe each one. */ rspec = 0; - for_each_compatible_node(np, NULL, "fsl,sec-v4.0-job-ring") - rspec++; - if (!rspec) { - /* for backward compatible with device trees */ - for_each_compatible_node(np, NULL, "fsl,sec4.0-job-ring") + for_each_available_child_of_node(nprop, np) + if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || + of_device_is_compatible(np, "fsl,sec4.0-job-ring")) rspec++; - } ctrlpriv->jrpdev = kzalloc(sizeof(struct platform_device *) * rspec, GFP_KERNEL); @@ -460,18 +441,9 @@ static int caam_probe(struct platform_device *pdev) ring = 0; ctrlpriv->total_jobrs = 0; - for_each_compatible_node(np, NULL, "fsl,sec-v4.0-job-ring") { - ctrlpriv->jrpdev[ring] = - of_platform_device_create(np, NULL, dev); - if (!ctrlpriv->jrpdev[ring]) { - pr_warn("JR%d Platform device creation error\n", ring); - continue; - } - ctrlpriv->total_jobrs++; - ring++; - } - if (!ring) { - for_each_compatible_node(np, NULL, "fsl,sec4.0-job-ring") { + for_each_available_child_of_node(nprop, np) + if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || + of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { ctrlpriv->jrpdev[ring] = of_platform_device_create(np, NULL, dev); if (!ctrlpriv->jrpdev[ring]) { @@ -482,7 +454,6 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->total_jobrs++; ring++; } - } /* Check to see if QI present. If so, enable */ ctrlpriv->qi_present = !!(rd_reg64(&topregs->ctrl.perfmon.comp_parms) & @@ -564,7 +535,7 @@ static int caam_probe(struct platform_device *pdev) /* Report "alive" for developer to see */ dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, - caam_get_era(caam_id)); + caam_get_era()); dev_info(dev, "job rings = %d, qi = %d\n", ctrlpriv->total_jobrs, ctrlpriv->qi_present); diff --git a/drivers/crypto/caam/ctrl.h b/drivers/crypto/caam/ctrl.h index 980d44eaaf4..cac5402a46e 100644 --- a/drivers/crypto/caam/ctrl.h +++ b/drivers/crypto/caam/ctrl.h @@ -8,6 +8,6 @@ #define CTRL_H /* Prototypes for backend-level services exposed to APIs */ -int caam_get_era(u64 caam_id); +int caam_get_era(void); #endif /* CTRL_H */ diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index cd5f678847c..7eec20bb384 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -155,21 +155,29 @@ static inline void append_cmd_data(u32 *desc, void *data, int len, append_data(desc, data, len); } -static inline u32 *append_jump(u32 *desc, u32 options) -{ - u32 *cmd = desc_end(desc); - - PRINT_POS; - append_cmd(desc, CMD_JUMP | options); - - return cmd; +#define APPEND_CMD_RET(cmd, op) \ +static inline u32 *append_##cmd(u32 *desc, u32 options) \ +{ \ + u32 *cmd = desc_end(desc); \ + PRINT_POS; \ + append_cmd(desc, CMD_##op | options); \ + return cmd; \ } +APPEND_CMD_RET(jump, JUMP) +APPEND_CMD_RET(move, MOVE) static inline void set_jump_tgt_here(u32 *desc, u32 *jump_cmd) { *jump_cmd = *jump_cmd | (desc_len(desc) - (jump_cmd - desc)); } +static inline void set_move_tgt_here(u32 *desc, u32 *move_cmd) +{ + *move_cmd &= ~MOVE_OFFSET_MASK; + *move_cmd = *move_cmd | ((desc_len(desc) << (MOVE_OFFSET_SHIFT + 2)) & + MOVE_OFFSET_MASK); +} + #define APPEND_CMD(cmd, op) \ static inline void append_##cmd(u32 *desc, u32 options) \ { \ @@ -177,7 +185,6 @@ static inline void append_##cmd(u32 *desc, u32 options) \ append_cmd(desc, CMD_##op | options); \ } APPEND_CMD(operation, OPERATION) -APPEND_CMD(move, MOVE) #define APPEND_CMD_LEN(cmd, op) \ static inline void append_##cmd(u32 *desc, unsigned int len, u32 options) \ @@ -328,7 +335,7 @@ append_cmd(desc, CMD_MATH | MATH_FUN_##op | MATH_DEST_##dest | \ do { \ APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ); \ append_cmd(desc, data); \ -} while (0); +} while (0) #define append_math_add_imm_u32(desc, dest, src0, src1, data) \ APPEND_MATH_IMM_u32(ADD, desc, dest, src0, src1, data) diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c index 9f25f529602..6531054a44c 100644 --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -11,243 +11,208 @@ #include "jr.h" #include "error.h" -#define SPRINTFCAT(str, format, param, max_alloc) \ -{ \ - char *tmp; \ - \ - tmp = kmalloc(sizeof(format) + max_alloc, GFP_ATOMIC); \ - sprintf(tmp, format, param); \ - strcat(str, tmp); \ - kfree(tmp); \ -} - -static void report_jump_idx(u32 status, char *outstr) +static const struct { + u8 value; + const char *error_text; +} desc_error_list[] = { + { 0x00, "No error." }, + { 0x01, "SGT Length Error. The descriptor is trying to read more data than is contained in the SGT table." }, + { 0x02, "SGT Null Entry Error." }, + { 0x03, "Job Ring Control Error. There is a bad value in the Job Ring Control register." }, + { 0x04, "Invalid Descriptor Command. The Descriptor Command field is invalid." }, + { 0x05, "Reserved." }, + { 0x06, "Invalid KEY Command" }, + { 0x07, "Invalid LOAD Command" }, + { 0x08, "Invalid STORE Command" }, + { 0x09, "Invalid OPERATION Command" }, + { 0x0A, "Invalid FIFO LOAD Command" }, + { 0x0B, "Invalid FIFO STORE Command" }, + { 0x0C, "Invalid MOVE/MOVE_LEN Command" }, + { 0x0D, "Invalid JUMP Command. A nonlocal JUMP Command is invalid because the target is not a Job Header Command, or the jump is from a Trusted Descriptor to a Job Descriptor, or because the target Descriptor contains a Shared Descriptor." }, + { 0x0E, "Invalid MATH Command" }, + { 0x0F, "Invalid SIGNATURE Command" }, + { 0x10, "Invalid Sequence Command. A SEQ IN PTR OR SEQ OUT PTR Command is invalid or a SEQ KEY, SEQ LOAD, SEQ FIFO LOAD, or SEQ FIFO STORE decremented the input or output sequence length below 0. This error may result if a built-in PROTOCOL Command has encountered a malformed PDU." }, + { 0x11, "Skip data type invalid. The type must be 0xE or 0xF."}, + { 0x12, "Shared Descriptor Header Error" }, + { 0x13, "Header Error. Invalid length or parity, or certain other problems." }, + { 0x14, "Burster Error. Burster has gotten to an illegal state" }, + { 0x15, "Context Register Length Error. The descriptor is trying to read or write past the end of the Context Register. A SEQ LOAD or SEQ STORE with the VLF bit set was executed with too large a length in the variable length register (VSOL for SEQ STORE or VSIL for SEQ LOAD)." }, + { 0x16, "DMA Error" }, + { 0x17, "Reserved." }, + { 0x1A, "Job failed due to JR reset" }, + { 0x1B, "Job failed due to Fail Mode" }, + { 0x1C, "DECO Watchdog timer timeout error" }, + { 0x1D, "DECO tried to copy a key from another DECO but the other DECO's Key Registers were locked" }, + { 0x1E, "DECO attempted to copy data from a DECO that had an unmasked Descriptor error" }, + { 0x1F, "LIODN error. DECO was trying to share from itself or from another DECO but the two Non-SEQ LIODN values didn't match or the 'shared from' DECO's Descriptor required that the SEQ LIODNs be the same and they aren't." }, + { 0x20, "DECO has completed a reset initiated via the DRR register" }, + { 0x21, "Nonce error. When using EKT (CCM) key encryption option in the FIFO STORE Command, the Nonce counter reached its maximum value and this encryption mode can no longer be used." }, + { 0x22, "Meta data is too large (> 511 bytes) for TLS decap (input frame; block ciphers) and IPsec decap (output frame, when doing the next header byte update) and DCRC (output frame)." }, + { 0x23, "Read Input Frame error" }, + { 0x24, "JDKEK, TDKEK or TDSK not loaded error" }, + { 0x80, "DNR (do not run) error" }, + { 0x81, "undefined protocol command" }, + { 0x82, "invalid setting in PDB" }, + { 0x83, "Anti-replay LATE error" }, + { 0x84, "Anti-replay REPLAY error" }, + { 0x85, "Sequence number overflow" }, + { 0x86, "Sigver invalid signature" }, + { 0x87, "DSA Sign Illegal test descriptor" }, + { 0x88, "Protocol Format Error - A protocol has seen an error in the format of data received. When running RSA, this means that formatting with random padding was used, and did not follow the form: 0x00, 0x02, 8-to-N bytes of non-zero pad, 0x00, F data." }, + { 0x89, "Protocol Size Error - A protocol has seen an error in size. When running RSA, pdb size N < (size of F) when no formatting is used; or pdb size N < (F + 11) when formatting is used." }, + { 0xC1, "Blob Command error: Undefined mode" }, + { 0xC2, "Blob Command error: Secure Memory Blob mode error" }, + { 0xC4, "Blob Command error: Black Blob key or input size error" }, + { 0xC5, "Blob Command error: Invalid key destination" }, + { 0xC8, "Blob Command error: Trusted/Secure mode error" }, + { 0xF0, "IPsec TTL or hop limit field either came in as 0, or was decremented to 0" }, + { 0xF1, "3GPP HFN matches or exceeds the Threshold" }, +}; + +static const char * const cha_id_list[] = { + "", + "AES", + "DES", + "ARC4", + "MDHA", + "RNG", + "SNOW f8", + "Kasumi f8/9", + "PKHA", + "CRCA", + "SNOW f9", + "ZUCE", + "ZUCA", +}; + +static const char * const err_id_list[] = { + "No error.", + "Mode error.", + "Data size error.", + "Key size error.", + "PKHA A memory size error.", + "PKHA B memory size error.", + "Data arrived out of sequence error.", + "PKHA divide-by-zero error.", + "PKHA modulus even error.", + "DES key parity error.", + "ICV check failed.", + "Hardware error.", + "Unsupported CCM AAD size.", + "Class 1 CHA is not reset", + "Invalid CHA combination was selected", + "Invalid CHA selected.", +}; + +static const char * const rng_err_id_list[] = { + "", + "", + "", + "Instantiate", + "Not instantiated", + "Test instantiate", + "Prediction resistance", + "Prediction resistance and test request", + "Uninstantiate", + "Secure key generation", +}; + +static void report_ccb_status(struct device *jrdev, const u32 status, + const char *error) { + u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >> + JRSTA_CCBERR_CHAID_SHIFT; + u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >> JRSTA_DECOERR_INDEX_SHIFT; + char *idx_str; + const char *cha_str = "unidentified cha_id value 0x"; + char cha_err_code[3] = { 0 }; + const char *err_str = "unidentified err_id value 0x"; + char err_err_code[3] = { 0 }; if (status & JRSTA_DECOERR_JUMP) - strcat(outstr, "jump tgt desc idx "); + idx_str = "jump tgt desc idx"; else - strcat(outstr, "desc idx "); - - SPRINTFCAT(outstr, "%d: ", idx, sizeof("255")); -} - -static void report_ccb_status(u32 status, char *outstr) -{ - static const char * const cha_id_list[] = { - "", - "AES", - "DES", - "ARC4", - "MDHA", - "RNG", - "SNOW f8", - "Kasumi f8/9", - "PKHA", - "CRCA", - "SNOW f9", - "ZUCE", - "ZUCA", - }; - static const char * const err_id_list[] = { - "No error.", - "Mode error.", - "Data size error.", - "Key size error.", - "PKHA A memory size error.", - "PKHA B memory size error.", - "Data arrived out of sequence error.", - "PKHA divide-by-zero error.", - "PKHA modulus even error.", - "DES key parity error.", - "ICV check failed.", - "Hardware error.", - "Unsupported CCM AAD size.", - "Class 1 CHA is not reset", - "Invalid CHA combination was selected", - "Invalid CHA selected.", - }; - static const char * const rng_err_id_list[] = { - "", - "", - "", - "Instantiate", - "Not instantiated", - "Test instantiate", - "Prediction resistance", - "Prediction resistance and test request", - "Uninstantiate", - "Secure key generation", - }; - u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >> - JRSTA_CCBERR_CHAID_SHIFT; - u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; + idx_str = "desc idx"; - report_jump_idx(status, outstr); - - if (cha_id < ARRAY_SIZE(cha_id_list)) { - SPRINTFCAT(outstr, "%s: ", cha_id_list[cha_id], - strlen(cha_id_list[cha_id])); - } else { - SPRINTFCAT(outstr, "unidentified cha_id value 0x%02x: ", - cha_id, sizeof("ff")); - } + if (cha_id < ARRAY_SIZE(cha_id_list)) + cha_str = cha_id_list[cha_id]; + else + snprintf(cha_err_code, sizeof(cha_err_code), "%02x", cha_id); if ((cha_id << JRSTA_CCBERR_CHAID_SHIFT) == JRSTA_CCBERR_CHAID_RNG && err_id < ARRAY_SIZE(rng_err_id_list) && strlen(rng_err_id_list[err_id])) { /* RNG-only error */ - SPRINTFCAT(outstr, "%s", rng_err_id_list[err_id], - strlen(rng_err_id_list[err_id])); - } else if (err_id < ARRAY_SIZE(err_id_list)) { - SPRINTFCAT(outstr, "%s", err_id_list[err_id], - strlen(err_id_list[err_id])); - } else { - SPRINTFCAT(outstr, "unidentified err_id value 0x%02x", - err_id, sizeof("ff")); - } + err_str = rng_err_id_list[err_id]; + } else if (err_id < ARRAY_SIZE(err_id_list)) + err_str = err_id_list[err_id]; + else + snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); + + dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n", + status, error, idx_str, idx, + cha_str, cha_err_code, + err_str, err_err_code); } -static void report_jump_status(u32 status, char *outstr) +static void report_jump_status(struct device *jrdev, const u32 status, + const char *error) { - SPRINTFCAT(outstr, "%s() not implemented", __func__, sizeof(__func__)); + dev_err(jrdev, "%08x: %s: %s() not implemented\n", + status, error, __func__); } -static void report_deco_status(u32 status, char *outstr) +static void report_deco_status(struct device *jrdev, const u32 status, + const char *error) { - static const struct { - u8 value; - char *error_text; - } desc_error_list[] = { - { 0x00, "No error." }, - { 0x01, "SGT Length Error. The descriptor is trying to read " - "more data than is contained in the SGT table." }, - { 0x02, "SGT Null Entry Error." }, - { 0x03, "Job Ring Control Error. There is a bad value in the " - "Job Ring Control register." }, - { 0x04, "Invalid Descriptor Command. The Descriptor Command " - "field is invalid." }, - { 0x05, "Reserved." }, - { 0x06, "Invalid KEY Command" }, - { 0x07, "Invalid LOAD Command" }, - { 0x08, "Invalid STORE Command" }, - { 0x09, "Invalid OPERATION Command" }, - { 0x0A, "Invalid FIFO LOAD Command" }, - { 0x0B, "Invalid FIFO STORE Command" }, - { 0x0C, "Invalid MOVE/MOVE_LEN Command" }, - { 0x0D, "Invalid JUMP Command. A nonlocal JUMP Command is " - "invalid because the target is not a Job Header " - "Command, or the jump is from a Trusted Descriptor to " - "a Job Descriptor, or because the target Descriptor " - "contains a Shared Descriptor." }, - { 0x0E, "Invalid MATH Command" }, - { 0x0F, "Invalid SIGNATURE Command" }, - { 0x10, "Invalid Sequence Command. A SEQ IN PTR OR SEQ OUT PTR " - "Command is invalid or a SEQ KEY, SEQ LOAD, SEQ FIFO " - "LOAD, or SEQ FIFO STORE decremented the input or " - "output sequence length below 0. This error may result " - "if a built-in PROTOCOL Command has encountered a " - "malformed PDU." }, - { 0x11, "Skip data type invalid. The type must be 0xE or 0xF."}, - { 0x12, "Shared Descriptor Header Error" }, - { 0x13, "Header Error. Invalid length or parity, or certain " - "other problems." }, - { 0x14, "Burster Error. Burster has gotten to an illegal " - "state" }, - { 0x15, "Context Register Length Error. The descriptor is " - "trying to read or write past the end of the Context " - "Register. A SEQ LOAD or SEQ STORE with the VLF bit " - "set was executed with too large a length in the " - "variable length register (VSOL for SEQ STORE or VSIL " - "for SEQ LOAD)." }, - { 0x16, "DMA Error" }, - { 0x17, "Reserved." }, - { 0x1A, "Job failed due to JR reset" }, - { 0x1B, "Job failed due to Fail Mode" }, - { 0x1C, "DECO Watchdog timer timeout error" }, - { 0x1D, "DECO tried to copy a key from another DECO but the " - "other DECO's Key Registers were locked" }, - { 0x1E, "DECO attempted to copy data from a DECO that had an " - "unmasked Descriptor error" }, - { 0x1F, "LIODN error. DECO was trying to share from itself or " - "from another DECO but the two Non-SEQ LIODN values " - "didn't match or the 'shared from' DECO's Descriptor " - "required that the SEQ LIODNs be the same and they " - "aren't." }, - { 0x20, "DECO has completed a reset initiated via the DRR " - "register" }, - { 0x21, "Nonce error. When using EKT (CCM) key encryption " - "option in the FIFO STORE Command, the Nonce counter " - "reached its maximum value and this encryption mode " - "can no longer be used." }, - { 0x22, "Meta data is too large (> 511 bytes) for TLS decap " - "(input frame; block ciphers) and IPsec decap (output " - "frame, when doing the next header byte update) and " - "DCRC (output frame)." }, - { 0x23, "Read Input Frame error" }, - { 0x24, "JDKEK, TDKEK or TDSK not loaded error" }, - { 0x80, "DNR (do not run) error" }, - { 0x81, "undefined protocol command" }, - { 0x82, "invalid setting in PDB" }, - { 0x83, "Anti-replay LATE error" }, - { 0x84, "Anti-replay REPLAY error" }, - { 0x85, "Sequence number overflow" }, - { 0x86, "Sigver invalid signature" }, - { 0x87, "DSA Sign Illegal test descriptor" }, - { 0x88, "Protocol Format Error - A protocol has seen an error " - "in the format of data received. When running RSA, " - "this means that formatting with random padding was " - "used, and did not follow the form: 0x00, 0x02, 8-to-N " - "bytes of non-zero pad, 0x00, F data." }, - { 0x89, "Protocol Size Error - A protocol has seen an error in " - "size. When running RSA, pdb size N < (size of F) when " - "no formatting is used; or pdb size N < (F + 11) when " - "formatting is used." }, - { 0xC1, "Blob Command error: Undefined mode" }, - { 0xC2, "Blob Command error: Secure Memory Blob mode error" }, - { 0xC4, "Blob Command error: Black Blob key or input size " - "error" }, - { 0xC5, "Blob Command error: Invalid key destination" }, - { 0xC8, "Blob Command error: Trusted/Secure mode error" }, - { 0xF0, "IPsec TTL or hop limit field either came in as 0, " - "or was decremented to 0" }, - { 0xF1, "3GPP HFN matches or exceeds the Threshold" }, - }; - u8 desc_error = status & JRSTA_DECOERR_ERROR_MASK; + u8 err_id = status & JRSTA_DECOERR_ERROR_MASK; + u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >> + JRSTA_DECOERR_INDEX_SHIFT; + char *idx_str; + const char *err_str = "unidentified error value 0x"; + char err_err_code[3] = { 0 }; int i; - report_jump_idx(status, outstr); + if (status & JRSTA_DECOERR_JUMP) + idx_str = "jump tgt desc idx"; + else + idx_str = "desc idx"; for (i = 0; i < ARRAY_SIZE(desc_error_list); i++) - if (desc_error_list[i].value == desc_error) + if (desc_error_list[i].value == err_id) break; - if (i != ARRAY_SIZE(desc_error_list) && desc_error_list[i].error_text) { - SPRINTFCAT(outstr, "%s", desc_error_list[i].error_text, - strlen(desc_error_list[i].error_text)); - } else { - SPRINTFCAT(outstr, "unidentified error value 0x%02x", - desc_error, sizeof("ff")); - } + if (i != ARRAY_SIZE(desc_error_list) && desc_error_list[i].error_text) + err_str = desc_error_list[i].error_text; + else + snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); + + dev_err(jrdev, "%08x: %s: %s %d: %s%s\n", + status, error, idx_str, idx, err_str, err_err_code); } -static void report_jr_status(u32 status, char *outstr) +static void report_jr_status(struct device *jrdev, const u32 status, + const char *error) { - SPRINTFCAT(outstr, "%s() not implemented", __func__, sizeof(__func__)); + dev_err(jrdev, "%08x: %s: %s() not implemented\n", + status, error, __func__); } -static void report_cond_code_status(u32 status, char *outstr) +static void report_cond_code_status(struct device *jrdev, const u32 status, + const char *error) { - SPRINTFCAT(outstr, "%s() not implemented", __func__, sizeof(__func__)); + dev_err(jrdev, "%08x: %s: %s() not implemented\n", + status, error, __func__); } -char *caam_jr_strstatus(char *outstr, u32 status) +void caam_jr_strstatus(struct device *jrdev, u32 status) { static const struct stat_src { - void (*report_ssed)(u32 status, char *outstr); - char *error; + void (*report_ssed)(struct device *jrdev, const u32 status, + const char *error); + const char *error; } status_src[] = { { NULL, "No error" }, { NULL, NULL }, @@ -259,12 +224,16 @@ char *caam_jr_strstatus(char *outstr, u32 status) { report_cond_code_status, "Condition Code" }, }; u32 ssrc = status >> JRSTA_SSRC_SHIFT; - - sprintf(outstr, "%s: ", status_src[ssrc].error); - - if (status_src[ssrc].report_ssed) - status_src[ssrc].report_ssed(status, outstr); - - return outstr; + const char *error = status_src[ssrc].error; + + /* + * If there is no further error handling function, just + * print the error code, error string and exit. Otherwise + * call the handler function. + */ + if (!status_src[ssrc].report_ssed) + dev_err(jrdev, "%08x: %s: \n", status, status_src[ssrc].error); + else + status_src[ssrc].report_ssed(jrdev, status, error); } EXPORT_SYMBOL(caam_jr_strstatus); diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h index 02c7baa1748..b6350b0d915 100644 --- a/drivers/crypto/caam/error.h +++ b/drivers/crypto/caam/error.h @@ -7,5 +7,5 @@ #ifndef CAAM_ERROR_H #define CAAM_ERROR_H #define CAAM_ERROR_STR_MAX 302 -extern char *caam_jr_strstatus(char *outstr, u32 status); +void caam_jr_strstatus(struct device *jrdev, u32 status); #endif /* CAAM_ERROR_H */ diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 1d80bd3636c..b512a4ba756 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -453,8 +453,8 @@ static int caam_jr_probe(struct platform_device *pdev) int error; jrdev = &pdev->dev; - jrpriv = kmalloc(sizeof(struct caam_drv_private_jr), - GFP_KERNEL); + jrpriv = devm_kmalloc(jrdev, sizeof(struct caam_drv_private_jr), + GFP_KERNEL); if (!jrpriv) return -ENOMEM; @@ -487,10 +487,8 @@ static int caam_jr_probe(struct platform_device *pdev) /* Now do the platform independent part */ error = caam_jr_init(jrdev); /* now turn on hardware */ - if (error) { - kfree(jrpriv); + if (error) return error; - } jrpriv->dev = jrdev; spin_lock(&driver_data.jr_alloc_lock); diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c index ea2e406610e..871703c49d2 100644 --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -19,11 +19,8 @@ void split_key_done(struct device *dev, u32 *desc, u32 err, dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); #endif - if (err) { - char tmp[CAAM_ERROR_STR_MAX]; - - dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err)); - } + if (err) + caam_jr_strstatus(dev, err); res->err = err; diff --git a/drivers/crypto/caam/regs.h b/drivers/crypto/caam/regs.h index d50174f45b2..cbde8b95a6f 100644 --- a/drivers/crypto/caam/regs.h +++ b/drivers/crypto/caam/regs.h @@ -74,10 +74,10 @@ #endif #else #ifdef __LITTLE_ENDIAN -#define wr_reg32(reg, data) __raw_writel(reg, data) +#define wr_reg32(reg, data) __raw_writel(data, reg) #define rd_reg32(reg) __raw_readl(reg) #ifdef CONFIG_64BIT -#define wr_reg64(reg, data) __raw_writeq(reg, data) +#define wr_reg64(reg, data) __raw_writeq(data, reg) #define rd_reg64(reg) __raw_readq(reg) #endif #endif diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig new file mode 100644 index 00000000000..7639ffc36c6 --- /dev/null +++ b/drivers/crypto/ccp/Kconfig @@ -0,0 +1,24 @@ +config CRYPTO_DEV_CCP_DD + tristate "Cryptographic Coprocessor device driver" + depends on CRYPTO_DEV_CCP + default m + select HW_RANDOM + help + Provides the interface to use the AMD Cryptographic Coprocessor + which can be used to accelerate or offload encryption operations + such as SHA, AES and more. If you choose 'M' here, this module + will be called ccp. + +config CRYPTO_DEV_CCP_CRYPTO + tristate "Encryption and hashing acceleration support" + depends on CRYPTO_DEV_CCP_DD + default m + select CRYPTO_ALGAPI + select CRYPTO_HASH + select CRYPTO_BLKCIPHER + select CRYPTO_AUTHENC + help + Support for using the cryptographic API with the AMD Cryptographic + Coprocessor. This module supports acceleration and offload of SHA + and AES algorithms. If you choose 'M' here, this module will be + called ccp_crypto. diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile new file mode 100644 index 00000000000..d3505a01872 --- /dev/null +++ b/drivers/crypto/ccp/Makefile @@ -0,0 +1,10 @@ +obj-$(CONFIG_CRYPTO_DEV_CCP_DD) += ccp.o +ccp-objs := ccp-dev.o ccp-ops.o +ccp-objs += ccp-pci.o + +obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o +ccp-crypto-objs := ccp-crypto-main.o \ + ccp-crypto-aes.o \ + ccp-crypto-aes-cmac.o \ + ccp-crypto-aes-xts.o \ + ccp-crypto-sha.o diff --git a/drivers/crypto/ccp/ccp-crypto-aes-cmac.c b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c new file mode 100644 index 00000000000..8e162ad8208 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes-cmac.c @@ -0,0 +1,365 @@ +/* + * AMD Cryptographic Coprocessor (CCP) AES CMAC crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/scatterlist.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/aes.h> +#include <crypto/hash.h> +#include <crypto/internal/hash.h> +#include <crypto/scatterwalk.h> + +#include "ccp-crypto.h" + + +static int ccp_aes_cmac_complete(struct crypto_async_request *async_req, + int ret) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req); + unsigned int digest_size = crypto_ahash_digestsize(tfm); + + if (ret) + goto e_free; + + if (rctx->hash_rem) { + /* Save remaining data to buffer */ + unsigned int offset = rctx->nbytes - rctx->hash_rem; + scatterwalk_map_and_copy(rctx->buf, rctx->src, + offset, rctx->hash_rem, 0); + rctx->buf_count = rctx->hash_rem; + } else + rctx->buf_count = 0; + + /* Update result area if supplied */ + if (req->result) + memcpy(req->result, rctx->iv, digest_size); + +e_free: + sg_free_table(&rctx->data_sg); + + return ret; +} + +static int ccp_do_cmac_update(struct ahash_request *req, unsigned int nbytes, + unsigned int final) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx(tfm); + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req); + struct scatterlist *sg, *cmac_key_sg = NULL; + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + unsigned int need_pad, sg_count; + gfp_t gfp; + u64 len; + int ret; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (nbytes) + rctx->null_msg = 0; + + len = (u64)rctx->buf_count + (u64)nbytes; + + if (!final && (len <= block_size)) { + scatterwalk_map_and_copy(rctx->buf + rctx->buf_count, req->src, + 0, nbytes, 0); + rctx->buf_count += nbytes; + + return 0; + } + + rctx->src = req->src; + rctx->nbytes = nbytes; + + rctx->final = final; + rctx->hash_rem = final ? 0 : len & (block_size - 1); + rctx->hash_cnt = len - rctx->hash_rem; + if (!final && !rctx->hash_rem) { + /* CCP can't do zero length final, so keep some data around */ + rctx->hash_cnt -= block_size; + rctx->hash_rem = block_size; + } + + if (final && (rctx->null_msg || (len & (block_size - 1)))) + need_pad = 1; + else + need_pad = 0; + + sg_init_one(&rctx->iv_sg, rctx->iv, sizeof(rctx->iv)); + + /* Build the data scatterlist table - allocate enough entries for all + * possible data pieces (buffer, input data, padding) + */ + sg_count = (nbytes) ? sg_nents(req->src) + 2 : 2; + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + ret = sg_alloc_table(&rctx->data_sg, sg_count, gfp); + if (ret) + return ret; + + sg = NULL; + if (rctx->buf_count) { + sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg); + } + + if (nbytes) + sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src); + + if (need_pad) { + int pad_length = block_size - (len & (block_size - 1)); + + rctx->hash_cnt += pad_length; + + memset(rctx->pad, 0, sizeof(rctx->pad)); + rctx->pad[0] = 0x80; + sg_init_one(&rctx->pad_sg, rctx->pad, pad_length); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->pad_sg); + } + if (sg) { + sg_mark_end(sg); + sg = rctx->data_sg.sgl; + } + + /* Initialize the K1/K2 scatterlist */ + if (final) + cmac_key_sg = (need_pad) ? &ctx->u.aes.k2_sg + : &ctx->u.aes.k1_sg; + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_AES; + rctx->cmd.u.aes.type = ctx->u.aes.type; + rctx->cmd.u.aes.mode = ctx->u.aes.mode; + rctx->cmd.u.aes.action = CCP_AES_ACTION_ENCRYPT; + rctx->cmd.u.aes.key = &ctx->u.aes.key_sg; + rctx->cmd.u.aes.key_len = ctx->u.aes.key_len; + rctx->cmd.u.aes.iv = &rctx->iv_sg; + rctx->cmd.u.aes.iv_len = AES_BLOCK_SIZE; + rctx->cmd.u.aes.src = sg; + rctx->cmd.u.aes.src_len = rctx->hash_cnt; + rctx->cmd.u.aes.dst = NULL; + rctx->cmd.u.aes.cmac_key = cmac_key_sg; + rctx->cmd.u.aes.cmac_key_len = ctx->u.aes.kn_len; + rctx->cmd.u.aes.cmac_final = final; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_aes_cmac_init(struct ahash_request *req) +{ + struct ccp_aes_cmac_req_ctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + + rctx->null_msg = 1; + + return 0; +} + +static int ccp_aes_cmac_update(struct ahash_request *req) +{ + return ccp_do_cmac_update(req, req->nbytes, 0); +} + +static int ccp_aes_cmac_final(struct ahash_request *req) +{ + return ccp_do_cmac_update(req, 0, 1); +} + +static int ccp_aes_cmac_finup(struct ahash_request *req) +{ + return ccp_do_cmac_update(req, req->nbytes, 1); +} + +static int ccp_aes_cmac_digest(struct ahash_request *req) +{ + int ret; + + ret = ccp_aes_cmac_init(req); + if (ret) + return ret; + + return ccp_aes_cmac_finup(req); +} + +static int ccp_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct ccp_crypto_ahash_alg *alg = + ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm)); + u64 k0_hi, k0_lo, k1_hi, k1_lo, k2_hi, k2_lo; + u64 rb_hi = 0x00, rb_lo = 0x87; + __be64 *gk; + int ret; + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->u.aes.type = CCP_AES_TYPE_128; + break; + case AES_KEYSIZE_192: + ctx->u.aes.type = CCP_AES_TYPE_192; + break; + case AES_KEYSIZE_256: + ctx->u.aes.type = CCP_AES_TYPE_256; + break; + default: + crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + ctx->u.aes.mode = alg->mode; + + /* Set to zero until complete */ + ctx->u.aes.key_len = 0; + + /* Set the key for the AES cipher used to generate the keys */ + ret = crypto_cipher_setkey(ctx->u.aes.tfm_cipher, key, key_len); + if (ret) + return ret; + + /* Encrypt a block of zeroes - use key area in context */ + memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key)); + crypto_cipher_encrypt_one(ctx->u.aes.tfm_cipher, ctx->u.aes.key, + ctx->u.aes.key); + + /* Generate K1 and K2 */ + k0_hi = be64_to_cpu(*((__be64 *)ctx->u.aes.key)); + k0_lo = be64_to_cpu(*((__be64 *)ctx->u.aes.key + 1)); + + k1_hi = (k0_hi << 1) | (k0_lo >> 63); + k1_lo = k0_lo << 1; + if (ctx->u.aes.key[0] & 0x80) { + k1_hi ^= rb_hi; + k1_lo ^= rb_lo; + } + gk = (__be64 *)ctx->u.aes.k1; + *gk = cpu_to_be64(k1_hi); + gk++; + *gk = cpu_to_be64(k1_lo); + + k2_hi = (k1_hi << 1) | (k1_lo >> 63); + k2_lo = k1_lo << 1; + if (ctx->u.aes.k1[0] & 0x80) { + k2_hi ^= rb_hi; + k2_lo ^= rb_lo; + } + gk = (__be64 *)ctx->u.aes.k2; + *gk = cpu_to_be64(k2_hi); + gk++; + *gk = cpu_to_be64(k2_lo); + + ctx->u.aes.kn_len = sizeof(ctx->u.aes.k1); + sg_init_one(&ctx->u.aes.k1_sg, ctx->u.aes.k1, sizeof(ctx->u.aes.k1)); + sg_init_one(&ctx->u.aes.k2_sg, ctx->u.aes.k2, sizeof(ctx->u.aes.k2)); + + /* Save the supplied key */ + memset(ctx->u.aes.key, 0, sizeof(ctx->u.aes.key)); + memcpy(ctx->u.aes.key, key, key_len); + ctx->u.aes.key_len = key_len; + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return ret; +} + +static int ccp_aes_cmac_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + struct crypto_cipher *cipher_tfm; + + ctx->complete = ccp_aes_cmac_complete; + ctx->u.aes.key_len = 0; + + crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_aes_cmac_req_ctx)); + + cipher_tfm = crypto_alloc_cipher("aes", 0, + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(cipher_tfm)) { + pr_warn("could not load aes cipher driver\n"); + return PTR_ERR(cipher_tfm); + } + ctx->u.aes.tfm_cipher = cipher_tfm; + + return 0; +} + +static void ccp_aes_cmac_cra_exit(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->u.aes.tfm_cipher) + crypto_free_cipher(ctx->u.aes.tfm_cipher); + ctx->u.aes.tfm_cipher = NULL; +} + +int ccp_register_aes_cmac_algs(struct list_head *head) +{ + struct ccp_crypto_ahash_alg *ccp_alg; + struct ahash_alg *alg; + struct hash_alg_common *halg; + struct crypto_alg *base; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + ccp_alg->mode = CCP_AES_MODE_CMAC; + + alg = &ccp_alg->alg; + alg->init = ccp_aes_cmac_init; + alg->update = ccp_aes_cmac_update; + alg->final = ccp_aes_cmac_final; + alg->finup = ccp_aes_cmac_finup; + alg->digest = ccp_aes_cmac_digest; + alg->setkey = ccp_aes_cmac_setkey; + + halg = &alg->halg; + halg->digestsize = AES_BLOCK_SIZE; + + base = &halg->base; + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "cmac(aes)"); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "cmac-aes-ccp"); + base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + base->cra_blocksize = AES_BLOCK_SIZE; + base->cra_ctxsize = sizeof(struct ccp_ctx); + base->cra_priority = CCP_CRA_PRIORITY; + base->cra_type = &crypto_ahash_type; + base->cra_init = ccp_aes_cmac_cra_init; + base->cra_exit = ccp_aes_cmac_cra_exit; + base->cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + pr_err("%s ahash algorithm registration error (%d)\n", + base->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-aes-xts.c b/drivers/crypto/ccp/ccp-crypto-aes-xts.c new file mode 100644 index 00000000000..0cc5594b7de --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes-xts.c @@ -0,0 +1,279 @@ +/* + * AMD Cryptographic Coprocessor (CCP) AES XTS crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/scatterlist.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/aes.h> +#include <crypto/scatterwalk.h> + +#include "ccp-crypto.h" + + +struct ccp_aes_xts_def { + const char *name; + const char *drv_name; +}; + +static struct ccp_aes_xts_def aes_xts_algs[] = { + { + .name = "xts(aes)", + .drv_name = "xts-aes-ccp", + }, +}; + +struct ccp_unit_size_map { + unsigned int size; + u32 value; +}; + +static struct ccp_unit_size_map unit_size_map[] = { + { + .size = 4096, + .value = CCP_XTS_AES_UNIT_SIZE_4096, + }, + { + .size = 2048, + .value = CCP_XTS_AES_UNIT_SIZE_2048, + }, + { + .size = 1024, + .value = CCP_XTS_AES_UNIT_SIZE_1024, + }, + { + .size = 512, + .value = CCP_XTS_AES_UNIT_SIZE_512, + }, + { + .size = 256, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, + { + .size = 128, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, + { + .size = 64, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, + { + .size = 32, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, + { + .size = 16, + .value = CCP_XTS_AES_UNIT_SIZE_16, + }, + { + .size = 1, + .value = CCP_XTS_AES_UNIT_SIZE__LAST, + }, +}; + +static int ccp_aes_xts_complete(struct crypto_async_request *async_req, int ret) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + + if (ret) + return ret; + + memcpy(req->info, rctx->iv, AES_BLOCK_SIZE); + + return 0; +} + +static int ccp_aes_xts_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); + + /* Only support 128-bit AES key with a 128-bit Tweak key, + * otherwise use the fallback + */ + switch (key_len) { + case AES_KEYSIZE_128 * 2: + memcpy(ctx->u.aes.key, key, key_len); + break; + } + ctx->u.aes.key_len = key_len / 2; + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return crypto_ablkcipher_setkey(ctx->u.aes.tfm_ablkcipher, key, + key_len); +} + +static int ccp_aes_xts_crypt(struct ablkcipher_request *req, + unsigned int encrypt) +{ + struct crypto_tfm *tfm = + crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + unsigned int unit; + int ret; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (req->nbytes & (AES_BLOCK_SIZE - 1)) + return -EINVAL; + + if (!req->info) + return -EINVAL; + + for (unit = 0; unit < ARRAY_SIZE(unit_size_map); unit++) + if (!(req->nbytes & (unit_size_map[unit].size - 1))) + break; + + if ((unit_size_map[unit].value == CCP_XTS_AES_UNIT_SIZE__LAST) || + (ctx->u.aes.key_len != AES_KEYSIZE_128)) { + /* Use the fallback to process the request for any + * unsupported unit sizes or key sizes + */ + ablkcipher_request_set_tfm(req, ctx->u.aes.tfm_ablkcipher); + ret = (encrypt) ? crypto_ablkcipher_encrypt(req) : + crypto_ablkcipher_decrypt(req); + ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + + return ret; + } + + memcpy(rctx->iv, req->info, AES_BLOCK_SIZE); + sg_init_one(&rctx->iv_sg, rctx->iv, AES_BLOCK_SIZE); + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_XTS_AES_128; + rctx->cmd.u.xts.action = (encrypt) ? CCP_AES_ACTION_ENCRYPT + : CCP_AES_ACTION_DECRYPT; + rctx->cmd.u.xts.unit_size = unit_size_map[unit].value; + rctx->cmd.u.xts.key = &ctx->u.aes.key_sg; + rctx->cmd.u.xts.key_len = ctx->u.aes.key_len; + rctx->cmd.u.xts.iv = &rctx->iv_sg; + rctx->cmd.u.xts.iv_len = AES_BLOCK_SIZE; + rctx->cmd.u.xts.src = req->src; + rctx->cmd.u.xts.src_len = req->nbytes; + rctx->cmd.u.xts.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_aes_xts_encrypt(struct ablkcipher_request *req) +{ + return ccp_aes_xts_crypt(req, 1); +} + +static int ccp_aes_xts_decrypt(struct ablkcipher_request *req) +{ + return ccp_aes_xts_crypt(req, 0); +} + +static int ccp_aes_xts_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ablkcipher *fallback_tfm; + + ctx->complete = ccp_aes_xts_complete; + ctx->u.aes.key_len = 0; + + fallback_tfm = crypto_alloc_ablkcipher(crypto_tfm_alg_name(tfm), 0, + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fallback_tfm)) { + pr_warn("could not load fallback driver %s\n", + crypto_tfm_alg_name(tfm)); + return PTR_ERR(fallback_tfm); + } + ctx->u.aes.tfm_ablkcipher = fallback_tfm; + + tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx) + + fallback_tfm->base.crt_ablkcipher.reqsize; + + return 0; +} + +static void ccp_aes_xts_cra_exit(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->u.aes.tfm_ablkcipher) + crypto_free_ablkcipher(ctx->u.aes.tfm_ablkcipher); + ctx->u.aes.tfm_ablkcipher = NULL; +} + + +static int ccp_register_aes_xts_alg(struct list_head *head, + const struct ccp_aes_xts_def *def) +{ + struct ccp_crypto_ablkcipher_alg *ccp_alg; + struct crypto_alg *alg; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + alg = &ccp_alg->alg; + + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->drv_name); + alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + alg->cra_blocksize = AES_BLOCK_SIZE; + alg->cra_ctxsize = sizeof(struct ccp_ctx); + alg->cra_priority = CCP_CRA_PRIORITY; + alg->cra_type = &crypto_ablkcipher_type; + alg->cra_ablkcipher.setkey = ccp_aes_xts_setkey; + alg->cra_ablkcipher.encrypt = ccp_aes_xts_encrypt; + alg->cra_ablkcipher.decrypt = ccp_aes_xts_decrypt; + alg->cra_ablkcipher.min_keysize = AES_MIN_KEY_SIZE * 2; + alg->cra_ablkcipher.max_keysize = AES_MAX_KEY_SIZE * 2; + alg->cra_ablkcipher.ivsize = AES_BLOCK_SIZE; + alg->cra_init = ccp_aes_xts_cra_init; + alg->cra_exit = ccp_aes_xts_cra_exit; + alg->cra_module = THIS_MODULE; + + ret = crypto_register_alg(alg); + if (ret) { + pr_err("%s ablkcipher algorithm registration error (%d)\n", + alg->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_aes_xts_algs(struct list_head *head) +{ + int i, ret; + + for (i = 0; i < ARRAY_SIZE(aes_xts_algs); i++) { + ret = ccp_register_aes_xts_alg(head, &aes_xts_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c new file mode 100644 index 00000000000..e46490db0f6 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -0,0 +1,369 @@ +/* + * AMD Cryptographic Coprocessor (CCP) AES crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/scatterlist.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/scatterwalk.h> + +#include "ccp-crypto.h" + + +static int ccp_aes_complete(struct crypto_async_request *async_req, int ret) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + + if (ret) + return ret; + + if (ctx->u.aes.mode != CCP_AES_MODE_ECB) + memcpy(req->info, rctx->iv, AES_BLOCK_SIZE); + + return 0; +} + +static int ccp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); + struct ccp_crypto_ablkcipher_alg *alg = + ccp_crypto_ablkcipher_alg(crypto_ablkcipher_tfm(tfm)); + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->u.aes.type = CCP_AES_TYPE_128; + break; + case AES_KEYSIZE_192: + ctx->u.aes.type = CCP_AES_TYPE_192; + break; + case AES_KEYSIZE_256: + ctx->u.aes.type = CCP_AES_TYPE_256; + break; + default: + crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + ctx->u.aes.mode = alg->mode; + ctx->u.aes.key_len = key_len; + + memcpy(ctx->u.aes.key, key, key_len); + sg_init_one(&ctx->u.aes.key_sg, ctx->u.aes.key, key_len); + + return 0; +} + +static int ccp_aes_crypt(struct ablkcipher_request *req, bool encrypt) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + struct scatterlist *iv_sg = NULL; + unsigned int iv_len = 0; + int ret; + + if (!ctx->u.aes.key_len) + return -EINVAL; + + if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) || + (ctx->u.aes.mode == CCP_AES_MODE_CBC) || + (ctx->u.aes.mode == CCP_AES_MODE_CFB)) && + (req->nbytes & (AES_BLOCK_SIZE - 1))) + return -EINVAL; + + if (ctx->u.aes.mode != CCP_AES_MODE_ECB) { + if (!req->info) + return -EINVAL; + + memcpy(rctx->iv, req->info, AES_BLOCK_SIZE); + iv_sg = &rctx->iv_sg; + iv_len = AES_BLOCK_SIZE; + sg_init_one(iv_sg, rctx->iv, iv_len); + } + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_AES; + rctx->cmd.u.aes.type = ctx->u.aes.type; + rctx->cmd.u.aes.mode = ctx->u.aes.mode; + rctx->cmd.u.aes.action = + (encrypt) ? CCP_AES_ACTION_ENCRYPT : CCP_AES_ACTION_DECRYPT; + rctx->cmd.u.aes.key = &ctx->u.aes.key_sg; + rctx->cmd.u.aes.key_len = ctx->u.aes.key_len; + rctx->cmd.u.aes.iv = iv_sg; + rctx->cmd.u.aes.iv_len = iv_len; + rctx->cmd.u.aes.src = req->src; + rctx->cmd.u.aes.src_len = req->nbytes; + rctx->cmd.u.aes.dst = req->dst; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_aes_encrypt(struct ablkcipher_request *req) +{ + return ccp_aes_crypt(req, true); +} + +static int ccp_aes_decrypt(struct ablkcipher_request *req) +{ + return ccp_aes_crypt(req, false); +} + +static int ccp_aes_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->complete = ccp_aes_complete; + ctx->u.aes.key_len = 0; + + tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx); + + return 0; +} + +static void ccp_aes_cra_exit(struct crypto_tfm *tfm) +{ +} + +static int ccp_aes_rfc3686_complete(struct crypto_async_request *async_req, + int ret) +{ + struct ablkcipher_request *req = ablkcipher_request_cast(async_req); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + + /* Restore the original pointer */ + req->info = rctx->rfc3686_info; + + return ccp_aes_complete(async_req, ret); +} + +static int ccp_aes_rfc3686_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ablkcipher_tfm(tfm)); + + if (key_len < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + key_len -= CTR_RFC3686_NONCE_SIZE; + memcpy(ctx->u.aes.nonce, key + key_len, CTR_RFC3686_NONCE_SIZE); + + return ccp_aes_setkey(tfm, key, key_len); +} + +static int ccp_aes_rfc3686_crypt(struct ablkcipher_request *req, bool encrypt) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct ccp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + u8 *iv; + + /* Initialize the CTR block */ + iv = rctx->rfc3686_iv; + memcpy(iv, ctx->u.aes.nonce, CTR_RFC3686_NONCE_SIZE); + + iv += CTR_RFC3686_NONCE_SIZE; + memcpy(iv, req->info, CTR_RFC3686_IV_SIZE); + + iv += CTR_RFC3686_IV_SIZE; + *(__be32 *)iv = cpu_to_be32(1); + + /* Point to the new IV */ + rctx->rfc3686_info = req->info; + req->info = rctx->rfc3686_iv; + + return ccp_aes_crypt(req, encrypt); +} + +static int ccp_aes_rfc3686_encrypt(struct ablkcipher_request *req) +{ + return ccp_aes_rfc3686_crypt(req, true); +} + +static int ccp_aes_rfc3686_decrypt(struct ablkcipher_request *req) +{ + return ccp_aes_rfc3686_crypt(req, false); +} + +static int ccp_aes_rfc3686_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + ctx->complete = ccp_aes_rfc3686_complete; + ctx->u.aes.key_len = 0; + + tfm->crt_ablkcipher.reqsize = sizeof(struct ccp_aes_req_ctx); + + return 0; +} + +static void ccp_aes_rfc3686_cra_exit(struct crypto_tfm *tfm) +{ +} + +static struct crypto_alg ccp_aes_defaults = { + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ccp_ctx), + .cra_priority = CCP_CRA_PRIORITY, + .cra_type = &crypto_ablkcipher_type, + .cra_init = ccp_aes_cra_init, + .cra_exit = ccp_aes_cra_exit, + .cra_module = THIS_MODULE, + .cra_ablkcipher = { + .setkey = ccp_aes_setkey, + .encrypt = ccp_aes_encrypt, + .decrypt = ccp_aes_decrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + }, +}; + +static struct crypto_alg ccp_aes_rfc3686_defaults = { + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = CTR_RFC3686_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ccp_ctx), + .cra_priority = CCP_CRA_PRIORITY, + .cra_type = &crypto_ablkcipher_type, + .cra_init = ccp_aes_rfc3686_cra_init, + .cra_exit = ccp_aes_rfc3686_cra_exit, + .cra_module = THIS_MODULE, + .cra_ablkcipher = { + .setkey = ccp_aes_rfc3686_setkey, + .encrypt = ccp_aes_rfc3686_encrypt, + .decrypt = ccp_aes_rfc3686_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + }, +}; + +struct ccp_aes_def { + enum ccp_aes_mode mode; + const char *name; + const char *driver_name; + unsigned int blocksize; + unsigned int ivsize; + struct crypto_alg *alg_defaults; +}; + +static struct ccp_aes_def aes_algs[] = { + { + .mode = CCP_AES_MODE_ECB, + .name = "ecb(aes)", + .driver_name = "ecb-aes-ccp", + .blocksize = AES_BLOCK_SIZE, + .ivsize = 0, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CBC, + .name = "cbc(aes)", + .driver_name = "cbc-aes-ccp", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CFB, + .name = "cfb(aes)", + .driver_name = "cfb-aes-ccp", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_OFB, + .name = "ofb(aes)", + .driver_name = "ofb-aes-ccp", + .blocksize = 1, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CTR, + .name = "ctr(aes)", + .driver_name = "ctr-aes-ccp", + .blocksize = 1, + .ivsize = AES_BLOCK_SIZE, + .alg_defaults = &ccp_aes_defaults, + }, + { + .mode = CCP_AES_MODE_CTR, + .name = "rfc3686(ctr(aes))", + .driver_name = "rfc3686-ctr-aes-ccp", + .blocksize = 1, + .ivsize = CTR_RFC3686_IV_SIZE, + .alg_defaults = &ccp_aes_rfc3686_defaults, + }, +}; + +static int ccp_register_aes_alg(struct list_head *head, + const struct ccp_aes_def *def) +{ + struct ccp_crypto_ablkcipher_alg *ccp_alg; + struct crypto_alg *alg; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + ccp_alg->mode = def->mode; + + /* Copy the defaults and override as necessary */ + alg = &ccp_alg->alg; + *alg = *def->alg_defaults; + snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->driver_name); + alg->cra_blocksize = def->blocksize; + alg->cra_ablkcipher.ivsize = def->ivsize; + + ret = crypto_register_alg(alg); + if (ret) { + pr_err("%s ablkcipher algorithm registration error (%d)\n", + alg->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return 0; +} + +int ccp_register_aes_algs(struct list_head *head) +{ + int i, ret; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + ret = ccp_register_aes_alg(head, &aes_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto-main.c b/drivers/crypto/ccp/ccp-crypto-main.c new file mode 100644 index 00000000000..20dc848481e --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-main.c @@ -0,0 +1,388 @@ +/* + * AMD Cryptographic Coprocessor (CCP) crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/ccp.h> +#include <linux/scatterlist.h> +#include <crypto/internal/hash.h> + +#include "ccp-crypto.h" + +MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0.0"); +MODULE_DESCRIPTION("AMD Cryptographic Coprocessor crypto API support"); + +static unsigned int aes_disable; +module_param(aes_disable, uint, 0444); +MODULE_PARM_DESC(aes_disable, "Disable use of AES - any non-zero value"); + +static unsigned int sha_disable; +module_param(sha_disable, uint, 0444); +MODULE_PARM_DESC(sha_disable, "Disable use of SHA - any non-zero value"); + + +/* List heads for the supported algorithms */ +static LIST_HEAD(hash_algs); +static LIST_HEAD(cipher_algs); + +/* For any tfm, requests for that tfm must be returned on the order + * received. With multiple queues available, the CCP can process more + * than one cmd at a time. Therefore we must maintain a cmd list to insure + * the proper ordering of requests on a given tfm. + */ +struct ccp_crypto_queue { + struct list_head cmds; + struct list_head *backlog; + unsigned int cmd_count; +}; +#define CCP_CRYPTO_MAX_QLEN 100 + +static struct ccp_crypto_queue req_queue; +static spinlock_t req_queue_lock; + +struct ccp_crypto_cmd { + struct list_head entry; + + struct ccp_cmd *cmd; + + /* Save the crypto_tfm and crypto_async_request addresses + * separately to avoid any reference to a possibly invalid + * crypto_async_request structure after invoking the request + * callback + */ + struct crypto_async_request *req; + struct crypto_tfm *tfm; + + /* Used for held command processing to determine state */ + int ret; +}; + +struct ccp_crypto_cpu { + struct work_struct work; + struct completion completion; + struct ccp_crypto_cmd *crypto_cmd; + int err; +}; + + +static inline bool ccp_crypto_success(int err) +{ + if (err && (err != -EINPROGRESS) && (err != -EBUSY)) + return false; + + return true; +} + +static struct ccp_crypto_cmd *ccp_crypto_cmd_complete( + struct ccp_crypto_cmd *crypto_cmd, struct ccp_crypto_cmd **backlog) +{ + struct ccp_crypto_cmd *held = NULL, *tmp; + unsigned long flags; + + *backlog = NULL; + + spin_lock_irqsave(&req_queue_lock, flags); + + /* Held cmds will be after the current cmd in the queue so start + * searching for a cmd with a matching tfm for submission. + */ + tmp = crypto_cmd; + list_for_each_entry_continue(tmp, &req_queue.cmds, entry) { + if (crypto_cmd->tfm != tmp->tfm) + continue; + held = tmp; + break; + } + + /* Process the backlog: + * Because cmds can be executed from any point in the cmd list + * special precautions have to be taken when handling the backlog. + */ + if (req_queue.backlog != &req_queue.cmds) { + /* Skip over this cmd if it is the next backlog cmd */ + if (req_queue.backlog == &crypto_cmd->entry) + req_queue.backlog = crypto_cmd->entry.next; + + *backlog = container_of(req_queue.backlog, + struct ccp_crypto_cmd, entry); + req_queue.backlog = req_queue.backlog->next; + + /* Skip over this cmd if it is now the next backlog cmd */ + if (req_queue.backlog == &crypto_cmd->entry) + req_queue.backlog = crypto_cmd->entry.next; + } + + /* Remove the cmd entry from the list of cmds */ + req_queue.cmd_count--; + list_del(&crypto_cmd->entry); + + spin_unlock_irqrestore(&req_queue_lock, flags); + + return held; +} + +static void ccp_crypto_complete(void *data, int err) +{ + struct ccp_crypto_cmd *crypto_cmd = data; + struct ccp_crypto_cmd *held, *next, *backlog; + struct crypto_async_request *req = crypto_cmd->req; + struct ccp_ctx *ctx = crypto_tfm_ctx(req->tfm); + int ret; + + if (err == -EINPROGRESS) { + /* Only propogate the -EINPROGRESS if necessary */ + if (crypto_cmd->ret == -EBUSY) { + crypto_cmd->ret = -EINPROGRESS; + req->complete(req, -EINPROGRESS); + } + + return; + } + + /* Operation has completed - update the queue before invoking + * the completion callbacks and retrieve the next cmd (cmd with + * a matching tfm) that can be submitted to the CCP. + */ + held = ccp_crypto_cmd_complete(crypto_cmd, &backlog); + if (backlog) { + backlog->ret = -EINPROGRESS; + backlog->req->complete(backlog->req, -EINPROGRESS); + } + + /* Transition the state from -EBUSY to -EINPROGRESS first */ + if (crypto_cmd->ret == -EBUSY) + req->complete(req, -EINPROGRESS); + + /* Completion callbacks */ + ret = err; + if (ctx->complete) + ret = ctx->complete(req, ret); + req->complete(req, ret); + + /* Submit the next cmd */ + while (held) { + /* Since we have already queued the cmd, we must indicate that + * we can backlog so as not to "lose" this request. + */ + held->cmd->flags |= CCP_CMD_MAY_BACKLOG; + ret = ccp_enqueue_cmd(held->cmd); + if (ccp_crypto_success(ret)) + break; + + /* Error occurred, report it and get the next entry */ + ctx = crypto_tfm_ctx(held->req->tfm); + if (ctx->complete) + ret = ctx->complete(held->req, ret); + held->req->complete(held->req, ret); + + next = ccp_crypto_cmd_complete(held, &backlog); + if (backlog) { + backlog->ret = -EINPROGRESS; + backlog->req->complete(backlog->req, -EINPROGRESS); + } + + kfree(held); + held = next; + } + + kfree(crypto_cmd); +} + +static int ccp_crypto_enqueue_cmd(struct ccp_crypto_cmd *crypto_cmd) +{ + struct ccp_crypto_cmd *active = NULL, *tmp; + unsigned long flags; + bool free_cmd = true; + int ret; + + spin_lock_irqsave(&req_queue_lock, flags); + + /* Check if the cmd can/should be queued */ + if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { + ret = -EBUSY; + if (!(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) + goto e_lock; + } + + /* Look for an entry with the same tfm. If there is a cmd + * with the same tfm in the list then the current cmd cannot + * be submitted to the CCP yet. + */ + list_for_each_entry(tmp, &req_queue.cmds, entry) { + if (crypto_cmd->tfm != tmp->tfm) + continue; + active = tmp; + break; + } + + ret = -EINPROGRESS; + if (!active) { + ret = ccp_enqueue_cmd(crypto_cmd->cmd); + if (!ccp_crypto_success(ret)) + goto e_lock; /* Error, don't queue it */ + if ((ret == -EBUSY) && + !(crypto_cmd->cmd->flags & CCP_CMD_MAY_BACKLOG)) + goto e_lock; /* Not backlogging, don't queue it */ + } + + if (req_queue.cmd_count >= CCP_CRYPTO_MAX_QLEN) { + ret = -EBUSY; + if (req_queue.backlog == &req_queue.cmds) + req_queue.backlog = &crypto_cmd->entry; + } + crypto_cmd->ret = ret; + + req_queue.cmd_count++; + list_add_tail(&crypto_cmd->entry, &req_queue.cmds); + + free_cmd = false; + +e_lock: + spin_unlock_irqrestore(&req_queue_lock, flags); + + if (free_cmd) + kfree(crypto_cmd); + + return ret; +} + +/** + * ccp_crypto_enqueue_request - queue an crypto async request for processing + * by the CCP + * + * @req: crypto_async_request struct to be processed + * @cmd: ccp_cmd struct to be sent to the CCP + */ +int ccp_crypto_enqueue_request(struct crypto_async_request *req, + struct ccp_cmd *cmd) +{ + struct ccp_crypto_cmd *crypto_cmd; + gfp_t gfp; + + gfp = req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; + + crypto_cmd = kzalloc(sizeof(*crypto_cmd), gfp); + if (!crypto_cmd) + return -ENOMEM; + + /* The tfm pointer must be saved and not referenced from the + * crypto_async_request (req) pointer because it is used after + * completion callback for the request and the req pointer + * might not be valid anymore. + */ + crypto_cmd->cmd = cmd; + crypto_cmd->req = req; + crypto_cmd->tfm = req->tfm; + + cmd->callback = ccp_crypto_complete; + cmd->data = crypto_cmd; + + if (req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) + cmd->flags |= CCP_CMD_MAY_BACKLOG; + else + cmd->flags &= ~CCP_CMD_MAY_BACKLOG; + + return ccp_crypto_enqueue_cmd(crypto_cmd); +} + +struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table, + struct scatterlist *sg_add) +{ + struct scatterlist *sg, *sg_last = NULL; + + for (sg = table->sgl; sg; sg = sg_next(sg)) + if (!sg_page(sg)) + break; + BUG_ON(!sg); + + for (; sg && sg_add; sg = sg_next(sg), sg_add = sg_next(sg_add)) { + sg_set_page(sg, sg_page(sg_add), sg_add->length, + sg_add->offset); + sg_last = sg; + } + BUG_ON(sg_add); + + return sg_last; +} + +static int ccp_register_algs(void) +{ + int ret; + + if (!aes_disable) { + ret = ccp_register_aes_algs(&cipher_algs); + if (ret) + return ret; + + ret = ccp_register_aes_cmac_algs(&hash_algs); + if (ret) + return ret; + + ret = ccp_register_aes_xts_algs(&cipher_algs); + if (ret) + return ret; + } + + if (!sha_disable) { + ret = ccp_register_sha_algs(&hash_algs); + if (ret) + return ret; + } + + return 0; +} + +static void ccp_unregister_algs(void) +{ + struct ccp_crypto_ahash_alg *ahash_alg, *ahash_tmp; + struct ccp_crypto_ablkcipher_alg *ablk_alg, *ablk_tmp; + + list_for_each_entry_safe(ahash_alg, ahash_tmp, &hash_algs, entry) { + crypto_unregister_ahash(&ahash_alg->alg); + list_del(&ahash_alg->entry); + kfree(ahash_alg); + } + + list_for_each_entry_safe(ablk_alg, ablk_tmp, &cipher_algs, entry) { + crypto_unregister_alg(&ablk_alg->alg); + list_del(&ablk_alg->entry); + kfree(ablk_alg); + } +} + +static int ccp_crypto_init(void) +{ + int ret; + + spin_lock_init(&req_queue_lock); + INIT_LIST_HEAD(&req_queue.cmds); + req_queue.backlog = &req_queue.cmds; + req_queue.cmd_count = 0; + + ret = ccp_register_algs(); + if (ret) + ccp_unregister_algs(); + + return ret; +} + +static void ccp_crypto_exit(void) +{ + ccp_unregister_algs(); +} + +module_init(ccp_crypto_init); +module_exit(ccp_crypto_exit); diff --git a/drivers/crypto/ccp/ccp-crypto-sha.c b/drivers/crypto/ccp/ccp-crypto-sha.c new file mode 100644 index 00000000000..873f2342524 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto-sha.c @@ -0,0 +1,437 @@ +/* + * AMD Cryptographic Coprocessor (CCP) SHA crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/sched.h> +#include <linux/delay.h> +#include <linux/scatterlist.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/hash.h> +#include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <crypto/scatterwalk.h> + +#include "ccp-crypto.h" + + +static int ccp_sha_complete(struct crypto_async_request *async_req, int ret) +{ + struct ahash_request *req = ahash_request_cast(async_req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req); + unsigned int digest_size = crypto_ahash_digestsize(tfm); + + if (ret) + goto e_free; + + if (rctx->hash_rem) { + /* Save remaining data to buffer */ + unsigned int offset = rctx->nbytes - rctx->hash_rem; + scatterwalk_map_and_copy(rctx->buf, rctx->src, + offset, rctx->hash_rem, 0); + rctx->buf_count = rctx->hash_rem; + } else + rctx->buf_count = 0; + + /* Update result area if supplied */ + if (req->result) + memcpy(req->result, rctx->ctx, digest_size); + +e_free: + sg_free_table(&rctx->data_sg); + + return ret; +} + +static int ccp_do_sha_update(struct ahash_request *req, unsigned int nbytes, + unsigned int final) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx(tfm); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct scatterlist *sg; + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + unsigned int sg_count; + gfp_t gfp; + u64 len; + int ret; + + len = (u64)rctx->buf_count + (u64)nbytes; + + if (!final && (len <= block_size)) { + scatterwalk_map_and_copy(rctx->buf + rctx->buf_count, req->src, + 0, nbytes, 0); + rctx->buf_count += nbytes; + + return 0; + } + + rctx->src = req->src; + rctx->nbytes = nbytes; + + rctx->final = final; + rctx->hash_rem = final ? 0 : len & (block_size - 1); + rctx->hash_cnt = len - rctx->hash_rem; + if (!final && !rctx->hash_rem) { + /* CCP can't do zero length final, so keep some data around */ + rctx->hash_cnt -= block_size; + rctx->hash_rem = block_size; + } + + /* Initialize the context scatterlist */ + sg_init_one(&rctx->ctx_sg, rctx->ctx, sizeof(rctx->ctx)); + + sg = NULL; + if (rctx->buf_count && nbytes) { + /* Build the data scatterlist table - allocate enough entries + * for both data pieces (buffer and input data) + */ + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + sg_count = sg_nents(req->src) + 1; + ret = sg_alloc_table(&rctx->data_sg, sg_count, gfp); + if (ret) + return ret; + + sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, &rctx->buf_sg); + sg = ccp_crypto_sg_table_add(&rctx->data_sg, req->src); + sg_mark_end(sg); + + sg = rctx->data_sg.sgl; + } else if (rctx->buf_count) { + sg_init_one(&rctx->buf_sg, rctx->buf, rctx->buf_count); + + sg = &rctx->buf_sg; + } else if (nbytes) { + sg = req->src; + } + + rctx->msg_bits += (rctx->hash_cnt << 3); /* Total in bits */ + + memset(&rctx->cmd, 0, sizeof(rctx->cmd)); + INIT_LIST_HEAD(&rctx->cmd.entry); + rctx->cmd.engine = CCP_ENGINE_SHA; + rctx->cmd.u.sha.type = rctx->type; + rctx->cmd.u.sha.ctx = &rctx->ctx_sg; + rctx->cmd.u.sha.ctx_len = sizeof(rctx->ctx); + rctx->cmd.u.sha.src = sg; + rctx->cmd.u.sha.src_len = rctx->hash_cnt; + rctx->cmd.u.sha.opad = ctx->u.sha.key_len ? + &ctx->u.sha.opad_sg : NULL; + rctx->cmd.u.sha.opad_len = ctx->u.sha.key_len ? + ctx->u.sha.opad_count : 0; + rctx->cmd.u.sha.first = rctx->first; + rctx->cmd.u.sha.final = rctx->final; + rctx->cmd.u.sha.msg_bits = rctx->msg_bits; + + rctx->first = 0; + + ret = ccp_crypto_enqueue_request(&req->base, &rctx->cmd); + + return ret; +} + +static int ccp_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct ccp_ctx *ctx = crypto_ahash_ctx(tfm); + struct ccp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct ccp_crypto_ahash_alg *alg = + ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm)); + unsigned int block_size = + crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); + + memset(rctx, 0, sizeof(*rctx)); + + rctx->type = alg->type; + rctx->first = 1; + + if (ctx->u.sha.key_len) { + /* Buffer the HMAC key for first update */ + memcpy(rctx->buf, ctx->u.sha.ipad, block_size); + rctx->buf_count = block_size; + } + + return 0; +} + +static int ccp_sha_update(struct ahash_request *req) +{ + return ccp_do_sha_update(req, req->nbytes, 0); +} + +static int ccp_sha_final(struct ahash_request *req) +{ + return ccp_do_sha_update(req, 0, 1); +} + +static int ccp_sha_finup(struct ahash_request *req) +{ + return ccp_do_sha_update(req, req->nbytes, 1); +} + +static int ccp_sha_digest(struct ahash_request *req) +{ + int ret; + + ret = ccp_sha_init(req); + if (ret) + return ret; + + return ccp_sha_finup(req); +} + +static int ccp_sha_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int key_len) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm)); + struct crypto_shash *shash = ctx->u.sha.hmac_tfm; + struct { + struct shash_desc sdesc; + char ctx[crypto_shash_descsize(shash)]; + } desc; + unsigned int block_size = crypto_shash_blocksize(shash); + unsigned int digest_size = crypto_shash_digestsize(shash); + int i, ret; + + /* Set to zero until complete */ + ctx->u.sha.key_len = 0; + + /* Clear key area to provide zero padding for keys smaller + * than the block size + */ + memset(ctx->u.sha.key, 0, sizeof(ctx->u.sha.key)); + + if (key_len > block_size) { + /* Must hash the input key */ + desc.sdesc.tfm = shash; + desc.sdesc.flags = crypto_ahash_get_flags(tfm) & + CRYPTO_TFM_REQ_MAY_SLEEP; + + ret = crypto_shash_digest(&desc.sdesc, key, key_len, + ctx->u.sha.key); + if (ret) { + crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + key_len = digest_size; + } else + memcpy(ctx->u.sha.key, key, key_len); + + for (i = 0; i < block_size; i++) { + ctx->u.sha.ipad[i] = ctx->u.sha.key[i] ^ 0x36; + ctx->u.sha.opad[i] = ctx->u.sha.key[i] ^ 0x5c; + } + + sg_init_one(&ctx->u.sha.opad_sg, ctx->u.sha.opad, block_size); + ctx->u.sha.opad_count = block_size; + + ctx->u.sha.key_len = key_len; + + return 0; +} + +static int ccp_sha_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); + + ctx->complete = ccp_sha_complete; + ctx->u.sha.key_len = 0; + + crypto_ahash_set_reqsize(ahash, sizeof(struct ccp_sha_req_ctx)); + + return 0; +} + +static void ccp_sha_cra_exit(struct crypto_tfm *tfm) +{ +} + +static int ccp_hmac_sha_cra_init(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + struct ccp_crypto_ahash_alg *alg = ccp_crypto_ahash_alg(tfm); + struct crypto_shash *hmac_tfm; + + hmac_tfm = crypto_alloc_shash(alg->child_alg, 0, 0); + if (IS_ERR(hmac_tfm)) { + pr_warn("could not load driver %s need for HMAC support\n", + alg->child_alg); + return PTR_ERR(hmac_tfm); + } + + ctx->u.sha.hmac_tfm = hmac_tfm; + + return ccp_sha_cra_init(tfm); +} + +static void ccp_hmac_sha_cra_exit(struct crypto_tfm *tfm) +{ + struct ccp_ctx *ctx = crypto_tfm_ctx(tfm); + + if (ctx->u.sha.hmac_tfm) + crypto_free_shash(ctx->u.sha.hmac_tfm); + + ccp_sha_cra_exit(tfm); +} + +struct ccp_sha_def { + const char *name; + const char *drv_name; + enum ccp_sha_type type; + u32 digest_size; + u32 block_size; +}; + +static struct ccp_sha_def sha_algs[] = { + { + .name = "sha1", + .drv_name = "sha1-ccp", + .type = CCP_SHA_TYPE_1, + .digest_size = SHA1_DIGEST_SIZE, + .block_size = SHA1_BLOCK_SIZE, + }, + { + .name = "sha224", + .drv_name = "sha224-ccp", + .type = CCP_SHA_TYPE_224, + .digest_size = SHA224_DIGEST_SIZE, + .block_size = SHA224_BLOCK_SIZE, + }, + { + .name = "sha256", + .drv_name = "sha256-ccp", + .type = CCP_SHA_TYPE_256, + .digest_size = SHA256_DIGEST_SIZE, + .block_size = SHA256_BLOCK_SIZE, + }, +}; + +static int ccp_register_hmac_alg(struct list_head *head, + const struct ccp_sha_def *def, + const struct ccp_crypto_ahash_alg *base_alg) +{ + struct ccp_crypto_ahash_alg *ccp_alg; + struct ahash_alg *alg; + struct hash_alg_common *halg; + struct crypto_alg *base; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + /* Copy the base algorithm and only change what's necessary */ + *ccp_alg = *base_alg; + INIT_LIST_HEAD(&ccp_alg->entry); + + strncpy(ccp_alg->child_alg, def->name, CRYPTO_MAX_ALG_NAME); + + alg = &ccp_alg->alg; + alg->setkey = ccp_sha_setkey; + + halg = &alg->halg; + + base = &halg->base; + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", def->name); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s", + def->drv_name); + base->cra_init = ccp_hmac_sha_cra_init; + base->cra_exit = ccp_hmac_sha_cra_exit; + + ret = crypto_register_ahash(alg); + if (ret) { + pr_err("%s ahash algorithm registration error (%d)\n", + base->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + return ret; +} + +static int ccp_register_sha_alg(struct list_head *head, + const struct ccp_sha_def *def) +{ + struct ccp_crypto_ahash_alg *ccp_alg; + struct ahash_alg *alg; + struct hash_alg_common *halg; + struct crypto_alg *base; + int ret; + + ccp_alg = kzalloc(sizeof(*ccp_alg), GFP_KERNEL); + if (!ccp_alg) + return -ENOMEM; + + INIT_LIST_HEAD(&ccp_alg->entry); + + ccp_alg->type = def->type; + + alg = &ccp_alg->alg; + alg->init = ccp_sha_init; + alg->update = ccp_sha_update; + alg->final = ccp_sha_final; + alg->finup = ccp_sha_finup; + alg->digest = ccp_sha_digest; + + halg = &alg->halg; + halg->digestsize = def->digest_size; + + base = &halg->base; + snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->drv_name); + base->cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + base->cra_blocksize = def->block_size; + base->cra_ctxsize = sizeof(struct ccp_ctx); + base->cra_priority = CCP_CRA_PRIORITY; + base->cra_type = &crypto_ahash_type; + base->cra_init = ccp_sha_cra_init; + base->cra_exit = ccp_sha_cra_exit; + base->cra_module = THIS_MODULE; + + ret = crypto_register_ahash(alg); + if (ret) { + pr_err("%s ahash algorithm registration error (%d)\n", + base->cra_name, ret); + kfree(ccp_alg); + return ret; + } + + list_add(&ccp_alg->entry, head); + + ret = ccp_register_hmac_alg(head, def, ccp_alg); + + return ret; +} + +int ccp_register_sha_algs(struct list_head *head) +{ + int i, ret; + + for (i = 0; i < ARRAY_SIZE(sha_algs); i++) { + ret = ccp_register_sha_alg(head, &sha_algs[i]); + if (ret) + return ret; + } + + return 0; +} diff --git a/drivers/crypto/ccp/ccp-crypto.h b/drivers/crypto/ccp/ccp-crypto.h new file mode 100644 index 00000000000..9aa4ae184f7 --- /dev/null +++ b/drivers/crypto/ccp/ccp-crypto.h @@ -0,0 +1,197 @@ +/* + * AMD Cryptographic Coprocessor (CCP) crypto API support + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __CCP_CRYPTO_H__ +#define __CCP_CRYPTO_H__ + + +#include <linux/list.h> +#include <linux/wait.h> +#include <linux/pci.h> +#include <linux/ccp.h> +#include <linux/crypto.h> +#include <crypto/algapi.h> +#include <crypto/aes.h> +#include <crypto/ctr.h> +#include <crypto/hash.h> +#include <crypto/sha.h> + + +#define CCP_CRA_PRIORITY 300 + +struct ccp_crypto_ablkcipher_alg { + struct list_head entry; + + u32 mode; + + struct crypto_alg alg; +}; + +struct ccp_crypto_ahash_alg { + struct list_head entry; + + const __be32 *init; + u32 type; + u32 mode; + + /* Child algorithm used for HMAC, CMAC, etc */ + char child_alg[CRYPTO_MAX_ALG_NAME]; + + struct ahash_alg alg; +}; + +static inline struct ccp_crypto_ablkcipher_alg * + ccp_crypto_ablkcipher_alg(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + + return container_of(alg, struct ccp_crypto_ablkcipher_alg, alg); +} + +static inline struct ccp_crypto_ahash_alg * + ccp_crypto_ahash_alg(struct crypto_tfm *tfm) +{ + struct crypto_alg *alg = tfm->__crt_alg; + struct ahash_alg *ahash_alg; + + ahash_alg = container_of(alg, struct ahash_alg, halg.base); + + return container_of(ahash_alg, struct ccp_crypto_ahash_alg, alg); +} + + +/***** AES related defines *****/ +struct ccp_aes_ctx { + /* Fallback cipher for XTS with unsupported unit sizes */ + struct crypto_ablkcipher *tfm_ablkcipher; + + /* Cipher used to generate CMAC K1/K2 keys */ + struct crypto_cipher *tfm_cipher; + + enum ccp_engine engine; + enum ccp_aes_type type; + enum ccp_aes_mode mode; + + struct scatterlist key_sg; + unsigned int key_len; + u8 key[AES_MAX_KEY_SIZE]; + + u8 nonce[CTR_RFC3686_NONCE_SIZE]; + + /* CMAC key structures */ + struct scatterlist k1_sg; + struct scatterlist k2_sg; + unsigned int kn_len; + u8 k1[AES_BLOCK_SIZE]; + u8 k2[AES_BLOCK_SIZE]; +}; + +struct ccp_aes_req_ctx { + struct scatterlist iv_sg; + u8 iv[AES_BLOCK_SIZE]; + + /* Fields used for RFC3686 requests */ + u8 *rfc3686_info; + u8 rfc3686_iv[AES_BLOCK_SIZE]; + + struct ccp_cmd cmd; +}; + +struct ccp_aes_cmac_req_ctx { + unsigned int null_msg; + unsigned int final; + + struct scatterlist *src; + unsigned int nbytes; + + u64 hash_cnt; + unsigned int hash_rem; + + struct sg_table data_sg; + + struct scatterlist iv_sg; + u8 iv[AES_BLOCK_SIZE]; + + struct scatterlist buf_sg; + unsigned int buf_count; + u8 buf[AES_BLOCK_SIZE]; + + struct scatterlist pad_sg; + unsigned int pad_count; + u8 pad[AES_BLOCK_SIZE]; + + struct ccp_cmd cmd; +}; + +/***** SHA related defines *****/ +#define MAX_SHA_CONTEXT_SIZE SHA256_DIGEST_SIZE +#define MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE + +struct ccp_sha_ctx { + struct scatterlist opad_sg; + unsigned int opad_count; + + unsigned int key_len; + u8 key[MAX_SHA_BLOCK_SIZE]; + u8 ipad[MAX_SHA_BLOCK_SIZE]; + u8 opad[MAX_SHA_BLOCK_SIZE]; + struct crypto_shash *hmac_tfm; +}; + +struct ccp_sha_req_ctx { + enum ccp_sha_type type; + + u64 msg_bits; + + unsigned int first; + unsigned int final; + + struct scatterlist *src; + unsigned int nbytes; + + u64 hash_cnt; + unsigned int hash_rem; + + struct sg_table data_sg; + + struct scatterlist ctx_sg; + u8 ctx[MAX_SHA_CONTEXT_SIZE]; + + struct scatterlist buf_sg; + unsigned int buf_count; + u8 buf[MAX_SHA_BLOCK_SIZE]; + + /* CCP driver command */ + struct ccp_cmd cmd; +}; + +/***** Common Context Structure *****/ +struct ccp_ctx { + int (*complete)(struct crypto_async_request *req, int ret); + + union { + struct ccp_aes_ctx aes; + struct ccp_sha_ctx sha; + } u; +}; + +int ccp_crypto_enqueue_request(struct crypto_async_request *req, + struct ccp_cmd *cmd); +struct scatterlist *ccp_crypto_sg_table_add(struct sg_table *table, + struct scatterlist *sg_add); + +int ccp_register_aes_algs(struct list_head *head); +int ccp_register_aes_cmac_algs(struct list_head *head); +int ccp_register_aes_xts_algs(struct list_head *head); +int ccp_register_sha_algs(struct list_head *head); + +#endif diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c new file mode 100644 index 00000000000..2c7816149b0 --- /dev/null +++ b/drivers/crypto/ccp/ccp-dev.c @@ -0,0 +1,608 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/delay.h> +#include <linux/hw_random.h> +#include <linux/cpu.h> +#include <asm/cpu_device_id.h> +#include <linux/ccp.h> + +#include "ccp-dev.h" + +MODULE_AUTHOR("Tom Lendacky <thomas.lendacky@amd.com>"); +MODULE_LICENSE("GPL"); +MODULE_VERSION("1.0.0"); +MODULE_DESCRIPTION("AMD Cryptographic Coprocessor driver"); + +struct ccp_tasklet_data { + struct completion completion; + struct ccp_cmd *cmd; +}; + + +static struct ccp_device *ccp_dev; +static inline struct ccp_device *ccp_get_device(void) +{ + return ccp_dev; +} + +static inline void ccp_add_device(struct ccp_device *ccp) +{ + ccp_dev = ccp; +} + +static inline void ccp_del_device(struct ccp_device *ccp) +{ + ccp_dev = NULL; +} + +/** + * ccp_enqueue_cmd - queue an operation for processing by the CCP + * + * @cmd: ccp_cmd struct to be processed + * + * Queue a cmd to be processed by the CCP. If queueing the cmd + * would exceed the defined length of the cmd queue the cmd will + * only be queued if the CCP_CMD_MAY_BACKLOG flag is set and will + * result in a return code of -EBUSY. + * + * The callback routine specified in the ccp_cmd struct will be + * called to notify the caller of completion (if the cmd was not + * backlogged) or advancement out of the backlog. If the cmd has + * advanced out of the backlog the "err" value of the callback + * will be -EINPROGRESS. Any other "err" value during callback is + * the result of the operation. + * + * The cmd has been successfully queued if: + * the return code is -EINPROGRESS or + * the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set + */ +int ccp_enqueue_cmd(struct ccp_cmd *cmd) +{ + struct ccp_device *ccp = ccp_get_device(); + unsigned long flags; + unsigned int i; + int ret; + + if (!ccp) + return -ENODEV; + + /* Caller must supply a callback routine */ + if (!cmd->callback) + return -EINVAL; + + cmd->ccp = ccp; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + i = ccp->cmd_q_count; + + if (ccp->cmd_count >= MAX_CMD_QLEN) { + ret = -EBUSY; + if (cmd->flags & CCP_CMD_MAY_BACKLOG) + list_add_tail(&cmd->entry, &ccp->backlog); + } else { + ret = -EINPROGRESS; + ccp->cmd_count++; + list_add_tail(&cmd->entry, &ccp->cmd); + + /* Find an idle queue */ + if (!ccp->suspending) { + for (i = 0; i < ccp->cmd_q_count; i++) { + if (ccp->cmd_q[i].active) + continue; + + break; + } + } + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* If we found an idle queue, wake it up */ + if (i < ccp->cmd_q_count) + wake_up_process(ccp->cmd_q[i].kthread); + + return ret; +} +EXPORT_SYMBOL_GPL(ccp_enqueue_cmd); + +static void ccp_do_cmd_backlog(struct work_struct *work) +{ + struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work); + struct ccp_device *ccp = cmd->ccp; + unsigned long flags; + unsigned int i; + + cmd->callback(cmd->data, -EINPROGRESS); + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->cmd_count++; + list_add_tail(&cmd->entry, &ccp->cmd); + + /* Find an idle queue */ + for (i = 0; i < ccp->cmd_q_count; i++) { + if (ccp->cmd_q[i].active) + continue; + + break; + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* If we found an idle queue, wake it up */ + if (i < ccp->cmd_q_count) + wake_up_process(ccp->cmd_q[i].kthread); +} + +static struct ccp_cmd *ccp_dequeue_cmd(struct ccp_cmd_queue *cmd_q) +{ + struct ccp_device *ccp = cmd_q->ccp; + struct ccp_cmd *cmd = NULL; + struct ccp_cmd *backlog = NULL; + unsigned long flags; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + cmd_q->active = 0; + + if (ccp->suspending) { + cmd_q->suspended = 1; + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + wake_up_interruptible(&ccp->suspend_queue); + + return NULL; + } + + if (ccp->cmd_count) { + cmd_q->active = 1; + + cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); + list_del(&cmd->entry); + + ccp->cmd_count--; + } + + if (!list_empty(&ccp->backlog)) { + backlog = list_first_entry(&ccp->backlog, struct ccp_cmd, + entry); + list_del(&backlog->entry); + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + if (backlog) { + INIT_WORK(&backlog->work, ccp_do_cmd_backlog); + schedule_work(&backlog->work); + } + + return cmd; +} + +static void ccp_do_cmd_complete(unsigned long data) +{ + struct ccp_tasklet_data *tdata = (struct ccp_tasklet_data *)data; + struct ccp_cmd *cmd = tdata->cmd; + + cmd->callback(cmd->data, cmd->ret); + complete(&tdata->completion); +} + +static int ccp_cmd_queue_thread(void *data) +{ + struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data; + struct ccp_cmd *cmd; + struct ccp_tasklet_data tdata; + struct tasklet_struct tasklet; + + tasklet_init(&tasklet, ccp_do_cmd_complete, (unsigned long)&tdata); + + set_current_state(TASK_INTERRUPTIBLE); + while (!kthread_should_stop()) { + schedule(); + + set_current_state(TASK_INTERRUPTIBLE); + + cmd = ccp_dequeue_cmd(cmd_q); + if (!cmd) + continue; + + __set_current_state(TASK_RUNNING); + + /* Execute the command */ + cmd->ret = ccp_run_cmd(cmd_q, cmd); + + /* Schedule the completion callback */ + tdata.cmd = cmd; + init_completion(&tdata.completion); + tasklet_schedule(&tasklet); + wait_for_completion(&tdata.completion); + } + + __set_current_state(TASK_RUNNING); + + return 0; +} + +static int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) +{ + struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); + u32 trng_value; + int len = min_t(int, sizeof(trng_value), max); + + /* + * Locking is provided by the caller so we can update device + * hwrng-related fields safely + */ + trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG); + if (!trng_value) { + /* Zero is returned if not data is available or if a + * bad-entropy error is present. Assume an error if + * we exceed TRNG_RETRIES reads of zero. + */ + if (ccp->hwrng_retries++ > TRNG_RETRIES) + return -EIO; + + return 0; + } + + /* Reset the counter and save the rng value */ + ccp->hwrng_retries = 0; + memcpy(data, &trng_value, len); + + return len; +} + +/** + * ccp_alloc_struct - allocate and initialize the ccp_device struct + * + * @dev: device struct of the CCP + */ +struct ccp_device *ccp_alloc_struct(struct device *dev) +{ + struct ccp_device *ccp; + + ccp = kzalloc(sizeof(*ccp), GFP_KERNEL); + if (ccp == NULL) { + dev_err(dev, "unable to allocate device struct\n"); + return NULL; + } + ccp->dev = dev; + + INIT_LIST_HEAD(&ccp->cmd); + INIT_LIST_HEAD(&ccp->backlog); + + spin_lock_init(&ccp->cmd_lock); + mutex_init(&ccp->req_mutex); + mutex_init(&ccp->ksb_mutex); + ccp->ksb_count = KSB_COUNT; + ccp->ksb_start = 0; + + return ccp; +} + +/** + * ccp_init - initialize the CCP device + * + * @ccp: ccp_device struct + */ +int ccp_init(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct ccp_cmd_queue *cmd_q; + struct dma_pool *dma_pool; + char dma_pool_name[MAX_DMAPOOL_NAME_LEN]; + unsigned int qmr, qim, i; + int ret; + + /* Find available queues */ + qim = 0; + qmr = ioread32(ccp->io_regs + Q_MASK_REG); + for (i = 0; i < MAX_HW_QUEUES; i++) { + if (!(qmr & (1 << i))) + continue; + + /* Allocate a dma pool for this queue */ + snprintf(dma_pool_name, sizeof(dma_pool_name), "ccp_q%d", i); + dma_pool = dma_pool_create(dma_pool_name, dev, + CCP_DMAPOOL_MAX_SIZE, + CCP_DMAPOOL_ALIGN, 0); + if (!dma_pool) { + dev_err(dev, "unable to allocate dma pool\n"); + ret = -ENOMEM; + goto e_pool; + } + + cmd_q = &ccp->cmd_q[ccp->cmd_q_count]; + ccp->cmd_q_count++; + + cmd_q->ccp = ccp; + cmd_q->id = i; + cmd_q->dma_pool = dma_pool; + + /* Reserve 2 KSB regions for the queue */ + cmd_q->ksb_key = KSB_START + ccp->ksb_start++; + cmd_q->ksb_ctx = KSB_START + ccp->ksb_start++; + ccp->ksb_count -= 2; + + /* Preset some register values and masks that are queue + * number dependent + */ + cmd_q->reg_status = ccp->io_regs + CMD_Q_STATUS_BASE + + (CMD_Q_STATUS_INCR * i); + cmd_q->reg_int_status = ccp->io_regs + CMD_Q_INT_STATUS_BASE + + (CMD_Q_STATUS_INCR * i); + cmd_q->int_ok = 1 << (i * 2); + cmd_q->int_err = 1 << ((i * 2) + 1); + + cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); + + init_waitqueue_head(&cmd_q->int_queue); + + /* Build queue interrupt mask (two interrupts per queue) */ + qim |= cmd_q->int_ok | cmd_q->int_err; + + dev_dbg(dev, "queue #%u available\n", i); + } + if (ccp->cmd_q_count == 0) { + dev_notice(dev, "no command queues available\n"); + ret = -EIO; + goto e_pool; + } + dev_notice(dev, "%u command queues available\n", ccp->cmd_q_count); + + /* Disable and clear interrupts until ready */ + iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + } + iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); + + /* Request an irq */ + ret = ccp->get_irq(ccp); + if (ret) { + dev_err(dev, "unable to allocate an IRQ\n"); + goto e_pool; + } + + /* Initialize the queues used to wait for KSB space and suspend */ + init_waitqueue_head(&ccp->ksb_queue); + init_waitqueue_head(&ccp->suspend_queue); + + /* Create a kthread for each queue */ + for (i = 0; i < ccp->cmd_q_count; i++) { + struct task_struct *kthread; + + cmd_q = &ccp->cmd_q[i]; + + kthread = kthread_create(ccp_cmd_queue_thread, cmd_q, + "ccp-q%u", cmd_q->id); + if (IS_ERR(kthread)) { + dev_err(dev, "error creating queue thread (%ld)\n", + PTR_ERR(kthread)); + ret = PTR_ERR(kthread); + goto e_kthread; + } + + cmd_q->kthread = kthread; + wake_up_process(kthread); + } + + /* Register the RNG */ + ccp->hwrng.name = "ccp-rng"; + ccp->hwrng.read = ccp_trng_read; + ret = hwrng_register(&ccp->hwrng); + if (ret) { + dev_err(dev, "error registering hwrng (%d)\n", ret); + goto e_kthread; + } + + /* Make the device struct available before enabling interrupts */ + ccp_add_device(ccp); + + /* Enable interrupts */ + iowrite32(qim, ccp->io_regs + IRQ_MASK_REG); + + return 0; + +e_kthread: + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + + ccp->free_irq(ccp); + +e_pool: + for (i = 0; i < ccp->cmd_q_count; i++) + dma_pool_destroy(ccp->cmd_q[i].dma_pool); + + return ret; +} + +/** + * ccp_destroy - tear down the CCP device + * + * @ccp: ccp_device struct + */ +void ccp_destroy(struct ccp_device *ccp) +{ + struct ccp_cmd_queue *cmd_q; + struct ccp_cmd *cmd; + unsigned int qim, i; + + /* Remove general access to the device struct */ + ccp_del_device(ccp); + + /* Unregister the RNG */ + hwrng_unregister(&ccp->hwrng); + + /* Stop the queue kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].kthread) + kthread_stop(ccp->cmd_q[i].kthread); + + /* Build queue interrupt mask (two interrupt masks per queue) */ + qim = 0; + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + qim |= cmd_q->int_ok | cmd_q->int_err; + } + + /* Disable and clear interrupts */ + iowrite32(0x00, ccp->io_regs + IRQ_MASK_REG); + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + ioread32(cmd_q->reg_int_status); + ioread32(cmd_q->reg_status); + } + iowrite32(qim, ccp->io_regs + IRQ_STATUS_REG); + + ccp->free_irq(ccp); + + for (i = 0; i < ccp->cmd_q_count; i++) + dma_pool_destroy(ccp->cmd_q[i].dma_pool); + + /* Flush the cmd and backlog queue */ + while (!list_empty(&ccp->cmd)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } + while (!list_empty(&ccp->backlog)) { + /* Invoke the callback directly with an error code */ + cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry); + list_del(&cmd->entry); + cmd->callback(cmd->data, -ENODEV); + } +} + +/** + * ccp_irq_handler - handle interrupts generated by the CCP device + * + * @irq: the irq associated with the interrupt + * @data: the data value supplied when the irq was created + */ +irqreturn_t ccp_irq_handler(int irq, void *data) +{ + struct device *dev = data; + struct ccp_device *ccp = dev_get_drvdata(dev); + struct ccp_cmd_queue *cmd_q; + u32 q_int, status; + unsigned int i; + + status = ioread32(ccp->io_regs + IRQ_STATUS_REG); + + for (i = 0; i < ccp->cmd_q_count; i++) { + cmd_q = &ccp->cmd_q[i]; + + q_int = status & (cmd_q->int_ok | cmd_q->int_err); + if (q_int) { + cmd_q->int_status = status; + cmd_q->q_status = ioread32(cmd_q->reg_status); + cmd_q->q_int_status = ioread32(cmd_q->reg_int_status); + + /* On error, only save the first error value */ + if ((q_int & cmd_q->int_err) && !cmd_q->cmd_error) + cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status); + + cmd_q->int_rcvd = 1; + + /* Acknowledge the interrupt and wake the kthread */ + iowrite32(q_int, ccp->io_regs + IRQ_STATUS_REG); + wake_up_interruptible(&cmd_q->int_queue); + } + } + + return IRQ_HANDLED; +} + +#ifdef CONFIG_PM +bool ccp_queues_suspended(struct ccp_device *ccp) +{ + unsigned int suspended = 0; + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + for (i = 0; i < ccp->cmd_q_count; i++) + if (ccp->cmd_q[i].suspended) + suspended++; + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + return ccp->cmd_q_count == suspended; +} +#endif + +static const struct x86_cpu_id ccp_support[] = { + { X86_VENDOR_AMD, 22, }, +}; + +static int __init ccp_mod_init(void) +{ + struct cpuinfo_x86 *cpuinfo = &boot_cpu_data; + int ret; + + if (!x86_match_cpu(ccp_support)) + return -ENODEV; + + switch (cpuinfo->x86) { + case 22: + if ((cpuinfo->x86_model < 48) || (cpuinfo->x86_model > 63)) + return -ENODEV; + + ret = ccp_pci_init(); + if (ret) + return ret; + + /* Don't leave the driver loaded if init failed */ + if (!ccp_get_device()) { + ccp_pci_exit(); + return -ENODEV; + } + + return 0; + + break; + } + + return -ENODEV; +} + +static void __exit ccp_mod_exit(void) +{ + struct cpuinfo_x86 *cpuinfo = &boot_cpu_data; + + switch (cpuinfo->x86) { + case 22: + ccp_pci_exit(); + break; + } +} + +module_init(ccp_mod_init); +module_exit(ccp_mod_exit); diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h new file mode 100644 index 00000000000..7ec536e702e --- /dev/null +++ b/drivers/crypto/ccp/ccp-dev.h @@ -0,0 +1,272 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#ifndef __CCP_DEV_H__ +#define __CCP_DEV_H__ + +#include <linux/device.h> +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/wait.h> +#include <linux/dmapool.h> +#include <linux/hw_random.h> + + +#define IO_OFFSET 0x20000 + +#define MAX_DMAPOOL_NAME_LEN 32 + +#define MAX_HW_QUEUES 5 +#define MAX_CMD_QLEN 100 + +#define TRNG_RETRIES 10 + + +/****** Register Mappings ******/ +#define Q_MASK_REG 0x000 +#define TRNG_OUT_REG 0x00c +#define IRQ_MASK_REG 0x040 +#define IRQ_STATUS_REG 0x200 + +#define DEL_CMD_Q_JOB 0x124 +#define DEL_Q_ACTIVE 0x00000200 +#define DEL_Q_ID_SHIFT 6 + +#define CMD_REQ0 0x180 +#define CMD_REQ_INCR 0x04 + +#define CMD_Q_STATUS_BASE 0x210 +#define CMD_Q_INT_STATUS_BASE 0x214 +#define CMD_Q_STATUS_INCR 0x20 + +#define CMD_Q_CACHE 0x228 +#define CMD_Q_CACHE_INC 0x20 + +#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f); +#define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f); + +/****** REQ0 Related Values ******/ +#define REQ0_WAIT_FOR_WRITE 0x00000004 +#define REQ0_INT_ON_COMPLETE 0x00000002 +#define REQ0_STOP_ON_COMPLETE 0x00000001 + +#define REQ0_CMD_Q_SHIFT 9 +#define REQ0_JOBID_SHIFT 3 + +/****** REQ1 Related Values ******/ +#define REQ1_PROTECT_SHIFT 27 +#define REQ1_ENGINE_SHIFT 23 +#define REQ1_KEY_KSB_SHIFT 2 + +#define REQ1_EOM 0x00000002 +#define REQ1_INIT 0x00000001 + +/* AES Related Values */ +#define REQ1_AES_TYPE_SHIFT 21 +#define REQ1_AES_MODE_SHIFT 18 +#define REQ1_AES_ACTION_SHIFT 17 +#define REQ1_AES_CFB_SIZE_SHIFT 10 + +/* XTS-AES Related Values */ +#define REQ1_XTS_AES_SIZE_SHIFT 10 + +/* SHA Related Values */ +#define REQ1_SHA_TYPE_SHIFT 21 + +/* RSA Related Values */ +#define REQ1_RSA_MOD_SIZE_SHIFT 10 + +/* Pass-Through Related Values */ +#define REQ1_PT_BW_SHIFT 12 +#define REQ1_PT_BS_SHIFT 10 + +/* ECC Related Values */ +#define REQ1_ECC_AFFINE_CONVERT 0x00200000 +#define REQ1_ECC_FUNCTION_SHIFT 18 + +/****** REQ4 Related Values ******/ +#define REQ4_KSB_SHIFT 18 +#define REQ4_MEMTYPE_SHIFT 16 + +/****** REQ6 Related Values ******/ +#define REQ6_MEMTYPE_SHIFT 16 + + +/****** Key Storage Block ******/ +#define KSB_START 77 +#define KSB_END 127 +#define KSB_COUNT (KSB_END - KSB_START + 1) +#define CCP_KSB_BITS 256 +#define CCP_KSB_BYTES 32 + +#define CCP_JOBID_MASK 0x0000003f + +#define CCP_DMAPOOL_MAX_SIZE 64 +#define CCP_DMAPOOL_ALIGN (1 << 5) + +#define CCP_REVERSE_BUF_SIZE 64 + +#define CCP_AES_KEY_KSB_COUNT 1 +#define CCP_AES_CTX_KSB_COUNT 1 + +#define CCP_XTS_AES_KEY_KSB_COUNT 1 +#define CCP_XTS_AES_CTX_KSB_COUNT 1 + +#define CCP_SHA_KSB_COUNT 1 + +#define CCP_RSA_MAX_WIDTH 4096 + +#define CCP_PASSTHRU_BLOCKSIZE 256 +#define CCP_PASSTHRU_MASKSIZE 32 +#define CCP_PASSTHRU_KSB_COUNT 1 + +#define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */ +#define CCP_ECC_MAX_OPERANDS 6 +#define CCP_ECC_MAX_OUTPUTS 3 +#define CCP_ECC_SRC_BUF_SIZE 448 +#define CCP_ECC_DST_BUF_SIZE 192 +#define CCP_ECC_OPERAND_SIZE 64 +#define CCP_ECC_OUTPUT_SIZE 64 +#define CCP_ECC_RESULT_OFFSET 60 +#define CCP_ECC_RESULT_SUCCESS 0x0001 + + +struct ccp_device; +struct ccp_cmd; + +struct ccp_cmd_queue { + struct ccp_device *ccp; + + /* Queue identifier */ + u32 id; + + /* Queue dma pool */ + struct dma_pool *dma_pool; + + /* Queue reserved KSB regions */ + u32 ksb_key; + u32 ksb_ctx; + + /* Queue processing thread */ + struct task_struct *kthread; + unsigned int active; + unsigned int suspended; + + /* Number of free command slots available */ + unsigned int free_slots; + + /* Interrupt masks */ + u32 int_ok; + u32 int_err; + + /* Register addresses for queue */ + void __iomem *reg_status; + void __iomem *reg_int_status; + + /* Status values from job */ + u32 int_status; + u32 q_status; + u32 q_int_status; + u32 cmd_error; + + /* Interrupt wait queue */ + wait_queue_head_t int_queue; + unsigned int int_rcvd; +} ____cacheline_aligned; + +struct ccp_device { + struct device *dev; + + /* + * Bus specific device information + */ + void *dev_specific; + int (*get_irq)(struct ccp_device *ccp); + void (*free_irq)(struct ccp_device *ccp); + + /* + * I/O area used for device communication. The register mapping + * starts at an offset into the mapped bar. + * The CMD_REQx registers and the Delete_Cmd_Queue_Job register + * need to be protected while a command queue thread is accessing + * them. + */ + struct mutex req_mutex ____cacheline_aligned; + void __iomem *io_map; + void __iomem *io_regs; + + /* + * Master lists that all cmds are queued on. Because there can be + * more than one CCP command queue that can process a cmd a separate + * backlog list is neeeded so that the backlog completion call + * completes before the cmd is available for execution. + */ + spinlock_t cmd_lock ____cacheline_aligned; + unsigned int cmd_count; + struct list_head cmd; + struct list_head backlog; + + /* + * The command queues. These represent the queues available on the + * CCP that are available for processing cmds + */ + struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES]; + unsigned int cmd_q_count; + + /* + * Support for the CCP True RNG + */ + struct hwrng hwrng; + unsigned int hwrng_retries; + + /* + * A counter used to generate job-ids for cmds submitted to the CCP + */ + atomic_t current_id ____cacheline_aligned; + + /* + * The CCP uses key storage blocks (KSB) to maintain context for certain + * operations. To prevent multiple cmds from using the same KSB range + * a command queue reserves a KSB range for the duration of the cmd. + * Each queue, will however, reserve 2 KSB blocks for operations that + * only require single KSB entries (eg. AES context/iv and key) in order + * to avoid allocation contention. This will reserve at most 10 KSB + * entries, leaving 40 KSB entries available for dynamic allocation. + */ + struct mutex ksb_mutex ____cacheline_aligned; + DECLARE_BITMAP(ksb, KSB_COUNT); + wait_queue_head_t ksb_queue; + unsigned int ksb_avail; + unsigned int ksb_count; + u32 ksb_start; + + /* Suspend support */ + unsigned int suspending; + wait_queue_head_t suspend_queue; +}; + + +int ccp_pci_init(void); +void ccp_pci_exit(void); + +struct ccp_device *ccp_alloc_struct(struct device *dev); +int ccp_init(struct ccp_device *ccp); +void ccp_destroy(struct ccp_device *ccp); +bool ccp_queues_suspended(struct ccp_device *ccp); + +irqreturn_t ccp_irq_handler(int irq, void *data); + +int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd); + +#endif diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c new file mode 100644 index 00000000000..9ae006d69df --- /dev/null +++ b/drivers/crypto/ccp/ccp-ops.c @@ -0,0 +1,2126 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/delay.h> +#include <linux/ccp.h> +#include <linux/scatterlist.h> +#include <crypto/scatterwalk.h> +#include <crypto/sha.h> + +#include "ccp-dev.h" + + +enum ccp_memtype { + CCP_MEMTYPE_SYSTEM = 0, + CCP_MEMTYPE_KSB, + CCP_MEMTYPE_LOCAL, + CCP_MEMTYPE__LAST, +}; + +struct ccp_dma_info { + dma_addr_t address; + unsigned int offset; + unsigned int length; + enum dma_data_direction dir; +}; + +struct ccp_dm_workarea { + struct device *dev; + struct dma_pool *dma_pool; + unsigned int length; + + u8 *address; + struct ccp_dma_info dma; +}; + +struct ccp_sg_workarea { + struct scatterlist *sg; + unsigned int nents; + unsigned int length; + + struct scatterlist *dma_sg; + struct device *dma_dev; + unsigned int dma_count; + enum dma_data_direction dma_dir; + + unsigned int sg_used; + + u64 bytes_left; +}; + +struct ccp_data { + struct ccp_sg_workarea sg_wa; + struct ccp_dm_workarea dm_wa; +}; + +struct ccp_mem { + enum ccp_memtype type; + union { + struct ccp_dma_info dma; + u32 ksb; + } u; +}; + +struct ccp_aes_op { + enum ccp_aes_type type; + enum ccp_aes_mode mode; + enum ccp_aes_action action; +}; + +struct ccp_xts_aes_op { + enum ccp_aes_action action; + enum ccp_xts_aes_unit_size unit_size; +}; + +struct ccp_sha_op { + enum ccp_sha_type type; + u64 msg_bits; +}; + +struct ccp_rsa_op { + u32 mod_size; + u32 input_len; +}; + +struct ccp_passthru_op { + enum ccp_passthru_bitwise bit_mod; + enum ccp_passthru_byteswap byte_swap; +}; + +struct ccp_ecc_op { + enum ccp_ecc_function function; +}; + +struct ccp_op { + struct ccp_cmd_queue *cmd_q; + + u32 jobid; + u32 ioc; + u32 soc; + u32 ksb_key; + u32 ksb_ctx; + u32 init; + u32 eom; + + struct ccp_mem src; + struct ccp_mem dst; + + union { + struct ccp_aes_op aes; + struct ccp_xts_aes_op xts; + struct ccp_sha_op sha; + struct ccp_rsa_op rsa; + struct ccp_passthru_op passthru; + struct ccp_ecc_op ecc; + } u; +}; + +/* SHA initial context values */ +static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { + cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1), + cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3), + cpu_to_be32(SHA1_H4), 0, 0, 0, +}; + +static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { + cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1), + cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3), + cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5), + cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7), +}; + +static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { + cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1), + cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3), + cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5), + cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), +}; + +/* The CCP cannot perform zero-length sha operations so the caller + * is required to buffer data for the final operation. However, a + * sha operation for a message with a total length of zero is valid + * so known values are required to supply the result. + */ +static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = { + 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, + 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, + 0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = { + 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, + 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, + 0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, + 0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00, +}; + +static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = { + 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, + 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, + 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, + 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55, +}; + +static u32 ccp_addr_lo(struct ccp_dma_info *info) +{ + return lower_32_bits(info->address + info->offset); +} + +static u32 ccp_addr_hi(struct ccp_dma_info *info) +{ + return upper_32_bits(info->address + info->offset) & 0x0000ffff; +} + +static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count) +{ + struct ccp_cmd_queue *cmd_q = op->cmd_q; + struct ccp_device *ccp = cmd_q->ccp; + void __iomem *cr_addr; + u32 cr0, cmd; + unsigned int i; + int ret = 0; + + /* We could read a status register to see how many free slots + * are actually available, but reading that register resets it + * and you could lose some error information. + */ + cmd_q->free_slots--; + + cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT) + | (op->jobid << REQ0_JOBID_SHIFT) + | REQ0_WAIT_FOR_WRITE; + + if (op->soc) + cr0 |= REQ0_STOP_ON_COMPLETE + | REQ0_INT_ON_COMPLETE; + + if (op->ioc || !cmd_q->free_slots) + cr0 |= REQ0_INT_ON_COMPLETE; + + /* Start at CMD_REQ1 */ + cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR; + + mutex_lock(&ccp->req_mutex); + + /* Write CMD_REQ1 through CMD_REQx first */ + for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR) + iowrite32(*(cr + i), cr_addr); + + /* Tell the CCP to start */ + wmb(); + iowrite32(cr0, ccp->io_regs + CMD_REQ0); + + mutex_unlock(&ccp->req_mutex); + + if (cr0 & REQ0_INT_ON_COMPLETE) { + /* Wait for the job to complete */ + ret = wait_event_interruptible(cmd_q->int_queue, + cmd_q->int_rcvd); + if (ret || cmd_q->cmd_error) { + /* On error delete all related jobs from the queue */ + cmd = (cmd_q->id << DEL_Q_ID_SHIFT) + | op->jobid; + + iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); + + if (!ret) + ret = -EIO; + } else if (op->soc) { + /* Delete just head job from the queue on SoC */ + cmd = DEL_Q_ACTIVE + | (cmd_q->id << DEL_Q_ID_SHIFT) + | op->jobid; + + iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB); + } + + cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status); + + cmd_q->int_rcvd = 0; + } + + return ret; +} + +static int ccp_perform_aes(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT) + | (op->u.aes.type << REQ1_AES_TYPE_SHIFT) + | (op->u.aes.mode << REQ1_AES_MODE_SHIFT) + | (op->u.aes.action << REQ1_AES_ACTION_SHIFT) + | (op->ksb_key << REQ1_KEY_KSB_SHIFT); + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + if (op->u.aes.mode == CCP_AES_MODE_CFB) + cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT); + + if (op->eom) + cr[0] |= REQ1_EOM; + + if (op->init) + cr[0] |= REQ1_INIT; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_xts_aes(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT) + | (op->u.xts.action << REQ1_AES_ACTION_SHIFT) + | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT) + | (op->ksb_key << REQ1_KEY_KSB_SHIFT); + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + if (op->eom) + cr[0] |= REQ1_EOM; + + if (op->init) + cr[0] |= REQ1_INIT; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_sha(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT) + | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT) + | REQ1_INIT; + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + + if (op->eom) { + cr[0] |= REQ1_EOM; + cr[4] = lower_32_bits(op->u.sha.msg_bits); + cr[5] = upper_32_bits(op->u.sha.msg_bits); + } else { + cr[4] = 0; + cr[5] = 0; + } + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_rsa(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT) + | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT) + | (op->ksb_key << REQ1_KEY_KSB_SHIFT) + | REQ1_EOM; + cr[1] = op->u.rsa.input_len - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT) + | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_passthru(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT) + | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT) + | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT); + + if (op->src.type == CCP_MEMTYPE_SYSTEM) + cr[1] = op->src.u.dma.length - 1; + else + cr[1] = op->dst.u.dma.length - 1; + + if (op->src.type == CCP_MEMTYPE_SYSTEM) { + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + + if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP) + cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT); + } else { + cr[2] = op->src.u.ksb * CCP_KSB_BYTES; + cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT); + } + + if (op->dst.type == CCP_MEMTYPE_SYSTEM) { + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + } else { + cr[4] = op->dst.u.ksb * CCP_KSB_BYTES; + cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT); + } + + if (op->eom) + cr[0] |= REQ1_EOM; + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static int ccp_perform_ecc(struct ccp_op *op) +{ + u32 cr[6]; + + /* Fill out the register contents for REQ1 through REQ6 */ + cr[0] = REQ1_ECC_AFFINE_CONVERT + | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT) + | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT) + | REQ1_EOM; + cr[1] = op->src.u.dma.length - 1; + cr[2] = ccp_addr_lo(&op->src.u.dma); + cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->src.u.dma); + cr[4] = ccp_addr_lo(&op->dst.u.dma); + cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT) + | ccp_addr_hi(&op->dst.u.dma); + + return ccp_do_cmd(op, cr, ARRAY_SIZE(cr)); +} + +static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count) +{ + int start; + + for (;;) { + mutex_lock(&ccp->ksb_mutex); + + start = (u32)bitmap_find_next_zero_area(ccp->ksb, + ccp->ksb_count, + ccp->ksb_start, + count, 0); + if (start <= ccp->ksb_count) { + bitmap_set(ccp->ksb, start, count); + + mutex_unlock(&ccp->ksb_mutex); + break; + } + + ccp->ksb_avail = 0; + + mutex_unlock(&ccp->ksb_mutex); + + /* Wait for KSB entries to become available */ + if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail)) + return 0; + } + + return KSB_START + start; +} + +static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start, + unsigned int count) +{ + if (!start) + return; + + mutex_lock(&ccp->ksb_mutex); + + bitmap_clear(ccp->ksb, start - KSB_START, count); + + ccp->ksb_avail = 1; + + mutex_unlock(&ccp->ksb_mutex); + + wake_up_interruptible_all(&ccp->ksb_queue); +} + +static u32 ccp_gen_jobid(struct ccp_device *ccp) +{ + return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK; +} + +static void ccp_sg_free(struct ccp_sg_workarea *wa) +{ + if (wa->dma_count) + dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir); + + wa->dma_count = 0; +} + +static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev, + struct scatterlist *sg, u64 len, + enum dma_data_direction dma_dir) +{ + memset(wa, 0, sizeof(*wa)); + + wa->sg = sg; + if (!sg) + return 0; + + wa->nents = sg_nents(sg); + wa->length = sg->length; + wa->bytes_left = len; + wa->sg_used = 0; + + if (len == 0) + return 0; + + if (dma_dir == DMA_NONE) + return 0; + + wa->dma_sg = sg; + wa->dma_dev = dev; + wa->dma_dir = dma_dir; + wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir); + if (!wa->dma_count) + return -ENOMEM; + + + return 0; +} + +static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len) +{ + unsigned int nbytes = min_t(u64, len, wa->bytes_left); + + if (!wa->sg) + return; + + wa->sg_used += nbytes; + wa->bytes_left -= nbytes; + if (wa->sg_used == wa->sg->length) { + wa->sg = sg_next(wa->sg); + wa->sg_used = 0; + } +} + +static void ccp_dm_free(struct ccp_dm_workarea *wa) +{ + if (wa->length <= CCP_DMAPOOL_MAX_SIZE) { + if (wa->address) + dma_pool_free(wa->dma_pool, wa->address, + wa->dma.address); + } else { + if (wa->dma.address) + dma_unmap_single(wa->dev, wa->dma.address, wa->length, + wa->dma.dir); + kfree(wa->address); + } + + wa->address = NULL; + wa->dma.address = 0; +} + +static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa, + struct ccp_cmd_queue *cmd_q, + unsigned int len, + enum dma_data_direction dir) +{ + memset(wa, 0, sizeof(*wa)); + + if (!len) + return 0; + + wa->dev = cmd_q->ccp->dev; + wa->length = len; + + if (len <= CCP_DMAPOOL_MAX_SIZE) { + wa->dma_pool = cmd_q->dma_pool; + + wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL, + &wa->dma.address); + if (!wa->address) + return -ENOMEM; + + wa->dma.length = CCP_DMAPOOL_MAX_SIZE; + + memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE); + } else { + wa->address = kzalloc(len, GFP_KERNEL); + if (!wa->address) + return -ENOMEM; + + wa->dma.address = dma_map_single(wa->dev, wa->address, len, + dir); + if (!wa->dma.address) + return -ENOMEM; + + wa->dma.length = len; + } + wa->dma.dir = dir; + + return 0; +} + +static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, + struct scatterlist *sg, unsigned int sg_offset, + unsigned int len) +{ + WARN_ON(!wa->address); + + scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, + 0); +} + +static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset, + struct scatterlist *sg, unsigned int sg_offset, + unsigned int len) +{ + WARN_ON(!wa->address); + + scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len, + 1); +} + +static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa, + struct scatterlist *sg, + unsigned int len, unsigned int se_len, + bool sign_extend) +{ + unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; + u8 buffer[CCP_REVERSE_BUF_SIZE]; + + BUG_ON(se_len > sizeof(buffer)); + + sg_offset = len; + dm_offset = 0; + nbytes = len; + while (nbytes) { + ksb_len = min_t(unsigned int, nbytes, se_len); + sg_offset -= ksb_len; + + scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0); + for (i = 0; i < ksb_len; i++) + wa->address[dm_offset + i] = buffer[ksb_len - i - 1]; + + dm_offset += ksb_len; + nbytes -= ksb_len; + + if ((ksb_len != se_len) && sign_extend) { + /* Must sign-extend to nearest sign-extend length */ + if (wa->address[dm_offset - 1] & 0x80) + memset(wa->address + dm_offset, 0xff, + se_len - ksb_len); + } + } +} + +static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa, + struct scatterlist *sg, + unsigned int len) +{ + unsigned int nbytes, sg_offset, dm_offset, ksb_len, i; + u8 buffer[CCP_REVERSE_BUF_SIZE]; + + sg_offset = 0; + dm_offset = len; + nbytes = len; + while (nbytes) { + ksb_len = min_t(unsigned int, nbytes, sizeof(buffer)); + dm_offset -= ksb_len; + + for (i = 0; i < ksb_len; i++) + buffer[ksb_len - i - 1] = wa->address[dm_offset + i]; + scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1); + + sg_offset += ksb_len; + nbytes -= ksb_len; + } +} + +static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q) +{ + ccp_dm_free(&data->dm_wa); + ccp_sg_free(&data->sg_wa); +} + +static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q, + struct scatterlist *sg, u64 sg_len, + unsigned int dm_len, + enum dma_data_direction dir) +{ + int ret; + + memset(data, 0, sizeof(*data)); + + ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len, + dir); + if (ret) + goto e_err; + + ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir); + if (ret) + goto e_err; + + return 0; + +e_err: + ccp_free_data(data, cmd_q); + + return ret; +} + +static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from) +{ + struct ccp_sg_workarea *sg_wa = &data->sg_wa; + struct ccp_dm_workarea *dm_wa = &data->dm_wa; + unsigned int buf_count, nbytes; + + /* Clear the buffer if setting it */ + if (!from) + memset(dm_wa->address, 0, dm_wa->length); + + if (!sg_wa->sg) + return 0; + + /* Perform the copy operation + * nbytes will always be <= UINT_MAX because dm_wa->length is + * an unsigned int + */ + nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length); + scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used, + nbytes, from); + + /* Update the structures and generate the count */ + buf_count = 0; + while (sg_wa->bytes_left && (buf_count < dm_wa->length)) { + nbytes = min(sg_wa->sg->length - sg_wa->sg_used, + dm_wa->length - buf_count); + nbytes = min_t(u64, sg_wa->bytes_left, nbytes); + + buf_count += nbytes; + ccp_update_sg_workarea(sg_wa, nbytes); + } + + return buf_count; +} + +static unsigned int ccp_fill_queue_buf(struct ccp_data *data) +{ + return ccp_queue_buf(data, 0); +} + +static unsigned int ccp_empty_queue_buf(struct ccp_data *data) +{ + return ccp_queue_buf(data, 1); +} + +static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst, + struct ccp_op *op, unsigned int block_size, + bool blocksize_op) +{ + unsigned int sg_src_len, sg_dst_len, op_len; + + /* The CCP can only DMA from/to one address each per operation. This + * requires that we find the smallest DMA area between the source + * and destination. The resulting len values will always be <= UINT_MAX + * because the dma length is an unsigned int. + */ + sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used; + sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len); + + if (dst) { + sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used; + sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len); + op_len = min(sg_src_len, sg_dst_len); + } else + op_len = sg_src_len; + + /* The data operation length will be at least block_size in length + * or the smaller of available sg room remaining for the source or + * the destination + */ + op_len = max(op_len, block_size); + + /* Unless we have to buffer data, there's no reason to wait */ + op->soc = 0; + + if (sg_src_len < block_size) { + /* Not enough data in the sg element, so it + * needs to be buffered into a blocksize chunk + */ + int cp_len = ccp_fill_queue_buf(src); + + op->soc = 1; + op->src.u.dma.address = src->dm_wa.dma.address; + op->src.u.dma.offset = 0; + op->src.u.dma.length = (blocksize_op) ? block_size : cp_len; + } else { + /* Enough data in the sg element, but we need to + * adjust for any previously copied data + */ + op->src.u.dma.address = sg_dma_address(src->sg_wa.sg); + op->src.u.dma.offset = src->sg_wa.sg_used; + op->src.u.dma.length = op_len & ~(block_size - 1); + + ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length); + } + + if (dst) { + if (sg_dst_len < block_size) { + /* Not enough room in the sg element or we're on the + * last piece of data (when using padding), so the + * output needs to be buffered into a blocksize chunk + */ + op->soc = 1; + op->dst.u.dma.address = dst->dm_wa.dma.address; + op->dst.u.dma.offset = 0; + op->dst.u.dma.length = op->src.u.dma.length; + } else { + /* Enough room in the sg element, but we need to + * adjust for any previously used area + */ + op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg); + op->dst.u.dma.offset = dst->sg_wa.sg_used; + op->dst.u.dma.length = op->src.u.dma.length; + } + } +} + +static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst, + struct ccp_op *op) +{ + op->init = 0; + + if (dst) { + if (op->dst.u.dma.address == dst->dm_wa.dma.address) + ccp_empty_queue_buf(dst); + else + ccp_update_sg_workarea(&dst->sg_wa, + op->dst.u.dma.length); + } +} + +static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q, + struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, + u32 byte_swap, bool from) +{ + struct ccp_op op; + + memset(&op, 0, sizeof(op)); + + op.cmd_q = cmd_q; + op.jobid = jobid; + op.eom = 1; + + if (from) { + op.soc = 1; + op.src.type = CCP_MEMTYPE_KSB; + op.src.u.ksb = ksb; + op.dst.type = CCP_MEMTYPE_SYSTEM; + op.dst.u.dma.address = wa->dma.address; + op.dst.u.dma.length = wa->length; + } else { + op.src.type = CCP_MEMTYPE_SYSTEM; + op.src.u.dma.address = wa->dma.address; + op.src.u.dma.length = wa->length; + op.dst.type = CCP_MEMTYPE_KSB; + op.dst.u.ksb = ksb; + } + + op.u.passthru.byte_swap = byte_swap; + + return ccp_perform_passthru(&op); +} + +static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q, + struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, + u32 byte_swap) +{ + return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false); +} + +static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q, + struct ccp_dm_workarea *wa, u32 jobid, u32 ksb, + u32 byte_swap) +{ + return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true); +} + +static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, + struct ccp_cmd *cmd) +{ + struct ccp_aes_engine *aes = &cmd->u.aes; + struct ccp_dm_workarea key, ctx; + struct ccp_data src; + struct ccp_op op; + unsigned int dm_offset; + int ret; + + if (!((aes->key_len == AES_KEYSIZE_128) || + (aes->key_len == AES_KEYSIZE_192) || + (aes->key_len == AES_KEYSIZE_256))) + return -EINVAL; + + if (aes->src_len & (AES_BLOCK_SIZE - 1)) + return -EINVAL; + + if (aes->iv_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!aes->key || !aes->iv || !aes->src) + return -EINVAL; + + if (aes->cmac_final) { + if (aes->cmac_key_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!aes->cmac_key) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); + BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); + + ret = -EIO; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_key = cmd_q->ksb_key; + op.ksb_ctx = cmd_q->ksb_ctx; + op.init = 1; + op.u.aes.type = aes->type; + op.u.aes.mode = aes->mode; + op.u.aes.action = aes->action; + + /* All supported key sizes fit in a single (32-byte) KSB entry + * and must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little + * endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_KSB_BYTES - aes->key_len; + ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); + ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* The AES context fits in a single (32-byte) KSB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + /* Send data to the CCP AES engine */ + ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, + AES_BLOCK_SIZE, DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true); + if (aes->cmac_final && !src.sg_wa.bytes_left) { + op.eom = 1; + + /* Push the K1/K2 key to the CCP now */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, + op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + + ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0, + aes->cmac_key_len); + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + } + + ret = ccp_perform_aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + + ccp_process_data(&src, NULL, &op); + } + + /* Retrieve the AES context - convert from LE to BE using + * 32-byte (256-bit) byteswapping + */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_src; + } + + /* ...but we only need AES_BLOCK_SIZE bytes */ + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_aes_engine *aes = &cmd->u.aes; + struct ccp_dm_workarea key, ctx; + struct ccp_data src, dst; + struct ccp_op op; + unsigned int dm_offset; + bool in_place = false; + int ret; + + if (aes->mode == CCP_AES_MODE_CMAC) + return ccp_run_aes_cmac_cmd(cmd_q, cmd); + + if (!((aes->key_len == AES_KEYSIZE_128) || + (aes->key_len == AES_KEYSIZE_192) || + (aes->key_len == AES_KEYSIZE_256))) + return -EINVAL; + + if (((aes->mode == CCP_AES_MODE_ECB) || + (aes->mode == CCP_AES_MODE_CBC) || + (aes->mode == CCP_AES_MODE_CFB)) && + (aes->src_len & (AES_BLOCK_SIZE - 1))) + return -EINVAL; + + if (!aes->key || !aes->src || !aes->dst) + return -EINVAL; + + if (aes->mode != CCP_AES_MODE_ECB) { + if (aes->iv_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!aes->iv) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1); + BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1); + + ret = -EIO; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_key = cmd_q->ksb_key; + op.ksb_ctx = cmd_q->ksb_ctx; + op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1; + op.u.aes.type = aes->type; + op.u.aes.mode = aes->mode; + op.u.aes.action = aes->action; + + /* All supported key sizes fit in a single (32-byte) KSB entry + * and must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little + * endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_KSB_BYTES - aes->key_len; + ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len); + ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* The AES context fits in a single (32-byte) KSB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + if (aes->mode != CCP_AES_MODE_ECB) { + /* Load the AES context - conver to LE */ + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + } + + /* Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(aes->src) == sg_virt(aes->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len, + AES_BLOCK_SIZE, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + if (in_place) + dst = src; + else { + ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len, + AES_BLOCK_SIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP AES engine */ + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true); + if (!src.sg_wa.bytes_left) { + op.eom = 1; + + /* Since we don't retrieve the AES context in ECB + * mode we have to wait for the operation to complete + * on the last piece of data + */ + if (aes->mode == CCP_AES_MODE_ECB) + op.soc = 1; + } + + ret = ccp_perform_aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&src, &dst, &op); + } + + if (aes->mode != CCP_AES_MODE_ECB) { + /* Retrieve the AES context - convert from LE to BE using + * 32-byte (256-bit) byteswapping + */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* ...but we only need AES_BLOCK_SIZE bytes */ + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len); + } + +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, + struct ccp_cmd *cmd) +{ + struct ccp_xts_aes_engine *xts = &cmd->u.xts; + struct ccp_dm_workarea key, ctx; + struct ccp_data src, dst; + struct ccp_op op; + unsigned int unit_size, dm_offset; + bool in_place = false; + int ret; + + switch (xts->unit_size) { + case CCP_XTS_AES_UNIT_SIZE_16: + unit_size = 16; + break; + case CCP_XTS_AES_UNIT_SIZE_512: + unit_size = 512; + break; + case CCP_XTS_AES_UNIT_SIZE_1024: + unit_size = 1024; + break; + case CCP_XTS_AES_UNIT_SIZE_2048: + unit_size = 2048; + break; + case CCP_XTS_AES_UNIT_SIZE_4096: + unit_size = 4096; + break; + + default: + return -EINVAL; + } + + if (xts->key_len != AES_KEYSIZE_128) + return -EINVAL; + + if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1))) + return -EINVAL; + + if (xts->iv_len != AES_BLOCK_SIZE) + return -EINVAL; + + if (!xts->key || !xts->iv || !xts->src || !xts->dst) + return -EINVAL; + + BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1); + BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1); + + ret = -EIO; + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_key = cmd_q->ksb_key; + op.ksb_ctx = cmd_q->ksb_ctx; + op.init = 1; + op.u.xts.action = xts->action; + op.u.xts.unit_size = xts->unit_size; + + /* All supported key sizes fit in a single (32-byte) KSB entry + * and must be in little endian format. Use the 256-bit byte + * swap passthru option to convert from big endian to little + * endian. + */ + ret = ccp_init_dm_workarea(&key, cmd_q, + CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128; + ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len); + ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len); + ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_key; + } + + /* The AES context fits in a single (32-byte) KSB entry and + * for XTS is already in little endian format so no byte swapping + * is needed. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + goto e_key; + + ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len); + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + /* Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(xts->src) == sg_virt(xts->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len, + unit_size, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + if (in_place) + dst = src; + else { + ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len, + unit_size, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP AES engine */ + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, &dst, &op, unit_size, true); + if (!src.sg_wa.bytes_left) + op.eom = 1; + + ret = ccp_perform_xts_aes(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_process_data(&src, &dst, &op); + } + + /* Retrieve the AES context - convert from LE to BE using + * 32-byte (256-bit) byteswapping + */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + /* ...but we only need AES_BLOCK_SIZE bytes */ + dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE; + ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len); + +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + +e_key: + ccp_dm_free(&key); + + return ret; +} + +static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_sha_engine *sha = &cmd->u.sha; + struct ccp_dm_workarea ctx; + struct ccp_data src; + struct ccp_op op; + int ret; + + if (sha->ctx_len != CCP_SHA_CTXSIZE) + return -EINVAL; + + if (!sha->ctx) + return -EINVAL; + + if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1))) + return -EINVAL; + + if (!sha->src_len) { + const u8 *sha_zero; + + /* Not final, just return */ + if (!sha->final) + return 0; + + /* CCP can't do a zero length sha operation so the caller + * must buffer the data. + */ + if (sha->msg_bits) + return -EINVAL; + + /* A sha operation for a message with a total length of zero, + * return known result. + */ + switch (sha->type) { + case CCP_SHA_TYPE_1: + sha_zero = ccp_sha1_zero; + break; + case CCP_SHA_TYPE_224: + sha_zero = ccp_sha224_zero; + break; + case CCP_SHA_TYPE_256: + sha_zero = ccp_sha256_zero; + break; + default: + return -EINVAL; + } + + scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0, + sha->ctx_len, 1); + + return 0; + } + + if (!sha->src) + return -EINVAL; + + BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1); + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_ctx = cmd_q->ksb_ctx; + op.u.sha.type = sha->type; + op.u.sha.msg_bits = sha->msg_bits; + + /* The SHA context fits in a single (32-byte) KSB entry and + * must be in little endian format. Use the 256-bit byte swap + * passthru option to convert from big endian to little endian. + */ + ret = ccp_init_dm_workarea(&ctx, cmd_q, + CCP_SHA_KSB_COUNT * CCP_KSB_BYTES, + DMA_BIDIRECTIONAL); + if (ret) + return ret; + + if (sha->first) { + const __be32 *init; + + switch (sha->type) { + case CCP_SHA_TYPE_1: + init = ccp_sha1_init; + break; + case CCP_SHA_TYPE_224: + init = ccp_sha224_init; + break; + case CCP_SHA_TYPE_256: + init = ccp_sha256_init; + break; + default: + ret = -EINVAL; + goto e_ctx; + } + memcpy(ctx.address, init, CCP_SHA_CTXSIZE); + } else + ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); + + ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_ctx; + } + + /* Send data to the CCP SHA engine */ + ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len, + CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE); + if (ret) + goto e_ctx; + + while (src.sg_wa.bytes_left) { + ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false); + if (sha->final && !src.sg_wa.bytes_left) + op.eom = 1; + + ret = ccp_perform_sha(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_data; + } + + ccp_process_data(&src, NULL, &op); + } + + /* Retrieve the SHA context - convert from LE to BE using + * 32-byte (256-bit) byteswapping to BE + */ + ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx, + CCP_PASSTHRU_BYTESWAP_256BIT); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_data; + } + + ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len); + + if (sha->final && sha->opad) { + /* HMAC operation, recursively perform final SHA */ + struct ccp_cmd hmac_cmd; + struct scatterlist sg; + u64 block_size, digest_size; + u8 *hmac_buf; + + switch (sha->type) { + case CCP_SHA_TYPE_1: + block_size = SHA1_BLOCK_SIZE; + digest_size = SHA1_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_224: + block_size = SHA224_BLOCK_SIZE; + digest_size = SHA224_DIGEST_SIZE; + break; + case CCP_SHA_TYPE_256: + block_size = SHA256_BLOCK_SIZE; + digest_size = SHA256_DIGEST_SIZE; + break; + default: + ret = -EINVAL; + goto e_data; + } + + if (sha->opad_len != block_size) { + ret = -EINVAL; + goto e_data; + } + + hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL); + if (!hmac_buf) { + ret = -ENOMEM; + goto e_data; + } + sg_init_one(&sg, hmac_buf, block_size + digest_size); + + scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0); + memcpy(hmac_buf + block_size, ctx.address, digest_size); + + memset(&hmac_cmd, 0, sizeof(hmac_cmd)); + hmac_cmd.engine = CCP_ENGINE_SHA; + hmac_cmd.u.sha.type = sha->type; + hmac_cmd.u.sha.ctx = sha->ctx; + hmac_cmd.u.sha.ctx_len = sha->ctx_len; + hmac_cmd.u.sha.src = &sg; + hmac_cmd.u.sha.src_len = block_size + digest_size; + hmac_cmd.u.sha.opad = NULL; + hmac_cmd.u.sha.opad_len = 0; + hmac_cmd.u.sha.first = 1; + hmac_cmd.u.sha.final = 1; + hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3; + + ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd); + if (ret) + cmd->engine_error = hmac_cmd.engine_error; + + kfree(hmac_buf); + } + +e_data: + ccp_free_data(&src, cmd_q); + +e_ctx: + ccp_dm_free(&ctx); + + return ret; +} + +static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_rsa_engine *rsa = &cmd->u.rsa; + struct ccp_dm_workarea exp, src; + struct ccp_data dst; + struct ccp_op op; + unsigned int ksb_count, i_len, o_len; + int ret; + + if (rsa->key_size > CCP_RSA_MAX_WIDTH) + return -EINVAL; + + if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst) + return -EINVAL; + + /* The RSA modulus must precede the message being acted upon, so + * it must be copied to a DMA area where the message and the + * modulus can be concatenated. Therefore the input buffer + * length required is twice the output buffer length (which + * must be a multiple of 256-bits). + */ + o_len = ((rsa->key_size + 255) / 256) * 32; + i_len = o_len * 2; + + ksb_count = o_len / CCP_KSB_BYTES; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count); + if (!op.ksb_key) + return -EIO; + + /* The RSA exponent may span multiple (32-byte) KSB entries and must + * be in little endian format. Reverse copy each 32-byte chunk + * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk) + * and each byte within that chunk and do not perform any byte swap + * operations on the passthru operation. + */ + ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE); + if (ret) + goto e_ksb; + + ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES, + true); + ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_exp; + } + + /* Concatenate the modulus and the message. Both the modulus and + * the operands must be in little endian format. Since the input + * is in big endian format it must be converted. + */ + ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE); + if (ret) + goto e_exp; + + ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES, + true); + src.address += o_len; /* Adjust the address for the copy operation */ + ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES, + true); + src.address -= o_len; /* Reset the address to original value */ + + /* Prepare the output area for the operation */ + ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len, + o_len, DMA_FROM_DEVICE); + if (ret) + goto e_src; + + op.soc = 1; + op.src.u.dma.address = src.dma.address; + op.src.u.dma.offset = 0; + op.src.u.dma.length = i_len; + op.dst.u.dma.address = dst.dm_wa.dma.address; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = o_len; + + op.u.rsa.mod_size = rsa->key_size; + op.u.rsa.input_len = i_len; + + ret = ccp_perform_rsa(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len); + +e_dst: + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_dm_free(&src); + +e_exp: + ccp_dm_free(&exp); + +e_ksb: + ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count); + + return ret; +} + +static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, + struct ccp_cmd *cmd) +{ + struct ccp_passthru_engine *pt = &cmd->u.passthru; + struct ccp_dm_workarea mask; + struct ccp_data src, dst; + struct ccp_op op; + bool in_place = false; + unsigned int i; + int ret; + + if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1))) + return -EINVAL; + + if (!pt->src || !pt->dst) + return -EINVAL; + + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { + if (pt->mask_len != CCP_PASSTHRU_MASKSIZE) + return -EINVAL; + if (!pt->mask) + return -EINVAL; + } + + BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1); + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) { + /* Load the mask */ + op.ksb_key = cmd_q->ksb_key; + + ret = ccp_init_dm_workarea(&mask, cmd_q, + CCP_PASSTHRU_KSB_COUNT * + CCP_KSB_BYTES, + DMA_TO_DEVICE); + if (ret) + return ret; + + ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len); + ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key, + CCP_PASSTHRU_BYTESWAP_NOOP); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_mask; + } + } + + /* Prepare the input and output data workareas. For in-place + * operations we need to set the dma direction to BIDIRECTIONAL + * and copy the src workarea to the dst workarea. + */ + if (sg_virt(pt->src) == sg_virt(pt->dst)) + in_place = true; + + ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len, + CCP_PASSTHRU_MASKSIZE, + in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); + if (ret) + goto e_mask; + + if (in_place) + dst = src; + else { + ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len, + CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE); + if (ret) + goto e_src; + } + + /* Send data to the CCP Passthru engine + * Because the CCP engine works on a single source and destination + * dma address at a time, each entry in the source scatterlist + * (after the dma_map_sg call) must be less than or equal to the + * (remaining) length in the destination scatterlist entry and the + * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE + */ + dst.sg_wa.sg_used = 0; + for (i = 1; i <= src.sg_wa.dma_count; i++) { + if (!dst.sg_wa.sg || + (dst.sg_wa.sg->length < src.sg_wa.sg->length)) { + ret = -EINVAL; + goto e_dst; + } + + if (i == src.sg_wa.dma_count) { + op.eom = 1; + op.soc = 1; + } + + op.src.type = CCP_MEMTYPE_SYSTEM; + op.src.u.dma.address = sg_dma_address(src.sg_wa.sg); + op.src.u.dma.offset = 0; + op.src.u.dma.length = sg_dma_len(src.sg_wa.sg); + + op.dst.type = CCP_MEMTYPE_SYSTEM; + op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg); + op.dst.u.dma.offset = dst.sg_wa.sg_used; + op.dst.u.dma.length = op.src.u.dma.length; + + ret = ccp_perform_passthru(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + dst.sg_wa.sg_used += src.sg_wa.sg->length; + if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) { + dst.sg_wa.sg = sg_next(dst.sg_wa.sg); + dst.sg_wa.sg_used = 0; + } + src.sg_wa.sg = sg_next(src.sg_wa.sg); + } + +e_dst: + if (!in_place) + ccp_free_data(&dst, cmd_q); + +e_src: + ccp_free_data(&src, cmd_q); + +e_mask: + if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) + ccp_dm_free(&mask); + + return ret; +} + +static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_ecc_engine *ecc = &cmd->u.ecc; + struct ccp_dm_workarea src, dst; + struct ccp_op op; + int ret; + u8 *save; + + if (!ecc->u.mm.operand_1 || + (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) + if (!ecc->u.mm.operand_2 || + (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (!ecc->u.mm.result || + (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + + /* Concatenate the modulus and the operands. Both the modulus and + * the operands must be in little endian format. Since the input + * is in big endian format it must be converted and placed in a + * fixed length buffer. + */ + ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, + DMA_TO_DEVICE); + if (ret) + return ret; + + /* Save the workarea address since it is updated in order to perform + * the concatenation + */ + save = src.address; + + /* Copy the ECC modulus */ + ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + /* Copy the first operand */ + ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1, + ecc->u.mm.operand_1_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) { + /* Copy the second operand */ + ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2, + ecc->u.mm.operand_2_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + } + + /* Restore the workarea address */ + src.address = save; + + /* Prepare the output area for the operation */ + ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, + DMA_FROM_DEVICE); + if (ret) + goto e_src; + + op.soc = 1; + op.src.u.dma.address = src.dma.address; + op.src.u.dma.offset = 0; + op.src.u.dma.length = src.length; + op.dst.u.dma.address = dst.dma.address; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = dst.length; + + op.u.ecc.function = cmd->u.ecc.function; + + ret = ccp_perform_ecc(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ecc->ecc_result = le16_to_cpup( + (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); + if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { + ret = -EIO; + goto e_dst; + } + + /* Save the ECC result */ + ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES); + +e_dst: + ccp_dm_free(&dst); + +e_src: + ccp_dm_free(&src); + + return ret; +} + +static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_ecc_engine *ecc = &cmd->u.ecc; + struct ccp_dm_workarea src, dst; + struct ccp_op op; + int ret; + u8 *save; + + if (!ecc->u.pm.point_1.x || + (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) || + !ecc->u.pm.point_1.y || + (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { + if (!ecc->u.pm.point_2.x || + (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) || + !ecc->u.pm.point_2.y || + (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + } else { + if (!ecc->u.pm.domain_a || + (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) + if (!ecc->u.pm.scalar || + (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + } + + if (!ecc->u.pm.result.x || + (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) || + !ecc->u.pm.result.y || + (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + memset(&op, 0, sizeof(op)); + op.cmd_q = cmd_q; + op.jobid = ccp_gen_jobid(cmd_q->ccp); + + /* Concatenate the modulus and the operands. Both the modulus and + * the operands must be in little endian format. Since the input + * is in big endian format it must be converted and placed in a + * fixed length buffer. + */ + ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE, + DMA_TO_DEVICE); + if (ret) + return ret; + + /* Save the workarea address since it is updated in order to perform + * the concatenation + */ + save = src.address; + + /* Copy the ECC modulus */ + ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + /* Copy the first point X and Y coordinate */ + ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x, + ecc->u.pm.point_1.x_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y, + ecc->u.pm.point_1.y_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + /* Set the first point Z coordianate to 1 */ + *(src.address) = 0x01; + src.address += CCP_ECC_OPERAND_SIZE; + + if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) { + /* Copy the second point X and Y coordinate */ + ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x, + ecc->u.pm.point_2.x_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y, + ecc->u.pm.point_2.y_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + /* Set the second point Z coordianate to 1 */ + *(src.address) = 0x01; + src.address += CCP_ECC_OPERAND_SIZE; + } else { + /* Copy the Domain "a" parameter */ + ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a, + ecc->u.pm.domain_a_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + + if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) { + /* Copy the scalar value */ + ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar, + ecc->u.pm.scalar_len, + CCP_ECC_OPERAND_SIZE, true); + src.address += CCP_ECC_OPERAND_SIZE; + } + } + + /* Restore the workarea address */ + src.address = save; + + /* Prepare the output area for the operation */ + ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE, + DMA_FROM_DEVICE); + if (ret) + goto e_src; + + op.soc = 1; + op.src.u.dma.address = src.dma.address; + op.src.u.dma.offset = 0; + op.src.u.dma.length = src.length; + op.dst.u.dma.address = dst.dma.address; + op.dst.u.dma.offset = 0; + op.dst.u.dma.length = dst.length; + + op.u.ecc.function = cmd->u.ecc.function; + + ret = ccp_perform_ecc(&op); + if (ret) { + cmd->engine_error = cmd_q->cmd_error; + goto e_dst; + } + + ecc->ecc_result = le16_to_cpup( + (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET)); + if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) { + ret = -EIO; + goto e_dst; + } + + /* Save the workarea address since it is updated as we walk through + * to copy the point math result + */ + save = dst.address; + + /* Save the ECC result X and Y coordinates */ + ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x, + CCP_ECC_MODULUS_BYTES); + dst.address += CCP_ECC_OUTPUT_SIZE; + ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y, + CCP_ECC_MODULUS_BYTES); + dst.address += CCP_ECC_OUTPUT_SIZE; + + /* Restore the workarea address */ + dst.address = save; + +e_dst: + ccp_dm_free(&dst); + +e_src: + ccp_dm_free(&src); + + return ret; +} + +static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + struct ccp_ecc_engine *ecc = &cmd->u.ecc; + + ecc->ecc_result = 0; + + if (!ecc->mod || + (ecc->mod_len > CCP_ECC_MODULUS_BYTES)) + return -EINVAL; + + switch (ecc->function) { + case CCP_ECC_FUNCTION_MMUL_384BIT: + case CCP_ECC_FUNCTION_MADD_384BIT: + case CCP_ECC_FUNCTION_MINV_384BIT: + return ccp_run_ecc_mm_cmd(cmd_q, cmd); + + case CCP_ECC_FUNCTION_PADD_384BIT: + case CCP_ECC_FUNCTION_PMUL_384BIT: + case CCP_ECC_FUNCTION_PDBL_384BIT: + return ccp_run_ecc_pm_cmd(cmd_q, cmd); + + default: + return -EINVAL; + } +} + +int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) +{ + int ret; + + cmd->engine_error = 0; + cmd_q->cmd_error = 0; + cmd_q->int_rcvd = 0; + cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status)); + + switch (cmd->engine) { + case CCP_ENGINE_AES: + ret = ccp_run_aes_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_XTS_AES_128: + ret = ccp_run_xts_aes_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_SHA: + ret = ccp_run_sha_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_RSA: + ret = ccp_run_rsa_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_PASSTHRU: + ret = ccp_run_passthru_cmd(cmd_q, cmd); + break; + case CCP_ENGINE_ECC: + ret = ccp_run_ecc_cmd(cmd_q, cmd); + break; + default: + ret = -EINVAL; + } + + return ret; +} diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c new file mode 100644 index 00000000000..0d746236df5 --- /dev/null +++ b/drivers/crypto/ccp/ccp-pci.c @@ -0,0 +1,360 @@ +/* + * AMD Cryptographic Coprocessor (CCP) driver + * + * Copyright (C) 2013 Advanced Micro Devices, Inc. + * + * Author: Tom Lendacky <thomas.lendacky@amd.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> +#include <linux/kthread.h> +#include <linux/sched.h> +#include <linux/interrupt.h> +#include <linux/spinlock.h> +#include <linux/delay.h> +#include <linux/ccp.h> + +#include "ccp-dev.h" + +#define IO_BAR 2 +#define MSIX_VECTORS 2 + +struct ccp_msix { + u32 vector; + char name[16]; +}; + +struct ccp_pci { + int msix_count; + struct ccp_msix msix[MSIX_VECTORS]; +}; + +static int ccp_get_msix_irqs(struct ccp_device *ccp) +{ + struct ccp_pci *ccp_pci = ccp->dev_specific; + struct device *dev = ccp->dev; + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + struct msix_entry msix_entry[MSIX_VECTORS]; + unsigned int name_len = sizeof(ccp_pci->msix[0].name) - 1; + int v, ret; + + for (v = 0; v < ARRAY_SIZE(msix_entry); v++) + msix_entry[v].entry = v; + + ret = pci_enable_msix_range(pdev, msix_entry, 1, v); + if (ret < 0) + return ret; + + ccp_pci->msix_count = ret; + for (v = 0; v < ccp_pci->msix_count; v++) { + /* Set the interrupt names and request the irqs */ + snprintf(ccp_pci->msix[v].name, name_len, "ccp-%u", v); + ccp_pci->msix[v].vector = msix_entry[v].vector; + ret = request_irq(ccp_pci->msix[v].vector, ccp_irq_handler, + 0, ccp_pci->msix[v].name, dev); + if (ret) { + dev_notice(dev, "unable to allocate MSI-X IRQ (%d)\n", + ret); + goto e_irq; + } + } + + return 0; + +e_irq: + while (v--) + free_irq(ccp_pci->msix[v].vector, dev); + + pci_disable_msix(pdev); + + ccp_pci->msix_count = 0; + + return ret; +} + +static int ccp_get_msi_irq(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + int ret; + + ret = pci_enable_msi(pdev); + if (ret) + return ret; + + ret = request_irq(pdev->irq, ccp_irq_handler, 0, "ccp", dev); + if (ret) { + dev_notice(dev, "unable to allocate MSI IRQ (%d)\n", ret); + goto e_msi; + } + + return 0; + +e_msi: + pci_disable_msi(pdev); + + return ret; +} + +static int ccp_get_irqs(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + int ret; + + ret = ccp_get_msix_irqs(ccp); + if (!ret) + return 0; + + /* Couldn't get MSI-X vectors, try MSI */ + dev_notice(dev, "could not enable MSI-X (%d), trying MSI\n", ret); + ret = ccp_get_msi_irq(ccp); + if (!ret) + return 0; + + /* Couldn't get MSI interrupt */ + dev_notice(dev, "could not enable MSI (%d)\n", ret); + + return ret; +} + +static void ccp_free_irqs(struct ccp_device *ccp) +{ + struct ccp_pci *ccp_pci = ccp->dev_specific; + struct device *dev = ccp->dev; + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + + if (ccp_pci->msix_count) { + while (ccp_pci->msix_count--) + free_irq(ccp_pci->msix[ccp_pci->msix_count].vector, + dev); + pci_disable_msix(pdev); + } else { + free_irq(pdev->irq, dev); + pci_disable_msi(pdev); + } +} + +static int ccp_find_mmio_area(struct ccp_device *ccp) +{ + struct device *dev = ccp->dev; + struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + resource_size_t io_len; + unsigned long io_flags; + int bar; + + io_flags = pci_resource_flags(pdev, IO_BAR); + io_len = pci_resource_len(pdev, IO_BAR); + if ((io_flags & IORESOURCE_MEM) && (io_len >= (IO_OFFSET + 0x800))) + return IO_BAR; + + for (bar = 0; bar < PCI_STD_RESOURCE_END; bar++) { + io_flags = pci_resource_flags(pdev, bar); + io_len = pci_resource_len(pdev, bar); + if ((io_flags & IORESOURCE_MEM) && + (io_len >= (IO_OFFSET + 0x800))) + return bar; + } + + return -EIO; +} + +static int ccp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct ccp_device *ccp; + struct ccp_pci *ccp_pci; + struct device *dev = &pdev->dev; + unsigned int bar; + int ret; + + ret = -ENOMEM; + ccp = ccp_alloc_struct(dev); + if (!ccp) + goto e_err; + + ccp_pci = kzalloc(sizeof(*ccp_pci), GFP_KERNEL); + if (!ccp_pci) { + ret = -ENOMEM; + goto e_free1; + } + ccp->dev_specific = ccp_pci; + ccp->get_irq = ccp_get_irqs; + ccp->free_irq = ccp_free_irqs; + + ret = pci_request_regions(pdev, "ccp"); + if (ret) { + dev_err(dev, "pci_request_regions failed (%d)\n", ret); + goto e_free2; + } + + ret = pci_enable_device(pdev); + if (ret) { + dev_err(dev, "pci_enable_device failed (%d)\n", ret); + goto e_regions; + } + + pci_set_master(pdev); + + ret = ccp_find_mmio_area(ccp); + if (ret < 0) + goto e_device; + bar = ret; + + ret = -EIO; + ccp->io_map = pci_iomap(pdev, bar, 0); + if (ccp->io_map == NULL) { + dev_err(dev, "pci_iomap failed\n"); + goto e_device; + } + ccp->io_regs = ccp->io_map + IO_OFFSET; + + ret = dma_set_mask(dev, DMA_BIT_MASK(48)); + if (ret == 0) { + ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(48)); + if (ret) { + dev_err(dev, + "pci_set_consistent_dma_mask failed (%d)\n", + ret); + goto e_bar0; + } + } else { + ret = dma_set_mask(dev, DMA_BIT_MASK(32)); + if (ret) { + dev_err(dev, "pci_set_dma_mask failed (%d)\n", ret); + goto e_bar0; + } + } + + dev_set_drvdata(dev, ccp); + + ret = ccp_init(ccp); + if (ret) + goto e_bar0; + + dev_notice(dev, "enabled\n"); + + return 0; + +e_bar0: + pci_iounmap(pdev, ccp->io_map); + +e_device: + pci_disable_device(pdev); + +e_regions: + pci_release_regions(pdev); + +e_free2: + kfree(ccp_pci); + +e_free1: + kfree(ccp); + +e_err: + dev_notice(dev, "initialization failed\n"); + return ret; +} + +static void ccp_pci_remove(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct ccp_device *ccp = dev_get_drvdata(dev); + + if (!ccp) + return; + + ccp_destroy(ccp); + + pci_iounmap(pdev, ccp->io_map); + + pci_disable_device(pdev); + + pci_release_regions(pdev); + + kfree(ccp); + + dev_notice(dev, "disabled\n"); +} + +#ifdef CONFIG_PM +static int ccp_pci_suspend(struct pci_dev *pdev, pm_message_t state) +{ + struct device *dev = &pdev->dev; + struct ccp_device *ccp = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->suspending = 1; + + /* Wake all the queue kthreads to prepare for suspend */ + for (i = 0; i < ccp->cmd_q_count; i++) + wake_up_process(ccp->cmd_q[i].kthread); + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + /* Wait for all queue kthreads to say they're done */ + while (!ccp_queues_suspended(ccp)) + wait_event_interruptible(ccp->suspend_queue, + ccp_queues_suspended(ccp)); + + return 0; +} + +static int ccp_pci_resume(struct pci_dev *pdev) +{ + struct device *dev = &pdev->dev; + struct ccp_device *ccp = dev_get_drvdata(dev); + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&ccp->cmd_lock, flags); + + ccp->suspending = 0; + + /* Wake up all the kthreads */ + for (i = 0; i < ccp->cmd_q_count; i++) { + ccp->cmd_q[i].suspended = 0; + wake_up_process(ccp->cmd_q[i].kthread); + } + + spin_unlock_irqrestore(&ccp->cmd_lock, flags); + + return 0; +} +#endif + +static DEFINE_PCI_DEVICE_TABLE(ccp_pci_table) = { + { PCI_VDEVICE(AMD, 0x1537), }, + /* Last entry must be zero */ + { 0, } +}; +MODULE_DEVICE_TABLE(pci, ccp_pci_table); + +static struct pci_driver ccp_pci_driver = { + .name = "AMD Cryptographic Coprocessor", + .id_table = ccp_pci_table, + .probe = ccp_pci_probe, + .remove = ccp_pci_remove, +#ifdef CONFIG_PM + .suspend = ccp_pci_suspend, + .resume = ccp_pci_resume, +#endif +}; + +int ccp_pci_init(void) +{ + return pci_register_driver(&ccp_pci_driver); +} + +void ccp_pci_exit(void) +{ + pci_unregister_driver(&ccp_pci_driver); +} diff --git a/drivers/crypto/dcp.c b/drivers/crypto/dcp.c deleted file mode 100644 index 247ab8048f5..00000000000 --- a/drivers/crypto/dcp.c +++ /dev/null @@ -1,903 +0,0 @@ -/* - * Cryptographic API. - * - * Support for DCP cryptographic accelerator. - * - * Copyright (c) 2013 - * Author: Tobias Rauter <tobias.rauter@gmail.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as published - * by the Free Software Foundation. - * - * Based on tegra-aes.c, dcp.c (from freescale SDK) and sahara.c - */ -#include <linux/module.h> -#include <linux/init.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/platform_device.h> -#include <linux/dma-mapping.h> -#include <linux/io.h> -#include <linux/mutex.h> -#include <linux/interrupt.h> -#include <linux/completion.h> -#include <linux/workqueue.h> -#include <linux/delay.h> -#include <linux/crypto.h> -#include <linux/miscdevice.h> - -#include <crypto/scatterwalk.h> -#include <crypto/aes.h> - - -/* IOCTL for DCP OTP Key AES - taken from Freescale's SDK*/ -#define DBS_IOCTL_BASE 'd' -#define DBS_ENC _IOW(DBS_IOCTL_BASE, 0x00, uint8_t[16]) -#define DBS_DEC _IOW(DBS_IOCTL_BASE, 0x01, uint8_t[16]) - -/* DCP channel used for AES */ -#define USED_CHANNEL 1 -/* Ring Buffers' maximum size */ -#define DCP_MAX_PKG 20 - -/* Control Register */ -#define DCP_REG_CTRL 0x000 -#define DCP_CTRL_SFRST (1<<31) -#define DCP_CTRL_CLKGATE (1<<30) -#define DCP_CTRL_CRYPTO_PRESENT (1<<29) -#define DCP_CTRL_SHA_PRESENT (1<<28) -#define DCP_CTRL_GATHER_RES_WRITE (1<<23) -#define DCP_CTRL_ENABLE_CONTEXT_CACHE (1<<22) -#define DCP_CTRL_ENABLE_CONTEXT_SWITCH (1<<21) -#define DCP_CTRL_CH_IRQ_E_0 0x01 -#define DCP_CTRL_CH_IRQ_E_1 0x02 -#define DCP_CTRL_CH_IRQ_E_2 0x04 -#define DCP_CTRL_CH_IRQ_E_3 0x08 - -/* Status register */ -#define DCP_REG_STAT 0x010 -#define DCP_STAT_OTP_KEY_READY (1<<28) -#define DCP_STAT_CUR_CHANNEL(stat) ((stat>>24)&0x0F) -#define DCP_STAT_READY_CHANNEL(stat) ((stat>>16)&0x0F) -#define DCP_STAT_IRQ(stat) (stat&0x0F) -#define DCP_STAT_CHAN_0 (0x01) -#define DCP_STAT_CHAN_1 (0x02) -#define DCP_STAT_CHAN_2 (0x04) -#define DCP_STAT_CHAN_3 (0x08) - -/* Channel Control Register */ -#define DCP_REG_CHAN_CTRL 0x020 -#define DCP_CHAN_CTRL_CH0_IRQ_MERGED (1<<16) -#define DCP_CHAN_CTRL_HIGH_PRIO_0 (0x0100) -#define DCP_CHAN_CTRL_HIGH_PRIO_1 (0x0200) -#define DCP_CHAN_CTRL_HIGH_PRIO_2 (0x0400) -#define DCP_CHAN_CTRL_HIGH_PRIO_3 (0x0800) -#define DCP_CHAN_CTRL_ENABLE_0 (0x01) -#define DCP_CHAN_CTRL_ENABLE_1 (0x02) -#define DCP_CHAN_CTRL_ENABLE_2 (0x04) -#define DCP_CHAN_CTRL_ENABLE_3 (0x08) - -/* - * Channel Registers: - * The DCP has 4 channels. Each of this channels - * has 4 registers (command pointer, semaphore, status and options). - * The address of register REG of channel CHAN is obtained by - * dcp_chan_reg(REG, CHAN) - */ -#define DCP_REG_CHAN_PTR 0x00000100 -#define DCP_REG_CHAN_SEMA 0x00000110 -#define DCP_REG_CHAN_STAT 0x00000120 -#define DCP_REG_CHAN_OPT 0x00000130 - -#define DCP_CHAN_STAT_NEXT_CHAIN_IS_0 0x010000 -#define DCP_CHAN_STAT_NO_CHAIN 0x020000 -#define DCP_CHAN_STAT_CONTEXT_ERROR 0x030000 -#define DCP_CHAN_STAT_PAYLOAD_ERROR 0x040000 -#define DCP_CHAN_STAT_INVALID_MODE 0x050000 -#define DCP_CHAN_STAT_PAGEFAULT 0x40 -#define DCP_CHAN_STAT_DST 0x20 -#define DCP_CHAN_STAT_SRC 0x10 -#define DCP_CHAN_STAT_PACKET 0x08 -#define DCP_CHAN_STAT_SETUP 0x04 -#define DCP_CHAN_STAT_MISMATCH 0x02 - -/* hw packet control*/ - -#define DCP_PKT_PAYLOAD_KEY (1<<11) -#define DCP_PKT_OTP_KEY (1<<10) -#define DCP_PKT_CIPHER_INIT (1<<9) -#define DCP_PKG_CIPHER_ENCRYPT (1<<8) -#define DCP_PKT_CIPHER_ENABLE (1<<5) -#define DCP_PKT_DECR_SEM (1<<1) -#define DCP_PKT_CHAIN (1<<2) -#define DCP_PKT_IRQ 1 - -#define DCP_PKT_MODE_CBC (1<<4) -#define DCP_PKT_KEYSELECT_OTP (0xFF<<8) - -/* cipher flags */ -#define DCP_ENC 0x0001 -#define DCP_DEC 0x0002 -#define DCP_ECB 0x0004 -#define DCP_CBC 0x0008 -#define DCP_CBC_INIT 0x0010 -#define DCP_NEW_KEY 0x0040 -#define DCP_OTP_KEY 0x0080 -#define DCP_AES 0x1000 - -/* DCP Flags */ -#define DCP_FLAG_BUSY 0x01 -#define DCP_FLAG_PRODUCING 0x02 - -/* clock defines */ -#define CLOCK_ON 1 -#define CLOCK_OFF 0 - -struct dcp_dev_req_ctx { - int mode; -}; - -struct dcp_op { - unsigned int flags; - u8 key[AES_KEYSIZE_128]; - int keylen; - - struct ablkcipher_request *req; - struct crypto_ablkcipher *fallback; - - uint32_t stat; - uint32_t pkt1; - uint32_t pkt2; - struct ablkcipher_walk walk; -}; - -struct dcp_dev { - struct device *dev; - void __iomem *dcp_regs_base; - - int dcp_vmi_irq; - int dcp_irq; - - spinlock_t queue_lock; - struct crypto_queue queue; - - uint32_t pkt_produced; - uint32_t pkt_consumed; - - struct dcp_hw_packet *hw_pkg[DCP_MAX_PKG]; - dma_addr_t hw_phys_pkg; - - /* [KEY][IV] Both with 16 Bytes */ - u8 *payload_base; - dma_addr_t payload_base_dma; - - - struct tasklet_struct done_task; - struct tasklet_struct queue_task; - struct timer_list watchdog; - - unsigned long flags; - - struct dcp_op *ctx; - - struct miscdevice dcp_bootstream_misc; -}; - -struct dcp_hw_packet { - uint32_t next; - uint32_t pkt1; - uint32_t pkt2; - uint32_t src; - uint32_t dst; - uint32_t size; - uint32_t payload; - uint32_t stat; -}; - -static struct dcp_dev *global_dev; - -static inline u32 dcp_chan_reg(u32 reg, int chan) -{ - return reg + (chan) * 0x40; -} - -static inline void dcp_write(struct dcp_dev *dev, u32 data, u32 reg) -{ - writel(data, dev->dcp_regs_base + reg); -} - -static inline void dcp_set(struct dcp_dev *dev, u32 data, u32 reg) -{ - writel(data, dev->dcp_regs_base + (reg | 0x04)); -} - -static inline void dcp_clear(struct dcp_dev *dev, u32 data, u32 reg) -{ - writel(data, dev->dcp_regs_base + (reg | 0x08)); -} - -static inline void dcp_toggle(struct dcp_dev *dev, u32 data, u32 reg) -{ - writel(data, dev->dcp_regs_base + (reg | 0x0C)); -} - -static inline unsigned int dcp_read(struct dcp_dev *dev, u32 reg) -{ - return readl(dev->dcp_regs_base + reg); -} - -static void dcp_dma_unmap(struct dcp_dev *dev, struct dcp_hw_packet *pkt) -{ - dma_unmap_page(dev->dev, pkt->src, pkt->size, DMA_TO_DEVICE); - dma_unmap_page(dev->dev, pkt->dst, pkt->size, DMA_FROM_DEVICE); - dev_dbg(dev->dev, "unmap packet %x", (unsigned int) pkt); -} - -static int dcp_dma_map(struct dcp_dev *dev, - struct ablkcipher_walk *walk, struct dcp_hw_packet *pkt) -{ - dev_dbg(dev->dev, "map packet %x", (unsigned int) pkt); - /* align to length = 16 */ - pkt->size = walk->nbytes - (walk->nbytes % 16); - - pkt->src = dma_map_page(dev->dev, walk->src.page, walk->src.offset, - pkt->size, DMA_TO_DEVICE); - - if (pkt->src == 0) { - dev_err(dev->dev, "Unable to map src"); - return -ENOMEM; - } - - pkt->dst = dma_map_page(dev->dev, walk->dst.page, walk->dst.offset, - pkt->size, DMA_FROM_DEVICE); - - if (pkt->dst == 0) { - dev_err(dev->dev, "Unable to map dst"); - dma_unmap_page(dev->dev, pkt->src, pkt->size, DMA_TO_DEVICE); - return -ENOMEM; - } - - return 0; -} - -static void dcp_op_one(struct dcp_dev *dev, struct dcp_hw_packet *pkt, - uint8_t last) -{ - struct dcp_op *ctx = dev->ctx; - pkt->pkt1 = ctx->pkt1; - pkt->pkt2 = ctx->pkt2; - - pkt->payload = (u32) dev->payload_base_dma; - pkt->stat = 0; - - if (ctx->flags & DCP_CBC_INIT) { - pkt->pkt1 |= DCP_PKT_CIPHER_INIT; - ctx->flags &= ~DCP_CBC_INIT; - } - - mod_timer(&dev->watchdog, jiffies + msecs_to_jiffies(500)); - pkt->pkt1 |= DCP_PKT_IRQ; - if (!last) - pkt->pkt1 |= DCP_PKT_CHAIN; - - dev->pkt_produced++; - - dcp_write(dev, 1, - dcp_chan_reg(DCP_REG_CHAN_SEMA, USED_CHANNEL)); -} - -static void dcp_op_proceed(struct dcp_dev *dev) -{ - struct dcp_op *ctx = dev->ctx; - struct dcp_hw_packet *pkt; - - while (ctx->walk.nbytes) { - int err = 0; - - pkt = dev->hw_pkg[dev->pkt_produced % DCP_MAX_PKG]; - err = dcp_dma_map(dev, &ctx->walk, pkt); - if (err) { - dev->ctx->stat |= err; - /* start timer to wait for already set up calls */ - mod_timer(&dev->watchdog, - jiffies + msecs_to_jiffies(500)); - break; - } - - - err = ctx->walk.nbytes - pkt->size; - ablkcipher_walk_done(dev->ctx->req, &dev->ctx->walk, err); - - dcp_op_one(dev, pkt, ctx->walk.nbytes == 0); - /* we have to wait if no space is left in buffer */ - if (dev->pkt_produced - dev->pkt_consumed == DCP_MAX_PKG) - break; - } - clear_bit(DCP_FLAG_PRODUCING, &dev->flags); -} - -static void dcp_op_start(struct dcp_dev *dev, uint8_t use_walk) -{ - struct dcp_op *ctx = dev->ctx; - - if (ctx->flags & DCP_NEW_KEY) { - memcpy(dev->payload_base, ctx->key, ctx->keylen); - ctx->flags &= ~DCP_NEW_KEY; - } - - ctx->pkt1 = 0; - ctx->pkt1 |= DCP_PKT_CIPHER_ENABLE; - ctx->pkt1 |= DCP_PKT_DECR_SEM; - - if (ctx->flags & DCP_OTP_KEY) - ctx->pkt1 |= DCP_PKT_OTP_KEY; - else - ctx->pkt1 |= DCP_PKT_PAYLOAD_KEY; - - if (ctx->flags & DCP_ENC) - ctx->pkt1 |= DCP_PKG_CIPHER_ENCRYPT; - - ctx->pkt2 = 0; - if (ctx->flags & DCP_CBC) - ctx->pkt2 |= DCP_PKT_MODE_CBC; - - dev->pkt_produced = 0; - dev->pkt_consumed = 0; - - ctx->stat = 0; - dcp_clear(dev, -1, dcp_chan_reg(DCP_REG_CHAN_STAT, USED_CHANNEL)); - dcp_write(dev, (u32) dev->hw_phys_pkg, - dcp_chan_reg(DCP_REG_CHAN_PTR, USED_CHANNEL)); - - set_bit(DCP_FLAG_PRODUCING, &dev->flags); - - if (use_walk) { - ablkcipher_walk_init(&ctx->walk, ctx->req->dst, - ctx->req->src, ctx->req->nbytes); - ablkcipher_walk_phys(ctx->req, &ctx->walk); - dcp_op_proceed(dev); - } else { - dcp_op_one(dev, dev->hw_pkg[0], 1); - clear_bit(DCP_FLAG_PRODUCING, &dev->flags); - } -} - -static void dcp_done_task(unsigned long data) -{ - struct dcp_dev *dev = (struct dcp_dev *)data; - struct dcp_hw_packet *last_packet; - int fin; - fin = 0; - - for (last_packet = dev->hw_pkg[(dev->pkt_consumed) % DCP_MAX_PKG]; - last_packet->stat == 1; - last_packet = - dev->hw_pkg[++(dev->pkt_consumed) % DCP_MAX_PKG]) { - - dcp_dma_unmap(dev, last_packet); - last_packet->stat = 0; - fin++; - } - /* the last call of this function already consumed this IRQ's packet */ - if (fin == 0) - return; - - dev_dbg(dev->dev, - "Packet(s) done with status %x; finished: %d, produced:%d, complete consumed: %d", - dev->ctx->stat, fin, dev->pkt_produced, dev->pkt_consumed); - - last_packet = dev->hw_pkg[(dev->pkt_consumed - 1) % DCP_MAX_PKG]; - if (!dev->ctx->stat && last_packet->pkt1 & DCP_PKT_CHAIN) { - if (!test_and_set_bit(DCP_FLAG_PRODUCING, &dev->flags)) - dcp_op_proceed(dev); - return; - } - - while (unlikely(dev->pkt_consumed < dev->pkt_produced)) { - dcp_dma_unmap(dev, - dev->hw_pkg[dev->pkt_consumed++ % DCP_MAX_PKG]); - } - - if (dev->ctx->flags & DCP_OTP_KEY) { - /* we used the miscdevice, no walk to finish */ - clear_bit(DCP_FLAG_BUSY, &dev->flags); - return; - } - - ablkcipher_walk_complete(&dev->ctx->walk); - dev->ctx->req->base.complete(&dev->ctx->req->base, - dev->ctx->stat); - dev->ctx->req = NULL; - /* in case there are other requests in the queue */ - tasklet_schedule(&dev->queue_task); -} - -static void dcp_watchdog(unsigned long data) -{ - struct dcp_dev *dev = (struct dcp_dev *)data; - dev->ctx->stat |= dcp_read(dev, - dcp_chan_reg(DCP_REG_CHAN_STAT, USED_CHANNEL)); - - dev_err(dev->dev, "Timeout, Channel status: %x", dev->ctx->stat); - - if (!dev->ctx->stat) - dev->ctx->stat = -ETIMEDOUT; - - dcp_done_task(data); -} - - -static irqreturn_t dcp_common_irq(int irq, void *context) -{ - u32 msk; - struct dcp_dev *dev = (struct dcp_dev *) context; - - del_timer(&dev->watchdog); - - msk = DCP_STAT_IRQ(dcp_read(dev, DCP_REG_STAT)); - dcp_clear(dev, msk, DCP_REG_STAT); - if (msk == 0) - return IRQ_NONE; - - dev->ctx->stat |= dcp_read(dev, - dcp_chan_reg(DCP_REG_CHAN_STAT, USED_CHANNEL)); - - if (msk & DCP_STAT_CHAN_1) - tasklet_schedule(&dev->done_task); - - return IRQ_HANDLED; -} - -static irqreturn_t dcp_vmi_irq(int irq, void *context) -{ - return dcp_common_irq(irq, context); -} - -static irqreturn_t dcp_irq(int irq, void *context) -{ - return dcp_common_irq(irq, context); -} - -static void dcp_crypt(struct dcp_dev *dev, struct dcp_op *ctx) -{ - dev->ctx = ctx; - - if ((ctx->flags & DCP_CBC) && ctx->req->info) { - ctx->flags |= DCP_CBC_INIT; - memcpy(dev->payload_base + AES_KEYSIZE_128, - ctx->req->info, AES_KEYSIZE_128); - } - - dcp_op_start(dev, 1); -} - -static void dcp_queue_task(unsigned long data) -{ - struct dcp_dev *dev = (struct dcp_dev *) data; - struct crypto_async_request *async_req, *backlog; - struct crypto_ablkcipher *tfm; - struct dcp_op *ctx; - struct dcp_dev_req_ctx *rctx; - struct ablkcipher_request *req; - unsigned long flags; - - spin_lock_irqsave(&dev->queue_lock, flags); - - backlog = crypto_get_backlog(&dev->queue); - async_req = crypto_dequeue_request(&dev->queue); - - spin_unlock_irqrestore(&dev->queue_lock, flags); - - if (!async_req) - goto ret_nothing_done; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - req = ablkcipher_request_cast(async_req); - tfm = crypto_ablkcipher_reqtfm(req); - rctx = ablkcipher_request_ctx(req); - ctx = crypto_ablkcipher_ctx(tfm); - - if (!req->src || !req->dst) - goto ret_nothing_done; - - ctx->flags |= rctx->mode; - ctx->req = req; - - dcp_crypt(dev, ctx); - - return; - -ret_nothing_done: - clear_bit(DCP_FLAG_BUSY, &dev->flags); -} - - -static int dcp_cra_init(struct crypto_tfm *tfm) -{ - const char *name = tfm->__crt_alg->cra_name; - struct dcp_op *ctx = crypto_tfm_ctx(tfm); - - tfm->crt_ablkcipher.reqsize = sizeof(struct dcp_dev_req_ctx); - - ctx->fallback = crypto_alloc_ablkcipher(name, 0, - CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK); - - if (IS_ERR(ctx->fallback)) { - dev_err(global_dev->dev, "Error allocating fallback algo %s\n", - name); - return PTR_ERR(ctx->fallback); - } - - return 0; -} - -static void dcp_cra_exit(struct crypto_tfm *tfm) -{ - struct dcp_op *ctx = crypto_tfm_ctx(tfm); - - if (ctx->fallback) - crypto_free_ablkcipher(ctx->fallback); - - ctx->fallback = NULL; -} - -/* async interface */ -static int dcp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, - unsigned int len) -{ - struct dcp_op *ctx = crypto_ablkcipher_ctx(tfm); - unsigned int ret = 0; - ctx->keylen = len; - ctx->flags = 0; - if (len == AES_KEYSIZE_128) { - if (memcmp(ctx->key, key, AES_KEYSIZE_128)) { - memcpy(ctx->key, key, len); - ctx->flags |= DCP_NEW_KEY; - } - return 0; - } - - ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; - ctx->fallback->base.crt_flags |= - (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); - - ret = crypto_ablkcipher_setkey(ctx->fallback, key, len); - if (ret) { - struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm); - - tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK; - tfm_aux->crt_flags |= - (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK); - } - return ret; -} - -static int dcp_aes_cbc_crypt(struct ablkcipher_request *req, int mode) -{ - struct dcp_dev_req_ctx *rctx = ablkcipher_request_ctx(req); - struct dcp_dev *dev = global_dev; - unsigned long flags; - int err = 0; - - if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) - return -EINVAL; - - rctx->mode = mode; - - spin_lock_irqsave(&dev->queue_lock, flags); - err = ablkcipher_enqueue_request(&dev->queue, req); - spin_unlock_irqrestore(&dev->queue_lock, flags); - - flags = test_and_set_bit(DCP_FLAG_BUSY, &dev->flags); - - if (!(flags & DCP_FLAG_BUSY)) - tasklet_schedule(&dev->queue_task); - - return err; -} - -static int dcp_aes_cbc_encrypt(struct ablkcipher_request *req) -{ - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); - struct dcp_op *ctx = crypto_ablkcipher_ctx( - crypto_ablkcipher_reqtfm(req)); - - if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - int err = 0; - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_encrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); - return err; - } - - return dcp_aes_cbc_crypt(req, DCP_AES | DCP_ENC | DCP_CBC); -} - -static int dcp_aes_cbc_decrypt(struct ablkcipher_request *req) -{ - struct crypto_tfm *tfm = - crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); - struct dcp_op *ctx = crypto_ablkcipher_ctx( - crypto_ablkcipher_reqtfm(req)); - - if (unlikely(ctx->keylen != AES_KEYSIZE_128)) { - int err = 0; - ablkcipher_request_set_tfm(req, ctx->fallback); - err = crypto_ablkcipher_decrypt(req); - ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); - return err; - } - return dcp_aes_cbc_crypt(req, DCP_AES | DCP_DEC | DCP_CBC); -} - -static struct crypto_alg algs[] = { - { - .cra_name = "cbc(aes)", - .cra_driver_name = "dcp-cbc-aes", - .cra_alignmask = 3, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = AES_KEYSIZE_128, - .cra_type = &crypto_ablkcipher_type, - .cra_priority = 300, - .cra_u.ablkcipher = { - .min_keysize = AES_KEYSIZE_128, - .max_keysize = AES_KEYSIZE_128, - .setkey = dcp_aes_setkey, - .encrypt = dcp_aes_cbc_encrypt, - .decrypt = dcp_aes_cbc_decrypt, - .ivsize = AES_KEYSIZE_128, - } - - }, -}; - -/* DCP bootstream verification interface: uses OTP key for crypto */ -static int dcp_bootstream_open(struct inode *inode, struct file *file) -{ - file->private_data = container_of((file->private_data), - struct dcp_dev, dcp_bootstream_misc); - return 0; -} - -static long dcp_bootstream_ioctl(struct file *file, - unsigned int cmd, unsigned long arg) -{ - struct dcp_dev *dev = (struct dcp_dev *) file->private_data; - void __user *argp = (void __user *)arg; - int ret; - - if (dev == NULL) - return -EBADF; - - if (cmd != DBS_ENC && cmd != DBS_DEC) - return -EINVAL; - - if (copy_from_user(dev->payload_base, argp, 16)) - return -EFAULT; - - if (test_and_set_bit(DCP_FLAG_BUSY, &dev->flags)) - return -EAGAIN; - - dev->ctx = kzalloc(sizeof(struct dcp_op), GFP_KERNEL); - if (!dev->ctx) { - dev_err(dev->dev, - "cannot allocate context for OTP crypto"); - clear_bit(DCP_FLAG_BUSY, &dev->flags); - return -ENOMEM; - } - - dev->ctx->flags = DCP_AES | DCP_ECB | DCP_OTP_KEY | DCP_CBC_INIT; - dev->ctx->flags |= (cmd == DBS_ENC) ? DCP_ENC : DCP_DEC; - dev->hw_pkg[0]->src = dev->payload_base_dma; - dev->hw_pkg[0]->dst = dev->payload_base_dma; - dev->hw_pkg[0]->size = 16; - - dcp_op_start(dev, 0); - - while (test_bit(DCP_FLAG_BUSY, &dev->flags)) - cpu_relax(); - - ret = dev->ctx->stat; - if (!ret && copy_to_user(argp, dev->payload_base, 16)) - ret = -EFAULT; - - kfree(dev->ctx); - - return ret; -} - -static const struct file_operations dcp_bootstream_fops = { - .owner = THIS_MODULE, - .unlocked_ioctl = dcp_bootstream_ioctl, - .open = dcp_bootstream_open, -}; - -static int dcp_probe(struct platform_device *pdev) -{ - struct dcp_dev *dev = NULL; - struct resource *r; - int i, ret, j; - - dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); - if (!dev) - return -ENOMEM; - - global_dev = dev; - dev->dev = &pdev->dev; - - platform_set_drvdata(pdev, dev); - - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dev->dcp_regs_base = devm_ioremap_resource(&pdev->dev, r); - if (IS_ERR(dev->dcp_regs_base)) - return PTR_ERR(dev->dcp_regs_base); - - dcp_set(dev, DCP_CTRL_SFRST, DCP_REG_CTRL); - udelay(10); - dcp_clear(dev, DCP_CTRL_SFRST | DCP_CTRL_CLKGATE, DCP_REG_CTRL); - - dcp_write(dev, DCP_CTRL_GATHER_RES_WRITE | - DCP_CTRL_ENABLE_CONTEXT_CACHE | DCP_CTRL_CH_IRQ_E_1, - DCP_REG_CTRL); - - dcp_write(dev, DCP_CHAN_CTRL_ENABLE_1, DCP_REG_CHAN_CTRL); - - for (i = 0; i < 4; i++) - dcp_clear(dev, -1, dcp_chan_reg(DCP_REG_CHAN_STAT, i)); - - dcp_clear(dev, -1, DCP_REG_STAT); - - - r = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!r) { - dev_err(&pdev->dev, "can't get IRQ resource (0)\n"); - return -EIO; - } - dev->dcp_vmi_irq = r->start; - ret = devm_request_irq(&pdev->dev, dev->dcp_vmi_irq, dcp_vmi_irq, 0, - "dcp", dev); - if (ret != 0) { - dev_err(&pdev->dev, "can't request_irq (0)\n"); - return -EIO; - } - - r = platform_get_resource(pdev, IORESOURCE_IRQ, 1); - if (!r) { - dev_err(&pdev->dev, "can't get IRQ resource (1)\n"); - return -EIO; - } - dev->dcp_irq = r->start; - ret = devm_request_irq(&pdev->dev, dev->dcp_irq, dcp_irq, 0, "dcp", - dev); - if (ret != 0) { - dev_err(&pdev->dev, "can't request_irq (1)\n"); - return -EIO; - } - - dev->hw_pkg[0] = dma_alloc_coherent(&pdev->dev, - DCP_MAX_PKG * sizeof(struct dcp_hw_packet), - &dev->hw_phys_pkg, - GFP_KERNEL); - if (!dev->hw_pkg[0]) { - dev_err(&pdev->dev, "Could not allocate hw descriptors\n"); - return -ENOMEM; - } - - for (i = 1; i < DCP_MAX_PKG; i++) { - dev->hw_pkg[i - 1]->next = dev->hw_phys_pkg - + i * sizeof(struct dcp_hw_packet); - dev->hw_pkg[i] = dev->hw_pkg[i - 1] + 1; - } - dev->hw_pkg[i - 1]->next = dev->hw_phys_pkg; - - - dev->payload_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, - &dev->payload_base_dma, GFP_KERNEL); - if (!dev->payload_base) { - dev_err(&pdev->dev, "Could not allocate memory for key\n"); - ret = -ENOMEM; - goto err_free_hw_packet; - } - tasklet_init(&dev->queue_task, dcp_queue_task, - (unsigned long) dev); - tasklet_init(&dev->done_task, dcp_done_task, - (unsigned long) dev); - spin_lock_init(&dev->queue_lock); - - crypto_init_queue(&dev->queue, 10); - - init_timer(&dev->watchdog); - dev->watchdog.function = &dcp_watchdog; - dev->watchdog.data = (unsigned long)dev; - - dev->dcp_bootstream_misc.minor = MISC_DYNAMIC_MINOR, - dev->dcp_bootstream_misc.name = "dcpboot", - dev->dcp_bootstream_misc.fops = &dcp_bootstream_fops, - ret = misc_register(&dev->dcp_bootstream_misc); - if (ret != 0) { - dev_err(dev->dev, "Unable to register misc device\n"); - goto err_free_key_iv; - } - - for (i = 0; i < ARRAY_SIZE(algs); i++) { - algs[i].cra_priority = 300; - algs[i].cra_ctxsize = sizeof(struct dcp_op); - algs[i].cra_module = THIS_MODULE; - algs[i].cra_init = dcp_cra_init; - algs[i].cra_exit = dcp_cra_exit; - if (crypto_register_alg(&algs[i])) { - dev_err(&pdev->dev, "register algorithm failed\n"); - ret = -ENOMEM; - goto err_unregister; - } - } - dev_notice(&pdev->dev, "DCP crypto enabled.!\n"); - - return 0; - -err_unregister: - for (j = 0; j < i; j++) - crypto_unregister_alg(&algs[j]); -err_free_key_iv: - tasklet_kill(&dev->done_task); - tasklet_kill(&dev->queue_task); - dma_free_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, dev->payload_base, - dev->payload_base_dma); -err_free_hw_packet: - dma_free_coherent(&pdev->dev, DCP_MAX_PKG * - sizeof(struct dcp_hw_packet), dev->hw_pkg[0], - dev->hw_phys_pkg); - - return ret; -} - -static int dcp_remove(struct platform_device *pdev) -{ - struct dcp_dev *dev; - int j; - dev = platform_get_drvdata(pdev); - - misc_deregister(&dev->dcp_bootstream_misc); - - for (j = 0; j < ARRAY_SIZE(algs); j++) - crypto_unregister_alg(&algs[j]); - - tasklet_kill(&dev->done_task); - tasklet_kill(&dev->queue_task); - - dma_free_coherent(&pdev->dev, 2 * AES_KEYSIZE_128, dev->payload_base, - dev->payload_base_dma); - - dma_free_coherent(&pdev->dev, - DCP_MAX_PKG * sizeof(struct dcp_hw_packet), - dev->hw_pkg[0], dev->hw_phys_pkg); - - return 0; -} - -static struct of_device_id fs_dcp_of_match[] = { - { .compatible = "fsl-dcp"}, - {}, -}; - -static struct platform_driver fs_dcp_driver = { - .probe = dcp_probe, - .remove = dcp_remove, - .driver = { - .name = "fsl-dcp", - .owner = THIS_MODULE, - .of_match_table = fs_dcp_of_match - } -}; - -module_platform_driver(fs_dcp_driver); - - -MODULE_AUTHOR("Tobias Rauter <tobias.rauter@gmail.com>"); -MODULE_DESCRIPTION("Freescale DCP Crypto Driver"); -MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/geode-aes.c b/drivers/crypto/geode-aes.c index 0c9ff497172..fe538e5287a 100644 --- a/drivers/crypto/geode-aes.c +++ b/drivers/crypto/geode-aes.c @@ -226,7 +226,7 @@ geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) op->dst = (void *) out; op->mode = AES_MODE_ECB; op->flags = 0; - op->len = AES_MIN_BLOCK_SIZE; + op->len = AES_BLOCK_SIZE; op->dir = AES_DIR_ENCRYPT; geode_aes_crypt(op); @@ -247,7 +247,7 @@ geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) op->dst = (void *) out; op->mode = AES_MODE_ECB; op->flags = 0; - op->len = AES_MIN_BLOCK_SIZE; + op->len = AES_BLOCK_SIZE; op->dir = AES_DIR_DECRYPT; geode_aes_crypt(op); @@ -255,7 +255,7 @@ geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) static int fallback_init_cip(struct crypto_tfm *tfm) { - const char *name = tfm->__crt_alg->cra_name; + const char *name = crypto_tfm_alg_name(tfm); struct geode_aes_op *op = crypto_tfm_ctx(tfm); op->fallback.cip = crypto_alloc_cipher(name, 0, @@ -286,7 +286,7 @@ static struct crypto_alg geode_alg = { CRYPTO_ALG_NEED_FALLBACK, .cra_init = fallback_init_cip, .cra_exit = fallback_exit_cip, - .cra_blocksize = AES_MIN_BLOCK_SIZE, + .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct geode_aes_op), .cra_module = THIS_MODULE, .cra_u = { @@ -320,7 +320,7 @@ geode_cbc_decrypt(struct blkcipher_desc *desc, op->src = walk.src.virt.addr, op->dst = walk.dst.virt.addr; op->mode = AES_MODE_CBC; - op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE); + op->len = nbytes - (nbytes % AES_BLOCK_SIZE); op->dir = AES_DIR_DECRYPT; ret = geode_aes_crypt(op); @@ -352,7 +352,7 @@ geode_cbc_encrypt(struct blkcipher_desc *desc, op->src = walk.src.virt.addr, op->dst = walk.dst.virt.addr; op->mode = AES_MODE_CBC; - op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE); + op->len = nbytes - (nbytes % AES_BLOCK_SIZE); op->dir = AES_DIR_ENCRYPT; ret = geode_aes_crypt(op); @@ -365,7 +365,7 @@ geode_cbc_encrypt(struct blkcipher_desc *desc, static int fallback_init_blk(struct crypto_tfm *tfm) { - const char *name = tfm->__crt_alg->cra_name; + const char *name = crypto_tfm_alg_name(tfm); struct geode_aes_op *op = crypto_tfm_ctx(tfm); op->fallback.blk = crypto_alloc_blkcipher(name, 0, @@ -396,7 +396,7 @@ static struct crypto_alg geode_cbc_alg = { CRYPTO_ALG_NEED_FALLBACK, .cra_init = fallback_init_blk, .cra_exit = fallback_exit_blk, - .cra_blocksize = AES_MIN_BLOCK_SIZE, + .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct geode_aes_op), .cra_alignmask = 15, .cra_type = &crypto_blkcipher_type, @@ -408,7 +408,7 @@ static struct crypto_alg geode_cbc_alg = { .setkey = geode_setkey_blk, .encrypt = geode_cbc_encrypt, .decrypt = geode_cbc_decrypt, - .ivsize = AES_IV_LENGTH, + .ivsize = AES_BLOCK_SIZE, } } }; @@ -432,7 +432,7 @@ geode_ecb_decrypt(struct blkcipher_desc *desc, op->src = walk.src.virt.addr, op->dst = walk.dst.virt.addr; op->mode = AES_MODE_ECB; - op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE); + op->len = nbytes - (nbytes % AES_BLOCK_SIZE); op->dir = AES_DIR_DECRYPT; ret = geode_aes_crypt(op); @@ -462,7 +462,7 @@ geode_ecb_encrypt(struct blkcipher_desc *desc, op->src = walk.src.virt.addr, op->dst = walk.dst.virt.addr; op->mode = AES_MODE_ECB; - op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE); + op->len = nbytes - (nbytes % AES_BLOCK_SIZE); op->dir = AES_DIR_ENCRYPT; ret = geode_aes_crypt(op); @@ -482,7 +482,7 @@ static struct crypto_alg geode_ecb_alg = { CRYPTO_ALG_NEED_FALLBACK, .cra_init = fallback_init_blk, .cra_exit = fallback_exit_blk, - .cra_blocksize = AES_MIN_BLOCK_SIZE, + .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct geode_aes_op), .cra_alignmask = 15, .cra_type = &crypto_blkcipher_type, @@ -547,7 +547,7 @@ static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) if (ret) goto eecb; - printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n"); + dev_notice(&dev->dev, "GEODE AES engine enabled.\n"); return 0; eecb: @@ -565,7 +565,7 @@ static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id) eenable: pci_disable_device(dev); - printk(KERN_ERR "geode-aes: GEODE AES initialization failed.\n"); + dev_err(&dev->dev, "GEODE AES initialization failed.\n"); return ret; } diff --git a/drivers/crypto/geode-aes.h b/drivers/crypto/geode-aes.h index f1855b50da4..f442ca972e3 100644 --- a/drivers/crypto/geode-aes.h +++ b/drivers/crypto/geode-aes.h @@ -10,10 +10,6 @@ #define _GEODE_AES_H_ /* driver logic flags */ -#define AES_IV_LENGTH 16 -#define AES_KEY_LENGTH 16 -#define AES_MIN_BLOCK_SIZE 16 - #define AES_MODE_ECB 0 #define AES_MODE_CBC 1 @@ -64,7 +60,7 @@ struct geode_aes_op { u32 flags; int len; - u8 key[AES_KEY_LENGTH]; + u8 key[AES_KEYSIZE_128]; u8 *iv; union { diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c index 8d1e6f8e9e9..29d0ee50490 100644 --- a/drivers/crypto/mv_cesa.c +++ b/drivers/crypto/mv_cesa.c @@ -622,8 +622,8 @@ static int queue_manag(void *data) } if (async_req) { - if (async_req->tfm->__crt_alg->cra_type != - &crypto_ahash_type) { + if (crypto_tfm_alg_type(async_req->tfm) != + CRYPTO_ALG_TYPE_AHASH) { struct ablkcipher_request *req = ablkcipher_request_cast(async_req); mv_start_new_crypt_req(req); @@ -843,7 +843,7 @@ static int mv_hash_setkey(struct crypto_ahash *tfm, const u8 * key, static int mv_cra_hash_init(struct crypto_tfm *tfm, const char *base_hash_name, enum hash_op op, int count_add) { - const char *fallback_driver_name = tfm->__crt_alg->cra_name; + const char *fallback_driver_name = crypto_tfm_alg_name(tfm); struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(tfm); struct crypto_shash *fallback_tfm = NULL; struct crypto_shash *base_hash = NULL; diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c new file mode 100644 index 00000000000..b5f7e6db24d --- /dev/null +++ b/drivers/crypto/mxs-dcp.c @@ -0,0 +1,1103 @@ +/* + * Freescale i.MX23/i.MX28 Data Co-Processor driver + * + * Copyright (C) 2013 Marek Vasut <marex@denx.de> + * + * The code contained herein is licensed under the GNU General Public + * License. You may obtain a copy of the GNU General Public License + * Version 2 or later at the following locations: + * + * http://www.opensource.org/licenses/gpl-license.html + * http://www.gnu.org/copyleft/gpl.html + */ + +#include <linux/crypto.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/kthread.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/stmp_device.h> + +#include <crypto/aes.h> +#include <crypto/sha.h> +#include <crypto/internal/hash.h> + +#define DCP_MAX_CHANS 4 +#define DCP_BUF_SZ PAGE_SIZE + +#define DCP_ALIGNMENT 64 + +/* DCP DMA descriptor. */ +struct dcp_dma_desc { + uint32_t next_cmd_addr; + uint32_t control0; + uint32_t control1; + uint32_t source; + uint32_t destination; + uint32_t size; + uint32_t payload; + uint32_t status; +}; + +/* Coherent aligned block for bounce buffering. */ +struct dcp_coherent_block { + uint8_t aes_in_buf[DCP_BUF_SZ]; + uint8_t aes_out_buf[DCP_BUF_SZ]; + uint8_t sha_in_buf[DCP_BUF_SZ]; + + uint8_t aes_key[2 * AES_KEYSIZE_128]; + + struct dcp_dma_desc desc[DCP_MAX_CHANS]; +}; + +struct dcp { + struct device *dev; + void __iomem *base; + + uint32_t caps; + + struct dcp_coherent_block *coh; + + struct completion completion[DCP_MAX_CHANS]; + struct mutex mutex[DCP_MAX_CHANS]; + struct task_struct *thread[DCP_MAX_CHANS]; + struct crypto_queue queue[DCP_MAX_CHANS]; +}; + +enum dcp_chan { + DCP_CHAN_HASH_SHA = 0, + DCP_CHAN_CRYPTO = 2, +}; + +struct dcp_async_ctx { + /* Common context */ + enum dcp_chan chan; + uint32_t fill; + + /* SHA Hash-specific context */ + struct mutex mutex; + uint32_t alg; + unsigned int hot:1; + + /* Crypto-specific context */ + struct crypto_ablkcipher *fallback; + unsigned int key_len; + uint8_t key[AES_KEYSIZE_128]; +}; + +struct dcp_aes_req_ctx { + unsigned int enc:1; + unsigned int ecb:1; +}; + +struct dcp_sha_req_ctx { + unsigned int init:1; + unsigned int fini:1; +}; + +/* + * There can even be only one instance of the MXS DCP due to the + * design of Linux Crypto API. + */ +static struct dcp *global_sdcp; + +/* DCP register layout. */ +#define MXS_DCP_CTRL 0x00 +#define MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES (1 << 23) +#define MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING (1 << 22) + +#define MXS_DCP_STAT 0x10 +#define MXS_DCP_STAT_CLR 0x18 +#define MXS_DCP_STAT_IRQ_MASK 0xf + +#define MXS_DCP_CHANNELCTRL 0x20 +#define MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK 0xff + +#define MXS_DCP_CAPABILITY1 0x40 +#define MXS_DCP_CAPABILITY1_SHA256 (4 << 16) +#define MXS_DCP_CAPABILITY1_SHA1 (1 << 16) +#define MXS_DCP_CAPABILITY1_AES128 (1 << 0) + +#define MXS_DCP_CONTEXT 0x50 + +#define MXS_DCP_CH_N_CMDPTR(n) (0x100 + ((n) * 0x40)) + +#define MXS_DCP_CH_N_SEMA(n) (0x110 + ((n) * 0x40)) + +#define MXS_DCP_CH_N_STAT(n) (0x120 + ((n) * 0x40)) +#define MXS_DCP_CH_N_STAT_CLR(n) (0x128 + ((n) * 0x40)) + +/* DMA descriptor bits. */ +#define MXS_DCP_CONTROL0_HASH_TERM (1 << 13) +#define MXS_DCP_CONTROL0_HASH_INIT (1 << 12) +#define MXS_DCP_CONTROL0_PAYLOAD_KEY (1 << 11) +#define MXS_DCP_CONTROL0_CIPHER_ENCRYPT (1 << 8) +#define MXS_DCP_CONTROL0_CIPHER_INIT (1 << 9) +#define MXS_DCP_CONTROL0_ENABLE_HASH (1 << 6) +#define MXS_DCP_CONTROL0_ENABLE_CIPHER (1 << 5) +#define MXS_DCP_CONTROL0_DECR_SEMAPHORE (1 << 1) +#define MXS_DCP_CONTROL0_INTERRUPT (1 << 0) + +#define MXS_DCP_CONTROL1_HASH_SELECT_SHA256 (2 << 16) +#define MXS_DCP_CONTROL1_HASH_SELECT_SHA1 (0 << 16) +#define MXS_DCP_CONTROL1_CIPHER_MODE_CBC (1 << 4) +#define MXS_DCP_CONTROL1_CIPHER_MODE_ECB (0 << 4) +#define MXS_DCP_CONTROL1_CIPHER_SELECT_AES128 (0 << 0) + +static int mxs_dcp_start_dma(struct dcp_async_ctx *actx) +{ + struct dcp *sdcp = global_sdcp; + const int chan = actx->chan; + uint32_t stat; + int ret; + struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; + + dma_addr_t desc_phys = dma_map_single(sdcp->dev, desc, sizeof(*desc), + DMA_TO_DEVICE); + + reinit_completion(&sdcp->completion[chan]); + + /* Clear status register. */ + writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(chan)); + + /* Load the DMA descriptor. */ + writel(desc_phys, sdcp->base + MXS_DCP_CH_N_CMDPTR(chan)); + + /* Increment the semaphore to start the DMA transfer. */ + writel(1, sdcp->base + MXS_DCP_CH_N_SEMA(chan)); + + ret = wait_for_completion_timeout(&sdcp->completion[chan], + msecs_to_jiffies(1000)); + if (!ret) { + dev_err(sdcp->dev, "Channel %i timeout (DCP_STAT=0x%08x)\n", + chan, readl(sdcp->base + MXS_DCP_STAT)); + return -ETIMEDOUT; + } + + stat = readl(sdcp->base + MXS_DCP_CH_N_STAT(chan)); + if (stat & 0xff) { + dev_err(sdcp->dev, "Channel %i error (CH_STAT=0x%08x)\n", + chan, stat); + return -EINVAL; + } + + dma_unmap_single(sdcp->dev, desc_phys, sizeof(*desc), DMA_TO_DEVICE); + + return 0; +} + +/* + * Encryption (AES128) + */ +static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, + struct ablkcipher_request *req, int init) +{ + struct dcp *sdcp = global_sdcp; + struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; + struct dcp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + int ret; + + dma_addr_t key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key, + 2 * AES_KEYSIZE_128, + DMA_TO_DEVICE); + dma_addr_t src_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_in_buf, + DCP_BUF_SZ, DMA_TO_DEVICE); + dma_addr_t dst_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_out_buf, + DCP_BUF_SZ, DMA_FROM_DEVICE); + + /* Fill in the DMA descriptor. */ + desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE | + MXS_DCP_CONTROL0_INTERRUPT | + MXS_DCP_CONTROL0_ENABLE_CIPHER; + + /* Payload contains the key. */ + desc->control0 |= MXS_DCP_CONTROL0_PAYLOAD_KEY; + + if (rctx->enc) + desc->control0 |= MXS_DCP_CONTROL0_CIPHER_ENCRYPT; + if (init) + desc->control0 |= MXS_DCP_CONTROL0_CIPHER_INIT; + + desc->control1 = MXS_DCP_CONTROL1_CIPHER_SELECT_AES128; + + if (rctx->ecb) + desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_ECB; + else + desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_CBC; + + desc->next_cmd_addr = 0; + desc->source = src_phys; + desc->destination = dst_phys; + desc->size = actx->fill; + desc->payload = key_phys; + desc->status = 0; + + ret = mxs_dcp_start_dma(actx); + + dma_unmap_single(sdcp->dev, key_phys, 2 * AES_KEYSIZE_128, + DMA_TO_DEVICE); + dma_unmap_single(sdcp->dev, src_phys, DCP_BUF_SZ, DMA_TO_DEVICE); + dma_unmap_single(sdcp->dev, dst_phys, DCP_BUF_SZ, DMA_FROM_DEVICE); + + return ret; +} + +static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq) +{ + struct dcp *sdcp = global_sdcp; + + struct ablkcipher_request *req = ablkcipher_request_cast(arq); + struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm); + struct dcp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + + struct scatterlist *dst = req->dst; + struct scatterlist *src = req->src; + const int nents = sg_nents(req->src); + + const int out_off = DCP_BUF_SZ; + uint8_t *in_buf = sdcp->coh->aes_in_buf; + uint8_t *out_buf = sdcp->coh->aes_out_buf; + + uint8_t *out_tmp, *src_buf, *dst_buf = NULL; + uint32_t dst_off = 0; + + uint8_t *key = sdcp->coh->aes_key; + + int ret = 0; + int split = 0; + unsigned int i, len, clen, rem = 0; + int init = 0; + + actx->fill = 0; + + /* Copy the key from the temporary location. */ + memcpy(key, actx->key, actx->key_len); + + if (!rctx->ecb) { + /* Copy the CBC IV just past the key. */ + memcpy(key + AES_KEYSIZE_128, req->info, AES_KEYSIZE_128); + /* CBC needs the INIT set. */ + init = 1; + } else { + memset(key + AES_KEYSIZE_128, 0, AES_KEYSIZE_128); + } + + for_each_sg(req->src, src, nents, i) { + src_buf = sg_virt(src); + len = sg_dma_len(src); + + do { + if (actx->fill + len > out_off) + clen = out_off - actx->fill; + else + clen = len; + + memcpy(in_buf + actx->fill, src_buf, clen); + len -= clen; + src_buf += clen; + actx->fill += clen; + + /* + * If we filled the buffer or this is the last SG, + * submit the buffer. + */ + if (actx->fill == out_off || sg_is_last(src)) { + ret = mxs_dcp_run_aes(actx, req, init); + if (ret) + return ret; + init = 0; + + out_tmp = out_buf; + while (dst && actx->fill) { + if (!split) { + dst_buf = sg_virt(dst); + dst_off = 0; + } + rem = min(sg_dma_len(dst) - dst_off, + actx->fill); + + memcpy(dst_buf + dst_off, out_tmp, rem); + out_tmp += rem; + dst_off += rem; + actx->fill -= rem; + + if (dst_off == sg_dma_len(dst)) { + dst = sg_next(dst); + split = 0; + } else { + split = 1; + } + } + } + } while (len); + } + + return ret; +} + +static int dcp_chan_thread_aes(void *data) +{ + struct dcp *sdcp = global_sdcp; + const int chan = DCP_CHAN_CRYPTO; + + struct crypto_async_request *backlog; + struct crypto_async_request *arq; + + int ret; + + do { + __set_current_state(TASK_INTERRUPTIBLE); + + mutex_lock(&sdcp->mutex[chan]); + backlog = crypto_get_backlog(&sdcp->queue[chan]); + arq = crypto_dequeue_request(&sdcp->queue[chan]); + mutex_unlock(&sdcp->mutex[chan]); + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + if (arq) { + ret = mxs_dcp_aes_block_crypt(arq); + arq->complete(arq, ret); + continue; + } + + schedule(); + } while (!kthread_should_stop()); + + return 0; +} + +static int mxs_dcp_block_fallback(struct ablkcipher_request *req, int enc) +{ + struct crypto_tfm *tfm = + crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req)); + struct dcp_async_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + int ret; + + ablkcipher_request_set_tfm(req, ctx->fallback); + + if (enc) + ret = crypto_ablkcipher_encrypt(req); + else + ret = crypto_ablkcipher_decrypt(req); + + ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm)); + + return ret; +} + +static int mxs_dcp_aes_enqueue(struct ablkcipher_request *req, int enc, int ecb) +{ + struct dcp *sdcp = global_sdcp; + struct crypto_async_request *arq = &req->base; + struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm); + struct dcp_aes_req_ctx *rctx = ablkcipher_request_ctx(req); + int ret; + + if (unlikely(actx->key_len != AES_KEYSIZE_128)) + return mxs_dcp_block_fallback(req, enc); + + rctx->enc = enc; + rctx->ecb = ecb; + actx->chan = DCP_CHAN_CRYPTO; + + mutex_lock(&sdcp->mutex[actx->chan]); + ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base); + mutex_unlock(&sdcp->mutex[actx->chan]); + + wake_up_process(sdcp->thread[actx->chan]); + + return -EINPROGRESS; +} + +static int mxs_dcp_aes_ecb_decrypt(struct ablkcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 0, 1); +} + +static int mxs_dcp_aes_ecb_encrypt(struct ablkcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 1, 1); +} + +static int mxs_dcp_aes_cbc_decrypt(struct ablkcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 0, 0); +} + +static int mxs_dcp_aes_cbc_encrypt(struct ablkcipher_request *req) +{ + return mxs_dcp_aes_enqueue(req, 1, 0); +} + +static int mxs_dcp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int len) +{ + struct dcp_async_ctx *actx = crypto_ablkcipher_ctx(tfm); + unsigned int ret; + + /* + * AES 128 is supposed by the hardware, store key into temporary + * buffer and exit. We must use the temporary buffer here, since + * there can still be an operation in progress. + */ + actx->key_len = len; + if (len == AES_KEYSIZE_128) { + memcpy(actx->key, key, len); + return 0; + } + + /* Check if the key size is supported by kernel at all. */ + if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) { + tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + /* + * If the requested AES key size is not supported by the hardware, + * but is supported by in-kernel software implementation, we use + * software fallback. + */ + actx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; + actx->fallback->base.crt_flags |= + tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK; + + ret = crypto_ablkcipher_setkey(actx->fallback, key, len); + if (!ret) + return 0; + + tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK; + tfm->base.crt_flags |= + actx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK; + + return ret; +} + +static int mxs_dcp_aes_fallback_init(struct crypto_tfm *tfm) +{ + const char *name = crypto_tfm_alg_name(tfm); + const uint32_t flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK; + struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm); + struct crypto_ablkcipher *blk; + + blk = crypto_alloc_ablkcipher(name, 0, flags); + if (IS_ERR(blk)) + return PTR_ERR(blk); + + actx->fallback = blk; + tfm->crt_ablkcipher.reqsize = sizeof(struct dcp_aes_req_ctx); + return 0; +} + +static void mxs_dcp_aes_fallback_exit(struct crypto_tfm *tfm) +{ + struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm); + + crypto_free_ablkcipher(actx->fallback); + actx->fallback = NULL; +} + +/* + * Hashing (SHA1/SHA256) + */ +static int mxs_dcp_run_sha(struct ahash_request *req) +{ + struct dcp *sdcp = global_sdcp; + int ret; + + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct hash_alg_common *halg = crypto_hash_alg_common(tfm); + + struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; + + dma_addr_t digest_phys = 0; + dma_addr_t buf_phys = dma_map_single(sdcp->dev, sdcp->coh->sha_in_buf, + DCP_BUF_SZ, DMA_TO_DEVICE); + + /* Fill in the DMA descriptor. */ + desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE | + MXS_DCP_CONTROL0_INTERRUPT | + MXS_DCP_CONTROL0_ENABLE_HASH; + if (rctx->init) + desc->control0 |= MXS_DCP_CONTROL0_HASH_INIT; + + desc->control1 = actx->alg; + desc->next_cmd_addr = 0; + desc->source = buf_phys; + desc->destination = 0; + desc->size = actx->fill; + desc->payload = 0; + desc->status = 0; + + /* Set HASH_TERM bit for last transfer block. */ + if (rctx->fini) { + digest_phys = dma_map_single(sdcp->dev, req->result, + halg->digestsize, DMA_FROM_DEVICE); + desc->control0 |= MXS_DCP_CONTROL0_HASH_TERM; + desc->payload = digest_phys; + } + + ret = mxs_dcp_start_dma(actx); + + if (rctx->fini) + dma_unmap_single(sdcp->dev, digest_phys, halg->digestsize, + DMA_FROM_DEVICE); + + dma_unmap_single(sdcp->dev, buf_phys, DCP_BUF_SZ, DMA_TO_DEVICE); + + return ret; +} + +static int dcp_sha_req_to_buf(struct crypto_async_request *arq) +{ + struct dcp *sdcp = global_sdcp; + + struct ahash_request *req = ahash_request_cast(arq); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct hash_alg_common *halg = crypto_hash_alg_common(tfm); + const int nents = sg_nents(req->src); + + uint8_t *in_buf = sdcp->coh->sha_in_buf; + + uint8_t *src_buf; + + struct scatterlist *src; + + unsigned int i, len, clen; + int ret; + + int fin = rctx->fini; + if (fin) + rctx->fini = 0; + + for_each_sg(req->src, src, nents, i) { + src_buf = sg_virt(src); + len = sg_dma_len(src); + + do { + if (actx->fill + len > DCP_BUF_SZ) + clen = DCP_BUF_SZ - actx->fill; + else + clen = len; + + memcpy(in_buf + actx->fill, src_buf, clen); + len -= clen; + src_buf += clen; + actx->fill += clen; + + /* + * If we filled the buffer and still have some + * more data, submit the buffer. + */ + if (len && actx->fill == DCP_BUF_SZ) { + ret = mxs_dcp_run_sha(req); + if (ret) + return ret; + actx->fill = 0; + rctx->init = 0; + } + } while (len); + } + + if (fin) { + rctx->fini = 1; + + /* Submit whatever is left. */ + if (!req->result) + return -EINVAL; + + ret = mxs_dcp_run_sha(req); + if (ret) + return ret; + + actx->fill = 0; + + /* For some reason, the result is flipped. */ + for (i = 0; i < halg->digestsize / 2; i++) { + swap(req->result[i], + req->result[halg->digestsize - i - 1]); + } + } + + return 0; +} + +static int dcp_chan_thread_sha(void *data) +{ + struct dcp *sdcp = global_sdcp; + const int chan = DCP_CHAN_HASH_SHA; + + struct crypto_async_request *backlog; + struct crypto_async_request *arq; + + struct dcp_sha_req_ctx *rctx; + + struct ahash_request *req; + int ret, fini; + + do { + __set_current_state(TASK_INTERRUPTIBLE); + + mutex_lock(&sdcp->mutex[chan]); + backlog = crypto_get_backlog(&sdcp->queue[chan]); + arq = crypto_dequeue_request(&sdcp->queue[chan]); + mutex_unlock(&sdcp->mutex[chan]); + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + if (arq) { + req = ahash_request_cast(arq); + rctx = ahash_request_ctx(req); + + ret = dcp_sha_req_to_buf(arq); + fini = rctx->fini; + arq->complete(arq, ret); + if (!fini) + continue; + } + + schedule(); + } while (!kthread_should_stop()); + + return 0; +} + +static int dcp_sha_init(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + + struct hash_alg_common *halg = crypto_hash_alg_common(tfm); + + /* + * Start hashing session. The code below only inits the + * hashing session context, nothing more. + */ + memset(actx, 0, sizeof(*actx)); + + if (strcmp(halg->base.cra_name, "sha1") == 0) + actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA1; + else + actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA256; + + actx->fill = 0; + actx->hot = 0; + actx->chan = DCP_CHAN_HASH_SHA; + + mutex_init(&actx->mutex); + + return 0; +} + +static int dcp_sha_update_fx(struct ahash_request *req, int fini) +{ + struct dcp *sdcp = global_sdcp; + + struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req); + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm); + + int ret; + + /* + * Ignore requests that have no data in them and are not + * the trailing requests in the stream of requests. + */ + if (!req->nbytes && !fini) + return 0; + + mutex_lock(&actx->mutex); + + rctx->fini = fini; + + if (!actx->hot) { + actx->hot = 1; + rctx->init = 1; + } + + mutex_lock(&sdcp->mutex[actx->chan]); + ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base); + mutex_unlock(&sdcp->mutex[actx->chan]); + + wake_up_process(sdcp->thread[actx->chan]); + mutex_unlock(&actx->mutex); + + return -EINPROGRESS; +} + +static int dcp_sha_update(struct ahash_request *req) +{ + return dcp_sha_update_fx(req, 0); +} + +static int dcp_sha_final(struct ahash_request *req) +{ + ahash_request_set_crypt(req, NULL, req->result, 0); + req->nbytes = 0; + return dcp_sha_update_fx(req, 1); +} + +static int dcp_sha_finup(struct ahash_request *req) +{ + return dcp_sha_update_fx(req, 1); +} + +static int dcp_sha_digest(struct ahash_request *req) +{ + int ret; + + ret = dcp_sha_init(req); + if (ret) + return ret; + + return dcp_sha_finup(req); +} + +static int dcp_sha_cra_init(struct crypto_tfm *tfm) +{ + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct dcp_sha_req_ctx)); + return 0; +} + +static void dcp_sha_cra_exit(struct crypto_tfm *tfm) +{ +} + +/* AES 128 ECB and AES 128 CBC */ +static struct crypto_alg dcp_aes_algs[] = { + { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-dcp", + .cra_priority = 400, + .cra_alignmask = 15, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_init = mxs_dcp_aes_fallback_init, + .cra_exit = mxs_dcp_aes_fallback_exit, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = mxs_dcp_aes_setkey, + .encrypt = mxs_dcp_aes_ecb_encrypt, + .decrypt = mxs_dcp_aes_ecb_decrypt + }, + }, + }, { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-dcp", + .cra_priority = 400, + .cra_alignmask = 15, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_NEED_FALLBACK, + .cra_init = mxs_dcp_aes_fallback_init, + .cra_exit = mxs_dcp_aes_fallback_exit, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = mxs_dcp_aes_setkey, + .encrypt = mxs_dcp_aes_cbc_encrypt, + .decrypt = mxs_dcp_aes_cbc_decrypt, + .ivsize = AES_BLOCK_SIZE, + }, + }, + }, +}; + +/* SHA1 */ +static struct ahash_alg dcp_sha1_alg = { + .init = dcp_sha_init, + .update = dcp_sha_update, + .final = dcp_sha_final, + .finup = dcp_sha_finup, + .digest = dcp_sha_digest, + .halg = { + .digestsize = SHA1_DIGEST_SIZE, + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-dcp", + .cra_priority = 400, + .cra_alignmask = 63, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_module = THIS_MODULE, + .cra_init = dcp_sha_cra_init, + .cra_exit = dcp_sha_cra_exit, + }, + }, +}; + +/* SHA256 */ +static struct ahash_alg dcp_sha256_alg = { + .init = dcp_sha_init, + .update = dcp_sha_update, + .final = dcp_sha_final, + .finup = dcp_sha_finup, + .digest = dcp_sha_digest, + .halg = { + .digestsize = SHA256_DIGEST_SIZE, + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-dcp", + .cra_priority = 400, + .cra_alignmask = 63, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dcp_async_ctx), + .cra_module = THIS_MODULE, + .cra_init = dcp_sha_cra_init, + .cra_exit = dcp_sha_cra_exit, + }, + }, +}; + +static irqreturn_t mxs_dcp_irq(int irq, void *context) +{ + struct dcp *sdcp = context; + uint32_t stat; + int i; + + stat = readl(sdcp->base + MXS_DCP_STAT); + stat &= MXS_DCP_STAT_IRQ_MASK; + if (!stat) + return IRQ_NONE; + + /* Clear the interrupts. */ + writel(stat, sdcp->base + MXS_DCP_STAT_CLR); + + /* Complete the DMA requests that finished. */ + for (i = 0; i < DCP_MAX_CHANS; i++) + if (stat & (1 << i)) + complete(&sdcp->completion[i]); + + return IRQ_HANDLED; +} + +static int mxs_dcp_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dcp *sdcp = NULL; + int i, ret; + + struct resource *iores; + int dcp_vmi_irq, dcp_irq; + + if (global_sdcp) { + dev_err(dev, "Only one DCP instance allowed!\n"); + return -ENODEV; + } + + iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); + dcp_vmi_irq = platform_get_irq(pdev, 0); + if (dcp_vmi_irq < 0) + return dcp_vmi_irq; + + dcp_irq = platform_get_irq(pdev, 1); + if (dcp_irq < 0) + return dcp_irq; + + sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL); + if (!sdcp) + return -ENOMEM; + + sdcp->dev = dev; + sdcp->base = devm_ioremap_resource(dev, iores); + if (IS_ERR(sdcp->base)) + return PTR_ERR(sdcp->base); + + + ret = devm_request_irq(dev, dcp_vmi_irq, mxs_dcp_irq, 0, + "dcp-vmi-irq", sdcp); + if (ret) { + dev_err(dev, "Failed to claim DCP VMI IRQ!\n"); + return ret; + } + + ret = devm_request_irq(dev, dcp_irq, mxs_dcp_irq, 0, + "dcp-irq", sdcp); + if (ret) { + dev_err(dev, "Failed to claim DCP IRQ!\n"); + return ret; + } + + /* Allocate coherent helper block. */ + sdcp->coh = devm_kzalloc(dev, sizeof(*sdcp->coh) + DCP_ALIGNMENT, + GFP_KERNEL); + if (!sdcp->coh) + return -ENOMEM; + + /* Re-align the structure so it fits the DCP constraints. */ + sdcp->coh = PTR_ALIGN(sdcp->coh, DCP_ALIGNMENT); + + /* Restart the DCP block. */ + ret = stmp_reset_block(sdcp->base); + if (ret) + return ret; + + /* Initialize control register. */ + writel(MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES | + MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING | 0xf, + sdcp->base + MXS_DCP_CTRL); + + /* Enable all DCP DMA channels. */ + writel(MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK, + sdcp->base + MXS_DCP_CHANNELCTRL); + + /* + * We do not enable context switching. Give the context buffer a + * pointer to an illegal address so if context switching is + * inadvertantly enabled, the DCP will return an error instead of + * trashing good memory. The DCP DMA cannot access ROM, so any ROM + * address will do. + */ + writel(0xffff0000, sdcp->base + MXS_DCP_CONTEXT); + for (i = 0; i < DCP_MAX_CHANS; i++) + writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(i)); + writel(0xffffffff, sdcp->base + MXS_DCP_STAT_CLR); + + global_sdcp = sdcp; + + platform_set_drvdata(pdev, sdcp); + + for (i = 0; i < DCP_MAX_CHANS; i++) { + mutex_init(&sdcp->mutex[i]); + init_completion(&sdcp->completion[i]); + crypto_init_queue(&sdcp->queue[i], 50); + } + + /* Create the SHA and AES handler threads. */ + sdcp->thread[DCP_CHAN_HASH_SHA] = kthread_run(dcp_chan_thread_sha, + NULL, "mxs_dcp_chan/sha"); + if (IS_ERR(sdcp->thread[DCP_CHAN_HASH_SHA])) { + dev_err(dev, "Error starting SHA thread!\n"); + return PTR_ERR(sdcp->thread[DCP_CHAN_HASH_SHA]); + } + + sdcp->thread[DCP_CHAN_CRYPTO] = kthread_run(dcp_chan_thread_aes, + NULL, "mxs_dcp_chan/aes"); + if (IS_ERR(sdcp->thread[DCP_CHAN_CRYPTO])) { + dev_err(dev, "Error starting SHA thread!\n"); + ret = PTR_ERR(sdcp->thread[DCP_CHAN_CRYPTO]); + goto err_destroy_sha_thread; + } + + /* Register the various crypto algorithms. */ + sdcp->caps = readl(sdcp->base + MXS_DCP_CAPABILITY1); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) { + ret = crypto_register_algs(dcp_aes_algs, + ARRAY_SIZE(dcp_aes_algs)); + if (ret) { + /* Failed to register algorithm. */ + dev_err(dev, "Failed to register AES crypto!\n"); + goto err_destroy_aes_thread; + } + } + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) { + ret = crypto_register_ahash(&dcp_sha1_alg); + if (ret) { + dev_err(dev, "Failed to register %s hash!\n", + dcp_sha1_alg.halg.base.cra_name); + goto err_unregister_aes; + } + } + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256) { + ret = crypto_register_ahash(&dcp_sha256_alg); + if (ret) { + dev_err(dev, "Failed to register %s hash!\n", + dcp_sha256_alg.halg.base.cra_name); + goto err_unregister_sha1; + } + } + + return 0; + +err_unregister_sha1: + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) + crypto_unregister_ahash(&dcp_sha1_alg); + +err_unregister_aes: + if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) + crypto_unregister_algs(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs)); + +err_destroy_aes_thread: + kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]); + +err_destroy_sha_thread: + kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]); + return ret; +} + +static int mxs_dcp_remove(struct platform_device *pdev) +{ + struct dcp *sdcp = platform_get_drvdata(pdev); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256) + crypto_unregister_ahash(&dcp_sha256_alg); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) + crypto_unregister_ahash(&dcp_sha1_alg); + + if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) + crypto_unregister_algs(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs)); + + kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]); + kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]); + + platform_set_drvdata(pdev, NULL); + + global_sdcp = NULL; + + return 0; +} + +static const struct of_device_id mxs_dcp_dt_ids[] = { + { .compatible = "fsl,imx23-dcp", .data = NULL, }, + { .compatible = "fsl,imx28-dcp", .data = NULL, }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(of, mxs_dcp_dt_ids); + +static struct platform_driver mxs_dcp_driver = { + .probe = mxs_dcp_probe, + .remove = mxs_dcp_remove, + .driver = { + .name = "mxs-dcp", + .owner = THIS_MODULE, + .of_match_table = mxs_dcp_dt_ids, + }, +}; + +module_platform_driver(mxs_dcp_driver); + +MODULE_AUTHOR("Marek Vasut <marex@denx.de>"); +MODULE_DESCRIPTION("Freescale MXS DCP Driver"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:mxs-dcp"); diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index e1f0ab413c3..7263c10a56e 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -356,7 +356,7 @@ static int n2_hash_async_finup(struct ahash_request *req) static int n2_hash_cra_init(struct crypto_tfm *tfm) { - const char *fallback_driver_name = tfm->__crt_alg->cra_name; + const char *fallback_driver_name = crypto_tfm_alg_name(tfm); struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct crypto_ahash *fallback_tfm; @@ -391,7 +391,7 @@ static void n2_hash_cra_exit(struct crypto_tfm *tfm) static int n2_hmac_cra_init(struct crypto_tfm *tfm) { - const char *fallback_driver_name = tfm->__crt_alg->cra_name; + const char *fallback_driver_name = crypto_tfm_alg_name(tfm); struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm); diff --git a/drivers/crypto/nx/nx-842.c b/drivers/crypto/nx/nx-842.c index 6c4c000671c..502edf0a293 100644 --- a/drivers/crypto/nx/nx-842.c +++ b/drivers/crypto/nx/nx-842.c @@ -158,6 +158,15 @@ static inline unsigned long nx842_get_scatterlist_size( return sl->entry_nr * sizeof(struct nx842_slentry); } +static inline unsigned long nx842_get_pa(void *addr) +{ + if (is_vmalloc_addr(addr)) + return page_to_phys(vmalloc_to_page(addr)) + + offset_in_page(addr); + else + return __pa(addr); +} + static int nx842_build_scatterlist(unsigned long buf, int len, struct nx842_scatterlist *sl) { @@ -168,7 +177,7 @@ static int nx842_build_scatterlist(unsigned long buf, int len, entry = sl->entries; while (len) { - entry->ptr = __pa(buf); + entry->ptr = nx842_get_pa((void *)buf); nextpage = ALIGN(buf + 1, NX842_HW_PAGE_SIZE); if (nextpage < buf + len) { /* we aren't at the end yet */ @@ -370,8 +379,8 @@ int nx842_compress(const unsigned char *in, unsigned int inlen, op.flags = NX842_OP_COMPRESS; csbcpb = &workmem->csbcpb; memset(csbcpb, 0, sizeof(*csbcpb)); - op.csbcpb = __pa(csbcpb); - op.out = __pa(slout.entries); + op.csbcpb = nx842_get_pa(csbcpb); + op.out = nx842_get_pa(slout.entries); for (i = 0; i < hdr->blocks_nr; i++) { /* @@ -401,13 +410,13 @@ int nx842_compress(const unsigned char *in, unsigned int inlen, */ if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) { /* Create direct DDE */ - op.in = __pa(inbuf); + op.in = nx842_get_pa((void *)inbuf); op.inlen = max_sync_size; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(inbuf, max_sync_size, &slin); - op.in = __pa(slin.entries); + op.in = nx842_get_pa(slin.entries); op.inlen = -nx842_get_scatterlist_size(&slin); } @@ -565,7 +574,7 @@ int nx842_decompress(const unsigned char *in, unsigned int inlen, op.flags = NX842_OP_DECOMPRESS; csbcpb = &workmem->csbcpb; memset(csbcpb, 0, sizeof(*csbcpb)); - op.csbcpb = __pa(csbcpb); + op.csbcpb = nx842_get_pa(csbcpb); /* * max_sync_size may have changed since compression, @@ -597,12 +606,12 @@ int nx842_decompress(const unsigned char *in, unsigned int inlen, if (likely((inbuf & NX842_HW_PAGE_MASK) == ((inbuf + hdr->sizes[i] - 1) & NX842_HW_PAGE_MASK))) { /* Create direct DDE */ - op.in = __pa(inbuf); + op.in = nx842_get_pa((void *)inbuf); op.inlen = hdr->sizes[i]; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(inbuf, hdr->sizes[i] , &slin); - op.in = __pa(slin.entries); + op.in = nx842_get_pa(slin.entries); op.inlen = -nx842_get_scatterlist_size(&slin); } @@ -613,12 +622,12 @@ int nx842_decompress(const unsigned char *in, unsigned int inlen, */ if (likely(max_sync_size == NX842_HW_PAGE_SIZE)) { /* Create direct DDE */ - op.out = __pa(outbuf); + op.out = nx842_get_pa((void *)outbuf); op.outlen = max_sync_size; } else { /* Create indirect DDE (scatterlist) */ nx842_build_scatterlist(outbuf, max_sync_size, &slout); - op.out = __pa(slout.entries); + op.out = nx842_get_pa(slout.entries); op.outlen = -nx842_get_scatterlist_size(&slout); } @@ -1188,12 +1197,7 @@ static int __init nx842_probe(struct vio_dev *viodev, } rcu_read_lock(); - if (dev_set_drvdata(&viodev->dev, rcu_dereference(devdata))) { - rcu_read_unlock(); - dev_err(&viodev->dev, "failed to set driver data for device\n"); - ret = -1; - goto error; - } + dev_set_drvdata(&viodev->dev, rcu_dereference(devdata)); rcu_read_unlock(); if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) { @@ -1225,7 +1229,7 @@ static int __exit nx842_remove(struct vio_dev *viodev) old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_mutex)); of_reconfig_notifier_unregister(&nx842_of_nb); - rcu_assign_pointer(devdata, NULL); + RCU_INIT_POINTER(devdata, NULL); spin_unlock_irqrestore(&devdata_mutex, flags); synchronize_rcu(); dev_set_drvdata(&viodev->dev, NULL); @@ -1276,7 +1280,7 @@ static void __exit nx842_exit(void) spin_lock_irqsave(&devdata_mutex, flags); old_devdata = rcu_dereference_check(devdata, lockdep_is_held(&devdata_mutex)); - rcu_assign_pointer(devdata, NULL); + RCU_INIT_POINTER(devdata, NULL); spin_unlock_irqrestore(&devdata_mutex, flags); synchronize_rcu(); if (old_devdata) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index a9ccbf14096..cb98fa54573 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -784,6 +784,7 @@ static int omap_aes_ctr_decrypt(struct ablkcipher_request *req) static int omap_aes_cra_init(struct crypto_tfm *tfm) { struct omap_aes_dev *dd = NULL; + int err; /* Find AES device, currently picks the first device */ spin_lock_bh(&list_lock); @@ -792,7 +793,13 @@ static int omap_aes_cra_init(struct crypto_tfm *tfm) } spin_unlock_bh(&list_lock); - pm_runtime_get_sync(dd->dev); + err = pm_runtime_get_sync(dd->dev); + if (err < 0) { + dev_err(dd->dev, "%s: failed to get_sync(%d)\n", + __func__, err); + return err; + } + tfm->crt_ablkcipher.reqsize = sizeof(struct omap_aes_reqctx); return 0; @@ -1182,7 +1189,12 @@ static int omap_aes_probe(struct platform_device *pdev) dd->phys_base = res.start; pm_runtime_enable(dev); - pm_runtime_get_sync(dev); + err = pm_runtime_get_sync(dev); + if (err < 0) { + dev_err(dev, "%s: failed to get_sync(%d)\n", + __func__, err); + goto err_res; + } omap_aes_dma_stop(dd); @@ -1295,9 +1307,7 @@ static int omap_aes_resume(struct device *dev) } #endif -static const struct dev_pm_ops omap_aes_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(omap_aes_suspend, omap_aes_resume) -}; +static SIMPLE_DEV_PM_OPS(omap_aes_pm_ops, omap_aes_suspend, omap_aes_resume); static struct platform_driver omap_aes_driver = { .probe = omap_aes_probe, diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c new file mode 100644 index 00000000000..b8bc84be874 --- /dev/null +++ b/drivers/crypto/omap-des.c @@ -0,0 +1,1235 @@ +/* + * Support for OMAP DES and Triple DES HW acceleration. + * + * Copyright (c) 2013 Texas Instruments Incorporated + * Author: Joel Fernandes <joelf@ti.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + */ + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#ifdef DEBUG +#define prn(num) printk(#num "=%d\n", num) +#define prx(num) printk(#num "=%x\n", num) +#else +#define prn(num) do { } while (0) +#define prx(num) do { } while (0) +#endif + +#include <linux/err.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/errno.h> +#include <linux/kernel.h> +#include <linux/platform_device.h> +#include <linux/scatterlist.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> +#include <linux/omap-dma.h> +#include <linux/pm_runtime.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_address.h> +#include <linux/io.h> +#include <linux/crypto.h> +#include <linux/interrupt.h> +#include <crypto/scatterwalk.h> +#include <crypto/des.h> + +#define DST_MAXBURST 2 + +#define DES_BLOCK_WORDS (DES_BLOCK_SIZE >> 2) + +#define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) + +#define DES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \ + ((x ^ 0x01) * 0x04)) + +#define DES_REG_IV(dd, x) ((dd)->pdata->iv_ofs + ((x) * 0x04)) + +#define DES_REG_CTRL(dd) ((dd)->pdata->ctrl_ofs) +#define DES_REG_CTRL_CBC BIT(4) +#define DES_REG_CTRL_TDES BIT(3) +#define DES_REG_CTRL_DIRECTION BIT(2) +#define DES_REG_CTRL_INPUT_READY BIT(1) +#define DES_REG_CTRL_OUTPUT_READY BIT(0) + +#define DES_REG_DATA_N(dd, x) ((dd)->pdata->data_ofs + ((x) * 0x04)) + +#define DES_REG_REV(dd) ((dd)->pdata->rev_ofs) + +#define DES_REG_MASK(dd) ((dd)->pdata->mask_ofs) + +#define DES_REG_LENGTH_N(x) (0x24 + ((x) * 0x04)) + +#define DES_REG_IRQ_STATUS(dd) ((dd)->pdata->irq_status_ofs) +#define DES_REG_IRQ_ENABLE(dd) ((dd)->pdata->irq_enable_ofs) +#define DES_REG_IRQ_DATA_IN BIT(1) +#define DES_REG_IRQ_DATA_OUT BIT(2) + +#define FLAGS_MODE_MASK 0x000f +#define FLAGS_ENCRYPT BIT(0) +#define FLAGS_CBC BIT(1) +#define FLAGS_INIT BIT(4) +#define FLAGS_BUSY BIT(6) + +struct omap_des_ctx { + struct omap_des_dev *dd; + + int keylen; + u32 key[(3 * DES_KEY_SIZE) / sizeof(u32)]; + unsigned long flags; +}; + +struct omap_des_reqctx { + unsigned long mode; +}; + +#define OMAP_DES_QUEUE_LENGTH 1 +#define OMAP_DES_CACHE_SIZE 0 + +struct omap_des_algs_info { + struct crypto_alg *algs_list; + unsigned int size; + unsigned int registered; +}; + +struct omap_des_pdata { + struct omap_des_algs_info *algs_info; + unsigned int algs_info_size; + + void (*trigger)(struct omap_des_dev *dd, int length); + + u32 key_ofs; + u32 iv_ofs; + u32 ctrl_ofs; + u32 data_ofs; + u32 rev_ofs; + u32 mask_ofs; + u32 irq_enable_ofs; + u32 irq_status_ofs; + + u32 dma_enable_in; + u32 dma_enable_out; + u32 dma_start; + + u32 major_mask; + u32 major_shift; + u32 minor_mask; + u32 minor_shift; +}; + +struct omap_des_dev { + struct list_head list; + unsigned long phys_base; + void __iomem *io_base; + struct omap_des_ctx *ctx; + struct device *dev; + unsigned long flags; + int err; + + /* spinlock used for queues */ + spinlock_t lock; + struct crypto_queue queue; + + struct tasklet_struct done_task; + struct tasklet_struct queue_task; + + struct ablkcipher_request *req; + /* + * total is used by PIO mode for book keeping so introduce + * variable total_save as need it to calc page_order + */ + size_t total; + size_t total_save; + + struct scatterlist *in_sg; + struct scatterlist *out_sg; + + /* Buffers for copying for unaligned cases */ + struct scatterlist in_sgl; + struct scatterlist out_sgl; + struct scatterlist *orig_out; + int sgs_copied; + + struct scatter_walk in_walk; + struct scatter_walk out_walk; + int dma_in; + struct dma_chan *dma_lch_in; + int dma_out; + struct dma_chan *dma_lch_out; + int in_sg_len; + int out_sg_len; + int pio_only; + const struct omap_des_pdata *pdata; +}; + +/* keep registered devices data here */ +static LIST_HEAD(dev_list); +static DEFINE_SPINLOCK(list_lock); + +#ifdef DEBUG +#define omap_des_read(dd, offset) \ + ({ \ + int _read_ret; \ + _read_ret = __raw_readl(dd->io_base + offset); \ + pr_err("omap_des_read(" #offset "=%#x)= %#x\n", \ + offset, _read_ret); \ + _read_ret; \ + }) +#else +static inline u32 omap_des_read(struct omap_des_dev *dd, u32 offset) +{ + return __raw_readl(dd->io_base + offset); +} +#endif + +#ifdef DEBUG +#define omap_des_write(dd, offset, value) \ + do { \ + pr_err("omap_des_write(" #offset "=%#x) value=%#x\n", \ + offset, value); \ + __raw_writel(value, dd->io_base + offset); \ + } while (0) +#else +static inline void omap_des_write(struct omap_des_dev *dd, u32 offset, + u32 value) +{ + __raw_writel(value, dd->io_base + offset); +} +#endif + +static inline void omap_des_write_mask(struct omap_des_dev *dd, u32 offset, + u32 value, u32 mask) +{ + u32 val; + + val = omap_des_read(dd, offset); + val &= ~mask; + val |= value; + omap_des_write(dd, offset, val); +} + +static void omap_des_write_n(struct omap_des_dev *dd, u32 offset, + u32 *value, int count) +{ + for (; count--; value++, offset += 4) + omap_des_write(dd, offset, *value); +} + +static int omap_des_hw_init(struct omap_des_dev *dd) +{ + int err; + + /* + * clocks are enabled when request starts and disabled when finished. + * It may be long delays between requests. + * Device might go to off mode to save power. + */ + err = pm_runtime_get_sync(dd->dev); + if (err < 0) { + pm_runtime_put_noidle(dd->dev); + dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err); + return err; + } + + if (!(dd->flags & FLAGS_INIT)) { + dd->flags |= FLAGS_INIT; + dd->err = 0; + } + + return 0; +} + +static int omap_des_write_ctrl(struct omap_des_dev *dd) +{ + unsigned int key32; + int i, err; + u32 val = 0, mask = 0; + + err = omap_des_hw_init(dd); + if (err) + return err; + + key32 = dd->ctx->keylen / sizeof(u32); + + /* it seems a key should always be set even if it has not changed */ + for (i = 0; i < key32; i++) { + omap_des_write(dd, DES_REG_KEY(dd, i), + __le32_to_cpu(dd->ctx->key[i])); + } + + if ((dd->flags & FLAGS_CBC) && dd->req->info) + omap_des_write_n(dd, DES_REG_IV(dd, 0), dd->req->info, 2); + + if (dd->flags & FLAGS_CBC) + val |= DES_REG_CTRL_CBC; + if (dd->flags & FLAGS_ENCRYPT) + val |= DES_REG_CTRL_DIRECTION; + if (key32 == 6) + val |= DES_REG_CTRL_TDES; + + mask |= DES_REG_CTRL_CBC | DES_REG_CTRL_DIRECTION | DES_REG_CTRL_TDES; + + omap_des_write_mask(dd, DES_REG_CTRL(dd), val, mask); + + return 0; +} + +static void omap_des_dma_trigger_omap4(struct omap_des_dev *dd, int length) +{ + u32 mask, val; + + omap_des_write(dd, DES_REG_LENGTH_N(0), length); + + val = dd->pdata->dma_start; + + if (dd->dma_lch_out != NULL) + val |= dd->pdata->dma_enable_out; + if (dd->dma_lch_in != NULL) + val |= dd->pdata->dma_enable_in; + + mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in | + dd->pdata->dma_start; + + omap_des_write_mask(dd, DES_REG_MASK(dd), val, mask); +} + +static void omap_des_dma_stop(struct omap_des_dev *dd) +{ + u32 mask; + + mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in | + dd->pdata->dma_start; + + omap_des_write_mask(dd, DES_REG_MASK(dd), 0, mask); +} + +static struct omap_des_dev *omap_des_find_dev(struct omap_des_ctx *ctx) +{ + struct omap_des_dev *dd = NULL, *tmp; + + spin_lock_bh(&list_lock); + if (!ctx->dd) { + list_for_each_entry(tmp, &dev_list, list) { + /* FIXME: take fist available des core */ + dd = tmp; + break; + } + ctx->dd = dd; + } else { + /* already found before */ + dd = ctx->dd; + } + spin_unlock_bh(&list_lock); + + return dd; +} + +static void omap_des_dma_out_callback(void *data) +{ + struct omap_des_dev *dd = data; + + /* dma_lch_out - completed */ + tasklet_schedule(&dd->done_task); +} + +static int omap_des_dma_init(struct omap_des_dev *dd) +{ + int err = -ENOMEM; + dma_cap_mask_t mask; + + dd->dma_lch_out = NULL; + dd->dma_lch_in = NULL; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + dd->dma_lch_in = dma_request_slave_channel_compat(mask, + omap_dma_filter_fn, + &dd->dma_in, + dd->dev, "rx"); + if (!dd->dma_lch_in) { + dev_err(dd->dev, "Unable to request in DMA channel\n"); + goto err_dma_in; + } + + dd->dma_lch_out = dma_request_slave_channel_compat(mask, + omap_dma_filter_fn, + &dd->dma_out, + dd->dev, "tx"); + if (!dd->dma_lch_out) { + dev_err(dd->dev, "Unable to request out DMA channel\n"); + goto err_dma_out; + } + + return 0; + +err_dma_out: + dma_release_channel(dd->dma_lch_in); +err_dma_in: + if (err) + pr_err("error: %d\n", err); + return err; +} + +static void omap_des_dma_cleanup(struct omap_des_dev *dd) +{ + dma_release_channel(dd->dma_lch_out); + dma_release_channel(dd->dma_lch_in); +} + +static void sg_copy_buf(void *buf, struct scatterlist *sg, + unsigned int start, unsigned int nbytes, int out) +{ + struct scatter_walk walk; + + if (!nbytes) + return; + + scatterwalk_start(&walk, sg); + scatterwalk_advance(&walk, start); + scatterwalk_copychunks(buf, &walk, nbytes, out); + scatterwalk_done(&walk, out, 0); +} + +static int omap_des_crypt_dma(struct crypto_tfm *tfm, + struct scatterlist *in_sg, struct scatterlist *out_sg, + int in_sg_len, int out_sg_len) +{ + struct omap_des_ctx *ctx = crypto_tfm_ctx(tfm); + struct omap_des_dev *dd = ctx->dd; + struct dma_async_tx_descriptor *tx_in, *tx_out; + struct dma_slave_config cfg; + int ret; + + if (dd->pio_only) { + scatterwalk_start(&dd->in_walk, dd->in_sg); + scatterwalk_start(&dd->out_walk, dd->out_sg); + + /* Enable DATAIN interrupt and let it take + care of the rest */ + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2); + return 0; + } + + dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE); + + memset(&cfg, 0, sizeof(cfg)); + + cfg.src_addr = dd->phys_base + DES_REG_DATA_N(dd, 0); + cfg.dst_addr = dd->phys_base + DES_REG_DATA_N(dd, 0); + cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + cfg.src_maxburst = DST_MAXBURST; + cfg.dst_maxburst = DST_MAXBURST; + + /* IN */ + ret = dmaengine_slave_config(dd->dma_lch_in, &cfg); + if (ret) { + dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", + ret); + return ret; + } + + tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len, + DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!tx_in) { + dev_err(dd->dev, "IN prep_slave_sg() failed\n"); + return -EINVAL; + } + + /* No callback necessary */ + tx_in->callback_param = dd; + + /* OUT */ + ret = dmaengine_slave_config(dd->dma_lch_out, &cfg); + if (ret) { + dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", + ret); + return ret; + } + + tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len, + DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!tx_out) { + dev_err(dd->dev, "OUT prep_slave_sg() failed\n"); + return -EINVAL; + } + + tx_out->callback = omap_des_dma_out_callback; + tx_out->callback_param = dd; + + dmaengine_submit(tx_in); + dmaengine_submit(tx_out); + + dma_async_issue_pending(dd->dma_lch_in); + dma_async_issue_pending(dd->dma_lch_out); + + /* start DMA */ + dd->pdata->trigger(dd, dd->total); + + return 0; +} + +static int omap_des_crypt_dma_start(struct omap_des_dev *dd) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm( + crypto_ablkcipher_reqtfm(dd->req)); + int err; + + pr_debug("total: %d\n", dd->total); + + if (!dd->pio_only) { + err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len, + DMA_TO_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + + err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + if (!err) { + dev_err(dd->dev, "dma_map_sg() error\n"); + return -EINVAL; + } + } + + err = omap_des_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len, + dd->out_sg_len); + if (err && !dd->pio_only) { + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + } + + return err; +} + +static void omap_des_finish_req(struct omap_des_dev *dd, int err) +{ + struct ablkcipher_request *req = dd->req; + + pr_debug("err: %d\n", err); + + pm_runtime_put(dd->dev); + dd->flags &= ~FLAGS_BUSY; + + req->base.complete(&req->base, err); +} + +static int omap_des_crypt_dma_stop(struct omap_des_dev *dd) +{ + int err = 0; + + pr_debug("total: %d\n", dd->total); + + omap_des_dma_stop(dd); + + dmaengine_terminate_all(dd->dma_lch_in); + dmaengine_terminate_all(dd->dma_lch_out); + + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE); + + return err; +} + +static int omap_des_copy_needed(struct scatterlist *sg) +{ + while (sg) { + if (!IS_ALIGNED(sg->offset, 4)) + return -1; + if (!IS_ALIGNED(sg->length, DES_BLOCK_SIZE)) + return -1; + sg = sg_next(sg); + } + return 0; +} + +static int omap_des_copy_sgs(struct omap_des_dev *dd) +{ + void *buf_in, *buf_out; + int pages; + + pages = dd->total >> PAGE_SHIFT; + + if (dd->total & (PAGE_SIZE-1)) + pages++; + + BUG_ON(!pages); + + buf_in = (void *)__get_free_pages(GFP_ATOMIC, pages); + buf_out = (void *)__get_free_pages(GFP_ATOMIC, pages); + + if (!buf_in || !buf_out) { + pr_err("Couldn't allocated pages for unaligned cases.\n"); + return -1; + } + + dd->orig_out = dd->out_sg; + + sg_copy_buf(buf_in, dd->in_sg, 0, dd->total, 0); + + sg_init_table(&dd->in_sgl, 1); + sg_set_buf(&dd->in_sgl, buf_in, dd->total); + dd->in_sg = &dd->in_sgl; + + sg_init_table(&dd->out_sgl, 1); + sg_set_buf(&dd->out_sgl, buf_out, dd->total); + dd->out_sg = &dd->out_sgl; + + return 0; +} + +static int omap_des_handle_queue(struct omap_des_dev *dd, + struct ablkcipher_request *req) +{ + struct crypto_async_request *async_req, *backlog; + struct omap_des_ctx *ctx; + struct omap_des_reqctx *rctx; + unsigned long flags; + int err, ret = 0; + + spin_lock_irqsave(&dd->lock, flags); + if (req) + ret = ablkcipher_enqueue_request(&dd->queue, req); + if (dd->flags & FLAGS_BUSY) { + spin_unlock_irqrestore(&dd->lock, flags); + return ret; + } + backlog = crypto_get_backlog(&dd->queue); + async_req = crypto_dequeue_request(&dd->queue); + if (async_req) + dd->flags |= FLAGS_BUSY; + spin_unlock_irqrestore(&dd->lock, flags); + + if (!async_req) + return ret; + + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + + req = ablkcipher_request_cast(async_req); + + /* assign new request to device */ + dd->req = req; + dd->total = req->nbytes; + dd->total_save = req->nbytes; + dd->in_sg = req->src; + dd->out_sg = req->dst; + + if (omap_des_copy_needed(dd->in_sg) || + omap_des_copy_needed(dd->out_sg)) { + if (omap_des_copy_sgs(dd)) + pr_err("Failed to copy SGs for unaligned cases\n"); + dd->sgs_copied = 1; + } else { + dd->sgs_copied = 0; + } + + dd->in_sg_len = scatterwalk_bytes_sglen(dd->in_sg, dd->total); + dd->out_sg_len = scatterwalk_bytes_sglen(dd->out_sg, dd->total); + BUG_ON(dd->in_sg_len < 0 || dd->out_sg_len < 0); + + rctx = ablkcipher_request_ctx(req); + ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); + rctx->mode &= FLAGS_MODE_MASK; + dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; + + dd->ctx = ctx; + ctx->dd = dd; + + err = omap_des_write_ctrl(dd); + if (!err) + err = omap_des_crypt_dma_start(dd); + if (err) { + /* des_task will not finish it, so do it here */ + omap_des_finish_req(dd, err); + tasklet_schedule(&dd->queue_task); + } + + return ret; /* return ret, which is enqueue return value */ +} + +static void omap_des_done_task(unsigned long data) +{ + struct omap_des_dev *dd = (struct omap_des_dev *)data; + void *buf_in, *buf_out; + int pages; + + pr_debug("enter done_task\n"); + + if (!dd->pio_only) { + dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE); + dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, + DMA_FROM_DEVICE); + omap_des_crypt_dma_stop(dd); + } + + if (dd->sgs_copied) { + buf_in = sg_virt(&dd->in_sgl); + buf_out = sg_virt(&dd->out_sgl); + + sg_copy_buf(buf_out, dd->orig_out, 0, dd->total_save, 1); + + pages = get_order(dd->total_save); + free_pages((unsigned long)buf_in, pages); + free_pages((unsigned long)buf_out, pages); + } + + omap_des_finish_req(dd, 0); + omap_des_handle_queue(dd, NULL); + + pr_debug("exit\n"); +} + +static void omap_des_queue_task(unsigned long data) +{ + struct omap_des_dev *dd = (struct omap_des_dev *)data; + + omap_des_handle_queue(dd, NULL); +} + +static int omap_des_crypt(struct ablkcipher_request *req, unsigned long mode) +{ + struct omap_des_ctx *ctx = crypto_ablkcipher_ctx( + crypto_ablkcipher_reqtfm(req)); + struct omap_des_reqctx *rctx = ablkcipher_request_ctx(req); + struct omap_des_dev *dd; + + pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes, + !!(mode & FLAGS_ENCRYPT), + !!(mode & FLAGS_CBC)); + + if (!IS_ALIGNED(req->nbytes, DES_BLOCK_SIZE)) { + pr_err("request size is not exact amount of DES blocks\n"); + return -EINVAL; + } + + dd = omap_des_find_dev(ctx); + if (!dd) + return -ENODEV; + + rctx->mode = mode; + + return omap_des_handle_queue(dd, req); +} + +/* ********************** ALG API ************************************ */ + +static int omap_des_setkey(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct omap_des_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if (keylen != DES_KEY_SIZE && keylen != (3*DES_KEY_SIZE)) + return -EINVAL; + + pr_debug("enter, keylen: %d\n", keylen); + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int omap_des_ecb_encrypt(struct ablkcipher_request *req) +{ + return omap_des_crypt(req, FLAGS_ENCRYPT); +} + +static int omap_des_ecb_decrypt(struct ablkcipher_request *req) +{ + return omap_des_crypt(req, 0); +} + +static int omap_des_cbc_encrypt(struct ablkcipher_request *req) +{ + return omap_des_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); +} + +static int omap_des_cbc_decrypt(struct ablkcipher_request *req) +{ + return omap_des_crypt(req, FLAGS_CBC); +} + +static int omap_des_cra_init(struct crypto_tfm *tfm) +{ + pr_debug("enter\n"); + + tfm->crt_ablkcipher.reqsize = sizeof(struct omap_des_reqctx); + + return 0; +} + +static void omap_des_cra_exit(struct crypto_tfm *tfm) +{ + pr_debug("enter\n"); +} + +/* ********************** ALGS ************************************ */ + +static struct crypto_alg algs_ecb_cbc[] = { +{ + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-omap", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_des_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = omap_des_cra_init, + .cra_exit = omap_des_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = omap_des_setkey, + .encrypt = omap_des_ecb_encrypt, + .decrypt = omap_des_ecb_decrypt, + } +}, +{ + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-omap", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_des_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = omap_des_cra_init, + .cra_exit = omap_des_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = omap_des_setkey, + .encrypt = omap_des_cbc_encrypt, + .decrypt = omap_des_cbc_decrypt, + } +}, +{ + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3-omap", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_des_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = omap_des_cra_init, + .cra_exit = omap_des_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = 3*DES_KEY_SIZE, + .max_keysize = 3*DES_KEY_SIZE, + .setkey = omap_des_setkey, + .encrypt = omap_des_ecb_encrypt, + .decrypt = omap_des_ecb_decrypt, + } +}, +{ + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-des3-omap", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct omap_des_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = omap_des_cra_init, + .cra_exit = omap_des_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = 3*DES_KEY_SIZE, + .max_keysize = 3*DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = omap_des_setkey, + .encrypt = omap_des_cbc_encrypt, + .decrypt = omap_des_cbc_decrypt, + } +} +}; + +static struct omap_des_algs_info omap_des_algs_info_ecb_cbc[] = { + { + .algs_list = algs_ecb_cbc, + .size = ARRAY_SIZE(algs_ecb_cbc), + }, +}; + +#ifdef CONFIG_OF +static const struct omap_des_pdata omap_des_pdata_omap4 = { + .algs_info = omap_des_algs_info_ecb_cbc, + .algs_info_size = ARRAY_SIZE(omap_des_algs_info_ecb_cbc), + .trigger = omap_des_dma_trigger_omap4, + .key_ofs = 0x14, + .iv_ofs = 0x18, + .ctrl_ofs = 0x20, + .data_ofs = 0x28, + .rev_ofs = 0x30, + .mask_ofs = 0x34, + .irq_status_ofs = 0x3c, + .irq_enable_ofs = 0x40, + .dma_enable_in = BIT(5), + .dma_enable_out = BIT(6), + .major_mask = 0x0700, + .major_shift = 8, + .minor_mask = 0x003f, + .minor_shift = 0, +}; + +static irqreturn_t omap_des_irq(int irq, void *dev_id) +{ + struct omap_des_dev *dd = dev_id; + u32 status, i; + u32 *src, *dst; + + status = omap_des_read(dd, DES_REG_IRQ_STATUS(dd)); + if (status & DES_REG_IRQ_DATA_IN) { + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0); + + BUG_ON(!dd->in_sg); + + BUG_ON(_calc_walked(in) > dd->in_sg->length); + + src = sg_virt(dd->in_sg) + _calc_walked(in); + + for (i = 0; i < DES_BLOCK_WORDS; i++) { + omap_des_write(dd, DES_REG_DATA_N(dd, i), *src); + + scatterwalk_advance(&dd->in_walk, 4); + if (dd->in_sg->length == _calc_walked(in)) { + dd->in_sg = scatterwalk_sg_next(dd->in_sg); + if (dd->in_sg) { + scatterwalk_start(&dd->in_walk, + dd->in_sg); + src = sg_virt(dd->in_sg) + + _calc_walked(in); + } + } else { + src++; + } + } + + /* Clear IRQ status */ + status &= ~DES_REG_IRQ_DATA_IN; + omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status); + + /* Enable DATA_OUT interrupt */ + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x4); + + } else if (status & DES_REG_IRQ_DATA_OUT) { + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0); + + BUG_ON(!dd->out_sg); + + BUG_ON(_calc_walked(out) > dd->out_sg->length); + + dst = sg_virt(dd->out_sg) + _calc_walked(out); + + for (i = 0; i < DES_BLOCK_WORDS; i++) { + *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i)); + scatterwalk_advance(&dd->out_walk, 4); + if (dd->out_sg->length == _calc_walked(out)) { + dd->out_sg = scatterwalk_sg_next(dd->out_sg); + if (dd->out_sg) { + scatterwalk_start(&dd->out_walk, + dd->out_sg); + dst = sg_virt(dd->out_sg) + + _calc_walked(out); + } + } else { + dst++; + } + } + + dd->total -= DES_BLOCK_SIZE; + + BUG_ON(dd->total < 0); + + /* Clear IRQ status */ + status &= ~DES_REG_IRQ_DATA_OUT; + omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status); + + if (!dd->total) + /* All bytes read! */ + tasklet_schedule(&dd->done_task); + else + /* Enable DATA_IN interrupt for next block */ + omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2); + } + + return IRQ_HANDLED; +} + +static const struct of_device_id omap_des_of_match[] = { + { + .compatible = "ti,omap4-des", + .data = &omap_des_pdata_omap4, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, omap_des_of_match); + +static int omap_des_get_of(struct omap_des_dev *dd, + struct platform_device *pdev) +{ + const struct of_device_id *match; + + match = of_match_device(of_match_ptr(omap_des_of_match), &pdev->dev); + if (!match) { + dev_err(&pdev->dev, "no compatible OF match\n"); + return -EINVAL; + } + + dd->dma_out = -1; /* Dummy value that's unused */ + dd->dma_in = -1; /* Dummy value that's unused */ + dd->pdata = match->data; + + return 0; +} +#else +static int omap_des_get_of(struct omap_des_dev *dd, + struct device *dev) +{ + return -EINVAL; +} +#endif + +static int omap_des_get_pdev(struct omap_des_dev *dd, + struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct resource *r; + int err = 0; + + /* Get the DMA out channel */ + r = platform_get_resource(pdev, IORESOURCE_DMA, 0); + if (!r) { + dev_err(dev, "no DMA out resource info\n"); + err = -ENODEV; + goto err; + } + dd->dma_out = r->start; + + /* Get the DMA in channel */ + r = platform_get_resource(pdev, IORESOURCE_DMA, 1); + if (!r) { + dev_err(dev, "no DMA in resource info\n"); + err = -ENODEV; + goto err; + } + dd->dma_in = r->start; + + /* non-DT devices get pdata from pdev */ + dd->pdata = pdev->dev.platform_data; + +err: + return err; +} + +static int omap_des_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct omap_des_dev *dd; + struct crypto_alg *algp; + struct resource *res; + int err = -ENOMEM, i, j, irq = -1; + u32 reg; + + dd = devm_kzalloc(dev, sizeof(struct omap_des_dev), GFP_KERNEL); + if (dd == NULL) { + dev_err(dev, "unable to alloc data struct.\n"); + goto err_data; + } + dd->dev = dev; + platform_set_drvdata(pdev, dd); + + spin_lock_init(&dd->lock); + crypto_init_queue(&dd->queue, OMAP_DES_QUEUE_LENGTH); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(dev, "no MEM resource info\n"); + goto err_res; + } + + err = (dev->of_node) ? omap_des_get_of(dd, pdev) : + omap_des_get_pdev(dd, pdev); + if (err) + goto err_res; + + dd->io_base = devm_ioremap_resource(dev, res); + if (IS_ERR(dd->io_base)) { + err = PTR_ERR(dd->io_base); + goto err_res; + } + dd->phys_base = res->start; + + pm_runtime_enable(dev); + err = pm_runtime_get_sync(dev); + if (err < 0) { + pm_runtime_put_noidle(dev); + dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err); + goto err_get; + } + + omap_des_dma_stop(dd); + + reg = omap_des_read(dd, DES_REG_REV(dd)); + + pm_runtime_put_sync(dev); + + dev_info(dev, "OMAP DES hw accel rev: %u.%u\n", + (reg & dd->pdata->major_mask) >> dd->pdata->major_shift, + (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift); + + tasklet_init(&dd->done_task, omap_des_done_task, (unsigned long)dd); + tasklet_init(&dd->queue_task, omap_des_queue_task, (unsigned long)dd); + + err = omap_des_dma_init(dd); + if (err && DES_REG_IRQ_STATUS(dd) && DES_REG_IRQ_ENABLE(dd)) { + dd->pio_only = 1; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(dev, "can't get IRQ resource\n"); + goto err_irq; + } + + err = devm_request_irq(dev, irq, omap_des_irq, 0, + dev_name(dev), dd); + if (err) { + dev_err(dev, "Unable to grab omap-des IRQ\n"); + goto err_irq; + } + } + + + INIT_LIST_HEAD(&dd->list); + spin_lock(&list_lock); + list_add_tail(&dd->list, &dev_list); + spin_unlock(&list_lock); + + for (i = 0; i < dd->pdata->algs_info_size; i++) { + for (j = 0; j < dd->pdata->algs_info[i].size; j++) { + algp = &dd->pdata->algs_info[i].algs_list[j]; + + pr_debug("reg alg: %s\n", algp->cra_name); + INIT_LIST_HEAD(&algp->cra_list); + + err = crypto_register_alg(algp); + if (err) + goto err_algs; + + dd->pdata->algs_info[i].registered++; + } + } + + return 0; +err_algs: + for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) + for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) + crypto_unregister_alg( + &dd->pdata->algs_info[i].algs_list[j]); + if (!dd->pio_only) + omap_des_dma_cleanup(dd); +err_irq: + tasklet_kill(&dd->done_task); + tasklet_kill(&dd->queue_task); +err_get: + pm_runtime_disable(dev); +err_res: + dd = NULL; +err_data: + dev_err(dev, "initialization failed.\n"); + return err; +} + +static int omap_des_remove(struct platform_device *pdev) +{ + struct omap_des_dev *dd = platform_get_drvdata(pdev); + int i, j; + + if (!dd) + return -ENODEV; + + spin_lock(&list_lock); + list_del(&dd->list); + spin_unlock(&list_lock); + + for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) + for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) + crypto_unregister_alg( + &dd->pdata->algs_info[i].algs_list[j]); + + tasklet_kill(&dd->done_task); + tasklet_kill(&dd->queue_task); + omap_des_dma_cleanup(dd); + pm_runtime_disable(dd->dev); + dd = NULL; + + return 0; +} + +#ifdef CONFIG_PM_SLEEP +static int omap_des_suspend(struct device *dev) +{ + pm_runtime_put_sync(dev); + return 0; +} + +static int omap_des_resume(struct device *dev) +{ + int err; + + err = pm_runtime_get_sync(dev); + if (err < 0) { + pm_runtime_put_noidle(dev); + dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err); + return err; + } + return 0; +} +#endif + +static SIMPLE_DEV_PM_OPS(omap_des_pm_ops, omap_des_suspend, omap_des_resume); + +static struct platform_driver omap_des_driver = { + .probe = omap_des_probe, + .remove = omap_des_remove, + .driver = { + .name = "omap-des", + .owner = THIS_MODULE, + .pm = &omap_des_pm_ops, + .of_match_table = of_match_ptr(omap_des_of_match), + }, +}; + +module_platform_driver(omap_des_driver); + +MODULE_DESCRIPTION("OMAP DES hw acceleration support."); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Joel Fernandes <joelf@ti.com>"); diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index e45aaaf0db3..710d8638696 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -636,11 +636,17 @@ static size_t omap_sham_append_buffer(struct omap_sham_reqctx *ctx, static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx) { size_t count; + const u8 *vaddr; while (ctx->sg) { + vaddr = kmap_atomic(sg_page(ctx->sg)); + count = omap_sham_append_buffer(ctx, - sg_virt(ctx->sg) + ctx->offset, + vaddr + ctx->offset, ctx->sg->length - ctx->offset); + + kunmap_atomic((void *)vaddr); + if (!count) break; ctx->offset += count; @@ -789,10 +795,13 @@ static int omap_sham_update_cpu(struct omap_sham_dev *dd) dev_dbg(dd->dev, "cpu: bufcnt: %u, digcnt: %d, final: %d\n", ctx->bufcnt, ctx->digcnt, final); - bufcnt = ctx->bufcnt; - ctx->bufcnt = 0; + if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) { + bufcnt = ctx->bufcnt; + ctx->bufcnt = 0; + return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, final); + } - return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, final); + return 0; } static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) @@ -1103,6 +1112,9 @@ static int omap_sham_update(struct ahash_request *req) return 0; } + if (dd->polling_mode) + ctx->flags |= BIT(FLAGS_CPU); + return omap_sham_enqueue(req, OP_UPDATE); } @@ -1970,7 +1982,8 @@ err_algs: crypto_unregister_ahash( &dd->pdata->algs_info[i].algs_list[j]); pm_runtime_disable(dev); - dma_release_channel(dd->dma_lch); + if (dd->dma_lch) + dma_release_channel(dd->dma_lch); data_err: dev_err(dev, "initialization failed.\n"); @@ -1994,7 +2007,9 @@ static int omap_sham_remove(struct platform_device *pdev) &dd->pdata->algs_info[i].algs_list[j]); tasklet_kill(&dd->done_task); pm_runtime_disable(&pdev->dev); - dma_release_channel(dd->dma_lch); + + if (dd->dma_lch) + dma_release_channel(dd->dma_lch); return 0; } @@ -2013,9 +2028,7 @@ static int omap_sham_resume(struct device *dev) } #endif -static const struct dev_pm_ops omap_sham_pm_ops = { - SET_SYSTEM_SLEEP_PM_OPS(omap_sham_suspend, omap_sham_resume) -}; +static SIMPLE_DEV_PM_OPS(omap_sham_pm_ops, omap_sham_suspend, omap_sham_resume); static struct platform_driver omap_sham_driver = { .probe = omap_sham_probe, diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index 9266c0e2549..bace885634f 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -211,7 +211,7 @@ static int padlock_sha256_final(struct shash_desc *desc, u8 *out) static int padlock_cra_init(struct crypto_tfm *tfm) { struct crypto_shash *hash = __crypto_shash_cast(tfm); - const char *fallback_driver_name = tfm->__crt_alg->cra_name; + const char *fallback_driver_name = crypto_tfm_alg_name(tfm); struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm); struct crypto_shash *fallback_tfm; int err = -ENOMEM; diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index a6175ba6d23..5da5b98b8f2 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -1720,22 +1720,16 @@ static int spacc_probe(struct platform_device *pdev) engine->name = dev_name(&pdev->dev); mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + engine->regs = devm_ioremap_resource(&pdev->dev, mem); + if (IS_ERR(engine->regs)) + return PTR_ERR(engine->regs); + irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!mem || !irq) { + if (!irq) { dev_err(&pdev->dev, "no memory/irq resource for engine\n"); return -ENXIO; } - if (!devm_request_mem_region(&pdev->dev, mem->start, resource_size(mem), - engine->name)) - return -ENOMEM; - - engine->regs = devm_ioremap(&pdev->dev, mem->start, resource_size(mem)); - if (!engine->regs) { - dev_err(&pdev->dev, "memory map failed\n"); - return -ENOMEM; - } - if (devm_request_irq(&pdev->dev, irq->start, spacc_spacc_irq, 0, engine->name, engine)) { dev_err(engine->dev, "failed to request IRQ\n"); diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index cf149b19ff4..4197ad9a711 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -22,6 +22,7 @@ #include <linux/scatterlist.h> #include <linux/dma-mapping.h> #include <linux/io.h> +#include <linux/of.h> #include <linux/crypto.h> #include <linux/interrupt.h> @@ -29,9 +30,6 @@ #include <crypto/aes.h> #include <crypto/ctr.h> -#include <plat/cpu.h> -#include <mach/dma.h> - #define _SBF(s, v) ((v) << (s)) #define _BIT(b) _SBF(b, 1) @@ -105,7 +103,7 @@ #define SSS_REG_FCPKDMAO 0x005C /* AES registers */ -#define SSS_REG_AES_CONTROL 0x4000 +#define SSS_REG_AES_CONTROL 0x00 #define SSS_AES_BYTESWAP_DI _BIT(11) #define SSS_AES_BYTESWAP_DO _BIT(10) #define SSS_AES_BYTESWAP_IV _BIT(9) @@ -121,21 +119,25 @@ #define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02) #define SSS_AES_MODE_DECRYPT _BIT(0) -#define SSS_REG_AES_STATUS 0x4004 +#define SSS_REG_AES_STATUS 0x04 #define SSS_AES_BUSY _BIT(2) #define SSS_AES_INPUT_READY _BIT(1) #define SSS_AES_OUTPUT_READY _BIT(0) -#define SSS_REG_AES_IN_DATA(s) (0x4010 + (s << 2)) -#define SSS_REG_AES_OUT_DATA(s) (0x4020 + (s << 2)) -#define SSS_REG_AES_IV_DATA(s) (0x4030 + (s << 2)) -#define SSS_REG_AES_CNT_DATA(s) (0x4040 + (s << 2)) -#define SSS_REG_AES_KEY_DATA(s) (0x4080 + (s << 2)) +#define SSS_REG_AES_IN_DATA(s) (0x10 + (s << 2)) +#define SSS_REG_AES_OUT_DATA(s) (0x20 + (s << 2)) +#define SSS_REG_AES_IV_DATA(s) (0x30 + (s << 2)) +#define SSS_REG_AES_CNT_DATA(s) (0x40 + (s << 2)) +#define SSS_REG_AES_KEY_DATA(s) (0x80 + (s << 2)) #define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg)) #define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg)) #define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg)) +#define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg) +#define SSS_AES_WRITE(dev, reg, val) __raw_writel((val), \ + SSS_AES_REG(dev, reg)) + /* HW engine modes */ #define FLAGS_AES_DECRYPT _BIT(0) #define FLAGS_AES_MODE_MASK _SBF(1, 0x03) @@ -145,6 +147,20 @@ #define AES_KEY_LEN 16 #define CRYPTO_QUEUE_LEN 1 +/** + * struct samsung_aes_variant - platform specific SSS driver data + * @has_hash_irq: true if SSS module uses hash interrupt, false otherwise + * @aes_offset: AES register offset from SSS module's base. + * + * Specifies platform specific configuration of SSS module. + * Note: A structure for driver specific platform data is used for future + * expansion of its usage. + */ +struct samsung_aes_variant { + bool has_hash_irq; + unsigned int aes_offset; +}; + struct s5p_aes_reqctx { unsigned long mode; }; @@ -161,6 +177,7 @@ struct s5p_aes_dev { struct device *dev; struct clk *clk; void __iomem *ioaddr; + void __iomem *aes_ioaddr; int irq_hash; int irq_fc; @@ -173,10 +190,48 @@ struct s5p_aes_dev { struct crypto_queue queue; bool busy; spinlock_t lock; + + struct samsung_aes_variant *variant; }; static struct s5p_aes_dev *s5p_dev; +static const struct samsung_aes_variant s5p_aes_data = { + .has_hash_irq = true, + .aes_offset = 0x4000, +}; + +static const struct samsung_aes_variant exynos_aes_data = { + .has_hash_irq = false, + .aes_offset = 0x200, +}; + +static const struct of_device_id s5p_sss_dt_match[] = { + { + .compatible = "samsung,s5pv210-secss", + .data = &s5p_aes_data, + }, + { + .compatible = "samsung,exynos4210-secss", + .data = &exynos_aes_data, + }, + { }, +}; +MODULE_DEVICE_TABLE(of, s5p_sss_dt_match); + +static inline struct samsung_aes_variant *find_s5p_sss_version + (struct platform_device *pdev) +{ + if (IS_ENABLED(CONFIG_OF) && (pdev->dev.of_node)) { + const struct of_device_id *match; + match = of_match_node(s5p_sss_dt_match, + pdev->dev.of_node); + return (struct samsung_aes_variant *)match->data; + } + return (struct samsung_aes_variant *) + platform_get_device_id(pdev)->driver_data; +} + static void s5p_set_dma_indata(struct s5p_aes_dev *dev, struct scatterlist *sg) { SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg)); @@ -272,8 +327,12 @@ static void s5p_aes_tx(struct s5p_aes_dev *dev) } s5p_set_dma_outdata(dev, dev->sg_dst); - } else + } else { s5p_aes_complete(dev, err); + + dev->busy = true; + tasklet_schedule(&dev->tasklet); + } } static void s5p_aes_rx(struct s5p_aes_dev *dev) @@ -322,14 +381,15 @@ static void s5p_set_aes(struct s5p_aes_dev *dev, { void __iomem *keystart; - memcpy(dev->ioaddr + SSS_REG_AES_IV_DATA(0), iv, 0x10); + if (iv) + memcpy(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv, 0x10); if (keylen == AES_KEYSIZE_256) - keystart = dev->ioaddr + SSS_REG_AES_KEY_DATA(0); + keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0); else if (keylen == AES_KEYSIZE_192) - keystart = dev->ioaddr + SSS_REG_AES_KEY_DATA(2); + keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(2); else - keystart = dev->ioaddr + SSS_REG_AES_KEY_DATA(4); + keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(4); memcpy(keystart, key, keylen); } @@ -379,7 +439,7 @@ static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode) if (err) goto outdata_error; - SSS_WRITE(dev, AES_CONTROL, aes_control); + SSS_AES_WRITE(dev, AES_CONTROL, aes_control); s5p_set_aes(dev, dev->ctx->aes_key, req->info, dev->ctx->keylen); s5p_set_dma_indata(dev, req->src); @@ -410,10 +470,13 @@ static void s5p_tasklet_cb(unsigned long data) spin_lock_irqsave(&dev->lock, flags); backlog = crypto_get_backlog(&dev->queue); async_req = crypto_dequeue_request(&dev->queue); - spin_unlock_irqrestore(&dev->lock, flags); - if (!async_req) + if (!async_req) { + dev->busy = false; + spin_unlock_irqrestore(&dev->lock, flags); return; + } + spin_unlock_irqrestore(&dev->lock, flags); if (backlog) backlog->complete(backlog, -EINPROGRESS); @@ -432,14 +495,13 @@ static int s5p_aes_handle_req(struct s5p_aes_dev *dev, int err; spin_lock_irqsave(&dev->lock, flags); + err = ablkcipher_enqueue_request(&dev->queue, req); if (dev->busy) { - err = -EAGAIN; spin_unlock_irqrestore(&dev->lock, flags); goto exit; } dev->busy = true; - err = ablkcipher_enqueue_request(&dev->queue, req); spin_unlock_irqrestore(&dev->lock, flags); tasklet_schedule(&dev->tasklet); @@ -564,21 +626,21 @@ static int s5p_aes_probe(struct platform_device *pdev) struct s5p_aes_dev *pdata; struct device *dev = &pdev->dev; struct resource *res; + struct samsung_aes_variant *variant; if (s5p_dev) return -EEXIST; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL); if (!pdata) return -ENOMEM; - if (!devm_request_mem_region(dev, res->start, - resource_size(res), pdev->name)) - return -EBUSY; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(pdata->ioaddr)) + return PTR_ERR(pdata->ioaddr); + + variant = find_s5p_sss_version(pdev); pdata->clk = devm_clk_get(dev, "secss"); if (IS_ERR(pdata->clk)) { @@ -586,26 +648,17 @@ static int s5p_aes_probe(struct platform_device *pdev) return -ENOENT; } - clk_enable(pdata->clk); + err = clk_prepare_enable(pdata->clk); + if (err < 0) { + dev_err(dev, "Enabling SSS clk failed, err %d\n", err); + return err; + } spin_lock_init(&pdata->lock); - pdata->ioaddr = devm_ioremap(dev, res->start, - resource_size(res)); - pdata->irq_hash = platform_get_irq_byname(pdev, "hash"); - if (pdata->irq_hash < 0) { - err = pdata->irq_hash; - dev_warn(dev, "hash interrupt is not available.\n"); - goto err_irq; - } - err = devm_request_irq(dev, pdata->irq_hash, s5p_aes_interrupt, - IRQF_SHARED, pdev->name, pdev); - if (err < 0) { - dev_warn(dev, "hash interrupt is not available.\n"); - goto err_irq; - } + pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset; - pdata->irq_fc = platform_get_irq_byname(pdev, "feed control"); + pdata->irq_fc = platform_get_irq(pdev, 0); if (pdata->irq_fc < 0) { err = pdata->irq_fc; dev_warn(dev, "feed control interrupt is not available.\n"); @@ -618,6 +671,23 @@ static int s5p_aes_probe(struct platform_device *pdev) goto err_irq; } + if (variant->has_hash_irq) { + pdata->irq_hash = platform_get_irq(pdev, 1); + if (pdata->irq_hash < 0) { + err = pdata->irq_hash; + dev_warn(dev, "hash interrupt is not available.\n"); + goto err_irq; + } + err = devm_request_irq(dev, pdata->irq_hash, s5p_aes_interrupt, + IRQF_SHARED, pdev->name, pdev); + if (err < 0) { + dev_warn(dev, "hash interrupt is not available.\n"); + goto err_irq; + } + } + + pdata->busy = false; + pdata->variant = variant; pdata->dev = dev; platform_set_drvdata(pdev, pdata); s5p_dev = pdata; @@ -644,7 +714,7 @@ static int s5p_aes_probe(struct platform_device *pdev) tasklet_kill(&pdata->tasklet); err_irq: - clk_disable(pdata->clk); + clk_disable_unprepare(pdata->clk); s5p_dev = NULL; @@ -664,7 +734,7 @@ static int s5p_aes_remove(struct platform_device *pdev) tasklet_kill(&pdata->tasklet); - clk_disable(pdata->clk); + clk_disable_unprepare(pdata->clk); s5p_dev = NULL; @@ -677,6 +747,7 @@ static struct platform_driver s5p_aes_crypto = { .driver = { .owner = THIS_MODULE, .name = "s5p-secss", + .of_match_table = s5p_sss_dt_match, }, }; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 785a9ded7bd..164e1ec624e 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -728,7 +728,7 @@ static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req) static int sahara_aes_cra_init(struct crypto_tfm *tfm) { - const char *name = tfm->__crt_alg->cra_name; + const char *name = crypto_tfm_alg_name(tfm); struct sahara_ctx *ctx = crypto_tfm_ctx(tfm); ctx->fallback = crypto_alloc_ablkcipher(name, 0, @@ -885,22 +885,9 @@ static int sahara_probe(struct platform_device *pdev) /* Get the base address */ res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(&pdev->dev, "failed to get memory region resource\n"); - return -ENODEV; - } - - if (devm_request_mem_region(&pdev->dev, res->start, - resource_size(res), SAHARA_NAME) == NULL) { - dev_err(&pdev->dev, "failed to request memory region\n"); - return -ENOENT; - } - dev->regs_base = devm_ioremap(&pdev->dev, res->start, - resource_size(res)); - if (!dev->regs_base) { - dev_err(&pdev->dev, "failed to ioremap address region\n"); - return -ENOENT; - } + dev->regs_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(dev->regs_base)) + return PTR_ERR(dev->regs_base); /* Get the IRQ */ irq = platform_get_irq(pdev, 0); @@ -909,10 +896,11 @@ static int sahara_probe(struct platform_device *pdev) return irq; } - if (devm_request_irq(&pdev->dev, irq, sahara_irq_handler, - 0, SAHARA_NAME, dev) < 0) { + err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler, + 0, dev_name(&pdev->dev), dev); + if (err) { dev_err(&pdev->dev, "failed to request irq\n"); - return -ENOENT; + return err; } /* clocks */ diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index b44f4ddc565..624b8be0c36 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -338,20 +338,29 @@ DEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE) static u32 current_desc_hdr(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); - int tail = priv->chan[ch].tail; + int tail, iter; dma_addr_t cur_desc; - cur_desc = in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); + cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; + cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); - while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) { - tail = (tail + 1) & (priv->fifo_len - 1); - if (tail == priv->chan[ch].tail) { + if (!cur_desc) { + dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); + return 0; + } + + tail = priv->chan[ch].tail; + + iter = tail; + while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) { + iter = (iter + 1) & (priv->fifo_len - 1); + if (iter == tail) { dev_err(dev, "couldn't locate current descriptor\n"); return 0; } } - return priv->chan[ch].fifo[tail].desc->hdr; + return priv->chan[ch].fifo[iter].desc->hdr; } /* @@ -2486,8 +2495,6 @@ static int talitos_remove(struct platform_device *ofdev) iounmap(priv->reg); - dev_set_drvdata(dev, NULL); - kfree(priv); return 0; @@ -2630,6 +2637,8 @@ static int talitos_probe(struct platform_device *ofdev) if (!priv) return -ENOMEM; + INIT_LIST_HEAD(&priv->alg_list); + dev_set_drvdata(dev, priv); priv->ofdev = ofdev; @@ -2650,8 +2659,6 @@ static int talitos_probe(struct platform_device *ofdev) (unsigned long)dev); } - INIT_LIST_HEAD(&priv->alg_list); - priv->reg = of_iomap(np, 0); if (!priv->reg) { dev_err(dev, "failed to of_iomap\n"); diff --git a/drivers/crypto/tegra-aes.c b/drivers/crypto/tegra-aes.c deleted file mode 100644 index 060eecc5dbc..00000000000 --- a/drivers/crypto/tegra-aes.c +++ /dev/null @@ -1,1087 +0,0 @@ -/* - * drivers/crypto/tegra-aes.c - * - * Driver for NVIDIA Tegra AES hardware engine residing inside the - * Bit Stream Engine for Video (BSEV) hardware block. - * - * The programming sequence for this engine is with the help - * of commands which travel via a command queue residing between the - * CPU and the BSEV block. The BSEV engine has an internal RAM (VRAM) - * where the final input plaintext, keys and the IV have to be copied - * before starting the encrypt/decrypt operation. - * - * Copyright (c) 2010, NVIDIA Corporation. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/module.h> -#include <linux/init.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/clk.h> -#include <linux/platform_device.h> -#include <linux/scatterlist.h> -#include <linux/dma-mapping.h> -#include <linux/io.h> -#include <linux/mutex.h> -#include <linux/interrupt.h> -#include <linux/completion.h> -#include <linux/workqueue.h> - -#include <crypto/scatterwalk.h> -#include <crypto/aes.h> -#include <crypto/internal/rng.h> - -#include "tegra-aes.h" - -#define FLAGS_MODE_MASK 0x00FF -#define FLAGS_ENCRYPT BIT(0) -#define FLAGS_CBC BIT(1) -#define FLAGS_GIV BIT(2) -#define FLAGS_RNG BIT(3) -#define FLAGS_OFB BIT(4) -#define FLAGS_NEW_KEY BIT(5) -#define FLAGS_NEW_IV BIT(6) -#define FLAGS_INIT BIT(7) -#define FLAGS_FAST BIT(8) -#define FLAGS_BUSY 9 - -/* - * Defines AES engine Max process bytes size in one go, which takes 1 msec. - * AES engine spends about 176 cycles/16-bytes or 11 cycles/byte - * The duration CPU can use the BSE to 1 msec, then the number of available - * cycles of AVP/BSE is 216K. In this duration, AES can process 216/11 ~= 19KB - * Based on this AES_HW_DMA_BUFFER_SIZE_BYTES is configured to 16KB. - */ -#define AES_HW_DMA_BUFFER_SIZE_BYTES 0x4000 - -/* - * The key table length is 64 bytes - * (This includes first upto 32 bytes key + 16 bytes original initial vector - * and 16 bytes updated initial vector) - */ -#define AES_HW_KEY_TABLE_LENGTH_BYTES 64 - -/* - * The memory being used is divides as follows: - * 1. Key - 32 bytes - * 2. Original IV - 16 bytes - * 3. Updated IV - 16 bytes - * 4. Key schedule - 256 bytes - * - * 1+2+3 constitute the hw key table. - */ -#define AES_HW_IV_SIZE 16 -#define AES_HW_KEYSCHEDULE_LEN 256 -#define AES_IVKEY_SIZE (AES_HW_KEY_TABLE_LENGTH_BYTES + AES_HW_KEYSCHEDULE_LEN) - -/* Define commands required for AES operation */ -enum { - CMD_BLKSTARTENGINE = 0x0E, - CMD_DMASETUP = 0x10, - CMD_DMACOMPLETE = 0x11, - CMD_SETTABLE = 0x15, - CMD_MEMDMAVD = 0x22, -}; - -/* Define sub-commands */ -enum { - SUBCMD_VRAM_SEL = 0x1, - SUBCMD_CRYPTO_TABLE_SEL = 0x3, - SUBCMD_KEY_TABLE_SEL = 0x8, -}; - -/* memdma_vd command */ -#define MEMDMA_DIR_DTOVRAM 0 /* sdram -> vram */ -#define MEMDMA_DIR_VTODRAM 1 /* vram -> sdram */ -#define MEMDMA_DIR_SHIFT 25 -#define MEMDMA_NUM_WORDS_SHIFT 12 - -/* command queue bit shifts */ -enum { - CMDQ_KEYTABLEADDR_SHIFT = 0, - CMDQ_KEYTABLEID_SHIFT = 17, - CMDQ_VRAMSEL_SHIFT = 23, - CMDQ_TABLESEL_SHIFT = 24, - CMDQ_OPCODE_SHIFT = 26, -}; - -/* - * The secure key slot contains a unique secure key generated - * and loaded by the bootloader. This slot is marked as non-accessible - * to the kernel. - */ -#define SSK_SLOT_NUM 4 - -#define AES_NR_KEYSLOTS 8 -#define TEGRA_AES_QUEUE_LENGTH 50 -#define DEFAULT_RNG_BLK_SZ 16 - -/* The command queue depth */ -#define AES_HW_MAX_ICQ_LENGTH 5 - -struct tegra_aes_slot { - struct list_head node; - int slot_num; -}; - -static struct tegra_aes_slot ssk = { - .slot_num = SSK_SLOT_NUM, -}; - -struct tegra_aes_reqctx { - unsigned long mode; -}; - -struct tegra_aes_dev { - struct device *dev; - void __iomem *io_base; - dma_addr_t ivkey_phys_base; - void __iomem *ivkey_base; - struct clk *aes_clk; - struct tegra_aes_ctx *ctx; - int irq; - unsigned long flags; - struct completion op_complete; - u32 *buf_in; - dma_addr_t dma_buf_in; - u32 *buf_out; - dma_addr_t dma_buf_out; - u8 *iv; - u8 dt[DEFAULT_RNG_BLK_SZ]; - int ivlen; - u64 ctr; - spinlock_t lock; - struct crypto_queue queue; - struct tegra_aes_slot *slots; - struct ablkcipher_request *req; - size_t total; - struct scatterlist *in_sg; - size_t in_offset; - struct scatterlist *out_sg; - size_t out_offset; -}; - -static struct tegra_aes_dev *aes_dev; - -struct tegra_aes_ctx { - struct tegra_aes_dev *dd; - unsigned long flags; - struct tegra_aes_slot *slot; - u8 key[AES_MAX_KEY_SIZE]; - size_t keylen; -}; - -static struct tegra_aes_ctx rng_ctx = { - .flags = FLAGS_NEW_KEY, - .keylen = AES_KEYSIZE_128, -}; - -/* keep registered devices data here */ -static struct list_head dev_list; -static DEFINE_SPINLOCK(list_lock); -static DEFINE_MUTEX(aes_lock); - -static void aes_workqueue_handler(struct work_struct *work); -static DECLARE_WORK(aes_work, aes_workqueue_handler); -static struct workqueue_struct *aes_wq; - -static inline u32 aes_readl(struct tegra_aes_dev *dd, u32 offset) -{ - return readl(dd->io_base + offset); -} - -static inline void aes_writel(struct tegra_aes_dev *dd, u32 val, u32 offset) -{ - writel(val, dd->io_base + offset); -} - -static int aes_start_crypt(struct tegra_aes_dev *dd, u32 in_addr, u32 out_addr, - int nblocks, int mode, bool upd_iv) -{ - u32 cmdq[AES_HW_MAX_ICQ_LENGTH]; - int i, eng_busy, icq_empty, ret; - u32 value; - - /* reset all the interrupt bits */ - aes_writel(dd, 0xFFFFFFFF, TEGRA_AES_INTR_STATUS); - - /* enable error, dma xfer complete interrupts */ - aes_writel(dd, 0x33, TEGRA_AES_INT_ENB); - - cmdq[0] = CMD_DMASETUP << CMDQ_OPCODE_SHIFT; - cmdq[1] = in_addr; - cmdq[2] = CMD_BLKSTARTENGINE << CMDQ_OPCODE_SHIFT | (nblocks-1); - cmdq[3] = CMD_DMACOMPLETE << CMDQ_OPCODE_SHIFT; - - value = aes_readl(dd, TEGRA_AES_CMDQUE_CONTROL); - /* access SDRAM through AHB */ - value &= ~TEGRA_AES_CMDQ_CTRL_SRC_STM_SEL_FIELD; - value &= ~TEGRA_AES_CMDQ_CTRL_DST_STM_SEL_FIELD; - value |= TEGRA_AES_CMDQ_CTRL_SRC_STM_SEL_FIELD | - TEGRA_AES_CMDQ_CTRL_DST_STM_SEL_FIELD | - TEGRA_AES_CMDQ_CTRL_ICMDQEN_FIELD; - aes_writel(dd, value, TEGRA_AES_CMDQUE_CONTROL); - dev_dbg(dd->dev, "cmd_q_ctrl=0x%x", value); - - value = (0x1 << TEGRA_AES_SECURE_INPUT_ALG_SEL_SHIFT) | - ((dd->ctx->keylen * 8) << - TEGRA_AES_SECURE_INPUT_KEY_LEN_SHIFT) | - ((u32)upd_iv << TEGRA_AES_SECURE_IV_SELECT_SHIFT); - - if (mode & FLAGS_CBC) { - value |= ((((mode & FLAGS_ENCRYPT) ? 2 : 3) - << TEGRA_AES_SECURE_XOR_POS_SHIFT) | - (((mode & FLAGS_ENCRYPT) ? 2 : 3) - << TEGRA_AES_SECURE_VCTRAM_SEL_SHIFT) | - ((mode & FLAGS_ENCRYPT) ? 1 : 0) - << TEGRA_AES_SECURE_CORE_SEL_SHIFT); - } else if (mode & FLAGS_OFB) { - value |= ((TEGRA_AES_SECURE_XOR_POS_FIELD) | - (2 << TEGRA_AES_SECURE_INPUT_SEL_SHIFT) | - (TEGRA_AES_SECURE_CORE_SEL_FIELD)); - } else if (mode & FLAGS_RNG) { - value |= (((mode & FLAGS_ENCRYPT) ? 1 : 0) - << TEGRA_AES_SECURE_CORE_SEL_SHIFT | - TEGRA_AES_SECURE_RNG_ENB_FIELD); - } else { - value |= (((mode & FLAGS_ENCRYPT) ? 1 : 0) - << TEGRA_AES_SECURE_CORE_SEL_SHIFT); - } - - dev_dbg(dd->dev, "secure_in_sel=0x%x", value); - aes_writel(dd, value, TEGRA_AES_SECURE_INPUT_SELECT); - - aes_writel(dd, out_addr, TEGRA_AES_SECURE_DEST_ADDR); - reinit_completion(&dd->op_complete); - - for (i = 0; i < AES_HW_MAX_ICQ_LENGTH - 1; i++) { - do { - value = aes_readl(dd, TEGRA_AES_INTR_STATUS); - eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD; - icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD; - } while (eng_busy && !icq_empty); - aes_writel(dd, cmdq[i], TEGRA_AES_ICMDQUE_WR); - } - - ret = wait_for_completion_timeout(&dd->op_complete, - msecs_to_jiffies(150)); - if (ret == 0) { - dev_err(dd->dev, "timed out (0x%x)\n", - aes_readl(dd, TEGRA_AES_INTR_STATUS)); - return -ETIMEDOUT; - } - - aes_writel(dd, cmdq[AES_HW_MAX_ICQ_LENGTH - 1], TEGRA_AES_ICMDQUE_WR); - return 0; -} - -static void aes_release_key_slot(struct tegra_aes_slot *slot) -{ - if (slot->slot_num == SSK_SLOT_NUM) - return; - - spin_lock(&list_lock); - list_add_tail(&slot->node, &dev_list); - slot = NULL; - spin_unlock(&list_lock); -} - -static struct tegra_aes_slot *aes_find_key_slot(void) -{ - struct tegra_aes_slot *slot = NULL; - struct list_head *new_head; - int empty; - - spin_lock(&list_lock); - empty = list_empty(&dev_list); - if (!empty) { - slot = list_entry(&dev_list, struct tegra_aes_slot, node); - new_head = dev_list.next; - list_del(&dev_list); - dev_list.next = new_head->next; - dev_list.prev = NULL; - } - spin_unlock(&list_lock); - - return slot; -} - -static int aes_set_key(struct tegra_aes_dev *dd) -{ - u32 value, cmdq[2]; - struct tegra_aes_ctx *ctx = dd->ctx; - int eng_busy, icq_empty, dma_busy; - bool use_ssk = false; - - /* use ssk? */ - if (!dd->ctx->slot) { - dev_dbg(dd->dev, "using ssk"); - dd->ctx->slot = &ssk; - use_ssk = true; - } - - /* enable key schedule generation in hardware */ - value = aes_readl(dd, TEGRA_AES_SECURE_CONFIG_EXT); - value &= ~TEGRA_AES_SECURE_KEY_SCH_DIS_FIELD; - aes_writel(dd, value, TEGRA_AES_SECURE_CONFIG_EXT); - - /* select the key slot */ - value = aes_readl(dd, TEGRA_AES_SECURE_CONFIG); - value &= ~TEGRA_AES_SECURE_KEY_INDEX_FIELD; - value |= (ctx->slot->slot_num << TEGRA_AES_SECURE_KEY_INDEX_SHIFT); - aes_writel(dd, value, TEGRA_AES_SECURE_CONFIG); - - if (use_ssk) - return 0; - - /* copy the key table from sdram to vram */ - cmdq[0] = CMD_MEMDMAVD << CMDQ_OPCODE_SHIFT | - MEMDMA_DIR_DTOVRAM << MEMDMA_DIR_SHIFT | - AES_HW_KEY_TABLE_LENGTH_BYTES / sizeof(u32) << - MEMDMA_NUM_WORDS_SHIFT; - cmdq[1] = (u32)dd->ivkey_phys_base; - - aes_writel(dd, cmdq[0], TEGRA_AES_ICMDQUE_WR); - aes_writel(dd, cmdq[1], TEGRA_AES_ICMDQUE_WR); - - do { - value = aes_readl(dd, TEGRA_AES_INTR_STATUS); - eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD; - icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD; - dma_busy = value & TEGRA_AES_DMA_BUSY_FIELD; - } while (eng_busy && !icq_empty && dma_busy); - - /* settable command to get key into internal registers */ - value = CMD_SETTABLE << CMDQ_OPCODE_SHIFT | - SUBCMD_CRYPTO_TABLE_SEL << CMDQ_TABLESEL_SHIFT | - SUBCMD_VRAM_SEL << CMDQ_VRAMSEL_SHIFT | - (SUBCMD_KEY_TABLE_SEL | ctx->slot->slot_num) << - CMDQ_KEYTABLEID_SHIFT; - aes_writel(dd, value, TEGRA_AES_ICMDQUE_WR); - - do { - value = aes_readl(dd, TEGRA_AES_INTR_STATUS); - eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD; - icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD; - } while (eng_busy && !icq_empty); - - return 0; -} - -static int tegra_aes_handle_req(struct tegra_aes_dev *dd) -{ - struct crypto_async_request *async_req, *backlog; - struct crypto_ablkcipher *tfm; - struct tegra_aes_ctx *ctx; - struct tegra_aes_reqctx *rctx; - struct ablkcipher_request *req; - unsigned long flags; - int dma_max = AES_HW_DMA_BUFFER_SIZE_BYTES; - int ret = 0, nblocks, total; - int count = 0; - dma_addr_t addr_in, addr_out; - struct scatterlist *in_sg, *out_sg; - - if (!dd) - return -EINVAL; - - spin_lock_irqsave(&dd->lock, flags); - backlog = crypto_get_backlog(&dd->queue); - async_req = crypto_dequeue_request(&dd->queue); - if (!async_req) - clear_bit(FLAGS_BUSY, &dd->flags); - spin_unlock_irqrestore(&dd->lock, flags); - - if (!async_req) - return -ENODATA; - - if (backlog) - backlog->complete(backlog, -EINPROGRESS); - - req = ablkcipher_request_cast(async_req); - - dev_dbg(dd->dev, "%s: get new req\n", __func__); - - if (!req->src || !req->dst) - return -EINVAL; - - /* take mutex to access the aes hw */ - mutex_lock(&aes_lock); - - /* assign new request to device */ - dd->req = req; - dd->total = req->nbytes; - dd->in_offset = 0; - dd->in_sg = req->src; - dd->out_offset = 0; - dd->out_sg = req->dst; - - in_sg = dd->in_sg; - out_sg = dd->out_sg; - - total = dd->total; - - tfm = crypto_ablkcipher_reqtfm(req); - rctx = ablkcipher_request_ctx(req); - ctx = crypto_ablkcipher_ctx(tfm); - rctx->mode &= FLAGS_MODE_MASK; - dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode; - - dd->iv = (u8 *)req->info; - dd->ivlen = crypto_ablkcipher_ivsize(tfm); - - /* assign new context to device */ - ctx->dd = dd; - dd->ctx = ctx; - - if (ctx->flags & FLAGS_NEW_KEY) { - /* copy the key */ - memcpy(dd->ivkey_base, ctx->key, ctx->keylen); - memset(dd->ivkey_base + ctx->keylen, 0, AES_HW_KEY_TABLE_LENGTH_BYTES - ctx->keylen); - aes_set_key(dd); - ctx->flags &= ~FLAGS_NEW_KEY; - } - - if (((dd->flags & FLAGS_CBC) || (dd->flags & FLAGS_OFB)) && dd->iv) { - /* set iv to the aes hw slot - * Hw generates updated iv only after iv is set in slot. - * So key and iv is passed asynchronously. - */ - memcpy(dd->buf_in, dd->iv, dd->ivlen); - - ret = aes_start_crypt(dd, (u32)dd->dma_buf_in, - dd->dma_buf_out, 1, FLAGS_CBC, false); - if (ret < 0) { - dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret); - goto out; - } - } - - while (total) { - dev_dbg(dd->dev, "remain: %d\n", total); - ret = dma_map_sg(dd->dev, in_sg, 1, DMA_TO_DEVICE); - if (!ret) { - dev_err(dd->dev, "dma_map_sg() error\n"); - goto out; - } - - ret = dma_map_sg(dd->dev, out_sg, 1, DMA_FROM_DEVICE); - if (!ret) { - dev_err(dd->dev, "dma_map_sg() error\n"); - dma_unmap_sg(dd->dev, dd->in_sg, - 1, DMA_TO_DEVICE); - goto out; - } - - addr_in = sg_dma_address(in_sg); - addr_out = sg_dma_address(out_sg); - dd->flags |= FLAGS_FAST; - count = min_t(int, sg_dma_len(in_sg), dma_max); - WARN_ON(sg_dma_len(in_sg) != sg_dma_len(out_sg)); - nblocks = DIV_ROUND_UP(count, AES_BLOCK_SIZE); - - ret = aes_start_crypt(dd, addr_in, addr_out, nblocks, - dd->flags, true); - - dma_unmap_sg(dd->dev, out_sg, 1, DMA_FROM_DEVICE); - dma_unmap_sg(dd->dev, in_sg, 1, DMA_TO_DEVICE); - - if (ret < 0) { - dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret); - goto out; - } - dd->flags &= ~FLAGS_FAST; - - dev_dbg(dd->dev, "out: copied %d\n", count); - total -= count; - in_sg = sg_next(in_sg); - out_sg = sg_next(out_sg); - WARN_ON(((total != 0) && (!in_sg || !out_sg))); - } - -out: - mutex_unlock(&aes_lock); - - dd->total = total; - - if (dd->req->base.complete) - dd->req->base.complete(&dd->req->base, ret); - - dev_dbg(dd->dev, "%s: exit\n", __func__); - return ret; -} - -static int tegra_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, - unsigned int keylen) -{ - struct tegra_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); - struct tegra_aes_dev *dd = aes_dev; - struct tegra_aes_slot *key_slot; - - if ((keylen != AES_KEYSIZE_128) && (keylen != AES_KEYSIZE_192) && - (keylen != AES_KEYSIZE_256)) { - dev_err(dd->dev, "unsupported key size\n"); - crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; - } - - dev_dbg(dd->dev, "keylen: %d\n", keylen); - - ctx->dd = dd; - - if (key) { - if (!ctx->slot) { - key_slot = aes_find_key_slot(); - if (!key_slot) { - dev_err(dd->dev, "no empty slot\n"); - return -ENOMEM; - } - - ctx->slot = key_slot; - } - - memcpy(ctx->key, key, keylen); - ctx->keylen = keylen; - } - - ctx->flags |= FLAGS_NEW_KEY; - dev_dbg(dd->dev, "done\n"); - return 0; -} - -static void aes_workqueue_handler(struct work_struct *work) -{ - struct tegra_aes_dev *dd = aes_dev; - int ret; - - ret = clk_prepare_enable(dd->aes_clk); - if (ret) - BUG_ON("clock enable failed"); - - /* empty the crypto queue and then return */ - do { - ret = tegra_aes_handle_req(dd); - } while (!ret); - - clk_disable_unprepare(dd->aes_clk); -} - -static irqreturn_t aes_irq(int irq, void *dev_id) -{ - struct tegra_aes_dev *dd = (struct tegra_aes_dev *)dev_id; - u32 value = aes_readl(dd, TEGRA_AES_INTR_STATUS); - int busy = test_bit(FLAGS_BUSY, &dd->flags); - - if (!busy) { - dev_dbg(dd->dev, "spurious interrupt\n"); - return IRQ_NONE; - } - - dev_dbg(dd->dev, "irq_stat: 0x%x\n", value); - if (value & TEGRA_AES_INT_ERROR_MASK) - aes_writel(dd, TEGRA_AES_INT_ERROR_MASK, TEGRA_AES_INTR_STATUS); - - if (!(value & TEGRA_AES_ENGINE_BUSY_FIELD)) - complete(&dd->op_complete); - else - return IRQ_NONE; - - return IRQ_HANDLED; -} - -static int tegra_aes_crypt(struct ablkcipher_request *req, unsigned long mode) -{ - struct tegra_aes_reqctx *rctx = ablkcipher_request_ctx(req); - struct tegra_aes_dev *dd = aes_dev; - unsigned long flags; - int err = 0; - int busy; - - dev_dbg(dd->dev, "nbytes: %d, enc: %d, cbc: %d, ofb: %d\n", - req->nbytes, !!(mode & FLAGS_ENCRYPT), - !!(mode & FLAGS_CBC), !!(mode & FLAGS_OFB)); - - rctx->mode = mode; - - spin_lock_irqsave(&dd->lock, flags); - err = ablkcipher_enqueue_request(&dd->queue, req); - busy = test_and_set_bit(FLAGS_BUSY, &dd->flags); - spin_unlock_irqrestore(&dd->lock, flags); - - if (!busy) - queue_work(aes_wq, &aes_work); - - return err; -} - -static int tegra_aes_ecb_encrypt(struct ablkcipher_request *req) -{ - return tegra_aes_crypt(req, FLAGS_ENCRYPT); -} - -static int tegra_aes_ecb_decrypt(struct ablkcipher_request *req) -{ - return tegra_aes_crypt(req, 0); -} - -static int tegra_aes_cbc_encrypt(struct ablkcipher_request *req) -{ - return tegra_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC); -} - -static int tegra_aes_cbc_decrypt(struct ablkcipher_request *req) -{ - return tegra_aes_crypt(req, FLAGS_CBC); -} - -static int tegra_aes_ofb_encrypt(struct ablkcipher_request *req) -{ - return tegra_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_OFB); -} - -static int tegra_aes_ofb_decrypt(struct ablkcipher_request *req) -{ - return tegra_aes_crypt(req, FLAGS_OFB); -} - -static int tegra_aes_get_random(struct crypto_rng *tfm, u8 *rdata, - unsigned int dlen) -{ - struct tegra_aes_dev *dd = aes_dev; - struct tegra_aes_ctx *ctx = &rng_ctx; - int ret, i; - u8 *dest = rdata, *dt = dd->dt; - - /* take mutex to access the aes hw */ - mutex_lock(&aes_lock); - - ret = clk_prepare_enable(dd->aes_clk); - if (ret) { - mutex_unlock(&aes_lock); - return ret; - } - - ctx->dd = dd; - dd->ctx = ctx; - dd->flags = FLAGS_ENCRYPT | FLAGS_RNG; - - memcpy(dd->buf_in, dt, DEFAULT_RNG_BLK_SZ); - - ret = aes_start_crypt(dd, (u32)dd->dma_buf_in, - (u32)dd->dma_buf_out, 1, dd->flags, true); - if (ret < 0) { - dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret); - dlen = ret; - goto out; - } - memcpy(dest, dd->buf_out, dlen); - - /* update the DT */ - for (i = DEFAULT_RNG_BLK_SZ - 1; i >= 0; i--) { - dt[i] += 1; - if (dt[i] != 0) - break; - } - -out: - clk_disable_unprepare(dd->aes_clk); - mutex_unlock(&aes_lock); - - dev_dbg(dd->dev, "%s: done\n", __func__); - return dlen; -} - -static int tegra_aes_rng_reset(struct crypto_rng *tfm, u8 *seed, - unsigned int slen) -{ - struct tegra_aes_dev *dd = aes_dev; - struct tegra_aes_ctx *ctx = &rng_ctx; - struct tegra_aes_slot *key_slot; - int ret = 0; - u8 tmp[16]; /* 16 bytes = 128 bits of entropy */ - u8 *dt; - - if (!ctx || !dd) { - pr_err("ctx=0x%x, dd=0x%x\n", - (unsigned int)ctx, (unsigned int)dd); - return -EINVAL; - } - - if (slen < (DEFAULT_RNG_BLK_SZ + AES_KEYSIZE_128)) { - dev_err(dd->dev, "seed size invalid"); - return -ENOMEM; - } - - /* take mutex to access the aes hw */ - mutex_lock(&aes_lock); - - if (!ctx->slot) { - key_slot = aes_find_key_slot(); - if (!key_slot) { - dev_err(dd->dev, "no empty slot\n"); - mutex_unlock(&aes_lock); - return -ENOMEM; - } - ctx->slot = key_slot; - } - - ctx->dd = dd; - dd->ctx = ctx; - dd->ctr = 0; - - ctx->keylen = AES_KEYSIZE_128; - ctx->flags |= FLAGS_NEW_KEY; - - /* copy the key to the key slot */ - memcpy(dd->ivkey_base, seed + DEFAULT_RNG_BLK_SZ, AES_KEYSIZE_128); - memset(dd->ivkey_base + AES_KEYSIZE_128, 0, AES_HW_KEY_TABLE_LENGTH_BYTES - AES_KEYSIZE_128); - - dd->iv = seed; - dd->ivlen = slen; - - dd->flags = FLAGS_ENCRYPT | FLAGS_RNG; - - ret = clk_prepare_enable(dd->aes_clk); - if (ret) { - mutex_unlock(&aes_lock); - return ret; - } - - aes_set_key(dd); - - /* set seed to the aes hw slot */ - memcpy(dd->buf_in, dd->iv, DEFAULT_RNG_BLK_SZ); - ret = aes_start_crypt(dd, (u32)dd->dma_buf_in, - dd->dma_buf_out, 1, FLAGS_CBC, false); - if (ret < 0) { - dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret); - goto out; - } - - if (dd->ivlen >= (2 * DEFAULT_RNG_BLK_SZ + AES_KEYSIZE_128)) { - dt = dd->iv + DEFAULT_RNG_BLK_SZ + AES_KEYSIZE_128; - } else { - get_random_bytes(tmp, sizeof(tmp)); - dt = tmp; - } - memcpy(dd->dt, dt, DEFAULT_RNG_BLK_SZ); - -out: - clk_disable_unprepare(dd->aes_clk); - mutex_unlock(&aes_lock); - - dev_dbg(dd->dev, "%s: done\n", __func__); - return ret; -} - -static int tegra_aes_cra_init(struct crypto_tfm *tfm) -{ - tfm->crt_ablkcipher.reqsize = sizeof(struct tegra_aes_reqctx); - - return 0; -} - -static void tegra_aes_cra_exit(struct crypto_tfm *tfm) -{ - struct tegra_aes_ctx *ctx = - crypto_ablkcipher_ctx((struct crypto_ablkcipher *)tfm); - - if (ctx && ctx->slot) - aes_release_key_slot(ctx->slot); -} - -static struct crypto_alg algs[] = { - { - .cra_name = "ecb(aes)", - .cra_driver_name = "ecb-aes-tegra", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_u.ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = tegra_aes_setkey, - .encrypt = tegra_aes_ecb_encrypt, - .decrypt = tegra_aes_ecb_decrypt, - }, - }, { - .cra_name = "cbc(aes)", - .cra_driver_name = "cbc-aes-tegra", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_u.ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_MIN_KEY_SIZE, - .setkey = tegra_aes_setkey, - .encrypt = tegra_aes_cbc_encrypt, - .decrypt = tegra_aes_cbc_decrypt, - } - }, { - .cra_name = "ofb(aes)", - .cra_driver_name = "ofb-aes-tegra", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_alignmask = 3, - .cra_type = &crypto_ablkcipher_type, - .cra_u.ablkcipher = { - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_MIN_KEY_SIZE, - .setkey = tegra_aes_setkey, - .encrypt = tegra_aes_ofb_encrypt, - .decrypt = tegra_aes_ofb_decrypt, - } - }, { - .cra_name = "ansi_cprng", - .cra_driver_name = "rng-aes-tegra", - .cra_flags = CRYPTO_ALG_TYPE_RNG, - .cra_ctxsize = sizeof(struct tegra_aes_ctx), - .cra_type = &crypto_rng_type, - .cra_u.rng = { - .rng_make_random = tegra_aes_get_random, - .rng_reset = tegra_aes_rng_reset, - .seedsize = AES_KEYSIZE_128 + (2 * DEFAULT_RNG_BLK_SZ), - } - } -}; - -static int tegra_aes_probe(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct tegra_aes_dev *dd; - struct resource *res; - int err = -ENOMEM, i = 0, j; - - dd = devm_kzalloc(dev, sizeof(struct tegra_aes_dev), GFP_KERNEL); - if (dd == NULL) { - dev_err(dev, "unable to alloc data struct.\n"); - return err; - } - - dd->dev = dev; - platform_set_drvdata(pdev, dd); - - dd->slots = devm_kzalloc(dev, sizeof(struct tegra_aes_slot) * - AES_NR_KEYSLOTS, GFP_KERNEL); - if (dd->slots == NULL) { - dev_err(dev, "unable to alloc slot struct.\n"); - goto out; - } - - spin_lock_init(&dd->lock); - crypto_init_queue(&dd->queue, TEGRA_AES_QUEUE_LENGTH); - - /* Get the module base address */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(dev, "invalid resource type: base\n"); - err = -ENODEV; - goto out; - } - - if (!devm_request_mem_region(&pdev->dev, res->start, - resource_size(res), - dev_name(&pdev->dev))) { - dev_err(&pdev->dev, "Couldn't request MEM resource\n"); - return -ENODEV; - } - - dd->io_base = devm_ioremap(dev, res->start, resource_size(res)); - if (!dd->io_base) { - dev_err(dev, "can't ioremap register space\n"); - err = -ENOMEM; - goto out; - } - - /* Initialize the vde clock */ - dd->aes_clk = devm_clk_get(dev, "vde"); - if (IS_ERR(dd->aes_clk)) { - dev_err(dev, "iclock intialization failed.\n"); - err = -ENODEV; - goto out; - } - - err = clk_set_rate(dd->aes_clk, ULONG_MAX); - if (err) { - dev_err(dd->dev, "iclk set_rate fail(%d)\n", err); - goto out; - } - - /* - * the foll contiguous memory is allocated as follows - - * - hardware key table - * - key schedule - */ - dd->ivkey_base = dma_alloc_coherent(dev, AES_HW_KEY_TABLE_LENGTH_BYTES, - &dd->ivkey_phys_base, - GFP_KERNEL); - if (!dd->ivkey_base) { - dev_err(dev, "can not allocate iv/key buffer\n"); - err = -ENOMEM; - goto out; - } - - dd->buf_in = dma_alloc_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES, - &dd->dma_buf_in, GFP_KERNEL); - if (!dd->buf_in) { - dev_err(dev, "can not allocate dma-in buffer\n"); - err = -ENOMEM; - goto out; - } - - dd->buf_out = dma_alloc_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES, - &dd->dma_buf_out, GFP_KERNEL); - if (!dd->buf_out) { - dev_err(dev, "can not allocate dma-out buffer\n"); - err = -ENOMEM; - goto out; - } - - init_completion(&dd->op_complete); - aes_wq = alloc_workqueue("tegra_aes_wq", WQ_HIGHPRI | WQ_UNBOUND, 1); - if (!aes_wq) { - dev_err(dev, "alloc_workqueue failed\n"); - err = -ENOMEM; - goto out; - } - - /* get the irq */ - res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); - if (!res) { - dev_err(dev, "invalid resource type: base\n"); - err = -ENODEV; - goto out; - } - dd->irq = res->start; - - err = devm_request_irq(dev, dd->irq, aes_irq, IRQF_TRIGGER_HIGH | - IRQF_SHARED, "tegra-aes", dd); - if (err) { - dev_err(dev, "request_irq failed\n"); - goto out; - } - - mutex_init(&aes_lock); - INIT_LIST_HEAD(&dev_list); - - spin_lock_init(&list_lock); - spin_lock(&list_lock); - for (i = 0; i < AES_NR_KEYSLOTS; i++) { - if (i == SSK_SLOT_NUM) - continue; - dd->slots[i].slot_num = i; - INIT_LIST_HEAD(&dd->slots[i].node); - list_add_tail(&dd->slots[i].node, &dev_list); - } - spin_unlock(&list_lock); - - aes_dev = dd; - for (i = 0; i < ARRAY_SIZE(algs); i++) { - algs[i].cra_priority = 300; - algs[i].cra_ctxsize = sizeof(struct tegra_aes_ctx); - algs[i].cra_module = THIS_MODULE; - algs[i].cra_init = tegra_aes_cra_init; - algs[i].cra_exit = tegra_aes_cra_exit; - - err = crypto_register_alg(&algs[i]); - if (err) - goto out; - } - - dev_info(dev, "registered"); - return 0; - -out: - for (j = 0; j < i; j++) - crypto_unregister_alg(&algs[j]); - if (dd->ivkey_base) - dma_free_coherent(dev, AES_HW_KEY_TABLE_LENGTH_BYTES, - dd->ivkey_base, dd->ivkey_phys_base); - if (dd->buf_in) - dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES, - dd->buf_in, dd->dma_buf_in); - if (dd->buf_out) - dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES, - dd->buf_out, dd->dma_buf_out); - if (aes_wq) - destroy_workqueue(aes_wq); - spin_lock(&list_lock); - list_del(&dev_list); - spin_unlock(&list_lock); - - aes_dev = NULL; - - dev_err(dev, "%s: initialization failed.\n", __func__); - return err; -} - -static int tegra_aes_remove(struct platform_device *pdev) -{ - struct device *dev = &pdev->dev; - struct tegra_aes_dev *dd = platform_get_drvdata(pdev); - int i; - - for (i = 0; i < ARRAY_SIZE(algs); i++) - crypto_unregister_alg(&algs[i]); - - cancel_work_sync(&aes_work); - destroy_workqueue(aes_wq); - spin_lock(&list_lock); - list_del(&dev_list); - spin_unlock(&list_lock); - - dma_free_coherent(dev, AES_HW_KEY_TABLE_LENGTH_BYTES, - dd->ivkey_base, dd->ivkey_phys_base); - dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES, - dd->buf_in, dd->dma_buf_in); - dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES, - dd->buf_out, dd->dma_buf_out); - aes_dev = NULL; - - return 0; -} - -static struct of_device_id tegra_aes_of_match[] = { - { .compatible = "nvidia,tegra20-aes", }, - { .compatible = "nvidia,tegra30-aes", }, - { }, -}; - -static struct platform_driver tegra_aes_driver = { - .probe = tegra_aes_probe, - .remove = tegra_aes_remove, - .driver = { - .name = "tegra-aes", - .owner = THIS_MODULE, - .of_match_table = tegra_aes_of_match, - }, -}; - -module_platform_driver(tegra_aes_driver); - -MODULE_DESCRIPTION("Tegra AES/OFB/CPRNG hw acceleration support."); -MODULE_AUTHOR("NVIDIA Corporation"); -MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/tegra-aes.h b/drivers/crypto/tegra-aes.h deleted file mode 100644 index 6006333a893..00000000000 --- a/drivers/crypto/tegra-aes.h +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2010, NVIDIA Corporation. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#ifndef __CRYPTODEV_TEGRA_AES_H -#define __CRYPTODEV_TEGRA_AES_H - -#define TEGRA_AES_ICMDQUE_WR 0x1000 -#define TEGRA_AES_CMDQUE_CONTROL 0x1008 -#define TEGRA_AES_INTR_STATUS 0x1018 -#define TEGRA_AES_INT_ENB 0x1040 -#define TEGRA_AES_CONFIG 0x1044 -#define TEGRA_AES_IRAM_ACCESS_CFG 0x10A0 -#define TEGRA_AES_SECURE_DEST_ADDR 0x1100 -#define TEGRA_AES_SECURE_INPUT_SELECT 0x1104 -#define TEGRA_AES_SECURE_CONFIG 0x1108 -#define TEGRA_AES_SECURE_CONFIG_EXT 0x110C -#define TEGRA_AES_SECURE_SECURITY 0x1110 -#define TEGRA_AES_SECURE_HASH_RESULT0 0x1120 -#define TEGRA_AES_SECURE_HASH_RESULT1 0x1124 -#define TEGRA_AES_SECURE_HASH_RESULT2 0x1128 -#define TEGRA_AES_SECURE_HASH_RESULT3 0x112C -#define TEGRA_AES_SECURE_SEC_SEL0 0x1140 -#define TEGRA_AES_SECURE_SEC_SEL1 0x1144 -#define TEGRA_AES_SECURE_SEC_SEL2 0x1148 -#define TEGRA_AES_SECURE_SEC_SEL3 0x114C -#define TEGRA_AES_SECURE_SEC_SEL4 0x1150 -#define TEGRA_AES_SECURE_SEC_SEL5 0x1154 -#define TEGRA_AES_SECURE_SEC_SEL6 0x1158 -#define TEGRA_AES_SECURE_SEC_SEL7 0x115C - -/* interrupt status reg masks and shifts */ -#define TEGRA_AES_ENGINE_BUSY_FIELD BIT(0) -#define TEGRA_AES_ICQ_EMPTY_FIELD BIT(3) -#define TEGRA_AES_DMA_BUSY_FIELD BIT(23) - -/* secure select reg masks and shifts */ -#define TEGRA_AES_SECURE_SEL0_KEYREAD_ENB0_FIELD BIT(0) - -/* secure config ext masks and shifts */ -#define TEGRA_AES_SECURE_KEY_SCH_DIS_FIELD BIT(15) - -/* secure config masks and shifts */ -#define TEGRA_AES_SECURE_KEY_INDEX_SHIFT 20 -#define TEGRA_AES_SECURE_KEY_INDEX_FIELD (0x1F << TEGRA_AES_SECURE_KEY_INDEX_SHIFT) -#define TEGRA_AES_SECURE_BLOCK_CNT_SHIFT 0 -#define TEGRA_AES_SECURE_BLOCK_CNT_FIELD (0xFFFFF << TEGRA_AES_SECURE_BLOCK_CNT_SHIFT) - -/* stream interface select masks and shifts */ -#define TEGRA_AES_CMDQ_CTRL_UCMDQEN_FIELD BIT(0) -#define TEGRA_AES_CMDQ_CTRL_ICMDQEN_FIELD BIT(1) -#define TEGRA_AES_CMDQ_CTRL_SRC_STM_SEL_FIELD BIT(4) -#define TEGRA_AES_CMDQ_CTRL_DST_STM_SEL_FIELD BIT(5) - -/* config register masks and shifts */ -#define TEGRA_AES_CONFIG_ENDIAN_ENB_FIELD BIT(10) -#define TEGRA_AES_CONFIG_MODE_SEL_SHIFT 0 -#define TEGRA_AES_CONFIG_MODE_SEL_FIELD (0x1F << TEGRA_AES_CONFIG_MODE_SEL_SHIFT) - -/* extended config */ -#define TEGRA_AES_SECURE_OFFSET_CNT_SHIFT 24 -#define TEGRA_AES_SECURE_OFFSET_CNT_FIELD (0xFF << TEGRA_AES_SECURE_OFFSET_CNT_SHIFT) -#define TEGRA_AES_SECURE_KEYSCHED_GEN_FIELD BIT(15) - -/* init vector select */ -#define TEGRA_AES_SECURE_IV_SELECT_SHIFT 10 -#define TEGRA_AES_SECURE_IV_SELECT_FIELD BIT(10) - -/* secure engine input */ -#define TEGRA_AES_SECURE_INPUT_ALG_SEL_SHIFT 28 -#define TEGRA_AES_SECURE_INPUT_ALG_SEL_FIELD (0xF << TEGRA_AES_SECURE_INPUT_ALG_SEL_SHIFT) -#define TEGRA_AES_SECURE_INPUT_KEY_LEN_SHIFT 16 -#define TEGRA_AES_SECURE_INPUT_KEY_LEN_FIELD (0xFFF << TEGRA_AES_SECURE_INPUT_KEY_LEN_SHIFT) -#define TEGRA_AES_SECURE_RNG_ENB_FIELD BIT(11) -#define TEGRA_AES_SECURE_CORE_SEL_SHIFT 9 -#define TEGRA_AES_SECURE_CORE_SEL_FIELD BIT(9) -#define TEGRA_AES_SECURE_VCTRAM_SEL_SHIFT 7 -#define TEGRA_AES_SECURE_VCTRAM_SEL_FIELD (0x3 << TEGRA_AES_SECURE_VCTRAM_SEL_SHIFT) -#define TEGRA_AES_SECURE_INPUT_SEL_SHIFT 5 -#define TEGRA_AES_SECURE_INPUT_SEL_FIELD (0x3 << TEGRA_AES_SECURE_INPUT_SEL_SHIFT) -#define TEGRA_AES_SECURE_XOR_POS_SHIFT 3 -#define TEGRA_AES_SECURE_XOR_POS_FIELD (0x3 << TEGRA_AES_SECURE_XOR_POS_SHIFT) -#define TEGRA_AES_SECURE_HASH_ENB_FIELD BIT(2) -#define TEGRA_AES_SECURE_ON_THE_FLY_FIELD BIT(0) - -/* interrupt error mask */ -#define TEGRA_AES_INT_ERROR_MASK 0xFFF000 - -#endif |