diff options
Diffstat (limited to 'drivers/crypto/talitos.c')
| -rw-r--r-- | drivers/crypto/talitos.c | 797 |
1 files changed, 361 insertions, 436 deletions
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 2d8c7890168..624b8be0c36 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -32,6 +32,8 @@ #include <linux/interrupt.h> #include <linux/crypto.h> #include <linux/hw_random.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> #include <linux/of_platform.h> #include <linux/dma-mapping.h> #include <linux/io.h> @@ -53,114 +55,6 @@ #include "talitos.h" -#define TALITOS_TIMEOUT 100000 -#define TALITOS_MAX_DATA_LEN 65535 - -#define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f) -#define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf) -#define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf) - -/* descriptor pointer entry */ -struct talitos_ptr { - __be16 len; /* length */ - u8 j_extent; /* jump to sg link table and/or extent */ - u8 eptr; /* extended address */ - __be32 ptr; /* address */ -}; - -static const struct talitos_ptr zero_entry = { - .len = 0, - .j_extent = 0, - .eptr = 0, - .ptr = 0 -}; - -/* descriptor */ -struct talitos_desc { - __be32 hdr; /* header high bits */ - __be32 hdr_lo; /* header low bits */ - struct talitos_ptr ptr[7]; /* ptr/len pair array */ -}; - -/** - * talitos_request - descriptor submission request - * @desc: descriptor pointer (kernel virtual) - * @dma_desc: descriptor's physical bus address - * @callback: whom to call when descriptor processing is done - * @context: caller context (optional) - */ -struct talitos_request { - struct talitos_desc *desc; - dma_addr_t dma_desc; - void (*callback) (struct device *dev, struct talitos_desc *desc, - void *context, int error); - void *context; -}; - -/* per-channel fifo management */ -struct talitos_channel { - void __iomem *reg; - - /* request fifo */ - struct talitos_request *fifo; - - /* number of requests pending in channel h/w fifo */ - atomic_t submit_count ____cacheline_aligned; - - /* request submission (head) lock */ - spinlock_t head_lock ____cacheline_aligned; - /* index to next free descriptor request */ - int head; - - /* request release (tail) lock */ - spinlock_t tail_lock ____cacheline_aligned; - /* index to next in-progress/done descriptor request */ - int tail; -}; - -struct talitos_private { - struct device *dev; - struct platform_device *ofdev; - void __iomem *reg; - int irq[2]; - - /* SEC version geometry (from device tree node) */ - unsigned int num_channels; - unsigned int chfifo_len; - unsigned int exec_units; - unsigned int desc_types; - - /* SEC Compatibility info */ - unsigned long features; - - /* - * length of the request fifo - * fifo_len is chfifo_len rounded up to next power of 2 - * so we can use bitwise ops to wrap - */ - unsigned int fifo_len; - - struct talitos_channel *chan; - - /* next channel to be assigned next incoming descriptor */ - atomic_t last_chan ____cacheline_aligned; - - /* request callback tasklet */ - struct tasklet_struct done_task[2]; - - /* list of registered algorithms */ - struct list_head alg_list; - - /* hwrng device */ - struct hwrng rng; -}; - -/* .features flag */ -#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001 -#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002 -#define TALITOS_FTR_SHA224_HWINIT 0x00000004 -#define TALITOS_FTR_HMAC_OK 0x00000008 - static void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr) { talitos_ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); @@ -300,11 +194,11 @@ static int init_device(struct device *dev) * callback must check err and feedback in descriptor header * for device processing status. */ -static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, - void (*callback)(struct device *dev, - struct talitos_desc *desc, - void *context, int error), - void *context) +int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, + void (*callback)(struct device *dev, + struct talitos_desc *desc, + void *context, int error), + void *context) { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request; @@ -345,6 +239,7 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, return -EINPROGRESS; } +EXPORT_SYMBOL(talitos_submit); /* * process what was done, notify callback of error if not @@ -412,6 +307,7 @@ static void talitos_done_##name(unsigned long data) \ { \ struct device *dev = (struct device *)data; \ struct talitos_private *priv = dev_get_drvdata(dev); \ + unsigned long flags; \ \ if (ch_done_mask & 1) \ flush_channel(dev, 0, 0, 0); \ @@ -427,8 +323,10 @@ static void talitos_done_##name(unsigned long data) \ out: \ /* At this point, all completed channels have been processed */ \ /* Unmask done interrupts for channels completed later on. */ \ + spin_lock_irqsave(&priv->reg_lock, flags); \ setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ } DEF_TALITOS_DONE(4ch, TALITOS_ISR_4CHDONE) DEF_TALITOS_DONE(ch0_2, TALITOS_ISR_CH_0_2_DONE) @@ -440,20 +338,29 @@ DEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE) static u32 current_desc_hdr(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); - int tail = priv->chan[ch].tail; + int tail, iter; dma_addr_t cur_desc; - cur_desc = in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); + cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32; + cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO); - while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) { - tail = (tail + 1) & (priv->fifo_len - 1); - if (tail == priv->chan[ch].tail) { + if (!cur_desc) { + dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n"); + return 0; + } + + tail = priv->chan[ch].tail; + + iter = tail; + while (priv->chan[ch].fifo[iter].dma_desc != cur_desc) { + iter = (iter + 1) & (priv->fifo_len - 1); + if (iter == tail) { dev_err(dev, "couldn't locate current descriptor\n"); return 0; } } - return priv->chan[ch].fifo[tail].desc->hdr; + return priv->chan[ch].fifo[iter].desc->hdr; } /* @@ -619,22 +526,28 @@ static irqreturn_t talitos_interrupt_##name(int irq, void *data) \ struct device *dev = data; \ struct talitos_private *priv = dev_get_drvdata(dev); \ u32 isr, isr_lo; \ + unsigned long flags; \ \ + spin_lock_irqsave(&priv->reg_lock, flags); \ isr = in_be32(priv->reg + TALITOS_ISR); \ isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \ /* Acknowledge interrupt */ \ out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \ out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \ \ - if (unlikely((isr & ~TALITOS_ISR_4CHDONE) & ch_err_mask || isr_lo)) \ - talitos_error(dev, isr, isr_lo); \ - else \ + if (unlikely(isr & ch_err_mask || isr_lo)) { \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ + talitos_error(dev, isr & ch_err_mask, isr_lo); \ + } \ + else { \ if (likely(isr & ch_done_mask)) { \ /* mask further done interrupts. */ \ clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \ /* done_task will unmask done interrupts at exit */ \ tasklet_schedule(&priv->done_task[tlet]); \ } \ + spin_unlock_irqrestore(&priv->reg_lock, flags); \ + } \ \ return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \ IRQ_NONE; \ @@ -721,7 +634,7 @@ static void talitos_unregister_rng(struct device *dev) * crypto alg */ #define TALITOS_CRA_PRIORITY 3000 -#define TALITOS_MAX_KEY_SIZE 64 +#define TALITOS_MAX_KEY_SIZE 96 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ #define MD5_BLOCK_SIZE 64 @@ -769,39 +682,20 @@ static int aead_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { struct talitos_ctx *ctx = crypto_aead_ctx(authenc); - struct rtattr *rta = (void *)key; - struct crypto_authenc_key_param *param; - unsigned int authkeylen; - unsigned int enckeylen; - - if (!RTA_OK(rta, keylen)) - goto badkey; - - if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM) - goto badkey; - - if (RTA_PAYLOAD(rta) < sizeof(*param)) - goto badkey; + struct crypto_authenc_keys keys; - param = RTA_DATA(rta); - enckeylen = be32_to_cpu(param->enckeylen); - - key += RTA_ALIGN(rta->rta_len); - keylen -= RTA_ALIGN(rta->rta_len); - - if (keylen < enckeylen) + if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; - authkeylen = keylen - enckeylen; - - if (keylen > TALITOS_MAX_KEY_SIZE) + if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE) goto badkey; - memcpy(&ctx->key, key, keylen); + memcpy(ctx->key, keys.authkey, keys.authkeylen); + memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen); - ctx->keylen = keylen; - ctx->enckeylen = enckeylen; - ctx->authkeylen = authkeylen; + ctx->keylen = keys.authkeylen + keys.enckeylen; + ctx->enckeylen = keys.enckeylen; + ctx->authkeylen = keys.authkeylen; return 0; @@ -812,8 +706,13 @@ badkey: /* * talitos_edesc - s/w-extended descriptor + * @assoc_nents: number of segments in associated data scatterlist * @src_nents: number of segments in input scatterlist * @dst_nents: number of segments in output scatterlist + * @assoc_chained: whether assoc is chained or not + * @src_chained: whether src is chained or not + * @dst_chained: whether dst is chained or not + * @iv_dma: dma address of iv for checking continuity and link table * @dma_len: length of dma mapped link_tbl space * @dma_link_tbl: bus physical address of link_tbl * @desc: h/w descriptor @@ -824,10 +723,13 @@ badkey: * of link_tbl data */ struct talitos_edesc { + int assoc_nents; int src_nents; int dst_nents; - int src_is_chained; - int dst_is_chained; + bool assoc_chained; + bool src_chained; + bool dst_chained; + dma_addr_t iv_dma; int dma_len; dma_addr_t dma_link_tbl; struct talitos_desc desc; @@ -836,7 +738,7 @@ struct talitos_edesc { static int talitos_map_sg(struct device *dev, struct scatterlist *sg, unsigned int nents, enum dma_data_direction dir, - int chained) + bool chained) { if (unlikely(chained)) while (sg) { @@ -866,13 +768,13 @@ static void talitos_sg_unmap(struct device *dev, unsigned int dst_nents = edesc->dst_nents ? : 1; if (src != dst) { - if (edesc->src_is_chained) + if (edesc->src_chained) talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE); else dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); if (dst) { - if (edesc->dst_is_chained) + if (edesc->dst_chained) talitos_unmap_sg_chain(dev, dst, DMA_FROM_DEVICE); else @@ -880,7 +782,7 @@ static void talitos_sg_unmap(struct device *dev, DMA_FROM_DEVICE); } } else - if (edesc->src_is_chained) + if (edesc->src_chained) talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL); else dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); @@ -895,7 +797,13 @@ static void ipsec_esp_unmap(struct device *dev, unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE); - dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE); + if (edesc->assoc_chained) + talitos_unmap_sg_chain(dev, areq->assoc, DMA_TO_DEVICE); + else if (areq->assoclen) + /* assoc_nents counts also for IV in non-contiguous cases */ + dma_unmap_sg(dev, areq->assoc, + edesc->assoc_nents ? edesc->assoc_nents - 1 : 1, + DMA_TO_DEVICE); talitos_sg_unmap(dev, edesc, areq->src, areq->dst); @@ -923,9 +831,10 @@ static void ipsec_esp_encrypt_done(struct device *dev, ipsec_esp_unmap(dev, edesc, areq); /* copy the generated ICV to dst */ - if (edesc->dma_len) { + if (edesc->dst_nents) { icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; + edesc->dst_nents + 2 + + edesc->assoc_nents]; sg = sg_last(areq->dst, edesc->dst_nents); memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize, icvdata, ctx->authsize); @@ -955,7 +864,8 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, /* auth check */ if (edesc->dma_len) icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; + edesc->dst_nents + 2 + + edesc->assoc_nents]; else icvdata = &edesc->link_tbl[0]; @@ -1017,8 +927,7 @@ static int sg_to_link_tbl(struct scatterlist *sg, int sg_count, sg_count--; link_tbl_ptr--; } - link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len) - + cryptlen); + be16_add_cpu(&link_tbl_ptr->len, cryptlen); /* tag end of link table */ link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; @@ -1030,10 +939,9 @@ static int sg_to_link_tbl(struct scatterlist *sg, int sg_count, * fill in and submit ipsec_esp descriptor */ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, - u8 *giv, u64 seq, - void (*callback) (struct device *dev, - struct talitos_desc *desc, - void *context, int error)) + u64 seq, void (*callback) (struct device *dev, + struct talitos_desc *desc, + void *context, int error)) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(aead); @@ -1048,12 +956,46 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, /* hmac key */ map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, 0, DMA_TO_DEVICE); + /* hmac data */ - map_single_talitos_ptr(dev, &desc->ptr[1], areq->assoclen + ivsize, - sg_virt(areq->assoc), 0, DMA_TO_DEVICE); + desc->ptr[1].len = cpu_to_be16(areq->assoclen + ivsize); + if (edesc->assoc_nents) { + int tbl_off = edesc->src_nents + edesc->dst_nents + 2; + struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; + + to_talitos_ptr(&desc->ptr[1], edesc->dma_link_tbl + tbl_off * + sizeof(struct talitos_ptr)); + desc->ptr[1].j_extent = DESC_PTR_LNKTBL_JUMP; + + /* assoc_nents - 1 entries for assoc, 1 for IV */ + sg_count = sg_to_link_tbl(areq->assoc, edesc->assoc_nents - 1, + areq->assoclen, tbl_ptr); + + /* add IV to link table */ + tbl_ptr += sg_count - 1; + tbl_ptr->j_extent = 0; + tbl_ptr++; + to_talitos_ptr(tbl_ptr, edesc->iv_dma); + tbl_ptr->len = cpu_to_be16(ivsize); + tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; + + dma_sync_single_for_device(dev, edesc->dma_link_tbl, + edesc->dma_len, DMA_BIDIRECTIONAL); + } else { + if (areq->assoclen) + to_talitos_ptr(&desc->ptr[1], + sg_dma_address(areq->assoc)); + else + to_talitos_ptr(&desc->ptr[1], edesc->iv_dma); + desc->ptr[1].j_extent = 0; + } + /* cipher iv */ - map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0, - DMA_TO_DEVICE); + to_talitos_ptr(&desc->ptr[2], edesc->iv_dma); + desc->ptr[2].len = cpu_to_be16(ivsize); + desc->ptr[2].j_extent = 0; + /* Sync needed for the aead_givencrypt case */ + dma_sync_single_for_device(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen, @@ -1072,7 +1014,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, - edesc->src_is_chained); + edesc->src_chained); if (sg_count == 1) { to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src)); @@ -1104,32 +1046,30 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, if (areq->src != areq->dst) sg_count = talitos_map_sg(dev, areq->dst, edesc->dst_nents ? : 1, - DMA_FROM_DEVICE, - edesc->dst_is_chained); + DMA_FROM_DEVICE, edesc->dst_chained); if (sg_count == 1) { to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst)); } else { - struct talitos_ptr *link_tbl_ptr = - &edesc->link_tbl[edesc->src_nents + 1]; + int tbl_off = edesc->src_nents + 1; + struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl + - (edesc->src_nents + 1) * - sizeof(struct talitos_ptr)); + tbl_off * sizeof(struct talitos_ptr)); sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, - link_tbl_ptr); + tbl_ptr); /* Add an entry to the link table for ICV data */ - link_tbl_ptr += sg_count - 1; - link_tbl_ptr->j_extent = 0; - sg_count++; - link_tbl_ptr++; - link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; - link_tbl_ptr->len = cpu_to_be16(authsize); + tbl_ptr += sg_count - 1; + tbl_ptr->j_extent = 0; + tbl_ptr++; + tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN; + tbl_ptr->len = cpu_to_be16(authsize); /* icv data follows link tables */ - to_talitos_ptr(link_tbl_ptr, edesc->dma_link_tbl + - (edesc->src_nents + edesc->dst_nents + 2) * + to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + + (tbl_off + edesc->dst_nents + 1 + + edesc->assoc_nents) * sizeof(struct talitos_ptr)); desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, @@ -1151,96 +1091,43 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, /* * derive number of elements in scatterlist */ -static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained) +static int sg_count(struct scatterlist *sg_list, int nbytes, bool *chained) { struct scatterlist *sg = sg_list; int sg_nents = 0; - *chained = 0; + *chained = false; while (nbytes > 0) { sg_nents++; nbytes -= sg->length; if (!sg_is_last(sg) && (sg + 1)->length == 0) - *chained = 1; + *chained = true; sg = scatterwalk_sg_next(sg); } return sg_nents; } -/** - * sg_copy_end_to_buffer - Copy end data from SG list to a linear buffer - * @sgl: The SG list - * @nents: Number of SG entries - * @buf: Where to copy to - * @buflen: The number of bytes to copy - * @skip: The number of bytes to skip before copying. - * Note: skip + buflen should equal SG total size. - * - * Returns the number of copied bytes. - * - **/ -static size_t sg_copy_end_to_buffer(struct scatterlist *sgl, unsigned int nents, - void *buf, size_t buflen, unsigned int skip) -{ - unsigned int offset = 0; - unsigned int boffset = 0; - struct sg_mapping_iter miter; - unsigned long flags; - unsigned int sg_flags = SG_MITER_ATOMIC; - size_t total_buffer = buflen + skip; - - sg_flags |= SG_MITER_FROM_SG; - - sg_miter_start(&miter, sgl, nents, sg_flags); - - local_irq_save(flags); - - while (sg_miter_next(&miter) && offset < total_buffer) { - unsigned int len; - unsigned int ignore; - - if ((offset + miter.length) > skip) { - if (offset < skip) { - /* Copy part of this segment */ - ignore = skip - offset; - len = miter.length - ignore; - if (boffset + len > buflen) - len = buflen - boffset; - memcpy(buf + boffset, miter.addr + ignore, len); - } else { - /* Copy all of this segment (up to buflen) */ - len = miter.length; - if (boffset + len > buflen) - len = buflen - boffset; - memcpy(buf + boffset, miter.addr, len); - } - boffset += len; - } - offset += miter.length; - } - - sg_miter_stop(&miter); - - local_irq_restore(flags); - return boffset; -} - /* * allocate and map the extended descriptor */ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, + struct scatterlist *assoc, struct scatterlist *src, struct scatterlist *dst, - int hash_result, + u8 *iv, + unsigned int assoclen, unsigned int cryptlen, unsigned int authsize, + unsigned int ivsize, int icv_stashing, - u32 cryptoflags) + u32 cryptoflags, + bool encrypt) { struct talitos_edesc *edesc; - int src_nents, dst_nents, alloc_len, dma_len; - int src_chained, dst_chained = 0; + int assoc_nents = 0, src_nents, dst_nents, alloc_len, dma_len; + bool assoc_chained = false, src_chained = false, dst_chained = false; + dma_addr_t iv_dma = 0; gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -1249,19 +1136,36 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, return ERR_PTR(-EINVAL); } - src_nents = sg_count(src, cryptlen + authsize, &src_chained); - src_nents = (src_nents == 1) ? 0 : src_nents; + if (ivsize) + iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); - if (hash_result) { - dst_nents = 0; - } else { - if (dst == src) { - dst_nents = src_nents; - } else { - dst_nents = sg_count(dst, cryptlen + authsize, - &dst_chained); - dst_nents = (dst_nents == 1) ? 0 : dst_nents; - } + if (assoclen) { + /* + * Currently it is assumed that iv is provided whenever assoc + * is. + */ + BUG_ON(!iv); + + assoc_nents = sg_count(assoc, assoclen, &assoc_chained); + talitos_map_sg(dev, assoc, assoc_nents, DMA_TO_DEVICE, + assoc_chained); + assoc_nents = (assoc_nents == 1) ? 0 : assoc_nents; + + if (assoc_nents || sg_dma_address(assoc) + assoclen != iv_dma) + assoc_nents = assoc_nents ? assoc_nents + 1 : 2; + } + + if (!dst || dst == src) { + src_nents = sg_count(src, cryptlen + authsize, &src_chained); + src_nents = (src_nents == 1) ? 0 : src_nents; + dst_nents = dst ? src_nents : 0; + } else { /* dst && dst != src*/ + src_nents = sg_count(src, cryptlen + (encrypt ? 0 : authsize), + &src_chained); + src_nents = (src_nents == 1) ? 0 : src_nents; + dst_nents = sg_count(dst, cryptlen + (encrypt ? authsize : 0), + &dst_chained); + dst_nents = (dst_nents == 1) ? 0 : dst_nents; } /* @@ -1270,9 +1174,9 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, * and the ICV data itself */ alloc_len = sizeof(struct talitos_edesc); - if (src_nents || dst_nents) { - dma_len = (src_nents + dst_nents + 2) * - sizeof(struct talitos_ptr) + authsize; + if (assoc_nents || src_nents || dst_nents) { + dma_len = (src_nents + dst_nents + 2 + assoc_nents) * + sizeof(struct talitos_ptr) + authsize; alloc_len += dma_len; } else { dma_len = 0; @@ -1281,14 +1185,27 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc = kmalloc(alloc_len, GFP_DMA | flags); if (!edesc) { + if (assoc_chained) + talitos_unmap_sg_chain(dev, assoc, DMA_TO_DEVICE); + else if (assoclen) + dma_unmap_sg(dev, assoc, + assoc_nents ? assoc_nents - 1 : 1, + DMA_TO_DEVICE); + + if (iv_dma) + dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dev_err(dev, "could not allocate edescriptor\n"); return ERR_PTR(-ENOMEM); } + edesc->assoc_nents = assoc_nents; edesc->src_nents = src_nents; edesc->dst_nents = dst_nents; - edesc->src_is_chained = src_chained; - edesc->dst_is_chained = dst_chained; + edesc->assoc_chained = assoc_chained; + edesc->src_chained = src_chained; + edesc->dst_chained = dst_chained; + edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; if (dma_len) edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], @@ -1298,15 +1215,17 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, return edesc; } -static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, - int icv_stashing) +static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, + int icv_stashing, bool encrypt) { struct crypto_aead *authenc = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); + unsigned int ivsize = crypto_aead_ivsize(authenc); - return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0, - areq->cryptlen, ctx->authsize, icv_stashing, - areq->base.flags); + return talitos_edesc_alloc(ctx->dev, areq->assoc, areq->src, areq->dst, + iv, areq->assoclen, areq->cryptlen, + ctx->authsize, ivsize, icv_stashing, + areq->base.flags, encrypt); } static int aead_encrypt(struct aead_request *req) @@ -1316,14 +1235,14 @@ static int aead_encrypt(struct aead_request *req) struct talitos_edesc *edesc; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, 0); + edesc = aead_edesc_alloc(req, req->iv, 0, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); /* set encrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; - return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done); + return ipsec_esp(edesc, req, 0, ipsec_esp_encrypt_done); } static int aead_decrypt(struct aead_request *req) @@ -1339,7 +1258,7 @@ static int aead_decrypt(struct aead_request *req) req->cryptlen -= authsize; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, 1); + edesc = aead_edesc_alloc(req, req->iv, 1, false); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1355,9 +1274,7 @@ static int aead_decrypt(struct aead_request *req) /* reset integrity check result bits */ edesc->desc.hdr_lo = 0; - return ipsec_esp(edesc, req, NULL, 0, - ipsec_esp_decrypt_hwauth_done); - + return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_hwauth_done); } /* Have to check the ICV with software */ @@ -1366,7 +1283,8 @@ static int aead_decrypt(struct aead_request *req) /* stash incoming ICV for later cmp with ICV generated by the h/w */ if (edesc->dma_len) icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; + edesc->dst_nents + 2 + + edesc->assoc_nents]; else icvdata = &edesc->link_tbl[0]; @@ -1375,7 +1293,7 @@ static int aead_decrypt(struct aead_request *req) memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize, ctx->authsize); - return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_swauth_done); + return ipsec_esp(edesc, req, 0, ipsec_esp_decrypt_swauth_done); } static int aead_givencrypt(struct aead_givcrypt_request *req) @@ -1386,7 +1304,7 @@ static int aead_givencrypt(struct aead_givcrypt_request *req) struct talitos_edesc *edesc; /* allocate extended descriptor */ - edesc = aead_edesc_alloc(areq, 0); + edesc = aead_edesc_alloc(areq, req->giv, 0, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1397,8 +1315,7 @@ static int aead_givencrypt(struct aead_givcrypt_request *req) /* avoid consecutive packets going out with same IV */ *(__be64 *)req->giv ^= cpu_to_be64(req->seq); - return ipsec_esp(edesc, areq, req->giv, req->seq, - ipsec_esp_encrypt_done); + return ipsec_esp(edesc, areq, req->seq, ipsec_esp_encrypt_done); } static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, @@ -1454,7 +1371,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; unsigned int cryptlen = areq->nbytes; - unsigned int ivsize; + unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); int sg_count, ret; /* first DWORD empty */ @@ -1463,9 +1380,9 @@ static int common_nonsnoop(struct talitos_edesc *edesc, desc->ptr[0].j_extent = 0; /* cipher iv */ - ivsize = crypto_ablkcipher_ivsize(cipher); - map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, areq->info, 0, - DMA_TO_DEVICE); + to_talitos_ptr(&desc->ptr[1], edesc->iv_dma); + desc->ptr[1].len = cpu_to_be16(ivsize); + desc->ptr[1].j_extent = 0; /* cipher key */ map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen, @@ -1480,7 +1397,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1, (areq->src == areq->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE, - edesc->src_is_chained); + edesc->src_chained); if (sg_count == 1) { to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src)); @@ -1507,8 +1424,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, if (areq->src != areq->dst) sg_count = talitos_map_sg(dev, areq->dst, edesc->dst_nents ? : 1, - DMA_FROM_DEVICE, - edesc->dst_is_chained); + DMA_FROM_DEVICE, edesc->dst_chained); if (sg_count == 1) { to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst)); @@ -1544,13 +1460,15 @@ static int common_nonsnoop(struct talitos_edesc *edesc, } static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request * - areq) + areq, bool encrypt) { struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); + unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); - return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0, - areq->nbytes, 0, 0, areq->base.flags); + return talitos_edesc_alloc(ctx->dev, NULL, areq->src, areq->dst, + areq->info, 0, areq->nbytes, 0, ivsize, 0, + areq->base.flags, encrypt); } static int ablkcipher_encrypt(struct ablkcipher_request *areq) @@ -1560,7 +1478,7 @@ static int ablkcipher_encrypt(struct ablkcipher_request *areq) struct talitos_edesc *edesc; /* allocate extended descriptor */ - edesc = ablkcipher_edesc_alloc(areq); + edesc = ablkcipher_edesc_alloc(areq, true); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1577,7 +1495,7 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq) struct talitos_edesc *edesc; /* allocate extended descriptor */ - edesc = ablkcipher_edesc_alloc(areq); + edesc = ablkcipher_edesc_alloc(areq, false); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1676,8 +1594,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, sg_count = talitos_map_sg(dev, req_ctx->psrc, edesc->src_nents ? : 1, - DMA_TO_DEVICE, - edesc->src_is_chained); + DMA_TO_DEVICE, edesc->src_chained); if (sg_count == 1) { to_talitos_ptr(&desc->ptr[3], sg_dma_address(req_ctx->psrc)); @@ -1729,8 +1646,8 @@ static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq, struct talitos_ctx *ctx = crypto_ahash_ctx(tfm); struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq); - return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, 1, - nbytes, 0, 0, areq->base.flags); + return talitos_edesc_alloc(ctx->dev, NULL, req_ctx->psrc, NULL, NULL, 0, + nbytes, 0, 0, 0, areq->base.flags, false); } static int ahash_init(struct ahash_request *areq) @@ -1788,7 +1705,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) unsigned int nbytes_to_hash; unsigned int to_hash_later; unsigned int nsg; - int chained; + bool chained; if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { /* Buffer up to one whole block */ @@ -1827,7 +1744,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) if (to_hash_later) { int nents = sg_count(areq->src, nbytes, &chained); - sg_copy_end_to_buffer(areq->src, nents, + sg_pcopy_to_buffer(areq->src, nents, req_ctx->bufnext, to_hash_later, nbytes - to_hash_later); @@ -2007,14 +1924,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_aead_type, .cra_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "<built-in>", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, } @@ -2033,14 +1943,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_aead_type, .cra_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "<built-in>", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, } @@ -2054,20 +1957,52 @@ static struct talitos_alg_template driver_algs[] = { DESC_HDR_MODE1_MDEU_PAD | DESC_HDR_MODE1_MDEU_SHA1_HMAC, }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.crypto = { + .cra_name = "authenc(hmac(sha224),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha224-cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_aead = { + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.crypto = { + .cra_name = "authenc(hmac(sha224),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha224-cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_aead = { + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA224_DIGEST_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUA | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEU_SHA224_HMAC, + }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.crypto = { .cra_name = "authenc(hmac(sha256),cbc(aes))", .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_aead_type, .cra_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "<built-in>", .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, } @@ -2086,14 +2021,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_aead_type, .cra_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "<built-in>", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, } @@ -2109,18 +2037,89 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AEAD, .alg.crypto = { + .cra_name = "authenc(hmac(sha384),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha384-cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_aead = { + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUB | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEUB_SHA384_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.crypto = { + .cra_name = "authenc(hmac(sha384),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha384-cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_aead = { + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA384_DIGEST_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUB | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEUB_SHA384_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.crypto = { + .cra_name = "authenc(hmac(sha512),cbc(aes))", + .cra_driver_name = "authenc-hmac-sha512-cbc-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_aead = { + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CBC | + DESC_HDR_SEL1_MDEUB | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEUB_SHA512_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.crypto = { + .cra_name = "authenc(hmac(sha512),cbc(des3_ede))", + .cra_driver_name = "authenc-hmac-sha512-cbc-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, + .cra_aead = { + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA512_DIGEST_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC | + DESC_HDR_MODE0_DEU_3DES | + DESC_HDR_SEL1_MDEUB | + DESC_HDR_MODE1_MDEU_INIT | + DESC_HDR_MODE1_MDEU_PAD | + DESC_HDR_MODE1_MDEUB_SHA512_HMAC, + }, + { .type = CRYPTO_ALG_TYPE_AEAD, + .alg.crypto = { .cra_name = "authenc(hmac(md5),cbc(aes))", .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_aead_type, .cra_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "<built-in>", .ivsize = AES_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, } @@ -2139,14 +2138,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_aead_type, .cra_aead = { - .setkey = aead_setkey, - .setauthsize = aead_setauthsize, - .encrypt = aead_encrypt, - .decrypt = aead_decrypt, - .givencrypt = aead_givencrypt, - .geniv = "<built-in>", .ivsize = DES3_EDE_BLOCK_SIZE, .maxauthsize = MD5_DIGEST_SIZE, } @@ -2168,12 +2160,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ablkcipher_type, .cra_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .geniv = "eseqiv", .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, @@ -2190,12 +2177,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ablkcipher_type, .cra_ablkcipher = { - .setkey = ablkcipher_setkey, - .encrypt = ablkcipher_encrypt, - .decrypt = ablkcipher_decrypt, - .geniv = "eseqiv", .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, .ivsize = DES3_EDE_BLOCK_SIZE, @@ -2209,11 +2191,6 @@ static struct talitos_alg_template driver_algs[] = { /* AHASH algorithms. */ { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, .halg.digestsize = MD5_DIGEST_SIZE, .halg.base = { .cra_name = "md5", @@ -2221,7 +2198,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = MD5_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2230,11 +2206,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, .halg.digestsize = SHA1_DIGEST_SIZE, .halg.base = { .cra_name = "sha1", @@ -2242,7 +2213,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA1_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2251,11 +2221,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, .halg.digestsize = SHA224_DIGEST_SIZE, .halg.base = { .cra_name = "sha224", @@ -2263,7 +2228,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA224_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2272,11 +2236,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, .halg.digestsize = SHA256_DIGEST_SIZE, .halg.base = { .cra_name = "sha256", @@ -2284,7 +2243,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA256_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2293,11 +2251,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, .halg.digestsize = SHA384_DIGEST_SIZE, .halg.base = { .cra_name = "sha384", @@ -2305,7 +2258,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA384_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2314,11 +2266,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, .halg.digestsize = SHA512_DIGEST_SIZE, .halg.base = { .cra_name = "sha512", @@ -2326,7 +2273,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA512_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2335,12 +2281,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, - .setkey = ahash_setkey, .halg.digestsize = MD5_DIGEST_SIZE, .halg.base = { .cra_name = "hmac(md5)", @@ -2348,7 +2288,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = MD5_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2357,12 +2296,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, - .setkey = ahash_setkey, .halg.digestsize = SHA1_DIGEST_SIZE, .halg.base = { .cra_name = "hmac(sha1)", @@ -2370,7 +2303,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA1_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2379,12 +2311,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, - .setkey = ahash_setkey, .halg.digestsize = SHA224_DIGEST_SIZE, .halg.base = { .cra_name = "hmac(sha224)", @@ -2392,7 +2318,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA224_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2401,12 +2326,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, - .setkey = ahash_setkey, .halg.digestsize = SHA256_DIGEST_SIZE, .halg.base = { .cra_name = "hmac(sha256)", @@ -2414,7 +2333,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA256_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2423,12 +2341,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, - .setkey = ahash_setkey, .halg.digestsize = SHA384_DIGEST_SIZE, .halg.base = { .cra_name = "hmac(sha384)", @@ -2436,7 +2348,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA384_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2445,12 +2356,6 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_AHASH, .alg.hash = { - .init = ahash_init, - .update = ahash_update, - .final = ahash_final, - .finup = ahash_finup, - .digest = ahash_digest, - .setkey = ahash_setkey, .halg.digestsize = SHA512_DIGEST_SIZE, .halg.base = { .cra_name = "hmac(sha512)", @@ -2458,7 +2363,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_blocksize = SHA512_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_type = &crypto_ahash_type } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2591,8 +2495,6 @@ static int talitos_remove(struct platform_device *ofdev) iounmap(priv->reg); - dev_set_drvdata(dev, NULL); - kfree(priv); return 0; @@ -2616,14 +2518,34 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, case CRYPTO_ALG_TYPE_ABLKCIPHER: alg = &t_alg->algt.alg.crypto; alg->cra_init = talitos_cra_init; + alg->cra_type = &crypto_ablkcipher_type; + alg->cra_ablkcipher.setkey = ablkcipher_setkey; + alg->cra_ablkcipher.encrypt = ablkcipher_encrypt; + alg->cra_ablkcipher.decrypt = ablkcipher_decrypt; + alg->cra_ablkcipher.geniv = "eseqiv"; break; case CRYPTO_ALG_TYPE_AEAD: alg = &t_alg->algt.alg.crypto; alg->cra_init = talitos_cra_init_aead; + alg->cra_type = &crypto_aead_type; + alg->cra_aead.setkey = aead_setkey; + alg->cra_aead.setauthsize = aead_setauthsize; + alg->cra_aead.encrypt = aead_encrypt; + alg->cra_aead.decrypt = aead_decrypt; + alg->cra_aead.givencrypt = aead_givencrypt; + alg->cra_aead.geniv = "<built-in>"; break; case CRYPTO_ALG_TYPE_AHASH: alg = &t_alg->algt.alg.hash.halg.base; alg->cra_init = talitos_cra_init_ahash; + alg->cra_type = &crypto_ahash_type; + t_alg->algt.alg.hash.init = ahash_init; + t_alg->algt.alg.hash.update = ahash_update; + t_alg->algt.alg.hash.final = ahash_final; + t_alg->algt.alg.hash.finup = ahash_finup; + t_alg->algt.alg.hash.digest = ahash_digest; + t_alg->algt.alg.hash.setkey = ahash_setkey; + if (!(priv->features & TALITOS_FTR_HMAC_OK) && !strncmp(alg->cra_name, "hmac", 4)) { kfree(t_alg); @@ -2648,6 +2570,7 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, alg->cra_priority = TALITOS_CRA_PRIORITY; alg->cra_alignmask = 0; alg->cra_ctxsize = sizeof(struct talitos_ctx); + alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; t_alg->dev = dev; @@ -2714,10 +2637,14 @@ static int talitos_probe(struct platform_device *ofdev) if (!priv) return -ENOMEM; + INIT_LIST_HEAD(&priv->alg_list); + dev_set_drvdata(dev, priv); priv->ofdev = ofdev; + spin_lock_init(&priv->reg_lock); + err = talitos_probe_irq(ofdev); if (err) goto err_out; @@ -2732,8 +2659,6 @@ static int talitos_probe(struct platform_device *ofdev) (unsigned long)dev); } - INIT_LIST_HEAD(&priv->alg_list); - priv->reg = of_iomap(np, 0); if (!priv->reg) { dev_err(dev, "failed to of_iomap\n"); |