diff options
Diffstat (limited to 'drivers/crypto/hifn_795x.c')
| -rw-r--r-- | drivers/crypto/hifn_795x.c | 860 |
1 files changed, 412 insertions, 448 deletions
diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 81f3f950cd7..12fea3e2234 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -29,7 +29,6 @@ #include <linux/dma-mapping.h> #include <linux/scatterlist.h> #include <linux/highmem.h> -#include <linux/interrupt.h> #include <linux/crypto.h> #include <linux/hw_random.h> #include <linux/ktime.h> @@ -39,9 +38,6 @@ #include <asm/kmap_types.h> -#undef dprintk - -#define HIFN_TEST //#define HIFN_DEBUG #ifdef HIFN_DEBUG @@ -325,7 +321,7 @@ static atomic_t hifn_dev_number; #define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */ #define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */ #define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */ -#define HIFN_PUBOPLEN_EXP_S 7 /* exponent lenght shift */ +#define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */ #define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */ #define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */ @@ -364,12 +360,14 @@ static atomic_t hifn_dev_number; #define HIFN_NAMESIZE 32 #define HIFN_MAX_RESULT_ORDER 5 -#define HIFN_D_CMD_RSIZE 24*4 -#define HIFN_D_SRC_RSIZE 80*4 -#define HIFN_D_DST_RSIZE 80*4 -#define HIFN_D_RES_RSIZE 24*4 +#define HIFN_D_CMD_RSIZE 24*1 +#define HIFN_D_SRC_RSIZE 80*1 +#define HIFN_D_DST_RSIZE 80*1 +#define HIFN_D_RES_RSIZE 24*1 + +#define HIFN_D_DST_DALIGN 4 -#define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-5 +#define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1) #define AES_MIN_KEY_SIZE 16 #define AES_MAX_KEY_SIZE 32 @@ -405,8 +403,6 @@ struct hifn_dma { u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND]; u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT]; - u64 test_src, test_dst; - /* * Our current positions for insertion and removal from the descriptor * rings. @@ -433,9 +429,6 @@ struct hifn_device struct pci_dev *pdev; void __iomem *bar[3]; - unsigned long result_mem; - dma_addr_t dst; - void *desc_virt; dma_addr_t desc_dma; @@ -445,8 +438,6 @@ struct hifn_device spinlock_t lock; - void *priv; - u32 flags; int active, started; struct delayed_work work; @@ -535,10 +526,10 @@ struct hifn_crypt_command */ struct hifn_mac_command { - volatile u16 masks; - volatile u16 header_skip; - volatile u16 source_count; - volatile u16 reserved; + volatile __le16 masks; + volatile __le16 header_skip; + volatile __le16 source_count; + volatile __le16 reserved; }; #define HIFN_MAC_CMD_ALG_MASK 0x0001 @@ -564,10 +555,10 @@ struct hifn_mac_command struct hifn_comp_command { - volatile u16 masks; - volatile u16 header_skip; - volatile u16 source_count; - volatile u16 reserved; + volatile __le16 masks; + volatile __le16 header_skip; + volatile __le16 source_count; + volatile __le16 reserved; }; #define HIFN_COMP_CMD_SRCLEN_M 0xc000 @@ -583,10 +574,10 @@ struct hifn_comp_command struct hifn_base_result { - volatile u16 flags; - volatile u16 session; - volatile u16 src_cnt; /* 15:0 of source count */ - volatile u16 dst_cnt; /* 15:0 of dest count */ + volatile __le16 flags; + volatile __le16 session; + volatile __le16 src_cnt; /* 15:0 of source count */ + volatile __le16 dst_cnt; /* 15:0 of dest count */ }; #define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */ @@ -597,8 +588,8 @@ struct hifn_base_result struct hifn_comp_result { - volatile u16 flags; - volatile u16 crc; + volatile __le16 flags; + volatile __le16 crc; }; #define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */ @@ -609,8 +600,8 @@ struct hifn_comp_result struct hifn_mac_result { - volatile u16 flags; - volatile u16 reserved; + volatile __le16 flags; + volatile __le16 reserved; /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */ }; @@ -619,8 +610,8 @@ struct hifn_mac_result struct hifn_crypt_result { - volatile u16 flags; - volatile u16 reserved; + volatile __le16 flags; + volatile __le16 reserved; }; #define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */ @@ -647,7 +638,7 @@ struct hifn_crypto_alg #define ASYNC_FLAGS_MISALIGNED (1<<0) -struct ablkcipher_walk +struct hifn_cipher_walk { struct scatterlist cache[ASYNC_SCATTERLIST_CACHE]; u32 flags; @@ -656,12 +647,17 @@ struct ablkcipher_walk struct hifn_context { - u8 key[HIFN_MAX_CRYPT_KEY_LENGTH], *iv; + u8 key[HIFN_MAX_CRYPT_KEY_LENGTH]; struct hifn_device *dev; - unsigned int keysize, ivsize; + unsigned int keysize; +}; + +struct hifn_request_context +{ + u8 *iv; + unsigned int ivsize; u8 op, type, mode, unused; - struct ablkcipher_walk walk; - atomic_t sg_num; + struct hifn_cipher_walk walk; }; #define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg) @@ -686,12 +682,12 @@ static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg) static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val) { - writel(val, dev->bar[0] + reg); + writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg); } static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val) { - writel(val, dev->bar[1] + reg); + writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg); } static void hifn_wait_puc(struct hifn_device *dev) @@ -825,8 +821,8 @@ static int hifn_register_rng(struct hifn_device *dev) /* * We must wait at least 256 Pk_clk cycles between two reads of the rng. */ - dev->rng_wait_time = DIV_ROUND_UP(NSEC_PER_SEC, dev->pk_clk_freq) * - 256; + dev->rng_wait_time = DIV_ROUND_UP_ULL(NSEC_PER_SEC, + dev->pk_clk_freq) * 256; dev->rng.name = dev->name; dev->rng.data_present = hifn_rng_data_present, @@ -867,7 +863,7 @@ static int hifn_init_pubrng(struct hifn_device *dev) dev->dmareg |= HIFN_DMAIER_PUBDONE; hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); - dprintk("Chip %s: Public key engine has been sucessfully " + dprintk("Chip %s: Public key engine has been successfully " "initialised.\n", dev->name); } @@ -894,7 +890,7 @@ static int hifn_enable_crypto(struct hifn_device *dev) char *offtbl = NULL; int i; - for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) { + for (i = 0; i < ARRAY_SIZE(pci2id); i++) { if (pci2id[i].pci_vendor == dev->pdev->vendor && pci2id[i].pci_prod == dev->pdev->device) { offtbl = pci2id[i].card_id; @@ -1037,14 +1033,14 @@ static void hifn_init_registers(struct hifn_device *dev) hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); /* write all 4 ring address registers */ - hifn_write_1(dev, HIFN_1_DMA_CRAR, __cpu_to_le32(dptr + - offsetof(struct hifn_dma, cmdr[0]))); - hifn_write_1(dev, HIFN_1_DMA_SRAR, __cpu_to_le32(dptr + - offsetof(struct hifn_dma, srcr[0]))); - hifn_write_1(dev, HIFN_1_DMA_DRAR, __cpu_to_le32(dptr + - offsetof(struct hifn_dma, dstr[0]))); - hifn_write_1(dev, HIFN_1_DMA_RRAR, __cpu_to_le32(dptr + - offsetof(struct hifn_dma, resr[0]))); + hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr + + offsetof(struct hifn_dma, cmdr[0])); + hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr + + offsetof(struct hifn_dma, srcr[0])); + hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr + + offsetof(struct hifn_dma, dstr[0])); + hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr + + offsetof(struct hifn_dma, resr[0])); mdelay(2); #if 0 @@ -1166,113 +1162,20 @@ static int hifn_setup_crypto_command(struct hifn_device *dev, return cmd_len; } -static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, - unsigned int offset, unsigned int size) -{ - struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - int idx; - dma_addr_t addr; - - addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE); - - idx = dma->srci; - - dma->srcr[idx].p = __cpu_to_le32(addr); - dma->srcr[idx].l = __cpu_to_le32(size) | HIFN_D_VALID | - HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST; - - if (++idx == HIFN_D_SRC_RSIZE) { - dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | - HIFN_D_JUMP | - HIFN_D_MASKDONEIRQ | HIFN_D_LAST); - idx = 0; - } - - dma->srci = idx; - dma->srcu++; - - if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { - hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); - dev->flags |= HIFN_FLAG_SRC_BUSY; - } - - return size; -} - -static void hifn_setup_res_desc(struct hifn_device *dev) -{ - struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - - dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | - HIFN_D_VALID | HIFN_D_LAST); - /* - * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | - * HIFN_D_LAST | HIFN_D_NOINVALID); - */ - - if (++dma->resi == HIFN_D_RES_RSIZE) { - dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | - HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST); - dma->resi = 0; - } - - dma->resu++; - - if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { - hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); - dev->flags |= HIFN_FLAG_RES_BUSY; - } -} - -static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, - unsigned offset, unsigned size) -{ - struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - int idx; - dma_addr_t addr; - - addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE); - - idx = dma->dsti; - dma->dstr[idx].p = __cpu_to_le32(addr); - dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | - HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST); - - if (++idx == HIFN_D_DST_RSIZE) { - dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | - HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | - HIFN_D_LAST | HIFN_D_NOINVALID); - idx = 0; - } - dma->dsti = idx; - dma->dstu++; - - if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { - hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); - dev->flags |= HIFN_FLAG_DST_BUSY; - } -} - -static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff, - struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv, - struct hifn_context *ctx) +static int hifn_setup_cmd_desc(struct hifn_device *dev, + struct hifn_context *ctx, struct hifn_request_context *rctx, + void *priv, unsigned int nbytes) { struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int cmd_len, sa_idx; u8 *buf, *buf_pos; u16 mask; - dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n", - dev->name, spage, soff, dpage, doff, nbytes, priv, ctx); - - sa_idx = dma->resi; - - hifn_setup_src_desc(dev, spage, soff, nbytes); - + sa_idx = dma->cmdi; buf_pos = buf = dma->command_bufs[dma->cmdi]; mask = 0; - switch (ctx->op) { + switch (rctx->op) { case ACRYPTO_OP_DECRYPT: mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; break; @@ -1289,15 +1192,15 @@ static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes, nbytes, mask, dev->snum); - if (ctx->op == ACRYPTO_OP_ENCRYPT || ctx->op == ACRYPTO_OP_DECRYPT) { + if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) { u16 md = 0; if (ctx->keysize) md |= HIFN_CRYPT_CMD_NEW_KEY; - if (ctx->iv && ctx->mode != ACRYPTO_MODE_ECB) + if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB) md |= HIFN_CRYPT_CMD_NEW_IV; - switch (ctx->mode) { + switch (rctx->mode) { case ACRYPTO_MODE_ECB: md |= HIFN_CRYPT_CMD_MODE_ECB; break; @@ -1314,7 +1217,7 @@ static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned goto err_out; } - switch (ctx->type) { + switch (rctx->type) { case ACRYPTO_TYPE_AES_128: if (ctx->keysize != 16) goto err_out; @@ -1349,17 +1252,18 @@ static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned buf_pos += hifn_setup_crypto_command(dev, buf_pos, nbytes, nbytes, ctx->key, ctx->keysize, - ctx->iv, ctx->ivsize, md); + rctx->iv, rctx->ivsize, md); } dev->sa[sa_idx] = priv; + dev->started++; cmd_len = buf_pos - buf; dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ); if (++dma->cmdi == HIFN_D_CMD_RSIZE) { - dma->cmdr[dma->cmdi].l = __cpu_to_le32(HIFN_MAX_COMMAND | + dma->cmdr[dma->cmdi].l = __cpu_to_le32( HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ | HIFN_D_JUMP); dma->cmdi = 0; @@ -1370,17 +1274,150 @@ static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); dev->flags |= HIFN_FLAG_CMD_BUSY; } - - hifn_setup_dst_desc(dev, dpage, doff, nbytes); - hifn_setup_res_desc(dev); - return 0; err_out: return -EINVAL; } -static int ablkcipher_walk_init(struct ablkcipher_walk *w, +static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, + unsigned int offset, unsigned int size, int last) +{ + struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; + int idx; + dma_addr_t addr; + + addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE); + + idx = dma->srci; + + dma->srcr[idx].p = __cpu_to_le32(addr); + dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | + HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); + + if (++idx == HIFN_D_SRC_RSIZE) { + dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | + (last ? HIFN_D_LAST : 0)); + idx = 0; + } + + dma->srci = idx; + dma->srcu++; + + if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); + dev->flags |= HIFN_FLAG_SRC_BUSY; + } + + return size; +} + +static void hifn_setup_res_desc(struct hifn_device *dev) +{ + struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; + + dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | + HIFN_D_VALID | HIFN_D_LAST); + /* + * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | + * HIFN_D_LAST); + */ + + if (++dma->resi == HIFN_D_RES_RSIZE) { + dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST); + dma->resi = 0; + } + + dma->resu++; + + if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); + dev->flags |= HIFN_FLAG_RES_BUSY; + } +} + +static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, + unsigned offset, unsigned size, int last) +{ + struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; + int idx; + dma_addr_t addr; + + addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE); + + idx = dma->dsti; + dma->dstr[idx].p = __cpu_to_le32(addr); + dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | + HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); + + if (++idx == HIFN_D_DST_RSIZE) { + dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | + (last ? HIFN_D_LAST : 0)); + idx = 0; + } + dma->dsti = idx; + dma->dstu++; + + if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); + dev->flags |= HIFN_FLAG_DST_BUSY; + } +} + +static int hifn_setup_dma(struct hifn_device *dev, + struct hifn_context *ctx, struct hifn_request_context *rctx, + struct scatterlist *src, struct scatterlist *dst, + unsigned int nbytes, void *priv) +{ + struct scatterlist *t; + struct page *spage, *dpage; + unsigned int soff, doff; + unsigned int n, len; + + n = nbytes; + while (n) { + spage = sg_page(src); + soff = src->offset; + len = min(src->length, n); + + hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); + + src++; + n -= len; + } + + t = &rctx->walk.cache[0]; + n = nbytes; + while (n) { + if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + BUG_ON(!sg_page(t)); + dpage = sg_page(t); + doff = 0; + len = t->length; + } else { + BUG_ON(!sg_page(dst)); + dpage = sg_page(dst); + doff = dst->offset; + len = dst->length; + } + len = min(len, n); + + hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); + + dst++; + t++; + n -= len; + } + + hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes); + hifn_setup_res_desc(dev); + return 0; +} + +static int hifn_cipher_walk_init(struct hifn_cipher_walk *w, int num, gfp_t gfp_flags) { int i; @@ -1405,7 +1442,7 @@ static int ablkcipher_walk_init(struct ablkcipher_walk *w, return i; } -static void ablkcipher_walk_exit(struct ablkcipher_walk *w) +static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w) { int i; @@ -1420,32 +1457,26 @@ static void ablkcipher_walk_exit(struct ablkcipher_walk *w) w->num = 0; } -static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist *src, +static int ablkcipher_add(unsigned int *drestp, struct scatterlist *dst, unsigned int size, unsigned int *nbytesp) { unsigned int copy, drest = *drestp, nbytes = *nbytesp; int idx = 0; - void *saddr; if (drest < size || size > nbytes) return -EINVAL; while (size) { - copy = min(drest, src->length); - - saddr = kmap_atomic(sg_page(src), KM_SOFTIRQ1); - memcpy(daddr, saddr + src->offset, copy); - kunmap_atomic(saddr, KM_SOFTIRQ1); + copy = min3(drest, size, dst->length); size -= copy; drest -= copy; nbytes -= copy; - daddr += copy; dprintk("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", __func__, copy, size, drest, nbytes); - src++; + dst++; idx++; } @@ -1455,15 +1486,10 @@ static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist return idx; } -static int ablkcipher_walk(struct ablkcipher_request *req, - struct ablkcipher_walk *w) +static int hifn_cipher_walk(struct ablkcipher_request *req, + struct hifn_cipher_walk *w) { - unsigned blocksize = - crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req)); - unsigned alignmask = - crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req)); - struct scatterlist *src, *dst, *t; - void *daddr; + struct scatterlist *dst, *t; unsigned int nbytes = req->nbytes, offset, copy, diff; int idx, tidx, err; @@ -1473,33 +1499,27 @@ static int ablkcipher_walk(struct ablkcipher_request *req, if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED)) return -EINVAL; - src = &req->src[idx]; dst = &req->dst[idx]; - dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, " - "blocksize: %u, nbytes: %u.\n", - __func__, src->length, dst->length, src->offset, - dst->offset, offset, blocksize, nbytes); - - if (src->length & (blocksize - 1) || - src->offset & (alignmask - 1) || - dst->length & (blocksize - 1) || - dst->offset & (alignmask - 1) || - offset) { - unsigned slen = src->length - offset; + dprintk("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", + __func__, dst->length, dst->offset, offset, nbytes); + + if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || + !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || + offset) { + unsigned slen = min(dst->length - offset, nbytes); unsigned dlen = PAGE_SIZE; t = &w->cache[idx]; - daddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0); - err = ablkcipher_add(daddr, &dlen, src, slen, &nbytes); + err = ablkcipher_add(&dlen, dst, slen, &nbytes); if (err < 0) - goto err_out_unmap; + return err; idx += err; - copy = slen & ~(blocksize - 1); - diff = slen & (blocksize - 1); + copy = slen & ~(HIFN_D_DST_DALIGN - 1); + diff = slen & (HIFN_D_DST_DALIGN - 1); if (dlen < nbytes) { /* @@ -1507,7 +1527,7 @@ static int ablkcipher_walk(struct ablkcipher_request *req, * to put there additional blocksized chunk, * so we mark that page as containing only * blocksize aligned chunks: - * t->length = (slen & ~(blocksize - 1)); + * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); * and increase number of bytes to be processed * in next chunk: * nbytes += diff; @@ -1530,21 +1550,19 @@ static int ablkcipher_walk(struct ablkcipher_request *req, } else { copy += diff + nbytes; - src = &req->src[idx]; + dst = &req->dst[idx]; - err = ablkcipher_add(daddr + slen, &dlen, src, nbytes, &nbytes); + err = ablkcipher_add(&dlen, dst, nbytes, &nbytes); if (err < 0) - goto err_out_unmap; + return err; idx += err; } t->length = copy; t->offset = offset; - - kunmap_atomic(daddr, KM_SOFTIRQ0); } else { - nbytes -= src->length; + nbytes -= min(dst->length, nbytes); idx++; } @@ -1552,58 +1570,46 @@ static int ablkcipher_walk(struct ablkcipher_request *req, } return tidx; - -err_out_unmap: - kunmap_atomic(daddr, KM_SOFTIRQ0); - return err; } static int hifn_setup_session(struct ablkcipher_request *req) { struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); struct hifn_device *dev = ctx->dev; - struct page *spage, *dpage; - unsigned long soff, doff, flags; - unsigned int nbytes = req->nbytes, idx = 0, len; + unsigned long dlen, flags; + unsigned int nbytes = req->nbytes, idx = 0; int err = -EINVAL, sg_num; - struct scatterlist *src, *dst, *t; - unsigned blocksize = - crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req)); - unsigned alignmask = - crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req)); + struct scatterlist *dst; - if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB) + if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB) goto err_out_exit; - ctx->walk.flags = 0; + rctx->walk.flags = 0; while (nbytes) { - src = &req->src[idx]; dst = &req->dst[idx]; + dlen = min(dst->length, nbytes); - if (src->length & (blocksize - 1) || - src->offset & (alignmask - 1) || - dst->length & (blocksize - 1) || - dst->offset & (alignmask - 1)) { - ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; - } + if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || + !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN)) + rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; - nbytes -= src->length; + nbytes -= dlen; idx++; } - if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { - err = ablkcipher_walk_init(&ctx->walk, idx, GFP_ATOMIC); + if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); if (err < 0) return err; } - nbytes = req->nbytes; - idx = 0; - - sg_num = ablkcipher_walk(req, &ctx->walk); - - atomic_set(&ctx->sg_num, sg_num); + sg_num = hifn_cipher_walk(req, &rctx->walk); + if (sg_num < 0) { + err = sg_num; + goto err_out_exit; + } spin_lock_irqsave(&dev->lock, flags); if (dev->started + sg_num > HIFN_QUEUE_LENGTH) { @@ -1611,37 +1617,11 @@ static int hifn_setup_session(struct ablkcipher_request *req) goto err_out; } - dev->snum++; - dev->started += sg_num; - - while (nbytes) { - src = &req->src[idx]; - dst = &req->dst[idx]; - t = &ctx->walk.cache[idx]; - - if (t->length) { - spage = dpage = sg_page(t); - soff = doff = 0; - len = t->length; - } else { - spage = sg_page(src); - soff = src->offset; - - dpage = sg_page(dst); - doff = dst->offset; - - len = dst->length; - } - - idx++; - - err = hifn_setup_dma(dev, spage, soff, dpage, doff, nbytes, - req, ctx); - if (err) - goto err_out; + err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->nbytes, req); + if (err) + goto err_out; - nbytes -= len; - } + dev->snum++; dev->active = HIFN_DEFAULT_ACTIVE_NUM; spin_unlock_irqrestore(&dev->lock, flags); @@ -1651,12 +1631,13 @@ static int hifn_setup_session(struct ablkcipher_request *req) err_out: spin_unlock_irqrestore(&dev->lock, flags); err_out_exit: - if (err && printk_ratelimit()) - dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, " + if (err) { + printk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, " "type: %u, err: %d.\n", - dev->name, ctx->iv, ctx->ivsize, + dev->name, rctx->iv, rctx->ivsize, ctx->key, ctx->keysize, - ctx->mode, ctx->op, ctx->type, err); + rctx->mode, rctx->op, rctx->type, err); + } return err; } @@ -1666,31 +1647,33 @@ static int hifn_test(struct hifn_device *dev, int encdec, u8 snum) int n, err; u8 src[16]; struct hifn_context ctx; + struct hifn_request_context rctx; u8 fips_aes_ecb_from_zero[16] = { 0x66, 0xE9, 0x4B, 0xD4, 0xEF, 0x8A, 0x2C, 0x3B, 0x88, 0x4C, 0xFA, 0x59, 0xCA, 0x34, 0x2B, 0x2E}; + struct scatterlist sg; memset(src, 0, sizeof(src)); memset(ctx.key, 0, sizeof(ctx.key)); ctx.dev = dev; ctx.keysize = 16; - ctx.ivsize = 0; - ctx.iv = NULL; - ctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT; - ctx.mode = ACRYPTO_MODE_ECB; - ctx.type = ACRYPTO_TYPE_AES_128; - atomic_set(&ctx.sg_num, 1); - - err = hifn_setup_dma(dev, - virt_to_page(src), offset_in_page(src), - virt_to_page(src), offset_in_page(src), - sizeof(src), NULL, &ctx); + rctx.ivsize = 0; + rctx.iv = NULL; + rctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT; + rctx.mode = ACRYPTO_MODE_ECB; + rctx.type = ACRYPTO_TYPE_AES_128; + rctx.walk.cache[0].length = 0; + + sg_init_one(&sg, &src, sizeof(src)); + + err = hifn_setup_dma(dev, &ctx, &rctx, &sg, &sg, sizeof(src), NULL); if (err) goto err_out; + dev->started = 0; msleep(200); dprintk("%s: decoded: ", dev->name); @@ -1717,6 +1700,7 @@ static int hifn_start_device(struct hifn_device *dev) { int err; + dev->started = dev->active = 0; hifn_reset_dma(dev, 1); err = hifn_enable_crypto(dev); @@ -1745,12 +1729,11 @@ static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset return -EINVAL; while (size) { + copy = min3(srest, dst->length, size); - copy = min(dst->length, srest); - - daddr = kmap_atomic(sg_page(dst), KM_IRQ0); + daddr = kmap_atomic(sg_page(dst)); memcpy(daddr + dst->offset + offset, saddr, copy); - kunmap_atomic(daddr, KM_IRQ0); + kunmap_atomic(daddr); nbytes -= copy; size -= copy; @@ -1771,90 +1754,65 @@ static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset return idx; } -static void hifn_process_ready(struct ablkcipher_request *req, int error) +static inline void hifn_complete_sa(struct hifn_device *dev, int i) { - struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); - struct hifn_device *dev; - - dprintk("%s: req: %p, ctx: %p.\n", __func__, req, ctx); + unsigned long flags; - dev = ctx->dev; - dprintk("%s: req: %p, started: %d, sg_num: %d.\n", - __func__, req, dev->started, atomic_read(&ctx->sg_num)); + spin_lock_irqsave(&dev->lock, flags); + dev->sa[i] = NULL; + dev->started--; + if (dev->started < 0) + printk("%s: started: %d.\n", __func__, dev->started); + spin_unlock_irqrestore(&dev->lock, flags); + BUG_ON(dev->started < 0); +} - if (--dev->started < 0) - BUG(); +static void hifn_process_ready(struct ablkcipher_request *req, int error) +{ + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); - if (atomic_dec_and_test(&ctx->sg_num)) { + if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { unsigned int nbytes = req->nbytes; int idx = 0, err; struct scatterlist *dst, *t; void *saddr; - if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { - while (nbytes) { - t = &ctx->walk.cache[idx]; - dst = &req->dst[idx]; - - dprintk("\n%s: sg_page(t): %p, t->length: %u, " - "sg_page(dst): %p, dst->length: %u, " - "nbytes: %u.\n", - __func__, sg_page(t), t->length, - sg_page(dst), dst->length, nbytes); + while (nbytes) { + t = &rctx->walk.cache[idx]; + dst = &req->dst[idx]; - if (!t->length) { - nbytes -= dst->length; - idx++; - continue; - } + dprintk("\n%s: sg_page(t): %p, t->length: %u, " + "sg_page(dst): %p, dst->length: %u, " + "nbytes: %u.\n", + __func__, sg_page(t), t->length, + sg_page(dst), dst->length, nbytes); - saddr = kmap_atomic(sg_page(t), KM_IRQ1); + if (!t->length) { + nbytes -= min(dst->length, nbytes); + idx++; + continue; + } - err = ablkcipher_get(saddr, &t->length, t->offset, - dst, nbytes, &nbytes); - if (err < 0) { - kunmap_atomic(saddr, KM_IRQ1); - break; - } + saddr = kmap_atomic(sg_page(t)); - idx += err; - kunmap_atomic(saddr, KM_IRQ1); + err = ablkcipher_get(saddr, &t->length, t->offset, + dst, nbytes, &nbytes); + if (err < 0) { + kunmap_atomic(saddr); + break; } - ablkcipher_walk_exit(&ctx->walk); + idx += err; + kunmap_atomic(saddr); } - req->base.complete(&req->base, error); + hifn_cipher_walk_exit(&rctx->walk); } -} - -static void hifn_check_for_completion(struct hifn_device *dev, int error) -{ - int i; - struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - - for (i=0; i<HIFN_D_RES_RSIZE; ++i) { - struct hifn_desc *d = &dma->resr[i]; - - if (!(d->l & __cpu_to_le32(HIFN_D_VALID)) && dev->sa[i]) { - dev->success++; - dev->reset = 0; - hifn_process_ready(dev->sa[i], error); - dev->sa[i] = NULL; - } - if (d->l & __cpu_to_le32(HIFN_D_DESTOVER | HIFN_D_OVER)) - if (printk_ratelimit()) - printk("%s: overflow detected [d: %u, o: %u] " - "at %d resr: l: %08x, p: %08x.\n", - dev->name, - !!(d->l & __cpu_to_le32(HIFN_D_DESTOVER)), - !!(d->l & __cpu_to_le32(HIFN_D_OVER)), - i, d->l, d->p); - } + req->base.complete(&req->base, error); } -static void hifn_clear_rings(struct hifn_device *dev) +static void hifn_clear_rings(struct hifn_device *dev, int error) { struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int i, u; @@ -1871,21 +1829,26 @@ static void hifn_clear_rings(struct hifn_device *dev) if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - if (i != HIFN_D_RES_RSIZE) - u--; + if (dev->sa[i]) { + dev->success++; + dev->reset = 0; + hifn_process_ready(dev->sa[i], error); + hifn_complete_sa(dev, i); + } - if (++i == (HIFN_D_RES_RSIZE + 1)) + if (++i == HIFN_D_RES_RSIZE) i = 0; + u--; } dma->resk = i; dma->resu = u; i = dma->srck; u = dma->srcu; while (u != 0) { - if (i == HIFN_D_SRC_RSIZE) - i = 0; if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - i++, u--; + if (++i == HIFN_D_SRC_RSIZE) + i = 0; + u--; } dma->srck = i; dma->srcu = u; @@ -1893,20 +1856,19 @@ static void hifn_clear_rings(struct hifn_device *dev) while (u != 0) { if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - if (i != HIFN_D_CMD_RSIZE) - u--; - if (++i == (HIFN_D_CMD_RSIZE + 1)) + if (++i == HIFN_D_CMD_RSIZE) i = 0; + u--; } dma->cmdk = i; dma->cmdu = u; i = dma->dstk; u = dma->dstu; while (u != 0) { - if (i == HIFN_D_DST_RSIZE) - i = 0; if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - i++, u--; + if (++i == HIFN_D_DST_RSIZE) + i = 0; + u--; } dma->dstk = i; dma->dstu = u; @@ -1920,7 +1882,7 @@ static void hifn_clear_rings(struct hifn_device *dev) static void hifn_work(struct work_struct *work) { - struct delayed_work *dw = container_of(work, struct delayed_work, work); + struct delayed_work *dw = to_delayed_work(work); struct hifn_device *dev = container_of(dw, struct hifn_device, work); unsigned long flags; int reset = 0; @@ -1951,30 +1913,39 @@ static void hifn_work(struct work_struct *work) } else dev->active--; - if (dev->prev_success == dev->success && dev->started) + if ((dev->prev_success == dev->success) && dev->started) reset = 1; dev->prev_success = dev->success; spin_unlock_irqrestore(&dev->lock, flags); if (reset) { - dprintk("%s: r: %08x, active: %d, started: %d, " - "success: %lu: reset: %d.\n", - dev->name, r, dev->active, dev->started, - dev->success, reset); - if (++dev->reset >= 5) { - dprintk("%s: really hard reset.\n", dev->name); + int i; + struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; + + printk("%s: r: %08x, active: %d, started: %d, " + "success: %lu: qlen: %u/%u, reset: %d.\n", + dev->name, r, dev->active, dev->started, + dev->success, dev->queue.qlen, dev->queue.max_qlen, + reset); + + printk("%s: res: ", __func__); + for (i=0; i<HIFN_D_RES_RSIZE; ++i) { + printk("%x.%p ", dma->resr[i].l, dev->sa[i]); + if (dev->sa[i]) { + hifn_process_ready(dev->sa[i], -ENODEV); + hifn_complete_sa(dev, i); + } + } + printk("\n"); + hifn_reset_dma(dev, 1); hifn_stop_device(dev); hifn_start_device(dev); dev->reset = 0; } - spin_lock_irqsave(&dev->lock, flags); - hifn_check_for_completion(dev, -EBUSY); - hifn_clear_rings(dev); - dev->started = 0; - spin_unlock_irqrestore(&dev->lock, flags); + tasklet_schedule(&dev->tasklet); } schedule_delayed_work(&dev->work, HZ); @@ -1991,8 +1962,8 @@ static irqreturn_t hifn_interrupt(int irq, void *data) dprintk("%s: 1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n", dev->name, dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, - dma->cmdu, dma->srcu, dma->dstu, dma->resu, - dma->cmdi, dma->srci, dma->dsti, dma->resi); + dma->cmdi, dma->srci, dma->dsti, dma->resi, + dma->cmdu, dma->srcu, dma->dstu, dma->resu); if ((dmacsr & dev->dmareg) == 0) return IRQ_NONE; @@ -2009,11 +1980,10 @@ static irqreturn_t hifn_interrupt(int irq, void *data) if (restart) { u32 puisr = hifn_read_0(dev, HIFN_0_PUISR); - if (printk_ratelimit()) - printk("%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", - dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER), - !!(dmacsr & HIFN_DMACSR_D_OVER), - puisr, !!(puisr & HIFN_PUISR_DSTOVER)); + printk(KERN_WARNING "%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", + dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER), + !!(dmacsr & HIFN_DMACSR_D_OVER), + puisr, !!(puisr & HIFN_PUISR_DSTOVER)); if (!!(puisr & HIFN_PUISR_DSTOVER)) hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER | @@ -2023,12 +1993,11 @@ static irqreturn_t hifn_interrupt(int irq, void *data) restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); if (restart) { - if (printk_ratelimit()) - printk("%s: abort: c: %d, s: %d, d: %d, r: %d.\n", - dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT), - !!(dmacsr & HIFN_DMACSR_S_ABORT), - !!(dmacsr & HIFN_DMACSR_D_ABORT), - !!(dmacsr & HIFN_DMACSR_R_ABORT)); + printk(KERN_WARNING "%s: abort: c: %d, s: %d, d: %d, r: %d.\n", + dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT), + !!(dmacsr & HIFN_DMACSR_S_ABORT), + !!(dmacsr & HIFN_DMACSR_D_ABORT), + !!(dmacsr & HIFN_DMACSR_R_ABORT)); hifn_reset_dma(dev, 1); hifn_init_dma(dev); hifn_init_registers(dev); @@ -2041,7 +2010,6 @@ static irqreturn_t hifn_interrupt(int irq, void *data) } tasklet_schedule(&dev->tasklet); - hifn_clear_rings(dev); return IRQ_HANDLED; } @@ -2050,26 +2018,28 @@ static void hifn_flush(struct hifn_device *dev) { unsigned long flags; struct crypto_async_request *async_req; - struct hifn_context *ctx; struct ablkcipher_request *req; struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int i; - spin_lock_irqsave(&dev->lock, flags); for (i=0; i<HIFN_D_RES_RSIZE; ++i) { struct hifn_desc *d = &dma->resr[i]; if (dev->sa[i]) { hifn_process_ready(dev->sa[i], (d->l & __cpu_to_le32(HIFN_D_VALID))?-ENODEV:0); + hifn_complete_sa(dev, i); } } + spin_lock_irqsave(&dev->lock, flags); while ((async_req = crypto_dequeue_request(&dev->queue))) { - ctx = crypto_tfm_ctx(async_req->tfm); req = container_of(async_req, struct ablkcipher_request, base); + spin_unlock_irqrestore(&dev->lock, flags); hifn_process_ready(req, -ENODEV); + + spin_lock_irqsave(&dev->lock, flags); } spin_unlock_irqrestore(&dev->lock, flags); } @@ -2128,6 +2098,7 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, u8 type, u8 mode) { struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); unsigned ivsize; ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req)); @@ -2148,11 +2119,11 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, type = ACRYPTO_TYPE_AES_256; } - ctx->op = op; - ctx->mode = mode; - ctx->type = type; - ctx->iv = req->info; - ctx->ivsize = ivsize; + rctx->op = op; + rctx->mode = mode; + rctx->type = type; + rctx->iv = req->info; + rctx->ivsize = ivsize; /* * HEAVY TODO: needs to kick Herbert XU to write documentation. @@ -2165,21 +2136,23 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, static int hifn_process_queue(struct hifn_device *dev) { - struct crypto_async_request *async_req; - struct hifn_context *ctx; + struct crypto_async_request *async_req, *backlog; struct ablkcipher_request *req; unsigned long flags; int err = 0; while (dev->started < HIFN_QUEUE_LENGTH) { spin_lock_irqsave(&dev->lock, flags); + backlog = crypto_get_backlog(&dev->queue); async_req = crypto_dequeue_request(&dev->queue); spin_unlock_irqrestore(&dev->lock, flags); if (!async_req) break; - ctx = crypto_tfm_ctx(async_req->tfm); + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + req = container_of(async_req, struct ablkcipher_request, base); err = hifn_handle_req(req); @@ -2202,9 +2175,9 @@ static int hifn_setup_crypto(struct ablkcipher_request *req, u8 op, return err; if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) - err = hifn_process_queue(dev); + hifn_process_queue(dev); - return err; + return -EINPROGRESS; } /* @@ -2364,7 +2337,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { * 3DES ECB, CBC, CFB and OFB modes. */ { - .name = "cfb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, + .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, @@ -2374,7 +2347,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ofb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, + .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, @@ -2384,8 +2357,9 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "cbc(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, + .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8, .ablkcipher = { + .ivsize = HIFN_IV_LENGTH, .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, .setkey = hifn_setkey, @@ -2394,7 +2368,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ecb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, + .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, @@ -2408,7 +2382,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { * DES ECB, CBC, CFB and OFB modes. */ { - .name = "cfb(des)", .drv_name = "hifn-des", .bsize = 8, + .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_DES_KEY_LENGTH, .max_keysize = HIFN_DES_KEY_LENGTH, @@ -2418,7 +2392,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ofb(des)", .drv_name = "hifn-des", .bsize = 8, + .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_DES_KEY_LENGTH, .max_keysize = HIFN_DES_KEY_LENGTH, @@ -2428,8 +2402,9 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "cbc(des)", .drv_name = "hifn-des", .bsize = 8, + .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8, .ablkcipher = { + .ivsize = HIFN_IV_LENGTH, .min_keysize = HIFN_DES_KEY_LENGTH, .max_keysize = HIFN_DES_KEY_LENGTH, .setkey = hifn_setkey, @@ -2438,7 +2413,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ecb(des)", .drv_name = "hifn-des", .bsize = 8, + .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_DES_KEY_LENGTH, .max_keysize = HIFN_DES_KEY_LENGTH, @@ -2452,7 +2427,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { * AES ECB, CBC, CFB and OFB modes. */ { - .name = "ecb(aes)", .drv_name = "hifn-aes", .bsize = 16, + .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16, .ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -2462,8 +2437,9 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "cbc(aes)", .drv_name = "hifn-aes", .bsize = 16, + .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16, .ablkcipher = { + .ivsize = HIFN_AES_IV_LENGTH, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = hifn_setkey, @@ -2472,7 +2448,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "cfb(aes)", .drv_name = "hifn-aes", .bsize = 16, + .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16, .ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -2482,7 +2458,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ofb(aes)", .drv_name = "hifn-aes", .bsize = 16, + .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16, .ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -2500,7 +2476,7 @@ static int hifn_cra_init(struct crypto_tfm *tfm) struct hifn_context *ctx = crypto_tfm_ctx(tfm); ctx->dev = ha->dev; - + tfm->crt_ablkcipher.reqsize = sizeof(struct hifn_request_context); return 0; } @@ -2514,15 +2490,15 @@ static int hifn_alg_alloc(struct hifn_device *dev, struct hifn_alg_template *t) return -ENOMEM; snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name); - snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", t->drv_name); + snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s", + t->drv_name, dev->name); alg->alg.cra_priority = 300; - alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; + alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC; alg->alg.cra_blocksize = t->bsize; alg->alg.cra_ctxsize = sizeof(struct hifn_context); - alg->alg.cra_alignmask = 15; - if (t->bsize == 8) - alg->alg.cra_alignmask = 3; + alg->alg.cra_alignmask = 0; alg->alg.cra_type = &crypto_ablkcipher_type; alg->alg.cra_module = THIS_MODULE; alg->alg.cra_u.ablkcipher = t->ablkcipher; @@ -2579,7 +2555,10 @@ static void hifn_tasklet_callback(unsigned long data) * (like dev->success), but they are used in process * context or update is atomic (like setting dev->sa[i] to NULL). */ - hifn_check_for_completion(dev, 0); + hifn_clear_rings(dev, 0); + + if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) + hifn_process_queue(dev); } static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -2593,7 +2572,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) return err; pci_set_master(pdev); - err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); + err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); if (err) goto err_out_disable_pci_device; @@ -2632,26 +2611,18 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) size = pci_resource_len(pdev, i); dev->bar[i] = ioremap_nocache(addr, size); - if (!dev->bar[i]) + if (!dev->bar[i]) { + err = -ENOMEM; goto err_out_unmap_bars; + } } - dev->result_mem = __get_free_pages(GFP_KERNEL, HIFN_MAX_RESULT_ORDER); - if (!dev->result_mem) { - dprintk("Failed to allocate %d pages for result_mem.\n", - HIFN_MAX_RESULT_ORDER); - goto err_out_unmap_bars; - } - memset((void *)dev->result_mem, 0, PAGE_SIZE*(1<<HIFN_MAX_RESULT_ORDER)); - - dev->dst = pci_map_single(pdev, (void *)dev->result_mem, - PAGE_SIZE << HIFN_MAX_RESULT_ORDER, PCI_DMA_FROMDEVICE); - dev->desc_virt = pci_alloc_consistent(pdev, sizeof(struct hifn_dma), &dev->desc_dma); if (!dev->desc_virt) { dprintk("Failed to allocate descriptor rings.\n"); - goto err_out_free_result_pages; + err = -ENOMEM; + goto err_out_unmap_bars; } memset(dev->desc_virt, 0, sizeof(struct hifn_dma)); @@ -2705,17 +2676,12 @@ err_out_stop_device: hifn_reset_dma(dev, 1); hifn_stop_device(dev); err_out_free_irq: - free_irq(dev->irq, dev->name); + free_irq(dev->irq, dev); tasklet_kill(&dev->tasklet); err_out_free_desc: pci_free_consistent(pdev, sizeof(struct hifn_dma), dev->desc_virt, dev->desc_dma); -err_out_free_result_pages: - pci_unmap_single(pdev, dev->dst, PAGE_SIZE << HIFN_MAX_RESULT_ORDER, - PCI_DMA_FROMDEVICE); - free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER); - err_out_unmap_bars: for (i=0; i<3; ++i) if (dev->bar[i]) @@ -2738,25 +2704,20 @@ static void hifn_remove(struct pci_dev *pdev) dev = pci_get_drvdata(pdev); if (dev) { - cancel_delayed_work(&dev->work); - flush_scheduled_work(); + cancel_delayed_work_sync(&dev->work); hifn_unregister_rng(dev); hifn_unregister_alg(dev); hifn_reset_dma(dev, 1); hifn_stop_device(dev); - free_irq(dev->irq, dev->name); + free_irq(dev->irq, dev); tasklet_kill(&dev->tasklet); hifn_flush(dev); pci_free_consistent(pdev, sizeof(struct hifn_dma), dev->desc_virt, dev->desc_dma); - pci_unmap_single(pdev, dev->dst, - PAGE_SIZE << HIFN_MAX_RESULT_ORDER, - PCI_DMA_FROMDEVICE); - free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER); for (i=0; i<3; ++i) if (dev->bar[i]) iounmap(dev->bar[i]); @@ -2779,14 +2740,17 @@ static struct pci_driver hifn_pci_driver = { .name = "hifn795x", .id_table = hifn_pci_tbl, .probe = hifn_probe, - .remove = __devexit_p(hifn_remove), + .remove = hifn_remove, }; -static int __devinit hifn_init(void) +static int __init hifn_init(void) { unsigned int freq; int err; + /* HIFN supports only 32-bit addresses */ + BUILD_BUG_ON(sizeof(dma_addr_t) != 4); + if (strncmp(hifn_pll_ref, "ext", 3) && strncmp(hifn_pll_ref, "pci", 3)) { printk(KERN_ERR "hifn795x: invalid hifn_pll_ref clock, " @@ -2822,7 +2786,7 @@ static int __devinit hifn_init(void) return 0; } -static void __devexit hifn_fini(void) +static void __exit hifn_fini(void) { pci_unregister_driver(&hifn_pci_driver); |