diff options
Diffstat (limited to 'crypto/async_tx')
| -rw-r--r-- | crypto/async_tx/Kconfig | 13 | ||||
| -rw-r--r-- | crypto/async_tx/Makefile | 4 | ||||
| -rw-r--r-- | crypto/async_tx/async_memcpy.c | 96 | ||||
| -rw-r--r-- | crypto/async_tx/async_memset.c | 109 | ||||
| -rw-r--r-- | crypto/async_tx/async_pq.c | 441 | ||||
| -rw-r--r-- | crypto/async_tx/async_raid6_recov.c | 531 | ||||
| -rw-r--r-- | crypto/async_tx/async_tx.c | 564 | ||||
| -rw-r--r-- | crypto/async_tx/async_xor.c | 453 | ||||
| -rw-r--r-- | crypto/async_tx/raid6test.c | 253 |
9 files changed, 1685 insertions, 779 deletions
diff --git a/crypto/async_tx/Kconfig b/crypto/async_tx/Kconfig index d8fb3914598..f38a58aef3e 100644 --- a/crypto/async_tx/Kconfig +++ b/crypto/async_tx/Kconfig @@ -10,7 +10,18 @@ config ASYNC_XOR select ASYNC_CORE select XOR_BLOCKS -config ASYNC_MEMSET +config ASYNC_PQ tristate select ASYNC_CORE +config ASYNC_RAID6_RECOV + tristate + select ASYNC_CORE + select ASYNC_PQ + select ASYNC_XOR + +config ASYNC_TX_DISABLE_PQ_VAL_DMA + bool + +config ASYNC_TX_DISABLE_XOR_VAL_DMA + bool diff --git a/crypto/async_tx/Makefile b/crypto/async_tx/Makefile index 27baa7d52fb..462e4abbfe6 100644 --- a/crypto/async_tx/Makefile +++ b/crypto/async_tx/Makefile @@ -1,4 +1,6 @@ obj-$(CONFIG_ASYNC_CORE) += async_tx.o obj-$(CONFIG_ASYNC_MEMCPY) += async_memcpy.o -obj-$(CONFIG_ASYNC_MEMSET) += async_memset.o obj-$(CONFIG_ASYNC_XOR) += async_xor.o +obj-$(CONFIG_ASYNC_PQ) += async_pq.o +obj-$(CONFIG_ASYNC_RAID6_RECOV) += async_raid6_recov.o +obj-$(CONFIG_ASYNC_RAID6_TEST) += raid6test.o diff --git a/crypto/async_tx/async_memcpy.c b/crypto/async_tx/async_memcpy.c index 84caa4efc0d..f8c0b8dbeb7 100644 --- a/crypto/async_tx/async_memcpy.c +++ b/crypto/async_tx/async_memcpy.c @@ -25,6 +25,7 @@ */ #include <linux/kernel.h> #include <linux/highmem.h> +#include <linux/module.h> #include <linux/mm.h> #include <linux/dma-mapping.h> #include <linux/async_tx.h> @@ -33,84 +34,77 @@ * async_memcpy - attempt to copy memory with a dma engine. * @dest: destination page * @src: src page - * @offset: offset in pages to start transaction + * @dest_offset: offset into 'dest' to start transaction + * @src_offset: offset into 'src' to start transaction * @len: length in bytes - * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK, - * @depend_tx: memcpy depends on the result of this transaction - * @cb_fn: function to call when the memcpy completes - * @cb_param: parameter to pass to the callback routine + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK */ struct dma_async_tx_descriptor * async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, - unsigned int src_offset, size_t len, enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) + unsigned int src_offset, size_t len, + struct async_submit_ctl *submit) { - struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMCPY, + struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY, &dest, 1, &src, 1, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx = NULL; - - if (device) { - dma_addr_t dma_dest, dma_src; - unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; - - dma_dest = dma_map_page(device->dev, dest, dest_offset, len, - DMA_FROM_DEVICE); - - dma_src = dma_map_page(device->dev, src, src_offset, len, - DMA_TO_DEVICE); - - tx = device->device_prep_dma_memcpy(chan, dma_dest, dma_src, - len, dma_prep_flags); + struct dmaengine_unmap_data *unmap = NULL; + + if (device) + unmap = dmaengine_get_unmap_data(device->dev, 2, GFP_NOIO); + + if (unmap && is_dma_copy_aligned(device, src_offset, dest_offset, len)) { + unsigned long dma_prep_flags = 0; + + if (submit->cb_fn) + dma_prep_flags |= DMA_PREP_INTERRUPT; + if (submit->flags & ASYNC_TX_FENCE) + dma_prep_flags |= DMA_PREP_FENCE; + + unmap->to_cnt = 1; + unmap->addr[0] = dma_map_page(device->dev, src, src_offset, len, + DMA_TO_DEVICE); + unmap->from_cnt = 1; + unmap->addr[1] = dma_map_page(device->dev, dest, dest_offset, len, + DMA_FROM_DEVICE); + unmap->len = len; + + tx = device->device_prep_dma_memcpy(chan, unmap->addr[1], + unmap->addr[0], len, + dma_prep_flags); } if (tx) { pr_debug("%s: (async) len: %zu\n", __func__, len); - async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); + + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); } else { void *dest_buf, *src_buf; pr_debug("%s: (sync) len: %zu\n", __func__, len); /* wait for any prerequisite operations */ - if (depend_tx) { - /* if ack is already set then we cannot be sure - * we are referring to the correct operation - */ - BUG_ON(depend_tx->ack); - if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR) - panic("%s: DMA_ERROR waiting for depend_tx\n", - __func__); - } - - dest_buf = kmap_atomic(dest, KM_USER0) + dest_offset; - src_buf = kmap_atomic(src, KM_USER1) + src_offset; + async_tx_quiesce(&submit->depend_tx); + + dest_buf = kmap_atomic(dest) + dest_offset; + src_buf = kmap_atomic(src) + src_offset; memcpy(dest_buf, src_buf, len); - kunmap_atomic(dest_buf, KM_USER0); - kunmap_atomic(src_buf, KM_USER1); + kunmap_atomic(src_buf); + kunmap_atomic(dest_buf); - async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param); + async_tx_sync_epilog(submit); } + dmaengine_unmap_put(unmap); + return tx; } EXPORT_SYMBOL_GPL(async_memcpy); -static int __init async_memcpy_init(void) -{ - return 0; -} - -static void __exit async_memcpy_exit(void) -{ - do { } while (0); -} - -module_init(async_memcpy_init); -module_exit(async_memcpy_exit); - MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("asynchronous memcpy api"); MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_memset.c b/crypto/async_tx/async_memset.c deleted file mode 100644 index f5ff3906b03..00000000000 --- a/crypto/async_tx/async_memset.c +++ /dev/null @@ -1,109 +0,0 @@ -/* - * memory fill offload engine support - * - * Copyright © 2006, Intel Corporation. - * - * Dan Williams <dan.j.williams@intel.com> - * - * with architecture considerations by: - * Neil Brown <neilb@suse.de> - * Jeff Garzik <jeff@garzik.org> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License along with - * this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. - * - */ -#include <linux/kernel.h> -#include <linux/interrupt.h> -#include <linux/mm.h> -#include <linux/dma-mapping.h> -#include <linux/async_tx.h> - -/** - * async_memset - attempt to fill memory with a dma engine. - * @dest: destination page - * @val: fill value - * @offset: offset in pages to start transaction - * @len: length in bytes - * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK - * @depend_tx: memset depends on the result of this transaction - * @cb_fn: function to call when the memcpy completes - * @cb_param: parameter to pass to the callback routine - */ -struct dma_async_tx_descriptor * -async_memset(struct page *dest, int val, unsigned int offset, - size_t len, enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) -{ - struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMSET, - &dest, 1, NULL, 0, len); - struct dma_device *device = chan ? chan->device : NULL; - struct dma_async_tx_descriptor *tx = NULL; - - if (device) { - dma_addr_t dma_dest; - unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; - - dma_dest = dma_map_page(device->dev, dest, offset, len, - DMA_FROM_DEVICE); - - tx = device->device_prep_dma_memset(chan, dma_dest, val, len, - dma_prep_flags); - } - - if (tx) { - pr_debug("%s: (async) len: %zu\n", __func__, len); - async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); - } else { /* run the memset synchronously */ - void *dest_buf; - pr_debug("%s: (sync) len: %zu\n", __func__, len); - - dest_buf = (void *) (((char *) page_address(dest)) + offset); - - /* wait for any prerequisite operations */ - if (depend_tx) { - /* if ack is already set then we cannot be sure - * we are referring to the correct operation - */ - BUG_ON(depend_tx->ack); - if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR) - panic("%s: DMA_ERROR waiting for depend_tx\n", - __func__); - } - - memset(dest_buf, val, len); - - async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param); - } - - return tx; -} -EXPORT_SYMBOL_GPL(async_memset); - -static int __init async_memset_init(void) -{ - return 0; -} - -static void __exit async_memset_exit(void) -{ - do { } while (0); -} - -module_init(async_memset_init); -module_exit(async_memset_exit); - -MODULE_AUTHOR("Intel Corporation"); -MODULE_DESCRIPTION("asynchronous memset api"); -MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c new file mode 100644 index 00000000000..d05327caf69 --- /dev/null +++ b/crypto/async_tx/async_pq.c @@ -0,0 +1,441 @@ +/* + * Copyright(c) 2007 Yuri Tikhonov <yur@emcraft.com> + * Copyright(c) 2009 Intel Corporation + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/dma-mapping.h> +#include <linux/raid/pq.h> +#include <linux/async_tx.h> +#include <linux/gfp.h> + +/** + * pq_scribble_page - space to hold throwaway P or Q buffer for + * synchronous gen_syndrome + */ +static struct page *pq_scribble_page; + +/* the struct page *blocks[] parameter passed to async_gen_syndrome() + * and async_syndrome_val() contains the 'P' destination address at + * blocks[disks-2] and the 'Q' destination address at blocks[disks-1] + * + * note: these are macros as they are used as lvalues + */ +#define P(b, d) (b[d-2]) +#define Q(b, d) (b[d-1]) + +/** + * do_async_gen_syndrome - asynchronously calculate P and/or Q + */ +static __async_inline struct dma_async_tx_descriptor * +do_async_gen_syndrome(struct dma_chan *chan, + const unsigned char *scfs, int disks, + struct dmaengine_unmap_data *unmap, + enum dma_ctrl_flags dma_flags, + struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct dma_device *dma = chan->device; + enum async_tx_flags flags_orig = submit->flags; + dma_async_tx_callback cb_fn_orig = submit->cb_fn; + dma_async_tx_callback cb_param_orig = submit->cb_param; + int src_cnt = disks - 2; + unsigned short pq_src_cnt; + dma_addr_t dma_dest[2]; + int src_off = 0; + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + + while (src_cnt > 0) { + submit->flags = flags_orig; + pq_src_cnt = min(src_cnt, dma_maxpq(dma, dma_flags)); + /* if we are submitting additional pqs, leave the chain open, + * clear the callback parameters, and leave the destination + * buffers mapped + */ + if (src_cnt > pq_src_cnt) { + submit->flags &= ~ASYNC_TX_ACK; + submit->flags |= ASYNC_TX_FENCE; + submit->cb_fn = NULL; + submit->cb_param = NULL; + } else { + submit->cb_fn = cb_fn_orig; + submit->cb_param = cb_param_orig; + if (cb_fn_orig) + dma_flags |= DMA_PREP_INTERRUPT; + } + + /* Drivers force forward progress in case they can not provide + * a descriptor + */ + for (;;) { + dma_dest[0] = unmap->addr[disks - 2]; + dma_dest[1] = unmap->addr[disks - 1]; + tx = dma->device_prep_dma_pq(chan, dma_dest, + &unmap->addr[src_off], + pq_src_cnt, + &scfs[src_off], unmap->len, + dma_flags); + if (likely(tx)) + break; + async_tx_quiesce(&submit->depend_tx); + dma_async_issue_pending(chan); + } + + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + submit->depend_tx = tx; + + /* drop completed sources */ + src_cnt -= pq_src_cnt; + src_off += pq_src_cnt; + + dma_flags |= DMA_PREP_CONTINUE; + } + + return tx; +} + +/** + * do_sync_gen_syndrome - synchronously calculate a raid6 syndrome + */ +static void +do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks, + size_t len, struct async_submit_ctl *submit) +{ + void **srcs; + int i; + + if (submit->scribble) + srcs = submit->scribble; + else + srcs = (void **) blocks; + + for (i = 0; i < disks; i++) { + if (blocks[i] == NULL) { + BUG_ON(i > disks - 3); /* P or Q can't be zero */ + srcs[i] = (void*)raid6_empty_zero_page; + } else + srcs[i] = page_address(blocks[i]) + offset; + } + raid6_call.gen_syndrome(disks, len, srcs); + async_tx_sync_epilog(submit); +} + +/** + * async_gen_syndrome - asynchronously calculate a raid6 syndrome + * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 + * @offset: common offset into each block (src and dest) to start transaction + * @disks: number of blocks (including missing P or Q, see below) + * @len: length of operation in bytes + * @submit: submission/completion modifiers + * + * General note: This routine assumes a field of GF(2^8) with a + * primitive polynomial of 0x11d and a generator of {02}. + * + * 'disks' note: callers can optionally omit either P or Q (but not + * both) from the calculation by setting blocks[disks-2] or + * blocks[disks-1] to NULL. When P or Q is omitted 'len' must be <= + * PAGE_SIZE as a temporary buffer of this size is used in the + * synchronous path. 'disks' always accounts for both destination + * buffers. If any source buffers (blocks[i] where i < disks - 2) are + * set to NULL those buffers will be replaced with the raid6_zero_page + * in the synchronous path and omitted in the hardware-asynchronous + * path. + */ +struct dma_async_tx_descriptor * +async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, + size_t len, struct async_submit_ctl *submit) +{ + int src_cnt = disks - 2; + struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, + &P(blocks, disks), 2, + blocks, src_cnt, len); + struct dma_device *device = chan ? chan->device : NULL; + struct dmaengine_unmap_data *unmap = NULL; + + BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks))); + + if (device) + unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOIO); + + if (unmap && + (src_cnt <= dma_maxpq(device, 0) || + dma_maxpq(device, DMA_PREP_CONTINUE) > 0) && + is_dma_pq_aligned(device, offset, 0, len)) { + struct dma_async_tx_descriptor *tx; + enum dma_ctrl_flags dma_flags = 0; + unsigned char coefs[src_cnt]; + int i, j; + + /* run the p+q asynchronously */ + pr_debug("%s: (async) disks: %d len: %zu\n", + __func__, disks, len); + + /* convert source addresses being careful to collapse 'empty' + * sources and update the coefficients accordingly + */ + unmap->len = len; + for (i = 0, j = 0; i < src_cnt; i++) { + if (blocks[i] == NULL) + continue; + unmap->addr[j] = dma_map_page(device->dev, blocks[i], offset, + len, DMA_TO_DEVICE); + coefs[j] = raid6_gfexp[i]; + unmap->to_cnt++; + j++; + } + + /* + * DMAs use destinations as sources, + * so use BIDIRECTIONAL mapping + */ + unmap->bidi_cnt++; + if (P(blocks, disks)) + unmap->addr[j++] = dma_map_page(device->dev, P(blocks, disks), + offset, len, DMA_BIDIRECTIONAL); + else { + unmap->addr[j++] = 0; + dma_flags |= DMA_PREP_PQ_DISABLE_P; + } + + unmap->bidi_cnt++; + if (Q(blocks, disks)) + unmap->addr[j++] = dma_map_page(device->dev, Q(blocks, disks), + offset, len, DMA_BIDIRECTIONAL); + else { + unmap->addr[j++] = 0; + dma_flags |= DMA_PREP_PQ_DISABLE_Q; + } + + tx = do_async_gen_syndrome(chan, coefs, j, unmap, dma_flags, submit); + dmaengine_unmap_put(unmap); + return tx; + } + + dmaengine_unmap_put(unmap); + + /* run the pq synchronously */ + pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len); + + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); + + if (!P(blocks, disks)) { + P(blocks, disks) = pq_scribble_page; + BUG_ON(len + offset > PAGE_SIZE); + } + if (!Q(blocks, disks)) { + Q(blocks, disks) = pq_scribble_page; + BUG_ON(len + offset > PAGE_SIZE); + } + do_sync_gen_syndrome(blocks, offset, disks, len, submit); + + return NULL; +} +EXPORT_SYMBOL_GPL(async_gen_syndrome); + +static inline struct dma_chan * +pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, size_t len) +{ + #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA + return NULL; + #endif + return async_tx_find_channel(submit, DMA_PQ_VAL, NULL, 0, blocks, + disks, len); +} + +/** + * async_syndrome_val - asynchronously validate a raid6 syndrome + * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 + * @offset: common offset into each block (src and dest) to start transaction + * @disks: number of blocks (including missing P or Q, see below) + * @len: length of operation in bytes + * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set + * @spare: temporary result buffer for the synchronous case + * @submit: submission / completion modifiers + * + * The same notes from async_gen_syndrome apply to the 'blocks', + * and 'disks' parameters of this routine. The synchronous path + * requires a temporary result buffer and submit->scribble to be + * specified. + */ +struct dma_async_tx_descriptor * +async_syndrome_val(struct page **blocks, unsigned int offset, int disks, + size_t len, enum sum_check_flags *pqres, struct page *spare, + struct async_submit_ctl *submit) +{ + struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len); + struct dma_device *device = chan ? chan->device : NULL; + struct dma_async_tx_descriptor *tx; + unsigned char coefs[disks-2]; + enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0; + struct dmaengine_unmap_data *unmap = NULL; + + BUG_ON(disks < 4); + + if (device) + unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOIO); + + if (unmap && disks <= dma_maxpq(device, 0) && + is_dma_pq_aligned(device, offset, 0, len)) { + struct device *dev = device->dev; + dma_addr_t pq[2]; + int i, j = 0, src_cnt = 0; + + pr_debug("%s: (async) disks: %d len: %zu\n", + __func__, disks, len); + + unmap->len = len; + for (i = 0; i < disks-2; i++) + if (likely(blocks[i])) { + unmap->addr[j] = dma_map_page(dev, blocks[i], + offset, len, + DMA_TO_DEVICE); + coefs[j] = raid6_gfexp[i]; + unmap->to_cnt++; + src_cnt++; + j++; + } + + if (!P(blocks, disks)) { + pq[0] = 0; + dma_flags |= DMA_PREP_PQ_DISABLE_P; + } else { + pq[0] = dma_map_page(dev, P(blocks, disks), + offset, len, + DMA_TO_DEVICE); + unmap->addr[j++] = pq[0]; + unmap->to_cnt++; + } + if (!Q(blocks, disks)) { + pq[1] = 0; + dma_flags |= DMA_PREP_PQ_DISABLE_Q; + } else { + pq[1] = dma_map_page(dev, Q(blocks, disks), + offset, len, + DMA_TO_DEVICE); + unmap->addr[j++] = pq[1]; + unmap->to_cnt++; + } + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + for (;;) { + tx = device->device_prep_dma_pq_val(chan, pq, + unmap->addr, + src_cnt, + coefs, + len, pqres, + dma_flags); + if (likely(tx)) + break; + async_tx_quiesce(&submit->depend_tx); + dma_async_issue_pending(chan); + } + + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + + return tx; + } else { + struct page *p_src = P(blocks, disks); + struct page *q_src = Q(blocks, disks); + enum async_tx_flags flags_orig = submit->flags; + dma_async_tx_callback cb_fn_orig = submit->cb_fn; + void *scribble = submit->scribble; + void *cb_param_orig = submit->cb_param; + void *p, *q, *s; + + pr_debug("%s: (sync) disks: %d len: %zu\n", + __func__, disks, len); + + /* caller must provide a temporary result buffer and + * allow the input parameters to be preserved + */ + BUG_ON(!spare || !scribble); + + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); + + /* recompute p and/or q into the temporary buffer and then + * check to see the result matches the current value + */ + tx = NULL; + *pqres = 0; + if (p_src) { + init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL, + NULL, NULL, scribble); + tx = async_xor(spare, blocks, offset, disks-2, len, submit); + async_tx_quiesce(&tx); + p = page_address(p_src) + offset; + s = page_address(spare) + offset; + *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P; + } + + if (q_src) { + P(blocks, disks) = NULL; + Q(blocks, disks) = spare; + init_async_submit(submit, 0, NULL, NULL, NULL, scribble); + tx = async_gen_syndrome(blocks, offset, disks, len, submit); + async_tx_quiesce(&tx); + q = page_address(q_src) + offset; + s = page_address(spare) + offset; + *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q; + } + + /* restore P, Q and submit */ + P(blocks, disks) = p_src; + Q(blocks, disks) = q_src; + + submit->cb_fn = cb_fn_orig; + submit->cb_param = cb_param_orig; + submit->flags = flags_orig; + async_tx_sync_epilog(submit); + + return NULL; + } +} +EXPORT_SYMBOL_GPL(async_syndrome_val); + +static int __init async_pq_init(void) +{ + pq_scribble_page = alloc_page(GFP_KERNEL); + + if (pq_scribble_page) + return 0; + + pr_err("%s: failed to allocate required spare page\n", __func__); + + return -ENOMEM; +} + +static void __exit async_pq_exit(void) +{ + put_page(pq_scribble_page); +} + +module_init(async_pq_init); +module_exit(async_pq_exit); + +MODULE_DESCRIPTION("asynchronous raid6 syndrome generation/validation"); +MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c new file mode 100644 index 00000000000..934a8498149 --- /dev/null +++ b/crypto/async_tx/async_raid6_recov.c @@ -0,0 +1,531 @@ +/* + * Asynchronous RAID-6 recovery calculations ASYNC_TX API. + * Copyright(c) 2009 Intel Corporation + * + * based on raid6recov.c: + * Copyright 2002 H. Peter Anvin + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/dma-mapping.h> +#include <linux/raid/pq.h> +#include <linux/async_tx.h> +#include <linux/dmaengine.h> + +static struct dma_async_tx_descriptor * +async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, + size_t len, struct async_submit_ctl *submit) +{ + struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, + &dest, 1, srcs, 2, len); + struct dma_device *dma = chan ? chan->device : NULL; + struct dmaengine_unmap_data *unmap = NULL; + const u8 *amul, *bmul; + u8 ax, bx; + u8 *a, *b, *c; + + if (dma) + unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO); + + if (unmap) { + struct device *dev = dma->dev; + dma_addr_t pq[2]; + struct dma_async_tx_descriptor *tx; + enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + unmap->addr[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE); + unmap->addr[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE); + unmap->to_cnt = 2; + + unmap->addr[2] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); + unmap->bidi_cnt = 1; + /* engine only looks at Q, but expects it to follow P */ + pq[1] = unmap->addr[2]; + + unmap->len = len; + tx = dma->device_prep_dma_pq(chan, pq, unmap->addr, 2, coef, + len, dma_flags); + if (tx) { + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + dmaengine_unmap_put(unmap); + return tx; + } + + /* could not get a descriptor, unmap and fall through to + * the synchronous path + */ + dmaengine_unmap_put(unmap); + } + + /* run the operation synchronously */ + async_tx_quiesce(&submit->depend_tx); + amul = raid6_gfmul[coef[0]]; + bmul = raid6_gfmul[coef[1]]; + a = page_address(srcs[0]); + b = page_address(srcs[1]); + c = page_address(dest); + + while (len--) { + ax = amul[*a++]; + bx = bmul[*b++]; + *c++ = ax ^ bx; + } + + return NULL; +} + +static struct dma_async_tx_descriptor * +async_mult(struct page *dest, struct page *src, u8 coef, size_t len, + struct async_submit_ctl *submit) +{ + struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, + &dest, 1, &src, 1, len); + struct dma_device *dma = chan ? chan->device : NULL; + struct dmaengine_unmap_data *unmap = NULL; + const u8 *qmul; /* Q multiplier table */ + u8 *d, *s; + + if (dma) + unmap = dmaengine_get_unmap_data(dma->dev, 3, GFP_NOIO); + + if (unmap) { + dma_addr_t dma_dest[2]; + struct device *dev = dma->dev; + struct dma_async_tx_descriptor *tx; + enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + unmap->addr[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE); + unmap->to_cnt++; + unmap->addr[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); + dma_dest[1] = unmap->addr[1]; + unmap->bidi_cnt++; + unmap->len = len; + + /* this looks funny, but the engine looks for Q at + * dma_dest[1] and ignores dma_dest[0] as a dest + * due to DMA_PREP_PQ_DISABLE_P + */ + tx = dma->device_prep_dma_pq(chan, dma_dest, unmap->addr, + 1, &coef, len, dma_flags); + + if (tx) { + dma_set_unmap(tx, unmap); + dmaengine_unmap_put(unmap); + async_tx_submit(chan, tx, submit); + return tx; + } + + /* could not get a descriptor, unmap and fall through to + * the synchronous path + */ + dmaengine_unmap_put(unmap); + } + + /* no channel available, or failed to allocate a descriptor, so + * perform the operation synchronously + */ + async_tx_quiesce(&submit->depend_tx); + qmul = raid6_gfmul[coef]; + d = page_address(dest); + s = page_address(src); + + while (len--) + *d++ = qmul[*s++]; + + return NULL; +} + +static struct dma_async_tx_descriptor * +__2data_recov_4(int disks, size_t bytes, int faila, int failb, + struct page **blocks, struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *a, *b; + struct page *srcs[2]; + unsigned char coef[2]; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + + p = blocks[disks-2]; + q = blocks[disks-1]; + + a = blocks[faila]; + b = blocks[failb]; + + /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */ + /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ + srcs[0] = p; + srcs[1] = q; + coef[0] = raid6_gfexi[failb-faila]; + coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_sum_product(b, srcs, coef, bytes, submit); + + /* Dy = P+Pxy+Dx */ + srcs[0] = p; + srcs[1] = b; + init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor(a, srcs, 0, 2, bytes, submit); + + return tx; + +} + +static struct dma_async_tx_descriptor * +__2data_recov_5(int disks, size_t bytes, int faila, int failb, + struct page **blocks, struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *g, *dp, *dq; + struct page *srcs[2]; + unsigned char coef[2]; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + int good_srcs, good, i; + + good_srcs = 0; + good = -1; + for (i = 0; i < disks-2; i++) { + if (blocks[i] == NULL) + continue; + if (i == faila || i == failb) + continue; + good = i; + good_srcs++; + } + BUG_ON(good_srcs > 1); + + p = blocks[disks-2]; + q = blocks[disks-1]; + g = blocks[good]; + + /* Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for delta p and + * delta q + */ + dp = blocks[faila]; + dq = blocks[failb]; + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_memcpy(dp, g, 0, 0, bytes, submit); + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit); + + /* compute P + Pxy */ + srcs[0] = dp; + srcs[1] = p; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dp, srcs, 0, 2, bytes, submit); + + /* compute Q + Qxy */ + srcs[0] = dq; + srcs[1] = q; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dq, srcs, 0, 2, bytes, submit); + + /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ + srcs[0] = dp; + srcs[1] = dq; + coef[0] = raid6_gfexi[failb-faila]; + coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_sum_product(dq, srcs, coef, bytes, submit); + + /* Dy = P+Pxy+Dx */ + srcs[0] = dp; + srcs[1] = dq; + init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor(dp, srcs, 0, 2, bytes, submit); + + return tx; +} + +static struct dma_async_tx_descriptor * +__2data_recov_n(int disks, size_t bytes, int faila, int failb, + struct page **blocks, struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *dp, *dq; + struct page *srcs[2]; + unsigned char coef[2]; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + + p = blocks[disks-2]; + q = blocks[disks-1]; + + /* Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for + * delta p and delta q + */ + dp = blocks[faila]; + blocks[faila] = NULL; + blocks[disks-2] = dp; + dq = blocks[failb]; + blocks[failb] = NULL; + blocks[disks-1] = dq; + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_gen_syndrome(blocks, 0, disks, bytes, submit); + + /* Restore pointer table */ + blocks[faila] = dp; + blocks[failb] = dq; + blocks[disks-2] = p; + blocks[disks-1] = q; + + /* compute P + Pxy */ + srcs[0] = dp; + srcs[1] = p; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dp, srcs, 0, 2, bytes, submit); + + /* compute Q + Qxy */ + srcs[0] = dq; + srcs[1] = q; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dq, srcs, 0, 2, bytes, submit); + + /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ + srcs[0] = dp; + srcs[1] = dq; + coef[0] = raid6_gfexi[failb-faila]; + coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_sum_product(dq, srcs, coef, bytes, submit); + + /* Dy = P+Pxy+Dx */ + srcs[0] = dp; + srcs[1] = dq; + init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor(dp, srcs, 0, 2, bytes, submit); + + return tx; +} + +/** + * async_raid6_2data_recov - asynchronously calculate two missing data blocks + * @disks: number of disks in the RAID-6 array + * @bytes: block size + * @faila: first failed drive index + * @failb: second failed drive index + * @blocks: array of source pointers where the last two entries are p and q + * @submit: submission/completion modifiers + */ +struct dma_async_tx_descriptor * +async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, + struct page **blocks, struct async_submit_ctl *submit) +{ + void *scribble = submit->scribble; + int non_zero_srcs, i; + + BUG_ON(faila == failb); + if (failb < faila) + swap(faila, failb); + + pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); + + /* if a dma resource is not available or a scribble buffer is not + * available punt to the synchronous path. In the 'dma not + * available' case be sure to use the scribble buffer to + * preserve the content of 'blocks' as the caller intended. + */ + if (!async_dma_find_channel(DMA_PQ) || !scribble) { + void **ptrs = scribble ? scribble : (void **) blocks; + + async_tx_quiesce(&submit->depend_tx); + for (i = 0; i < disks; i++) + if (blocks[i] == NULL) + ptrs[i] = (void *) raid6_empty_zero_page; + else + ptrs[i] = page_address(blocks[i]); + + raid6_2data_recov(disks, bytes, faila, failb, ptrs); + + async_tx_sync_epilog(submit); + + return NULL; + } + + non_zero_srcs = 0; + for (i = 0; i < disks-2 && non_zero_srcs < 4; i++) + if (blocks[i]) + non_zero_srcs++; + switch (non_zero_srcs) { + case 0: + case 1: + /* There must be at least 2 sources - the failed devices. */ + BUG(); + + case 2: + /* dma devices do not uniformly understand a zero source pq + * operation (in contrast to the synchronous case), so + * explicitly handle the special case of a 4 disk array with + * both data disks missing. + */ + return __2data_recov_4(disks, bytes, faila, failb, blocks, submit); + case 3: + /* dma devices do not uniformly understand a single + * source pq operation (in contrast to the synchronous + * case), so explicitly handle the special case of a 5 disk + * array with 2 of 3 data disks missing. + */ + return __2data_recov_5(disks, bytes, faila, failb, blocks, submit); + default: + return __2data_recov_n(disks, bytes, faila, failb, blocks, submit); + } +} +EXPORT_SYMBOL_GPL(async_raid6_2data_recov); + +/** + * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block + * @disks: number of disks in the RAID-6 array + * @bytes: block size + * @faila: failed drive index + * @blocks: array of source pointers where the last two entries are p and q + * @submit: submission/completion modifiers + */ +struct dma_async_tx_descriptor * +async_raid6_datap_recov(int disks, size_t bytes, int faila, + struct page **blocks, struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *dq; + u8 coef; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + int good_srcs, good, i; + struct page *srcs[2]; + + pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); + + /* if a dma resource is not available or a scribble buffer is not + * available punt to the synchronous path. In the 'dma not + * available' case be sure to use the scribble buffer to + * preserve the content of 'blocks' as the caller intended. + */ + if (!async_dma_find_channel(DMA_PQ) || !scribble) { + void **ptrs = scribble ? scribble : (void **) blocks; + + async_tx_quiesce(&submit->depend_tx); + for (i = 0; i < disks; i++) + if (blocks[i] == NULL) + ptrs[i] = (void*)raid6_empty_zero_page; + else + ptrs[i] = page_address(blocks[i]); + + raid6_datap_recov(disks, bytes, faila, ptrs); + + async_tx_sync_epilog(submit); + + return NULL; + } + + good_srcs = 0; + good = -1; + for (i = 0; i < disks-2; i++) { + if (i == faila) + continue; + if (blocks[i]) { + good = i; + good_srcs++; + if (good_srcs > 1) + break; + } + } + BUG_ON(good_srcs == 0); + + p = blocks[disks-2]; + q = blocks[disks-1]; + + /* Compute syndrome with zero for the missing data page + * Use the dead data page as temporary storage for delta q + */ + dq = blocks[faila]; + blocks[faila] = NULL; + blocks[disks-1] = dq; + + /* in the 4-disk case we only need to perform a single source + * multiplication with the one good data block. + */ + if (good_srcs == 1) { + struct page *g = blocks[good]; + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, + scribble); + tx = async_memcpy(p, g, 0, 0, bytes, submit); + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, + scribble); + tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit); + } else { + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, + scribble); + tx = async_gen_syndrome(blocks, 0, disks, bytes, submit); + } + + /* Restore pointer table */ + blocks[faila] = dq; + blocks[disks-1] = q; + + /* calculate g^{-faila} */ + coef = raid6_gfinv[raid6_gfexp[faila]]; + + srcs[0] = dq; + srcs[1] = q; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dq, srcs, 0, 2, bytes, submit); + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_mult(dq, dq, coef, bytes, submit); + + srcs[0] = p; + srcs[1] = dq; + init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor(p, srcs, 0, 2, bytes, submit); + + return tx; +} +EXPORT_SYMBOL_GPL(async_raid6_datap_recov); + +MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>"); +MODULE_DESCRIPTION("asynchronous RAID-6 recovery api"); +MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c index 2be3bae8993..39ea4791a3c 100644 --- a/crypto/async_tx/async_tx.c +++ b/crypto/async_tx/async_tx.c @@ -23,439 +23,220 @@ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. * */ +#include <linux/rculist.h> +#include <linux/module.h> #include <linux/kernel.h> #include <linux/async_tx.h> #ifdef CONFIG_DMA_ENGINE -static enum dma_state_client -dma_channel_add_remove(struct dma_client *client, - struct dma_chan *chan, enum dma_state state); - -static struct dma_client async_tx_dma = { - .event_callback = dma_channel_add_remove, - /* .cap_mask == 0 defaults to all channels */ -}; - -/** - * dma_cap_mask_all - enable iteration over all operation types - */ -static dma_cap_mask_t dma_cap_mask_all; - -/** - * chan_ref_percpu - tracks channel allocations per core/opertion - */ -struct chan_ref_percpu { - struct dma_chan_ref *ref; -}; - -static int channel_table_initialized; -static struct chan_ref_percpu *channel_table[DMA_TX_TYPE_END]; - -/** - * async_tx_lock - protect modification of async_tx_master_list and serialize - * rebalance operations - */ -static spinlock_t async_tx_lock; - -static LIST_HEAD(async_tx_master_list); - -/* async_tx_issue_pending_all - start all transactions on all channels */ -void async_tx_issue_pending_all(void) -{ - struct dma_chan_ref *ref; - - rcu_read_lock(); - list_for_each_entry_rcu(ref, &async_tx_master_list, node) - ref->chan->device->device_issue_pending(ref->chan); - rcu_read_unlock(); -} -EXPORT_SYMBOL_GPL(async_tx_issue_pending_all); - -/* dma_wait_for_async_tx - spin wait for a transcation to complete - * @tx: transaction to wait on - */ -enum dma_status -dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx) +static int __init async_tx_init(void) { - enum dma_status status; - struct dma_async_tx_descriptor *iter; - struct dma_async_tx_descriptor *parent; - - if (!tx) - return DMA_SUCCESS; - - /* poll through the dependency chain, return when tx is complete */ - do { - iter = tx; - - /* find the root of the unsubmitted dependency chain */ - while (iter->cookie == -EBUSY) { - parent = iter->parent; - if (parent && parent->cookie == -EBUSY) - iter = iter->parent; - else - break; - } - - status = dma_sync_wait(iter->chan, iter->cookie); - } while (status == DMA_IN_PROGRESS || (iter != tx)); + async_dmaengine_get(); - return status; -} -EXPORT_SYMBOL_GPL(dma_wait_for_async_tx); - -/* async_tx_run_dependencies - helper routine for dma drivers to process - * (start) dependent operations on their target channel - * @tx: transaction with dependencies - */ -void -async_tx_run_dependencies(struct dma_async_tx_descriptor *tx) -{ - struct dma_async_tx_descriptor *dep_tx, *_dep_tx; - struct dma_device *dev; - struct dma_chan *chan; + printk(KERN_INFO "async_tx: api initialized (async)\n"); - list_for_each_entry_safe(dep_tx, _dep_tx, &tx->depend_list, - depend_node) { - chan = dep_tx->chan; - dev = chan->device; - /* we can't depend on ourselves */ - BUG_ON(chan == tx->chan); - list_del(&dep_tx->depend_node); - tx->tx_submit(dep_tx); - - /* we need to poke the engine as client code does not - * know about dependency submission events - */ - dev->device_issue_pending(chan); - } + return 0; } -EXPORT_SYMBOL_GPL(async_tx_run_dependencies); -static void -free_dma_chan_ref(struct rcu_head *rcu) +static void __exit async_tx_exit(void) { - struct dma_chan_ref *ref; - ref = container_of(rcu, struct dma_chan_ref, rcu); - kfree(ref); + async_dmaengine_put(); } -static void -init_dma_chan_ref(struct dma_chan_ref *ref, struct dma_chan *chan) -{ - INIT_LIST_HEAD(&ref->node); - INIT_RCU_HEAD(&ref->rcu); - ref->chan = chan; - atomic_set(&ref->count, 0); -} +module_init(async_tx_init); +module_exit(async_tx_exit); /** - * get_chan_ref_by_cap - returns the nth channel of the given capability - * defaults to returning the channel with the desired capability and the - * lowest reference count if the index can not be satisfied - * @cap: capability to match - * @index: nth channel desired, passing -1 has the effect of forcing the - * default return value + * __async_tx_find_channel - find a channel to carry out the operation or let + * the transaction execute synchronously + * @submit: transaction dependency and submission modifiers + * @tx_type: transaction type */ -static struct dma_chan_ref * -get_chan_ref_by_cap(enum dma_transaction_type cap, int index) +struct dma_chan * +__async_tx_find_channel(struct async_submit_ctl *submit, + enum dma_transaction_type tx_type) { - struct dma_chan_ref *ret_ref = NULL, *min_ref = NULL, *ref; - - rcu_read_lock(); - list_for_each_entry_rcu(ref, &async_tx_master_list, node) - if (dma_has_cap(cap, ref->chan->device->cap_mask)) { - if (!min_ref) - min_ref = ref; - else if (atomic_read(&ref->count) < - atomic_read(&min_ref->count)) - min_ref = ref; - - if (index-- == 0) { - ret_ref = ref; - break; - } - } - rcu_read_unlock(); - - if (!ret_ref) - ret_ref = min_ref; + struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; - if (ret_ref) - atomic_inc(&ret_ref->count); - - return ret_ref; + /* see if we can keep the chain on one channel */ + if (depend_tx && + dma_has_cap(tx_type, depend_tx->chan->device->cap_mask)) + return depend_tx->chan; + return async_dma_find_channel(tx_type); } +EXPORT_SYMBOL_GPL(__async_tx_find_channel); +#endif + /** - * async_tx_rebalance - redistribute the available channels, optimize - * for cpu isolation in the SMP case, and opertaion isolation in the - * uniprocessor case + * async_tx_channel_switch - queue an interrupt descriptor with a dependency + * pre-attached. + * @depend_tx: the operation that must finish before the new operation runs + * @tx: the new operation */ -static void async_tx_rebalance(void) +static void +async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx, + struct dma_async_tx_descriptor *tx) { - int cpu, cap, cpu_idx = 0; - unsigned long flags; + struct dma_chan *chan = depend_tx->chan; + struct dma_device *device = chan->device; + struct dma_async_tx_descriptor *intr_tx = (void *) ~0; + + /* first check to see if we can still append to depend_tx */ + txd_lock(depend_tx); + if (txd_parent(depend_tx) && depend_tx->chan == tx->chan) { + txd_chain(depend_tx, tx); + intr_tx = NULL; + } + txd_unlock(depend_tx); - if (!channel_table_initialized) + /* attached dependency, flush the parent channel */ + if (!intr_tx) { + device->device_issue_pending(chan); return; - - spin_lock_irqsave(&async_tx_lock, flags); - - /* undo the last distribution */ - for_each_dma_cap_mask(cap, dma_cap_mask_all) - for_each_possible_cpu(cpu) { - struct dma_chan_ref *ref = - per_cpu_ptr(channel_table[cap], cpu)->ref; - if (ref) { - atomic_set(&ref->count, 0); - per_cpu_ptr(channel_table[cap], cpu)->ref = - NULL; - } - } - - for_each_dma_cap_mask(cap, dma_cap_mask_all) - for_each_online_cpu(cpu) { - struct dma_chan_ref *new; - if (NR_CPUS > 1) - new = get_chan_ref_by_cap(cap, cpu_idx++); - else - new = get_chan_ref_by_cap(cap, -1); - - per_cpu_ptr(channel_table[cap], cpu)->ref = new; - } - - spin_unlock_irqrestore(&async_tx_lock, flags); -} - -static enum dma_state_client -dma_channel_add_remove(struct dma_client *client, - struct dma_chan *chan, enum dma_state state) -{ - unsigned long found, flags; - struct dma_chan_ref *master_ref, *ref; - enum dma_state_client ack = DMA_DUP; /* default: take no action */ - - switch (state) { - case DMA_RESOURCE_AVAILABLE: - found = 0; - rcu_read_lock(); - list_for_each_entry_rcu(ref, &async_tx_master_list, node) - if (ref->chan == chan) { - found = 1; - break; - } - rcu_read_unlock(); - - pr_debug("async_tx: dma resource available [%s]\n", - found ? "old" : "new"); - - if (!found) - ack = DMA_ACK; - else - break; - - /* add the channel to the generic management list */ - master_ref = kmalloc(sizeof(*master_ref), GFP_KERNEL); - if (master_ref) { - /* keep a reference until async_tx is unloaded */ - dma_chan_get(chan); - init_dma_chan_ref(master_ref, chan); - spin_lock_irqsave(&async_tx_lock, flags); - list_add_tail_rcu(&master_ref->node, - &async_tx_master_list); - spin_unlock_irqrestore(&async_tx_lock, - flags); - } else { - printk(KERN_WARNING "async_tx: unable to create" - " new master entry in response to" - " a DMA_RESOURCE_ADDED event" - " (-ENOMEM)\n"); - return 0; - } - - async_tx_rebalance(); - break; - case DMA_RESOURCE_REMOVED: - found = 0; - spin_lock_irqsave(&async_tx_lock, flags); - list_for_each_entry_rcu(ref, &async_tx_master_list, node) - if (ref->chan == chan) { - /* permit backing devices to go away */ - dma_chan_put(ref->chan); - list_del_rcu(&ref->node); - call_rcu(&ref->rcu, free_dma_chan_ref); - found = 1; - break; - } - spin_unlock_irqrestore(&async_tx_lock, flags); - - pr_debug("async_tx: dma resource removed [%s]\n", - found ? "ours" : "not ours"); - - if (found) - ack = DMA_ACK; - else - break; - - async_tx_rebalance(); - break; - case DMA_RESOURCE_SUSPEND: - case DMA_RESOURCE_RESUME: - printk(KERN_WARNING "async_tx: does not support dma channel" - " suspend/resume\n"); - break; - default: - BUG(); } - return ack; -} - -static int __init -async_tx_init(void) -{ - enum dma_transaction_type cap; - - spin_lock_init(&async_tx_lock); - bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END); - - /* an interrupt will never be an explicit operation type. - * clearing this bit prevents allocation to a slot in 'channel_table' + /* see if we can schedule an interrupt + * otherwise poll for completion */ - clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits); + if (dma_has_cap(DMA_INTERRUPT, device->cap_mask)) + intr_tx = device->device_prep_dma_interrupt(chan, 0); + else + intr_tx = NULL; + + if (intr_tx) { + intr_tx->callback = NULL; + intr_tx->callback_param = NULL; + /* safe to chain outside the lock since we know we are + * not submitted yet + */ + txd_chain(intr_tx, tx); + + /* check if we need to append */ + txd_lock(depend_tx); + if (txd_parent(depend_tx)) { + txd_chain(depend_tx, intr_tx); + async_tx_ack(intr_tx); + intr_tx = NULL; + } + txd_unlock(depend_tx); - for_each_dma_cap_mask(cap, dma_cap_mask_all) { - channel_table[cap] = alloc_percpu(struct chan_ref_percpu); - if (!channel_table[cap]) - goto err; + if (intr_tx) { + txd_clear_parent(intr_tx); + intr_tx->tx_submit(intr_tx); + async_tx_ack(intr_tx); + } + device->device_issue_pending(chan); + } else { + if (dma_wait_for_async_tx(depend_tx) != DMA_COMPLETE) + panic("%s: DMA error waiting for depend_tx\n", + __func__); + tx->tx_submit(tx); } - - channel_table_initialized = 1; - dma_async_client_register(&async_tx_dma); - dma_async_client_chan_request(&async_tx_dma); - - printk(KERN_INFO "async_tx: api initialized (async)\n"); - - return 0; -err: - printk(KERN_ERR "async_tx: initialization failure\n"); - - while (--cap >= 0) - free_percpu(channel_table[cap]); - - return 1; } -static void __exit async_tx_exit(void) -{ - enum dma_transaction_type cap; - - channel_table_initialized = 0; - - for_each_dma_cap_mask(cap, dma_cap_mask_all) - if (channel_table[cap]) - free_percpu(channel_table[cap]); - - dma_async_client_unregister(&async_tx_dma); -} /** - * __async_tx_find_channel - find a channel to carry out the operation or let - * the transaction execute synchronously - * @depend_tx: transaction dependency - * @tx_type: transaction type + * submit_disposition - flags for routing an incoming operation + * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock + * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch + * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly + * + * while holding depend_tx->lock we must avoid submitting new operations + * to prevent a circular locking dependency with drivers that already + * hold a channel lock when calling async_tx_run_dependencies. */ -struct dma_chan * -__async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx, - enum dma_transaction_type tx_type) -{ - /* see if we can keep the chain on one channel */ - if (depend_tx && - dma_has_cap(tx_type, depend_tx->chan->device->cap_mask)) - return depend_tx->chan; - else if (likely(channel_table_initialized)) { - struct dma_chan_ref *ref; - int cpu = get_cpu(); - ref = per_cpu_ptr(channel_table[tx_type], cpu)->ref; - put_cpu(); - return ref ? ref->chan : NULL; - } else - return NULL; -} -EXPORT_SYMBOL_GPL(__async_tx_find_channel); -#else -static int __init async_tx_init(void) -{ - printk(KERN_INFO "async_tx: api initialized (sync-only)\n"); - return 0; -} - -static void __exit async_tx_exit(void) -{ - do { } while (0); -} -#endif +enum submit_disposition { + ASYNC_TX_SUBMITTED, + ASYNC_TX_CHANNEL_SWITCH, + ASYNC_TX_DIRECT_SUBMIT, +}; void async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, - enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) + struct async_submit_ctl *submit) { - tx->callback = cb_fn; - tx->callback_param = cb_param; + struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; - /* set this new tx to run after depend_tx if: - * 1/ a dependency exists (depend_tx is !NULL) - * 2/ the tx can not be submitted to the current channel - */ - if (depend_tx && depend_tx->chan != chan) { - /* if ack is already set then we cannot be sure + tx->callback = submit->cb_fn; + tx->callback_param = submit->cb_param; + + if (depend_tx) { + enum submit_disposition s; + + /* sanity check the dependency chain: + * 1/ if ack is already set then we cannot be sure * we are referring to the correct operation + * 2/ dependencies are 1:1 i.e. two transactions can + * not depend on the same parent + */ + BUG_ON(async_tx_test_ack(depend_tx) || txd_next(depend_tx) || + txd_parent(tx)); + + /* the lock prevents async_tx_run_dependencies from missing + * the setting of ->next when ->parent != NULL */ - BUG_ON(depend_tx->ack); - - tx->parent = depend_tx; - spin_lock_bh(&depend_tx->lock); - list_add_tail(&tx->depend_node, &depend_tx->depend_list); - if (depend_tx->cookie == 0) { - struct dma_chan *dep_chan = depend_tx->chan; - struct dma_device *dep_dev = dep_chan->device; - dep_dev->device_dependency_added(dep_chan); + txd_lock(depend_tx); + if (txd_parent(depend_tx)) { + /* we have a parent so we can not submit directly + * if we are staying on the same channel: append + * else: channel switch + */ + if (depend_tx->chan == chan) { + txd_chain(depend_tx, tx); + s = ASYNC_TX_SUBMITTED; + } else + s = ASYNC_TX_CHANNEL_SWITCH; + } else { + /* we do not have a parent so we may be able to submit + * directly if we are staying on the same channel + */ + if (depend_tx->chan == chan) + s = ASYNC_TX_DIRECT_SUBMIT; + else + s = ASYNC_TX_CHANNEL_SWITCH; } - spin_unlock_bh(&depend_tx->lock); + txd_unlock(depend_tx); - /* schedule an interrupt to trigger the channel switch */ - async_trigger_callback(ASYNC_TX_ACK, depend_tx, NULL, NULL); + switch (s) { + case ASYNC_TX_SUBMITTED: + break; + case ASYNC_TX_CHANNEL_SWITCH: + async_tx_channel_switch(depend_tx, tx); + break; + case ASYNC_TX_DIRECT_SUBMIT: + txd_clear_parent(tx); + tx->tx_submit(tx); + break; + } } else { - tx->parent = NULL; + txd_clear_parent(tx); tx->tx_submit(tx); } - if (flags & ASYNC_TX_ACK) + if (submit->flags & ASYNC_TX_ACK) async_tx_ack(tx); - if (depend_tx && (flags & ASYNC_TX_DEP_ACK)) + if (depend_tx) async_tx_ack(depend_tx); } EXPORT_SYMBOL_GPL(async_tx_submit); /** - * async_trigger_callback - schedules the callback function to be run after - * any dependent operations have been completed. - * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK - * @depend_tx: 'callback' requires the completion of this transaction - * @cb_fn: function to call after depend_tx completes - * @cb_param: parameter to pass to the callback routine + * async_trigger_callback - schedules the callback function to be run + * @submit: submission and completion parameters + * + * honored flags: ASYNC_TX_ACK + * + * The callback is run after any dependent operations have completed. */ struct dma_async_tx_descriptor * -async_trigger_callback(enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) +async_trigger_callback(struct async_submit_ctl *submit) { struct dma_chan *chan; struct dma_device *device; struct dma_async_tx_descriptor *tx; + struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; if (depend_tx) { chan = depend_tx->chan; @@ -467,37 +248,46 @@ async_trigger_callback(enum async_tx_flags flags, if (device && !dma_has_cap(DMA_INTERRUPT, device->cap_mask)) device = NULL; - tx = device ? device->device_prep_dma_interrupt(chan) : NULL; + tx = device ? device->device_prep_dma_interrupt(chan, 0) : NULL; } else tx = NULL; if (tx) { pr_debug("%s: (async)\n", __func__); - async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); + async_tx_submit(chan, tx, submit); } else { pr_debug("%s: (sync)\n", __func__); /* wait for any prerequisite operations */ - if (depend_tx) { - /* if ack is already set then we cannot be sure - * we are referring to the correct operation - */ - BUG_ON(depend_tx->ack); - if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR) - panic("%s: DMA_ERROR waiting for depend_tx\n", - __func__); - } + async_tx_quiesce(&submit->depend_tx); - async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param); + async_tx_sync_epilog(submit); } return tx; } EXPORT_SYMBOL_GPL(async_trigger_callback); -module_init(async_tx_init); -module_exit(async_tx_exit); +/** + * async_tx_quiesce - ensure tx is complete and freeable upon return + * @tx - transaction to quiesce + */ +void async_tx_quiesce(struct dma_async_tx_descriptor **tx) +{ + if (*tx) { + /* if ack is already set then we cannot be sure + * we are referring to the correct operation + */ + BUG_ON(async_tx_test_ack(*tx)); + if (dma_wait_for_async_tx(*tx) != DMA_COMPLETE) + panic("%s: DMA error waiting for transaction\n", + __func__); + async_tx_ack(*tx); + *tx = NULL; + } +} +EXPORT_SYMBOL_GPL(async_tx_quiesce); MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API"); diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c index 1c445c7bdab..3c562f5a60b 100644 --- a/crypto/async_tx/async_xor.c +++ b/crypto/async_tx/async_xor.c @@ -25,328 +25,321 @@ */ #include <linux/kernel.h> #include <linux/interrupt.h> +#include <linux/module.h> #include <linux/mm.h> #include <linux/dma-mapping.h> #include <linux/raid/xor.h> #include <linux/async_tx.h> -/* do_async_xor - dma map the pages and perform the xor with an engine. - * This routine is marked __always_inline so it can be compiled away - * when CONFIG_DMA_ENGINE=n - */ -static __always_inline struct dma_async_tx_descriptor * -do_async_xor(struct dma_device *device, - struct dma_chan *chan, struct page *dest, struct page **src_list, - unsigned int offset, unsigned int src_cnt, size_t len, - enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) +/* do_async_xor - dma map the pages and perform the xor with an engine */ +static __async_inline struct dma_async_tx_descriptor * +do_async_xor(struct dma_chan *chan, struct dmaengine_unmap_data *unmap, + struct async_submit_ctl *submit) { - dma_addr_t dma_dest; - dma_addr_t *dma_src = (dma_addr_t *) src_list; - struct dma_async_tx_descriptor *tx; - int i; - unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; + struct dma_device *dma = chan->device; + struct dma_async_tx_descriptor *tx = NULL; + dma_async_tx_callback cb_fn_orig = submit->cb_fn; + void *cb_param_orig = submit->cb_param; + enum async_tx_flags flags_orig = submit->flags; + enum dma_ctrl_flags dma_flags = 0; + int src_cnt = unmap->to_cnt; + int xor_src_cnt; + dma_addr_t dma_dest = unmap->addr[unmap->to_cnt]; + dma_addr_t *src_list = unmap->addr; - pr_debug("%s: len: %zu\n", __func__, len); + while (src_cnt) { + dma_addr_t tmp; - dma_dest = dma_map_page(device->dev, dest, offset, len, - DMA_FROM_DEVICE); + submit->flags = flags_orig; + xor_src_cnt = min(src_cnt, (int)dma->max_xor); + /* if we are submitting additional xors, leave the chain open + * and clear the callback parameters + */ + if (src_cnt > xor_src_cnt) { + submit->flags &= ~ASYNC_TX_ACK; + submit->flags |= ASYNC_TX_FENCE; + submit->cb_fn = NULL; + submit->cb_param = NULL; + } else { + submit->cb_fn = cb_fn_orig; + submit->cb_param = cb_param_orig; + } + if (submit->cb_fn) + dma_flags |= DMA_PREP_INTERRUPT; + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; - for (i = 0; i < src_cnt; i++) - dma_src[i] = dma_map_page(device->dev, src_list[i], offset, - len, DMA_TO_DEVICE); - - /* Since we have clobbered the src_list we are committed - * to doing this asynchronously. Drivers force forward progress - * in case they can not provide a descriptor - */ - tx = device->device_prep_dma_xor(chan, dma_dest, dma_src, src_cnt, len, - dma_prep_flags); - if (!tx) { - if (depend_tx) - dma_wait_for_async_tx(depend_tx); - - while (!tx) - tx = device->device_prep_dma_xor(chan, dma_dest, - dma_src, src_cnt, len, - dma_prep_flags); - } + /* Drivers force forward progress in case they can not provide a + * descriptor + */ + tmp = src_list[0]; + if (src_list > unmap->addr) + src_list[0] = dma_dest; + tx = dma->device_prep_dma_xor(chan, dma_dest, src_list, + xor_src_cnt, unmap->len, + dma_flags); + src_list[0] = tmp; + + + if (unlikely(!tx)) + async_tx_quiesce(&submit->depend_tx); + + /* spin wait for the preceding transactions to complete */ + while (unlikely(!tx)) { + dma_async_issue_pending(chan); + tx = dma->device_prep_dma_xor(chan, dma_dest, + src_list, + xor_src_cnt, unmap->len, + dma_flags); + } - async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); + submit->depend_tx = tx; + + if (src_cnt > xor_src_cnt) { + /* drop completed sources */ + src_cnt -= xor_src_cnt; + /* use the intermediate result a source */ + src_cnt++; + src_list += xor_src_cnt - 1; + } else + break; + } return tx; } static void do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, - unsigned int src_cnt, size_t len, enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) + int src_cnt, size_t len, struct async_submit_ctl *submit) { - void *_dest; int i; + int xor_src_cnt = 0; + int src_off = 0; + void *dest_buf; + void **srcs; - pr_debug("%s: len: %zu\n", __func__, len); + if (submit->scribble) + srcs = submit->scribble; + else + srcs = (void **) src_list; - /* reuse the 'src_list' array to convert to buffer pointers */ + /* convert to buffer pointers */ for (i = 0; i < src_cnt; i++) - src_list[i] = (struct page *) - (page_address(src_list[i]) + offset); - + if (src_list[i]) + srcs[xor_src_cnt++] = page_address(src_list[i]) + offset; + src_cnt = xor_src_cnt; /* set destination address */ - _dest = page_address(dest) + offset; + dest_buf = page_address(dest) + offset; - if (flags & ASYNC_TX_XOR_ZERO_DST) - memset(_dest, 0, len); + if (submit->flags & ASYNC_TX_XOR_ZERO_DST) + memset(dest_buf, 0, len); - xor_blocks(src_cnt, len, _dest, - (void **) src_list); + while (src_cnt > 0) { + /* process up to 'MAX_XOR_BLOCKS' sources */ + xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS); + xor_blocks(xor_src_cnt, len, dest_buf, &srcs[src_off]); - async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param); + /* drop completed sources */ + src_cnt -= xor_src_cnt; + src_off += xor_src_cnt; + } + + async_tx_sync_epilog(submit); } /** * async_xor - attempt to xor a set of blocks with a dma engine. - * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST - * flag must be set to not include dest data in the calculation. The - * assumption with dma eninges is that they only use the destination - * buffer as a source when it is explicity specified in the source list. * @dest: destination page - * @src_list: array of source pages (if the dest is also a source it must be - * at index zero). The contents of this array may be overwritten. - * @offset: offset in pages to start transaction + * @src_list: array of source pages + * @offset: common src/dst offset to start transaction * @src_cnt: number of source pages * @len: length in bytes - * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST, - * ASYNC_TX_ACK, ASYNC_TX_DEP_ACK - * @depend_tx: xor depends on the result of this transaction. - * @cb_fn: function to call when the xor completes - * @cb_param: parameter to pass to the callback routine + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST + * + * xor_blocks always uses the dest as a source so the + * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in + * the calculation. The assumption with dma eninges is that they only + * use the destination buffer as a source when it is explicity specified + * in the source list. + * + * src_list note: if the dest is also a source it must be at index zero. + * The contents of this array will be overwritten if a scribble region + * is not specified. */ struct dma_async_tx_descriptor * async_xor(struct page *dest, struct page **src_list, unsigned int offset, - int src_cnt, size_t len, enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) + int src_cnt, size_t len, struct async_submit_ctl *submit) { - struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR, + struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR, &dest, 1, src_list, src_cnt, len); struct dma_device *device = chan ? chan->device : NULL; - struct dma_async_tx_descriptor *tx = NULL; - dma_async_tx_callback _cb_fn; - void *_cb_param; - unsigned long local_flags; - int xor_src_cnt; - int i = 0, src_off = 0; + struct dmaengine_unmap_data *unmap = NULL; BUG_ON(src_cnt <= 1); - while (src_cnt) { - local_flags = flags; - if (device) { /* run the xor asynchronously */ - xor_src_cnt = min(src_cnt, device->max_xor); - /* if we are submitting additional xors - * only set the callback on the last transaction - */ - if (src_cnt > xor_src_cnt) { - local_flags &= ~ASYNC_TX_ACK; - _cb_fn = NULL; - _cb_param = NULL; - } else { - _cb_fn = cb_fn; - _cb_param = cb_param; - } - - tx = do_async_xor(device, chan, dest, - &src_list[src_off], offset, - xor_src_cnt, len, local_flags, - depend_tx, _cb_fn, _cb_param); - } else { /* run the xor synchronously */ - /* in the sync case the dest is an implied source - * (assumes the dest is at the src_off index) - */ - if (flags & ASYNC_TX_XOR_DROP_DST) { - src_cnt--; - src_off++; - } + if (device) + unmap = dmaengine_get_unmap_data(device->dev, src_cnt+1, GFP_NOIO); - /* process up to 'MAX_XOR_BLOCKS' sources */ - xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS); - - /* if we are submitting additional xors - * only set the callback on the last transaction - */ - if (src_cnt > xor_src_cnt) { - local_flags &= ~ASYNC_TX_ACK; - _cb_fn = NULL; - _cb_param = NULL; - } else { - _cb_fn = cb_fn; - _cb_param = cb_param; - } + if (unmap && is_dma_xor_aligned(device, offset, 0, len)) { + struct dma_async_tx_descriptor *tx; + int i, j; - /* wait for any prerequisite operations */ - if (depend_tx) { - /* if ack is already set then we cannot be sure - * we are referring to the correct operation - */ - BUG_ON(depend_tx->ack); - if (dma_wait_for_async_tx(depend_tx) == - DMA_ERROR) - panic("%s: DMA_ERROR waiting for " - "depend_tx\n", - __func__); - } + /* run the xor asynchronously */ + pr_debug("%s (async): len: %zu\n", __func__, len); - do_sync_xor(dest, &src_list[src_off], offset, - xor_src_cnt, len, local_flags, depend_tx, - _cb_fn, _cb_param); + unmap->len = len; + for (i = 0, j = 0; i < src_cnt; i++) { + if (!src_list[i]) + continue; + unmap->to_cnt++; + unmap->addr[j++] = dma_map_page(device->dev, src_list[i], + offset, len, DMA_TO_DEVICE); } - /* the previous tx is hidden from the client, - * so ack it - */ - if (i && depend_tx) - async_tx_ack(depend_tx); + /* map it bidirectional as it may be re-used as a source */ + unmap->addr[j] = dma_map_page(device->dev, dest, offset, len, + DMA_BIDIRECTIONAL); + unmap->bidi_cnt = 1; - depend_tx = tx; + tx = do_async_xor(chan, unmap, submit); + dmaengine_unmap_put(unmap); + return tx; + } else { + dmaengine_unmap_put(unmap); + /* run the xor synchronously */ + pr_debug("%s (sync): len: %zu\n", __func__, len); + WARN_ONCE(chan, "%s: no space for dma address conversion\n", + __func__); + + /* in the sync case the dest is an implied source + * (assumes the dest is the first source) + */ + if (submit->flags & ASYNC_TX_XOR_DROP_DST) { + src_cnt--; + src_list++; + } - if (src_cnt > xor_src_cnt) { - /* drop completed sources */ - src_cnt -= xor_src_cnt; - src_off += xor_src_cnt; + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); - /* unconditionally preserve the destination */ - flags &= ~ASYNC_TX_XOR_ZERO_DST; + do_sync_xor(dest, src_list, offset, src_cnt, len, submit); - /* use the intermediate result a source, but remember - * it's dropped, because it's implied, in the sync case - */ - src_list[--src_off] = dest; - src_cnt++; - flags |= ASYNC_TX_XOR_DROP_DST; - } else - src_cnt = 0; - i++; + return NULL; } - - return tx; } EXPORT_SYMBOL_GPL(async_xor); static int page_is_zero(struct page *p, unsigned int offset, size_t len) { - char *a = page_address(p) + offset; - return ((*(u32 *) a) == 0 && - memcmp(a, a + 4, len - 4) == 0); + return !memchr_inv(page_address(p) + offset, 0, len); +} + +static inline struct dma_chan * +xor_val_chan(struct async_submit_ctl *submit, struct page *dest, + struct page **src_list, int src_cnt, size_t len) +{ + #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA + return NULL; + #endif + return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list, + src_cnt, len); } /** - * async_xor_zero_sum - attempt a xor parity check with a dma engine. + * async_xor_val - attempt a xor parity check with a dma engine. * @dest: destination page used if the xor is performed synchronously - * @src_list: array of source pages. The dest page must be listed as a source - * at index zero. The contents of this array may be overwritten. + * @src_list: array of source pages * @offset: offset in pages to start transaction * @src_cnt: number of source pages * @len: length in bytes * @result: 0 if sum == 0 else non-zero - * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK - * @depend_tx: xor depends on the result of this transaction. - * @cb_fn: function to call when the xor completes - * @cb_param: parameter to pass to the callback routine + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK + * + * src_list note: if the dest is also a source it must be at index zero. + * The contents of this array will be overwritten if a scribble region + * is not specified. */ struct dma_async_tx_descriptor * -async_xor_zero_sum(struct page *dest, struct page **src_list, - unsigned int offset, int src_cnt, size_t len, - u32 *result, enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) +async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, + int src_cnt, size_t len, enum sum_check_flags *result, + struct async_submit_ctl *submit) { - struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_ZERO_SUM, - &dest, 1, src_list, - src_cnt, len); + struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx = NULL; + struct dmaengine_unmap_data *unmap = NULL; BUG_ON(src_cnt <= 1); - if (device && src_cnt <= device->max_xor) { - dma_addr_t *dma_src = (dma_addr_t *) src_list; - unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; + if (device) + unmap = dmaengine_get_unmap_data(device->dev, src_cnt, GFP_NOIO); + + if (unmap && src_cnt <= device->max_xor && + is_dma_xor_aligned(device, offset, 0, len)) { + unsigned long dma_prep_flags = 0; int i; pr_debug("%s: (async) len: %zu\n", __func__, len); - for (i = 0; i < src_cnt; i++) - dma_src[i] = dma_map_page(device->dev, src_list[i], - offset, len, DMA_TO_DEVICE); - - tx = device->device_prep_dma_zero_sum(chan, dma_src, src_cnt, - len, result, - dma_prep_flags); - if (!tx) { - if (depend_tx) - dma_wait_for_async_tx(depend_tx); + if (submit->cb_fn) + dma_prep_flags |= DMA_PREP_INTERRUPT; + if (submit->flags & ASYNC_TX_FENCE) + dma_prep_flags |= DMA_PREP_FENCE; - while (!tx) - tx = device->device_prep_dma_zero_sum(chan, - dma_src, src_cnt, len, result, + for (i = 0; i < src_cnt; i++) { + unmap->addr[i] = dma_map_page(device->dev, src_list[i], + offset, len, DMA_TO_DEVICE); + unmap->to_cnt++; + } + unmap->len = len; + + tx = device->device_prep_dma_xor_val(chan, unmap->addr, src_cnt, + len, result, + dma_prep_flags); + if (unlikely(!tx)) { + async_tx_quiesce(&submit->depend_tx); + + while (!tx) { + dma_async_issue_pending(chan); + tx = device->device_prep_dma_xor_val(chan, + unmap->addr, src_cnt, len, result, dma_prep_flags); + } } - - async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); + dma_set_unmap(tx, unmap); + async_tx_submit(chan, tx, submit); } else { - unsigned long xor_flags = flags; + enum async_tx_flags flags_orig = submit->flags; pr_debug("%s: (sync) len: %zu\n", __func__, len); + WARN_ONCE(device && src_cnt <= device->max_xor, + "%s: no space for dma address conversion\n", + __func__); - xor_flags |= ASYNC_TX_XOR_DROP_DST; - xor_flags &= ~ASYNC_TX_ACK; - - tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags, - depend_tx, NULL, NULL); + submit->flags |= ASYNC_TX_XOR_DROP_DST; + submit->flags &= ~ASYNC_TX_ACK; - if (tx) { - if (dma_wait_for_async_tx(tx) == DMA_ERROR) - panic("%s: DMA_ERROR waiting for tx\n", - __func__); - async_tx_ack(tx); - } + tx = async_xor(dest, src_list, offset, src_cnt, len, submit); - *result = page_is_zero(dest, offset, len) ? 0 : 1; + async_tx_quiesce(&tx); - tx = NULL; + *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P; - async_tx_sync_epilog(flags, depend_tx, cb_fn, cb_param); + async_tx_sync_epilog(submit); + submit->flags = flags_orig; } + dmaengine_unmap_put(unmap); return tx; } -EXPORT_SYMBOL_GPL(async_xor_zero_sum); - -static int __init async_xor_init(void) -{ - #ifdef CONFIG_DMA_ENGINE - /* To conserve stack space the input src_list (array of page pointers) - * is reused to hold the array of dma addresses passed to the driver. - * This conversion is only possible when dma_addr_t is less than the - * the size of a pointer. HIGHMEM64G is known to violate this - * assumption. - */ - BUILD_BUG_ON(sizeof(dma_addr_t) > sizeof(struct page *)); - #endif - - return 0; -} - -static void __exit async_xor_exit(void) -{ - do { } while (0); -} - -module_init(async_xor_init); -module_exit(async_xor_exit); +EXPORT_SYMBOL_GPL(async_xor_val); MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api"); diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c new file mode 100644 index 00000000000..dad95f45b88 --- /dev/null +++ b/crypto/async_tx/raid6test.c @@ -0,0 +1,253 @@ +/* + * asynchronous raid6 recovery self test + * Copyright (c) 2009, Intel Corporation. + * + * based on drivers/md/raid6test/test.c: + * Copyright 2002-2007 H. Peter Anvin + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +#include <linux/async_tx.h> +#include <linux/gfp.h> +#include <linux/mm.h> +#include <linux/random.h> +#include <linux/module.h> + +#undef pr +#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args) + +#define NDISKS 64 /* Including P and Q */ + +static struct page *dataptrs[NDISKS]; +static addr_conv_t addr_conv[NDISKS]; +static struct page *data[NDISKS+3]; +static struct page *spare; +static struct page *recovi; +static struct page *recovj; + +static void callback(void *param) +{ + struct completion *cmp = param; + + complete(cmp); +} + +static void makedata(int disks) +{ + int i; + + for (i = 0; i < disks; i++) { + prandom_bytes(page_address(data[i]), PAGE_SIZE); + dataptrs[i] = data[i]; + } +} + +static char disk_type(int d, int disks) +{ + if (d == disks - 2) + return 'P'; + else if (d == disks - 1) + return 'Q'; + else + return 'D'; +} + +/* Recover two failed blocks. */ +static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, struct page **ptrs) +{ + struct async_submit_ctl submit; + struct completion cmp; + struct dma_async_tx_descriptor *tx = NULL; + enum sum_check_flags result = ~0; + + if (faila > failb) + swap(faila, failb); + + if (failb == disks-1) { + if (faila == disks-2) { + /* P+Q failure. Just rebuild the syndrome. */ + init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); + tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit); + } else { + struct page *blocks[disks]; + struct page *dest; + int count = 0; + int i; + + /* data+Q failure. Reconstruct data from P, + * then rebuild syndrome + */ + for (i = disks; i-- ; ) { + if (i == faila || i == failb) + continue; + blocks[count++] = ptrs[i]; + } + dest = ptrs[faila]; + init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, + NULL, NULL, addr_conv); + tx = async_xor(dest, blocks, 0, count, bytes, &submit); + + init_async_submit(&submit, 0, tx, NULL, NULL, addr_conv); + tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit); + } + } else { + if (failb == disks-2) { + /* data+P failure. */ + init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); + tx = async_raid6_datap_recov(disks, bytes, faila, ptrs, &submit); + } else { + /* data+data failure. */ + init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); + tx = async_raid6_2data_recov(disks, bytes, faila, failb, ptrs, &submit); + } + } + init_completion(&cmp); + init_async_submit(&submit, ASYNC_TX_ACK, tx, callback, &cmp, addr_conv); + tx = async_syndrome_val(ptrs, 0, disks, bytes, &result, spare, &submit); + async_tx_issue_pending(tx); + + if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) + pr("%s: timeout! (faila: %d failb: %d disks: %d)\n", + __func__, faila, failb, disks); + + if (result != 0) + pr("%s: validation failure! faila: %d failb: %d sum_check_flags: %x\n", + __func__, faila, failb, result); +} + +static int test_disks(int i, int j, int disks) +{ + int erra, errb; + + memset(page_address(recovi), 0xf0, PAGE_SIZE); + memset(page_address(recovj), 0xba, PAGE_SIZE); + + dataptrs[i] = recovi; + dataptrs[j] = recovj; + + raid6_dual_recov(disks, PAGE_SIZE, i, j, dataptrs); + + erra = memcmp(page_address(data[i]), page_address(recovi), PAGE_SIZE); + errb = memcmp(page_address(data[j]), page_address(recovj), PAGE_SIZE); + + pr("%s(%d, %d): faila=%3d(%c) failb=%3d(%c) %s\n", + __func__, i, j, i, disk_type(i, disks), j, disk_type(j, disks), + (!erra && !errb) ? "OK" : !erra ? "ERRB" : !errb ? "ERRA" : "ERRAB"); + + dataptrs[i] = data[i]; + dataptrs[j] = data[j]; + + return erra || errb; +} + +static int test(int disks, int *tests) +{ + struct dma_async_tx_descriptor *tx; + struct async_submit_ctl submit; + struct completion cmp; + int err = 0; + int i, j; + + recovi = data[disks]; + recovj = data[disks+1]; + spare = data[disks+2]; + + makedata(disks); + + /* Nuke syndromes */ + memset(page_address(data[disks-2]), 0xee, PAGE_SIZE); + memset(page_address(data[disks-1]), 0xee, PAGE_SIZE); + + /* Generate assumed good syndrome */ + init_completion(&cmp); + init_async_submit(&submit, ASYNC_TX_ACK, NULL, callback, &cmp, addr_conv); + tx = async_gen_syndrome(dataptrs, 0, disks, PAGE_SIZE, &submit); + async_tx_issue_pending(tx); + + if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) { + pr("error: initial gen_syndrome(%d) timed out\n", disks); + return 1; + } + + pr("testing the %d-disk case...\n", disks); + for (i = 0; i < disks-1; i++) + for (j = i+1; j < disks; j++) { + (*tests)++; + err += test_disks(i, j, disks); + } + + return err; +} + + +static int raid6_test(void) +{ + int err = 0; + int tests = 0; + int i; + + for (i = 0; i < NDISKS+3; i++) { + data[i] = alloc_page(GFP_KERNEL); + if (!data[i]) { + while (i--) + put_page(data[i]); + return -ENOMEM; + } + } + + /* the 4-disk and 5-disk cases are special for the recovery code */ + if (NDISKS > 4) + err += test(4, &tests); + if (NDISKS > 5) + err += test(5, &tests); + /* the 11 and 12 disk cases are special for ioatdma (p-disabled + * q-continuation without extended descriptor) + */ + if (NDISKS > 12) { + err += test(11, &tests); + err += test(12, &tests); + } + + /* the 24 disk case is special for ioatdma as it is the boudary point + * at which it needs to switch from 8-source ops to 16-source + * ops for continuation (assumes DMA_HAS_PQ_CONTINUE is not set) + */ + if (NDISKS > 24) + err += test(24, &tests); + + err += test(NDISKS, &tests); + + pr("\n"); + pr("complete (%d tests, %d failure%s)\n", + tests, err, err == 1 ? "" : "s"); + + for (i = 0; i < NDISKS+3; i++) + put_page(data[i]); + + return 0; +} + +static void raid6_test_exit(void) +{ +} + +/* when compiled-in wait for drivers to load first (assumes dma drivers + * are also compliled-in) + */ +late_initcall(raid6_test); +module_exit(raid6_test_exit); +MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>"); +MODULE_DESCRIPTION("asynchronous RAID-6 recovery self tests"); +MODULE_LICENSE("GPL"); |