diff options
Diffstat (limited to 'drivers/dma')
| -rw-r--r-- | drivers/dma/Kconfig | 23 | ||||
| -rw-r--r-- | drivers/dma/Makefile | 8 | ||||
| -rw-r--r-- | drivers/dma/dmatest.c | 8 | ||||
| -rw-r--r-- | drivers/dma/fsldma.c | 1177 | ||||
| -rw-r--r-- | drivers/dma/fsldma.h | 35 | ||||
| -rw-r--r-- | drivers/dma/ioat/dma.c | 46 | ||||
| -rw-r--r-- | drivers/dma/ioat/dma.h | 11 | ||||
| -rw-r--r-- | drivers/dma/ioat/dma_v2.c | 70 | ||||
| -rw-r--r-- | drivers/dma/ioat/dma_v2.h | 6 | ||||
| -rw-r--r-- | drivers/dma/ioat/dma_v3.c | 64 | ||||
| -rw-r--r-- | drivers/dma/ioat/registers.h | 2 | ||||
| -rw-r--r-- | drivers/dma/ipu/ipu_idmac.c | 15 | ||||
| -rw-r--r-- | drivers/dma/mpc512x_dma.c | 800 | ||||
| -rw-r--r-- | drivers/dma/ppc4xx/adma.c | 2 |
14 files changed, 1597 insertions, 670 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index e02d74b1e89..c27f80e5d53 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -13,6 +13,22 @@ menuconfig DMADEVICES DMA Device drivers supported by the configured arch, it may be empty in some cases. +config DMADEVICES_DEBUG + bool "DMA Engine debugging" + depends on DMADEVICES != n + help + This is an option for use by developers; most people should + say N here. This enables DMA engine core and driver debugging. + +config DMADEVICES_VDEBUG + bool "DMA Engine verbose debugging" + depends on DMADEVICES_DEBUG != n + help + This is an option for use by developers; most people should + say N here. This enables deeper (more verbose) debugging of + the DMA engine core and drivers. + + if DMADEVICES comment "DMA Devices" @@ -69,6 +85,13 @@ config FSL_DMA The Elo is the DMA controller on some 82xx and 83xx parts, and the Elo Plus is the DMA controller on 85xx and 86xx parts. +config MPC512X_DMA + tristate "Freescale MPC512x built-in DMA engine support" + depends on PPC_MPC512x + select DMA_ENGINE + ---help--- + Enable support for the Freescale MPC512x built-in DMA engine. + config MV_XOR bool "Marvell XOR engine support" depends on PLAT_ORION diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index 807053d4823..22bba3d5e2b 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -1,9 +1,17 @@ +ifeq ($(CONFIG_DMADEVICES_DEBUG),y) + EXTRA_CFLAGS += -DDEBUG +endif +ifeq ($(CONFIG_DMADEVICES_VDEBUG),y) + EXTRA_CFLAGS += -DVERBOSE_DEBUG +endif + obj-$(CONFIG_DMA_ENGINE) += dmaengine.o obj-$(CONFIG_NET_DMA) += iovlock.o obj-$(CONFIG_DMATEST) += dmatest.o obj-$(CONFIG_INTEL_IOATDMA) += ioat/ obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o obj-$(CONFIG_FSL_DMA) += fsldma.o +obj-$(CONFIG_MPC512X_DMA) += mpc512x_dma.o obj-$(CONFIG_MV_XOR) += mv_xor.o obj-$(CONFIG_DW_DMAC) += dw_dmac.o obj-$(CONFIG_AT_HDMAC) += at_hdmac.o diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index 948d563941c..6fa55fe3dd2 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -237,7 +237,7 @@ static int dmatest_func(void *data) dma_cookie_t cookie; enum dma_status status; enum dma_ctrl_flags flags; - u8 pq_coefs[pq_sources]; + u8 pq_coefs[pq_sources + 1]; int ret; int src_cnt; int dst_cnt; @@ -257,7 +257,7 @@ static int dmatest_func(void *data) } else if (thread->type == DMA_PQ) { src_cnt = pq_sources | 1; /* force odd to ensure dst = src */ dst_cnt = 2; - for (i = 0; i < pq_sources; i++) + for (i = 0; i < src_cnt; i++) pq_coefs[i] = 1; } else goto err_srcs; @@ -347,7 +347,7 @@ static int dmatest_func(void *data) else if (thread->type == DMA_XOR) tx = dev->device_prep_dma_xor(chan, dma_dsts[0] + dst_off, - dma_srcs, xor_sources, + dma_srcs, src_cnt, len, flags); else if (thread->type == DMA_PQ) { dma_addr_t dma_pq[dst_cnt]; @@ -355,7 +355,7 @@ static int dmatest_func(void *data) for (i = 0; i < dst_cnt; i++) dma_pq[i] = dma_dsts[i] + dst_off; tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, - pq_sources, pq_coefs, + src_cnt, pq_coefs, len, flags); } diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index 296f9e747fa..bbb4be5a3ff 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -37,19 +37,19 @@ #include <asm/fsldma.h> #include "fsldma.h" -static void dma_init(struct fsl_dma_chan *fsl_chan) +static void dma_init(struct fsldma_chan *chan) { /* Reset the channel */ - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, 0, 32); + DMA_OUT(chan, &chan->regs->mr, 0, 32); - switch (fsl_chan->feature & FSL_DMA_IP_MASK) { + switch (chan->feature & FSL_DMA_IP_MASK) { case FSL_DMA_IP_85XX: /* Set the channel to below modes: * EIE - Error interrupt enable * EOSIE - End of segments interrupt enable (basic mode) * EOLNIE - End of links interrupt enable */ - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EIE + DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EIE | FSL_DMA_MR_EOLNIE | FSL_DMA_MR_EOSIE, 32); break; case FSL_DMA_IP_83XX: @@ -57,170 +57,146 @@ static void dma_init(struct fsl_dma_chan *fsl_chan) * EOTIE - End-of-transfer interrupt enable * PRC_RM - PCI read multiple */ - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, FSL_DMA_MR_EOTIE + DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE | FSL_DMA_MR_PRC_RM, 32); break; } - } -static void set_sr(struct fsl_dma_chan *fsl_chan, u32 val) +static void set_sr(struct fsldma_chan *chan, u32 val) { - DMA_OUT(fsl_chan, &fsl_chan->reg_base->sr, val, 32); + DMA_OUT(chan, &chan->regs->sr, val, 32); } -static u32 get_sr(struct fsl_dma_chan *fsl_chan) +static u32 get_sr(struct fsldma_chan *chan) { - return DMA_IN(fsl_chan, &fsl_chan->reg_base->sr, 32); + return DMA_IN(chan, &chan->regs->sr, 32); } -static void set_desc_cnt(struct fsl_dma_chan *fsl_chan, +static void set_desc_cnt(struct fsldma_chan *chan, struct fsl_dma_ld_hw *hw, u32 count) { - hw->count = CPU_TO_DMA(fsl_chan, count, 32); + hw->count = CPU_TO_DMA(chan, count, 32); } -static void set_desc_src(struct fsl_dma_chan *fsl_chan, +static void set_desc_src(struct fsldma_chan *chan, struct fsl_dma_ld_hw *hw, dma_addr_t src) { u64 snoop_bits; - snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) + snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0; - hw->src_addr = CPU_TO_DMA(fsl_chan, snoop_bits | src, 64); + hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64); } -static void set_desc_dest(struct fsl_dma_chan *fsl_chan, - struct fsl_dma_ld_hw *hw, dma_addr_t dest) +static void set_desc_dst(struct fsldma_chan *chan, + struct fsl_dma_ld_hw *hw, dma_addr_t dst) { u64 snoop_bits; - snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) + snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0; - hw->dst_addr = CPU_TO_DMA(fsl_chan, snoop_bits | dest, 64); + hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64); } -static void set_desc_next(struct fsl_dma_chan *fsl_chan, +static void set_desc_next(struct fsldma_chan *chan, struct fsl_dma_ld_hw *hw, dma_addr_t next) { u64 snoop_bits; - snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) + snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) ? FSL_DMA_SNEN : 0; - hw->next_ln_addr = CPU_TO_DMA(fsl_chan, snoop_bits | next, 64); -} - -static void set_cdar(struct fsl_dma_chan *fsl_chan, dma_addr_t addr) -{ - DMA_OUT(fsl_chan, &fsl_chan->reg_base->cdar, addr | FSL_DMA_SNEN, 64); + hw->next_ln_addr = CPU_TO_DMA(chan, snoop_bits | next, 64); } -static dma_addr_t get_cdar(struct fsl_dma_chan *fsl_chan) +static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr) { - return DMA_IN(fsl_chan, &fsl_chan->reg_base->cdar, 64) & ~FSL_DMA_SNEN; + DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64); } -static void set_ndar(struct fsl_dma_chan *fsl_chan, dma_addr_t addr) +static dma_addr_t get_cdar(struct fsldma_chan *chan) { - DMA_OUT(fsl_chan, &fsl_chan->reg_base->ndar, addr, 64); + return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN; } -static dma_addr_t get_ndar(struct fsl_dma_chan *fsl_chan) +static dma_addr_t get_ndar(struct fsldma_chan *chan) { - return DMA_IN(fsl_chan, &fsl_chan->reg_base->ndar, 64); + return DMA_IN(chan, &chan->regs->ndar, 64); } -static u32 get_bcr(struct fsl_dma_chan *fsl_chan) +static u32 get_bcr(struct fsldma_chan *chan) { - return DMA_IN(fsl_chan, &fsl_chan->reg_base->bcr, 32); + return DMA_IN(chan, &chan->regs->bcr, 32); } -static int dma_is_idle(struct fsl_dma_chan *fsl_chan) +static int dma_is_idle(struct fsldma_chan *chan) { - u32 sr = get_sr(fsl_chan); + u32 sr = get_sr(chan); return (!(sr & FSL_DMA_SR_CB)) || (sr & FSL_DMA_SR_CH); } -static void dma_start(struct fsl_dma_chan *fsl_chan) +static void dma_start(struct fsldma_chan *chan) { - u32 mr_set = 0; - - if (fsl_chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { - DMA_OUT(fsl_chan, &fsl_chan->reg_base->bcr, 0, 32); - mr_set |= FSL_DMA_MR_EMP_EN; - } else if ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) - & ~FSL_DMA_MR_EMP_EN, 32); + u32 mode; + + mode = DMA_IN(chan, &chan->regs->mr, 32); + + if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { + if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { + DMA_OUT(chan, &chan->regs->bcr, 0, 32); + mode |= FSL_DMA_MR_EMP_EN; + } else { + mode &= ~FSL_DMA_MR_EMP_EN; + } } - if (fsl_chan->feature & FSL_DMA_CHAN_START_EXT) - mr_set |= FSL_DMA_MR_EMS_EN; + if (chan->feature & FSL_DMA_CHAN_START_EXT) + mode |= FSL_DMA_MR_EMS_EN; else - mr_set |= FSL_DMA_MR_CS; + mode |= FSL_DMA_MR_CS; - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) - | mr_set, 32); + DMA_OUT(chan, &chan->regs->mr, mode, 32); } -static void dma_halt(struct fsl_dma_chan *fsl_chan) +static void dma_halt(struct fsldma_chan *chan) { + u32 mode; int i; - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | FSL_DMA_MR_CA, - 32); - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & ~(FSL_DMA_MR_CS - | FSL_DMA_MR_EMS_EN | FSL_DMA_MR_CA), 32); + mode = DMA_IN(chan, &chan->regs->mr, 32); + mode |= FSL_DMA_MR_CA; + DMA_OUT(chan, &chan->regs->mr, mode, 32); + + mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN | FSL_DMA_MR_CA); + DMA_OUT(chan, &chan->regs->mr, mode, 32); for (i = 0; i < 100; i++) { - if (dma_is_idle(fsl_chan)) - break; + if (dma_is_idle(chan)) + return; + udelay(10); } - if (i >= 100 && !dma_is_idle(fsl_chan)) - dev_err(fsl_chan->dev, "DMA halt timeout!\n"); + + if (!dma_is_idle(chan)) + dev_err(chan->dev, "DMA halt timeout!\n"); } -static void set_ld_eol(struct fsl_dma_chan *fsl_chan, +static void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc) { u64 snoop_bits; - snoop_bits = ((fsl_chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) + snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_83XX) ? FSL_DMA_SNEN : 0; - desc->hw.next_ln_addr = CPU_TO_DMA(fsl_chan, - DMA_TO_CPU(fsl_chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL + desc->hw.next_ln_addr = CPU_TO_DMA(chan, + DMA_TO_CPU(chan, desc->hw.next_ln_addr, 64) | FSL_DMA_EOL | snoop_bits, 64); } -static void append_ld_queue(struct fsl_dma_chan *fsl_chan, - struct fsl_desc_sw *new_desc) -{ - struct fsl_desc_sw *queue_tail = to_fsl_desc(fsl_chan->ld_queue.prev); - - if (list_empty(&fsl_chan->ld_queue)) - return; - - /* Link to the new descriptor physical address and - * Enable End-of-segment interrupt for - * the last link descriptor. - * (the previous node's next link descriptor) - * - * For FSL_DMA_IP_83xx, the snoop enable bit need be set. - */ - queue_tail->hw.next_ln_addr = CPU_TO_DMA(fsl_chan, - new_desc->async_tx.phys | FSL_DMA_EOSIE | - (((fsl_chan->feature & FSL_DMA_IP_MASK) - == FSL_DMA_IP_83XX) ? FSL_DMA_SNEN : 0), 64); -} - /** * fsl_chan_set_src_loop_size - Set source address hold transfer size - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * @size : Address loop size, 0 for disable loop * * The set source address hold transfer size. The source @@ -229,29 +205,30 @@ static void append_ld_queue(struct fsl_dma_chan *fsl_chan, * read data from SA, SA + 1, SA + 2, SA + 3, then loop back to SA, * SA + 1 ... and so on. */ -static void fsl_chan_set_src_loop_size(struct fsl_dma_chan *fsl_chan, int size) +static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size) { + u32 mode; + + mode = DMA_IN(chan, &chan->regs->mr, 32); + switch (size) { case 0: - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & - (~FSL_DMA_MR_SAHE), 32); + mode &= ~FSL_DMA_MR_SAHE; break; case 1: case 2: case 4: case 8: - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | - FSL_DMA_MR_SAHE | (__ilog2(size) << 14), - 32); + mode |= FSL_DMA_MR_SAHE | (__ilog2(size) << 14); break; } + + DMA_OUT(chan, &chan->regs->mr, mode, 32); } /** - * fsl_chan_set_dest_loop_size - Set destination address hold transfer size - * @fsl_chan : Freescale DMA channel + * fsl_chan_set_dst_loop_size - Set destination address hold transfer size + * @chan : Freescale DMA channel * @size : Address loop size, 0 for disable loop * * The set destination address hold transfer size. The destination @@ -260,29 +237,30 @@ static void fsl_chan_set_src_loop_size(struct fsl_dma_chan *fsl_chan, int size) * write data to TA, TA + 1, TA + 2, TA + 3, then loop back to TA, * TA + 1 ... and so on. */ -static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size) +static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size) { + u32 mode; + + mode = DMA_IN(chan, &chan->regs->mr, 32); + switch (size) { case 0: - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) & - (~FSL_DMA_MR_DAHE), 32); + mode &= ~FSL_DMA_MR_DAHE; break; case 1: case 2: case 4: case 8: - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) | - FSL_DMA_MR_DAHE | (__ilog2(size) << 16), - 32); + mode |= FSL_DMA_MR_DAHE | (__ilog2(size) << 16); break; } + + DMA_OUT(chan, &chan->regs->mr, mode, 32); } /** * fsl_chan_set_request_count - Set DMA Request Count for external control - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * @size : Number of bytes to transfer in a single request * * The Freescale DMA channel can be controlled by the external signal DREQ#. @@ -292,35 +270,38 @@ static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size) * * A size of 0 disables external pause control. The maximum size is 1024. */ -static void fsl_chan_set_request_count(struct fsl_dma_chan *fsl_chan, int size) +static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size) { + u32 mode; + BUG_ON(size > 1024); - DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr, - DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32) - | ((__ilog2(size) << 24) & 0x0f000000), - 32); + + mode = DMA_IN(chan, &chan->regs->mr, 32); + mode |= (__ilog2(size) << 24) & 0x0f000000; + + DMA_OUT(chan, &chan->regs->mr, mode, 32); } /** * fsl_chan_toggle_ext_pause - Toggle channel external pause status - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * @enable : 0 is disabled, 1 is enabled. * * The Freescale DMA channel can be controlled by the external signal DREQ#. * The DMA Request Count feature should be used in addition to this feature * to set the number of bytes to transfer before pausing the channel. */ -static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int enable) +static void fsl_chan_toggle_ext_pause(struct fsldma_chan *chan, int enable) { if (enable) - fsl_chan->feature |= FSL_DMA_CHAN_PAUSE_EXT; + chan->feature |= FSL_DMA_CHAN_PAUSE_EXT; else - fsl_chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT; + chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT; } /** * fsl_chan_toggle_ext_start - Toggle channel external start status - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * @enable : 0 is disabled, 1 is enabled. * * If enable the external start, the channel can be started by an @@ -328,141 +309,196 @@ static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int enable) * transfer immediately. The DMA channel will wait for the * control pin asserted. */ -static void fsl_chan_toggle_ext_start(struct fsl_dma_chan *fsl_chan, int enable) +static void fsl_chan_toggle_ext_start(struct fsldma_chan *chan, int enable) { if (enable) - fsl_chan->feature |= FSL_DMA_CHAN_START_EXT; + chan->feature |= FSL_DMA_CHAN_START_EXT; else - fsl_chan->feature &= ~FSL_DMA_CHAN_START_EXT; + chan->feature &= ~FSL_DMA_CHAN_START_EXT; +} + +static void append_ld_queue(struct fsldma_chan *chan, + struct fsl_desc_sw *desc) +{ + struct fsl_desc_sw *tail = to_fsl_desc(chan->ld_pending.prev); + + if (list_empty(&chan->ld_pending)) + goto out_splice; + + /* + * Add the hardware descriptor to the chain of hardware descriptors + * that already exists in memory. + * + * This will un-set the EOL bit of the existing transaction, and the + * last link in this transaction will become the EOL descriptor. + */ + set_desc_next(chan, &tail->hw, desc->async_tx.phys); + + /* + * Add the software descriptor and all children to the list + * of pending transactions + */ +out_splice: + list_splice_tail_init(&desc->tx_list, &chan->ld_pending); } static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx) { - struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan); + struct fsldma_chan *chan = to_fsl_chan(tx->chan); struct fsl_desc_sw *desc = tx_to_fsl_desc(tx); struct fsl_desc_sw *child; unsigned long flags; dma_cookie_t cookie; - /* cookie increment and adding to ld_queue must be atomic */ - spin_lock_irqsave(&fsl_chan->desc_lock, flags); + spin_lock_irqsave(&chan->desc_lock, flags); - cookie = fsl_chan->common.cookie; + /* + * assign cookies to all of the software descriptors + * that make up this transaction + */ + cookie = chan->common.cookie; list_for_each_entry(child, &desc->tx_list, node) { cookie++; if (cookie < 0) cookie = 1; - desc->async_tx.cookie = cookie; + child->async_tx.cookie = cookie; } - fsl_chan->common.cookie = cookie; - append_ld_queue(fsl_chan, desc); - list_splice_init(&desc->tx_list, fsl_chan->ld_queue.prev); + chan->common.cookie = cookie; + + /* put this transaction onto the tail of the pending queue */ + append_ld_queue(chan, desc); - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); + spin_unlock_irqrestore(&chan->desc_lock, flags); return cookie; } /** * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool. - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * * Return - The descriptor allocated. NULL for failed. */ static struct fsl_desc_sw *fsl_dma_alloc_descriptor( - struct fsl_dma_chan *fsl_chan) + struct fsldma_chan *chan) { + struct fsl_desc_sw *desc; dma_addr_t pdesc; - struct fsl_desc_sw *desc_sw; - - desc_sw = dma_pool_alloc(fsl_chan->desc_pool, GFP_ATOMIC, &pdesc); - if (desc_sw) { - memset(desc_sw, 0, sizeof(struct fsl_desc_sw)); - INIT_LIST_HEAD(&desc_sw->tx_list); - dma_async_tx_descriptor_init(&desc_sw->async_tx, - &fsl_chan->common); - desc_sw->async_tx.tx_submit = fsl_dma_tx_submit; - desc_sw->async_tx.phys = pdesc; + + desc = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &pdesc); + if (!desc) { + dev_dbg(chan->dev, "out of memory for link desc\n"); + return NULL; } - return desc_sw; + memset(desc, 0, sizeof(*desc)); + INIT_LIST_HEAD(&desc->tx_list); + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = fsl_dma_tx_submit; + desc->async_tx.phys = pdesc; + + return desc; } /** * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel. - * @fsl_chan : Freescale DMA channel + * @chan : Freescale DMA channel * * This function will create a dma pool for descriptor allocation. * * Return - The number of descriptors allocated. */ -static int fsl_dma_alloc_chan_resources(struct dma_chan *chan) +static int fsl_dma_alloc_chan_resources(struct dma_chan *dchan) { - struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); + struct fsldma_chan *chan = to_fsl_chan(dchan); /* Has this channel already been allocated? */ - if (fsl_chan->desc_pool) + if (chan->desc_pool) return 1; - /* We need the descriptor to be aligned to 32bytes + /* + * We need the descriptor to be aligned to 32bytes * for meeting FSL DMA specification requirement. */ - fsl_chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool", - fsl_chan->dev, sizeof(struct fsl_desc_sw), - 32, 0); - if (!fsl_chan->desc_pool) { - dev_err(fsl_chan->dev, "No memory for channel %d " - "descriptor dma pool.\n", fsl_chan->id); - return 0; + chan->desc_pool = dma_pool_create("fsl_dma_engine_desc_pool", + chan->dev, + sizeof(struct fsl_desc_sw), + __alignof__(struct fsl_desc_sw), 0); + if (!chan->desc_pool) { + dev_err(chan->dev, "unable to allocate channel %d " + "descriptor pool\n", chan->id); + return -ENOMEM; } + /* there is at least one descriptor free to be allocated */ return 1; } /** - * fsl_dma_free_chan_resources - Free all resources of the channel. - * @fsl_chan : Freescale DMA channel + * fsldma_free_desc_list - Free all descriptors in a queue + * @chan: Freescae DMA channel + * @list: the list to free + * + * LOCKING: must hold chan->desc_lock */ -static void fsl_dma_free_chan_resources(struct dma_chan *chan) +static void fsldma_free_desc_list(struct fsldma_chan *chan, + struct list_head *list) { - struct fsl_dma_chan *fsl_chan = to_fsl_chan(chan); struct fsl_desc_sw *desc, *_desc; - unsigned long flags; - dev_dbg(fsl_chan->dev, "Free all channel resources.\n"); - spin_lock_irqsave(&fsl_chan->desc_lock, flags); - list_for_each_entry_safe(desc, _desc, &fsl_chan->ld_queue, node) { -#ifdef FSL_DMA_LD_DEBUG - dev_dbg(fsl_chan->dev, - "LD %p will be released.\n", desc); -#endif + list_for_each_entry_safe(desc, _desc, list, node) { + list_del(&desc->node); + dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); + } +} + +static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan, + struct list_head *list) +{ + struct fsl_desc_sw *desc, *_desc; + + list_for_each_entry_safe_reverse(desc, _desc, list, node) { list_del(&desc->node); - /* free link descriptor */ - dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys); + dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); } - spin_unlock_irqrestore(&fsl_chan->desc_lock, flags); - dma_pool_destroy(fsl_chan->desc_pool); +} + +/** + * fsl_dma_free_chan_resources - Free all resources of the channel. + * @chan : Freescale DMA channel + */ +static void fsl_dma_free_chan_resources(struct dma_chan *dchan) +{ + struct fsldma_chan *chan = to_fsl_chan(dchan); + unsigned long flags; + + dev_dbg(chan->dev, "Free all channel resources.\n"); + spin_lock_irqsave(&chan->desc_lock, flags); + fsldma_free_desc_list(chan, &chan->ld_pending); + fsldma_free_desc_list(chan, &chan->ld_running); + spin_unlock_irqrestore(&chan->desc_lock, flags); - fsl_chan->desc_pool = NULL; + dma_pool_destroy(chan->desc_pool); + chan->desc_pool = NULL; } static struct dma_async_tx_descriptor * -fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags) +fsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags) { - struct fsl_dma_chan *fsl_chan; + struct fsldma_chan *chan; struct fsl_desc_sw *new; - if (!chan) + if (!dchan) return NULL; - fsl_chan = to_fsl_chan(chan); + chan = to_fsl_chan(dchan); - new = fsl_dma_alloc_descriptor(fsl_chan); + new = fsl_dma_alloc_descriptor(chan); if (!new) { - dev_err(fsl_chan->dev, "No free memory for link descriptor\n"); + dev_err(chan->dev, "No free memory for link descriptor\n"); return NULL; } @@ -473,51 +509,50 @@ fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags) list_add_tail(&new->node, &new->tx_list); /* Set End-of-link to the last link descriptor of new list*/ - set_ld_eol(fsl_chan, new); + set_ld_eol(chan, new); return &new->async_tx; } static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy( - struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, + struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src, size_t len, unsigned long flags) { - struct fsl_dma_chan *fsl_chan; + struct fsldma_chan *chan; struct fsl_desc_sw *first = NULL, *prev = NULL, *new; - struct list_head *list; size_t copy; - if (!chan) + if (!dchan) return NULL; if (!len) return NULL; - fsl_chan = to_fsl_chan(chan); + chan = to_fsl_chan(dchan); do { /* Allocate the link descriptor from DMA pool */ - new = fsl_dma_alloc_descriptor(fsl_chan); + new = fsl_dma_alloc_descriptor(chan); if (!new) { - dev_err(fsl_chan->dev, + dev_err(chan->dev, "No free memory for link descriptor\n"); goto fail; } #ifdef FSL_DMA_LD_DEBUG - dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new); + dev_dbg(chan->dev, "new link desc alloc %p\n", new); #endif copy = min(len, (size_t)FSL_DMA_BCR_MAX_CNT); - set_desc_cnt(fsl_chan, &new->hw, copy); - set_desc_src(fsl_chan, &new->hw, dma_src); - set_desc_dest(fsl_chan, &new->hw, dma_dest); + set_desc_cnt(chan, &new->hw, copy); + set_desc_src(chan, &new->hw, dma_src); + set_desc_dst(chan, &new->hw, dma_dst); if (!first) first = new; else - set_desc_next(fsl_chan, &prev->hw, new->async_tx.phys); + set_desc_next(chan, &prev->hw, new->async_tx.phys); new->async_tx.cookie = 0; async_tx_ack(&new->async_tx); @@ -525,7 +560,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy( prev = new; len -= copy; dma_src += copy; - dma_dest += copy; + dma_dst += copy; /* Insert the link descriptor to the LD ring */ list_add_tail(&new->node, &first->tx_list); @@ -535,7 +570,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy( new->async_tx.cookie = -EBUSY; /* Set End-of-link to the last link descriptor of new list*/ - set_ld_eol(fsl_chan, new); + set_ld_eol(chan, new); return &first->async_tx; @@ -543,12 +578,7 @@ fail: if (!first) return NULL; - list = &first->tx_list; - list_for_each_entry_safe_reverse(new, prev, list, node) { - list_del(&new->node); - dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys); - } - + fsldma_free_desc_list_reverse(chan, &first->tx_list); return NULL; } @@ -565,13 +595,12 @@ fail: * chan->private variable. */ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, + struct dma_chan *dchan, struct scatterlist *sgl, unsigned int sg_len, enum dma_data_direction direction, unsigned long flags) { - struct fsl_dma_chan *fsl_chan; + struct fsldma_chan *chan; struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL; struct fsl_dma_slave *slave; - struct list_head *tx_list; size_t copy; int i; @@ -581,14 +610,14 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( struct fsl_dma_hw_addr *hw; dma_addr_t dma_dst, dma_src; - if (!chan) + if (!dchan) return NULL; - if (!chan->private) + if (!dchan->private) return NULL; - fsl_chan = to_fsl_chan(chan); - slave = chan->private; + chan = to_fsl_chan(dchan); + slave = dchan->private; if (list_empty(&slave->addresses)) return NULL; @@ -637,14 +666,14 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( } /* Allocate the link descriptor from DMA pool */ - new = fsl_dma_alloc_descriptor(fsl_chan); + new = fsl_dma_alloc_descriptor(chan); if (!new) { - dev_err(fsl_chan->dev, "No free memory for " + dev_err(chan->dev, "No free memory for " "link descriptor\n"); goto fail; } #ifdef FSL_DMA_LD_DEBUG - dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new); + dev_dbg(chan->dev, "new link desc alloc %p\n", new); #endif /* @@ -671,9 +700,9 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( } /* Fill in the descriptor */ - set_desc_cnt(fsl_chan, &new->hw, copy); - set_desc_src(fsl_chan, &new->hw, dma_src); - set_desc_dest(fsl_chan, &new->hw, dma_dst); + set_desc_cnt(chan, &new->hw, copy); + set_desc_src(chan, &new->hw, dma_src); + set_desc_dst(chan, &new->hw, dma_dst); /* * If this is not the first descriptor, chain the @@ -682,7 +711,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg( if (!first) { first = new; } else { - set_desc_next(fsl_chan, &prev->hw, + set_desc_next(chan, &prev->hw, new->async_tx.phys); } @@ -708,23 +737,23 @@ finished: new->async_tx.cookie = -EBUSY; /* Set End-of-link to the last link descriptor of new list */ - set_ld_eol(fsl_chan, new); + set_ld_eol(chan, new); /* Enable extra controller features */ - if (fsl_chan->set_src_loop_size) - fsl_chan->set_src_loop_size(fsl_chan, slave->src_loop_size); + if (chan->set_src_loop_size) + chan->set_src_loop_size(chan, slave->src_loop_size); - if (fsl_chan->set_dest_loop_size) - fsl_chan->set_dest_loop_size(fsl_chan, slave->dst_loop_size); + if (chan->set_dst_loop_size) + chan->set_dst_loop_size(chan, slave->dst_loop_size); - if (fsl_chan->toggle_ext_start) - fsl_chan->toggle_ext_start(fsl_chan, slave->external_start); + if (chan->toggle_ext_start) + chan->toggle_ext_start(chan, slave->external_start); - if (fsl_chan->toggle_ext_pause) - fsl_chan->toggle_ext_pause(fsl_chan, slave->external_pause); + if (chan->toggle_ext_pause) + chan->toggle_ext_pause(chan, slave->external_pause); - if (fsl_chan->set_request_count) - fsl_chan->set_request_count(fsl_chan, slave->request_count); + if (chan->set_request_count) + chan->set_request_count(chan, slave->request_count); return &first->async_tx; @@ -741,215 +770,216 @@ fail: * * We're re-using variables for the loop, oh well */ - tx_list = &first->tx_list; - list_for_each_entry_safe_reverse(new, prev, tx_list, node) { - list_del_init(&new->node); - dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys); - } - + fsldma_free_desc_list_reverse(chan, &first->tx_list); return NULL; } -static void fsl_dma_device_terminate_all(struct dma_chan *chan) +s |