diff options
27 files changed, 1177 insertions, 492 deletions
diff --git a/Documentation/devicetree/bindings/dma/dma.txt b/Documentation/devicetree/bindings/dma/dma.txt index 8f504e6bae1..82104271e75 100644 --- a/Documentation/devicetree/bindings/dma/dma.txt +++ b/Documentation/devicetree/bindings/dma/dma.txt @@ -14,7 +14,7 @@ Required property: Optional properties: - dma-channels: Number of DMA channels supported by the controller. -- dma-requests: Number of DMA requests signals supported by the +- dma-requests: Number of DMA request signals supported by the controller. Example: @@ -44,7 +44,7 @@ Required property: #dma-cells property in the node referenced by phandle containing DMA controller specific information. This typically contains a DMA request line number or a - channel number, but can contain any data that is used + channel number, but can contain any data that is required for configuring a channel. - dma-names: Contains one identifier string for each DMA specifier in the dmas property. The specific strings that can be used diff --git a/Documentation/devicetree/bindings/dma/mmp-dma.txt b/Documentation/devicetree/bindings/dma/mmp-dma.txt index a4fa4efa1d8..7a802f64e5b 100644 --- a/Documentation/devicetree/bindings/dma/mmp-dma.txt +++ b/Documentation/devicetree/bindings/dma/mmp-dma.txt @@ -1,17 +1,20 @@ * MARVELL MMP DMA controller Marvell Peripheral DMA Controller -Used platfroms: pxa688, pxa910, pxa3xx, etc +Used platforms: pxa688, pxa910, pxa3xx, etc Required properties: - compatible: Should be "marvell,pdma-1.0" - reg: Should contain DMA registers location and length. - interrupts: Either contain all of the per-channel DMA interrupts or one irq for pdma device -- #dma-channels: Number of DMA channels supported by the controller. + +Optional properties: +- #dma-channels: Number of DMA channels supported by the controller (defaults + to 32 when not specified) "marvell,pdma-1.0" -Used platfroms: pxa25x, pxa27x, pxa3xx, pxa93x, pxa168, pxa910, pxa688. +Used platforms: pxa25x, pxa27x, pxa3xx, pxa93x, pxa168, pxa910, pxa688. Examples: @@ -45,7 +48,7 @@ pdma: dma-controller@d4000000 { Marvell Two Channel DMA Controller used specifically for audio -Used platfroms: pxa688, pxa910 +Used platforms: pxa688, pxa910 Required properties: - compatible: Should be "marvell,adma-1.0" or "marvell,pxa910-squ" diff --git a/arch/arm/common/edma.c b/arch/arm/common/edma.c index 41bca32409f..25fa735abc6 100644 --- a/arch/arm/common/edma.c +++ b/arch/arm/common/edma.c @@ -994,29 +994,23 @@ void edma_set_dest(unsigned slot, dma_addr_t dest_port, EXPORT_SYMBOL(edma_set_dest); /** - * edma_get_position - returns the current transfer points + * edma_get_position - returns the current transfer point * @slot: parameter RAM slot being examined - * @src: pointer to source port position - * @dst: pointer to destination port position + * @dst: true selects the dest position, false the source * - * Returns current source and destination addresses for a particular - * parameter RAM slot. Its channel should not be active when this is called. + * Returns the position of the current active slot */ -void edma_get_position(unsigned slot, dma_addr_t *src, dma_addr_t *dst) +dma_addr_t edma_get_position(unsigned slot, bool dst) { - struct edmacc_param temp; - unsigned ctlr; + u32 offs, ctlr = EDMA_CTLR(slot); - ctlr = EDMA_CTLR(slot); slot = EDMA_CHAN_SLOT(slot); - edma_read_slot(EDMA_CTLR_CHAN(ctlr, slot), &temp); - if (src != NULL) - *src = temp.src; - if (dst != NULL) - *dst = temp.dst; + offs = PARM_OFFSET(slot); + offs += dst ? PARM_DST : PARM_SRC; + + return edma_read(ctlr, offs); } -EXPORT_SYMBOL(edma_get_position); /** * edma_set_src_index - configure DMA source address indexing @@ -1574,6 +1568,7 @@ static struct edma_soc_info *edma_setup_info_from_dt(struct device *dev, return ERR_PTR(ret); dma_cap_set(DMA_SLAVE, edma_filter_info.dma_cap); + dma_cap_set(DMA_CYCLIC, edma_filter_info.dma_cap); of_dma_controller_register(dev->of_node, of_dma_simple_xlate, &edma_filter_info); @@ -1768,6 +1763,9 @@ static int edma_probe(struct platform_device *pdev) map_queue_tc(j, queue_tc_mapping[i][0], queue_tc_mapping[i][1]); + /* Save the number of TCs */ + edma_cc[j]->num_tc = i; + /* Event queue priority mapping */ for (i = 0; queue_priority_mapping[i][0] != -1; i++) assign_priority_to_queue(j, diff --git a/arch/powerpc/boot/dts/mpc8308_p1m.dts b/arch/powerpc/boot/dts/mpc8308_p1m.dts index 651e4f55acd..57f86cdf9f3 100644 --- a/arch/powerpc/boot/dts/mpc8308_p1m.dts +++ b/arch/powerpc/boot/dts/mpc8308_p1m.dts @@ -296,7 +296,7 @@ }; dma@2c000 { - compatible = "fsl,mpc8308-dma", "fsl,mpc5121-dma"; + compatible = "fsl,mpc8308-dma"; reg = <0x2c000 0x1800>; interrupts = <3 0x8 94 0x8>; diff --git a/arch/powerpc/boot/dts/mpc8308rdb.dts b/arch/powerpc/boot/dts/mpc8308rdb.dts index 9ce45f2efd3..d0211f0413c 100644 --- a/arch/powerpc/boot/dts/mpc8308rdb.dts +++ b/arch/powerpc/boot/dts/mpc8308rdb.dts @@ -265,7 +265,7 @@ }; dma@2c000 { - compatible = "fsl,mpc8308-dma", "fsl,mpc5121-dma"; + compatible = "fsl,mpc8308-dma"; reg = <0x2c000 0x1800>; interrupts = <3 0x8 94 0x8>; diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index b30b7ed89fb..1eca7b9760e 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -234,7 +234,7 @@ config PL330_DMA config PCH_DMA tristate "Intel EG20T PCH / LAPIS Semicon IOH(ML7213/ML7223/ML7831) DMA" - depends on PCI && X86 + depends on PCI && (X86_32 || COMPILE_TEST) select DMA_ENGINE help Enable support for Intel EG20T PCH DMA engine. @@ -269,7 +269,7 @@ config MXS_DMA select DMA_ENGINE help Support the MXS DMA engine. This engine including APBH-DMA - and APBX-DMA is integrated into Freescale i.MX23/28 chips. + and APBX-DMA is integrated into Freescale i.MX23/28/MX6Q/MX6DL chips. config EP93XX_DMA bool "Cirrus Logic EP93xx DMA support" diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index cfdbb92aae1..a27ded53ab4 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -1493,6 +1493,13 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) dw->regs = chip->regs; chip->dw = dw; + dw->clk = devm_clk_get(chip->dev, "hclk"); + if (IS_ERR(dw->clk)) + return PTR_ERR(dw->clk); + err = clk_prepare_enable(dw->clk); + if (err) + return err; + dw_params = dma_read_byaddr(chip->regs, DW_PARAMS); autocfg = dw_params >> DW_PARAMS_EN & 0x1; @@ -1500,15 +1507,19 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) if (!pdata && autocfg) { pdata = devm_kzalloc(chip->dev, sizeof(*pdata), GFP_KERNEL); - if (!pdata) - return -ENOMEM; + if (!pdata) { + err = -ENOMEM; + goto err_pdata; + } /* Fill platform data with the default values */ pdata->is_private = true; pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING; pdata->chan_priority = CHAN_PRIORITY_ASCENDING; - } else if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) - return -EINVAL; + } else if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) { + err = -EINVAL; + goto err_pdata; + } if (autocfg) nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 0x7) + 1; @@ -1517,13 +1528,10 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) dw->chan = devm_kcalloc(chip->dev, nr_channels, sizeof(*dw->chan), GFP_KERNEL); - if (!dw->chan) - return -ENOMEM; - - dw->clk = devm_clk_get(chip->dev, "hclk"); - if (IS_ERR(dw->clk)) - return PTR_ERR(dw->clk); - clk_prepare_enable(dw->clk); + if (!dw->chan) { + err = -ENOMEM; + goto err_pdata; + } /* Get hardware configuration parameters */ if (autocfg) { @@ -1548,21 +1556,22 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) /* Disable BLOCK interrupts as well */ channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask); - err = devm_request_irq(chip->dev, chip->irq, dw_dma_interrupt, - IRQF_SHARED, "dw_dmac", dw); - if (err) - return err; - /* Create a pool of consistent memory blocks for hardware descriptors */ dw->desc_pool = dmam_pool_create("dw_dmac_desc_pool", chip->dev, sizeof(struct dw_desc), 4, 0); if (!dw->desc_pool) { dev_err(chip->dev, "No memory for descriptors dma pool\n"); - return -ENOMEM; + err = -ENOMEM; + goto err_pdata; } tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw); + err = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, + "dw_dmac", dw); + if (err) + goto err_pdata; + INIT_LIST_HEAD(&dw->dma.channels); for (i = 0; i < nr_channels; i++) { struct dw_dma_chan *dwc = &dw->chan[i]; @@ -1650,12 +1659,20 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) dma_writel(dw, CFG, DW_CFG_DMA_EN); + err = dma_async_device_register(&dw->dma); + if (err) + goto err_dma_register; + dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n", nr_channels); - dma_async_device_register(&dw->dma); - return 0; + +err_dma_register: + free_irq(chip->irq, dw); +err_pdata: + clk_disable_unprepare(dw->clk); + return err; } EXPORT_SYMBOL_GPL(dw_dma_probe); @@ -1667,6 +1684,7 @@ int dw_dma_remove(struct dw_dma_chip *chip) dw_dma_off(dw); dma_async_device_unregister(&dw->dma); + free_irq(chip->irq, dw); tasklet_kill(&dw->tasklet); list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels, @@ -1675,6 +1693,8 @@ int dw_dma_remove(struct dw_dma_chip *chip) channel_clear_bit(dw, CH_EN, dwc->mask); } + clk_disable_unprepare(dw->clk); + return 0; } EXPORT_SYMBOL_GPL(dw_dma_remove); diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c index fec59f1a77b..39e30c3c7a9 100644 --- a/drivers/dma/dw/pci.c +++ b/drivers/dma/dw/pci.c @@ -93,19 +93,13 @@ static int dw_pci_resume_early(struct device *dev) return dw_dma_resume(chip); }; -#else /* !CONFIG_PM_SLEEP */ - -#define dw_pci_suspend_late NULL -#define dw_pci_resume_early NULL - -#endif /* !CONFIG_PM_SLEEP */ +#endif /* CONFIG_PM_SLEEP */ static const struct dev_pm_ops dw_pci_dev_pm_ops = { - .suspend_late = dw_pci_suspend_late, - .resume_early = dw_pci_resume_early, + SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_pci_suspend_late, dw_pci_resume_early) }; -static DEFINE_PCI_DEVICE_TABLE(dw_pci_id_table) = { +static const struct pci_device_id dw_pci_id_table[] = { /* Medfield */ { PCI_VDEVICE(INTEL, 0x0827), (kernel_ulong_t)&dw_pci_pdata }, { PCI_VDEVICE(INTEL, 0x0830), (kernel_ulong_t)&dw_pci_pdata }, diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c index 453822cc4f9..c5b339af6be 100644 --- a/drivers/dma/dw/platform.c +++ b/drivers/dma/dw/platform.c @@ -256,7 +256,7 @@ MODULE_DEVICE_TABLE(acpi, dw_dma_acpi_id_table); #ifdef CONFIG_PM_SLEEP -static int dw_suspend_noirq(struct device *dev) +static int dw_suspend_late(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct dw_dma_chip *chip = platform_get_drvdata(pdev); @@ -264,7 +264,7 @@ static int dw_suspend_noirq(struct device *dev) return dw_dma_suspend(chip); } -static int dw_resume_noirq(struct device *dev) +static int dw_resume_early(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct dw_dma_chip *chip = platform_get_drvdata(pdev); @@ -272,20 +272,10 @@ static int dw_resume_noirq(struct device *dev) return dw_dma_resume(chip); } -#else /* !CONFIG_PM_SLEEP */ - -#define dw_suspend_noirq NULL -#define dw_resume_noirq NULL - -#endif /* !CONFIG_PM_SLEEP */ +#endif /* CONFIG_PM_SLEEP */ static const struct dev_pm_ops dw_dev_pm_ops = { - .suspend_noirq = dw_suspend_noirq, - .resume_noirq = dw_resume_noirq, - .freeze_noirq = dw_suspend_noirq, - .thaw_noirq = dw_resume_noirq, - .restore_noirq = dw_resume_noirq, - .poweroff_noirq = dw_suspend_noirq, + SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_suspend_late, dw_resume_early) }; static struct platform_driver dw_driver = { diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index 926360c2db6..d08c4dedef3 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -57,14 +57,48 @@ #define EDMA_MAX_SLOTS MAX_NR_SG #define EDMA_DESCRIPTORS 16 +struct edma_pset { + u32 len; + dma_addr_t addr; + struct edmacc_param param; +}; + struct edma_desc { struct virt_dma_desc vdesc; struct list_head node; + enum dma_transfer_direction direction; int cyclic; int absync; int pset_nr; + struct edma_chan *echan; int processed; - struct edmacc_param pset[0]; + + /* + * The following 4 elements are used for residue accounting. + * + * - processed_stat: the number of SG elements we have traversed + * so far to cover accounting. This is updated directly to processed + * during edma_callback and is always <= processed, because processed + * refers to the number of pending transfer (programmed to EDMA + * controller), where as processed_stat tracks number of transfers + * accounted for so far. + * + * - residue: The amount of bytes we have left to transfer for this desc + * + * - residue_stat: The residue in bytes of data we have covered + * so far for accounting. This is updated directly to residue + * during callbacks to keep it current. + * + * - sg_len: Tracks the length of the current intermediate transfer, + * this is required to update the residue during intermediate transfer + * completion callback. + */ + int processed_stat; + u32 sg_len; + u32 residue; + u32 residue_stat; + + struct edma_pset pset[0]; }; struct edma_cc; @@ -136,12 +170,14 @@ static void edma_execute(struct edma_chan *echan) /* Find out how many left */ left = edesc->pset_nr - edesc->processed; nslots = min(MAX_NR_SG, left); + edesc->sg_len = 0; /* Write descriptor PaRAM set(s) */ for (i = 0; i < nslots; i++) { j = i + edesc->processed; - edma_write_slot(echan->slot[i], &edesc->pset[j]); - dev_dbg(echan->vchan.chan.device->dev, + edma_write_slot(echan->slot[i], &edesc->pset[j].param); + edesc->sg_len += edesc->pset[j].len; + dev_vdbg(echan->vchan.chan.device->dev, "\n pset[%d]:\n" " chnum\t%d\n" " slot\t%d\n" @@ -154,14 +190,14 @@ static void edma_execute(struct edma_chan *echan) " cidx\t%08x\n" " lkrld\t%08x\n", j, echan->ch_num, echan->slot[i], - edesc->pset[j].opt, - edesc->pset[j].src, - edesc->pset[j].dst, - edesc->pset[j].a_b_cnt, - edesc->pset[j].ccnt, - edesc->pset[j].src_dst_bidx, - edesc->pset[j].src_dst_cidx, - edesc->pset[j].link_bcntrld); + edesc->pset[j].param.opt, + edesc->pset[j].param.src, + edesc->pset[j].param.dst, + edesc->pset[j].param.a_b_cnt, + edesc->pset[j].param.ccnt, + edesc->pset[j].param.src_dst_bidx, + edesc->pset[j].param.src_dst_cidx, + edesc->pset[j].param.link_bcntrld); /* Link to the previous slot if not the last set */ if (i != (nslots - 1)) edma_link(echan->slot[i], echan->slot[i+1]); @@ -183,7 +219,8 @@ static void edma_execute(struct edma_chan *echan) } if (edesc->processed <= MAX_NR_SG) { - dev_dbg(dev, "first transfer starting %d\n", echan->ch_num); + dev_dbg(dev, "first transfer starting on channel %d\n", + echan->ch_num); edma_start(echan->ch_num); } else { dev_dbg(dev, "chan: %d: completed %d elements, resuming\n", @@ -197,7 +234,7 @@ static void edma_execute(struct edma_chan *echan) * MAX_NR_SG */ if (echan->missed) { - dev_dbg(dev, "missed event in execute detected\n"); + dev_dbg(dev, "missed event on channel %d\n", echan->ch_num); edma_clean_channel(echan->ch_num); edma_stop(echan->ch_num); edma_start(echan->ch_num); @@ -242,6 +279,26 @@ static int edma_slave_config(struct edma_chan *echan, return 0; } +static int edma_dma_pause(struct edma_chan *echan) +{ + /* Pause/Resume only allowed with cyclic mode */ + if (!echan->edesc->cyclic) + return -EINVAL; + + edma_pause(echan->ch_num); + return 0; +} + +static int edma_dma_resume(struct edma_chan *echan) +{ + /* Pause/Resume only allowed with cyclic mode */ + if (!echan->edesc->cyclic) + return -EINVAL; + + edma_resume(echan->ch_num); + return 0; +} + static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg) { @@ -257,6 +314,14 @@ static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, config = (struct dma_slave_config *)arg; ret = edma_slave_config(echan, config); break; + case DMA_PAUSE: + ret = edma_dma_pause(echan); + break; + + case DMA_RESUME: + ret = edma_dma_resume(echan); + break; + default: ret = -ENOSYS; } @@ -275,18 +340,23 @@ static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, * @dma_length: Total length of the DMA transfer * @direction: Direction of the transfer */ -static int edma_config_pset(struct dma_chan *chan, struct edmacc_param *pset, +static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset, dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst, enum dma_slave_buswidth dev_width, unsigned int dma_length, enum dma_transfer_direction direction) { struct edma_chan *echan = to_edma_chan(chan); struct device *dev = chan->device->dev; + struct edmacc_param *param = &epset->param; int acnt, bcnt, ccnt, cidx; int src_bidx, dst_bidx, src_cidx, dst_cidx; int absync; acnt = dev_width; + + /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */ + if (!burst) + burst = 1; /* * If the maxburst is equal to the fifo width, use * A-synced transfers. This allows for large contiguous @@ -337,41 +407,50 @@ static int edma_config_pset(struct dma_chan *chan, struct edmacc_param *pset, cidx = acnt * bcnt; } + epset->len = dma_length; + if (direction == DMA_MEM_TO_DEV) { src_bidx = acnt; src_cidx = cidx; dst_bidx = 0; dst_cidx = 0; + epset->addr = src_addr; } else if (direction == DMA_DEV_TO_MEM) { src_bidx = 0; src_cidx = 0; dst_bidx = acnt; dst_cidx = cidx; + epset->addr = dst_addr; + } else if (direction == DMA_MEM_TO_MEM) { + src_bidx = acnt; + src_cidx = cidx; + dst_bidx = acnt; + dst_cidx = cidx; } else { dev_err(dev, "%s: direction not implemented yet\n", __func__); return -EINVAL; } - pset->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num)); + param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num)); /* Configure A or AB synchronized transfers */ if (absync) - pset->opt |= SYNCDIM; + param->opt |= SYNCDIM; - pset->src = src_addr; - pset->dst = dst_addr; + param->src = src_addr; + param->dst = dst_addr; - pset->src_dst_bidx = (dst_bidx << 16) | src_bidx; - pset->src_dst_cidx = (dst_cidx << 16) | src_cidx; + param->src_dst_bidx = (dst_bidx << 16) | src_bidx; + param->src_dst_cidx = (dst_cidx << 16) | src_cidx; - pset->a_b_cnt = bcnt << 16 | acnt; - pset->ccnt = ccnt; + param->a_b_cnt = bcnt << 16 | acnt; + param->ccnt = ccnt; /* * Only time when (bcntrld) auto reload is required is for * A-sync case, and in this case, a requirement of reload value * of SZ_64K-1 only is assured. 'link' is initially set to NULL * and then later will be populated by edma_execute. */ - pset->link_bcntrld = 0xffffffff; + param->link_bcntrld = 0xffffffff; return absync; } @@ -401,23 +480,26 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( dev_width = echan->cfg.dst_addr_width; burst = echan->cfg.dst_maxburst; } else { - dev_err(dev, "%s: bad direction?\n", __func__); + dev_err(dev, "%s: bad direction: %d\n", __func__, direction); return NULL; } if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) { - dev_err(dev, "Undefined slave buswidth\n"); + dev_err(dev, "%s: Undefined slave buswidth\n", __func__); return NULL; } edesc = kzalloc(sizeof(*edesc) + sg_len * sizeof(edesc->pset[0]), GFP_ATOMIC); if (!edesc) { - dev_dbg(dev, "Failed to allocate a descriptor\n"); + dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__); return NULL; } edesc->pset_nr = sg_len; + edesc->residue = 0; + edesc->direction = direction; + edesc->echan = echan; /* Allocate a PaRAM slot, if needed */ nslots = min_t(unsigned, MAX_NR_SG, sg_len); @@ -429,7 +511,8 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( EDMA_SLOT_ANY); if (echan->slot[i] < 0) { kfree(edesc); - dev_err(dev, "Failed to allocate slot\n"); + dev_err(dev, "%s: Failed to allocate slot\n", + __func__); return NULL; } } @@ -452,16 +535,56 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( } edesc->absync = ret; + edesc->residue += sg_dma_len(sg); /* If this is the last in a current SG set of transactions, enable interrupts so that next set is processed */ if (!((i+1) % MAX_NR_SG)) - edesc->pset[i].opt |= TCINTEN; + edesc->pset[i].param.opt |= TCINTEN; /* If this is the last set, enable completion interrupt flag */ if (i == sg_len - 1) - edesc->pset[i].opt |= TCINTEN; + edesc->pset[i].param.opt |= TCINTEN; } + edesc->residue_stat = edesc->residue; + + return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); +} + +struct dma_async_tx_descriptor *edma_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long tx_flags) +{ + int ret; + struct edma_desc *edesc; + struct device *dev = chan->device->dev; + struct edma_chan *echan = to_edma_chan(chan); + + if (unlikely(!echan || !len)) + return NULL; + + edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC); + if (!edesc) { + dev_dbg(dev, "Failed to allocate a descriptor\n"); + return NULL; + } + + edesc->pset_nr = 1; + + ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1, + DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM); + if (ret < 0) + return NULL; + + edesc->absync = ret; + + /* + * Enable intermediate transfer chaining to re-trigger channel + * on completion of every TR, and enable transfer-completion + * interrupt on completion of the whole transfer. + */ + edesc->pset[0].param.opt |= ITCCHEN; + edesc->pset[0].param.opt |= TCINTEN; return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); } @@ -493,12 +616,12 @@ static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( dev_width = echan->cfg.dst_addr_width; burst = echan->cfg.dst_maxburst; } else { - dev_err(dev, "%s: bad direction?\n", __func__); + dev_err(dev, "%s: bad direction: %d\n", __func__, direction); return NULL; } if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) { - dev_err(dev, "Undefined slave buswidth\n"); + dev_err(dev, "%s: Undefined slave buswidth\n", __func__); return NULL; } @@ -523,16 +646,18 @@ static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( edesc = kzalloc(sizeof(*edesc) + nslots * sizeof(edesc->pset[0]), GFP_ATOMIC); if (!edesc) { - dev_dbg(dev, "Failed to allocate a descriptor\n"); + dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__); return NULL; } edesc->cyclic = 1; edesc->pset_nr = nslots; + edesc->residue = edesc->residue_stat = buf_len; + edesc->direction = direction; + edesc->echan = echan; - dev_dbg(dev, "%s: nslots=%d\n", __func__, nslots); - dev_dbg(dev, "%s: period_len=%d\n", __func__, period_len); - dev_dbg(dev, "%s: buf_len=%d\n", __func__, buf_len); + dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n", + __func__, echan->ch_num, nslots, period_len, buf_len); for (i = 0; i < nslots; i++) { /* Allocate a PaRAM slot, if needed */ @@ -542,7 +667,8 @@ static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( EDMA_SLOT_ANY); if (echan->slot[i] < 0) { kfree(edesc); - dev_err(dev, "Failed to allocate slot\n"); + dev_err(dev, "%s: Failed to allocate slot\n", + __func__); return NULL; } } @@ -566,8 +692,8 @@ static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( else src_addr += period_len; - dev_dbg(dev, "%s: Configure period %d of buf:\n", __func__, i); - dev_dbg(dev, + dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i); + dev_vdbg(dev, "\n pset[%d]:\n" " chnum\t%d\n" " slot\t%d\n" @@ -580,14 +706,14 @@ static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( " cidx\t%08x\n" " lkrld\t%08x\n", i, echan->ch_num, echan->slot[i], - edesc->pset[i].opt, - edesc->pset[i].src, - edesc->pset[i].dst, - edesc->pset[i].a_b_cnt, - edesc->pset[i].ccnt, - edesc->pset[i].src_dst_bidx, - edesc->pset[i].src_dst_cidx, - edesc->pset[i].link_bcntrld); + edesc->pset[i].param.opt, + edesc->pset[i].param.src, + edesc->pset[i].param.dst, + edesc->pset[i].param.a_b_cnt, + edesc->pset[i].param.ccnt, + edesc->pset[i].param.src_dst_bidx, + edesc->pset[i].param.src_dst_cidx, + edesc->pset[i].param.link_bcntrld); edesc->absync = ret; @@ -595,7 +721,7 @@ static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( * Enable interrupts for every period because callback * has to be called for every period. */ - edesc->pset[i].opt |= TCINTEN; + edesc->pset[i].param.opt |= TCINTEN; } return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); @@ -606,7 +732,6 @@ static void edma_callback(unsigned ch_num, u16 ch_status, void *data) struct edma_chan *echan = data; struct device *dev = echan->vchan.chan.device->dev; struct edma_desc *edesc; - unsigned long flags; struct edmacc_param p; edesc = echan->edesc; @@ -617,27 +742,34 @@ static void edma_callback(unsigned ch_num, u16 ch_status, void *data) switch (ch_status) { case EDMA_DMA_COMPLETE: - spin_lock_irqsave(&echan->vchan.lock, flags); + spin_lock(&echan->vchan.lock); if (edesc) { if (edesc->cyclic) { vchan_cyclic_callback(&edesc->vdesc); } else if (edesc->processed == edesc->pset_nr) { dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num); + edesc->residue = 0; edma_stop(echan->ch_num); vchan_cookie_complete(&edesc->vdesc); edma_execute(echan); } else { dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num); + + /* Update statistics for tx_status */ + edesc->residue -= edesc->sg_len; + edesc->residue_stat = edesc->residue; + edesc->processed_stat = edesc->processed; + edma_execute(echan); } } - spin_unlock_irqrestore(&echan->vchan.lock, flags); + spin_unlock(&echan->vchan.lock); break; case EDMA_DMA_CC_ERROR: - spin_lock_irqsave(&echan->vchan.lock, flags); + spin_lock(&echan->vchan.lock); edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p); @@ -668,7 +800,7 @@ static void edma_callback(unsigned ch_num, u16 ch_status, void *data) edma_trigger_channel(echan->ch_num); } - spin_unlock_irqrestore(&echan->vchan.lock, flags); + spin_unlock(&echan->vchan.lock); break; default: @@ -704,7 +836,7 @@ static int edma_alloc_chan_resources(struct dma_chan *chan) echan->alloced = true; echan->slot[0] = echan->ch_num; - dev_dbg(dev, "allocated channel for %u:%u\n", + dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num, EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num)); return 0; @@ -756,23 +888,52 @@ static void edma_issue_pending(struct d |