diff options
Diffstat (limited to 'drivers/dma')
69 files changed, 14309 insertions, 2807 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index 526ec77c7ba..d761ad3ba09 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -62,6 +62,7 @@ config INTEL_IOATDMA tristate "Intel I/OAT DMA support" depends on PCI && X86 select DMA_ENGINE + select DMA_ENGINE_RAID select DCA help Enable support for the Intel(R) I/OAT DMA engine present @@ -89,14 +90,15 @@ config AT_HDMAC Support the Atmel AHB DMA controller. config FSL_DMA - tristate "Freescale Elo and Elo Plus DMA support" + tristate "Freescale Elo series DMA support" depends on FSL_SOC select DMA_ENGINE select ASYNC_TX_ENABLE_CHANNEL_SWITCH ---help--- - Enable support for the Freescale Elo and Elo Plus DMA controllers. - The Elo is the DMA controller on some 82xx and 83xx parts, and the - Elo Plus is the DMA controller on 85xx and 86xx parts. + Enable support for the Freescale Elo series DMA controllers. + The Elo is the DMA controller on some mpc82xx and mpc83xx parts, the + EloPlus is on mpc85xx and mpc86xx and Pxxx parts, and the Elo3 is on + some Txxx and Bxxx parts. config MPC512X_DMA tristate "Freescale MPC512x built-in DMA engine support" @@ -111,6 +113,7 @@ config MV_XOR bool "Marvell XOR engine support" depends on PLAT_ORION select DMA_ENGINE + select DMA_ENGINE_RAID select ASYNC_TX_ENABLE_CHANNEL_SWITCH ---help--- Enable support for the Marvell XOR engine. @@ -154,6 +157,18 @@ config TEGRA20_APB_DMA This DMA controller transfers data from memory to peripheral fifo or vice versa. It does not support memory to memory data transfer. +config S3C24XX_DMAC + tristate "Samsung S3C24XX DMA support" + depends on ARCH_S3C24XX && !S3C24XX_DMA + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support for the Samsung S3C24XX DMA controller driver. The + DMA controller is having multiple DMA channels which can be + configured for different peripherals like audio, UART, SPI. + The DMA controller can transfer data from memory to peripheral, + periphal to memory, periphal to periphal and memory to memory. + source "drivers/dma/sh/Kconfig" config COH901318 @@ -174,14 +189,32 @@ config AMCC_PPC440SPE_ADMA tristate "AMCC PPC440SPe ADMA support" depends on 440SPe || 440SP select DMA_ENGINE + select DMA_ENGINE_RAID select ARCH_HAS_ASYNC_TX_FIND_CHANNEL select ASYNC_TX_ENABLE_CHANNEL_SWITCH help Enable support for the AMCC PPC440SPe RAID engines. +config AMCC_PPC460EX_460GT_4CHAN_DMA + tristate "AMCC PPC460EX PPC460GT PLB DMA support" + depends on 460EX || 460GT || APM821xx + select DMA_ENGINE + default y + +config APM82181_ADMA + tristate "APM82181 Asynchonous DMA support" + depends on APM821xx + select ASYNC_CORE + select ASYNC_TX_DMA + select DMA_ENGINE + select ARCH_HAS_ASYNC_TX_FIND_CHANNEL + default y + ---help--- + Enable support for the APM82181 Asynchonous DMA engines. + config TIMB_DMA tristate "Timberdale FPGA DMA support" - depends on MFD_TIMBERDALE || HAS_IOMEM + depends on MFD_TIMBERDALE select DMA_ENGINE help Enable support for the Timberdale FPGA DMA engine. @@ -195,9 +228,10 @@ config SIRF_DMA config TI_EDMA bool "TI EDMA support" - depends on ARCH_DAVINCI || ARCH_OMAP + depends on ARCH_DAVINCI || ARCH_OMAP || ARCH_KEYSTONE select DMA_ENGINE select DMA_VIRTUAL_CHANNELS + select TI_PRIV_EDMA default n help Enable support for the TI EDMA controller. This DMA @@ -217,7 +251,7 @@ config PL330_DMA config PCH_DMA tristate "Intel EG20T PCH / LAPIS Semicon IOH(ML7213/ML7223/ML7831) DMA" - depends on PCI && X86 + depends on PCI && (X86_32 || COMPILE_TEST) select DMA_ENGINE help Enable support for Intel EG20T PCH DMA engine. @@ -252,7 +286,7 @@ config MXS_DMA select DMA_ENGINE help Support the MXS DMA engine. This engine including APBH-DMA - and APBX-DMA is integrated into Freescale i.MX23/28 chips. + and APBX-DMA is integrated into Freescale i.MX23/28/MX6Q/MX6DL chips. config EP93XX_DMA bool "Cirrus Logic EP93xx DMA support" @@ -275,9 +309,11 @@ config MMP_TDMA bool "MMP Two-Channel DMA support" depends on ARCH_MMP select DMA_ENGINE + select MMP_SRAM help Support the MMP Two-Channel DMA engine. This engine used for MMP Audio DMA and pxa910 SQU. + It needs sram driver under mach-mmp. Say Y here if you enabled MMP ADMA, otherwise say N. @@ -287,6 +323,12 @@ config DMA_OMAP select DMA_ENGINE select DMA_VIRTUAL_CHANNELS +config DMA_BCM2835 + tristate "BCM2835 DMA engine support" + depends on ARCH_BCM2835 + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + config TI_CPPI41 tristate "AM33xx CPPI41 DMA support" depends on ARCH_OMAP @@ -300,7 +342,7 @@ config MMP_PDMA depends on (ARCH_MMP || ARCH_PXA) select DMA_ENGINE help - Support the MMP PDMA engine for PXA and MMP platfrom. + Support the MMP PDMA engine for PXA and MMP platform. config DMA_JZ4740 tristate "JZ4740 DMA support" @@ -317,6 +359,39 @@ config K3_DMA Support the DMA engine for Hisilicon K3 platform devices. +config MOXART_DMA + tristate "MOXART DMA support" + depends on ARCH_MOXART + select DMA_ENGINE + select DMA_OF + select DMA_VIRTUAL_CHANNELS + help + Enable support for the MOXA ART SoC DMA controller. + +config FSL_EDMA + tristate "Freescale eDMA engine support" + depends on OF + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + help + Support the Freescale eDMA engine with programmable channel + multiplexing capability for DMA request sources(slot). + This module can be found on Freescale Vybrid and LS-1 SoCs. + +config XILINX_VDMA + tristate "Xilinx AXI VDMA Engine" + depends on (ARCH_ZYNQ || MICROBLAZE) + select DMA_ENGINE + help + Enable support for Xilinx AXI VDMA Soft IP. + + This engine provides high-bandwidth direct memory access + between memory and AXI4-Stream video type target + peripherals including peripherals which support AXI4- + Stream Video Protocol. It has two stream interfaces/ + channels, Memory Mapped to Stream (MM2S) and Stream to + Memory Mapped (S2MM) for the data transfers. + config DMA_ENGINE bool @@ -338,6 +413,7 @@ config NET_DMA bool "Network: TCP receive copy offload" depends on DMA_ENGINE && NET default (INTEL_IOATDMA || FSL_DMA) + depends on BROKEN help This enables the use of DMA engines in the network stack to offload receive copy-to-user operations, freeing CPU cycles. @@ -363,4 +439,16 @@ config DMATEST Simple DMA test client. Say N unless you're debugging a DMA Device driver. +config DMA_ENGINE_RAID + bool + +config QCOM_BAM_DMA + tristate "QCOM BAM DMA support" + depends on ARCH_QCOM || (COMPILE_TEST && OF && ARM) + select DMA_ENGINE + select DMA_VIRTUAL_CHANNELS + ---help--- + Enable support for the QCOM BAM DMA controller. This controller + provides DMA capabilities for a variety of on-chip devices. + endif diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index db89035b362..7acb4c437fa 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -21,7 +21,7 @@ obj-$(CONFIG_MX3_IPU) += ipu/ obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_SH_DMAE_BASE) += sh/ obj-$(CONFIG_COH901318) += coh901318.o coh901318_lli.o -obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += ppc4xx/ +obj-y += ppc4xx/ obj-$(CONFIG_IMX_SDMA) += imx-sdma.o obj-$(CONFIG_IMX_DMA) += imx-dma.o obj-$(CONFIG_MXS_DMA) += mxs-dma.o @@ -30,6 +30,7 @@ obj-$(CONFIG_SIRF_DMA) += sirf-dma.o obj-$(CONFIG_TI_EDMA) += edma.o obj-$(CONFIG_STE_DMA40) += ste_dma40.o ste_dma40_ll.o obj-$(CONFIG_TEGRA20_APB_DMA) += tegra20-apb-dma.o +obj-$(CONFIG_S3C24XX_DMAC) += s3c24xx-dma.o obj-$(CONFIG_PL330_DMA) += pl330.o obj-$(CONFIG_PCH_DMA) += pch_dma.o obj-$(CONFIG_AMBA_PL08X) += amba-pl08x.o @@ -37,7 +38,12 @@ obj-$(CONFIG_EP93XX_DMA) += ep93xx_dma.o obj-$(CONFIG_DMA_SA11X0) += sa11x0-dma.o obj-$(CONFIG_MMP_TDMA) += mmp_tdma.o obj-$(CONFIG_DMA_OMAP) += omap-dma.o +obj-$(CONFIG_DMA_BCM2835) += bcm2835-dma.o obj-$(CONFIG_MMP_PDMA) += mmp_pdma.o obj-$(CONFIG_DMA_JZ4740) += dma-jz4740.o obj-$(CONFIG_TI_CPPI41) += cppi41.o obj-$(CONFIG_K3_DMA) += k3dma.o +obj-$(CONFIG_MOXART_DMA) += moxart-dma.o +obj-$(CONFIG_FSL_EDMA) += fsl-edma.o +obj-$(CONFIG_QCOM_BAM_DMA) += qcom_bam_dma.o +obj-y += xilinx/ diff --git a/drivers/dma/acpi-dma.c b/drivers/dma/acpi-dma.c index e69b03c0fa5..de361a156b3 100644 --- a/drivers/dma/acpi-dma.c +++ b/drivers/dma/acpi-dma.c @@ -13,6 +13,7 @@ */ #include <linux/device.h> +#include <linux/err.h> #include <linux/module.h> #include <linux/list.h> #include <linux/mutex.h> @@ -30,11 +31,12 @@ static DEFINE_MUTEX(acpi_dma_lock); * @adev: ACPI device to match with * @adma: struct acpi_dma of the given DMA controller * - * Returns 1 on success, 0 when no information is available, or appropriate - * errno value on error. - * * In order to match a device from DSDT table to the corresponding CSRT device * we use MMIO address and IRQ. + * + * Return: + * 1 on success, 0 when no information is available, or appropriate errno value + * on error. */ static int acpi_dma_parse_resource_group(const struct acpi_csrt_group *grp, struct acpi_device *adev, struct acpi_dma *adma) @@ -101,7 +103,6 @@ static int acpi_dma_parse_resource_group(const struct acpi_csrt_group *grp, * * We are using this table to get the request line range of the specific DMA * controller to be used later. - * */ static void acpi_dma_parse_csrt(struct acpi_device *adev, struct acpi_dma *adma) { @@ -141,10 +142,11 @@ static void acpi_dma_parse_csrt(struct acpi_device *adev, struct acpi_dma *adma) * @data pointer to controller specific data to be used by * translation function * - * Returns 0 on success or appropriate errno value on error. - * * Allocated memory should be freed with appropriate acpi_dma_controller_free() * call. + * + * Return: + * 0 on success or appropriate errno value on error. */ int acpi_dma_controller_register(struct device *dev, struct dma_chan *(*acpi_dma_xlate) @@ -188,6 +190,9 @@ EXPORT_SYMBOL_GPL(acpi_dma_controller_register); * @dev: struct device of DMA controller * * Memory allocated by acpi_dma_controller_register() is freed here. + * + * Return: + * 0 on success or appropriate errno value on error. */ int acpi_dma_controller_free(struct device *dev) { @@ -225,6 +230,9 @@ static void devm_acpi_dma_release(struct device *dev, void *res) * Managed acpi_dma_controller_register(). DMA controller registered by this * function are automatically freed on driver detach. See * acpi_dma_controller_register() for more information. + * + * Return: + * 0 on success or appropriate errno value on error. */ int devm_acpi_dma_controller_register(struct device *dev, struct dma_chan *(*acpi_dma_xlate) @@ -258,7 +266,7 @@ EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_register); */ void devm_acpi_dma_controller_free(struct device *dev) { - WARN_ON(devres_destroy(dev, devm_acpi_dma_release, NULL, NULL)); + WARN_ON(devres_release(dev, devm_acpi_dma_release, NULL, NULL)); } EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_free); @@ -267,8 +275,6 @@ EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_free); * @adma: struct acpi_dma of DMA controller * @dma_spec: dma specifier to update * - * Returns 0, if no information is avaiable, -1 on mismatch, and 1 otherwise. - * * Accordingly to ACPI 5.0 Specification Table 6-170 "Fixed DMA Resource * Descriptor": * DMA Request Line bits is a platform-relative number uniquely @@ -276,6 +282,9 @@ EXPORT_SYMBOL_GPL(devm_acpi_dma_controller_free); * mapping is done in a controller-specific OS driver. * That's why we can safely adjust slave_id when the appropriate controller is * found. + * + * Return: + * 0, if no information is avaiable, -1 on mismatch, and 1 otherwise. */ static int acpi_dma_update_dma_spec(struct acpi_dma *adma, struct acpi_dma_spec *dma_spec) @@ -334,7 +343,8 @@ static int acpi_dma_parse_fixed_dma(struct acpi_resource *res, void *data) * @dev: struct device to get DMA request from * @index: index of FixedDMA descriptor for @dev * - * Returns pointer to appropriate dma channel on success or NULL on error. + * Return: + * Pointer to appropriate dma channel on success or an error pointer. */ struct dma_chan *acpi_dma_request_slave_chan_by_index(struct device *dev, size_t index) @@ -349,10 +359,10 @@ struct dma_chan *acpi_dma_request_slave_chan_by_index(struct device *dev, /* Check if the device was enumerated by ACPI */ if (!dev || !ACPI_HANDLE(dev)) - return NULL; + return ERR_PTR(-ENODEV); if (acpi_bus_get_device(ACPI_HANDLE(dev), &adev)) - return NULL; + return ERR_PTR(-ENODEV); memset(&pdata, 0, sizeof(pdata)); pdata.index = index; @@ -367,7 +377,7 @@ struct dma_chan *acpi_dma_request_slave_chan_by_index(struct device *dev, acpi_dev_free_resource_list(&resource_list); if (dma_spec->slave_id < 0 || dma_spec->chan_id < 0) - return NULL; + return ERR_PTR(-ENODEV); mutex_lock(&acpi_dma_lock); @@ -390,7 +400,7 @@ struct dma_chan *acpi_dma_request_slave_chan_by_index(struct device *dev, } mutex_unlock(&acpi_dma_lock); - return chan; + return chan ? chan : ERR_PTR(-EPROBE_DEFER); } EXPORT_SYMBOL_GPL(acpi_dma_request_slave_chan_by_index); @@ -403,7 +413,8 @@ EXPORT_SYMBOL_GPL(acpi_dma_request_slave_chan_by_index); * translate the names "tx" and "rx" here based on the most common case where * the first FixedDMA descriptor is TX and second is RX. * - * Returns pointer to appropriate dma channel on success or NULL on error. + * Return: + * Pointer to appropriate dma channel on success or an error pointer. */ struct dma_chan *acpi_dma_request_slave_chan_by_name(struct device *dev, const char *name) @@ -415,7 +426,7 @@ struct dma_chan *acpi_dma_request_slave_chan_by_name(struct device *dev, else if (!strcmp(name, "rx")) index = 1; else - return NULL; + return ERR_PTR(-ENODEV); return acpi_dma_request_slave_chan_by_index(dev, index); } @@ -427,8 +438,10 @@ EXPORT_SYMBOL_GPL(acpi_dma_request_slave_chan_by_name); * @adma: pointer to ACPI DMA controller data * * A simple translation function for ACPI based devices. Passes &struct - * dma_spec to the DMA controller driver provided filter function. Returns - * pointer to the channel if found or %NULL otherwise. + * dma_spec to the DMA controller driver provided filter function. + * + * Return: + * Pointer to the channel if found or %NULL otherwise. */ struct dma_chan *acpi_dma_simple_xlate(struct acpi_dma_spec *dma_spec, struct acpi_dma *adma) diff --git a/drivers/dma/amba-pl08x.c b/drivers/dma/amba-pl08x.c index fce46c5bf1c..8114731a1c6 100644 --- a/drivers/dma/amba-pl08x.c +++ b/drivers/dma/amba-pl08x.c @@ -83,6 +83,7 @@ #include <linux/dmaengine.h> #include <linux/dmapool.h> #include <linux/dma-mapping.h> +#include <linux/export.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/module.h> @@ -1164,42 +1165,12 @@ static void pl08x_free_txd(struct pl08x_driver_data *pl08x, kfree(txd); } -static void pl08x_unmap_buffers(struct pl08x_txd *txd) -{ - struct device *dev = txd->vd.tx.chan->device->dev; - struct pl08x_sg *dsg; - - if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - if (txd->vd.tx.flags & DMA_COMPL_SRC_UNMAP_SINGLE) - list_for_each_entry(dsg, &txd->dsg_list, node) - dma_unmap_single(dev, dsg->src_addr, dsg->len, - DMA_TO_DEVICE); - else { - list_for_each_entry(dsg, &txd->dsg_list, node) - dma_unmap_page(dev, dsg->src_addr, dsg->len, - DMA_TO_DEVICE); - } - } - if (!(txd->vd.tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (txd->vd.tx.flags & DMA_COMPL_DEST_UNMAP_SINGLE) - list_for_each_entry(dsg, &txd->dsg_list, node) - dma_unmap_single(dev, dsg->dst_addr, dsg->len, - DMA_FROM_DEVICE); - else - list_for_each_entry(dsg, &txd->dsg_list, node) - dma_unmap_page(dev, dsg->dst_addr, dsg->len, - DMA_FROM_DEVICE); - } -} - static void pl08x_desc_free(struct virt_dma_desc *vd) { struct pl08x_txd *txd = to_pl08x_txd(&vd->tx); struct pl08x_dma_chan *plchan = to_pl08x_chan(vd->tx.chan); - if (!plchan->slave) - pl08x_unmap_buffers(txd); - + dma_descriptor_unmap(&vd->tx); if (!txd->done) pl08x_release_mux(plchan); @@ -1252,7 +1223,7 @@ static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, size_t bytes = 0; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; /* @@ -1267,7 +1238,7 @@ static enum dma_status pl08x_dma_tx_status(struct dma_chan *chan, spin_lock_irqsave(&plchan->vc.lock, flags); ret = dma_cookie_status(chan, cookie, txstate); - if (ret != DMA_SUCCESS) { + if (ret != DMA_COMPLETE) { vd = vchan_find_desc(&plchan->vc, cookie); if (vd) { /* On the issued list, so hasn't been processed yet */ @@ -1801,6 +1772,7 @@ bool pl08x_filter_id(struct dma_chan *chan, void *chan_id) return false; } +EXPORT_SYMBOL_GPL(pl08x_filter_id); /* * Just check that the device is there and active @@ -2055,6 +2027,11 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) if (ret) return ret; + /* Ensure that we can do DMA */ + ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); + if (ret) + goto out_no_pl08x; + /* Create the driver state holder */ pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL); if (!pl08x) { @@ -2133,8 +2110,7 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR); writel(0x000000FF, pl08x->base + PL080_TC_CLEAR); - ret = request_irq(adev->irq[0], pl08x_irq, IRQF_DISABLED, - DRIVER_NAME, pl08x); + ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x); if (ret) { dev_err(&adev->dev, "%s failed to request interrupt %d\n", __func__, adev->irq[0]); @@ -2193,7 +2169,7 @@ static int pl08x_probe(struct amba_device *adev, const struct amba_id *id) /* Register slave channels */ ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave, pl08x->pd->num_slave_channels, true); - if (ret <= 0) { + if (ret < 0) { dev_warn(&pl08x->adev->dev, "%s failed to enumerate slave channels - %d\n", __func__, ret); diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index c787f38a186..c13a3bb0f59 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -344,31 +344,7 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) /* move myself to free_list */ list_move(&desc->desc_node, &atchan->free_list); - /* unmap dma addresses (not on slave channels) */ - if (!atchan->chan_common.private) { - struct device *parent = chan2parent(&atchan->chan_common); - if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) - dma_unmap_single(parent, - desc->lli.daddr, - desc->len, DMA_FROM_DEVICE); - else - dma_unmap_page(parent, - desc->lli.daddr, - desc->len, DMA_FROM_DEVICE); - } - if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) - dma_unmap_single(parent, - desc->lli.saddr, - desc->len, DMA_TO_DEVICE); - else - dma_unmap_page(parent, - desc->lli.saddr, - desc->len, DMA_TO_DEVICE); - } - } - + dma_descriptor_unmap(txd); /* for cyclic transfers, * no need to replay callback function while stopping */ if (!atc_chan_is_cyclic(atchan)) { @@ -1102,7 +1078,7 @@ atc_tx_status(struct dma_chan *chan, int bytes = 0; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; /* * There's no point calculating the residue if there's @@ -1593,7 +1569,6 @@ static int at_dma_remove(struct platform_device *pdev) /* Disable interrupts */ atc_disable_chan_irq(atdma, chan->chan_id); - tasklet_disable(&atchan->tasklet); tasklet_kill(&atchan->tasklet); list_del(&chan->device_node); diff --git a/drivers/dma/at_hdmac_regs.h b/drivers/dma/at_hdmac_regs.h index f31d647acdf..2787aba60c6 100644 --- a/drivers/dma/at_hdmac_regs.h +++ b/drivers/dma/at_hdmac_regs.h @@ -347,10 +347,6 @@ static struct device *chan2dev(struct dma_chan *chan) { return &chan->dev->device; } -static struct device *chan2parent(struct dma_chan *chan) -{ - return chan->dev->device.parent; -} #if defined(VERBOSE_DEBUG) static void vdbg_dump_regs(struct at_dma_chan *atchan) diff --git a/drivers/dma/bcm2835-dma.c b/drivers/dma/bcm2835-dma.c new file mode 100644 index 00000000000..a03602164e3 --- /dev/null +++ b/drivers/dma/bcm2835-dma.c @@ -0,0 +1,707 @@ +/* + * BCM2835 DMA engine support + * + * This driver only supports cyclic DMA transfers + * as needed for the I2S module. + * + * Author: Florian Meier <florian.meier@koalo.de> + * Copyright 2013 + * + * Based on + * OMAP DMAengine support by Russell King + * + * BCM2708 DMA Driver + * Copyright (C) 2010 Broadcom + * + * Raspberry Pi PCM I2S ALSA Driver + * Copyright (c) by Phil Poole 2013 + * + * MARVELL MMP Peripheral DMA Driver + * Copyright 2012 Marvell International Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/io.h> +#include <linux/spinlock.h> +#include <linux/of.h> +#include <linux/of_dma.h> + +#include "virt-dma.h" + +struct bcm2835_dmadev { + struct dma_device ddev; + spinlock_t lock; + void __iomem *base; + struct device_dma_parameters dma_parms; +}; + +struct bcm2835_dma_cb { + uint32_t info; + uint32_t src; + uint32_t dst; + uint32_t length; + uint32_t stride; + uint32_t next; + uint32_t pad[2]; +}; + +struct bcm2835_chan { + struct virt_dma_chan vc; + struct list_head node; + + struct dma_slave_config cfg; + bool cyclic; + unsigned int dreq; + + int ch; + struct bcm2835_desc *desc; + + void __iomem *chan_base; + int irq_number; +}; + +struct bcm2835_desc { + struct virt_dma_desc vd; + enum dma_transfer_direction dir; + + unsigned int control_block_size; + struct bcm2835_dma_cb *control_block_base; + dma_addr_t control_block_base_phys; + + unsigned int frames; + size_t size; +}; + +#define BCM2835_DMA_CS 0x00 +#define BCM2835_DMA_ADDR 0x04 +#define BCM2835_DMA_SOURCE_AD 0x0c +#define BCM2835_DMA_DEST_AD 0x10 +#define BCM2835_DMA_NEXTCB 0x1C + +/* DMA CS Control and Status bits */ +#define BCM2835_DMA_ACTIVE BIT(0) +#define BCM2835_DMA_INT BIT(2) +#define BCM2835_DMA_ISPAUSED BIT(4) /* Pause requested or not active */ +#define BCM2835_DMA_ISHELD BIT(5) /* Is held by DREQ flow control */ +#define BCM2835_DMA_ERR BIT(8) +#define BCM2835_DMA_ABORT BIT(30) /* Stop current CB, go to next, WO */ +#define BCM2835_DMA_RESET BIT(31) /* WO, self clearing */ + +#define BCM2835_DMA_INT_EN BIT(0) +#define BCM2835_DMA_D_INC BIT(4) +#define BCM2835_DMA_D_DREQ BIT(6) +#define BCM2835_DMA_S_INC BIT(8) +#define BCM2835_DMA_S_DREQ BIT(10) + +#define BCM2835_DMA_PER_MAP(x) ((x) << 16) + +#define BCM2835_DMA_DATA_TYPE_S8 1 +#define BCM2835_DMA_DATA_TYPE_S16 2 +#define BCM2835_DMA_DATA_TYPE_S32 4 +#define BCM2835_DMA_DATA_TYPE_S128 16 + +#define BCM2835_DMA_BULK_MASK BIT(0) +#define BCM2835_DMA_FIQ_MASK (BIT(2) | BIT(3)) + +/* Valid only for channels 0 - 14, 15 has its own base address */ +#define BCM2835_DMA_CHAN(n) ((n) << 8) /* Base address */ +#define BCM2835_DMA_CHANIO(base, n) ((base) + BCM2835_DMA_CHAN(n)) + +static inline struct bcm2835_dmadev *to_bcm2835_dma_dev(struct dma_device *d) +{ + return container_of(d, struct bcm2835_dmadev, ddev); +} + +static inline struct bcm2835_chan *to_bcm2835_dma_chan(struct dma_chan *c) +{ + return container_of(c, struct bcm2835_chan, vc.chan); +} + +static inline struct bcm2835_desc *to_bcm2835_dma_desc( + struct dma_async_tx_descriptor *t) +{ + return container_of(t, struct bcm2835_desc, vd.tx); +} + +static void bcm2835_dma_desc_free(struct virt_dma_desc *vd) +{ + struct bcm2835_desc *desc = container_of(vd, struct bcm2835_desc, vd); + dma_free_coherent(desc->vd.tx.chan->device->dev, + desc->control_block_size, + desc->control_block_base, + desc->control_block_base_phys); + kfree(desc); +} + +static int bcm2835_dma_abort(void __iomem *chan_base) +{ + unsigned long cs; + long int timeout = 10000; + + cs = readl(chan_base + BCM2835_DMA_CS); + if (!(cs & BCM2835_DMA_ACTIVE)) + return 0; + + /* Write 0 to the active bit - Pause the DMA */ + writel(0, chan_base + BCM2835_DMA_CS); + + /* Wait for any current AXI transfer to complete */ + while ((cs & BCM2835_DMA_ISPAUSED) && --timeout) { + cpu_relax(); + cs = readl(chan_base + BCM2835_DMA_CS); + } + + /* We'll un-pause when we set of our next DMA */ + if (!timeout) + return -ETIMEDOUT; + + if (!(cs & BCM2835_DMA_ACTIVE)) + return 0; + + /* Terminate the control block chain */ + writel(0, chan_base + BCM2835_DMA_NEXTCB); + + /* Abort the whole DMA */ + writel(BCM2835_DMA_ABORT | BCM2835_DMA_ACTIVE, + chan_base + BCM2835_DMA_CS); + + return 0; +} + +static void bcm2835_dma_start_desc(struct bcm2835_chan *c) +{ + struct virt_dma_desc *vd = vchan_next_desc(&c->vc); + struct bcm2835_desc *d; + + if (!vd) { + c->desc = NULL; + return; + } + + list_del(&vd->node); + + c->desc = d = to_bcm2835_dma_desc(&vd->tx); + + writel(d->control_block_base_phys, c->chan_base + BCM2835_DMA_ADDR); + writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS); +} + +static irqreturn_t bcm2835_dma_callback(int irq, void *data) +{ + struct bcm2835_chan *c = data; + struct bcm2835_desc *d; + unsigned long flags; + + spin_lock_irqsave(&c->vc.lock, flags); + + /* Acknowledge interrupt */ + writel(BCM2835_DMA_INT, c->chan_base + BCM2835_DMA_CS); + + d = c->desc; + + if (d) { + /* TODO Only works for cyclic DMA */ + vchan_cyclic_callback(&d->vd); + } + + /* Keep the DMA engine running */ + writel(BCM2835_DMA_ACTIVE, c->chan_base + BCM2835_DMA_CS); + + spin_unlock_irqrestore(&c->vc.lock, flags); + + return IRQ_HANDLED; +} + +static int bcm2835_dma_alloc_chan_resources(struct dma_chan *chan) +{ + struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); + + dev_dbg(c->vc.chan.device->dev, + "Allocating DMA channel %d\n", c->ch); + + return request_irq(c->irq_number, + bcm2835_dma_callback, 0, "DMA IRQ", c); +} + +static void bcm2835_dma_free_chan_resources(struct dma_chan *chan) +{ + struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); + + vchan_free_chan_resources(&c->vc); + free_irq(c->irq_number, c); + + dev_dbg(c->vc.chan.device->dev, "Freeing DMA channel %u\n", c->ch); +} + +static size_t bcm2835_dma_desc_size(struct bcm2835_desc *d) +{ + return d->size; +} + +static size_t bcm2835_dma_desc_size_pos(struct bcm2835_desc *d, dma_addr_t addr) +{ + unsigned int i; + size_t size; + + for (size = i = 0; i < d->frames; i++) { + struct bcm2835_dma_cb *control_block = + &d->control_block_base[i]; + size_t this_size = control_block->length; + dma_addr_t dma; + + if (d->dir == DMA_DEV_TO_MEM) + dma = control_block->dst; + else + dma = control_block->src; + + if (size) + size += this_size; + else if (addr >= dma && addr < dma + this_size) + size += dma + this_size - addr; + } + + return size; +} + +static enum dma_status bcm2835_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); + struct virt_dma_desc *vd; + enum dma_status ret; + unsigned long flags; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE || !txstate) + return ret; + + spin_lock_irqsave(&c->vc.lock, flags); + vd = vchan_find_desc(&c->vc, cookie); + if (vd) { + txstate->residue = + bcm2835_dma_desc_size(to_bcm2835_dma_desc(&vd->tx)); + } else if (c->desc && c->desc->vd.tx.cookie == cookie) { + struct bcm2835_desc *d = c->desc; + dma_addr_t pos; + + if (d->dir == DMA_MEM_TO_DEV) + pos = readl(c->chan_base + BCM2835_DMA_SOURCE_AD); + else if (d->dir == DMA_DEV_TO_MEM) + pos = readl(c->chan_base + BCM2835_DMA_DEST_AD); + else + pos = 0; + + txstate->residue = bcm2835_dma_desc_size_pos(d, pos); + } else { + txstate->residue = 0; + } + + spin_unlock_irqrestore(&c->vc.lock, flags); + + return ret; +} + +static void bcm2835_dma_issue_pending(struct dma_chan *chan) +{ + struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); + unsigned long flags; + + c->cyclic = true; /* Nothing else is implemented */ + + spin_lock_irqsave(&c->vc.lock, flags); + if (vchan_issue_pending(&c->vc) && !c->desc) + bcm2835_dma_start_desc(c); + + spin_unlock_irqrestore(&c->vc.lock, flags); +} + +static struct dma_async_tx_descriptor *bcm2835_dma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); + enum dma_slave_buswidth dev_width; + struct bcm2835_desc *d; + dma_addr_t dev_addr; + unsigned int es, sync_type; + unsigned int frame; + + /* Grab configuration */ + if (!is_slave_direction(direction)) { + dev_err(chan->device->dev, "%s: bad direction?\n", __func__); + return NULL; + } + + if (direction == DMA_DEV_TO_MEM) { + dev_addr = c->cfg.src_addr; + dev_width = c->cfg.src_addr_width; + sync_type = BCM2835_DMA_S_DREQ; + } else { + dev_addr = c->cfg.dst_addr; + dev_width = c->cfg.dst_addr_width; + sync_type = BCM2835_DMA_D_DREQ; + } + + /* Bus width translates to the element size (ES) */ + switch (dev_width) { + case DMA_SLAVE_BUSWIDTH_4_BYTES: + es = BCM2835_DMA_DATA_TYPE_S32; + break; + default: + return NULL; + } + + /* Now allocate and setup the descriptor. */ + d = kzalloc(sizeof(*d), GFP_NOWAIT); + if (!d) + return NULL; + + d->dir = direction; + d->frames = buf_len / period_len; + + /* Allocate memory for control blocks */ + d->control_block_size = d->frames * sizeof(struct bcm2835_dma_cb); + d->control_block_base = dma_zalloc_coherent(chan->device->dev, + d->control_block_size, &d->control_block_base_phys, + GFP_NOWAIT); + + if (!d->control_block_base) { + kfree(d); + return NULL; + } + + /* + * Iterate over all frames, create a control block + * for each frame and link them together. + */ + for (frame = 0; frame < d->frames; frame++) { + struct bcm2835_dma_cb *control_block = + &d->control_block_base[frame]; + + /* Setup adresses */ + if (d->dir == DMA_DEV_TO_MEM) { + control_block->info = BCM2835_DMA_D_INC; + control_block->src = dev_addr; + control_block->dst = buf_addr + frame * period_len; + } else { + control_block->info = BCM2835_DMA_S_INC; + control_block->src = buf_addr + frame * period_len; + control_block->dst = dev_addr; + } + + /* Enable interrupt */ + control_block->info |= BCM2835_DMA_INT_EN; + + /* Setup synchronization */ + if (sync_type != 0) + control_block->info |= sync_type; + + /* Setup DREQ channel */ + if (c->dreq != 0) + control_block->info |= + BCM2835_DMA_PER_MAP(c->dreq); + + /* Length of a frame */ + control_block->length = period_len; + d->size += control_block->length; + + /* + * Next block is the next frame. + * This DMA engine driver currently only supports cyclic DMA. + * Therefore, wrap around at number of frames. + */ + control_block->next = d->control_block_base_phys + + sizeof(struct bcm2835_dma_cb) + * ((frame + 1) % d->frames); + } + + return vchan_tx_prep(&c->vc, &d->vd, flags); +} + +static int bcm2835_dma_slave_config(struct bcm2835_chan *c, + struct dma_slave_config *cfg) +{ + if ((cfg->direction == DMA_DEV_TO_MEM && + cfg->src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) || + (cfg->direction == DMA_MEM_TO_DEV && + cfg->dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES) || + !is_slave_direction(cfg->direction)) { + return -EINVAL; + } + + c->cfg = *cfg; + + return 0; +} + +static int bcm2835_dma_terminate_all(struct bcm2835_chan *c) +{ + struct bcm2835_dmadev *d = to_bcm2835_dma_dev(c->vc.chan.device); + unsigned long flags; + int timeout = 10000; + LIST_HEAD(head); + + spin_lock_irqsave(&c->vc.lock, flags); + + /* Prevent this channel being scheduled */ + spin_lock(&d->lock); + list_del_init(&c->node); + spin_unlock(&d->lock); + + /* + * Stop DMA activity: we assume the callback will not be called + * after bcm_dma_abort() returns (even if it does, it will see + * c->desc is NULL and exit.) + */ + if (c->desc) { + c->desc = NULL; + bcm2835_dma_abort(c->chan_base); + + /* Wait for stopping */ + while (--timeout) { + if (!(readl(c->chan_base + BCM2835_DMA_CS) & + BCM2835_DMA_ACTIVE)) + break; + + cpu_relax(); + } + + if (!timeout) + dev_err(d->ddev.dev, "DMA transfer could not be terminated\n"); + } + + vchan_get_all_descriptors(&c->vc, &head); + spin_unlock_irqrestore(&c->vc.lock, flags); + vchan_dma_desc_free_list(&c->vc, &head); + + return 0; +} + +static int bcm2835_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct bcm2835_chan *c = to_bcm2835_dma_chan(chan); + + switch (cmd) { + case DMA_SLAVE_CONFIG: + return bcm2835_dma_slave_config(c, + (struct dma_slave_config *)arg); + + case DMA_TERMINATE_ALL: + return bcm2835_dma_terminate_all(c); + + default: + return -ENXIO; + } +} + +static int bcm2835_dma_chan_init(struct bcm2835_dmadev *d, int chan_id, int irq) +{ + struct bcm2835_chan *c; + + c = devm_kzalloc(d->ddev.dev, sizeof(*c), GFP_KERNEL); + if (!c) + return -ENOMEM; + + c->vc.desc_free = bcm2835_dma_desc_free; + vchan_init(&c->vc, &d->ddev); + INIT_LIST_HEAD(&c->node); + + d->ddev.chancnt++; + + c->chan_base = BCM2835_DMA_CHANIO(d->base, chan_id); + c->ch = chan_id; + c->irq_number = irq; + + return 0; +} + +static void bcm2835_dma_free(struct bcm2835_dmadev *od) +{ + struct bcm2835_chan *c, *next; + + list_for_each_entry_safe(c, next, &od->ddev.channels, + vc.chan.device_node) { + list_del(&c->vc.chan.device_node); + tasklet_kill(&c->vc.task); + } +} + +static const struct of_device_id bcm2835_dma_of_match[] = { + { .compatible = "brcm,bcm2835-dma", }, + {}, +}; +MODULE_DEVICE_TABLE(of, bcm2835_dma_of_match); + +static struct dma_chan *bcm2835_dma_xlate(struct of_phandle_args *spec, + struct of_dma *ofdma) +{ + struct bcm2835_dmadev *d = ofdma->of_dma_data; + struct dma_chan *chan; + + chan = dma_get_any_slave_channel(&d->ddev); + if (!chan) + return NULL; + + /* Set DREQ from param */ + to_bcm2835_dma_chan(chan)->dreq = spec->args[0]; + + return chan; +} + +static int bcm2835_dma_device_slave_caps(struct dma_chan *dchan, + struct dma_slave_caps *caps) +{ + caps->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + caps->dstn_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES); + caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + caps->cmd_pause = false; + caps->cmd_terminate = true; + + return 0; +} + +static int bcm2835_dma_probe(struct platform_device *pdev) +{ + struct bcm2835_dmadev *od; + struct resource *res; + void __iomem *base; + int rc; + int i; + int irq; + uint32_t chans_available; + + if (!pdev->dev.dma_mask) + pdev->dev.dma_mask = &pdev->dev.coherent_dma_mask; + + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (rc) + return rc; + + od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); + if (!od) + return -ENOMEM; + + pdev->dev.dma_parms = &od->dma_parms; + dma_set_max_seg_size(&pdev->dev, 0x3FFFFFFF); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(base)) + return PTR_ERR(base); + + od->base = base; + + dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); + dma_cap_set(DMA_PRIVATE, od->ddev.cap_mask); + dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); + od->ddev.device_alloc_chan_resources = bcm2835_dma_alloc_chan_resources; + od->ddev.device_free_chan_resources = bcm2835_dma_free_chan_resources; + od->ddev.device_tx_status = bcm2835_dma_tx_status; + od->ddev.device_issue_pending = bcm2835_dma_issue_pending; + od->ddev.device_slave_caps = bcm2835_dma_device_slave_caps; + od->ddev.device_prep_dma_cyclic = bcm2835_dma_prep_dma_cyclic; + od->ddev.device_control = bcm2835_dma_control; + od->ddev.dev = &pdev->dev; + INIT_LIST_HEAD(&od->ddev.channels); + spin_lock_init(&od->lock); + + platform_set_drvdata(pdev, od); + + /* Request DMA channel mask from device tree */ + if (of_property_read_u32(pdev->dev.of_node, + "brcm,dma-channel-mask", + &chans_available)) { + dev_err(&pdev->dev, "Failed to get channel mask\n"); + rc = -EINVAL; + goto err_no_dma; + } + + /* + * Do not use the FIQ and BULK channels, + * because they are used by the GPU. + */ + chans_available &= ~(BCM2835_DMA_FIQ_MASK | BCM2835_DMA_BULK_MASK); + + for (i = 0; i < pdev->num_resources; i++) { + irq = platform_get_irq(pdev, i); + if (irq < 0) + break; + + if (chans_available & (1 << i)) { + rc = bcm2835_dma_chan_init(od, i, irq); + if (rc) + goto err_no_dma; + } + } + + dev_dbg(&pdev->dev, "Initialized %i DMA channels\n", i); + + /* Device-tree DMA controller registration */ + rc = of_dma_controller_register(pdev->dev.of_node, + bcm2835_dma_xlate, od); + if (rc) { + dev_err(&pdev->dev, "Failed to register DMA controller\n"); + goto err_no_dma; + } + + rc = dma_async_device_register(&od->ddev); + if (rc) { + dev_err(&pdev->dev, + "Failed to register slave DMA engine device: %d\n", rc); + goto err_no_dma; + } + + dev_dbg(&pdev->dev, "Load BCM2835 DMA engine driver\n"); + + return 0; + +err_no_dma: + bcm2835_dma_free(od); + return rc; +} + +static int bcm2835_dma_remove(struct platform_device *pdev) +{ + struct bcm2835_dmadev *od = platform_get_drvdata(pdev); + + dma_async_device_unregister(&od->ddev); + bcm2835_dma_free(od); + + return 0; +} + +static struct platform_driver bcm2835_dma_driver = { + .probe = bcm2835_dma_probe, + .remove = bcm2835_dma_remove, + .driver = { + .name = "bcm2835-dma", + .owner = THIS_MODULE, + .of_match_table = of_match_ptr(bcm2835_dma_of_match), + }, +}; + +module_platform_driver(bcm2835_dma_driver); + +MODULE_ALIAS("platform:bcm2835-dma"); +MODULE_DESCRIPTION("BCM2835 DMA engine driver"); +MODULE_AUTHOR("Florian Meier <florian.meier@koalo.de>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/bestcomm/sram.c b/drivers/dma/bestcomm/sram.c index 5e2ed30ba2c..2074e0e3fa2 100644 --- a/drivers/dma/bestcomm/sram.c +++ b/drivers/dma/bestcomm/sram.c @@ -19,6 +19,7 @@ #include <linux/string.h> #include <linux/ioport.h> #include <linux/of.h> +#include <linux/of_address.h> #include <asm/io.h> #include <asm/mmu.h> diff --git a/drivers/dma/coh901318.c b/drivers/dma/coh901318.c index 31011d2a26f..3c6716e0b78 100644 --- a/drivers/dma/coh901318.c +++ b/drivers/dma/coh901318.c @@ -2369,7 +2369,7 @@ coh901318_tx_status(struct dma_chan *chan, dma_cookie_t cookie, enum dma_status ret; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; dma_set_residue(txstate, coh901318_get_bytes_left(chan)); @@ -2694,7 +2694,7 @@ static int __init coh901318_probe(struct platform_device *pdev) if (irq < 0) return irq; - err = devm_request_irq(&pdev->dev, irq, dma_irq_handler, IRQF_DISABLED, + err = devm_request_irq(&pdev->dev, irq, dma_irq_handler, 0, "coh901318", base); if (err) return err; diff --git a/drivers/dma/cppi41.c b/drivers/dma/cppi41.c index 7c82b92f9b1..8f8b0b60887 100644 --- a/drivers/dma/cppi41.c +++ b/drivers/dma/cppi41.c @@ -86,6 +86,9 @@ #define USBSS_IRQ_PD_COMP (1 << 2) +/* Packet Descriptor */ +#define PD2_ZERO_LENGTH (1 << 19) + struct cppi41_channel { struct dma_chan chan; struct dma_async_tx_descriptor txd; @@ -141,6 +144,9 @@ struct cppi41_dd { const struct chan_queues *queues_rx; const struct chan_queues *queues_tx; struct chan_queues td_queue; + + /* context for suspend/resume */ + unsigned int dma_tdfdq; }; #define FIST_COMPLETION_QUEUE 93 @@ -263,6 +269,15 @@ static u32 pd_trans_len(u32 val) return val & ((1 << (DESC_LENGTH_BITS_NUM + 1)) - 1); } +static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num) +{ + u32 desc; + + desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num)); + desc &= ~0x1f; + return desc; +} + static irqreturn_t cppi41_irq(int irq, void *data) { struct cppi41_dd *cdd = data; @@ -295,22 +310,25 @@ static irqreturn_t cppi41_irq(int irq, void *data) __iormb(); while (val) { - u32 desc; + u32 desc, len; q_num = __fls(val); val &= ~(1 << q_num); q_num += 32 * i; - desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(q_num)); - desc &= ~0x1f; + desc = cppi41_pop_desc(cdd, q_num); c = desc_to_chan(cdd, desc); if (WARN_ON(!c)) { pr_err("%s() q %d desc %08x\n", __func__, q_num, desc); continue; } - c->residue = pd_trans_len(c->desc->pd6) - - pd_trans_len(c->desc->pd0); + if (c->desc->pd2 & PD2_ZERO_LENGTH) + len = 0; + else + len = pd_trans_len(c->desc->pd0); + + c->residue = pd_trans_len(c->desc->pd6) - len; dma_cookie_complete(&c->txd); c->txd.callback(c->txd.callback_param); } @@ -353,7 +371,7 @@ static enum dma_status cppi41_dma_tx_status(struct dma_chan *chan, /* lock */ ret = dma_cookie_status(chan, cookie, txstate); - if (txstate && ret == DMA_SUCCESS) + if (txstate && ret == DMA_COMPLETE) txstate->residue = c->residue; /* unlock */ @@ -517,15 +535,6 @@ static void cppi41_compute_td_desc(struct cppi41_desc *d) d->pd0 = DESC_TYPE_TEARD << DESC_TYPE; } -static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num) -{ - u32 desc; - - desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num)); - desc &= ~0x1f; - return desc; -} - static int cppi41_tear_down_chan(struct cppi41_channel *c) { struct cppi41_dd *cdd = c->cdd; @@ -561,36 +570,26 @@ static int cppi41_tear_down_chan(struct cppi41_channel *c) c->td_retry = 100; } - if (!c->td_seen) { - unsigned td_comp_queue; + if (!c->td_seen || !c->td_desc_seen) { - if (c->is_tx) - td_comp_queue = cdd->td_queue.complete; - else - td_comp_queue = c->q_comp_num; + desc_phys = cppi41_pop_desc(cdd, cdd->td_queue.complete); + if (!desc_phys) + desc_phys = cppi41_pop_desc(cdd, c->q_comp_num); - desc_phys = cppi41_pop_desc(cdd, td_comp_queue); - if (desc_phys) { - __iormb(); + if (desc_phys == c->desc_phys) { + c->td_desc_seen = 1; + + } else if (desc_phys == td_desc_phys) { + u32 pd0; - if (desc_phys == td_desc_phys) { - u32 pd0; - pd0 = td->pd0; - WARN_ON((pd0 >> DESC_TYPE) != DESC_TYPE_TEARD); - WARN_ON(!c->is_tx && !(pd0 & TD_DESC_IS_RX)); - WARN_ON((pd0 & 0x1f) != c->port_num); - } else { - WARN_ON_ONCE(1); - } - c->td_seen = 1; - } - } - if (!c->td_desc_seen) { - desc_phys = cppi41_pop_desc(cdd, c->q_comp_num); - if (desc_phys) { __iormb(); - WARN_ON(c->desc_phys != desc_phys); - c->td_desc_seen = 1; + pd0 = td->pd0; + WARN_ON((pd0 >> DESC_TYPE) != DESC_TYPE_TEARD); + WARN_ON(!c->is_tx && !(pd0 & TD_DESC_IS_RX)); + WARN_ON((pd0 & 0x1f) != c->port_num); + c->td_seen = 1; + } else if (desc_phys) { + WARN_ON_ONCE(1); } } c->td_retry--; @@ -609,7 +608,7 @@ static int cppi41_tear_down_chan(struct cppi41_channel *c) WARN_ON(!c->td_retry); if (!c->td_desc_seen) { - desc_phys = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num)); + desc_phys = cppi41_pop_desc(cdd, c->q_num); WARN_ON(!desc_phys); } @@ -628,12 +627,15 @@ static int cppi41_stop_chan(struct dma_chan *chan) u32 desc_phys; int ret; + desc_phys = lower_32_bits(c->desc_phys); + desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc); + if (!cdd->chan_busy[desc_num]) + return 0; + ret = cppi41_tear_down_chan(c); if (ret) return ret; - desc_phys = lower_32_bits(c->desc_phys); - desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc); WARN_ON(!cdd->chan_busy[desc_num]); cdd->chan_busy[desc_num] = NULL; @@ -674,14 +676,14 @@ static void cleanup_chans(struct cppi41_dd *cdd) } } -static int cppi41_add_chans(struct platform_device *pdev, struct cppi41_dd *cdd) +static int cppi41_add_chans(struct device *dev, struct cppi41_dd *cdd) { struct cppi41_channel *cchan; int i; int ret; u32 n_chans; - ret = of_property_read_u32(pdev->dev.of_node, "#dma-channels", + ret = of_property_read_u32(dev->of_node, "#dma-channels", &n_chans); if (ret) return ret; @@ -719,7 +721,7 @@ err: return -ENOMEM; } -static void purge_descs(struct platform_device *pdev, struct cppi41_dd *cdd) +static void purge_descs(struct device *dev, struct cppi41_dd *cdd) { unsigned int mem_decs; int i; @@ -731,7 +733,7 @@ static void purge_descs(struct platform_device *pdev, struct cppi41_dd *cdd) cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i)); cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i)); - dma_free_coherent(&pdev->dev, mem_decs, cdd->cd, + dma_free_coherent(dev, mem_decs, cdd->cd, cdd->descs_phys); } } @@ -741,19 +743,19 @@ static void disable_sched(struct cppi41_dd *cdd) cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL); } -static void deinit_cpii41(struct platform_device *pdev, struct cppi41_dd *cdd) +static void deinit_cppi41(struct device *dev, struct cppi41_dd *cdd) { disable_sched(cdd); - purge_descs(pdev, cdd); + purge_descs(dev, cdd); cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE); cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE); - dma_free_coherent(&pdev->dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch, + dma_free_coherent(dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch, cdd->scratch_phys); } -static int init_descs(struct platform_device *pdev, struct cppi41_dd *cdd) +static int init_descs(struct device *dev, struct cppi41_dd *cdd) { unsigned int desc_size; unsigned int mem_decs; @@ -777,7 +779,7 @@ static int init_descs(struct platform_device *pdev, struct cppi41_dd *cdd) reg |= ilog2(ALLOC_DECS_NUM) - 5; BUILD_BUG_ON(DESCS_AREAS != 1); - cdd->cd = dma_alloc_coherent(&pdev->dev, mem_decs, + cdd->cd = dma_alloc_coherent(dev, mem_decs, &cdd->descs_phys, GFP_KERNEL); if (!cdd->cd) return -ENOMEM; @@ -813,12 +815,12 @@ static void init_sched(struct cppi41_dd *cdd) cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL); } -static int init_cppi41(struct platform_device *pdev, struct cppi41_dd *cdd) +static int init_cppi41(struct device *dev, struct cppi41_dd *cdd) { int ret; BUILD_BUG_ON(QMGR_SCRATCH_SIZE > ((1 << 14) - 1)); - cdd->qmgr_scratch = dma_alloc_coherent(&pdev->dev, QMGR_SCRATCH_SIZE, + cdd->qmgr_scratch = dma_alloc_coherent(dev, QMGR_SCRATCH_SIZE, &cdd->scratch_phys, GFP_KERNEL); if (!cdd->qmgr_scratch) return -ENOMEM; @@ -827,7 +829,7 @@ static int init_cppi41(struct platform_device *pdev, struct cppi41_dd *cdd) cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE); cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE); - ret = init_descs(pdev, cdd); + ret = init_descs(dev, cdd); if (ret) goto err_td; @@ -835,7 +837,7 @@ static int init_cppi41(struct platform_device *pdev, struct cppi41_dd *cdd) init_sched(cdd); return 0; err_td: - deinit_cpii41(pdev, cdd); + deinit_cppi41(dev, cdd); return ret; } @@ -914,11 +916,11 @@ static const struct of_device_id cppi41_dma_ids[] = { }; MODULE_DEVICE_TABLE(of, cppi41_dma_ids); -static const struct cppi_glue_infos *get_glue_info(struct platform_device *pdev) +static const struct cppi_glue_infos *get_glue_info(struct device *dev) { const struct of_device_id *of_id; - of_id = of_match_node(cppi41_dma_ids, pdev->dev.of_node); + of_id = of_match_node(cppi41_dma_ids, dev->of_node); if (!of_id) return NULL; return of_id->data; @@ -927,11 +929,12 @@ static const struct cppi_glue_infos *get_glue_info(struct platform_device *pdev) static int cppi41_dma_probe(struct platform_device *pdev) { struct cppi41_dd *cdd; + struct device *dev = &pdev->dev; const struct cppi_glue_infos *glue_info; int irq; int ret; - glue_info = get_glue_info(pdev); + glue_info = get_glue_info(dev); if (!glue_info) return -EINVAL; @@ -946,14 +949,14 @@ static int cppi41_dma_probe(struct platform_device *pdev) cdd->ddev.device_issue_pending = cppi41_dma_issue_pending; cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg; cdd->ddev.device_control = cppi41_dma_control; - cdd->ddev.dev = &pdev->dev; + cdd->ddev.dev = dev; INIT_LIST_HEAD(&cdd->ddev.channels); cpp41_dma_info.dma_cap = cdd->ddev.cap_mask; - cdd->usbss_mem = of_iomap(pdev->dev.of_node, 0); - cdd->ctrl_mem = of_iomap(pdev->dev.of_node, 1); - cdd->sched_mem = of_iomap(pdev->dev.of_node, 2); - cdd->qmgr_mem = of_iomap(pdev->dev.of_node, 3); + cdd->usbss_mem = of_iomap(dev->of_node, 0); + cdd->ctrl_mem = of_iomap(dev->of_node, 1); + cdd->sched_mem = of_iomap(dev->of_node, 2); + cdd->qmgr_mem = of_iomap(dev->of_node, 3); if (!cdd->usbss_mem || !cdd->ctrl_mem || !cdd->sched_mem || !cdd->qmgr_mem) { @@ -961,31 +964,33 @@ static int cppi41_dma_probe(struct platform_device *pdev) goto err_remap; } - pm_runtime_enable(&pdev->dev); - ret = pm_runtime_get_sync(&pdev->dev); - if (ret) + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) goto err_get_sync; cdd->queues_rx = glue_info->queues_rx; cdd->queues_tx = glue_info->queues_tx; cdd->td_queue = glue_info->td_queue; - ret = init_cppi41(pdev, cdd); + ret = init_cppi41(dev, cdd); if (ret) goto err_init_cppi; - ret = cppi41_add_chans(pdev, cdd); + ret = cppi41_add_chans(dev, cdd); if (ret) goto err_chans; - irq = irq_of_parse_and_map(pdev->dev.of_node, 0); - if (!irq) + irq = irq_of_parse_and_map(dev->of_node, 0); + if (!irq) { + ret = -EINVAL; goto err_irq; + } cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER); ret = request_irq(irq, glue_info->isr, IRQF_SHARED, - dev_name(&pdev->dev), cdd); + dev_name(dev), cdd); if (ret) goto err_irq; cdd->irq = irq; @@ -994,7 +999,7 @@ static int cppi41_dma_probe(struct platform_device *pdev) if (ret) goto err_dma_reg; - ret = of_dma_controller_register(pdev->dev.of_node, + ret = of_dma_controller_register(dev->of_node, cppi41_dma_xlate, &cpp41_dma_info); if (ret) goto err_of; @@ -1009,11 +1014,11 @@ err_irq: cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR); cleanup_chans(cdd); err_chans: - deinit_cpii41(pdev, cdd); + deinit_cppi41(dev, cdd); err_init_cppi: - pm_runtime_put(&pdev->dev); + pm_runtime_put(dev); err_get_sync: - pm_runtime_disable(&pdev->dev); + pm_runtime_disable(dev); iounmap(cdd->usbss_mem); iounmap(cdd->ctrl_mem); iounmap(cdd->sched_mem); @@ -1033,7 +1038,7 @@ static int cppi41_dma_remove(struct platform_device *pdev) cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR); free_irq(cdd->irq, cdd); cleanup_chans(cdd); - deinit_cpii41(pdev, cdd); + deinit_cppi41(&pdev->dev, cdd); iounmap(cdd->usbss_mem); iounmap(cdd->ctrl_mem); iounmap(cdd->sched_mem); @@ -1044,12 +1049,53 @@ static int cppi41_dma_remove(struct platform_device *pdev) return 0; } +#ifdef CONFIG_PM_SLEEP +static int cppi41_suspend(struct device *dev) +{ + struct cppi41_dd *cdd = dev_get_drvdata(dev); + + cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ); + cppi_writel(0, cdd->usbss_mem + USBSS_IRQ_CLEARR); + disable_sched(cdd); + + return 0; +} + +static int cppi41_resume(struct device *dev) +{ + struct cppi41_dd *cdd = dev_get_drvdata(dev); + struct cppi41_channel *c; + int i; + + for (i = 0; i < DESCS_AREAS; i++) + cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i)); + + list_for_each_entry(c, &cdd->ddev.channels, chan.device_node) + if (!c->is_tx) + cppi_writel(c->q_num, c->gcr_reg + RXHPCRA0); + + init_sched(cdd); + + cppi_writel(cdd->dma_tdfdq, cdd->ctrl_mem + DMA_TDFDQ); + cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE); + cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE); + cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE); + + cppi_writel(USBSS_IRQ_PD_COMP, cdd->usbss_mem + USBSS_IRQ_ENABLER); + + return 0; +} +#endif + +static SIMPLE_DEV_PM_OPS(cppi41_pm_ops, cppi41_suspend, cppi41_resume); + static struct platform_driver cpp41_dma_driver = { .probe = cppi41_dma_probe, .remove = cppi41_dma_remove, .driver = { .name = "cppi41-dma-engine", .owner = THIS_MODULE, + .pm = &cppi41_pm_ops, .of_match_table = of_match_ptr(cppi41_dma_ids), }, }; diff --git a/drivers/dma/dma-jz4740.c b/drivers/dma/dma-jz4740.c index b0c0c8268d4..94c380f0753 100644 --- a/drivers/dma/dma-jz4740.c +++ b/drivers/dma/dma-jz4740.c @@ -491,7 +491,7 @@ static enum dma_status jz4740_dma_tx_status(struct dma_chan *c, unsigned long flags; status = dma_cookie_status(c, cookie, state); - if (status == DMA_SUCCESS || !state) + if (status == DMA_COMPLETE || !state) return status; spin_lock_irqsave(&chan->vchan.lock, flags); diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 9162ac80c18..d5d30ed863c 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -65,6 +65,7 @@ #include <linux/acpi.h> #include <linux/acpi_dma.h> #include <linux/of_dma.h> +#include <linux/mempool.h> static DEFINE_MUTEX(dma_list_mutex); static DEFINE_IDR(dma_idr); @@ -534,11 +535,41 @@ struct dma_chan *dma_get_slave_channel(struct dma_chan *chan) } EXPORT_SYMBOL_GPL(dma_get_slave_channel); +struct dma_chan *dma_get_any_slave_channel(struct dma_device *device) +{ + dma_cap_mask_t mask; + struct dma_chan *chan; + int err; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + /* lock against __dma_request_channel */ + mutex_lock(&dma_list_mutex); + + chan = private_candidate(&mask, device, NULL, NULL); + if (chan) { + err = dma_chan_get(chan); + if (err) { + pr_debug("%s: failed to get %s: (%d)\n", + __func__, dma_chan_name(chan), err); + chan = NULL; + } + } + + mutex_unlock(&dma_list_mutex); + + return chan; +} +EXPORT_SYMBOL_GPL(dma_get_any_slave_channel); + /** * __dma_request_channel - try to allocate an exclusive channel * @mask: capabilities that the channel must satisfy * @fn: optional callback to disposition available channels * @fn_param: opaque parameter to pass to dma_filter_fn + * + * Returns pointer to appropriate DMA channel on success or NULL. */ struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, dma_filter_fn fn, void *fn_param) @@ -590,8 +621,11 @@ EXPORT_SYMBOL_GPL(__dma_request_channel); * dma_request_slave_channel - try to allocate an exclusive slave channel * @dev: pointer to client device structure * @name: slave channel name + * + * Returns pointer to appropriate DMA channel on success or an error pointer. */ -struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name) +struct dma_chan *dma_request_slave_channel_reason(struct device *dev, + const char *name) { /* If device-tree is present get slave info from here */ if (dev->of_node) @@ -601,7 +635,24 @@ struct dma_chan *dma_request_slave_channel(struct device *dev, const char *name) if (ACPI_HANDLE(dev)) return acpi_dma_request_slave_chan_by_name(dev, name); - return NULL; + return ERR_PTR(-ENODEV); +} +EXPORT_SYMBOL_GPL(dma_request_slave_channel_reason); + +/** + * dma_request_slave_channel - try to allocate an exclusive slave channel + * @dev: pointer to client device structure + * @name: slave channel name + * + * Returns pointer to appropriate DMA channel on success or NULL. + */ +struct dma_chan *dma_request_slave_channel(struct device *dev, + const char *name) +{ + struct dma_chan *ch = dma_request_slave_channel_reason(dev, name); + if (IS_ERR(ch)) + return NULL; + return ch; } EXPORT_SYMBOL_GPL(dma_request_slave_channel); @@ -901,98 +952,134 @@ void dma_async_device_unregister(struct dma_device *device) } EXPORT_SYMBOL(dma_async_device_unregister); -/** - * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses - * @chan: DMA channel to offload copy to - * @dest: destination address (virtual) - * @src: source address (virtual) - * @len: length - * - * Both @dest and @src must be mappable to a bus address according to the - * DMA mapping API rules for streaming mappings. - * Both @dest and @src must stay memory resident (kernel memory or locked - * user space pages). - */ -dma_cookie_t -dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, - void *src, size_t len) -{ - struct dma_device *dev = chan->device; - struct dma_async_tx_descriptor *tx; - dma_addr_t dma_dest, dma_src; - dma_cookie_t cookie; - unsigned long flags; +struct dmaengine_unmap_pool { + struct kmem_cache *cache; + const char *name; + mempool_t *pool; + size_t size; +}; - dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); - dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE); - flags = DMA_CTRL_ACK | - DMA_COMPL_SRC_UNMAP_SINGLE | - DMA_COMPL_DEST_UNMAP_SINGLE; - tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); +#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) } +static struct dmaengine_unmap_pool unmap_pool[] = { + __UNMAP_POOL(2), + #if IS_ENABLED(CONFIG_DMA_ENGINE_RAID) + __UNMAP_POOL(16), + __UNMAP_POOL(128), + __UNMAP_POOL(256), + #endif +}; - if (!tx) { - dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); - dma_unmap_single(dev->dev, dma_dest, len, DMA_FROM_DEVICE); - return -ENOMEM; +static struct dmaengine_unmap_pool *__get_unmap_pool(int nr) +{ + int order = get_count_order(nr); + + switch (order) { + case 0 ... 1: + return &unmap_pool[0]; + case 2 ... 4: + return &unmap_pool[1]; + case 5 ... 7: + return &unmap_pool[2]; + case 8: + return &unmap_pool[3]; + default: + BUG(); + return NULL; } +} - tx->callback = NULL; - cookie = tx->tx_submit(tx); +static void dmaengine_unmap(struct kref *kref) +{ + struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref); + struct device *dev = unmap->dev; + int cnt, i; + + cnt = unmap->to_cnt; + for (i = 0; i < cnt; i++) + dma_unmap_page(dev, unmap->addr[i], unmap->len, + DMA_TO_DEVICE); + cnt += unmap->from_cnt; + for (; i < cnt; i++) + dma_unmap_page(dev, unmap->addr[i], unmap->len, + DMA_FROM_DEVICE); + cnt += unmap->bidi_cnt; + for (; i < cnt; i++) { + if (unmap->addr[i] == 0) + continue; + dma_unmap_page(dev, unmap->addr[i], unmap->len, + DMA_BIDIRECTIONAL); + } + cnt = unmap->map_cnt; + mempool_free(unmap, __get_unmap_pool(cnt)->pool); +} - preempt_disable(); - __this_cpu_add(chan->local->bytes_transferred, len); - __this_cpu_inc(chan->local->memcpy_count); - preempt_enable(); +void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap) +{ + if (unmap) + kref_put(&unmap->kref, dmaengine_unmap); +} +EXPORT_SYMBOL_GPL(dmaengine_unmap_put); - return cookie; +static void dmaengine_destroy_unmap_pool(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) { + struct dmaengine_unmap_pool *p = &unmap_pool[i]; + + if (p->pool) + mempool_destroy(p->pool); + p->pool = NULL; + if (p->cache) + kmem_cache_destroy(p->cache); + p->cache = NULL; + } } -EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf); -/** - * dma_async_memcpy_buf_to_pg - offloaded copy from address to page - * @chan: DMA channel to offload copy to - * @page: destination page - * @offset: offset in page to copy to - * @kdata: source address (virtual) - * @len: length - * - * Both @page/@offset and @kdata must be mappable to a bus address according - * to the DMA mapping API rules for streaming mappings. - * Both @page/@offset and @kdata must stay memory resident (kernel memory or - * locked user space pages) - */ -dma_cookie_t -dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, - unsigned int offset, void *kdata, size_t len) +static int __init dmaengine_init_unmap_pool(void) { - struct dma_device *dev = chan->device; - struct dma_async_tx_descriptor *tx; - dma_addr_t dma_dest, dma_src; - dma_cookie_t cookie; - unsigned long flags; + int i; - dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); - dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE); - flags = DMA_CTRL_ACK | DMA_COMPL_SRC_UNMAP_SINGLE; - tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); + for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) { + struct dmaengine_unmap_pool *p = &unmap_pool[i]; + size_t size; - if (!tx) { - dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); - dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); - return -ENOMEM; + size = sizeof(struct dmaengine_unmap_data) + + sizeof(dma_addr_t) * p->size; + + p->cache = kmem_cache_create(p->name, size, 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!p->cache) + break; + p->pool = mempool_create_slab_pool(1, p->cache); + if (!p->pool) + break; } - tx->callback = NULL; - cookie = tx->tx_submit(tx); + if (i == ARRAY_SIZE(unmap_pool)) + return 0; - preempt_disable(); - __this_cpu_add(chan->local->bytes_transferred, len); - __this_cpu_inc(chan->local->memcpy_count); - preempt_enable(); + dmaengine_destroy_unmap_pool(); + return -ENOMEM; +} - return cookie; +struct dmaengine_unmap_data * +dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags) +{ + struct dmaengine_unmap_data *unmap; + + unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags); + if (!unmap) + return NULL; + + memset(unmap, 0, sizeof(*unmap)); + kref_init(&unmap->kref); + unmap->dev = dev; + unmap->map_cnt = nr; + + return unmap; } -EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); +EXPORT_SYMBOL(dmaengine_get_unmap_data); /** * dma_async_memcpy_pg_to_pg - offloaded copy from page to page @@ -1015,24 +1102,33 @@ dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, { struct dma_device *dev = chan->device; struct dma_async_tx_descriptor *tx; - dma_addr_t dma_dest, dma_src; + struct dmaengine_unmap_data *unmap; dma_cookie_t cookie; unsigned long flags; - dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE); - dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len, - DMA_FROM_DEVICE); + unmap = dmaengine_get_unmap_data(dev->dev, 2, GFP_NOWAIT); + if (!unmap) + return -ENOMEM; + + unmap->to_cnt = 1; + unmap->from_cnt = 1; + unmap->addr[0] = dma_map_page(dev->dev, src_pg, src_off, len, + DMA_TO_DEVICE); + unmap->addr[1] = dma_map_page(dev->dev, dest_pg, dest_off, len, + DMA_FROM_DEVICE); + unmap->len = len; flags = DMA_CTRL_ACK; - tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); + tx = dev->device_prep_dma_memcpy(chan, unmap->addr[1], unmap->addr[0], + len, flags); if (!tx) { - dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE); - dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); + dmaengine_unmap_put(unmap); return -ENOMEM; } - tx->callback = NULL; + dma_set_unmap(tx, unmap); cookie = tx->tx_submit(tx); + dmaengine_unmap_put(unmap); preempt_disable(); __this_cpu_add(chan->local->bytes_transferred, len); @@ -1043,6 +1139,52 @@ dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, } EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg); +/** + * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses + * @chan: DMA channel to offload copy to + * @dest: destination address (virtual) + * @src: source address (virtual) + * @len: length + * + * Both @dest and @src must be mappable to a bus address according to the + * DMA mapping API rules for streaming mappings. + * Both @dest and @src must stay memory resident (kernel memory or locked + * user space pages). + */ +dma_cookie_t +dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, + void *src, size_t len) +{ + return dma_async_memcpy_pg_to_pg(chan, virt_to_page(dest), + (unsigned long) dest & ~PAGE_MASK, + virt_to_page(src), + (unsigned long) src & ~PAGE_MASK, len); +} +EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf); + +/** + * dma_async_memcpy_buf_to_pg - offloaded copy from address to page + * @chan: DMA channel to offload copy to + * @page: destination page + * @offset: offset in page to copy to + * @kdata: source address (virtual) + * @len: length + * + * Both @page/@offset and @kdata must be mappable to a bus address according + * to the DMA mapping API rules for streaming mappings. + * Both @page/@offset and @kdata must stay memory resident (kernel memory or + * locked user space pages) + */ +dma_cookie_t +dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, + unsigned int offset, void *kdata, size_t len) +{ + return dma_async_memcpy_pg_to_pg(chan, page, offset, + virt_to_page(kdata), + (unsigned long) kdata & ~PAGE_MASK, len); +} +EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); + void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, struct dma_chan *chan) { @@ -1062,7 +1204,7 @@ dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx) unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); if (!tx) - return DMA_SUCCESS; + return DMA_COMPLETE; while (tx->cookie == -EBUSY) { if (time_after_eq(jiffies, dma_sync_wait_timeout)) { @@ -1116,6 +1258,10 @@ EXPORT_SYMBOL_GPL(dma_run_dependencies); static int __init dma_bus_init(void) { + int err = dmaengine_init_unmap_pool(); + + if (err) + return err; return class_register(&dma_devclass); } arch_initcall(dma_bus_init); diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c index 92f796cdc6a..e27cec25c59 100644 --- a/drivers/dma/dmatest.c +++ b/drivers/dma/dmatest.c @@ -8,6 +8,8 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/dmaengine.h> @@ -19,10 +21,6 @@ #include <linux/random.h> #include <linux/slab.h> #include <linux/wait.h> -#include <linux/ctype.h> -#include <linux/debugfs.h> -#include <linux/uaccess.h> -#include <linux/seq_file.h> static unsigned int test_buf_size = 16384; module_param(test_buf_size, uint, S_IRUGO | S_IWUSR); @@ -33,7 +31,7 @@ module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO | S_IWUSR); MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); -static char test_device[20]; +static char test_device[32]; module_param_string(device, test_device, sizeof(test_device), S_IRUGO | S_IWUSR); MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); @@ -68,6 +66,68 @@ module_param(timeout, uint, S_IRUGO | S_IWUSR); MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " "Pass -1 for infinite timeout"); +static bool noverify; +module_param(noverify, bool, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(noverify, "Disable random data setup and verification"); + +static bool verbose; +module_param(verbose, bool, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(verbose, "Enable \"success\" result messages (default: off)"); + +/** + * struct dmatest_params - test parameters. + * @buf_size: size of the memcpy test buffer + * @channel: bus ID of the channel to test + * @device: bus ID of the DMA Engine to test + * @threads_per_chan: number of threads to start per channel + * @max_channels: maximum number of channels to use + * @iterations: iterations before stopping test + * @xor_sources: number of xor source buffers + * @pq_sources: number of p+q source buffers + * @timeout: transfer timeout in msec, -1 for infinite timeout + */ +struct dmatest_params { + unsigned int buf_size; + char channel[20]; + char device[32]; + unsigned int threads_per_chan; + unsigned int max_channels; + unsigned int iterations; + unsigned int xor_sources; + unsigned int pq_sources; + int timeout; + bool noverify; +}; + +/** + * struct dmatest_info - test information. + * @params: test parameters + * @lock: access protection to the fields of this structure + */ +static struct dmatest_info { + /* Test parameters */ + struct dmatest_params params; + + /* Internal state */ + struct list_head channels; + unsigned int nr_channels; + struct mutex lock; + bool did_init; +} test_info = { + .channels = LIST_HEAD_INIT(test_info.channels), + .lock = __MUTEX_INITIALIZER(test_info.lock), +}; + +static int dmatest_run_set(const char *val, const struct kernel_param *kp); +static int dmatest_run_get(char *val, const struct kernel_param *kp); +static struct kernel_param_ops run_ops = { + .set = dmatest_run_set, + .get = dmatest_run_get, +}; +static bool dmatest_run; +module_param_cb(run, &run_ops, &dmatest_run, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(run, "Run the test (default: false)"); + /* Maximum amount of mismatched bytes in buffer to print */ #define MAX_ERROR_COUNT 32 @@ -88,56 +148,6 @@ MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " #define PATTERN_OVERWRITE 0x20 #define PATTERN_COUNT_MASK 0x1f -enum dmatest_error_type { - DMATEST_ET_OK, - DMATEST_ET_MAP_SRC, - DMATEST_ET_MAP_DST, - DMATEST_ET_PREP, - DMATEST_ET_SUBMIT, - DMATEST_ET_TIMEOUT, - DMATEST_ET_DMA_ERROR, - DMATEST_ET_DMA_IN_PROGRESS, - DMATEST_ET_VERIFY, - DMATEST_ET_VERIFY_BUF, -}; - -struct dmatest_verify_buffer { - unsigned int index; - u8 expected; - u8 actual; -}; - -struct dmatest_verify_result { - unsigned int error_count; - struct dmatest_verify_buffer data[MAX_ERROR_COUNT]; - u8 pattern; - bool is_srcbuf; -}; - -struct dmatest_thread_result { - struct list_head node; - unsigned int n; - unsigned int src_off; - unsigned int dst_off; - unsigned int len; - enum dmatest_error_type type; - union { - unsigned long data; - dma_cookie_t cookie; - enum dma_status status; - int error; - struct dmatest_verify_result *vr; - }; -}; - -struct dmatest_result { - struct list_head node; - char *name; - struct list_head results; -}; - -struct dmatest_info; - struct dmatest_thread { struct list_head node; struct dmatest_info *info; @@ -155,53 +165,42 @@ struct dmatest_chan { struct list_head threads; }; -/** - * struct dmatest_params - test parameters. - * @buf_size: size of the memcpy test buffer - * @channel: bus ID of the channel to test - * @device: bus ID of the DMA Engine to test - * @threads_per_chan: number of threads to start per channel - * @max_channels: maximum number of channels to use - * @iterations: iterations before stopping test - * @xor_sources: number of xor source buffers - * @pq_sources: number of p+q source buffers - * @timeout: transfer timeout in msec, -1 for infinite timeout - */ -struct dmatest_params { - unsigned int buf_size; - char channel[20]; - char device[20]; - unsigned int threads_per_chan; - unsigned int max_channels; - unsigned int iterations; - unsigned int xor_sources; - unsigned int pq_sources; - int timeout; -}; +static DECLARE_WAIT_QUEUE_HEAD(thread_wait); +static bool wait; -/** - * struct dmatest_info - test information. - * @params: test parameters - * @lock: access protection to the fields of this structure - */ -struct dmatest_info { - /* Test parameters */ - struct dmatest_params params; +static bool is_threaded_test_run(struct dmatest_info *info) +{ + struct dmatest_chan *dtc; - /* Internal state */ - struct list_head channels; - unsigned int nr_channels; - struct mutex lock; + list_for_each_entry(dtc, &info->channels, node) { + struct dmatest_thread *thread; - /* debugfs related stuff */ - struct dentry *root; + list_for_each_entry(thread, &dtc->threads, node) { + if (!thread->done) + return true; + } + } - /* Test results */ - struct list_head results; - struct mutex results_lock; -}; + return false; +} -static struct dmatest_info test_info; +static int dmatest_wait_get(char *val, const struct kernel_param *kp) +{ + struct dmatest_info *info = &test_info; + struct dmatest_params *params = &info->params; + + if (params->iterations) + wait_event(thread_wait, !is_threaded_test_run(info)); + wait = true; + return param_get_bool(val, kp); +} + +static struct kernel_param_ops wait_ops = { + .get = dmatest_wait_get, + .set = param_set_bool, +}; +module_param_cb(wait, &wait_ops, &wait, S_IRUGO); +MODULE_PARM_DESC(wait, "Wait for tests to complete (default: false)"); static bool dmatest_match_channel(struct dmatest_params *params, struct dma_chan *chan) @@ -223,7 +222,7 @@ static unsigned long dmatest_random(void) { unsigned long buf; - get_random_bytes(&buf, sizeof(buf)); + prandom_bytes(&buf, sizeof(buf)); return buf; } @@ -262,9 +261,31 @@ static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len, } } -static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs, - unsigned int start, unsigned int end, unsigned int counter, - u8 pattern, bool is_srcbuf) +static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, + unsigned int counter, bool is_srcbuf) +{ + u8 diff = actual ^ pattern; + u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); + const char *thread_name = current->comm; + + if (is_srcbuf) + pr_warn("%s: srcbuf[0x%x] overwritten! Expected %02x, got %02x\n", + thread_name, index, expected, actual); + else if ((pattern & PATTERN_COPY) + && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) + pr_warn("%s: dstbuf[0x%x] not copied! Expected %02x, got %02x\n", + thread_name, index, expected, actual); + else if (diff & PATTERN_SRC) + pr_warn("%s: dstbuf[0x%x] was copied! Expected %02x, got %02x\n", + thread_name, index, expected, actual); + else + pr_warn("%s: dstbuf[0x%x] mismatch! Expected %02x, got %02x\n", + thread_name, index, expected, actual); +} + +static unsigned int dmatest_verify(u8 **bufs, unsigned int start, + unsigned int end, unsigned int counter, u8 pattern, + bool is_srcbuf) { unsigned int i; unsigned int error_count = 0; @@ -272,7 +293,6 @@ static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs, u8 expected; u8 *buf; unsigned int counter_orig = counter; - struct dmatest_verify_buffer *vb; for (; (buf = *bufs); bufs++) { counter = counter_orig; @@ -280,12 +300,9 @@ static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs, actual = buf[i]; expected = pattern | (~counter & PATTERN_COUNT_MASK); if (actual != expected) { - if (error_count < MAX_ERROR_COUNT && vr) { - vb = &vr->data[error_count]; - vb->index = i; - vb->expected = expected; - vb->actual = actual; - } + if (error_count < MAX_ERROR_COUNT) + dmatest_mismatch(actual, pattern, i, + counter, is_srcbuf); error_count++; } counter++; @@ -293,7 +310,7 @@ static unsigned int dmatest_verify(struct dmatest_verify_result *vr, u8 **bufs, } if (error_count > MAX_ERROR_COUNT) - pr_warning("%s: %u errors suppressed\n", + pr_warn("%s: %u errors suppressed\n", current->comm, error_count - MAX_ERROR_COUNT); return error_count; @@ -313,20 +330,6 @@ static void dmatest_callback(void *arg) wake_up_all(done->wait); } -static inline void unmap_src(struct device *dev, dma_addr_t *addr, size_t len, - unsigned int count) -{ - while (count--) - dma_unmap_single(dev, addr[count], len, DMA_TO_DEVICE); -} - -static inline void unmap_dst(struct device *dev, dma_addr_t *addr, size_t len, - unsigned int count) -{ - while (count--) - dma_unmap_single(dev, addr[count], len, DMA_BIDIRECTIONAL); -} - static unsigned int min_odd(unsigned int x, unsigned int y) { unsigned int val = min(x, y); @@ -334,172 +337,49 @@ static unsigned int min_odd(unsigned int x, unsigned int y) return val % 2 ? val : val - 1; } -static char *verify_result_get_one(struct dmatest_verify_result *vr, - unsigned int i) +static void result(const char *err, unsigned int n, unsigned int src_off, + unsigned int dst_off, unsigned int len, unsigned long data) { - struct dmatest_verify_buffer *vb = &vr->data[i]; - u8 diff = vb->actual ^ vr->pattern; - static char buf[512]; - char *msg; - - if (vr->is_srcbuf) - msg = "srcbuf overwritten!"; - else if ((vr->pattern & PATTERN_COPY) - && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) - msg = "dstbuf not copied!"; - else if (diff & PATTERN_SRC) - msg = "dstbuf was copied!"; - else - msg = "dstbuf mismatch!"; - - snprintf(buf, sizeof(buf) - 1, "%s [0x%x] Expected %02x, got %02x", msg, - vb->index, vb->expected, vb->actual); - - return buf; + pr_info("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n", + current->comm, n, err, src_off, dst_off, len, data); } -static char *thread_result_get(const char *name, - struct dmatest_thread_result *tr) +static void dbg_result(const char *err, unsigned int n, unsigned int src_off, + unsigned int dst_off, unsigned int len, + unsigned long data) { - static const char * const messages[] = { - [DMATEST_ET_OK] = "No errors", - [DMATEST_ET_MAP_SRC] = "src mapping error", - [DMATEST_ET_MAP_DST] = "dst mapping error", - [DMATEST_ET_PREP] = "prep error", - [DMATEST_ET_SUBMIT] = "submit error", - [DMATEST_ET_TIMEOUT] = "test timed out", - [DMATEST_ET_DMA_ERROR] = - "got completion callback (DMA_ERROR)", - [DMATEST_ET_DMA_IN_PROGRESS] = - "got completion callback (DMA_IN_PROGRESS)", - [DMATEST_ET_VERIFY] = "errors", - [DMATEST_ET_VERIFY_BUF] = "verify errors", - }; - static char buf[512]; - - snprintf(buf, sizeof(buf) - 1, - "%s: #%u: %s with src_off=0x%x ""dst_off=0x%x len=0x%x (%lu)", - name, tr->n, messages[tr->type], tr->src_off, tr->dst_off, - tr->len, tr->data); - - return buf; + pr_debug("%s: result #%u: '%s' with src_off=0x%x dst_off=0x%x len=0x%x (%lu)\n", + current->comm, n, err, src_off, dst_off, len, data); } -static int thread_result_add(struct dmatest_info *info, - struct dmatest_result *r, enum dmatest_error_type type, - unsigned int n, unsigned int src_off, unsigned int dst_off, - unsigned int len, unsigned long data) -{ - struct dmatest_thread_result *tr; +#define verbose_result(err, n, src_off, dst_off, len, data) ({ \ + if (verbose) \ + result(err, n, src_off, dst_off, len, data); \ + else \ + dbg_result(err, n, src_off, dst_off, len, data); \ +}) - tr = kzalloc(sizeof(*tr), GFP_KERNEL); - if (!tr) - return -ENOMEM; - - tr->type = type; - tr->n = n; - tr->src_off = src_off; - tr->dst_off = dst_off; - tr->len = len; - tr->data = data; - - mutex_lock(&info->results_lock); - list_add_tail(&tr->node, &r->results); - mutex_unlock(&info->results_lock); - - if (tr->type == DMATEST_ET_OK) - pr_debug("%s\n", thread_result_get(r->name, tr)); - else - pr_warn("%s\n", thread_result_get(r->name, tr)); - - return 0; -} - -static unsigned int verify_result_add(struct dmatest_info *info, - struct dmatest_result *r, unsigned int n, - unsigned int src_off, unsigned int dst_off, unsigned int len, - u8 **bufs, int whence, unsigned int counter, u8 pattern, - bool is_srcbuf) +static unsigned long long dmatest_persec(s64 runtime, unsigned int val) { - struct dmatest_verify_result *vr; - unsigned int error_count; - unsigned int buf_off = is_srcbuf ? src_off : dst_off; - unsigned int start, end; - - if (whence < 0) { - start = 0; - end = buf_off; - } else if (whence > 0) { - start = buf_off + len; - end = info->params.buf_size; - } else { - start = buf_off; - end = buf_off + len; - } + unsigned long long per_sec = 1000000; - vr = kmalloc(sizeof(*vr), GFP_KERNEL); - if (!vr) { - pr_warn("dmatest: No memory to store verify result\n"); - return dmatest_verify(NULL, bufs, start, end, counter, pattern, - is_srcbuf); - } - - vr->pattern = pattern; - vr->is_srcbuf = is_srcbuf; - - error_count = dmatest_verify(vr, bufs, start, end, counter, pattern, - is_srcbuf); - if (error_count) { - vr->error_count = error_count; - thread_result_add(info, r, DMATEST_ET_VERIFY_BUF, n, src_off, - dst_off, len, (unsigned long)vr); - return error_count; - } - - kfree(vr); - return 0; -} - -static void result_free(struct dmatest_info *info, const char *name) -{ - struct dmatest_result *r, *_r; - - mutex_lock(&info->results_lock); - list_for_each_entry_safe(r, _r, &info->results, node) { - struct dmatest_thread_result *tr, *_tr; - - if (name && strcmp(r->name, name)) - continue; - - list_for_each_entry_safe(tr, _tr, &r->results, node) { - if (tr->type == DMATEST_ET_VERIFY_BUF) - kfree(tr->vr); - list_del(&tr->node); - kfree(tr); - } + if (runtime <= 0) + return 0; - kfree(r->name); - list_del(&r->node); - kfree(r); + /* drop precision until runtime is 32-bits */ + while (runtime > UINT_MAX) { + runtime >>= 1; + per_sec <<= 1; } - mutex_unlock(&info->results_lock); + per_sec *= val; + do_div(per_sec, runtime); + return per_sec; } -static struct dmatest_result *result_init(struct dmatest_info *info, - const char *name) +static unsigned long long dmatest_KBs(s64 runtime, unsigned long long len) { - struct dmatest_result *r; - - r = kzalloc(sizeof(*r), GFP_KERNEL); - if (r) { - r->name = kstrdup(name, GFP_KERNEL); - INIT_LIST_HEAD(&r->results); - mutex_lock(&info->results_lock); - list_add_tail(&r->node, &info->results); - mutex_unlock(&info->results_lock); - } - return r; + return dmatest_persec(runtime, len >> 10); } /* @@ -525,7 +405,6 @@ static int dmatest_func(void *data) struct dmatest_params *params; struct dma_chan *chan; struct dma_device *dev; - const char *thread_name; unsigned int src_off, dst_off, len; unsigned int error_count; unsigned int failed_tests = 0; @@ -538,9 +417,10 @@ static int dmatest_func(void *data) int src_cnt; int dst_cnt; int i; - struct dmatest_result *result; + ktime_t ktime; + s64 runtime = 0; + unsigned long long total_len = 0; - thread_name = current->comm; set_freezable(); ret = -ENOMEM; @@ -570,10 +450,6 @@ static int dmatest_func(void *data) } else goto err_thread_type; - result = result_init(info, thread_name); - if (!result) - goto err_srcs; - thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); if (!thread->srcs) goto err_srcs; @@ -597,17 +473,17 @@ static int dmatest_func(void *data) set_user_nice(current, 10); /* - * src buffers are freed by the DMAEngine code with dma_unmap_single() - * dst buffers are freed by ourselves below + * src and dst buffers are freed by ourselves below */ - flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT - | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE; + flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT; + ktime = ktime_get(); while (!kthread_should_stop() && !(params->iterations && total_tests >= params->iterations)) { struct dma_async_tx_descriptor *tx = NULL; - dma_addr_t dma_srcs[src_cnt]; - dma_addr_t dma_dsts[dst_cnt]; + struct dmaengine_unmap_data *um; + dma_addr_t srcs[src_cnt]; + dma_addr_t *dsts; u8 align = 0; total_tests++; @@ -626,81 +502,103 @@ static int dmatest_func(void *data) break; } - len = dmatest_random() % params->buf_size + 1; + if (params->noverify) { + len = params->buf_size; + src_off = 0; + dst_off = 0; + } else { + len = dmatest_random() % params->buf_size + 1; + len = (len >> align) << align; + if (!len) + len = 1 << align; + src_off = dmatest_random() % (params->buf_size - len + 1); + dst_off = dmatest_random() % (params->buf_size - len + 1); + + src_off = (src_off >> align) << align; + dst_off = (dst_off >> align) << align; + + dmatest_init_srcs(thread->srcs, src_off, len, + params->buf_size); + dmatest_init_dsts(thread->dsts, dst_off, len, + params->buf_size); + } + len = (len >> align) << align; if (!len) len = 1 << align; - src_off = dmatest_random() % (params->buf_size - len + 1); - dst_off = dmatest_random() % (params->buf_size - len + 1); - - src_off = (src_off >> align) << align; - dst_off = (dst_off >> align) << align; + total_len += len; - dmatest_init_srcs(thread->srcs, src_off, len, params->buf_size); - dmatest_init_dsts(thread->dsts, dst_off, len, params->buf_size); + um = dmaengine_get_unmap_data(dev->dev, src_cnt+dst_cnt, + GFP_KERNEL); + if (!um) { + failed_tests++; + result("unmap data NULL", total_tests, + src_off, dst_off, len, ret); + continue; + } + um->len = params->buf_size; for (i = 0; i < src_cnt; i++) { - u8 *buf = thread->srcs[i] + src_off; - - dma_srcs[i] = dma_map_single(dev->dev, buf, len, - DMA_TO_DEVICE); - ret = dma_mapping_error(dev->dev, dma_srcs[i]); + void *buf = thread->srcs[i]; + struct page *pg = virt_to_page(buf); + unsigned pg_off = (unsigned long) buf & ~PAGE_MASK; + + um->addr[i] = dma_map_page(dev->dev, pg, pg_off, + um->len, DMA_TO_DEVICE); + srcs[i] = um->addr[i] + src_off; + ret = dma_mapping_error(dev->dev, um->addr[i]); if (ret) { - unmap_src(dev->dev, dma_srcs, len, i); - thread_result_add(info, result, - DMATEST_ET_MAP_SRC, - total_tests, src_off, dst_off, - len, ret); + dmaengine_unmap_put(um); + result("src mapping error", total_tests, + src_off, dst_off, len, ret); failed_tests++; continue; } + um->to_cnt++; } /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ + dsts = &um->addr[src_cnt]; for (i = 0; i < dst_cnt; i++) { - dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], - params->buf_size, - DMA_BIDIRECTIONAL); - ret = dma_mapping_error(dev->dev, dma_dsts[i]); + void *buf = thread->dsts[i]; + struct page *pg = virt_to_page(buf); + unsigned pg_off = (unsigned long) buf & ~PAGE_MASK; + + dsts[i] = dma_map_page(dev->dev, pg, pg_off, um->len, + DMA_BIDIRECTIONAL); + ret = dma_mapping_error(dev->dev, dsts[i]); if (ret) { - unmap_src(dev->dev, dma_srcs, len, src_cnt); - unmap_dst(dev->dev, dma_dsts, params->buf_size, - i); - thread_result_add(info, result, - DMATEST_ET_MAP_DST, - total_tests, src_off, dst_off, - len, ret); + dmaengine_unmap_put(um); + result("dst mapping error", total_tests, + src_off, dst_off, len, ret); failed_tests++; continue; } + um->bidi_cnt++; } if (thread->type == DMA_MEMCPY) tx = dev->device_prep_dma_memcpy(chan, - dma_dsts[0] + dst_off, - dma_srcs[0], len, - flags); + dsts[0] + dst_off, + srcs[0], len, flags); else if (thread->type == DMA_XOR) tx = dev->device_prep_dma_xor(chan, - dma_dsts[0] + dst_off, - dma_srcs, src_cnt, + dsts[0] + dst_off, + srcs, src_cnt, len, flags); else if (thread->type == DMA_PQ) { dma_addr_t dma_pq[dst_cnt]; for (i = 0; i < dst_cnt; i++) - dma_pq[i] = dma_dsts[i] + dst_off; - tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, + dma_pq[i] = dsts[i] + dst_off; + tx = dev->device_prep_dma_pq(chan, dma_pq, srcs, src_cnt, pq_coefs, len, flags); } if (!tx) { - unmap_src(dev->dev, dma_srcs, len, src_cnt); - unmap_dst(dev->dev, dma_dsts, params->buf_size, - dst_cnt); - thread_result_add(info, result, DMATEST_ET_PREP, - total_tests, src_off, dst_off, - len, 0); + dmaengine_unmap_put(um); + result("prep error", total_tests, src_off, + dst_off, len, ret); msleep(100); failed_tests++; continue; @@ -712,9 +610,9 @@ static int dmatest_func(void *data) cookie = tx->tx_submit(tx); if (dma_submit_error(cookie)) { - thread_result_add(info, result, DMATEST_ET_SUBMIT, - total_tests, src_off, dst_off, - len, cookie); + dmaengine_unmap_put(um); + result("submit error", total_tests, src_off, + dst_off, len, ret); msleep(100); failed_tests++; continue; @@ -735,59 +633,59 @@ static int dmatest_func(void *data) * free it this time?" dancing. For now, just * leave it dangling. */ - thread_result_add(info, result, DMATEST_ET_TIMEOUT, - total_tests, src_off, dst_off, - len, 0); + dmaengine_unmap_put(um); + result("test timed out", total_tests, src_off, dst_off, + len, 0); failed_tests++; continue; - } else if (status != DMA_SUCCESS) { - enum dmatest_error_type type = (status == DMA_ERROR) ? - DMATEST_ET_DMA_ERROR : DMATEST_ET_DMA_IN_PROGRESS; - thread_result_add(info, result, type, - total_tests, src_off, dst_off, - len, status); + } else if (status != DMA_COMPLETE) { + dmaengine_unmap_put(um); + result(status == DMA_ERROR ? + "completion error status" : + "completion busy status", total_tests, src_off, + dst_off, len, ret); failed_tests++; continue; } - /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ - unmap_dst(dev->dev, dma_dsts, params->buf_size, dst_cnt); + dmaengine_unmap_put(um); - error_count = 0; + if (params->noverify) { + verbose_result("test passed", total_tests, src_off, + dst_off, len, 0); + continue; + } - pr_debug("%s: verifying source buffer...\n", thread_name); - error_count += verify_result_add(info, result, total_tests, - src_off, dst_off, len, thread->srcs, -1, + pr_debug("%s: verifying source buffer...\n", current->comm); + error_count = dmatest_verify(thread->srcs, 0, src_off, 0, PATTERN_SRC, true); - error_count += verify_result_add(info, result, total_tests, - src_off, dst_off, len, thread->srcs, 0, - src_off, PATTERN_SRC | PATTERN_COPY, true); - error_count += verify_result_add(info, result, total_tests, - src_off, dst_off, len, thread->srcs, 1, - src_off + len, PATTERN_SRC, true); - - pr_debug("%s: verifying dest buffer...\n", thread_name); - error_count += verify_result_add(info, result, total_tests, - src_off, dst_off, len, thread->dsts, -1, + error_count += dmatest_verify(thread->srcs, src_off, + src_off + len, src_off, + PATTERN_SRC | PATTERN_COPY, true); + error_count += dmatest_verify(thread->srcs, src_off + len, + params->buf_size, src_off + len, + PATTERN_SRC, true); + + pr_debug("%s: verifying dest buffer...\n", current->comm); + error_count += dmatest_verify(thread->dsts, 0, dst_off, 0, PATTERN_DST, false); - error_count += verify_result_add(info, result, total_tests, - src_off, dst_off, len, thread->dsts, 0, - src_off, PATTERN_SRC | PATTERN_COPY, false); - error_count += verify_result_add(info, result, total_tests, - src_off, dst_off, len, thread->dsts, 1, - dst_off + len, PATTERN_DST, false); + error_count += dmatest_verify(thread->dsts, dst_off, + dst_off + len, src_off, + PATTERN_SRC | PATTERN_COPY, false); + error_count += dmatest_verify(thread->dsts, dst_off + len, + params->buf_size, dst_off + len, + PATTERN_DST, false); if (error_count) { - thread_result_add(info, result, DMATEST_ET_VERIFY, - total_tests, src_off, dst_off, - len, error_count); + result("data error", total_tests, src_off, dst_off, + len, error_count); failed_tests++; } else { - thread_result_add(info, result, DMATEST_ET_OK, - total_tests, src_off, dst_off, - len, 0); + verbose_result("test passed", total_tests, src_off, + dst_off, len, 0); } } + runtime = ktime_us_delta(ktime_get(), ktime); ret = 0; for (i = 0; thread->dsts[i]; i++) @@ -802,20 +700,17 @@ err_srcbuf: err_srcs: kfree(pq_coefs); err_thread_type: - pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", - thread_name, total_tests, failed_tests, ret); + pr_info("%s: summary %u tests, %u failures %llu iops %llu KB/s (%d)\n", + current->comm, total_tests, failed_tests, + dmatest_persec(runtime, total_tests), + dmatest_KBs(runtime, total_len), ret); /* terminate all transfers on specified channels */ if (ret) dmaengine_terminate_all(chan); thread->done = true; - - if (params->iterations > 0) - while (!kthread_should_stop()) { - DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit); - interruptible_sleep_on(&wait_dmatest_exit); - } + wake_up(&thread_wait); return ret; } @@ -828,9 +723,10 @@ static void dmatest_cleanup_channel(struct dmatest_chan *dtc) list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { ret = kthread_stop(thread->task); - pr_debug("dmatest: thread %s exited with status %d\n", - thread->task->comm, ret); + pr_debug("thread %s exited with status %d\n", + thread->task->comm, ret); list_del(&thread->node); + put_task_struct(thread->task); kfree(thread); } @@ -861,27 +757,27 @@ static int dmatest_add_threads(struct dmatest_info *info, for (i = 0; i < params->threads_per_chan; i++) { thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); if (!thread) { - pr_warning("dmatest: No memory for %s-%s%u\n", - dma_chan_name(chan), op, i); - + pr_warn("No memory for %s-%s%u\n", + dma_chan_name(chan), op, i); break; } thread->info = info; thread->chan = dtc->chan; thread->type = type; smp_wmb(); - thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", + thread->task = kthread_create(dmatest_func, thread, "%s-%s%u", dma_chan_name(chan), op, i); if (IS_ERR(thread->task)) { - pr_warning("dmatest: Failed to run thread %s-%s%u\n", - dma_chan_name(chan), op, i); + pr_warn("Failed to create thread %s-%s%u\n", + dma_chan_name(chan), op, i); kfree(thread); break; } /* srcbuf and dstbuf are allocated by the thread itself */ - + get_task_struct(thread->task); list_add_tail(&thread->node, &dtc->threads); + wake_up_process(thread->task); } return i; @@ -897,7 +793,7 @@ static int dmatest_add_channel(struct dmatest_info *info, dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); if (!dtc) { - pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); + pr_warn("No memory for %s\n", dma_chan_name(chan)); return -ENOMEM; } @@ -917,7 +813,7 @@ static int dmatest_add_channel(struct dmatest_info *info, thread_count += cnt > 0 ? cnt : 0; } - pr_info("dmatest: Started %u threads using %s\n", + pr_info("Started %u threads using %s\n", thread_count, dma_chan_name(chan)); list_add_tail(&dtc->node, &info->channels); @@ -937,20 +833,20 @@ static bool filter(struct dma_chan *chan, void *param) return true; } -static int __run_threaded_test(struct dmatest_info *info) +static void request_channels(struct dmatest_info *info, + enum dma_transaction_type type) { dma_cap_mask_t mask; - struct dma_chan *chan; - struct dmatest_params *params = &info->params; - int err = 0; dma_cap_zero(mask); - dma_cap_set(DMA_MEMCPY, mask); + dma_cap_set(type, mask); for (;;) { + struct dmatest_params *params = &info->params; + struct dma_chan *chan; + chan = dma_request_channel(mask, filter, params); if (chan) { - err = dmatest_add_channel(info, chan); - if (err) { + if (dmatest_add_channel(info, chan)) { dma_release_channel(chan); break; /* add_channel failed, punt */ } @@ -960,22 +856,30 @@ static int __run_threaded_test(struct dmatest_info *info) info->nr_channels >= params->max_channels) break; /* we have all we need */ } - return err; } -#ifndef MODULE -static int run_threaded_test(struct dmatest_info *info) +static void run_threaded_test(struct dmatest_info *info) { - int ret; + struct dmatest_params *params = &info->params; - mutex_lock(&info->lock); - ret = __run_threaded_test(info); - mutex_unlock(&info->lock); - return ret; + /* Copy test parameters */ + params->buf_size = test_buf_size; + strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); + strlcpy(params->device, strim(test_device), sizeof(params->device)); + params->threads_per_chan = threads_per_chan; + params->max_channels = max_channels; + params->iterations = iterations; + params->xor_sources = xor_sources; + params->pq_sources = pq_sources; + params->timeout = timeout; + params->noverify = noverify; + + request_channels(info, DMA_MEMCPY); + request_channels(info, DMA_XOR); + request_channels(info, DMA_PQ); } -#endif -static void __stop_threaded_test(struct dmatest_info *info) +static void stop_threaded_test(struct dmatest_info *info) { struct dmatest_chan *dtc, *_dtc; struct dma_chan *chan; @@ -984,203 +888,86 @@ static void __stop_threaded_test(struct dmatest_info *info) list_del(&dtc->node); chan = dtc->chan; dmatest_cleanup_channel(dtc); - pr_debug("dmatest: dropped channel %s\n", dma_chan_name(chan)); + pr_debug("dropped channel %s\n", dma_chan_name(chan)); dma_release_channel(chan); } info->nr_channels = 0; } -static void stop_threaded_test(struct dmatest_info *info) +static void restart_threaded_test(struct dmatest_info *info, bool run) { - mutex_lock(&info->lock); - __stop_threaded_test(info); - mutex_unlock(&info->lock); -} - -static int __restart_threaded_test(struct dmatest_info *info, bool run) -{ - struct dmatest_params *params = &info->params; + /* we might be called early to set run=, defer running until all + * parameters have been evaluated + */ + if (!info->did_init) + return; /* Stop any running test first */ - __stop_threaded_test(info); - - if (run == false) - return 0; - - /* Clear results from previous run */ - result_free(info, NULL); - - /* Copy test parameters */ - params->buf_size = test_buf_size; - strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); - strlcpy(params->device, strim(test_device), sizeof(params->device)); - params->threads_per_chan = threads_per_chan; - params->max_channels = max_channels; - params->iterations = iterations; - params->xor_sources = xor_sources; - params->pq_sources = pq_sources; - params->timeout = timeout; + stop_threaded_test(info); /* Run test with new parameters */ - return __run_threaded_test(info); -} - -static bool __is_threaded_test_run(struct dmatest_info *info) -{ - struct dmatest_chan *dtc; - - list_for_each_entry(dtc, &info->channels, node) { - struct dmatest_thread *thread; - - list_for_each_entry(thread, &dtc->threads, node) { - if (!thread->done) - return true; - } - } - - return false; + run_threaded_test(info); } -static ssize_t dtf_read_run(struct file *file, char __user *user_buf, - size_t count, loff_t *ppos) +static int dmatest_run_get(char *val, const struct kernel_param *kp) { - struct dmatest_info *info = file->private_data; - char buf[3]; + struct dmatest_info *info = &test_info; mutex_lock(&info->lock); - - if (__is_threaded_test_run(info)) { - buf[0] = 'Y'; + if (is_threaded_test_run(info)) { + dmatest_run = true; } else { - __stop_threaded_test(info); - buf[0] = 'N'; + stop_threaded_test(info); + dmatest_run = false; } - mutex_unlock(&info->lock); - buf[1] = '\n'; - buf[2] = 0x00; - return simple_read_from_buffer(user_buf, count, ppos, buf, 2); -} - -static ssize_t dtf_write_run(struct file *file, const char __user *user_buf, - size_t count, loff_t *ppos) -{ - struct dmatest_info *info = file->private_data; - char buf[16]; - bool bv; - int ret = 0; - if (copy_from_user(buf, user_buf, min(count, (sizeof(buf) - 1)))) - return -EFAULT; - - if (strtobool(buf, &bv) == 0) { - mutex_lock(&info->lock); - - if (__is_threaded_test_run(info)) - ret = -EBUSY; - else - ret = __restart_threaded_test(info, bv); - - mutex_unlock(&info->lock); - } - - return ret ? ret : count; + return param_get_bool(val, kp); } -static const struct file_operations dtf_run_fops = { - .read = dtf_read_run, - .write = dtf_write_run, - .open = simple_open, - .llseek = default_llseek, -}; - -static int dtf_results_show(struct seq_file *sf, void *data) +static int dmatest_run_set(const char *val, const struct kernel_param *kp) { - struct dmatest_info *info = sf->private; - struct dmatest_result *result; - struct dmatest_thread_result *tr; - unsigned int i; + struct dmatest_info *info = &test_info; + int ret; - mutex_lock(&info->results_lock); - list_for_each_entry(result, &info->results, node) { - list_for_each_entry(tr, &result->results, node) { - seq_printf(sf, "%s\n", - thread_result_get(result->name, tr)); - if (tr->type == DMATEST_ET_VERIFY_BUF) { - for (i = 0; i < tr->vr->error_count; i++) { - seq_printf(sf, "\t%s\n", - verify_result_get_one(tr->vr, i)); - } - } - } + mutex_lock(&info->lock); + ret = param_set_bool(val, kp); + if (ret) { + mutex_unlock(&info->lock); + return ret; } - mutex_unlock(&info->results_lock); - return 0; -} - -static int dtf_results_open(struct inode *inode, struct file *file) -{ - return single_open(file, dtf_results_show, inode->i_private); -} - -static const struct file_operations dtf_results_fops = { - .open = dtf_results_open, - .read = seq_read, - .llseek = seq_lseek, - .release = single_release, -}; - -static int dmatest_register_dbgfs(struct dmatest_info *info) -{ - struct dentry *d; - - d = debugfs_create_dir("dmatest", NULL); - if (IS_ERR(d)) - return PTR_ERR(d); - if (!d) - goto err_root; + if (is_threaded_test_run(info)) + ret = -EBUSY; + else if (dmatest_run) + restart_threaded_test(info, dmatest_run); - info->root = d; - - /* Run or stop threaded test */ - debugfs_create_file("run", S_IWUSR | S_IRUGO, info->root, info, - &dtf_run_fops); - - /* Results of test in progress */ - debugfs_create_file("results", S_IRUGO, info->root, info, - &dtf_results_fops); - - return 0; + mutex_unlock(&info->lock); -err_root: - pr_err("dmatest: Failed to initialize debugfs\n"); - return -ENOMEM; + return ret; } static int __init dmatest_init(void) { struct dmatest_info *info = &test_info; - int ret; - - memset(info, 0, sizeof(*info)); + struct dmatest_params *params = &info->params; - mutex_init(&info->lock); - INIT_LIST_HEAD(&info->channels); + if (dmatest_run) { + mutex_lock(&info->lock); + run_threaded_test(info); + mutex_unlock(&info->lock); + } - mutex_init(&info->results_lock); - INIT_LIST_HEAD(&info->results); + if (params->iterations && wait) + wait_event(thread_wait, !is_threaded_test_run(info)); - ret = dmatest_register_dbgfs(info); - if (ret) - return ret; + /* module parameters are stable, inittime tests are started, + * let userspace take over 'run' control + */ + info->did_init = true; -#ifdef MODULE return 0; -#else - return run_threaded_test(info); -#endif } /* when compiled-in wait for drivers to load first */ late_initcall(dmatest_init); @@ -1189,9 +976,9 @@ static void __exit dmatest_exit(void) { struct dmatest_info *info = &test_info; - debugfs_remove_recursive(info->root); + mutex_lock(&info->lock); stop_threaded_test(info); - result_free(info, NULL); + mutex_unlock(&info->lock); } module_exit(dmatest_exit); diff --git a/drivers/dma/dw/core.c b/drivers/dma/dw/core.c index 89eb89f2228..a27ded53ab4 100644 --- a/drivers/dma/dw/core.c +++ b/drivers/dma/dw/core.c @@ -33,8 +33,8 @@ * of which use ARM any more). See the "Databook" from Synopsys for * information beyond what licensees probably provide. * - * The driver has currently been tested only with the Atmel AT32AP7000, - * which does not support descriptor writeback. + * The driver has been tested with the Atmel AT32AP7000, which does not + * support descriptor writeback. */ static inline bool is_request_line_unset(struct dw_dma_chan *dwc) @@ -85,10 +85,6 @@ static struct device *chan2dev(struct dma_chan *chan) { return &chan->dev->device; } -static struct device *chan2parent(struct dma_chan *chan) -{ - return chan->dev->device.parent; -} static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc) { @@ -222,8 +218,10 @@ static inline void dwc_do_single_block(struct dw_dma_chan *dwc, struct dw_dma *dw = to_dw_dma(dwc->chan.device); u32 ctllo; - /* Software emulation of LLP mode relies on interrupts to continue - * multi block transfer. */ + /* + * Software emulation of LLP mode relies on interrupts to continue + * multi block transfer. + */ ctllo = desc->lli.ctllo | DWC_CTLL_INT_EN; channel_writel(dwc, SAR, desc->lli.sar); @@ -257,8 +255,7 @@ static void dwc_dostart(struct dw_dma_chan *dwc, struct dw_desc *first) &dwc->flags); if (was_soft_llp) { dev_err(chan2dev(&dwc->chan), - "BUG: Attempted to start new LLP transfer " - "inside ongoing one\n"); + "BUG: Attempted to start new LLP transfer inside ongoing one\n"); return; } @@ -311,26 +308,7 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc, list_splice_init(&desc->tx_list, &dwc->free_list); list_move(&desc->desc_node, &dwc->free_list); - if (!is_slave_direction(dwc->direction)) { - struct device *parent = chan2parent(&dwc->chan); - if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) - dma_unmap_single(parent, desc->lli.dar, - desc->total_len, DMA_FROM_DEVICE); - else - dma_unmap_page(parent, desc->lli.dar, - desc->total_len, DMA_FROM_DEVICE); - } - if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) - dma_unmap_single(parent, desc->lli.sar, - desc->total_len, DMA_TO_DEVICE); - else - dma_unmap_page(parent, desc->lli.sar, - desc->total_len, DMA_TO_DEVICE); - } - } - + dma_descriptor_unmap(txd); spin_unlock_irqrestore(&dwc->lock, flags); if (callback) @@ -443,8 +421,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc) return; } - dev_vdbg(chan2dev(&dwc->chan), "%s: llp=0x%llx\n", __func__, - (unsigned long long)llp); + dev_vdbg(chan2dev(&dwc->chan), "%s: llp=%pad\n", __func__, &llp); list_for_each_entry_safe(desc, _desc, &dwc->active_list, desc_node) { /* Initial residue value */ @@ -590,9 +567,9 @@ static void dwc_handle_cyclic(struct dw_dma *dw, struct dw_dma_chan *dwc, unlikely(status_xfer & dwc->mask)) { int i; - dev_err(chan2dev(&dwc->chan), "cyclic DMA unexpected %s " - "interrupt, stopping DMA transfer\n", - status_xfer ? "xfer" : "error"); + dev_err(chan2dev(&dwc->chan), + "cyclic DMA unexpected %s interrupt, stopping DMA transfer\n", + status_xfer ? "xfer" : "error"); spin_lock_irqsave(&dwc->lock, flags); @@ -734,9 +711,8 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, u32 ctllo; dev_vdbg(chan2dev(chan), - "%s: d0x%llx s0x%llx l0x%zx f0x%lx\n", __func__, - (unsigned long long)dest, (unsigned long long)src, - len, flags); + "%s: d%pad s%pad l0x%zx f0x%lx\n", __func__, + &dest, &src, len, flags); if (unlikely(!len)) { dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__); @@ -1098,13 +1074,13 @@ dwc_tx_status(struct dma_chan *chan, enum dma_status ret; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; dwc_scan_descriptors(to_dw_dma(chan->device), dwc); ret = dma_cookie_status(chan, cookie, txstate); - if (ret != DMA_SUCCESS) + if (ret != DMA_COMPLETE) dma_set_residue(txstate, dwc_get_residue(dwc)); if (dwc->paused && ret == DMA_IN_PROGRESS) @@ -1424,9 +1400,9 @@ struct dw_cyclic_desc *dw_dma_cyclic_prep(struct dma_chan *chan, /* Let's make a cyclic list */ last->lli.llp = cdesc->desc[0]->txd.phys; - dev_dbg(chan2dev(&dwc->chan), "cyclic prepared buf 0x%llx len %zu " - "period %zu periods %d\n", (unsigned long long)buf_addr, - buf_len, period_len, periods); + dev_dbg(chan2dev(&dwc->chan), + "cyclic prepared buf %pad len %zu period %zu periods %d\n", + &buf_addr, buf_len, period_len, periods); cdesc->periods = periods; dwc->cdesc = cdesc; @@ -1503,7 +1479,6 @@ static void dw_dma_off(struct dw_dma *dw) int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) { struct dw_dma *dw; - size_t size; bool autocfg; unsigned int dw_params; unsigned int nr_channels; @@ -1511,6 +1486,20 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) int err; int i; + dw = devm_kzalloc(chip->dev, sizeof(*dw), GFP_KERNEL); + if (!dw) + return -ENOMEM; + + dw->regs = chip->regs; + chip->dw = dw; + + dw->clk = devm_clk_get(chip->dev, "hclk"); + if (IS_ERR(dw->clk)) + return PTR_ERR(dw->clk); + err = clk_prepare_enable(dw->clk); + if (err) + return err; + dw_params = dma_read_byaddr(chip->regs, DW_PARAMS); autocfg = dw_params >> DW_PARAMS_EN & 0x1; @@ -1518,33 +1507,31 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) if (!pdata && autocfg) { pdata = devm_kzalloc(chip->dev, sizeof(*pdata), GFP_KERNEL); - if (!pdata) - return -ENOMEM; + if (!pdata) { + err = -ENOMEM; + goto err_pdata; + } /* Fill platform data with the default values */ pdata->is_private = true; pdata->chan_allocation_order = CHAN_ALLOCATION_ASCENDING; pdata->chan_priority = CHAN_PRIORITY_ASCENDING; - } else if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) - return -EINVAL; + } else if (!pdata || pdata->nr_channels > DW_DMA_MAX_NR_CHANNELS) { + err = -EINVAL; + goto err_pdata; + } if (autocfg) nr_channels = (dw_params >> DW_PARAMS_NR_CHAN & 0x7) + 1; else nr_channels = pdata->nr_channels; - size = sizeof(struct dw_dma) + nr_channels * sizeof(struct dw_dma_chan); - dw = devm_kzalloc(chip->dev, size, GFP_KERNEL); - if (!dw) - return -ENOMEM; - - dw->clk = devm_clk_get(chip->dev, "hclk"); - if (IS_ERR(dw->clk)) - return PTR_ERR(dw->clk); - clk_prepare_enable(dw->clk); - - dw->regs = chip->regs; - chip->dw = dw; + dw->chan = devm_kcalloc(chip->dev, nr_channels, sizeof(*dw->chan), + GFP_KERNEL); + if (!dw->chan) { + err = -ENOMEM; + goto err_pdata; + } /* Get hardware configuration parameters */ if (autocfg) { @@ -1569,21 +1556,22 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) /* Disable BLOCK interrupts as well */ channel_clear_bit(dw, MASK.BLOCK, dw->all_chan_mask); - err = devm_request_irq(chip->dev, chip->irq, dw_dma_interrupt, - IRQF_SHARED, "dw_dmac", dw); - if (err) - return err; - /* Create a pool of consistent memory blocks for hardware descriptors */ dw->desc_pool = dmam_pool_create("dw_dmac_desc_pool", chip->dev, sizeof(struct dw_desc), 4, 0); if (!dw->desc_pool) { dev_err(chip->dev, "No memory for descriptors dma pool\n"); - return -ENOMEM; + err = -ENOMEM; + goto err_pdata; } tasklet_init(&dw->tasklet, dw_dma_tasklet, (unsigned long)dw); + err = request_irq(chip->irq, dw_dma_interrupt, IRQF_SHARED, + "dw_dmac", dw); + if (err) + goto err_pdata; + INIT_LIST_HEAD(&dw->dma.channels); for (i = 0; i < nr_channels; i++) { struct dw_dma_chan *dwc = &dw->chan[i]; @@ -1626,9 +1614,11 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) dev_dbg(chip->dev, "DWC_PARAMS[%d]: 0x%08x\n", i, dwc_params); - /* Decode maximum block size for given channel. The + /* + * Decode maximum block size for given channel. The * stored 4 bit value represents blocks from 0x00 for 3 - * up to 0x0a for 4095. */ + * up to 0x0a for 4095. + */ dwc->block_size = (4 << ((max_blk_size >> 4 * i) & 0xf)) - 1; dwc->nollp = @@ -1669,12 +1659,20 @@ int dw_dma_probe(struct dw_dma_chip *chip, struct dw_dma_platform_data *pdata) dma_writel(dw, CFG, DW_CFG_DMA_EN); + err = dma_async_device_register(&dw->dma); + if (err) + goto err_dma_register; + dev_info(chip->dev, "DesignWare DMA Controller, %d channels\n", nr_channels); - dma_async_device_register(&dw->dma); - return 0; + +err_dma_register: + free_irq(chip->irq, dw); +err_pdata: + clk_disable_unprepare(dw->clk); + return err; } EXPORT_SYMBOL_GPL(dw_dma_probe); @@ -1686,6 +1684,7 @@ int dw_dma_remove(struct dw_dma_chip *chip) dw_dma_off(dw); dma_async_device_unregister(&dw->dma); + free_irq(chip->irq, dw); tasklet_kill(&dw->tasklet); list_for_each_entry_safe(dwc, _dwc, &dw->dma.channels, @@ -1694,6 +1693,8 @@ int dw_dma_remove(struct dw_dma_chip *chip) channel_clear_bit(dw, CH_EN, dwc->mask); } + clk_disable_unprepare(dw->clk); + return 0; } EXPORT_SYMBOL_GPL(dw_dma_remove); diff --git a/drivers/dma/dw/pci.c b/drivers/dma/dw/pci.c index e89fc24b829..39e30c3c7a9 100644 --- a/drivers/dma/dw/pci.c +++ b/drivers/dma/dw/pci.c @@ -75,7 +75,31 @@ static void dw_pci_remove(struct pci_dev *pdev) dev_warn(&pdev->dev, "can't remove device properly: %d\n", ret); } -static DEFINE_PCI_DEVICE_TABLE(dw_pci_id_table) = { +#ifdef CONFIG_PM_SLEEP + +static int dw_pci_suspend_late(struct device *dev) +{ + struct pci_dev *pci = to_pci_dev(dev); + struct dw_dma_chip *chip = pci_get_drvdata(pci); + + return dw_dma_suspend(chip); +}; + +static int dw_pci_resume_early(struct device *dev) +{ + struct pci_dev *pci = to_pci_dev(dev); + struct dw_dma_chip *chip = pci_get_drvdata(pci); + + return dw_dma_resume(chip); +}; + +#endif /* CONFIG_PM_SLEEP */ + +static const struct dev_pm_ops dw_pci_dev_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_pci_suspend_late, dw_pci_resume_early) +}; + +static const struct pci_device_id dw_pci_id_table[] = { /* Medfield */ { PCI_VDEVICE(INTEL, 0x0827), (kernel_ulong_t)&dw_pci_pdata }, { PCI_VDEVICE(INTEL, 0x0830), (kernel_ulong_t)&dw_pci_pdata }, @@ -83,6 +107,9 @@ static DEFINE_PCI_DEVICE_TABLE(dw_pci_id_table) = { /* BayTrail */ { PCI_VDEVICE(INTEL, 0x0f06), (kernel_ulong_t)&dw_pci_pdata }, { PCI_VDEVICE(INTEL, 0x0f40), (kernel_ulong_t)&dw_pci_pdata }, + + /* Haswell */ + { PCI_VDEVICE(INTEL, 0x9c60), (kernel_ulong_t)&dw_pci_pdata }, { } }; MODULE_DEVICE_TABLE(pci, dw_pci_id_table); @@ -92,6 +119,9 @@ static struct pci_driver dw_pci_driver = { .id_table = dw_pci_id_table, .probe = dw_pci_probe, .remove = dw_pci_remove, + .driver = { + .pm = &dw_pci_dev_pm_ops, + }, }; module_pci_driver(dw_pci_driver); diff --git a/drivers/dma/dw/platform.c b/drivers/dma/dw/platform.c index e35d9759031..c5b339af6be 100644 --- a/drivers/dma/dw/platform.c +++ b/drivers/dma/dw/platform.c @@ -191,11 +191,9 @@ static int dw_probe(struct platform_device *pdev) if (IS_ERR(chip->regs)) return PTR_ERR(chip->regs); - /* Apply default dma_mask if needed */ - if (!dev->dma_mask) { - dev->dma_mask = &dev->coherent_dma_mask; - dev->coherent_dma_mask = DMA_BIT_MASK(32); - } + err = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (err) + return err; pdata = dev_get_platdata(dev); if (!pdata) @@ -258,7 +256,7 @@ MODULE_DEVICE_TABLE(acpi, dw_dma_acpi_id_table); #ifdef CONFIG_PM_SLEEP -static int dw_suspend_noirq(struct device *dev) +static int dw_suspend_late(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct dw_dma_chip *chip = platform_get_drvdata(pdev); @@ -266,7 +264,7 @@ static int dw_suspend_noirq(struct device *dev) return dw_dma_suspend(chip); } -static int dw_resume_noirq(struct device *dev) +static int dw_resume_early(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct dw_dma_chip *chip = platform_get_drvdata(pdev); @@ -274,20 +272,10 @@ static int dw_resume_noirq(struct device *dev) return dw_dma_resume(chip); } -#else /* !CONFIG_PM_SLEEP */ - -#define dw_suspend_noirq NULL -#define dw_resume_noirq NULL - -#endif /* !CONFIG_PM_SLEEP */ +#endif /* CONFIG_PM_SLEEP */ static const struct dev_pm_ops dw_dev_pm_ops = { - .suspend_noirq = dw_suspend_noirq, - .resume_noirq = dw_resume_noirq, - .freeze_noirq = dw_suspend_noirq, - .thaw_noirq = dw_resume_noirq, - .restore_noirq = dw_resume_noirq, - .poweroff_noirq = dw_suspend_noirq, + SET_LATE_SYSTEM_SLEEP_PM_OPS(dw_suspend_late, dw_resume_early) }; static struct platform_driver dw_driver = { diff --git a/drivers/dma/dw/regs.h b/drivers/dma/dw/regs.h index deb4274f80f..bb98d3e91e8 100644 --- a/drivers/dma/dw/regs.h +++ b/drivers/dma/dw/regs.h @@ -252,13 +252,13 @@ struct dw_dma { struct tasklet_struct tasklet; struct clk *clk; + /* channels */ + struct dw_dma_chan *chan; u8 all_chan_mask; /* hardware configuration */ unsigned char nr_masters; unsigned char data_width[4]; - - struct dw_dma_chan chan[0]; }; static inline struct dw_dma_regs __iomem *__dw_regs(struct dw_dma *dw) diff --git a/drivers/dma/edma.c b/drivers/dma/edma.c index ff50ff4c6a5..d08c4dedef3 100644 --- a/drivers/dma/edma.c +++ b/drivers/dma/edma.c @@ -46,18 +46,59 @@ #define EDMA_CHANS 64 #endif /* CONFIG_ARCH_DAVINCI_DA8XX */ -/* Max of 16 segments per channel to conserve PaRAM slots */ -#define MAX_NR_SG 16 +/* + * Max of 20 segments per channel to conserve PaRAM slots + * Also note that MAX_NR_SG should be atleast the no.of periods + * that are required for ASoC, otherwise DMA prep calls will + * fail. Today davinci-pcm is the only user of this driver and + * requires atleast 17 slots, so we setup the default to 20. + */ +#define MAX_NR_SG 20 #define EDMA_MAX_SLOTS MAX_NR_SG #define EDMA_DESCRIPTORS 16 +struct edma_pset { + u32 len; + dma_addr_t addr; + struct edmacc_param param; +}; + struct edma_desc { struct virt_dma_desc vdesc; struct list_head node; + enum dma_transfer_direction direction; + int cyclic; int absync; int pset_nr; + struct edma_chan *echan; int processed; - struct edmacc_param pset[0]; + + /* + * The following 4 elements are used for residue accounting. + * + * - processed_stat: the number of SG elements we have traversed + * so far to cover accounting. This is updated directly to processed + * during edma_callback and is always <= processed, because processed + * refers to the number of pending transfer (programmed to EDMA + * controller), where as processed_stat tracks number of transfers + * accounted for so far. + * + * - residue: The amount of bytes we have left to transfer for this desc + * + * - residue_stat: The residue in bytes of data we have covered + * so far for accounting. This is updated directly to residue + * during callbacks to keep it current. + * + * - sg_len: Tracks the length of the current intermediate transfer, + * this is required to update the residue during intermediate transfer + * completion callback. + */ + int processed_stat; + u32 sg_len; + u32 residue; + u32 residue_stat; + + struct edma_pset pset[0]; }; struct edma_cc; @@ -129,12 +170,14 @@ static void edma_execute(struct edma_chan *echan) /* Find out how many left */ left = edesc->pset_nr - edesc->processed; nslots = min(MAX_NR_SG, left); + edesc->sg_len = 0; /* Write descriptor PaRAM set(s) */ for (i = 0; i < nslots; i++) { j = i + edesc->processed; - edma_write_slot(echan->slot[i], &edesc->pset[j]); - dev_dbg(echan->vchan.chan.device->dev, + edma_write_slot(echan->slot[i], &edesc->pset[j].param); + edesc->sg_len += edesc->pset[j].len; + dev_vdbg(echan->vchan.chan.device->dev, "\n pset[%d]:\n" " chnum\t%d\n" " slot\t%d\n" @@ -147,14 +190,14 @@ static void edma_execute(struct edma_chan *echan) " cidx\t%08x\n" " lkrld\t%08x\n", j, echan->ch_num, echan->slot[i], - edesc->pset[j].opt, - edesc->pset[j].src, - edesc->pset[j].dst, - edesc->pset[j].a_b_cnt, - edesc->pset[j].ccnt, - edesc->pset[j].src_dst_bidx, - edesc->pset[j].src_dst_cidx, - edesc->pset[j].link_bcntrld); + edesc->pset[j].param.opt, + edesc->pset[j].param.src, + edesc->pset[j].param.dst, + edesc->pset[j].param.a_b_cnt, + edesc->pset[j].param.ccnt, + edesc->pset[j].param.src_dst_bidx, + edesc->pset[j].param.src_dst_cidx, + edesc->pset[j].param.link_bcntrld); /* Link to the previous slot if not the last set */ if (i != (nslots - 1)) edma_link(echan->slot[i], echan->slot[i+1]); @@ -167,14 +210,22 @@ static void edma_execute(struct edma_chan *echan) * then setup a link to the dummy slot, this results in all future * events being absorbed and that's OK because we're done */ - if (edesc->processed == edesc->pset_nr) - edma_link(echan->slot[nslots-1], echan->ecc->dummy_slot); - - edma_resume(echan->ch_num); + if (edesc->processed == edesc->pset_nr) { + if (edesc->cyclic) + edma_link(echan->slot[nslots-1], echan->slot[1]); + else + edma_link(echan->slot[nslots-1], + echan->ecc->dummy_slot); + } if (edesc->processed <= MAX_NR_SG) { - dev_dbg(dev, "first transfer starting %d\n", echan->ch_num); + dev_dbg(dev, "first transfer starting on channel %d\n", + echan->ch_num); edma_start(echan->ch_num); + } else { + dev_dbg(dev, "chan: %d: completed %d elements, resuming\n", + echan->ch_num, edesc->processed); + edma_resume(echan->ch_num); } /* @@ -183,7 +234,7 @@ static void edma_execute(struct edma_chan *echan) * MAX_NR_SG */ if (echan->missed) { - dev_dbg(dev, "missed event in execute detected\n"); + dev_dbg(dev, "missed event on channel %d\n", echan->ch_num); edma_clean_channel(echan->ch_num); edma_stop(echan->ch_num); edma_start(echan->ch_num); @@ -228,6 +279,26 @@ static int edma_slave_config(struct edma_chan *echan, return 0; } +static int edma_dma_pause(struct edma_chan *echan) +{ + /* Pause/Resume only allowed with cyclic mode */ + if (!echan->edesc->cyclic) + return -EINVAL; + + edma_pause(echan->ch_num); + return 0; +} + +static int edma_dma_resume(struct edma_chan *echan) +{ + /* Pause/Resume only allowed with cyclic mode */ + if (!echan->edesc->cyclic) + return -EINVAL; + + edma_resume(echan->ch_num); + return 0; +} + static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg) { @@ -243,6 +314,14 @@ static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, config = (struct dma_slave_config *)arg; ret = edma_slave_config(echan, config); break; + case DMA_PAUSE: + ret = edma_dma_pause(echan); + break; + + case DMA_RESUME: + ret = edma_dma_resume(echan); + break; + default: ret = -ENOSYS; } @@ -250,6 +329,131 @@ static int edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, return ret; } +/* + * A PaRAM set configuration abstraction used by other modes + * @chan: Channel who's PaRAM set we're configuring + * @pset: PaRAM set to initialize and setup. + * @src_addr: Source address of the DMA + * @dst_addr: Destination address of the DMA + * @burst: In units of dev_width, how much to send + * @dev_width: How much is the dev_width + * @dma_length: Total length of the DMA transfer + * @direction: Direction of the transfer + */ +static int edma_config_pset(struct dma_chan *chan, struct edma_pset *epset, + dma_addr_t src_addr, dma_addr_t dst_addr, u32 burst, + enum dma_slave_buswidth dev_width, unsigned int dma_length, + enum dma_transfer_direction direction) +{ + struct edma_chan *echan = to_edma_chan(chan); + struct device *dev = chan->device->dev; + struct edmacc_param *param = &epset->param; + int acnt, bcnt, ccnt, cidx; + int src_bidx, dst_bidx, src_cidx, dst_cidx; + int absync; + + acnt = dev_width; + + /* src/dst_maxburst == 0 is the same case as src/dst_maxburst == 1 */ + if (!burst) + burst = 1; + /* + * If the maxburst is equal to the fifo width, use + * A-synced transfers. This allows for large contiguous + * buffer transfers using only one PaRAM set. + */ + if (burst == 1) { + /* + * For the A-sync case, bcnt and ccnt are the remainder + * and quotient respectively of the division of: + * (dma_length / acnt) by (SZ_64K -1). This is so + * that in case bcnt over flows, we have ccnt to use. + * Note: In A-sync tranfer only, bcntrld is used, but it + * only applies for sg_dma_len(sg) >= SZ_64K. + * In this case, the best way adopted is- bccnt for the + * first frame will be the remainder below. Then for + * every successive frame, bcnt will be SZ_64K-1. This + * is assured as bcntrld = 0xffff in end of function. + */ + absync = false; + ccnt = dma_length / acnt / (SZ_64K - 1); + bcnt = dma_length / acnt - ccnt * (SZ_64K - 1); + /* + * If bcnt is non-zero, we have a remainder and hence an + * extra frame to transfer, so increment ccnt. + */ + if (bcnt) + ccnt++; + else + bcnt = SZ_64K - 1; + cidx = acnt; + } else { + /* + * If maxburst is greater than the fifo address_width, + * use AB-synced transfers where A count is the fifo + * address_width and B count is the maxburst. In this + * case, we are limited to transfers of C count frames + * of (address_width * maxburst) where C count is limited + * to SZ_64K-1. This places an upper bound on the length + * of an SG segment that can be handled. + */ + absync = true; + bcnt = burst; + ccnt = dma_length / (acnt * bcnt); + if (ccnt > (SZ_64K - 1)) { + dev_err(dev, "Exceeded max SG segment size\n"); + return -EINVAL; + } + cidx = acnt * bcnt; + } + + epset->len = dma_length; + + if (direction == DMA_MEM_TO_DEV) { + src_bidx = acnt; + src_cidx = cidx; + dst_bidx = 0; + dst_cidx = 0; + epset->addr = src_addr; + } else if (direction == DMA_DEV_TO_MEM) { + src_bidx = 0; + src_cidx = 0; + dst_bidx = acnt; + dst_cidx = cidx; + epset->addr = dst_addr; + } else if (direction == DMA_MEM_TO_MEM) { + src_bidx = acnt; + src_cidx = cidx; + dst_bidx = acnt; + dst_cidx = cidx; + } else { + dev_err(dev, "%s: direction not implemented yet\n", __func__); + return -EINVAL; + } + + param->opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num)); + /* Configure A or AB synchronized transfers */ + if (absync) + param->opt |= SYNCDIM; + + param->src = src_addr; + param->dst = dst_addr; + + param->src_dst_bidx = (dst_bidx << 16) | src_bidx; + param->src_dst_cidx = (dst_cidx << 16) | src_cidx; + + param->a_b_cnt = bcnt << 16 | acnt; + param->ccnt = ccnt; + /* + * Only time when (bcntrld) auto reload is required is for + * A-sync case, and in this case, a requirement of reload value + * of SZ_64K-1 only is assured. 'link' is initially set to NULL + * and then later will be populated by edma_execute. + */ + param->link_bcntrld = 0xffffffff; + return absync; +} + static struct dma_async_tx_descriptor *edma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, @@ -258,43 +462,44 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( struct edma_chan *echan = to_edma_chan(chan); struct device *dev = chan->device->dev; struct edma_desc *edesc; - dma_addr_t dev_addr; + dma_addr_t src_addr = 0, dst_addr = 0; enum dma_slave_buswidth dev_width; u32 burst; struct scatterlist *sg; - int acnt, bcnt, ccnt, src, dst, cidx; - int src_bidx, dst_bidx, src_cidx, dst_cidx; - int i, nslots; + int i, nslots, ret; if (unlikely(!echan || !sgl || !sg_len)) return NULL; if (direction == DMA_DEV_TO_MEM) { - dev_addr = echan->cfg.src_addr; + src_addr = echan->cfg.src_addr; dev_width = echan->cfg.src_addr_width; burst = echan->cfg.src_maxburst; } else if (direction == DMA_MEM_TO_DEV) { - dev_addr = echan->cfg.dst_addr; + dst_addr = echan->cfg.dst_addr; dev_width = echan->cfg.dst_addr_width; burst = echan->cfg.dst_maxburst; } else { - dev_err(dev, "%s: bad direction?\n", __func__); + dev_err(dev, "%s: bad direction: %d\n", __func__, direction); return NULL; } if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) { - dev_err(dev, "Undefined slave buswidth\n"); + dev_err(dev, "%s: Undefined slave buswidth\n", __func__); return NULL; } edesc = kzalloc(sizeof(*edesc) + sg_len * sizeof(edesc->pset[0]), GFP_ATOMIC); if (!edesc) { - dev_dbg(dev, "Failed to allocate a descriptor\n"); + dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__); return NULL; } edesc->pset_nr = sg_len; + edesc->residue = 0; + edesc->direction = direction; + edesc->echan = echan; /* Allocate a PaRAM slot, if needed */ nslots = min_t(unsigned, MAX_NR_SG, sg_len); @@ -305,7 +510,9 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( edma_alloc_slot(EDMA_CTLR(echan->ch_num), EDMA_SLOT_ANY); if (echan->slot[i] < 0) { - dev_err(dev, "Failed to allocate slot\n"); + kfree(edesc); + dev_err(dev, "%s: Failed to allocate slot\n", + __func__); return NULL; } } @@ -313,83 +520,208 @@ static struct dma_async_tx_descriptor *edma_prep_slave_sg( /* Configure PaRAM sets for each SG */ for_each_sg(sgl, sg, sg_len, i) { - - acnt = dev_width; - - /* - * If the maxburst is equal to the fifo width, use - * A-synced transfers. This allows for large contiguous - * buffer transfers using only one PaRAM set. - */ - if (burst == 1) { - edesc->absync = false; - ccnt = sg_dma_len(sg) / acnt / (SZ_64K - 1); - bcnt = sg_dma_len(sg) / acnt - ccnt * (SZ_64K - 1); - if (bcnt) - ccnt++; - else - bcnt = SZ_64K - 1; - cidx = acnt; - /* - * If maxburst is greater than the fifo address_width, - * use AB-synced transfers where A count is the fifo - * address_width and B count is the maxburst. In this - * case, we are limited to transfers of C count frames - * of (address_width * maxburst) where C count is limited - * to SZ_64K-1. This places an upper bound on the length - * of an SG segment that can be handled. - */ - } else { - edesc->absync = true; - bcnt = burst; - ccnt = sg_dma_len(sg) / (acnt * bcnt); - if (ccnt > (SZ_64K - 1)) { - dev_err(dev, "Exceeded max SG segment size\n"); - return NULL; - } - cidx = acnt * bcnt; - } - - if (direction == DMA_MEM_TO_DEV) { - src = sg_dma_address(sg); - dst = dev_addr; - src_bidx = acnt; - src_cidx = cidx; - dst_bidx = 0; - dst_cidx = 0; - } else { - src = dev_addr; - dst = sg_dma_address(sg); - src_bidx = 0; - src_cidx = 0; - dst_bidx = acnt; - dst_cidx = cidx; + /* Get address for each SG */ + if (direction == DMA_DEV_TO_MEM) + dst_addr = sg_dma_address(sg); + else + src_addr = sg_dma_address(sg); + + ret = edma_config_pset(chan, &edesc->pset[i], src_addr, + dst_addr, burst, dev_width, + sg_dma_len(sg), direction); + if (ret < 0) { + kfree(edesc); + return NULL; } - edesc->pset[i].opt = EDMA_TCC(EDMA_CHAN_SLOT(echan->ch_num)); - /* Configure A or AB synchronized transfers */ - if (edesc->absync) - edesc->pset[i].opt |= SYNCDIM; + edesc->absync = ret; + edesc->residue += sg_dma_len(sg); /* If this is the last in a current SG set of transactions, enable interrupts so that next set is processed */ if (!((i+1) % MAX_NR_SG)) - edesc->pset[i].opt |= TCINTEN; + edesc->pset[i].param.opt |= TCINTEN; /* If this is the last set, enable completion interrupt flag */ if (i == sg_len - 1) - edesc->pset[i].opt |= TCINTEN; + edesc->pset[i].param.opt |= TCINTEN; + } + edesc->residue_stat = edesc->residue; + + return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); +} + +struct dma_async_tx_descriptor *edma_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long tx_flags) +{ + int ret; + struct edma_desc *edesc; + struct device *dev = chan->device->dev; + struct edma_chan *echan = to_edma_chan(chan); + + if (unlikely(!echan || !len)) + return NULL; + + edesc = kzalloc(sizeof(*edesc) + sizeof(edesc->pset[0]), GFP_ATOMIC); + if (!edesc) { + dev_dbg(dev, "Failed to allocate a descriptor\n"); + return NULL; + } + + edesc->pset_nr = 1; + + ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1, + DMA_SLAVE_BUSWIDTH_4_BYTES, len, DMA_MEM_TO_MEM); + if (ret < 0) + return NULL; + + edesc->absync = ret; + + /* + * Enable intermediate transfer chaining to re-trigger channel + * on completion of every TR, and enable transfer-completion + * interrupt on completion of the whole transfer. + */ + edesc->pset[0].param.opt |= ITCCHEN; + edesc->pset[0].param.opt |= TCINTEN; + + return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); +} + +static struct dma_async_tx_descriptor *edma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long tx_flags, void *context) +{ + struct edma_chan *echan = to_edma_chan(chan); + struct device *dev = chan->device->dev; + struct edma_desc *edesc; + dma_addr_t src_addr, dst_addr; + enum dma_slave_buswidth dev_width; + u32 burst; + int i, ret, nslots; + + if (unlikely(!echan || !buf_len || !period_len)) + return NULL; + + if (direction == DMA_DEV_TO_MEM) { + src_addr = echan->cfg.src_addr; + dst_addr = buf_addr; + dev_width = echan->cfg.src_addr_width; + burst = echan->cfg.src_maxburst; + } else if (direction == DMA_MEM_TO_DEV) { + src_addr = buf_addr; + dst_addr = echan->cfg.dst_addr; + dev_width = echan->cfg.dst_addr_width; + burst = echan->cfg.dst_maxburst; + } else { + dev_err(dev, "%s: bad direction: %d\n", __func__, direction); + return NULL; + } + + if (dev_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) { + dev_err(dev, "%s: Undefined slave buswidth\n", __func__); + return NULL; + } + + if (unlikely(buf_len % period_len)) { + dev_err(dev, "Period should be multiple of Buffer length\n"); + return NULL; + } + + nslots = (buf_len / period_len) + 1; + + /* + * Cyclic DMA users such as audio cannot tolerate delays introduced + * by cases where the number of periods is more than the maximum + * number of SGs the EDMA driver can handle at a time. For DMA types + * such as Slave SGs, such delays are tolerable and synchronized, + * but the synchronization is difficult to achieve with Cyclic and + * cannot be guaranteed, so we error out early. + */ + if (nslots > MAX_NR_SG) + return NULL; + + edesc = kzalloc(sizeof(*edesc) + nslots * + sizeof(edesc->pset[0]), GFP_ATOMIC); + if (!edesc) { + dev_err(dev, "%s: Failed to allocate a descriptor\n", __func__); + return NULL; + } + + edesc->cyclic = 1; + edesc->pset_nr = nslots; + edesc->residue = edesc->residue_stat = buf_len; + edesc->direction = direction; + edesc->echan = echan; + + dev_dbg(dev, "%s: channel=%d nslots=%d period_len=%zu buf_len=%zu\n", + __func__, echan->ch_num, nslots, period_len, buf_len); + + for (i = 0; i < nslots; i++) { + /* Allocate a PaRAM slot, if needed */ + if (echan->slot[i] < 0) { + echan->slot[i] = + edma_alloc_slot(EDMA_CTLR(echan->ch_num), + EDMA_SLOT_ANY); + if (echan->slot[i] < 0) { + kfree(edesc); + dev_err(dev, "%s: Failed to allocate slot\n", + __func__); + return NULL; + } + } - edesc->pset[i].src = src; - edesc->pset[i].dst = dst; + if (i == nslots - 1) { + memcpy(&edesc->pset[i], &edesc->pset[0], + sizeof(edesc->pset[0])); + break; + } - edesc->pset[i].src_dst_bidx = (dst_bidx << 16) | src_bidx; - edesc->pset[i].src_dst_cidx = (dst_cidx << 16) | src_cidx; + ret = edma_config_pset(chan, &edesc->pset[i], src_addr, + dst_addr, burst, dev_width, period_len, + direction); + if (ret < 0) { + kfree(edesc); + return NULL; + } - edesc->pset[i].a_b_cnt = bcnt << 16 | acnt; - edesc->pset[i].ccnt = ccnt; - edesc->pset[i].link_bcntrld = 0xffffffff; + if (direction == DMA_DEV_TO_MEM) + dst_addr += period_len; + else + src_addr += period_len; + dev_vdbg(dev, "%s: Configure period %d of buf:\n", __func__, i); + dev_vdbg(dev, + "\n pset[%d]:\n" + " chnum\t%d\n" + " slot\t%d\n" + " opt\t%08x\n" + " src\t%08x\n" + " dst\t%08x\n" + " abcnt\t%08x\n" + " ccnt\t%08x\n" + " bidx\t%08x\n" + " cidx\t%08x\n" + " lkrld\t%08x\n", + i, echan->ch_num, echan->slot[i], + edesc->pset[i].param.opt, + edesc->pset[i].param.src, + edesc->pset[i].param.dst, + edesc->pset[i].param.a_b_cnt, + edesc->pset[i].param.ccnt, + edesc->pset[i].param.src_dst_bidx, + edesc->pset[i].param.src_dst_cidx, + edesc->pset[i].param.link_bcntrld); + + edesc->absync = ret; + + /* + * Enable interrupts for every period because callback + * has to be called for every period. + */ + edesc->pset[i].param.opt |= TCINTEN; } return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags); @@ -400,34 +732,44 @@ static void edma_callback(unsigned ch_num, u16 ch_status, void *data) struct edma_chan *echan = data; struct device *dev = echan->vchan.chan.device->dev; struct edma_desc *edesc; - unsigned long flags; struct edmacc_param p; - /* Pause the channel */ - edma_pause(echan->ch_num); + edesc = echan->edesc; + + /* Pause the channel for non-cyclic */ + if (!edesc || (edesc && !edesc->cyclic)) + edma_pause(echan->ch_num); switch (ch_status) { - case DMA_COMPLETE: - spin_lock_irqsave(&echan->vchan.lock, flags); + case EDMA_DMA_COMPLETE: + spin_lock(&echan->vchan.lock); - edesc = echan->edesc; if (edesc) { - if (edesc->processed == edesc->pset_nr) { + if (edesc->cyclic) { + vchan_cyclic_callback(&edesc->vdesc); + } else if (edesc->processed == edesc->pset_nr) { dev_dbg(dev, "Transfer complete, stopping channel %d\n", ch_num); + edesc->residue = 0; edma_stop(echan->ch_num); vchan_cookie_complete(&edesc->vdesc); + edma_execute(echan); } else { dev_dbg(dev, "Intermediate transfer complete on channel %d\n", ch_num); - } - edma_execute(echan); + /* Update statistics for tx_status */ + edesc->residue -= edesc->sg_len; + edesc->residue_stat = edesc->residue; + edesc->processed_stat = edesc->processed; + + edma_execute(echan); + } } - spin_unlock_irqrestore(&echan->vchan.lock, flags); + spin_unlock(&echan->vchan.lock); break; - case DMA_CC_ERROR: - spin_lock_irqsave(&echan->vchan.lock, flags); + case EDMA_DMA_CC_ERROR: + spin_lock(&echan->vchan.lock); edma_read_slot(EDMA_CHAN_SLOT(echan->slot[0]), &p); @@ -458,7 +800,7 @@ static void edma_callback(unsigned ch_num, u16 ch_status, void *data) edma_trigger_channel(echan->ch_num); } - spin_unlock_irqrestore(&echan->vchan.lock, flags); + spin_unlock(&echan->vchan.lock); break; default: @@ -494,8 +836,8 @@ static int edma_alloc_chan_resources(struct dma_chan *chan) echan->alloced = true; echan->slot[0] = echan->ch_num; - dev_info(dev, "allocated channel for %u:%u\n", - EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num)); + dev_dbg(dev, "allocated channel %d for %u:%u\n", echan->ch_num, + EDMA_CTLR(echan->ch_num), EDMA_CHAN_SLOT(echan->ch_num)); return 0; @@ -531,7 +873,7 @@ static void edma_free_chan_resources(struct dma_chan *chan) echan->alloced = false; } - dev_info(dev, "freeing channel for %u\n", echan->ch_num); + dev_dbg(dev, "freeing channel for %u\n", echan->ch_num); } /* Send pending descriptor to hardware */ @@ -546,23 +888,52 @@ static void edma_issue_pending(struct dma_chan *chan) spin_unlock_irqrestore(&echan->vchan.lock, flags); } -static size_t edma_desc_size(struct edma_desc *edesc) +static u32 edma_residue(struct edma_desc *edesc) { + bool dst = edesc->direction == DMA_DEV_TO_MEM; + struct edma_pset *pset = edesc->pset; + dma_addr_t done, pos; int i; - size_t size; - - if (edesc->absync) - for (size = i = 0; i < edesc->pset_nr; i++) - size += (edesc->pset[i].a_b_cnt & 0xffff) * - (edesc->pset[i].a_b_cnt >> 16) * - edesc->pset[i].ccnt; - else - size = (edesc->pset[0].a_b_cnt & 0xffff) * - (edesc->pset[0].a_b_cnt >> 16) + - (edesc->pset[0].a_b_cnt & 0xffff) * - (SZ_64K - 1) * edesc->pset[0].ccnt; - - return size; + + /* + * We always read the dst/src position from the first RamPar + * pset. That's the one which is active now. + */ + pos = edma_get_position(edesc->echan->slot[0], dst); + + /* + * Cyclic is simple. Just subtract pset[0].addr from pos. + * + * We never update edesc->residue in the cyclic case, so we + * can tell the remaining room to the end of the circular + * buffer. + */ + if (edesc->cyclic) { + done = pos - pset->addr; + edesc->residue_stat = edesc->residue - done; + return edesc->residue_stat; + } + + /* + * For SG operation we catch up with the last processed + * status. + */ + pset += edesc->processed_stat; + + for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) { + /* + * If we are inside this pset address range, we know + * this is the active one. Get the current delta and + * stop walking the psets. + */ + if (pos >= pset->addr && pos < pset->addr + pset->len) + return edesc->residue_stat - (pos - pset->addr); + + /* Otherwise mark it done and update residue_stat. */ + edesc->processed_stat++; + edesc->residue_stat -= pset->len; + } + return edesc->residue_stat; } /* Check request completion status */ @@ -576,17 +947,14 @@ static enum dma_status edma_tx_status(struct dma_chan *chan, unsigned long flags; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS || !txstate) + if (ret == DMA_COMPLETE || !txstate) return ret; spin_lock_irqsave(&echan->vchan.lock, flags); - vdesc = vchan_find_desc(&echan->vchan, cookie); - if (vdesc) { - txstate->residue = edma_desc_size(to_edma_desc(&vdesc->tx)); - } else if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie) { - struct edma_desc *edesc = echan->edesc; - txstate->residue = edma_desc_size(edesc); - } + if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie) + txstate->residue = edma_residue(echan->edesc); + else if ((vdesc = vchan_find_desc(&echan->vchan, cookie))) + txstate->residue = to_edma_desc(&vdesc->tx)->residue; spin_unlock_irqrestore(&echan->vchan.lock, flags); return ret; @@ -612,17 +980,43 @@ static void __init edma_chan_init(struct edma_cc *ecc, } } +#define EDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) + +static int edma_dma_device_slave_caps(struct dma_chan *dchan, + struct dma_slave_caps *caps) +{ + caps->src_addr_widths = EDMA_DMA_BUSWIDTHS; + caps->dstn_addr_widths = EDMA_DMA_BUSWIDTHS; + caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + caps->cmd_pause = true; + caps->cmd_terminate = true; + caps->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; + + return 0; +} + static void edma_dma_init(struct edma_cc *ecc, struct dma_device *dma, struct device *dev) { dma->device_prep_slave_sg = edma_prep_slave_sg; + dma->device_prep_dma_cyclic = edma_prep_dma_cyclic; + dma->device_prep_dma_memcpy = edma_prep_dma_memcpy; dma->device_alloc_chan_resources = edma_alloc_chan_resources; dma->device_free_chan_resources = edma_free_chan_resources; dma->device_issue_pending = edma_issue_pending; dma->device_tx_status = edma_tx_status; dma->device_control = edma_control; + dma->device_slave_caps = edma_dma_device_slave_caps; dma->dev = dev; + /* + * code using dma memcpy must make sure alignment of + * length is at dma->copy_align boundary. + */ + dma->copy_align = DMA_SLAVE_BUSWIDTH_4_BYTES; + INIT_LIST_HEAD(&dma->channels); } @@ -631,6 +1025,10 @@ static int edma_probe(struct platform_device *pdev) struct edma_cc *ecc; int ret; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + ecc = devm_kzalloc(&pdev->dev, sizeof(*ecc), GFP_KERNEL); if (!ecc) { dev_err(&pdev->dev, "Can't allocate controller\n"); @@ -646,6 +1044,8 @@ static int edma_probe(struct platform_device *pdev) dma_cap_zero(ecc->dma_slave.cap_mask); dma_cap_set(DMA_SLAVE, ecc->dma_slave.cap_mask); + dma_cap_set(DMA_CYCLIC, ecc->dma_slave.cap_mask); + dma_cap_set(DMA_MEMCPY, ecc->dma_slave.cap_mask); edma_dma_init(ecc, &ecc->dma_slave, &pdev->dev); @@ -702,11 +1102,13 @@ static struct platform_device *pdev0, *pdev1; static const struct platform_device_info edma_dev_info0 = { .name = "edma-dma-engine", .id = 0, + .dma_mask = DMA_BIT_MASK(32), }; static const struct platform_device_info edma_dev_info1 = { .name = "edma-dma-engine", .id = 1, + .dma_mask = DMA_BIT_MASK(32), }; static int edma_init(void) @@ -720,8 +1122,6 @@ static int edma_init(void) ret = PTR_ERR(pdev0); goto out; } - pdev0->dev.dma_mask = &pdev0->dev.coherent_dma_mask; - pdev0->dev.coherent_dma_mask = DMA_BIT_MASK(32); } if (EDMA_CTLRS == 2) { @@ -731,8 +1131,6 @@ static int edma_init(void) platform_device_unregister(pdev0); ret = PTR_ERR(pdev1); } - pdev1->dev.dma_mask = &pdev1->dev.coherent_dma_mask; - pdev1->dev.coherent_dma_mask = DMA_BIT_MASK(32); } out: @@ -749,6 +1147,6 @@ static void __exit edma_exit(void) } module_exit(edma_exit); -MODULE_AUTHOR("Matt Porter <mporter@ti.com>"); +MODULE_AUTHOR("Matt Porter <matt.porter@linaro.org>"); MODULE_DESCRIPTION("TI EDMA DMA engine driver"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/ep93xx_dma.c b/drivers/dma/ep93xx_dma.c index 591cd8c63ab..cb4bf682a70 100644 --- a/drivers/dma/ep93xx_dma.c +++ b/drivers/dma/ep93xx_dma.c @@ -733,28 +733,6 @@ static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac) spin_unlock_irqrestore(&edmac->lock, flags); } -static void ep93xx_dma_unmap_buffers(struct ep93xx_dma_desc *desc) -{ - struct device *dev = desc->txd.chan->device->dev; - - if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE) - dma_unmap_single(dev, desc->src_addr, desc->size, - DMA_TO_DEVICE); - else - dma_unmap_page(dev, desc->src_addr, desc->size, - DMA_TO_DEVICE); - } - if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE) - dma_unmap_single(dev, desc->dst_addr, desc->size, - DMA_FROM_DEVICE); - else - dma_unmap_page(dev, desc->dst_addr, desc->size, - DMA_FROM_DEVICE); - } -} - static void ep93xx_dma_tasklet(unsigned long data) { struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data; @@ -787,13 +765,7 @@ static void ep93xx_dma_tasklet(unsigned long data) /* Now we can release all the chained descriptors */ list_for_each_entry_safe(desc, d, &list, node) { - /* - * For the memcpy channels the API requires us to unmap the - * buffers unless requested otherwise. - */ - if (!edmac->chan.private) - ep93xx_dma_unmap_buffers(desc); - + dma_descriptor_unmap(&desc->txd); ep93xx_dma_desc_put(edmac, desc); } diff --git a/drivers/dma/fsl-edma.c b/drivers/dma/fsl-edma.c new file mode 100644 index 00000000000..b396a7fb53a --- /dev/null +++ b/drivers/dma/fsl-edma.c @@ -0,0 +1,985 @@ +/* + * drivers/dma/fsl-edma.c + * + * Copyright 2013-2014 Freescale Semiconductor, Inc. + * + * Driver for the Freescale eDMA engine with flexible channel multiplexing + * capability for DMA request sources. The eDMA block can be found on some + * Vybrid and Layerscape SoCs. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/interrupt.h> +#include <linux/clk.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_dma.h> + +#include "virt-dma.h" + +#define EDMA_CR 0x00 +#define EDMA_ES 0x04 +#define EDMA_ERQ 0x0C +#define EDMA_EEI 0x14 +#define EDMA_SERQ 0x1B +#define EDMA_CERQ 0x1A +#define EDMA_SEEI 0x19 +#define EDMA_CEEI 0x18 +#define EDMA_CINT 0x1F +#define EDMA_CERR 0x1E +#define EDMA_SSRT 0x1D +#define EDMA_CDNE 0x1C +#define EDMA_INTR 0x24 +#define EDMA_ERR 0x2C + +#define EDMA_TCD_SADDR(x) (0x1000 + 32 * (x)) +#define EDMA_TCD_SOFF(x) (0x1004 + 32 * (x)) +#define EDMA_TCD_ATTR(x) (0x1006 + 32 * (x)) +#define EDMA_TCD_NBYTES(x) (0x1008 + 32 * (x)) +#define EDMA_TCD_SLAST(x) (0x100C + 32 * (x)) +#define EDMA_TCD_DADDR(x) (0x1010 + 32 * (x)) +#define EDMA_TCD_DOFF(x) (0x1014 + 32 * (x)) +#define EDMA_TCD_CITER_ELINK(x) (0x1016 + 32 * (x)) +#define EDMA_TCD_CITER(x) (0x1016 + 32 * (x)) +#define EDMA_TCD_DLAST_SGA(x) (0x1018 + 32 * (x)) +#define EDMA_TCD_CSR(x) (0x101C + 32 * (x)) +#define EDMA_TCD_BITER_ELINK(x) (0x101E + 32 * (x)) +#define EDMA_TCD_BITER(x) (0x101E + 32 * (x)) + +#define EDMA_CR_EDBG BIT(1) +#define EDMA_CR_ERCA BIT(2) +#define EDMA_CR_ERGA BIT(3) +#define EDMA_CR_HOE BIT(4) +#define EDMA_CR_HALT BIT(5) +#define EDMA_CR_CLM BIT(6) +#define EDMA_CR_EMLM BIT(7) +#define EDMA_CR_ECX BIT(16) +#define EDMA_CR_CX BIT(17) + +#define EDMA_SEEI_SEEI(x) ((x) & 0x1F) +#define EDMA_CEEI_CEEI(x) ((x) & 0x1F) +#define EDMA_CINT_CINT(x) ((x) & 0x1F) +#define EDMA_CERR_CERR(x) ((x) & 0x1F) + +#define EDMA_TCD_ATTR_DSIZE(x) (((x) & 0x0007)) +#define EDMA_TCD_ATTR_DMOD(x) (((x) & 0x001F) << 3) +#define EDMA_TCD_ATTR_SSIZE(x) (((x) & 0x0007) << 8) +#define EDMA_TCD_ATTR_SMOD(x) (((x) & 0x001F) << 11) +#define EDMA_TCD_ATTR_SSIZE_8BIT (0x0000) +#define EDMA_TCD_ATTR_SSIZE_16BIT (0x0100) +#define EDMA_TCD_ATTR_SSIZE_32BIT (0x0200) +#define EDMA_TCD_ATTR_SSIZE_64BIT (0x0300) +#define EDMA_TCD_ATTR_SSIZE_32BYTE (0x0500) +#define EDMA_TCD_ATTR_DSIZE_8BIT (0x0000) +#define EDMA_TCD_ATTR_DSIZE_16BIT (0x0001) +#define EDMA_TCD_ATTR_DSIZE_32BIT (0x0002) +#define EDMA_TCD_ATTR_DSIZE_64BIT (0x0003) +#define EDMA_TCD_ATTR_DSIZE_32BYTE (0x0005) + +#define EDMA_TCD_SOFF_SOFF(x) (x) +#define EDMA_TCD_NBYTES_NBYTES(x) (x) +#define EDMA_TCD_SLAST_SLAST(x) (x) +#define EDMA_TCD_DADDR_DADDR(x) (x) +#define EDMA_TCD_CITER_CITER(x) ((x) & 0x7FFF) +#define EDMA_TCD_DOFF_DOFF(x) (x) +#define EDMA_TCD_DLAST_SGA_DLAST_SGA(x) (x) +#define EDMA_TCD_BITER_BITER(x) ((x) & 0x7FFF) + +#define EDMA_TCD_CSR_START BIT(0) +#define EDMA_TCD_CSR_INT_MAJOR BIT(1) +#define EDMA_TCD_CSR_INT_HALF BIT(2) +#define EDMA_TCD_CSR_D_REQ BIT(3) +#define EDMA_TCD_CSR_E_SG BIT(4) +#define EDMA_TCD_CSR_E_LINK BIT(5) +#define EDMA_TCD_CSR_ACTIVE BIT(6) +#define EDMA_TCD_CSR_DONE BIT(7) + +#define EDMAMUX_CHCFG_DIS 0x0 +#define EDMAMUX_CHCFG_ENBL 0x80 +#define EDMAMUX_CHCFG_SOURCE(n) ((n) & 0x3F) + +#define DMAMUX_NR 2 + +#define FSL_EDMA_BUSWIDTHS BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES) + +struct fsl_edma_hw_tcd { + u32 saddr; + u16 soff; + u16 attr; + u32 nbytes; + u32 slast; + u32 daddr; + u16 doff; + u16 citer; + u32 dlast_sga; + u16 csr; + u16 biter; +}; + +struct fsl_edma_sw_tcd { + dma_addr_t ptcd; + struct fsl_edma_hw_tcd *vtcd; +}; + +struct fsl_edma_slave_config { + enum dma_transfer_direction dir; + enum dma_slave_buswidth addr_width; + u32 dev_addr; + u32 burst; + u32 attr; +}; + +struct fsl_edma_chan { + struct virt_dma_chan vchan; + enum dma_status status; + struct fsl_edma_engine *edma; + struct fsl_edma_desc *edesc; + struct fsl_edma_slave_config fsc; + struct dma_pool *tcd_pool; +}; + +struct fsl_edma_desc { + struct virt_dma_desc vdesc; + struct fsl_edma_chan *echan; + bool iscyclic; + unsigned int n_tcds; + struct fsl_edma_sw_tcd tcd[]; +}; + +struct fsl_edma_engine { + struct dma_device dma_dev; + void __iomem *membase; + void __iomem *muxbase[DMAMUX_NR]; + struct clk *muxclk[DMAMUX_NR]; + struct mutex fsl_edma_mutex; + u32 n_chans; + int txirq; + int errirq; + bool big_endian; + struct fsl_edma_chan chans[]; +}; + +/* + * R/W functions for big- or little-endian registers + * the eDMA controller's endian is independent of the CPU core's endian. + */ + +static u16 edma_readw(struct fsl_edma_engine *edma, void __iomem *addr) +{ + if (edma->big_endian) + return ioread16be(addr); + else + return ioread16(addr); +} + +static u32 edma_readl(struct fsl_edma_engine *edma, void __iomem *addr) +{ + if (edma->big_endian) + return ioread32be(addr); + else + return ioread32(addr); +} + +static void edma_writeb(struct fsl_edma_engine *edma, u8 val, void __iomem *addr) +{ + iowrite8(val, addr); +} + +static void edma_writew(struct fsl_edma_engine *edma, u16 val, void __iomem *addr) +{ + if (edma->big_endian) + iowrite16be(val, addr); + else + iowrite16(val, addr); +} + +static void edma_writel(struct fsl_edma_engine *edma, u32 val, void __iomem *addr) +{ + if (edma->big_endian) + iowrite32be(val, addr); + else + iowrite32(val, addr); +} + +static struct fsl_edma_chan *to_fsl_edma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct fsl_edma_chan, vchan.chan); +} + +static struct fsl_edma_desc *to_fsl_edma_desc(struct virt_dma_desc *vd) +{ + return container_of(vd, struct fsl_edma_desc, vdesc); +} + +static void fsl_edma_enable_request(struct fsl_edma_chan *fsl_chan) +{ + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + + edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), addr + EDMA_SEEI); + edma_writeb(fsl_chan->edma, ch, addr + EDMA_SERQ); +} + +static void fsl_edma_disable_request(struct fsl_edma_chan *fsl_chan) +{ + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + + edma_writeb(fsl_chan->edma, ch, addr + EDMA_CERQ); + edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), addr + EDMA_CEEI); +} + +static void fsl_edma_chan_mux(struct fsl_edma_chan *fsl_chan, + unsigned int slot, bool enable) +{ + u32 ch = fsl_chan->vchan.chan.chan_id; + void __iomem *muxaddr = fsl_chan->edma->muxbase[ch / DMAMUX_NR]; + unsigned chans_per_mux, ch_off; + + chans_per_mux = fsl_chan->edma->n_chans / DMAMUX_NR; + ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux; + + if (enable) + edma_writeb(fsl_chan->edma, + EDMAMUX_CHCFG_ENBL | EDMAMUX_CHCFG_SOURCE(slot), + muxaddr + ch_off); + else + edma_writeb(fsl_chan->edma, EDMAMUX_CHCFG_DIS, muxaddr + ch_off); +} + +static unsigned int fsl_edma_get_tcd_attr(enum dma_slave_buswidth addr_width) +{ + switch (addr_width) { + case 1: + return EDMA_TCD_ATTR_SSIZE_8BIT | EDMA_TCD_ATTR_DSIZE_8BIT; + case 2: + return EDMA_TCD_ATTR_SSIZE_16BIT | EDMA_TCD_ATTR_DSIZE_16BIT; + case 4: + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; + case 8: + return EDMA_TCD_ATTR_SSIZE_64BIT | EDMA_TCD_ATTR_DSIZE_64BIT; + default: + return EDMA_TCD_ATTR_SSIZE_32BIT | EDMA_TCD_ATTR_DSIZE_32BIT; + } +} + +static void fsl_edma_free_desc(struct virt_dma_desc *vdesc) +{ + struct fsl_edma_desc *fsl_desc; + int i; + + fsl_desc = to_fsl_edma_desc(vdesc); + for (i = 0; i < fsl_desc->n_tcds; i++) + dma_pool_free(fsl_desc->echan->tcd_pool, + fsl_desc->tcd[i].vtcd, + fsl_desc->tcd[i].ptcd); + kfree(fsl_desc); +} + +static int fsl_edma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct dma_slave_config *cfg = (void *)arg; + unsigned long flags; + LIST_HEAD(head); + + switch (cmd) { + case DMA_TERMINATE_ALL: + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + fsl_edma_disable_request(fsl_chan); + fsl_chan->edesc = NULL; + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + return 0; + + case DMA_SLAVE_CONFIG: + fsl_chan->fsc.dir = cfg->direction; + if (cfg->direction == DMA_DEV_TO_MEM) { + fsl_chan->fsc.dev_addr = cfg->src_addr; + fsl_chan->fsc.addr_width = cfg->src_addr_width; + fsl_chan->fsc.burst = cfg->src_maxburst; + fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->src_addr_width); + } else if (cfg->direction == DMA_MEM_TO_DEV) { + fsl_chan->fsc.dev_addr = cfg->dst_addr; + fsl_chan->fsc.addr_width = cfg->dst_addr_width; + fsl_chan->fsc.burst = cfg->dst_maxburst; + fsl_chan->fsc.attr = fsl_edma_get_tcd_attr(cfg->dst_addr_width); + } else { + return -EINVAL; + } + return 0; + + case DMA_PAUSE: + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (fsl_chan->edesc) { + fsl_edma_disable_request(fsl_chan); + fsl_chan->status = DMA_PAUSED; + } + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + return 0; + + case DMA_RESUME: + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + if (fsl_chan->edesc) { + fsl_edma_enable_request(fsl_chan); + fsl_chan->status = DMA_IN_PROGRESS; + } + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + return 0; + + default: + return -ENXIO; + } +} + +static size_t fsl_edma_desc_residue(struct fsl_edma_chan *fsl_chan, + struct virt_dma_desc *vdesc, bool in_progress) +{ + struct fsl_edma_desc *edesc = fsl_chan->edesc; + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + enum dma_transfer_direction dir = fsl_chan->fsc.dir; + dma_addr_t cur_addr, dma_addr; + size_t len, size; + int i; + + /* calculate the total size in this desc */ + for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++) + len += edma_readl(fsl_chan->edma, &(edesc->tcd[i].vtcd->nbytes)) + * edma_readw(fsl_chan->edma, &(edesc->tcd[i].vtcd->biter)); + + if (!in_progress) + return len; + + if (dir == DMA_MEM_TO_DEV) + cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_SADDR(ch)); + else + cur_addr = edma_readl(fsl_chan->edma, addr + EDMA_TCD_DADDR(ch)); + + /* figure out the finished and calculate the residue */ + for (i = 0; i < fsl_chan->edesc->n_tcds; i++) { + size = edma_readl(fsl_chan->edma, &(edesc->tcd[i].vtcd->nbytes)) + * edma_readw(fsl_chan->edma, &(edesc->tcd[i].vtcd->biter)); + if (dir == DMA_MEM_TO_DEV) + dma_addr = edma_readl(fsl_chan->edma, + &(edesc->tcd[i].vtcd->saddr)); + else + dma_addr = edma_readl(fsl_chan->edma, + &(edesc->tcd[i].vtcd->daddr)); + + len -= size; + if (cur_addr > dma_addr && cur_addr < dma_addr + size) { + len += dma_addr + size - cur_addr; + break; + } + } + + return len; +} + +static enum dma_status fsl_edma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct virt_dma_desc *vdesc; + enum dma_status status; + unsigned long flags; + + status = dma_cookie_status(chan, cookie, txstate); + if (status == DMA_COMPLETE) + return status; + + if (!txstate) + return fsl_chan->status; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + vdesc = vchan_find_desc(&fsl_chan->vchan, cookie); + if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie) + txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, true); + else if (vdesc) + txstate->residue = fsl_edma_desc_residue(fsl_chan, vdesc, false); + else + txstate->residue = 0; + + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + + return fsl_chan->status; +} + +static void fsl_edma_set_tcd_params(struct fsl_edma_chan *fsl_chan, + u32 src, u32 dst, u16 attr, u16 soff, u32 nbytes, + u32 slast, u16 citer, u16 biter, u32 doff, u32 dlast_sga, + u16 csr) +{ + void __iomem *addr = fsl_chan->edma->membase; + u32 ch = fsl_chan->vchan.chan.chan_id; + + /* + * TCD parameters have been swapped in fill_tcd_params(), + * so just write them to registers in the cpu endian here + */ + writew(0, addr + EDMA_TCD_CSR(ch)); + writel(src, addr + EDMA_TCD_SADDR(ch)); + writel(dst, addr + EDMA_TCD_DADDR(ch)); + writew(attr, addr + EDMA_TCD_ATTR(ch)); + writew(soff, addr + EDMA_TCD_SOFF(ch)); + writel(nbytes, addr + EDMA_TCD_NBYTES(ch)); + writel(slast, addr + EDMA_TCD_SLAST(ch)); + writew(citer, addr + EDMA_TCD_CITER(ch)); + writew(biter, addr + EDMA_TCD_BITER(ch)); + writew(doff, addr + EDMA_TCD_DOFF(ch)); + writel(dlast_sga, addr + EDMA_TCD_DLAST_SGA(ch)); + writew(csr, addr + EDMA_TCD_CSR(ch)); +} + +static void fill_tcd_params(struct fsl_edma_engine *edma, + struct fsl_edma_hw_tcd *tcd, u32 src, u32 dst, + u16 attr, u16 soff, u32 nbytes, u32 slast, u16 citer, + u16 biter, u16 doff, u32 dlast_sga, bool major_int, + bool disable_req, bool enable_sg) +{ + u16 csr = 0; + + /* + * eDMA hardware SGs require the TCD parameters stored in memory + * the same endian as the eDMA module so that they can be loaded + * automatically by the engine + */ + edma_writel(edma, src, &(tcd->saddr)); + edma_writel(edma, dst, &(tcd->daddr)); + edma_writew(edma, attr, &(tcd->attr)); + edma_writew(edma, EDMA_TCD_SOFF_SOFF(soff), &(tcd->soff)); + edma_writel(edma, EDMA_TCD_NBYTES_NBYTES(nbytes), &(tcd->nbytes)); + edma_writel(edma, EDMA_TCD_SLAST_SLAST(slast), &(tcd->slast)); + edma_writew(edma, EDMA_TCD_CITER_CITER(citer), &(tcd->citer)); + edma_writew(edma, EDMA_TCD_DOFF_DOFF(doff), &(tcd->doff)); + edma_writel(edma, EDMA_TCD_DLAST_SGA_DLAST_SGA(dlast_sga), &(tcd->dlast_sga)); + edma_writew(edma, EDMA_TCD_BITER_BITER(biter), &(tcd->biter)); + if (major_int) + csr |= EDMA_TCD_CSR_INT_MAJOR; + + if (disable_req) + csr |= EDMA_TCD_CSR_D_REQ; + + if (enable_sg) + csr |= EDMA_TCD_CSR_E_SG; + + edma_writew(edma, csr, &(tcd->csr)); +} + +static struct fsl_edma_desc *fsl_edma_alloc_desc(struct fsl_edma_chan *fsl_chan, + int sg_len) +{ + struct fsl_edma_desc *fsl_desc; + int i; + + fsl_desc = kzalloc(sizeof(*fsl_desc) + sizeof(struct fsl_edma_sw_tcd) * sg_len, + GFP_NOWAIT); + if (!fsl_desc) + return NULL; + + fsl_desc->echan = fsl_chan; + fsl_desc->n_tcds = sg_len; + for (i = 0; i < sg_len; i++) { + fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool, + GFP_NOWAIT, &fsl_desc->tcd[i].ptcd); + if (!fsl_desc->tcd[i].vtcd) + goto err; + } + return fsl_desc; + +err: + while (--i >= 0) + dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd, + fsl_desc->tcd[i].ptcd); + kfree(fsl_desc); + return NULL; +} + +static struct dma_async_tx_descriptor *fsl_edma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t dma_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct fsl_edma_desc *fsl_desc; + dma_addr_t dma_buf_next; + int sg_len, i; + u32 src_addr, dst_addr, last_sg, nbytes; + u16 soff, doff, iter; + + if (!is_slave_direction(fsl_chan->fsc.dir)) + return NULL; + + sg_len = buf_len / period_len; + fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = true; + + dma_buf_next = dma_addr; + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; + iter = period_len / nbytes; + + for (i = 0; i < sg_len; i++) { + if (dma_buf_next >= dma_addr + buf_len) + dma_buf_next = dma_addr; + + /* get next sg's physical address */ + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; + + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { + src_addr = dma_buf_next; + dst_addr = fsl_chan->fsc.dev_addr; + soff = fsl_chan->fsc.addr_width; + doff = 0; + } else { + src_addr = fsl_chan->fsc.dev_addr; + dst_addr = dma_buf_next; + soff = 0; + doff = fsl_chan->fsc.addr_width; + } + + fill_tcd_params(fsl_chan->edma, fsl_desc->tcd[i].vtcd, src_addr, + dst_addr, fsl_chan->fsc.attr, soff, nbytes, 0, + iter, iter, doff, last_sg, true, false, true); + dma_buf_next += period_len; + } + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); +} + +static struct dma_async_tx_descriptor *fsl_edma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + struct fsl_edma_desc *fsl_desc; + struct scatterlist *sg; + u32 src_addr, dst_addr, last_sg, nbytes; + u16 soff, doff, iter; + int i; + + if (!is_slave_direction(fsl_chan->fsc.dir)) + return NULL; + + fsl_desc = fsl_edma_alloc_desc(fsl_chan, sg_len); + if (!fsl_desc) + return NULL; + fsl_desc->iscyclic = false; + + nbytes = fsl_chan->fsc.addr_width * fsl_chan->fsc.burst; + for_each_sg(sgl, sg, sg_len, i) { + /* get next sg's physical address */ + last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd; + + if (fsl_chan->fsc.dir == DMA_MEM_TO_DEV) { + src_addr = sg_dma_address(sg); + dst_addr = fsl_chan->fsc.dev_addr; + soff = fsl_chan->fsc.addr_width; + doff = 0; + } else { + src_addr = fsl_chan->fsc.dev_addr; + dst_addr = sg_dma_address(sg); + soff = 0; + doff = fsl_chan->fsc.addr_width; + } + + iter = sg_dma_len(sg) / nbytes; + if (i < sg_len - 1) { + last_sg = fsl_desc->tcd[(i + 1)].ptcd; + fill_tcd_params(fsl_chan->edma, fsl_desc->tcd[i].vtcd, + src_addr, dst_addr, fsl_chan->fsc.attr, + soff, nbytes, 0, iter, iter, doff, last_sg, + false, false, true); + } else { + last_sg = 0; + fill_tcd_params(fsl_chan->edma, fsl_desc->tcd[i].vtcd, + src_addr, dst_addr, fsl_chan->fsc.attr, + soff, nbytes, 0, iter, iter, doff, last_sg, + true, true, false); + } + } + + return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags); +} + +static void fsl_edma_xfer_desc(struct fsl_edma_chan *fsl_chan) +{ + struct fsl_edma_hw_tcd *tcd; + struct virt_dma_desc *vdesc; + + vdesc = vchan_next_desc(&fsl_chan->vchan); + if (!vdesc) + return; + fsl_chan->edesc = to_fsl_edma_desc(vdesc); + tcd = fsl_chan->edesc->tcd[0].vtcd; + fsl_edma_set_tcd_params(fsl_chan, tcd->saddr, tcd->daddr, tcd->attr, + tcd->soff, tcd->nbytes, tcd->slast, tcd->citer, + tcd->biter, tcd->doff, tcd->dlast_sga, tcd->csr); + fsl_edma_enable_request(fsl_chan); + fsl_chan->status = DMA_IN_PROGRESS; +} + +static irqreturn_t fsl_edma_tx_handler(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + unsigned int intr, ch; + void __iomem *base_addr; + struct fsl_edma_chan *fsl_chan; + + base_addr = fsl_edma->membase; + + intr = edma_readl(fsl_edma, base_addr + EDMA_INTR); + if (!intr) + return IRQ_NONE; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + if (intr & (0x1 << ch)) { + edma_writeb(fsl_edma, EDMA_CINT_CINT(ch), + base_addr + EDMA_CINT); + + fsl_chan = &fsl_edma->chans[ch]; + + spin_lock(&fsl_chan->vchan.lock); + if (!fsl_chan->edesc->iscyclic) { + list_del(&fsl_chan->edesc->vdesc.node); + vchan_cookie_complete(&fsl_chan->edesc->vdesc); + fsl_chan->edesc = NULL; + fsl_chan->status = DMA_COMPLETE; + } else { + vchan_cyclic_callback(&fsl_chan->edesc->vdesc); + } + + if (!fsl_chan->edesc) + fsl_edma_xfer_desc(fsl_chan); + + spin_unlock(&fsl_chan->vchan.lock); + } + } + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma_err_handler(int irq, void *dev_id) +{ + struct fsl_edma_engine *fsl_edma = dev_id; + unsigned int err, ch; + + err = edma_readl(fsl_edma, fsl_edma->membase + EDMA_ERR); + if (!err) + return IRQ_NONE; + + for (ch = 0; ch < fsl_edma->n_chans; ch++) { + if (err & (0x1 << ch)) { + fsl_edma_disable_request(&fsl_edma->chans[ch]); + edma_writeb(fsl_edma, EDMA_CERR_CERR(ch), + fsl_edma->membase + EDMA_CERR); + fsl_edma->chans[ch].status = DMA_ERROR; + } + } + return IRQ_HANDLED; +} + +static irqreturn_t fsl_edma_irq_handler(int irq, void *dev_id) +{ + if (fsl_edma_tx_handler(irq, dev_id) == IRQ_HANDLED) + return IRQ_HANDLED; + + return fsl_edma_err_handler(irq, dev_id); +} + +static void fsl_edma_issue_pending(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + + if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc) + fsl_edma_xfer_desc(fsl_chan); + + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); +} + +static struct dma_chan *fsl_edma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct fsl_edma_engine *fsl_edma = ofdma->of_dma_data; + struct dma_chan *chan, *_chan; + + if (dma_spec->args_count != 2) + return NULL; + + mutex_lock(&fsl_edma->fsl_edma_mutex); + list_for_each_entry_safe(chan, _chan, &fsl_edma->dma_dev.channels, device_node) { + if (chan->client_count) + continue; + if ((chan->chan_id / DMAMUX_NR) == dma_spec->args[0]) { + chan = dma_get_slave_channel(chan); + if (chan) { + chan->device->privatecnt++; + fsl_edma_chan_mux(to_fsl_edma_chan(chan), + dma_spec->args[1], true); + mutex_unlock(&fsl_edma->fsl_edma_mutex); + return chan; + } + } + } + mutex_unlock(&fsl_edma->fsl_edma_mutex); + return NULL; +} + +static int fsl_edma_alloc_chan_resources(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + + fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev, + sizeof(struct fsl_edma_hw_tcd), + 32, 0); + return 0; +} + +static void fsl_edma_free_chan_resources(struct dma_chan *chan) +{ + struct fsl_edma_chan *fsl_chan = to_fsl_edma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + + spin_lock_irqsave(&fsl_chan->vchan.lock, flags); + fsl_edma_disable_request(fsl_chan); + fsl_edma_chan_mux(fsl_chan, 0, false); + fsl_chan->edesc = NULL; + vchan_get_all_descriptors(&fsl_chan->vchan, &head); + spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags); + + vchan_dma_desc_free_list(&fsl_chan->vchan, &head); + dma_pool_destroy(fsl_chan->tcd_pool); + fsl_chan->tcd_pool = NULL; +} + +static int fsl_dma_device_slave_caps(struct dma_chan *dchan, + struct dma_slave_caps *caps) +{ + caps->src_addr_widths = FSL_EDMA_BUSWIDTHS; + caps->dstn_addr_widths = FSL_EDMA_BUSWIDTHS; + caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + caps->cmd_pause = true; + caps->cmd_terminate = true; + + return 0; +} + +static int +fsl_edma_irq_init(struct platform_device *pdev, struct fsl_edma_engine *fsl_edma) +{ + int ret; + + fsl_edma->txirq = platform_get_irq_byname(pdev, "edma-tx"); + if (fsl_edma->txirq < 0) { + dev_err(&pdev->dev, "Can't get edma-tx irq.\n"); + return fsl_edma->txirq; + } + + fsl_edma->errirq = platform_get_irq_byname(pdev, "edma-err"); + if (fsl_edma->errirq < 0) { + dev_err(&pdev->dev, "Can't get edma-err irq.\n"); + return fsl_edma->errirq; + } + + if (fsl_edma->txirq == fsl_edma->errirq) { + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, + fsl_edma_irq_handler, 0, "eDMA", fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register eDMA IRQ.\n"); + return ret; + } + } else { + ret = devm_request_irq(&pdev->dev, fsl_edma->txirq, + fsl_edma_tx_handler, 0, "eDMA tx", fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register eDMA tx IRQ.\n"); + return ret; + } + + ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, + fsl_edma_err_handler, 0, "eDMA err", fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register eDMA err IRQ.\n"); + return ret; + } + } + + return 0; +} + +static int fsl_edma_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct fsl_edma_engine *fsl_edma; + struct fsl_edma_chan *fsl_chan; + struct resource *res; + int len, chans; + int ret, i; + + ret = of_property_read_u32(np, "dma-channels", &chans); + if (ret) { + dev_err(&pdev->dev, "Can't get dma-channels.\n"); + return ret; + } + + len = sizeof(*fsl_edma) + sizeof(*fsl_chan) * chans; + fsl_edma = devm_kzalloc(&pdev->dev, len, GFP_KERNEL); + if (!fsl_edma) + return -ENOMEM; + + fsl_edma->n_chans = chans; + mutex_init(&fsl_edma->fsl_edma_mutex); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + fsl_edma->membase = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(fsl_edma->membase)) + return PTR_ERR(fsl_edma->membase); + + for (i = 0; i < DMAMUX_NR; i++) { + char clkname[32]; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 1 + i); + fsl_edma->muxbase[i] = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(fsl_edma->muxbase[i])) + return PTR_ERR(fsl_edma->muxbase[i]); + + sprintf(clkname, "dmamux%d", i); + fsl_edma->muxclk[i] = devm_clk_get(&pdev->dev, clkname); + if (IS_ERR(fsl_edma->muxclk[i])) { + dev_err(&pdev->dev, "Missing DMAMUX block clock.\n"); + return PTR_ERR(fsl_edma->muxclk[i]); + } + + ret = clk_prepare_enable(fsl_edma->muxclk[i]); + if (ret) { + dev_err(&pdev->dev, "DMAMUX clk block failed.\n"); + return ret; + } + + } + + ret = fsl_edma_irq_init(pdev, fsl_edma); + if (ret) + return ret; + + fsl_edma->big_endian = of_property_read_bool(np, "big-endian"); + + INIT_LIST_HEAD(&fsl_edma->dma_dev.channels); + for (i = 0; i < fsl_edma->n_chans; i++) { + struct fsl_edma_chan *fsl_chan = &fsl_edma->chans[i]; + + fsl_chan->edma = fsl_edma; + + fsl_chan->vchan.desc_free = fsl_edma_free_desc; + vchan_init(&fsl_chan->vchan, &fsl_edma->dma_dev); + + edma_writew(fsl_edma, 0x0, fsl_edma->membase + EDMA_TCD_CSR(i)); + fsl_edma_chan_mux(fsl_chan, 0, false); + } + + dma_cap_set(DMA_PRIVATE, fsl_edma->dma_dev.cap_mask); + dma_cap_set(DMA_SLAVE, fsl_edma->dma_dev.cap_mask); + dma_cap_set(DMA_CYCLIC, fsl_edma->dma_dev.cap_mask); + + fsl_edma->dma_dev.dev = &pdev->dev; + fsl_edma->dma_dev.device_alloc_chan_resources + = fsl_edma_alloc_chan_resources; + fsl_edma->dma_dev.device_free_chan_resources + = fsl_edma_free_chan_resources; + fsl_edma->dma_dev.device_tx_status = fsl_edma_tx_status; + fsl_edma->dma_dev.device_prep_slave_sg = fsl_edma_prep_slave_sg; + fsl_edma->dma_dev.device_prep_dma_cyclic = fsl_edma_prep_dma_cyclic; + fsl_edma->dma_dev.device_control = fsl_edma_control; + fsl_edma->dma_dev.device_issue_pending = fsl_edma_issue_pending; + fsl_edma->dma_dev.device_slave_caps = fsl_dma_device_slave_caps; + + platform_set_drvdata(pdev, fsl_edma); + + ret = dma_async_device_register(&fsl_edma->dma_dev); + if (ret) { + dev_err(&pdev->dev, "Can't register Freescale eDMA engine.\n"); + return ret; + } + + ret = of_dma_controller_register(np, fsl_edma_xlate, fsl_edma); + if (ret) { + dev_err(&pdev->dev, "Can't register Freescale eDMA of_dma.\n"); + dma_async_device_unregister(&fsl_edma->dma_dev); + return ret; + } + + /* enable round robin arbitration */ + edma_writel(fsl_edma, EDMA_CR_ERGA | EDMA_CR_ERCA, fsl_edma->membase + EDMA_CR); + + return 0; +} + +static int fsl_edma_remove(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct fsl_edma_engine *fsl_edma = platform_get_drvdata(pdev); + int i; + + of_dma_controller_free(np); + dma_async_device_unregister(&fsl_edma->dma_dev); + + for (i = 0; i < DMAMUX_NR; i++) + clk_disable_unprepare(fsl_edma->muxclk[i]); + + return 0; +} + +static const struct of_device_id fsl_edma_dt_ids[] = { + { .compatible = "fsl,vf610-edma", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, fsl_edma_dt_ids); + +static struct platform_driver fsl_edma_driver = { + .driver = { + .name = "fsl-edma", + .owner = THIS_MODULE, + .of_match_table = fsl_edma_dt_ids, + }, + .probe = fsl_edma_probe, + .remove = fsl_edma_remove, +}; + +static int __init fsl_edma_init(void) +{ + return platform_driver_register(&fsl_edma_driver); +} +subsys_initcall(fsl_edma_init); + +static void __exit fsl_edma_exit(void) +{ + platform_driver_unregister(&fsl_edma_driver); +} +module_exit(fsl_edma_exit); + +MODULE_ALIAS("platform:fsl-edma"); +MODULE_DESCRIPTION("Freescale eDMA engine driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/fsldma.c b/drivers/dma/fsldma.c index b3f3e90054f..e0fec68aed2 100644 --- a/drivers/dma/fsldma.c +++ b/drivers/dma/fsldma.c @@ -33,6 +33,8 @@ #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/dmapool.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> #include <linux/of_platform.h> #include "dmaengine.h" @@ -59,6 +61,16 @@ static u32 get_sr(struct fsldma_chan *chan) return DMA_IN(chan, &chan->regs->sr, 32); } +static void set_mr(struct fsldma_chan *chan, u32 val) +{ + DMA_OUT(chan, &chan->regs->mr, val, 32); +} + +static u32 get_mr(struct fsldma_chan *chan) +{ + return DMA_IN(chan, &chan->regs->mr, 32); +} + static void set_cdar(struct fsldma_chan *chan, dma_addr_t addr) { DMA_OUT(chan, &chan->regs->cdar, addr | FSL_DMA_SNEN, 64); @@ -69,6 +81,11 @@ static dma_addr_t get_cdar(struct fsldma_chan *chan) return DMA_IN(chan, &chan->regs->cdar, 64) & ~FSL_DMA_SNEN; } +static void set_bcr(struct fsldma_chan *chan, u32 val) +{ + DMA_OUT(chan, &chan->regs->bcr, val, 32); +} + static u32 get_bcr(struct fsldma_chan *chan) { return DMA_IN(chan, &chan->regs->bcr, 32); @@ -84,11 +101,6 @@ static void set_desc_cnt(struct fsldma_chan *chan, hw->count = CPU_TO_DMA(chan, count, 32); } -static u32 get_desc_cnt(struct fsldma_chan *chan, struct fsl_desc_sw *desc) -{ - return DMA_TO_CPU(chan, desc->hw.count, 32); -} - static void set_desc_src(struct fsldma_chan *chan, struct fsl_dma_ld_hw *hw, dma_addr_t src) { @@ -99,16 +111,6 @@ static void set_desc_src(struct fsldma_chan *chan, hw->src_addr = CPU_TO_DMA(chan, snoop_bits | src, 64); } -static dma_addr_t get_desc_src(struct fsldma_chan *chan, - struct fsl_desc_sw *desc) -{ - u64 snoop_bits; - - snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) - ? ((u64)FSL_DMA_SATR_SREADTYPE_SNOOP_READ << 32) : 0; - return DMA_TO_CPU(chan, desc->hw.src_addr, 64) & ~snoop_bits; -} - static void set_desc_dst(struct fsldma_chan *chan, struct fsl_dma_ld_hw *hw, dma_addr_t dst) { @@ -119,16 +121,6 @@ static void set_desc_dst(struct fsldma_chan *chan, hw->dst_addr = CPU_TO_DMA(chan, snoop_bits | dst, 64); } -static dma_addr_t get_desc_dst(struct fsldma_chan *chan, - struct fsl_desc_sw *desc) -{ - u64 snoop_bits; - - snoop_bits = ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) - ? ((u64)FSL_DMA_DATR_DWRITETYPE_SNOOP_WRITE << 32) : 0; - return DMA_TO_CPU(chan, desc->hw.dst_addr, 64) & ~snoop_bits; -} - static void set_desc_next(struct fsldma_chan *chan, struct fsl_dma_ld_hw *hw, dma_addr_t next) { @@ -158,7 +150,7 @@ static void set_ld_eol(struct fsldma_chan *chan, struct fsl_desc_sw *desc) static void dma_init(struct fsldma_chan *chan) { /* Reset the channel */ - DMA_OUT(chan, &chan->regs->mr, 0, 32); + set_mr(chan, 0); switch (chan->feature & FSL_DMA_IP_MASK) { case FSL_DMA_IP_85XX: @@ -167,16 +159,15 @@ static void dma_init(struct fsldma_chan *chan) * EOLNIE - End of links interrupt enable * BWC - Bandwidth sharing among channels */ - DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_BWC - | FSL_DMA_MR_EIE | FSL_DMA_MR_EOLNIE, 32); + set_mr(chan, FSL_DMA_MR_BWC | FSL_DMA_MR_EIE + | FSL_DMA_MR_EOLNIE); break; case FSL_DMA_IP_83XX: /* Set the channel to below modes: * EOTIE - End-of-transfer interrupt enable * PRC_RM - PCI read multiple */ - DMA_OUT(chan, &chan->regs->mr, FSL_DMA_MR_EOTIE - | FSL_DMA_MR_PRC_RM, 32); + set_mr(chan, FSL_DMA_MR_EOTIE | FSL_DMA_MR_PRC_RM); break; } } @@ -198,10 +189,10 @@ static void dma_start(struct fsldma_chan *chan) { u32 mode; - mode = DMA_IN(chan, &chan->regs->mr, 32); + mode = get_mr(chan); if (chan->feature & FSL_DMA_CHAN_PAUSE_EXT) { - DMA_OUT(chan, &chan->regs->bcr, 0, 32); + set_bcr(chan, 0); mode |= FSL_DMA_MR_EMP_EN; } else { mode &= ~FSL_DMA_MR_EMP_EN; @@ -214,7 +205,7 @@ static void dma_start(struct fsldma_chan *chan) mode |= FSL_DMA_MR_CS; } - DMA_OUT(chan, &chan->regs->mr, mode, 32); + set_mr(chan, mode); } static void dma_halt(struct fsldma_chan *chan) @@ -223,7 +214,7 @@ static void dma_halt(struct fsldma_chan *chan) int i; /* read the mode register */ - mode = DMA_IN(chan, &chan->regs->mr, 32); + mode = get_mr(chan); /* * The 85xx controller supports channel abort, which will stop @@ -232,14 +223,14 @@ static void dma_halt(struct fsldma_chan *chan) */ if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { mode |= FSL_DMA_MR_CA; - DMA_OUT(chan, &chan->regs->mr, mode, 32); + set_mr(chan, mode); mode &= ~FSL_DMA_MR_CA; } /* stop the DMA controller */ mode &= ~(FSL_DMA_MR_CS | FSL_DMA_MR_EMS_EN); - DMA_OUT(chan, &chan->regs->mr, mode, 32); + set_mr(chan, mode); /* wait for the DMA controller to become idle */ for (i = 0; i < 100; i++) { @@ -268,7 +259,7 @@ static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size) { u32 mode; - mode = DMA_IN(chan, &chan->regs->mr, 32); + mode = get_mr(chan); switch (size) { case 0: @@ -282,7 +273,7 @@ static void fsl_chan_set_src_loop_size(struct fsldma_chan *chan, int size) break; } - DMA_OUT(chan, &chan->regs->mr, mode, 32); + set_mr(chan, mode); } /** @@ -300,7 +291,7 @@ static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size) { u32 mode; - mode = DMA_IN(chan, &chan->regs->mr, 32); + mode = get_mr(chan); switch (size) { case 0: @@ -314,7 +305,7 @@ static void fsl_chan_set_dst_loop_size(struct fsldma_chan *chan, int size) break; } - DMA_OUT(chan, &chan->regs->mr, mode, 32); + set_mr(chan, mode); } /** @@ -335,10 +326,10 @@ static void fsl_chan_set_request_count(struct fsldma_chan *chan, int size) BUG_ON(size > 1024); - mode = DMA_IN(chan, &chan->regs->mr, 32); + mode = get_mr(chan); mode |= (__ilog2(size) << 24) & 0x0f000000; - DMA_OUT(chan, &chan->regs->mr, mode, 32); + set_mr(chan, mode); } /** @@ -406,7 +397,7 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx) struct fsl_desc_sw *desc = tx_to_fsl_desc(tx); struct fsl_desc_sw *child; unsigned long flags; - dma_cookie_t cookie; + dma_cookie_t cookie = -EINVAL; spin_lock_irqsave(&chan->desc_lock, flags); @@ -427,6 +418,19 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx) } /** + * fsl_dma_free_descriptor - Free descriptor from channel's DMA pool. + * @chan : Freescale DMA channel + * @desc: descriptor to be freed + */ +static void fsl_dma_free_descriptor(struct fsldma_chan *chan, + struct fsl_desc_sw *desc) +{ + list_del(&desc->node); + chan_dbg(chan, "LD %p free\n", desc); + dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); +} + +/** * fsl_dma_alloc_descriptor - Allocate descriptor from channel's DMA pool. * @chan : Freescale DMA channel * @@ -449,14 +453,107 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(struct fsldma_chan *chan) desc->async_tx.tx_submit = fsl_dma_tx_submit; desc->async_tx.phys = pdesc; -#ifdef FSL_DMA_LD_DEBUG chan_dbg(chan, "LD %p allocated\n", desc); -#endif return desc; } /** + * fsl_chan_xfer_ld_queue - transfer any pending transactions + * @chan : Freescale DMA channel + * + * HARDWARE STATE: idle + * LOCKING: must hold chan->desc_lock + */ +static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan) +{ + struct fsl_desc_sw *desc; + + /* + * If the list of pending descriptors is empty, then we + * don't need to do any work at all + */ + if (list_empty(&chan->ld_pending)) { + chan_dbg(chan, "no pending LDs\n"); + return; + } + + /* + * The DMA controller is not idle, which means that the interrupt + * handler will start any queued transactions when it runs after + * this transaction finishes + */ + if (!chan->idle) { + chan_dbg(chan, "DMA controller still busy\n"); + return; + } + + /* + * If there are some link descriptors which have not been + * transferred, we need to start the controller + */ + + /* + * Move all elements from the queue of pending transactions + * onto the list of running transactions + */ + chan_dbg(chan, "idle, starting controller\n"); + desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node); + list_splice_tail_init(&chan->ld_pending, &chan->ld_running); + + /* + * The 85xx DMA controller doesn't clear the channel start bit + * automatically at the end of a transfer. Therefore we must clear + * it in software before starting the transfer. + */ + if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { + u32 mode; + + mode = get_mr(chan); + mode &= ~FSL_DMA_MR_CS; + set_mr(chan, mode); + } + + /* + * Program the descriptor's address into the DMA controller, + * then start the DMA transaction + */ + set_cdar(chan, desc->async_tx.phys); + get_cdar(chan); + + dma_start(chan); + chan->idle = false; +} + +/** + * fsldma_cleanup_descriptor - cleanup and free a single link descriptor + * @chan: Freescale DMA channel + * @desc: descriptor to cleanup and free + * + * This function is used on a descriptor which has been executed by the DMA + * controller. It will run any callbacks, submit any dependencies, and then + * free the descriptor. + */ +static void fsldma_cleanup_descriptor(struct fsldma_chan *chan, + struct fsl_desc_sw *desc) +{ + struct dma_async_tx_descriptor *txd = &desc->async_tx; + + /* Run the link descriptor callback function */ + if (txd->callback) { + chan_dbg(chan, "LD %p callback\n", desc); + txd->callback(txd->callback_param); + } + + /* Run any dependencies */ + dma_run_dependencies(txd); + + dma_descriptor_unmap(txd); + chan_dbg(chan, "LD %p free\n", desc); + dma_pool_free(chan->desc_pool, desc, txd->phys); +} + +/** * fsl_dma_alloc_chan_resources - Allocate resources for DMA channel. * @chan : Freescale DMA channel * @@ -500,13 +597,8 @@ static void fsldma_free_desc_list(struct fsldma_chan *chan, { struct fsl_desc_sw *desc, *_desc; - list_for_each_entry_safe(desc, _desc, list, node) { - list_del(&desc->node); -#ifdef FSL_DMA_LD_DEBUG - chan_dbg(chan, "LD %p free\n", desc); -#endif - dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); - } + list_for_each_entry_safe(desc, _desc, list, node) + fsl_dma_free_descriptor(chan, desc); } static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan, @@ -514,13 +606,8 @@ static void fsldma_free_desc_list_reverse(struct fsldma_chan *chan, { struct fsl_desc_sw *desc, *_desc; - list_for_each_entry_safe_reverse(desc, _desc, list, node) { - list_del(&desc->node); -#ifdef FSL_DMA_LD_DEBUG - chan_dbg(chan, "LD %p free\n", desc); -#endif - dma_pool_free(chan->desc_pool, desc, desc->async_tx.phys); - } + list_for_each_entry_safe_reverse(desc, _desc, list, node) + fsl_dma_free_descriptor(chan, desc); } /** @@ -543,35 +630,6 @@ static void fsl_dma_free_chan_resources(struct dma_chan *dchan) } static struct dma_async_tx_descriptor * -fsl_dma_prep_interrupt(struct dma_chan *dchan, unsigned long flags) -{ - struct fsldma_chan *chan; - struct fsl_desc_sw *new; - - if (!dchan) - return NULL; - - chan = to_fsl_chan(dchan); - - new = fsl_dma_alloc_descriptor(chan); - if (!new) { - chan_err(chan, "%s\n", msg_ld_oom); - return NULL; - } - - new->async_tx.cookie = -EBUSY; - new->async_tx.flags = flags; - - /* Insert the link descriptor to the LD ring */ - list_add_tail(&new->node, &new->tx_list); - - /* Set End-of-link to the last link descriptor of new list */ - set_ld_eol(chan, new); - - return &new->async_tx; -} - -static struct dma_async_tx_descriptor * fsl_dma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst, dma_addr_t dma_src, size_t len, unsigned long flags) @@ -840,124 +898,6 @@ static int fsl_dma_device_control(struct dma_chan *dchan, } /** - * fsldma_cleanup_descriptor - cleanup and free a single link descriptor - * @chan: Freescale DMA channel - * @desc: descriptor to cleanup and free - * - * This function is used on a descriptor which has been executed by the DMA - * controller. It will run any callbacks, submit any dependencies, and then - * free the descriptor. - */ -static void fsldma_cleanup_descriptor(struct fsldma_chan *chan, - struct fsl_desc_sw *desc) -{ - struct dma_async_tx_descriptor *txd = &desc->async_tx; - struct device *dev = chan->common.device->dev; - dma_addr_t src = get_desc_src(chan, desc); - dma_addr_t dst = get_desc_dst(chan, desc); - u32 len = get_desc_cnt(chan, desc); - - /* Run the link descriptor callback function */ - if (txd->callback) { -#ifdef FSL_DMA_LD_DEBUG - chan_dbg(chan, "LD %p callback\n", desc); -#endif - txd->callback(txd->callback_param); - } - - /* Run any dependencies */ - dma_run_dependencies(txd); - - /* Unmap the dst buffer, if requested */ - if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) - dma_unmap_single(dev, dst, len, DMA_FROM_DEVICE); - else - dma_unmap_page(dev, dst, len, DMA_FROM_DEVICE); - } - - /* Unmap the src buffer, if requested */ - if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) - dma_unmap_single(dev, src, len, DMA_TO_DEVICE); - else - dma_unmap_page(dev, src, len, DMA_TO_DEVICE); - } - -#ifdef FSL_DMA_LD_DEBUG - chan_dbg(chan, "LD %p free\n", desc); -#endif - dma_pool_free(chan->desc_pool, desc, txd->phys); -} - -/** - * fsl_chan_xfer_ld_queue - transfer any pending transactions - * @chan : Freescale DMA channel - * - * HARDWARE STATE: idle - * LOCKING: must hold chan->desc_lock - */ -static void fsl_chan_xfer_ld_queue(struct fsldma_chan *chan) -{ - struct fsl_desc_sw *desc; - - /* - * If the list of pending descriptors is empty, then we - * don't need to do any work at all - */ - if (list_empty(&chan->ld_pending)) { - chan_dbg(chan, "no pending LDs\n"); - return; - } - - /* - * The DMA controller is not idle, which means that the interrupt - * handler will start any queued transactions when it runs after - * this transaction finishes - */ - if (!chan->idle) { - chan_dbg(chan, "DMA controller still busy\n"); - return; - } - - /* - * If there are some link descriptors which have not been - * transferred, we need to start the controller - */ - - /* - * Move all elements from the queue of pending transactions - * onto the list of running transactions - */ - chan_dbg(chan, "idle, starting controller\n"); - desc = list_first_entry(&chan->ld_pending, struct fsl_desc_sw, node); - list_splice_tail_init(&chan->ld_pending, &chan->ld_running); - - /* - * The 85xx DMA controller doesn't clear the channel start bit - * automatically at the end of a transfer. Therefore we must clear - * it in software before starting the transfer. - */ - if ((chan->feature & FSL_DMA_IP_MASK) == FSL_DMA_IP_85XX) { - u32 mode; - - mode = DMA_IN(chan, &chan->regs->mr, 32); - mode &= ~FSL_DMA_MR_CS; - DMA_OUT(chan, &chan->regs->mr, mode, 32); - } - - /* - * Program the descriptor's address into the DMA controller, - * then start the DMA transaction - */ - set_cdar(chan, desc->async_tx.phys); - get_cdar(chan); - - dma_start(chan); - chan->idle = false; -} - -/** * fsl_dma_memcpy_issue_pending - Issue the DMA start command * @chan : Freescale DMA channel */ @@ -1253,7 +1193,9 @@ static int fsl_dma_chan_probe(struct fsldma_device *fdev, WARN_ON(fdev->feature != chan->feature); chan->dev = fdev->dev; - chan->id = ((res.start - 0x100) & 0xfff) >> 7; + chan->id = (res.start & 0xfff) < 0x300 ? + ((res.start - 0x100) & 0xfff) >> 7 : + ((res.start - 0x200) & 0xfff) >> 7; if (chan->id >= FSL_DMA_MAX_CHANS_PER_DEVICE) { dev_err(fdev->dev, "too many channels for device\n"); err = -EINVAL; @@ -1344,12 +1286,10 @@ static int fsldma_of_probe(struct platform_device *op) fdev->irq = irq_of_parse_and_map(op->dev.of_node, 0); dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask); - dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask); dma_cap_set(DMA_SG, fdev->common.cap_mask); dma_cap_set(DMA_SLAVE, fdev->common.cap_mask); fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources; fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources; - fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt; fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy; fdev->common.device_prep_dma_sg = fsl_dma_prep_sg; fdev->common.device_tx_status = fsl_tx_status; @@ -1426,6 +1366,7 @@ static int fsldma_of_remove(struct platform_device *op) } static const struct of_device_id fsldma_of_ids[] = { + { .compatible = "fsl,elo3-dma", }, { .compatible = "fsl,eloplus-dma", }, { .compatible = "fsl,elo-dma", }, {} @@ -1447,7 +1388,7 @@ static struct platform_driver fsldma_of_driver = { static __init int fsldma_init(void) { - pr_info("Freescale Elo / Elo Plus DMA driver\n"); + pr_info("Freescale Elo series DMA driver\n"); return platform_driver_register(&fsldma_of_driver); } @@ -1459,5 +1400,5 @@ static void __exit fsldma_exit(void) subsys_initcall(fsldma_init); module_exit(fsldma_exit); -MODULE_DESCRIPTION("Freescale Elo / Elo Plus DMA driver"); +MODULE_DESCRIPTION("Freescale Elo series DMA driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/dma/fsldma.h b/drivers/dma/fsldma.h index f5c38791fc7..d56e8359982 100644 --- a/drivers/dma/fsldma.h +++ b/drivers/dma/fsldma.h @@ -41,7 +41,7 @@ * channel is allowed to transfer before the DMA engine pauses * the current channel and switches to the next channel */ -#define FSL_DMA_MR_BWC 0x08000000 +#define FSL_DMA_MR_BWC 0x0A000000 /* Special MR definition for MPC8349 */ #define FSL_DMA_MR_EOTIE 0x00000080 @@ -112,7 +112,7 @@ struct fsldma_chan_regs { }; struct fsldma_chan; -#define FSL_DMA_MAX_CHANS_PER_DEVICE 4 +#define FSL_DMA_MAX_CHANS_PER_DEVICE 8 struct fsldma_device { void __iomem *regs; /* DGSR register base */ diff --git a/drivers/dma/imx-dma.c b/drivers/dma/imx-dma.c index 78f8ca5fcce..286660a12cc 100644 --- a/drivers/dma/imx-dma.c +++ b/drivers/dma/imx-dma.c @@ -422,12 +422,12 @@ static irqreturn_t imxdma_err_handler(int irq, void *dev_id) /* Tasklet error handler */ tasklet_schedule(&imxdma->channel[i].dma_tasklet); - printk(KERN_WARNING - "DMA timeout on channel %d -%s%s%s%s\n", i, - errcode & IMX_DMA_ERR_BURST ? " burst" : "", - errcode & IMX_DMA_ERR_REQUEST ? " request" : "", - errcode & IMX_DMA_ERR_TRANSFER ? " transfer" : "", - errcode & IMX_DMA_ERR_BUFFER ? " buffer" : ""); + dev_warn(imxdma->dev, + "DMA timeout on channel %d -%s%s%s%s\n", i, + errcode & IMX_DMA_ERR_BURST ? " burst" : "", + errcode & IMX_DMA_ERR_REQUEST ? " request" : "", + errcode & IMX_DMA_ERR_TRANSFER ? " transfer" : "", + errcode & IMX_DMA_ERR_BUFFER ? " buffer" : ""); } return IRQ_HANDLED; } @@ -437,17 +437,18 @@ static void dma_irq_handle_channel(struct imxdma_channel *imxdmac) struct imxdma_engine *imxdma = imxdmac->imxdma; int chno = imxdmac->channel; struct imxdma_desc *desc; + unsigned long flags; - spin_lock(&imxdma->lock); + spin_lock_irqsave(&imxdma->lock, flags); if (list_empty(&imxdmac->ld_active)) { - spin_unlock(&imxdma->lock); + spin_unlock_irqrestore(&imxdma->lock, flags); goto out; } desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node); - spin_unlock(&imxdma->lock); + spin_unlock_irqrestore(&imxdma->lock, flags); if (desc->sg) { u32 tmp; @@ -519,7 +520,6 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) { struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan); struct imxdma_engine *imxdma = imxdmac->imxdma; - unsigned long flags; int slot = -1; int i; @@ -527,7 +527,6 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) switch (d->type) { case IMXDMA_DESC_INTERLEAVED: /* Try to get a free 2D slot */ - spin_lock_irqsave(&imxdma->lock, flags); for (i = 0; i < IMX_DMA_2D_SLOTS; i++) { if ((imxdma->slots_2d[i].count > 0) && ((imxdma->slots_2d[i].xsr != d->x) || @@ -537,10 +536,8 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) slot = i; break; } - if (slot < 0) { - spin_unlock_irqrestore(&imxdma->lock, flags); + if (slot < 0) return -EBUSY; - } imxdma->slots_2d[slot].xsr = d->x; imxdma->slots_2d[slot].ysr = d->y; @@ -549,7 +546,6 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) imxdmac->slot_2d = slot; imxdmac->enabled_2d = true; - spin_unlock_irqrestore(&imxdma->lock, flags); if (slot == IMX_DMA_2D_SLOT_A) { d->config_mem &= ~CCR_MSEL_B; @@ -576,9 +572,11 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) imx_dmav1_writel(imxdma, d->len, DMA_CNTR(imxdmac->channel)); - dev_dbg(imxdma->dev, "%s channel: %d dest=0x%08x src=0x%08x " - "dma_length=%d\n", __func__, imxdmac->channel, - d->dest, d->src, d->len); + dev_dbg(imxdma->dev, + "%s channel: %d dest=0x%08llx src=0x%08llx dma_length=%zu\n", + __func__, imxdmac->channel, + (unsigned long long)d->dest, + (unsigned long long)d->src, d->len); break; /* Cyclic transfer is the same as slave_sg with special sg configuration. */ @@ -590,20 +588,22 @@ static int imxdma_xfer_desc(struct imxdma_desc *d) imx_dmav1_writel(imxdma, imxdmac->ccr_from_device, DMA_CCR(imxdmac->channel)); - dev_dbg(imxdma->dev, "%s channel: %d sg=%p sgcount=%d " - "total length=%d dev_addr=0x%08x (dev2mem)\n", - __func__, imxdmac->channel, d->sg, d->sgcount, - d->len, imxdmac->per_address); + dev_dbg(imxdma->dev, + "%s channel: %d sg=%p sgcount=%d total length=%zu dev_addr=0x%08llx (dev2mem)\n", + __func__, imxdmac->channel, + d->sg, d->sgcount, d->len, + (unsigned long long)imxdmac->per_address); } else if (d->direction == DMA_MEM_TO_DEV) { imx_dmav1_writel(imxdma, imxdmac->per_address, DMA_DAR(imxdmac->channel)); imx_dmav1_writel(imxdma, imxdmac->ccr_to_device, DMA_CCR(imxdmac->channel)); - dev_dbg(imxdma->dev, "%s channel: %d sg=%p sgcount=%d " - "total length=%d dev_addr=0x%08x (mem2dev)\n", - __func__, imxdmac->channel, d->sg, d->sgcount, - d->len, imxdmac->per_address); + dev_dbg(imxdma->dev, + "%s channel: %d sg=%p sgcount=%d total length=%zu dev_addr=0x%08llx (mem2dev)\n", + __func__, imxdmac->channel, + d->sg, d->sgcount, d->len, + (unsigned long long)imxdmac->per_address); } else { dev_err(imxdma->dev, "%s channel: %d bad dma mode\n", __func__, imxdmac->channel); @@ -625,18 +625,17 @@ static void imxdma_tasklet(unsigned long data) struct imxdma_channel *imxdmac = (void *)data; struct imxdma_engine *imxdma = imxdmac->imxdma; struct imxdma_desc *desc; + unsigned long flags; - spin_lock(&imxdma->lock); + spin_lock_irqsave(&imxdma->lock, flags); if (list_empty(&imxdmac->ld_active)) { /* Someone might have called terminate all */ - goto out; + spin_unlock_irqrestore(&imxdma->lock, flags); + return; } desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node); - if (desc->desc.callback) - desc->desc.callback(desc->desc.callback_param); - /* If we are dealing with a cyclic descriptor, keep it on ld_active * and dont mark the descriptor as complete. * Only in non-cyclic cases it would be marked as complete @@ -663,7 +662,11 @@ static void imxdma_tasklet(unsigned long data) __func__, imxdmac->channel); } out: - spin_unlock(&imxdma->lock); + spin_unlock_irqrestore(&imxdma->lock, flags); + + if (desc->desc.callback) + desc->desc.callback(desc->desc.callback_param); + } static int imxdma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, @@ -772,7 +775,7 @@ static int imxdma_alloc_chan_resources(struct dma_chan *chan) desc->desc.tx_submit = imxdma_tx_submit; /* txd.flags will be overwritten in prep funcs */ desc->desc.flags = DMA_CTRL_ACK; - desc->status = DMA_SUCCESS; + desc->status = DMA_COMPLETE; list_add_tail(&desc->node, &imxdmac->ld_free); imxdmac->descs_allocated++; @@ -871,7 +874,7 @@ static struct dma_async_tx_descriptor *imxdma_prep_dma_cyclic( int i; unsigned int periods = buf_len / period_len; - dev_dbg(imxdma->dev, "%s channel: %d buf_len=%d period_len=%d\n", + dev_dbg(imxdma->dev, "%s channel: %d buf_len=%zu period_len=%zu\n", __func__, imxdmac->channel, buf_len, period_len); if (list_empty(&imxdmac->ld_free) || @@ -883,7 +886,7 @@ static struct dma_async_tx_descriptor *imxdma_prep_dma_cyclic( kfree(imxdmac->sg_list); imxdmac->sg_list = kcalloc(periods + 1, - sizeof(struct scatterlist), GFP_KERNEL); + sizeof(struct scatterlist), GFP_ATOMIC); if (!imxdmac->sg_list) return NULL; @@ -927,8 +930,9 @@ static struct dma_async_tx_descriptor *imxdma_prep_dma_memcpy( struct imxdma_engine *imxdma = imxdmac->imxdma; struct imxdma_desc *desc; - dev_dbg(imxdma->dev, "%s channel: %d src=0x%x dst=0x%x len=%d\n", - __func__, imxdmac->channel, src, dest, len); + dev_dbg(imxdma->dev, "%s channel: %d src=0x%llx dst=0x%llx len=%zu\n", + __func__, imxdmac->channel, (unsigned long long)src, + (unsigned long long)dest, len); if (list_empty(&imxdmac->ld_free) || imxdma_chan_is_doing_cyclic(imxdmac)) @@ -957,9 +961,10 @@ static struct dma_async_tx_descriptor *imxdma_prep_dma_interleaved( struct imxdma_engine *imxdma = imxdmac->imxdma; struct imxdma_desc *desc; - dev_dbg(imxdma->dev, "%s channel: %d src_start=0x%x dst_start=0x%x\n" - " src_sgl=%s dst_sgl=%s numf=%d frame_size=%d\n", __func__, - imxdmac->channel, xt->src_start, xt->dst_start, + dev_dbg(imxdma->dev, "%s channel: %d src_start=0x%llx dst_start=0x%llx\n" + " src_sgl=%s dst_sgl=%s numf=%zu frame_size=%zu\n", __func__, + imxdmac->channel, (unsigned long long)xt->src_start, + (unsigned long long) xt->dst_start, xt->src_sgl ? "true" : "false", xt->dst_sgl ? "true" : "false", xt->numf, xt->frame_size); @@ -1231,6 +1236,7 @@ static int imxdma_remove(struct platform_device *pdev) static struct platform_driver imxdma_driver = { .driver = { .name = "imx-dma", + .owner = THIS_MODULE, .of_match_table = imx_dma_of_dev_id, }, .id_table = imx_dma_devtype, diff --git a/drivers/dma/imx-sdma.c b/drivers/dma/imx-sdma.c index fc43603cf0b..14867e3ac8f 100644 --- a/drivers/dma/imx-sdma.c +++ b/drivers/dma/imx-sdma.c @@ -255,6 +255,7 @@ struct sdma_channel { enum dma_slave_buswidth word_size; unsigned int buf_tail; unsigned int num_bd; + unsigned int period_len; struct sdma_buffer_descriptor *bd; dma_addr_t bd_phys; unsigned int pc_from_device, pc_to_device; @@ -323,6 +324,7 @@ struct sdma_engine { struct clk *clk_ipg; struct clk *clk_ahb; spinlock_t channel_0_lock; + u32 script_number; struct sdma_script_start_addrs *script_addrs; const struct sdma_driver_data *drvdata; }; @@ -448,6 +450,7 @@ static const struct of_device_id sdma_dt_ids[] = { { .compatible = "fsl,imx51-sdma", .data = &sdma_imx51, }, { .compatible = "fsl,imx35-sdma", .data = &sdma_imx35, }, { .compatible = "fsl,imx31-sdma", .data = &sdma_imx31, }, + { .compatible = "fsl,imx25-sdma", .data = &sdma_imx25, }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, sdma_dt_ids); @@ -591,6 +594,12 @@ static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event) static void sdma_handle_channel_loop(struct sdma_channel *sdmac) { + if (sdmac->desc.callback) + sdmac->desc.callback(sdmac->desc.callback_param); +} + +static void sdma_update_channel_loop(struct sdma_channel *sdmac) +{ struct sdma_buffer_descriptor *bd; /* @@ -605,15 +614,10 @@ static void sdma_handle_channel_loop(struct sdma_channel *sdmac) if (bd->mode.status & BD_RROR) sdmac->status = DMA_ERROR; - else - sdmac->status = DMA_IN_PROGRESS; bd->mode.status |= BD_DONE; sdmac->buf_tail++; sdmac->buf_tail %= sdmac->num_bd; - - if (sdmac->desc.callback) - sdmac->desc.callback(sdmac->desc.callback_param); } } @@ -638,7 +642,7 @@ static void mxc_sdma_handle_channel_normal(struct sdma_channel *sdmac) if (error) sdmac->status = DMA_ERROR; else - sdmac->status = DMA_SUCCESS; + sdmac->status = DMA_COMPLETE; dma_cookie_complete(&sdmac->desc); if (sdmac->desc.callback) @@ -669,6 +673,9 @@ static irqreturn_t sdma_int_handler(int irq, void *dev_id) int channel = fls(stat) - 1; struct sdma_channel *sdmac = &sdma->channel[channel]; + if (sdmac->flags & IMX_DMA_SG_LOOP) + sdma_update_channel_loop(sdmac); + tasklet_schedule(&sdmac->tasklet); __clear_bit(channel, &stat); @@ -724,6 +731,10 @@ static void sdma_get_pc(struct sdma_channel *sdmac, per_2_emi = sdma->script_addrs->app_2_mcu_addr; emi_2_per = sdma->script_addrs->mcu_2_app_addr; break; + case IMX_DMATYPE_SSI_DUAL: + per_2_emi = sdma->script_addrs->ssish_2_mcu_addr; + emi_2_per = sdma->script_addrs->mcu_2_ssish_addr; + break; case IMX_DMATYPE_SSI_SP: case IMX_DMATYPE_MMC: case IMX_DMATYPE_SDHC: @@ -1089,8 +1100,8 @@ static struct dma_async_tx_descriptor *sdma_prep_slave_sg( param &= ~BD_CONT; } - dev_dbg(sdma->dev, "entry %d: count: %d dma: 0x%08x %s%s\n", - i, count, sg->dma_address, + dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n", + i, count, (u64)sg->dma_address, param & BD_WRAP ? "wrap" : "", param & BD_INTR ? " intr" : ""); @@ -1125,6 +1136,7 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( sdmac->status = DMA_IN_PROGRESS; sdmac->buf_tail = 0; + sdmac->period_len = period_len; sdmac->flags |= IMX_DMA_SG_LOOP; sdmac->direction = direction; @@ -1163,8 +1175,8 @@ static struct dma_async_tx_descriptor *sdma_prep_dma_cyclic( if (i + 1 == num_periods) param |= BD_WRAP; - dev_dbg(sdma->dev, "entry %d: count: %d dma: 0x%08x %s%s\n", - i, period_len, dma_addr, + dev_dbg(sdma->dev, "entry %d: count: %d dma: %#llx %s%s\n", + i, period_len, (u64)dma_addr, param & BD_WRAP ? "wrap" : "", param & BD_INTR ? " intr" : ""); @@ -1221,9 +1233,15 @@ static enum dma_status sdma_tx_status(struct dma_chan *chan, struct dma_tx_state *txstate) { struct sdma_channel *sdmac = to_sdma_chan(chan); + u32 residue; + + if (sdmac->flags & IMX_DMA_SG_LOOP) + residue = (sdmac->num_bd - sdmac->buf_tail) * sdmac->period_len; + else + residue = sdmac->chn_count - sdmac->chn_real_count; dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, - sdmac->chn_count - sdmac->chn_real_count); + residue); return sdmac->status; } @@ -1238,6 +1256,7 @@ static void sdma_issue_pending(struct dma_chan *chan) } #define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1 34 +#define SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2 38 static void sdma_add_scripts(struct sdma_engine *sdma, const struct sdma_script_start_addrs *addr) @@ -1246,7 +1265,11 @@ static void sdma_add_scripts(struct sdma_engine *sdma, s32 *saddr_arr = (u32 *)sdma->script_addrs; int i; - for (i = 0; i < SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; i++) + /* use the default firmware in ROM if missing external firmware */ + if (!sdma->script_number) + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; + + for (i = 0; i < sdma->script_number; i++) if (addr_arr[i] > 0) saddr_arr[i] = addr_arr[i]; } @@ -1272,6 +1295,17 @@ static void sdma_load_firmware(const struct firmware *fw, void *context) goto err_firmware; if (header->ram_code_start + header->ram_code_size > fw->size) goto err_firmware; + switch (header->version_major) { + case 1: + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V1; + break; + case 2: + sdma->script_number = SDMA_SCRIPT_ADDRS_ARRAY_SIZE_V2; + break; + default: + dev_err(sdma->dev, "unknown firmware version\n"); + goto err_firmware; + } addr = (void *)header + header->script_addrs_start; ram_code = (void *)header + header->ram_code_start; @@ -1432,6 +1466,10 @@ static int __init sdma_probe(struct platform_device *pdev) return -EINVAL; } + ret = dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + sdma = kzalloc(sizeof(*sdma), GFP_KERNEL); if (!sdma) return -ENOMEM; diff --git a/drivers/dma/intel_mid_dma.c b/drivers/dma/intel_mid_dma.c index a975ebebea8..1aab8130efa 100644 --- a/drivers/dma/intel_mid_dma.c +++ b/drivers/dma/intel_mid_dma.c @@ -309,7 +309,7 @@ static void midc_descriptor_complete(struct intel_mid_dma_chan *midc, callback_txd(param_txd); } if (midc->raw_tfr) { - desc->status = DMA_SUCCESS; + desc->status = DMA_COMPLETE; if (desc->lli != NULL) { pci_pool_free(desc->lli_pool, desc->lli, desc->lli_phys); @@ -481,7 +481,7 @@ static enum dma_status intel_mid_dma_tx_status(struct dma_chan *chan, enum dma_status ret; ret = dma_cookie_status(chan, cookie, txstate); - if (ret != DMA_SUCCESS) { + if (ret != DMA_COMPLETE) { spin_lock_bh(&midc->lock); midc_scan_descriptors(to_middma_device(chan->device), midc); spin_unlock_bh(&midc->lock); diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c index 5ff6fc1819d..4e3549a1613 100644 --- a/drivers/dma/ioat/dma.c +++ b/drivers/dma/ioat/dma.c @@ -77,7 +77,8 @@ static irqreturn_t ioat_dma_do_interrupt(int irq, void *data) attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET); for_each_set_bit(bit, &attnstatus, BITS_PER_LONG) { chan = ioat_chan_by_index(instance, bit); - tasklet_schedule(&chan->cleanup_task); + if (test_bit(IOAT_RUN, &chan->state)) + tasklet_schedule(&chan->cleanup_task); } writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET); @@ -93,7 +94,8 @@ static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data) { struct ioat_chan_common *chan = data; - tasklet_schedule(&chan->cleanup_task); + if (test_bit(IOAT_RUN, &chan->state)) + tasklet_schedule(&chan->cleanup_task); return IRQ_HANDLED; } @@ -116,7 +118,6 @@ void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *c chan->timer.function = device->timer_fn; chan->timer.data = data; tasklet_init(&chan->cleanup_task, device->cleanup_fn, data); - tasklet_disable(&chan->cleanup_task); } /** @@ -354,13 +355,49 @@ static int ioat1_dma_alloc_chan_resources(struct dma_chan *c) writel(((u64) chan->completion_dma) >> 32, chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH); - tasklet_enable(&chan->cleanup_task); + set_bit(IOAT_RUN, &chan->state); ioat1_dma_start_null_desc(ioat); /* give chain to dma device */ dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n", __func__, ioat->desccount); return ioat->desccount; } +void ioat_stop(struct ioat_chan_common *chan) +{ + struct ioatdma_device *device = chan->device; + struct pci_dev *pdev = device->pdev; + int chan_id = chan_num(chan); + struct msix_entry *msix; + + /* 1/ stop irq from firing tasklets + * 2/ stop the tasklet from re-arming irqs + */ + clear_bit(IOAT_RUN, &chan->state); + + /* flush inflight interrupts */ + switch (device->irq_mode) { + case IOAT_MSIX: + msix = &device->msix_entries[chan_id]; + synchronize_irq(msix->vector); + break; + case IOAT_MSI: + case IOAT_INTX: + synchronize_irq(pdev->irq); + break; + default: + break; + } + + /* flush inflight timers */ + del_timer_sync(&chan->timer); + + /* flush inflight tasklet runs */ + tasklet_kill(&chan->cleanup_task); + + /* final cleanup now that everything is quiesced and can't re-arm */ + device->cleanup_fn((unsigned long) &chan->common); +} + /** * ioat1_dma_free_chan_resources - release all the descriptors * @chan: the channel to be cleaned @@ -379,9 +416,7 @@ static void ioat1_dma_free_chan_resources(struct dma_chan *c) if (ioat->desccount == 0) return; - tasklet_disable(&chan->cleanup_task); - del_timer_sync(&chan->timer); - ioat1_cleanup(ioat); + ioat_stop(chan); /* Delay 100ms after reset to allow internal DMA logic to quiesce * before removing DMA descriptor resources. @@ -526,26 +561,14 @@ ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest, static void ioat1_cleanup_event(unsigned long data) { struct ioat_dma_chan *ioat = to_ioat_chan((void *) data); + struct ioat_chan_common *chan = &ioat->base; ioat1_cleanup(ioat); + if (!test_bit(IOAT_RUN, &chan->state)) + return; writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); } -void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags, - size_t len, struct ioat_dma_descriptor *hw) -{ - struct pci_dev *pdev = chan->device->pdev; - size_t offset = len - hw->size; - - if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) - ioat_unmap(pdev, hw->dst_addr - offset, len, - PCI_DMA_FROMDEVICE, flags, 1); - - if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) - ioat_unmap(pdev, hw->src_addr - offset, len, - PCI_DMA_TODEVICE, flags, 0); -} - dma_addr_t ioat_get_current_completion(struct ioat_chan_common *chan) { dma_addr_t phys_complete; @@ -602,7 +625,7 @@ static void __cleanup(struct ioat_dma_chan *ioat, dma_addr_t phys_complete) dump_desc_dbg(ioat, desc); if (tx->cookie) { dma_cookie_complete(tx); - ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw); + dma_descriptor_unmap(tx); ioat->active -= desc->hw->tx_cnt; if (tx->callback) { tx->callback(tx->callback_param); @@ -733,7 +756,7 @@ ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, enum dma_status ret; ret = dma_cookie_status(c, cookie, txstate); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; device->cleanup_fn((unsigned long) c); @@ -832,9 +855,16 @@ int ioat_dma_self_test(struct ioatdma_device *device) } dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma_src)) { + dev_err(dev, "mapping src buffer failed\n"); + goto free_resources; + } dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); - flags = DMA_COMPL_SKIP_SRC_UNMAP | DMA_COMPL_SKIP_DEST_UNMAP | - DMA_PREP_INTERRUPT; + if (dma_mapping_error(dev, dma_dest)) { + dev_err(dev, "mapping dest buffer failed\n"); + goto unmap_src; + } + flags = DMA_PREP_INTERRUPT; tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src, IOAT_TEST_SIZE, flags); if (!tx) { @@ -859,7 +889,7 @@ int ioat_dma_self_test(struct ioatdma_device *device) if (tmo == 0 || dma->device_tx_status(dma_chan, cookie, NULL) - != DMA_SUCCESS) { + != DMA_COMPLETE) { dev_err(dev, "Self-test copy timed out, disabling\n"); err = -ENODEV; goto unmap_dma; @@ -871,8 +901,9 @@ int ioat_dma_self_test(struct ioatdma_device *device) } unmap_dma: - dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE); dma_unmap_single(dev, dma_dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE); +unmap_src: + dma_unmap_single(dev, dma_src, IOAT_TEST_SIZE, DMA_TO_DEVICE); free_resources: dma->device_free_chan_resources(dma_chan); out: @@ -885,8 +916,7 @@ static char ioat_interrupt_style[32] = "msix"; module_param_string(ioat_interrupt_style, ioat_interrupt_style, sizeof(ioat_interrupt_style), 0644); MODULE_PARM_DESC(ioat_interrupt_style, - "set ioat interrupt style: msix (default), " - "msix-single-vector, msi, intx)"); + "set ioat interrupt style: msix (default), msi, intx"); /** * ioat_dma_setup_interrupts - setup interrupt handler @@ -904,8 +934,6 @@ int ioat_dma_setup_interrupts(struct ioatdma_device *device) if (!strcmp(ioat_interrupt_style, "msix")) goto msix; - if (!strcmp(ioat_interrupt_style, "msix-single-vector")) - goto msix_single_vector; if (!strcmp(ioat_interrupt_style, "msi")) goto msi; if (!strcmp(ioat_interrupt_style, "intx")) @@ -920,10 +948,8 @@ msix: device->msix_entries[i].entry = i; err = pci_enable_msix(pdev, device->msix_entries, msixcnt); - if (err < 0) + if (err) goto msi; - if (err > 0) - goto msix_single_vector; for (i = 0; i < msixcnt; i++) { msix = &device->msix_entries[i]; @@ -937,29 +963,13 @@ msix: chan = ioat_chan_by_index(device, j); devm_free_irq(dev, msix->vector, chan); } - goto msix_single_vector; + goto msi; } } intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL; device->irq_mode = IOAT_MSIX; goto done; -msix_single_vector: - msix = &device->msix_entries[0]; - msix->entry = 0; - err = pci_enable_msix(pdev, device->msix_entries, 1); - if (err) - goto msi; - - err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0, - "ioat-msix", device); - if (err) { - pci_disable_msix(pdev); - goto msi; - } - device->irq_mode = IOAT_MSIX_SINGLE; - goto done; - msi: err = pci_enable_msi(pdev); if (err) @@ -971,7 +981,7 @@ msi: pci_disable_msi(pdev); goto intx; } - device->irq_mode = IOAT_MSIX; + device->irq_mode = IOAT_MSI; goto done; intx: diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h index 54fb7b9ff9a..e982f00a984 100644 --- a/drivers/dma/ioat/dma.h +++ b/drivers/dma/ioat/dma.h @@ -52,7 +52,6 @@ enum ioat_irq_mode { IOAT_NOIRQ = 0, IOAT_MSIX, - IOAT_MSIX_SINGLE, IOAT_MSI, IOAT_INTX }; @@ -83,7 +82,6 @@ struct ioatdma_device { struct pci_pool *completion_pool; #define MAX_SED_POOLS 5 struct dma_pool *sed_hw_pool[MAX_SED_POOLS]; - struct kmem_cache *sed_pool; struct dma_device common; u8 version; struct msix_entry msix_entries[4]; @@ -342,16 +340,6 @@ static inline bool is_ioat_bug(unsigned long err) return !!err; } -static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len, - int direction, enum dma_ctrl_flags flags, bool dst) -{ - if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) || - (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE))) - pci_unmap_single(pdev, addr, len, direction); - else - pci_unmap_page(pdev, addr, len, direction); -} - int ioat_probe(struct ioatdma_device *device); int ioat_register(struct ioatdma_device *device); int ioat1_dma_probe(struct ioatdma_device *dev, int dca); @@ -363,13 +351,12 @@ void ioat_init_channel(struct ioatdma_device *device, struct ioat_chan_common *chan, int idx); enum dma_status ioat_dma_tx_status(struct dma_chan *c, dma_cookie_t cookie, struct dma_tx_state *txstate); -void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags, - size_t len, struct ioat_dma_descriptor *hw); bool ioat_cleanup_preamble(struct ioat_chan_common *chan, dma_addr_t *phys_complete); void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type); void ioat_kobject_del(struct ioatdma_device *device); int ioat_dma_setup_interrupts(struct ioatdma_device *device); +void ioat_stop(struct ioat_chan_common *chan); extern const struct sysfs_ops ioat_sysfs_ops; extern struct ioat_sysfs_entry ioat_version_attr; extern struct ioat_sysfs_entry ioat_cap_attr; diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c index b925e1b1d13..8d1058085ee 100644 --- a/drivers/dma/ioat/dma_v2.c +++ b/drivers/dma/ioat/dma_v2.c @@ -148,7 +148,7 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete) tx = &desc->txd; dump_desc_dbg(ioat, desc); if (tx->cookie) { - ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw); + dma_descriptor_unmap(tx); dma_cookie_complete(tx); if (tx->callback) { tx->callback(tx->callback_param); @@ -190,8 +190,11 @@ static void ioat2_cleanup(struct ioat2_dma_chan *ioat) void ioat2_cleanup_event(unsigned long data) { struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); + struct ioat_chan_common *chan = &ioat->base; ioat2_cleanup(ioat); + if (!test_bit(IOAT_RUN, &chan->state)) + return; writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); } @@ -553,10 +556,10 @@ int ioat2_alloc_chan_resources(struct dma_chan *c) ioat->issued = 0; ioat->tail = 0; ioat->alloc_order = order; + set_bit(IOAT_RUN, &chan->state); spin_unlock_bh(&ioat->prep_lock); spin_unlock_bh(&chan->cleanup_lock); - tasklet_enable(&chan->cleanup_task); ioat2_start_null_desc(ioat); /* check that we got off the ground */ @@ -566,7 +569,6 @@ int ioat2_alloc_chan_resources(struct dma_chan *c) } while (i++ < 20 && !is_ioat_active(status) && !is_ioat_idle(status)); if (is_ioat_active(status) || is_ioat_idle(status)) { - set_bit(IOAT_RUN, &chan->state); return 1 << ioat->alloc_order; } else { u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET); @@ -809,11 +811,8 @@ void ioat2_free_chan_resources(struct dma_chan *c) if (!ioat->ring) return; - tasklet_disable(&chan->cleanup_task); - del_timer_sync(&chan->timer); - device->cleanup_fn((unsigned long) c); + ioat_stop(chan); device->reset_hw(chan); - clear_bit(IOAT_RUN, &chan->state); spin_lock_bh(&chan->cleanup_lock); spin_lock_bh(&ioat->prep_lock); diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h index 212d584fe42..470292767e6 100644 --- a/drivers/dma/ioat/dma_v2.h +++ b/drivers/dma/ioat/dma_v2.h @@ -157,7 +157,6 @@ static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr) int ioat2_dma_probe(struct ioatdma_device *dev, int dca); int ioat3_dma_probe(struct ioatdma_device *dev, int dca); -void ioat3_dma_remove(struct ioatdma_device *dev); struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase); struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase); int ioat2_check_space_lock(struct ioat2_dma_chan *ioat, int num_descs); diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c index d8ececaf1b5..b9b38a1cf92 100644 --- a/drivers/dma/ioat/dma_v3.c +++ b/drivers/dma/ioat/dma_v3.c @@ -67,6 +67,8 @@ #include "dma.h" #include "dma_v2.h" +extern struct kmem_cache *ioat3_sed_cache; + /* ioat hardware assumes at least two sources for raid operations */ #define src_cnt_to_sw(x) ((x) + 2) #define src_cnt_to_hw(x) ((x) - 2) @@ -87,22 +89,8 @@ static const u8 pq_idx_to_field[] = { 1, 4, 5, 0, 1, 2, 4, 5 }; static const u8 pq16_idx_to_field[] = { 1, 4, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6 }; -/* - * technically sources 1 and 2 do not require SED, but the op will have - * at least 9 descriptors so that's irrelevant. - */ -static const u8 pq16_idx_to_sed[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, 1 }; - static void ioat3_eh(struct ioat2_dma_chan *ioat); -static dma_addr_t xor_get_src(struct ioat_raw_descriptor *descs[2], int idx) -{ - struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1]; - - return raw->field[xor_idx_to_field[idx]]; -} - static void xor_set_src(struct ioat_raw_descriptor *descs[2], dma_addr_t addr, u32 offset, int idx) { @@ -135,12 +123,6 @@ static void pq_set_src(struct ioat_raw_descriptor *descs[2], pq->coef[idx] = coef; } -static int sed_get_pq16_pool_idx(int src_cnt) -{ - - return pq16_idx_to_sed[src_cnt]; -} - static bool is_jf_ioat(struct pci_dev *pdev) { switch (pdev->device) { @@ -272,7 +254,7 @@ ioat3_alloc_sed(struct ioatdma_device *device, unsigned int hw_pool) struct ioat_sed_ent *sed; gfp_t flags = __GFP_ZERO | GFP_ATOMIC; - sed = kmem_cache_alloc(device->sed_pool, flags); + sed = kmem_cache_alloc(ioat3_sed_cache, flags); if (!sed) return NULL; @@ -280,7 +262,7 @@ ioat3_alloc_sed(struct ioatdma_device *device, unsigned int hw_pool) sed->hw = dma_pool_alloc(device->sed_hw_pool[hw_pool], flags, &sed->dma); if (!sed->hw) { - kmem_cache_free(device->sed_pool, sed); + kmem_cache_free(ioat3_sed_cache, sed); return NULL; } @@ -293,165 +275,7 @@ static void ioat3_free_sed(struct ioatdma_device *device, struct ioat_sed_ent *s return; dma_pool_free(device->sed_hw_pool[sed->hw_pool], sed->hw, sed->dma); - kmem_cache_free(device->sed_pool, sed); -} - -static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat, - struct ioat_ring_ent *desc, int idx) -{ - struct ioat_chan_common *chan = &ioat->base; - struct pci_dev *pdev = chan->device->pdev; - size_t len = desc->len; - size_t offset = len - desc->hw->size; - struct dma_async_tx_descriptor *tx = &desc->txd; - enum dma_ctrl_flags flags = tx->flags; - - switch (desc->hw->ctl_f.op) { - case IOAT_OP_COPY: - if (!desc->hw->ctl_f.null) /* skip 'interrupt' ops */ - ioat_dma_unmap(chan, flags, len, desc->hw); - break; - case IOAT_OP_XOR_VAL: - case IOAT_OP_XOR: { - struct ioat_xor_descriptor *xor = desc->xor; - struct ioat_ring_ent *ext; - struct ioat_xor_ext_descriptor *xor_ex = NULL; - int src_cnt = src_cnt_to_sw(xor->ctl_f.src_cnt); - struct ioat_raw_descriptor *descs[2]; - int i; - - if (src_cnt > 5) { - ext = ioat2_get_ring_ent(ioat, idx + 1); - xor_ex = ext->xor_ex; - } - - if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - descs[0] = (struct ioat_raw_descriptor *) xor; - descs[1] = (struct ioat_raw_descriptor *) xor_ex; - for (i = 0; i < src_cnt; i++) { - dma_addr_t src = xor_get_src(descs, i); - - ioat_unmap(pdev, src - offset, len, - PCI_DMA_TODEVICE, flags, 0); - } - - /* dest is a source in xor validate operations */ - if (xor->ctl_f.op == IOAT_OP_XOR_VAL) { - ioat_unmap(pdev, xor->dst_addr - offset, len, - PCI_DMA_TODEVICE, flags, 1); - break; - } - } - - if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) - ioat_unmap(pdev, xor->dst_addr - offset, len, - PCI_DMA_FROMDEVICE, flags, 1); - break; - } - case IOAT_OP_PQ_VAL: - case IOAT_OP_PQ: { - struct ioat_pq_descriptor *pq = desc->pq; - struct ioat_ring_ent *ext; - struct ioat_pq_ext_descriptor *pq_ex = NULL; - int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt); - struct ioat_raw_descriptor *descs[2]; - int i; - - if (src_cnt > 3) { - ext = ioat2_get_ring_ent(ioat, idx + 1); - pq_ex = ext->pq_ex; - } - - /* in the 'continue' case don't unmap the dests as sources */ - if (dmaf_p_disabled_continue(flags)) - src_cnt--; - else if (dmaf_continue(flags)) - src_cnt -= 3; - - if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - descs[0] = (struct ioat_raw_descriptor *) pq; - descs[1] = (struct ioat_raw_descriptor *) pq_ex; - for (i = 0; i < src_cnt; i++) { - dma_addr_t src = pq_get_src(descs, i); - - ioat_unmap(pdev, src - offset, len, - PCI_DMA_TODEVICE, flags, 0); - } - - /* the dests are sources in pq validate operations */ - if (pq->ctl_f.op == IOAT_OP_XOR_VAL) { - if (!(flags & DMA_PREP_PQ_DISABLE_P)) - ioat_unmap(pdev, pq->p_addr - offset, - len, PCI_DMA_TODEVICE, flags, 0); - if (!(flags & DMA_PREP_PQ_DISABLE_Q)) - ioat_unmap(pdev, pq->q_addr - offset, - len, PCI_DMA_TODEVICE, flags, 0); - break; - } - } - - if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (!(flags & DMA_PREP_PQ_DISABLE_P)) - ioat_unmap(pdev, pq->p_addr - offset, len, - PCI_DMA_BIDIRECTIONAL, flags, 1); - if (!(flags & DMA_PREP_PQ_DISABLE_Q)) - ioat_unmap(pdev, pq->q_addr - offset, len, - PCI_DMA_BIDIRECTIONAL, flags, 1); - } - break; - } - case IOAT_OP_PQ_16S: - case IOAT_OP_PQ_VAL_16S: { - struct ioat_pq_descriptor *pq = desc->pq; - int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt); - struct ioat_raw_descriptor *descs[4]; - int i; - - /* in the 'continue' case don't unmap the dests as sources */ - if (dmaf_p_disabled_continue(flags)) - src_cnt--; - else if (dmaf_continue(flags)) - src_cnt -= 3; - - if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - descs[0] = (struct ioat_raw_descriptor *)pq; - descs[1] = (struct ioat_raw_descriptor *)(desc->sed->hw); - descs[2] = (struct ioat_raw_descriptor *)(&desc->sed->hw->b[0]); - for (i = 0; i < src_cnt; i++) { - dma_addr_t src = pq16_get_src(descs, i); - - ioat_unmap(pdev, src - offset, len, - PCI_DMA_TODEVICE, flags, 0); - } - - /* the dests are sources in pq validate operations */ - if (pq->ctl_f.op == IOAT_OP_XOR_VAL) { - if (!(flags & DMA_PREP_PQ_DISABLE_P)) - ioat_unmap(pdev, pq->p_addr - offset, - len, PCI_DMA_TODEVICE, - flags, 0); - if (!(flags & DMA_PREP_PQ_DISABLE_Q)) - ioat_unmap(pdev, pq->q_addr - offset, - len, PCI_DMA_TODEVICE, - flags, 0); - break; - } - } - - if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (!(flags & DMA_PREP_PQ_DISABLE_P)) - ioat_unmap(pdev, pq->p_addr - offset, len, - PCI_DMA_BIDIRECTIONAL, flags, 1); - if (!(flags & DMA_PREP_PQ_DISABLE_Q)) - ioat_unmap(pdev, pq->q_addr - offset, len, - PCI_DMA_BIDIRECTIONAL, flags, 1); - } - break; - } - default: - dev_err(&pdev->dev, "%s: unknown op type: %#x\n", - __func__, desc->hw->ctl_f.op); - } + kmem_cache_free(ioat3_sed_cache, sed); } static bool desc_has_ext(struct ioat_ring_ent *desc) @@ -577,7 +401,7 @@ static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete) tx = &desc->txd; if (tx->cookie) { dma_cookie_complete(tx); - ioat3_dma_unmap(ioat, desc, idx + i); + dma_descriptor_unmap(tx); if (tx->callback) { tx->callback(tx->callback_param); tx->callback = NULL; @@ -640,8 +464,11 @@ static void ioat3_cleanup(struct ioat2_dma_chan *ioat) static void ioat3_cleanup_event(unsigned long data) { struct ioat2_dma_chan *ioat = to_ioat2_chan((void *) data); + struct ioat_chan_common *chan = &ioat->base; ioat3_cleanup(ioat); + if (!test_bit(IOAT_RUN, &chan->state)) + return; writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET); } @@ -807,7 +634,7 @@ ioat3_tx_status(struct dma_chan *c, dma_cookie_t cookie, enum dma_status ret; ret = dma_cookie_status(c, cookie, txstate); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; ioat3_cleanup(ioat); @@ -1129,9 +956,6 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, u8 op; int i, s, idx, num_descs; - /* this function only handles src_cnt 9 - 16 */ - BUG_ON(src_cnt < 9); - /* this function is only called with 9-16 sources */ op = result ? IOAT_OP_PQ_VAL_16S : IOAT_OP_PQ_16S; @@ -1159,8 +983,7 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, descs[0] = (struct ioat_raw_descriptor *) pq; - desc->sed = ioat3_alloc_sed(device, - sed_get_pq16_pool_idx(src_cnt)); + desc->sed = ioat3_alloc_sed(device, (src_cnt-2) >> 3); if (!desc->sed) { dev_err(to_dev(chan), "%s: no free sed entries\n", __func__); @@ -1218,13 +1041,21 @@ __ioat3_prep_pq16_lock(struct dma_chan *c, enum sum_check_flags *result, return &desc->txd; } +static int src_cnt_flags(unsigned int src_cnt, unsigned long flags) +{ + if (dmaf_p_disabled_continue(flags)) + return src_cnt + 1; + else if (dmaf_continue(flags)) + return src_cnt + 3; + else + return src_cnt; +} + static struct dma_async_tx_descriptor * ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, unsigned long flags) { - struct dma_device *dma = chan->device; - /* specify valid address for disabled result */ if (flags & DMA_PREP_PQ_DISABLE_P) dst[0] = dst[1]; @@ -1244,7 +1075,7 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, single_source_coef[0] = scf[0]; single_source_coef[1] = 0; - return (src_cnt > 8) && (dma->max_pq > 8) ? + return src_cnt_flags(src_cnt, flags) > 8 ? __ioat3_prep_pq16_lock(chan, NULL, dst, single_source, 2, single_source_coef, len, flags) : @@ -1252,7 +1083,7 @@ ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, single_source_coef, len, flags); } else { - return (src_cnt > 8) && (dma->max_pq > 8) ? + return src_cnt_flags(src_cnt, flags) > 8 ? __ioat3_prep_pq16_lock(chan, NULL, dst, src, src_cnt, scf, len, flags) : __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, @@ -1265,8 +1096,6 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, unsigned int src_cnt, const unsigned char *scf, size_t len, enum sum_check_flags *pqres, unsigned long flags) { - struct dma_device *dma = chan->device; - /* specify valid address for disabled result */ if (flags & DMA_PREP_PQ_DISABLE_P) pq[0] = pq[1]; @@ -1278,7 +1107,7 @@ ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, */ *pqres = 0; - return (src_cnt > 8) && (dma->max_pq > 8) ? + return src_cnt_flags(src_cnt, flags) > 8 ? __ioat3_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len, flags) : __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len, @@ -1289,7 +1118,6 @@ static struct dma_async_tx_descriptor * ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, unsigned int src_cnt, size_t len, unsigned long flags) { - struct dma_device *dma = chan->device; unsigned char scf[src_cnt]; dma_addr_t pq[2]; @@ -1298,7 +1126,7 @@ ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, flags |= DMA_PREP_PQ_DISABLE_Q; pq[1] = dst; /* specify valid address for disabled result */ - return (src_cnt > 8) && (dma->max_pq > 8) ? + return src_cnt_flags(src_cnt, flags) > 8 ? __ioat3_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len, flags) : __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len, @@ -1310,7 +1138,6 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, size_t len, enum sum_check_flags *result, unsigned long flags) { - struct dma_device *dma = chan->device; unsigned char scf[src_cnt]; dma_addr_t pq[2]; @@ -1324,8 +1151,7 @@ ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src, flags |= DMA_PREP_PQ_DISABLE_Q; pq[1] = pq[0]; /* specify valid address for disabled result */ - - return (src_cnt > 8) && (dma->max_pq > 8) ? + return src_cnt_flags(src_cnt, flags) > 8 ? __ioat3_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1, scf, len, flags) : __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, @@ -1444,9 +1270,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) DMA_TO_DEVICE); tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs, IOAT_NUM_SRC_TEST, PAGE_SIZE, - DMA_PREP_INTERRUPT | - DMA_COMPL_SKIP_SRC_UNMAP | - DMA_COMPL_SKIP_DEST_UNMAP); + DMA_PREP_INTERRUPT); if (!tx) { dev_err(dev, "Self-test xor prep failed\n"); @@ -1468,7 +1292,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { + if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { dev_err(dev, "Self-test xor timed out\n"); err = -ENODEV; goto dma_unmap; @@ -1507,9 +1331,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) DMA_TO_DEVICE); tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, - &xor_val_result, DMA_PREP_INTERRUPT | - DMA_COMPL_SKIP_SRC_UNMAP | - DMA_COMPL_SKIP_DEST_UNMAP); + &xor_val_result, DMA_PREP_INTERRUPT); if (!tx) { dev_err(dev, "Self-test zero prep failed\n"); err = -ENODEV; @@ -1530,7 +1352,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { + if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { dev_err(dev, "Self-test validate timed out\n"); err = -ENODEV; goto dma_unmap; @@ -1545,6 +1367,8 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) goto free_resources; } + memset(page_address(dest), 0, PAGE_SIZE); + /* test for non-zero parity sum */ op = IOAT_OP_XOR_VAL; @@ -1554,9 +1378,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) DMA_TO_DEVICE); tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs, IOAT_NUM_SRC_TEST + 1, PAGE_SIZE, - &xor_val_result, DMA_PREP_INTERRUPT | - DMA_COMPL_SKIP_SRC_UNMAP | - DMA_COMPL_SKIP_DEST_UNMAP); + &xor_val_result, DMA_PREP_INTERRUPT); if (!tx) { dev_err(dev, "Self-test 2nd zero prep failed\n"); err = -ENODEV; @@ -1577,7 +1399,7 @@ static int ioat_xor_val_self_test(struct ioatdma_device *device) tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)); - if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { + if (dma->device_tx_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { dev_err(dev, "Self-test 2nd validate timed out\n"); err = -ENODEV; goto dma_unmap; @@ -1630,52 +1452,36 @@ static int ioat3_dma_self_test(struct ioatdma_device *device) static int ioat3_irq_reinit(struct ioatdma_device *device) { - int msixcnt = device->common.chancnt; struct pci_dev *pdev = device->pdev; - int i; - struct msix_entry *msix; - struct ioat_chan_common *chan; - int err = 0; + int irq = pdev->irq, i; + + if (!is_bwd_ioat(pdev)) + return 0; switch (device->irq_mode) { case IOAT_MSIX: + for (i = 0; i < device->common.chancnt; i++) { + struct msix_entry *msix = &device->msix_entries[i]; + struct ioat_chan_common *chan; - for (i = 0; i < msixcnt; i++) { - msix = &device->msix_entries[i]; chan = ioat_chan_by_index(device, i); devm_free_irq(&pdev->dev, msix->vector, chan); } pci_disable_msix(pdev); break; - - case IOAT_MSIX_SINGLE: - msix = &device->msix_entries[0]; - chan = ioat_chan_by_index(device, 0); - devm_free_irq(&pdev->dev, msix->vector, chan); - pci_disable_msix(pdev); - break; - case IOAT_MSI: - chan = ioat_chan_by_index(device, 0); - devm_free_irq(&pdev->dev, pdev->irq, chan); pci_disable_msi(pdev); - break; - + /* fall through */ case IOAT_INTX: - chan = ioat_chan_by_index(device, 0); - devm_free_irq(&pdev->dev, pdev->irq, chan); + devm_free_irq(&pdev->dev, irq, device); break; - default: return 0; } - device->irq_mode = IOAT_NOIRQ; - err = ioat_dma_setup_interrupts(device); - - return err; + return ioat_dma_setup_interrupts(device); } static int ioat3_reset_hw(struct ioat_chan_common *chan) @@ -1718,14 +1524,12 @@ static int ioat3_reset_hw(struct ioat_chan_common *chan) } err = ioat2_reset_sync(chan, msecs_to_jiffies(200)); - if (err) { - dev_err(&pdev->dev, "Failed to reset!\n"); - return err; - } - - if (device->irq_mode != IOAT_NOIRQ && is_bwd_ioat(pdev)) + if (!err) err = ioat3_irq_reinit(device); + if (err) + dev_err(&pdev->dev, "Failed to reset: %d\n", err); + return err; } @@ -1835,21 +1639,15 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) char pool_name[14]; int i; - /* allocate sw descriptor pool for SED */ - device->sed_pool = kmem_cache_create("ioat_sed", - sizeof(struct ioat_sed_ent), 0, 0, NULL); - if (!device->sed_pool) - return -ENOMEM; - for (i = 0; i < MAX_SED_POOLS; i++) { snprintf(pool_name, 14, "ioat_hw%d_sed", i); /* allocate SED DMA pool */ - device->sed_hw_pool[i] = dma_pool_create(pool_name, + device->sed_hw_pool[i] = dmam_pool_create(pool_name, &pdev->dev, SED_SIZE * (i + 1), 64, 0); if (!device->sed_hw_pool[i]) - goto sed_pool_cleanup; + return -ENOMEM; } } @@ -1875,28 +1673,4 @@ int ioat3_dma_probe(struct ioatdma_device *device, int dca) device->dca = ioat3_dca_init(pdev, device->reg_base); return 0; - -sed_pool_cleanup: - if (device->sed_pool) { - int i; - kmem_cache_destroy(device->sed_pool); - - for (i = 0; i < MAX_SED_POOLS; i++) - if (device->sed_hw_pool[i]) - dma_pool_destroy(device->sed_hw_pool[i]); - } - - return -ENOMEM; -} - -void ioat3_dma_remove(struct ioatdma_device *device) -{ - if (device->sed_pool) { - int i; - kmem_cache_destroy(device->sed_pool); - - for (i = 0; i < MAX_SED_POOLS; i++) - if (device->sed_hw_pool[i]) - dma_pool_destroy(device->sed_hw_pool[i]); - } } diff --git a/drivers/dma/ioat/pci.c b/drivers/dma/ioat/pci.c index 2c8d560e633..1d051cd045d 100644 --- a/drivers/dma/ioat/pci.c +++ b/drivers/dma/ioat/pci.c @@ -123,6 +123,7 @@ module_param(ioat_dca_enabled, int, 0644); MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)"); struct kmem_cache *ioat2_cache; +struct kmem_cache *ioat3_sed_cache; #define DRV_NAME "ioatdma" @@ -207,9 +208,6 @@ static void ioat_remove(struct pci_dev *pdev) if (!device) return; - if (device->version >= IOAT_VER_3_0) - ioat3_dma_remove(device); - dev_err(&pdev->dev, "Removing dma and dca services\n"); if (device->dca) { unregister_dca_provider(device->dca, &pdev->dev); @@ -221,7 +219,7 @@ static void ioat_remove(struct pci_dev *pdev) static int __init ioat_init_module(void) { - int err; + int err = -ENOMEM; pr_info("%s: Intel(R) QuickData Technology Driver %s\n", DRV_NAME, IOAT_DMA_VERSION); @@ -231,9 +229,21 @@ static int __init ioat_init_module(void) if (!ioat2_cache) return -ENOMEM; + ioat3_sed_cache = KMEM_CACHE(ioat_sed_ent, 0); + if (!ioat3_sed_cache) + goto err_ioat2_cache; + err = pci_register_driver(&ioat_pci_driver); if (err) - kmem_cache_destroy(ioat2_cache); + goto err_ioat3_cache; + + return 0; + + err_ioat3_cache: + kmem_cache_destroy(ioat3_sed_cache); + + err_ioat2_cache: + kmem_cache_destroy(ioat2_cache); return err; } diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c index dd8b44a56e5..c56137bc386 100644 --- a/drivers/dma/iop-adma.c +++ b/drivers/dma/iop-adma.c @@ -61,80 +61,6 @@ static void iop_adma_free_slots(struct iop_adma_desc_slot *slot) } } -static void -iop_desc_unmap(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc) -{ - struct dma_async_tx_descriptor *tx = &desc->async_tx; - struct iop_adma_desc_slot *unmap = desc->group_head; - struct device *dev = &iop_chan->device->pdev->dev; - u32 len = unmap->unmap_len; - enum dma_ctrl_flags flags = tx->flags; - u32 src_cnt; - dma_addr_t addr; - dma_addr_t dest; - - src_cnt = unmap->unmap_src_cnt; - dest = iop_desc_get_dest_addr(unmap, iop_chan); - if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - enum dma_data_direction dir; - - if (src_cnt > 1) /* is xor? */ - dir = DMA_BIDIRECTIONAL; - else - dir = DMA_FROM_DEVICE; - - dma_unmap_page(dev, dest, len, dir); - } - - if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - while (src_cnt--) { - addr = iop_desc_get_src_addr(unmap, iop_chan, src_cnt); - if (addr == dest) - continue; - dma_unmap_page(dev, addr, len, DMA_TO_DEVICE); - } - } - desc->group_head = NULL; -} - -static void -iop_desc_unmap_pq(struct iop_adma_chan *iop_chan, struct iop_adma_desc_slot *desc) -{ - struct dma_async_tx_descriptor *tx = &desc->async_tx; - struct iop_adma_desc_slot *unmap = desc->group_head; - struct device *dev = &iop_chan->device->pdev->dev; - u32 len = unmap->unmap_len; - enum dma_ctrl_flags flags = tx->flags; - u32 src_cnt = unmap->unmap_src_cnt; - dma_addr_t pdest = iop_desc_get_dest_addr(unmap, iop_chan); - dma_addr_t qdest = iop_desc_get_qdest_addr(unmap, iop_chan); - int i; - - if (tx->flags & DMA_PREP_CONTINUE) - src_cnt -= 3; - - if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP) && !desc->pq_check_result) { - dma_unmap_page(dev, pdest, len, DMA_BIDIRECTIONAL); - dma_unmap_page(dev, qdest, len, DMA_BIDIRECTIONAL); - } - - if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - dma_addr_t addr; - - for (i = 0; i < src_cnt; i++) { - addr = iop_desc_get_src_addr(unmap, iop_chan, i); - dma_unmap_page(dev, addr, len, DMA_TO_DEVICE); - } - if (desc->pq_check_result) { - dma_unmap_page(dev, pdest, len, DMA_TO_DEVICE); - dma_unmap_page(dev, qdest, len, DMA_TO_DEVICE); - } - } - - desc->group_head = NULL; -} - - static dma_cookie_t iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, struct iop_adma_chan *iop_chan, dma_cookie_t cookie) @@ -152,15 +78,9 @@ iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc, if (tx->callback) tx->callback(tx->callback_param); - /* unmap dma addresses - * (unmap_single vs unmap_page?) - */ - if (desc->group_head && desc->unmap_len) { - if (iop_desc_is_pq(desc)) - iop_desc_unmap_pq(iop_chan, desc); - else - iop_desc_unmap(iop_chan, desc); - } + dma_descriptor_unmap(tx); + if (desc->group_head) + desc->group_head = NULL; } /* run dependent operations */ @@ -591,7 +511,6 @@ iop_adma_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) if (sw_desc) { grp_start = sw_desc->group_head; iop_desc_init_interrupt(grp_start, iop_chan); - grp_start->unmap_len = 0; sw_desc->async_tx.flags = flags; } spin_unlock_bh(&iop_chan->lock); @@ -623,8 +542,6 @@ iop_adma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dma_dest, iop_desc_set_byte_count(grp_start, iop_chan, len); iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); iop_desc_set_memcpy_src_addr(grp_start, dma_src); - sw_desc->unmap_src_cnt = 1; - sw_desc->unmap_len = len; sw_desc->async_tx.flags = flags; } spin_unlock_bh(&iop_chan->lock); @@ -657,8 +574,6 @@ iop_adma_prep_dma_xor(struct dma_chan *chan, dma_addr_t dma_dest, iop_desc_init_xor(grp_start, src_cnt, flags); iop_desc_set_byte_count(grp_start, iop_chan, len); iop_desc_set_dest_addr(grp_start, iop_chan, dma_dest); - sw_desc->unmap_src_cnt = src_cnt; - sw_desc->unmap_len = len; sw_desc->async_tx.flags = flags; while (src_cnt--) iop_desc_set_xor_src_addr(grp_start, src_cnt, @@ -694,8 +609,6 @@ iop_adma_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *dma_src, grp_start->xor_check_result = result; pr_debug("\t%s: grp_start->xor_check_result: %p\n", __func__, grp_start->xor_check_result); - sw_desc->unmap_src_cnt = src_cnt; - sw_desc->unmap_len = len; sw_desc->async_tx.flags = flags; while (src_cnt--) iop_desc_set_zero_sum_src_addr(grp_start, src_cnt, @@ -748,8 +661,6 @@ iop_adma_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, dst[0] = dst[1] & 0x7; iop_desc_set_pq_addr(g, dst); - sw_desc->unmap_src_cnt = src_cnt; - sw_desc->unmap_len = len; sw_desc->async_tx.flags = flags; for (i = 0; i < src_cnt; i++) iop_desc_set_pq_src_addr(g, i, src[i], scf[i]); @@ -804,8 +715,6 @@ iop_adma_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, g->pq_check_result = pqres; pr_debug("\t%s: g->pq_check_result: %p\n", __func__, g->pq_check_result); - sw_desc->unmap_src_cnt = src_cnt+2; - sw_desc->unmap_len = len; sw_desc->async_tx.flags = flags; while (src_cnt--) iop_desc_set_pq_zero_sum_src_addr(g, src_cnt, @@ -864,7 +773,7 @@ static enum dma_status iop_adma_status(struct dma_chan *chan, int ret; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; iop_adma_slot_cleanup(iop_chan); @@ -983,7 +892,7 @@ static int iop_adma_memcpy_self_test(struct iop_adma_device *device) msleep(1); if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_SUCCESS) { + DMA_COMPLETE) { dev_err(dma_chan->device->dev, "Self-test copy timed out, disabling\n"); err = -ENODEV; @@ -1083,7 +992,7 @@ iop_adma_xor_val_self_test(struct iop_adma_device *device) msleep(8); if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_SUCCESS) { + DMA_COMPLETE) { dev_err(dma_chan->device->dev, "Self-test xor timed out, disabling\n"); err = -ENODEV; @@ -1129,7 +1038,7 @@ iop_adma_xor_val_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(8); - if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { dev_err(dma_chan->device->dev, "Self-test zero sum timed out, disabling\n"); err = -ENODEV; @@ -1158,7 +1067,7 @@ iop_adma_xor_val_self_test(struct iop_adma_device *device) iop_adma_issue_pending(dma_chan); msleep(8); - if (iop_adma_status(dma_chan, cookie, NULL) != DMA_SUCCESS) { + if (iop_adma_status(dma_chan, cookie, NULL) != DMA_COMPLETE) { dev_err(dma_chan->device->dev, "Self-test non-zero sum timed out, disabling\n"); err = -ENODEV; @@ -1254,7 +1163,7 @@ iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) msleep(8); if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_SUCCESS) { + DMA_COMPLETE) { dev_err(dev, "Self-test pq timed out, disabling\n"); err = -ENODEV; goto free_resources; @@ -1291,7 +1200,7 @@ iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) msleep(8); if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_SUCCESS) { + DMA_COMPLETE) { dev_err(dev, "Self-test pq-zero-sum timed out, disabling\n"); err = -ENODEV; goto free_resources; @@ -1323,7 +1232,7 @@ iop_adma_pq_zero_sum_self_test(struct iop_adma_device *device) msleep(8); if (iop_adma_status(dma_chan, cookie, NULL) != - DMA_SUCCESS) { + DMA_COMPLETE) { dev_err(dev, "Self-test !pq-zero-sum timed out, disabling\n"); err = -ENODEV; goto free_resources; diff --git a/drivers/dma/ipu/ipu_idmac.c b/drivers/dma/ipu/ipu_idmac.c index cb9c0bc317e..128ca143486 100644 --- a/drivers/dma/ipu/ipu_idmac.c +++ b/drivers/dma/ipu/ipu_idmac.c @@ -1232,8 +1232,10 @@ static irqreturn_t idmac_interrupt(int irq, void *dev_id) desc = list_entry(ichan->queue.next, struct idmac_tx_desc, list); descnew = desc; - dev_dbg(dev, "IDMAC irq %d, dma 0x%08x, next dma 0x%08x, current %d, curbuf 0x%08x\n", - irq, sg_dma_address(*sg), sgnext ? sg_dma_address(sgnext) : 0, ichan->active_buffer, curbuf); + dev_dbg(dev, "IDMAC irq %d, dma %#llx, next dma %#llx, current %d, curbuf %#x\n", + irq, (u64)sg_dma_address(*sg), + sgnext ? (u64)sg_dma_address(sgnext) : 0, + ichan->active_buffer, curbuf); /* Find the descriptor of sgnext */ sgnew = idmac_sg_next(ichan, &descnew, *sg); diff --git a/drivers/dma/k3dma.c b/drivers/dma/k3dma.c index a2c330f5f95..a1f911aaf22 100644 --- a/drivers/dma/k3dma.c +++ b/drivers/dma/k3dma.c @@ -344,7 +344,7 @@ static enum dma_status k3_dma_tx_status(struct dma_chan *chan, size_t bytes = 0; ret = dma_cookie_status(&c->vc.chan, cookie, state); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; spin_lock_irqsave(&c->vc.lock, flags); @@ -477,7 +477,7 @@ static struct dma_async_tx_descriptor *k3_dma_prep_slave_sg( dma_addr_t addr, src = 0, dst = 0; int num = sglen, i; - if (sgl == 0) + if (sgl == NULL) return NULL; for_each_sg(sgl, sg, sglen, i) { @@ -693,7 +693,7 @@ static int k3_dma_probe(struct platform_device *op) irq = platform_get_irq(op, 0); ret = devm_request_irq(&op->dev, irq, - k3_dma_int_handler, IRQF_DISABLED, DRIVER_NAME, d); + k3_dma_int_handler, 0, DRIVER_NAME, d); if (ret) return ret; @@ -817,7 +817,7 @@ static int k3_dma_resume(struct device *dev) return 0; } -SIMPLE_DEV_PM_OPS(k3_dma_pmops, k3_dma_suspend, k3_dma_resume); +static SIMPLE_DEV_PM_OPS(k3_dma_pmops, k3_dma_suspend, k3_dma_resume); static struct platform_driver k3_pdma_driver = { .driver = { diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c index ff8d7827f8c..a7b186d536b 100644 --- a/drivers/dma/mmp_pdma.c +++ b/drivers/dma/mmp_pdma.c @@ -5,6 +5,7 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ + #include <linux/err.h> #include <linux/module.h> #include <linux/init.h> @@ -28,42 +29,41 @@ #define DALGN 0x00a0 #define DINT 0x00f0 #define DDADR 0x0200 -#define DSADR 0x0204 -#define DTADR 0x0208 +#define DSADR(n) (0x0204 + ((n) << 4)) +#define DTADR(n) (0x0208 + ((n) << 4)) #define DCMD 0x020c -#define DCSR_RUN (1 << 31) /* Run Bit (read / write) */ -#define DCSR_NODESC (1 << 30) /* No-Descriptor Fetch (read / write) */ -#define DCSR_STOPIRQEN (1 << 29) /* Stop Interrupt Enable (read / write) */ -#define DCSR_REQPEND (1 << 8) /* Request Pending (read-only) */ -#define DCSR_STOPSTATE (1 << 3) /* Stop State (read-only) */ -#define DCSR_ENDINTR (1 << 2) /* End Interrupt (read / write) */ -#define DCSR_STARTINTR (1 << 1) /* Start Interrupt (read / write) */ -#define DCSR_BUSERR (1 << 0) /* Bus Error Interrupt (read / write) */ - -#define DCSR_EORIRQEN (1 << 28) /* End of Receive Interrupt Enable (R/W) */ -#define DCSR_EORJMPEN (1 << 27) /* Jump to next descriptor on EOR */ -#define DCSR_EORSTOPEN (1 << 26) /* STOP on an EOR */ -#define DCSR_SETCMPST (1 << 25) /* Set Descriptor Compare Status */ -#define DCSR_CLRCMPST (1 << 24) /* Clear Descriptor Compare Status */ -#define DCSR_CMPST (1 << 10) /* The Descriptor Compare Status */ -#define DCSR_EORINTR (1 << 9) /* The end of Receive */ - -#define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + \ - (((n) & 0x3f) << 2)) -#define DRCMR_MAPVLD (1 << 7) /* Map Valid (read / write) */ -#define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */ +#define DCSR_RUN BIT(31) /* Run Bit (read / write) */ +#define DCSR_NODESC BIT(30) /* No-Descriptor Fetch (read / write) */ +#define DCSR_STOPIRQEN BIT(29) /* Stop Interrupt Enable (read / write) */ +#define DCSR_REQPEND BIT(8) /* Request Pending (read-only) */ +#define DCSR_STOPSTATE BIT(3) /* Stop State (read-only) */ +#define DCSR_ENDINTR BIT(2) /* End Interrupt (read / write) */ +#define DCSR_STARTINTR BIT(1) /* Start Interrupt (read / write) */ +#define DCSR_BUSERR BIT(0) /* Bus Error Interrupt (read / write) */ + +#define DCSR_EORIRQEN BIT(28) /* End of Receive Interrupt Enable (R/W) */ +#define DCSR_EORJMPEN BIT(27) /* Jump to next descriptor on EOR */ +#define DCSR_EORSTOPEN BIT(26) /* STOP on an EOR */ +#define DCSR_SETCMPST BIT(25) /* Set Descriptor Compare Status */ +#define DCSR_CLRCMPST BIT(24) /* Clear Descriptor Compare Status */ +#define DCSR_CMPST BIT(10) /* The Descriptor Compare Status */ +#define DCSR_EORINTR BIT(9) /* The end of Receive */ + +#define DRCMR(n) ((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2)) +#define DRCMR_MAPVLD BIT(7) /* Map Valid (read / write) */ +#define DRCMR_CHLNUM 0x1f /* mask for Channel Number (read / write) */ #define DDADR_DESCADDR 0xfffffff0 /* Address of next descriptor (mask) */ -#define DDADR_STOP (1 << 0) /* Stop (read / write) */ - -#define DCMD_INCSRCADDR (1 << 31) /* Source Address Increment Setting. */ -#define DCMD_INCTRGADDR (1 << 30) /* Target Address Increment Setting. */ -#define DCMD_FLOWSRC (1 << 29) /* Flow Control by the source. */ -#define DCMD_FLOWTRG (1 << 28) /* Flow Control by the target. */ -#define DCMD_STARTIRQEN (1 << 22) /* Start Interrupt Enable */ -#define DCMD_ENDIRQEN (1 << 21) /* End Interrupt Enable */ -#define DCMD_ENDIAN (1 << 18) /* Device Endian-ness. */ +#define DDADR_STOP BIT(0) /* Stop (read / write) */ + +#define DCMD_INCSRCADDR BIT(31) /* Source Address Increment Setting. */ +#define DCMD_INCTRGADDR BIT(30) /* Target Address Increment Setting. */ +#define DCMD_FLOWSRC BIT(29) /* Flow Control by the source. */ +#define DCMD_FLOWTRG BIT(28) /* Flow Control by the target. */ +#define DCMD_STARTIRQEN BIT(22) /* Start Interrupt Enable */ +#define DCMD_ENDIRQEN BIT(21) /* End Interrupt Enable */ +#define DCMD_ENDIAN BIT(18) /* Device Endian-ness. */ #define DCMD_BURST8 (1 << 16) /* 8 byte burst */ #define DCMD_BURST16 (2 << 16) /* 16 byte burst */ #define DCMD_BURST32 (3 << 16) /* 32 byte burst */ @@ -132,10 +132,14 @@ struct mmp_pdma_device { spinlock_t phy_lock; /* protect alloc/free phy channels */ }; -#define tx_to_mmp_pdma_desc(tx) container_of(tx, struct mmp_pdma_desc_sw, async_tx) -#define to_mmp_pdma_desc(lh) container_of(lh, struct mmp_pdma_desc_sw, node) -#define to_mmp_pdma_chan(dchan) container_of(dchan, struct mmp_pdma_chan, chan) -#define to_mmp_pdma_dev(dmadev) container_of(dmadev, struct mmp_pdma_device, device) +#define tx_to_mmp_pdma_desc(tx) \ + container_of(tx, struct mmp_pdma_desc_sw, async_tx) +#define to_mmp_pdma_desc(lh) \ + container_of(lh, struct mmp_pdma_desc_sw, node) +#define to_mmp_pdma_chan(dchan) \ + container_of(dchan, struct mmp_pdma_chan, chan) +#define to_mmp_pdma_dev(dmadev) \ + container_of(dmadev, struct mmp_pdma_device, device) static void set_desc(struct mmp_pdma_phy *phy, dma_addr_t addr) { @@ -162,19 +166,18 @@ static void enable_chan(struct mmp_pdma_phy *phy) writel(dalgn, phy->base + DALGN); reg = (phy->idx << 2) + DCSR; - writel(readl(phy->base + reg) | DCSR_RUN, - phy->base + reg); + writel(readl(phy->base + reg) | DCSR_RUN, phy->base + reg); } static void disable_chan(struct mmp_pdma_phy *phy) { u32 reg; - if (phy) { - reg = (phy->idx << 2) + DCSR; - writel(readl(phy->base + reg) & ~DCSR_RUN, - phy->base + reg); - } + if (!phy) + return; + + reg = (phy->idx << 2) + DCSR; + writel(readl(phy->base + reg) & ~DCSR_RUN, phy->base + reg); } static int clear_chan_irq(struct mmp_pdma_phy *phy) @@ -183,26 +186,27 @@ static int clear_chan_irq(struct mmp_pdma_phy *phy) u32 dint = readl(phy->base + DINT); u32 reg = (phy->idx << 2) + DCSR; - if (dint & BIT(phy->idx)) { - /* clear irq */ - dcsr = readl(phy->base + reg); - writel(dcsr, phy->base + reg); - if ((dcsr & DCSR_BUSERR) && (phy->vchan)) - dev_warn(phy->vchan->dev, "DCSR_BUSERR\n"); - return 0; - } - return -EAGAIN; + if (!(dint & BIT(phy->idx))) + return -EAGAIN; + + /* clear irq */ + dcsr = readl(phy->base + reg); + writel(dcsr, phy->base + reg); + if ((dcsr & DCSR_BUSERR) && (phy->vchan)) + dev_warn(phy->vchan->dev, "DCSR_BUSERR\n"); + + return 0; } static irqreturn_t mmp_pdma_chan_handler(int irq, void *dev_id) { struct mmp_pdma_phy *phy = dev_id; - if (clear_chan_irq(phy) == 0) { - tasklet_schedule(&phy->vchan->tasklet); - return IRQ_HANDLED; - } else + if (clear_chan_irq(phy) != 0) return IRQ_NONE; + + tasklet_schedule(&phy->vchan->tasklet); + return IRQ_HANDLED; } static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id) @@ -224,8 +228,8 @@ static irqreturn_t mmp_pdma_int_handler(int irq, void *dev_id) if (irq_num) return IRQ_HANDLED; - else - return IRQ_NONE; + + return IRQ_NONE; } /* lookup free phy channel as descending priority */ @@ -245,9 +249,9 @@ static struct mmp_pdma_phy *lookup_phy(struct mmp_pdma_chan *pchan) */ spin_lock_irqsave(&pdev->phy_lock, flags); - for (prio = 0; prio <= (((pdev->dma_channels - 1) & 0xf) >> 2); prio++) { + for (prio = 0; prio <= ((pdev->dma_channels - 1) & 0xf) >> 2; prio++) { for (i = 0; i < pdev->dma_channels; i++) { - if (prio != ((i & 0xf) >> 2)) + if (prio != (i & 0xf) >> 2) continue; phy = &pdev->phy[i]; if (!phy->vchan) { @@ -273,7 +277,7 @@ static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan) return; /* clear the channel mapping in DRCMR */ - reg = DRCMR(pchan->phy->vchan->drcmr); + reg = DRCMR(pchan->drcmr); writel(0, pchan->phy->base + reg); spin_lock_irqsave(&pdev->phy_lock, flags); @@ -389,14 +393,16 @@ static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan) if (chan->desc_pool) return 1; - chan->desc_pool = - dma_pool_create(dev_name(&dchan->dev->device), chan->dev, - sizeof(struct mmp_pdma_desc_sw), - __alignof__(struct mmp_pdma_desc_sw), 0); + chan->desc_pool = dma_pool_create(dev_name(&dchan->dev->device), + chan->dev, + sizeof(struct mmp_pdma_desc_sw), + __alignof__(struct mmp_pdma_desc_sw), + 0); if (!chan->desc_pool) { dev_err(chan->dev, "unable to allocate descriptor pool\n"); return -ENOMEM; } + mmp_pdma_free_phy(chan); chan->idle = true; chan->dev_addr = 0; @@ -404,7 +410,7 @@ static int mmp_pdma_alloc_chan_resources(struct dma_chan *dchan) } static void mmp_pdma_free_desc_list(struct mmp_pdma_chan *chan, - struct list_head *list) + struct list_head *list) { struct mmp_pdma_desc_sw *desc, *_desc; @@ -434,8 +440,8 @@ static void mmp_pdma_free_chan_resources(struct dma_chan *dchan) static struct dma_async_tx_descriptor * mmp_pdma_prep_memcpy(struct dma_chan *dchan, - dma_addr_t dma_dst, dma_addr_t dma_src, - size_t len, unsigned long flags) + dma_addr_t dma_dst, dma_addr_t dma_src, + size_t len, unsigned long flags) { struct mmp_pdma_chan *chan; struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; @@ -515,8 +521,8 @@ fail: static struct dma_async_tx_descriptor * mmp_pdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, - unsigned int sg_len, enum dma_transfer_direction dir, - unsigned long flags, void *context) + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, void *context) { struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new = NULL; @@ -591,10 +597,11 @@ fail: return NULL; } -static struct dma_async_tx_descriptor *mmp_pdma_prep_dma_cyclic( - struct dma_chan *dchan, dma_addr_t buf_addr, size_t len, - size_t period_len, enum dma_transfer_direction direction, - unsigned long flags, void *context) +static struct dma_async_tx_descriptor * +mmp_pdma_prep_dma_cyclic(struct dma_chan *dchan, + dma_addr_t buf_addr, size_t len, size_t period_len, + enum dma_transfer_direction direction, + unsigned long flags, void *context) { struct mmp_pdma_chan *chan; struct mmp_pdma_desc_sw *first = NULL, *prev = NULL, *new; @@ -636,8 +643,8 @@ static struct dma_async_tx_descriptor *mmp_pdma_prep_dma_cyclic( goto fail; } - new->desc.dcmd = chan->dcmd | DCMD_ENDIRQEN | - (DCMD_LENGTH & period_len); + new->desc.dcmd = (chan->dcmd | DCMD_ENDIRQEN | + (DCMD_LENGTH & period_len)); new->desc.dsadr = dma_src; new->desc.dtadr = dma_dst; @@ -677,12 +684,11 @@ fail: } static int mmp_pdma_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd, - unsigned long arg) + unsigned long arg) { struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); struct dma_slave_config *cfg = (void *)arg; unsigned long flags; - int ret = 0; u32 maxburst = 0, addr = 0; enum dma_slave_buswidth width = DMA_SLAVE_BUSWIDTH_UNDEFINED; @@ -739,13 +745,95 @@ static int mmp_pdma_control(struct dma_chan *dchan, enum dma_ctrl_cmd cmd, return -ENOSYS; } - return ret; + return 0; +} + +static unsigned int mmp_pdma_residue(struct mmp_pdma_chan *chan, + dma_cookie_t cookie) +{ + struct mmp_pdma_desc_sw *sw; + u32 curr, residue = 0; + bool passed = false; + bool cyclic = chan->cyclic_first != NULL; + + /* + * If the channel does not have a phy pointer anymore, it has already + * been completed. Therefore, its residue is 0. + */ + if (!chan->phy) + return 0; + + if (chan->dir == DMA_DEV_TO_MEM) + curr = readl(chan->phy->base + DTADR(chan->phy->idx)); + else + curr = readl(chan->phy->base + DSADR(chan->phy->idx)); + + list_for_each_entry(sw, &chan->chain_running, node) { + u32 start, end, len; + + if (chan->dir == DMA_DEV_TO_MEM) + start = sw->desc.dtadr; + else + start = sw->desc.dsadr; + + len = sw->desc.dcmd & DCMD_LENGTH; + end = start + len; + + /* + * 'passed' will be latched once we found the descriptor which + * lies inside the boundaries of the curr pointer. All + * descriptors that occur in the list _after_ we found that + * partially handled descriptor are still to be processed and + * are hence added to the residual bytes counter. + */ + + if (passed) { + residue += len; + } else if (curr >= start && curr <= end) { + residue += end - curr; + passed = true; + } + + /* + * Descriptors that have the ENDIRQEN bit set mark the end of a + * transaction chain, and the cookie assigned with it has been + * returned previously from mmp_pdma_tx_submit(). + * + * In case we have multiple transactions in the running chain, + * and the cookie does not match the one the user asked us + * about, reset the state variables and start over. + * + * This logic does not apply to cyclic transactions, where all + * descriptors have the ENDIRQEN bit set, and for which we + * can't have multiple transactions on one channel anyway. + */ + if (cyclic || !(sw->desc.dcmd & DCMD_ENDIRQEN)) + continue; + + if (sw->async_tx.cookie == cookie) { + return residue; + } else { + residue = 0; + passed = false; + } + } + + /* We should only get here in case of cyclic transactions */ + return residue; } static enum dma_status mmp_pdma_tx_status(struct dma_chan *dchan, - dma_cookie_t cookie, struct dma_tx_state *txstate) + dma_cookie_t cookie, + struct dma_tx_state *txstate) { - return dma_cookie_status(dchan, cookie, txstate); + struct mmp_pdma_chan *chan = to_mmp_pdma_chan(dchan); + enum dma_status ret; + + ret = dma_cookie_status(dchan, cookie, txstate); + if (likely(ret != DMA_ERROR)) + dma_set_residue(txstate, mmp_pdma_residue(chan, cookie)); + + return ret; } /** @@ -798,8 +886,7 @@ static void dma_do_tasklet(unsigned long data) * move the descriptors to a temporary list so we can drop * the lock during the entire cleanup operation */ - list_del(&desc->node); - list_add(&desc->node, &chain_cleanup); + list_move(&desc->node, &chain_cleanup); /* * Look for the first list entry which has the ENDIRQEN flag @@ -846,15 +933,13 @@ static int mmp_pdma_remove(struct platform_device *op) return 0; } -static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, - int idx, int irq) +static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, int idx, int irq) { struct mmp_pdma_phy *phy = &pdev->phy[idx]; struct mmp_pdma_chan *chan; int ret; - chan = devm_kzalloc(pdev->dev, - sizeof(struct mmp_pdma_chan), GFP_KERNEL); + chan = devm_kzalloc(pdev->dev, sizeof(*chan), GFP_KERNEL); if (chan == NULL) return -ENOMEM; @@ -862,8 +947,8 @@ static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, phy->base = pdev->base; if (irq) { - ret = devm_request_irq(pdev->dev, irq, - mmp_pdma_chan_handler, IRQF_DISABLED, "pdma", phy); + ret = devm_request_irq(pdev->dev, irq, mmp_pdma_chan_handler, + IRQF_SHARED, "pdma", phy); if (ret) { dev_err(pdev->dev, "channel request irq fail!\n"); return ret; @@ -878,8 +963,7 @@ static int mmp_pdma_chan_init(struct mmp_pdma_device *pdev, INIT_LIST_HEAD(&chan->chain_running); /* register virt channel to dma engine */ - list_add_tail(&chan->chan.device_node, - &pdev->device.channels); + list_add_tail(&chan->chan.device_node, &pdev->device.channels); return 0; } @@ -894,33 +978,15 @@ static struct dma_chan *mmp_pdma_dma_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct mmp_pdma_device *d = ofdma->of_dma_data; - struct dma_chan *chan, *candidate; + struct dma_chan *chan; -retry: - candidate = NULL; - - /* walk the list of channels registered with the current instance and - * find one that is currently unused */ - list_for_each_entry(chan, &d->device.channels, device_node) - if (chan->client_count == 0) { - candidate = chan; - break; - } - - if (!candidate) + chan = dma_get_any_slave_channel(&d->device); + if (!chan) return NULL; - /* dma_get_slave_channel will return NULL if we lost a race between - * the lookup and the reservation */ - chan = dma_get_slave_channel(candidate); + to_mmp_pdma_chan(chan)->drcmr = dma_spec->args[0]; - if (chan) { - struct mmp_pdma_chan *c = to_mmp_pdma_chan(chan); - c->drcmr = dma_spec->args[0]; - return chan; - } - - goto retry; + return chan; } static int mmp_pdma_probe(struct platform_device *op) @@ -935,6 +1001,7 @@ static int mmp_pdma_probe(struct platform_device *op) pdev = devm_kzalloc(&op->dev, sizeof(*pdev), GFP_KERNEL); if (!pdev) return -ENOMEM; + pdev->dev = &op->dev; spin_lock_init(&pdev->phy_lock); @@ -946,8 +1013,8 @@ static int mmp_pdma_probe(struct platform_device *op) of_id = of_match_device(mmp_pdma_dt_ids, pdev->dev); if (of_id) - of_property_read_u32(pdev->dev->of_node, - "#dma-channels", &dma_channels); + of_property_read_u32(pdev->dev->of_node, "#dma-channels", + &dma_channels); else if (pdata && pdata->dma_channels) dma_channels = pdata->dma_channels; else @@ -959,8 +1026,8 @@ static int mmp_pdma_probe(struct platform_device *op) irq_num++; } - pdev->phy = devm_kzalloc(pdev->dev, - dma_channels * sizeof(struct mmp_pdma_chan), GFP_KERNEL); + pdev->phy = devm_kcalloc(pdev->dev, dma_channels, sizeof(*pdev->phy), + GFP_KERNEL); if (pdev->phy == NULL) return -ENOMEM; @@ -969,8 +1036,8 @@ static int mmp_pdma_probe(struct platform_device *op) if (irq_num != dma_channels) { /* all chan share one irq, demux inside */ irq = platform_get_irq(op, 0); - ret = devm_request_irq(pdev->dev, irq, - mmp_pdma_int_handler, IRQF_DISABLED, "pdma", pdev); + ret = devm_request_irq(pdev->dev, irq, mmp_pdma_int_handler, + IRQF_SHARED, "pdma", pdev); if (ret) return ret; } @@ -1018,6 +1085,7 @@ static int mmp_pdma_probe(struct platform_device *op) } } + platform_set_drvdata(op, pdev); dev_info(pdev->device.dev, "initialized %d channels\n", dma_channels); return 0; } @@ -1045,7 +1113,7 @@ bool mmp_pdma_filter_fn(struct dma_chan *chan, void *param) if (chan->device->dev->driver != &mmp_pdma_driver.driver) return false; - c->drcmr = *(unsigned int *) param; + c->drcmr = *(unsigned int *)param; return true; } @@ -1053,6 +1121,6 @@ EXPORT_SYMBOL_GPL(mmp_pdma_filter_fn); module_platform_driver(mmp_pdma_driver); -MODULE_DESCRIPTION("MARVELL MMP Periphera DMA Driver"); +MODULE_DESCRIPTION("MARVELL MMP Peripheral DMA Driver"); MODULE_AUTHOR("Marvell International Ltd."); MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/mmp_tdma.c b/drivers/dma/mmp_tdma.c index 38cb517fb2e..724f7f4c972 100644 --- a/drivers/dma/mmp_tdma.c +++ b/drivers/dma/mmp_tdma.c @@ -22,6 +22,7 @@ #include <mach/regs-icu.h> #include <linux/platform_data/dma-mmp_tdma.h> #include <linux/of_device.h> +#include <linux/of_dma.h> #include "dmaengine.h" @@ -62,6 +63,11 @@ #define TDCR_BURSTSZ_16B (0x3 << 6) #define TDCR_BURSTSZ_32B (0x6 << 6) #define TDCR_BURSTSZ_64B (0x7 << 6) +#define TDCR_BURSTSZ_SQU_1B (0x5 << 6) +#define TDCR_BURSTSZ_SQU_2B (0x6 << 6) +#define TDCR_BURSTSZ_SQU_4B (0x0 << 6) +#define TDCR_BURSTSZ_SQU_8B (0x1 << 6) +#define TDCR_BURSTSZ_SQU_16B (0x3 << 6) #define TDCR_BURSTSZ_SQU_32B (0x7 << 6) #define TDCR_BURSTSZ_128B (0x5 << 6) #define TDCR_DSTDIR_MSK (0x3 << 4) /* Dst Direction */ @@ -116,11 +122,13 @@ struct mmp_tdma_chan { int idx; enum mmp_tdma_type type; int irq; - unsigned long reg_base; + void __iomem *reg_base; size_t buf_len; size_t period_len; size_t pos; + + struct gen_pool *pool; }; #define TDMA_CHANNEL_NUM 2 @@ -158,7 +166,7 @@ static void mmp_tdma_disable_chan(struct mmp_tdma_chan *tdmac) /* disable irq */ writel(0, tdmac->reg_base + TDIMR); - tdmac->status = DMA_SUCCESS; + tdmac->status = DMA_COMPLETE; } static void mmp_tdma_resume_chan(struct mmp_tdma_chan *tdmac) @@ -177,7 +185,7 @@ static void mmp_tdma_pause_chan(struct mmp_tdma_chan *tdmac) static int mmp_tdma_config_chan(struct mmp_tdma_chan *tdmac) { - unsigned int tdcr; + unsigned int tdcr = 0; mmp_tdma_disable_chan(tdmac); @@ -228,8 +236,31 @@ static int mmp_tdma_config_chan(struct mmp_tdma_chan *tdmac) return -EINVAL; } } else if (tdmac->type == PXA910_SQU) { - tdcr |= TDCR_BURSTSZ_SQU_32B; tdcr |= TDCR_SSPMOD; + + switch (tdmac->burst_sz) { + case 1: + tdcr |= TDCR_BURSTSZ_SQU_1B; + break; + case 2: + tdcr |= TDCR_BURSTSZ_SQU_2B; + break; + case 4: + tdcr |= TDCR_BURSTSZ_SQU_4B; + break; + case 8: + tdcr |= TDCR_BURSTSZ_SQU_8B; + break; + case 16: + tdcr |= TDCR_BURSTSZ_SQU_16B; + break; + case 32: + tdcr |= TDCR_BURSTSZ_SQU_32B; + break; + default: + dev_err(tdmac->dev, "mmp_tdma: unknown burst size.\n"); + return -EINVAL; + } } writel(tdcr, tdmac->reg_base + TDCR); @@ -296,7 +327,7 @@ static void mmp_tdma_free_descriptor(struct mmp_tdma_chan *tdmac) struct gen_pool *gpool; int size = tdmac->desc_num * sizeof(struct mmp_tdma_desc); - gpool = sram_get_gpool("asram"); + gpool = tdmac->pool; if (tdmac->desc_arr) gen_pool_free(gpool, (unsigned long)tdmac->desc_arr, size); @@ -324,7 +355,7 @@ static int mmp_tdma_alloc_chan_resources(struct dma_chan *chan) if (tdmac->irq) { ret = devm_request_irq(tdmac->dev, tdmac->irq, - mmp_tdma_chan_handler, IRQF_DISABLED, "tdma", tdmac); + mmp_tdma_chan_handler, 0, "tdma", tdmac); if (ret) return ret; } @@ -346,16 +377,11 @@ struct mmp_tdma_desc *mmp_tdma_alloc_descriptor(struct mmp_tdma_chan *tdmac) struct gen_pool *gpool; int size = tdmac->desc_num * sizeof(struct mmp_tdma_desc); - gpool = sram_get_gpool("asram"); + gpool = tdmac->pool; if (!gpool) return NULL; - tdmac->desc_arr = (void *)gen_pool_alloc(gpool, size); - if (!tdmac->desc_arr) - return NULL; - - tdmac->desc_arr_phys = gen_pool_virt_to_phys(gpool, - (unsigned long)tdmac->desc_arr); + tdmac->desc_arr = gen_pool_dma_alloc(gpool, size, &tdmac->desc_arr_phys); return tdmac->desc_arr; } @@ -370,7 +396,7 @@ static struct dma_async_tx_descriptor *mmp_tdma_prep_dma_cyclic( int num_periods = buf_len / period_len; int i = 0, buf = 0; - if (tdmac->status != DMA_SUCCESS) + if (tdmac->status != DMA_COMPLETE) return NULL; if (period_len > TDMA_MAX_XFER_BYTES) { @@ -482,7 +508,8 @@ static int mmp_tdma_remove(struct platform_device *pdev) } static int mmp_tdma_chan_init(struct mmp_tdma_device *tdev, - int idx, int irq, int type) + int idx, int irq, + int type, struct gen_pool *pool) { struct mmp_tdma_chan *tdmac; @@ -503,8 +530,9 @@ static int mmp_tdma_chan_init(struct mmp_tdma_device *tdev, tdmac->chan.device = &tdev->device; tdmac->idx = idx; tdmac->type = type; - tdmac->reg_base = (unsigned long)tdev->base + idx * 4; - tdmac->status = DMA_SUCCESS; + tdmac->reg_base = tdev->base + idx * 4; + tdmac->pool = pool; + tdmac->status = DMA_COMPLETE; tdev->tdmac[tdmac->idx] = tdmac; tasklet_init(&tdmac->tasklet, dma_do_tasklet, (unsigned long)tdmac); @@ -514,6 +542,45 @@ static int mmp_tdma_chan_init(struct mmp_tdma_device *tdev, return 0; } +struct mmp_tdma_filter_param { + struct device_node *of_node; + unsigned int chan_id; +}; + +static bool mmp_tdma_filter_fn(struct dma_chan *chan, void *fn_param) +{ + struct mmp_tdma_filter_param *param = fn_param; + struct mmp_tdma_chan *tdmac = to_mmp_tdma_chan(chan); + struct dma_device *pdma_device = tdmac->chan.device; + + if (pdma_device->dev->of_node != param->of_node) + return false; + + if (chan->chan_id != param->chan_id) + return false; + + return true; +} + +struct dma_chan *mmp_tdma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct mmp_tdma_device *tdev = ofdma->of_dma_data; + dma_cap_mask_t mask = tdev->device.cap_mask; + struct mmp_tdma_filter_param param; + + if (dma_spec->args_count != 1) + return NULL; + + param.of_node = ofdma->of_node; + param.chan_id = dma_spec->args[0]; + + if (param.chan_id >= TDMA_CHANNEL_NUM) + return NULL; + + return dma_request_channel(mask, mmp_tdma_filter_fn, ¶m); +} + static struct of_device_id mmp_tdma_dt_ids[] = { { .compatible = "marvell,adma-1.0", .data = (void *)MMP_AUD_TDMA}, { .compatible = "marvell,pxa910-squ", .data = (void *)PXA910_SQU}, @@ -530,6 +597,7 @@ static int mmp_tdma_probe(struct platform_device *pdev) int i, ret; int irq = 0, irq_num = 0; int chan_num = TDMA_CHANNEL_NUM; + struct gen_pool *pool; of_id = of_match_device(mmp_tdma_dt_ids, &pdev->dev); if (of_id) @@ -556,10 +624,19 @@ static int mmp_tdma_probe(struct platform_device *pdev) INIT_LIST_HEAD(&tdev->device.channels); + if (pdev->dev.of_node) + pool = of_get_named_gen_pool(pdev->dev.of_node, "asram", 0); + else + pool = sram_get_gpool("asram"); + if (!pool) { + dev_err(&pdev->dev, "asram pool not available\n"); + return -ENOMEM; + } + if (irq_num != chan_num) { irq = platform_get_irq(pdev, 0); ret = devm_request_irq(&pdev->dev, irq, - mmp_tdma_int_handler, IRQF_DISABLED, "tdma", tdev); + mmp_tdma_int_handler, 0, "tdma", tdev); if (ret) return ret; } @@ -567,7 +644,7 @@ static int mmp_tdma_probe(struct platform_device *pdev) /* initialize channel parameters */ for (i = 0; i < chan_num; i++) { irq = (irq_num != chan_num) ? 0 : platform_get_irq(pdev, i); - ret = mmp_tdma_chan_init(tdev, i, irq, type); + ret = mmp_tdma_chan_init(tdev, i, irq, type, pool); if (ret) return ret; } @@ -594,6 +671,16 @@ static int mmp_tdma_probe(struct platform_device *pdev) return ret; } + if (pdev->dev.of_node) { + ret = of_dma_controller_register(pdev->dev.of_node, + mmp_tdma_xlate, tdev); + if (ret) { + dev_err(tdev->device.dev, + "failed to register controller\n"); + dma_async_device_unregister(&tdev->device); + } + } + dev_info(tdev->device.dev, "initialized\n"); return 0; } diff --git a/drivers/dma/moxart-dma.c b/drivers/dma/moxart-dma.c new file mode 100644 index 00000000000..3258e484e4f --- /dev/null +++ b/drivers/dma/moxart-dma.c @@ -0,0 +1,699 @@ +/* + * MOXA ART SoCs DMA Engine support. + * + * Copyright (C) 2013 Jonas Jensen + * + * Jonas Jensen <jonas.jensen@gmail.com> + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any + * warranty of any kind, whether express or implied. + */ + +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_dma.h> +#include <linux/bitops.h> + +#include <asm/cacheflush.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +#define APB_DMA_MAX_CHANNEL 4 + +#define REG_OFF_ADDRESS_SOURCE 0 +#define REG_OFF_ADDRESS_DEST 4 +#define REG_OFF_CYCLES 8 +#define REG_OFF_CTRL 12 +#define REG_OFF_CHAN_SIZE 16 + +#define APB_DMA_ENABLE BIT(0) +#define APB_DMA_FIN_INT_STS BIT(1) +#define APB_DMA_FIN_INT_EN BIT(2) +#define APB_DMA_BURST_MODE BIT(3) +#define APB_DMA_ERR_INT_STS BIT(4) +#define APB_DMA_ERR_INT_EN BIT(5) + +/* + * Unset: APB + * Set: AHB + */ +#define APB_DMA_SOURCE_SELECT 0x40 +#define APB_DMA_DEST_SELECT 0x80 + +#define APB_DMA_SOURCE 0x100 +#define APB_DMA_DEST 0x1000 + +#define APB_DMA_SOURCE_MASK 0x700 +#define APB_DMA_DEST_MASK 0x7000 + +/* + * 000: No increment + * 001: +1 (Burst=0), +4 (Burst=1) + * 010: +2 (Burst=0), +8 (Burst=1) + * 011: +4 (Burst=0), +16 (Burst=1) + * 101: -1 (Burst=0), -4 (Burst=1) + * 110: -2 (Burst=0), -8 (Burst=1) + * 111: -4 (Burst=0), -16 (Burst=1) + */ +#define APB_DMA_SOURCE_INC_0 0 +#define APB_DMA_SOURCE_INC_1_4 0x100 +#define APB_DMA_SOURCE_INC_2_8 0x200 +#define APB_DMA_SOURCE_INC_4_16 0x300 +#define APB_DMA_SOURCE_DEC_1_4 0x500 +#define APB_DMA_SOURCE_DEC_2_8 0x600 +#define APB_DMA_SOURCE_DEC_4_16 0x700 +#define APB_DMA_DEST_INC_0 0 +#define APB_DMA_DEST_INC_1_4 0x1000 +#define APB_DMA_DEST_INC_2_8 0x2000 +#define APB_DMA_DEST_INC_4_16 0x3000 +#define APB_DMA_DEST_DEC_1_4 0x5000 +#define APB_DMA_DEST_DEC_2_8 0x6000 +#define APB_DMA_DEST_DEC_4_16 0x7000 + +/* + * Request signal select source/destination address for DMA hardware handshake. + * + * The request line number is a property of the DMA controller itself, + * e.g. MMC must always request channels where dma_slave_config->slave_id is 5. + * + * 0: No request / Grant signal + * 1-15: Request / Grant signal + */ +#define APB_DMA_SOURCE_REQ_NO 0x1000000 +#define APB_DMA_SOURCE_REQ_NO_MASK 0xf000000 +#define APB_DMA_DEST_REQ_NO 0x10000 +#define APB_DMA_DEST_REQ_NO_MASK 0xf0000 + +#define APB_DMA_DATA_WIDTH 0x100000 +#define APB_DMA_DATA_WIDTH_MASK 0x300000 +/* + * Data width of transfer: + * + * 00: Word + * 01: Half + * 10: Byte + */ +#define APB_DMA_DATA_WIDTH_4 0 +#define APB_DMA_DATA_WIDTH_2 0x100000 +#define APB_DMA_DATA_WIDTH_1 0x200000 + +#define APB_DMA_CYCLES_MASK 0x00ffffff + +#define MOXART_DMA_DATA_TYPE_S8 0x00 +#define MOXART_DMA_DATA_TYPE_S16 0x01 +#define MOXART_DMA_DATA_TYPE_S32 0x02 + +struct moxart_sg { + dma_addr_t addr; + uint32_t len; +}; + +struct moxart_desc { + enum dma_transfer_direction dma_dir; + dma_addr_t dev_addr; + unsigned int sglen; + unsigned int dma_cycles; + struct virt_dma_desc vd; + uint8_t es; + struct moxart_sg sg[0]; +}; + +struct moxart_chan { + struct virt_dma_chan vc; + + void __iomem *base; + struct moxart_desc *desc; + + struct dma_slave_config cfg; + + bool allocated; + bool error; + int ch_num; + unsigned int line_reqno; + unsigned int sgidx; +}; + +struct moxart_dmadev { + struct dma_device dma_slave; + struct moxart_chan slave_chans[APB_DMA_MAX_CHANNEL]; +}; + +struct moxart_filter_data { + struct moxart_dmadev *mdc; + struct of_phandle_args *dma_spec; +}; + +static const unsigned int es_bytes[] = { + [MOXART_DMA_DATA_TYPE_S8] = 1, + [MOXART_DMA_DATA_TYPE_S16] = 2, + [MOXART_DMA_DATA_TYPE_S32] = 4, +}; + +static struct device *chan2dev(struct dma_chan *chan) +{ + return &chan->dev->device; +} + +static inline struct moxart_chan *to_moxart_dma_chan(struct dma_chan *c) +{ + return container_of(c, struct moxart_chan, vc.chan); +} + +static inline struct moxart_desc *to_moxart_dma_desc( + struct dma_async_tx_descriptor *t) +{ + return container_of(t, struct moxart_desc, vd.tx); +} + +static void moxart_dma_desc_free(struct virt_dma_desc *vd) +{ + kfree(container_of(vd, struct moxart_desc, vd)); +} + +static int moxart_terminate_all(struct dma_chan *chan) +{ + struct moxart_chan *ch = to_moxart_dma_chan(chan); + unsigned long flags; + LIST_HEAD(head); + u32 ctrl; + + dev_dbg(chan2dev(chan), "%s: ch=%p\n", __func__, ch); + + spin_lock_irqsave(&ch->vc.lock, flags); + + if (ch->desc) + ch->desc = NULL; + + ctrl = readl(ch->base + REG_OFF_CTRL); + ctrl &= ~(APB_DMA_ENABLE | APB_DMA_FIN_INT_EN | APB_DMA_ERR_INT_EN); + writel(ctrl, ch->base + REG_OFF_CTRL); + + vchan_get_all_descriptors(&ch->vc, &head); + spin_unlock_irqrestore(&ch->vc.lock, flags); + vchan_dma_desc_free_list(&ch->vc, &head); + + return 0; +} + +static int moxart_slave_config(struct dma_chan *chan, + struct dma_slave_config *cfg) +{ + struct moxart_chan *ch = to_moxart_dma_chan(chan); + u32 ctrl; + + ch->cfg = *cfg; + + ctrl = readl(ch->base + REG_OFF_CTRL); + ctrl |= APB_DMA_BURST_MODE; + ctrl &= ~(APB_DMA_DEST_MASK | APB_DMA_SOURCE_MASK); + ctrl &= ~(APB_DMA_DEST_REQ_NO_MASK | APB_DMA_SOURCE_REQ_NO_MASK); + + switch (ch->cfg.src_addr_width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + ctrl |= APB_DMA_DATA_WIDTH_1; + if (ch->cfg.direction != DMA_MEM_TO_DEV) + ctrl |= APB_DMA_DEST_INC_1_4; + else + ctrl |= APB_DMA_SOURCE_INC_1_4; + break; + case DMA_SLAVE_BUSWIDTH_2_BYTES: + ctrl |= APB_DMA_DATA_WIDTH_2; + if (ch->cfg.direction != DMA_MEM_TO_DEV) + ctrl |= APB_DMA_DEST_INC_2_8; + else + ctrl |= APB_DMA_SOURCE_INC_2_8; + break; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + ctrl &= ~APB_DMA_DATA_WIDTH; + if (ch->cfg.direction != DMA_MEM_TO_DEV) + ctrl |= APB_DMA_DEST_INC_4_16; + else + ctrl |= APB_DMA_SOURCE_INC_4_16; + break; + default: + return -EINVAL; + } + + if (ch->cfg.direction == DMA_MEM_TO_DEV) { + ctrl &= ~APB_DMA_DEST_SELECT; + ctrl |= APB_DMA_SOURCE_SELECT; + ctrl |= (ch->line_reqno << 16 & + APB_DMA_DEST_REQ_NO_MASK); + } else { + ctrl |= APB_DMA_DEST_SELECT; + ctrl &= ~APB_DMA_SOURCE_SELECT; + ctrl |= (ch->line_reqno << 24 & + APB_DMA_SOURCE_REQ_NO_MASK); + } + + writel(ctrl, ch->base + REG_OFF_CTRL); + + return 0; +} + +static int moxart_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + int ret = 0; + + switch (cmd) { + case DMA_PAUSE: + case DMA_RESUME: + return -EINVAL; + case DMA_TERMINATE_ALL: + moxart_terminate_all(chan); + break; + case DMA_SLAVE_CONFIG: + ret = moxart_slave_config(chan, (struct dma_slave_config *)arg); + break; + default: + ret = -ENOSYS; + } + + return ret; +} + +static struct dma_async_tx_descriptor *moxart_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long tx_flags, void *context) +{ + struct moxart_chan *ch = to_moxart_dma_chan(chan); + struct moxart_desc *d; + enum dma_slave_buswidth dev_width; + dma_addr_t dev_addr; + struct scatterlist *sgent; + unsigned int es; + unsigned int i; + + if (!is_slave_direction(dir)) { + dev_err(chan2dev(chan), "%s: invalid DMA direction\n", + __func__); + return NULL; + } + + if (dir == DMA_DEV_TO_MEM) { + dev_addr = ch->cfg.src_addr; + dev_width = ch->cfg.src_addr_width; + } else { + dev_addr = ch->cfg.dst_addr; + dev_width = ch->cfg.dst_addr_width; + } + + switch (dev_width) { + case DMA_SLAVE_BUSWIDTH_1_BYTE: + es = MOXART_DMA_DATA_TYPE_S8; + break; + case DMA_SLAVE_BUSWIDTH_2_BYTES: + es = MOXART_DMA_DATA_TYPE_S16; + break; + case DMA_SLAVE_BUSWIDTH_4_BYTES: + es = MOXART_DMA_DATA_TYPE_S32; + break; + default: + dev_err(chan2dev(chan), "%s: unsupported data width (%u)\n", + __func__, dev_width); + return NULL; + } + + d = kzalloc(sizeof(*d) + sg_len * sizeof(d->sg[0]), GFP_ATOMIC); + if (!d) + return NULL; + + d->dma_dir = dir; + d->dev_addr = dev_addr; + d->es = es; + + for_each_sg(sgl, sgent, sg_len, i) { + d->sg[i].addr = sg_dma_address(sgent); + d->sg[i].len = sg_dma_len(sgent); + } + + d->sglen = sg_len; + + ch->error = 0; + + return vchan_tx_prep(&ch->vc, &d->vd, tx_flags); +} + +static struct dma_chan *moxart_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct moxart_dmadev *mdc = ofdma->of_dma_data; + struct dma_chan *chan; + struct moxart_chan *ch; + + chan = dma_get_any_slave_channel(&mdc->dma_slave); + if (!chan) + return NULL; + + ch = to_moxart_dma_chan(chan); + ch->line_reqno = dma_spec->args[0]; + + return chan; +} + +static int moxart_alloc_chan_resources(struct dma_chan *chan) +{ + struct moxart_chan *ch = to_moxart_dma_chan(chan); + + dev_dbg(chan2dev(chan), "%s: allocating channel #%u\n", + __func__, ch->ch_num); + ch->allocated = 1; + + return 0; +} + +static void moxart_free_chan_resources(struct dma_chan *chan) +{ + struct moxart_chan *ch = to_moxart_dma_chan(chan); + + vchan_free_chan_resources(&ch->vc); + + dev_dbg(chan2dev(chan), "%s: freeing channel #%u\n", + __func__, ch->ch_num); + ch->allocated = 0; +} + +static void moxart_dma_set_params(struct moxart_chan *ch, dma_addr_t src_addr, + dma_addr_t dst_addr) +{ + writel(src_addr, ch->base + REG_OFF_ADDRESS_SOURCE); + writel(dst_addr, ch->base + REG_OFF_ADDRESS_DEST); +} + +static void moxart_set_transfer_params(struct moxart_chan *ch, unsigned int len) +{ + struct moxart_desc *d = ch->desc; + unsigned int sglen_div = es_bytes[d->es]; + + d->dma_cycles = len >> sglen_div; + + /* + * There are 4 cycles on 64 bytes copied, i.e. one cycle copies 16 + * bytes ( when width is APB_DMAB_DATA_WIDTH_4 ). + */ + writel(d->dma_cycles, ch->base + REG_OFF_CYCLES); + + dev_dbg(chan2dev(&ch->vc.chan), "%s: set %u DMA cycles (len=%u)\n", + __func__, d->dma_cycles, len); +} + +static void moxart_start_dma(struct moxart_chan *ch) +{ + u32 ctrl; + + ctrl = readl(ch->base + REG_OFF_CTRL); + ctrl |= (APB_DMA_ENABLE | APB_DMA_FIN_INT_EN | APB_DMA_ERR_INT_EN); + writel(ctrl, ch->base + REG_OFF_CTRL); +} + +static void moxart_dma_start_sg(struct moxart_chan *ch, unsigned int idx) +{ + struct moxart_desc *d = ch->desc; + struct moxart_sg *sg = ch->desc->sg + idx; + + if (ch->desc->dma_dir == DMA_MEM_TO_DEV) + moxart_dma_set_params(ch, sg->addr, d->dev_addr); + else if (ch->desc->dma_dir == DMA_DEV_TO_MEM) + moxart_dma_set_params(ch, d->dev_addr, sg->addr); + + moxart_set_transfer_params(ch, sg->len); + + moxart_start_dma(ch); +} + +static void moxart_dma_start_desc(struct dma_chan *chan) +{ + struct moxart_chan *ch = to_moxart_dma_chan(chan); + struct virt_dma_desc *vd; + + vd = vchan_next_desc(&ch->vc); + + if (!vd) { + ch->desc = NULL; + return; + } + + list_del(&vd->node); + + ch->desc = to_moxart_dma_desc(&vd->tx); + ch->sgidx = 0; + + moxart_dma_start_sg(ch, 0); +} + +static void moxart_issue_pending(struct dma_chan *chan) +{ + struct moxart_chan *ch = to_moxart_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&ch->vc.lock, flags); + if (vchan_issue_pending(&ch->vc) && !ch->desc) + moxart_dma_start_desc(chan); + spin_unlock_irqrestore(&ch->vc.lock, flags); +} + +static size_t moxart_dma_desc_size(struct moxart_desc *d, + unsigned int completed_sgs) +{ + unsigned int i; + size_t size; + + for (size = i = completed_sgs; i < d->sglen; i++) + size += d->sg[i].len; + + return size; +} + +static size_t moxart_dma_desc_size_in_flight(struct moxart_chan *ch) +{ + size_t size; + unsigned int completed_cycles, cycles; + + size = moxart_dma_desc_size(ch->desc, ch->sgidx); + cycles = readl(ch->base + REG_OFF_CYCLES); + completed_cycles = (ch->desc->dma_cycles - cycles); + size -= completed_cycles << es_bytes[ch->desc->es]; + + dev_dbg(chan2dev(&ch->vc.chan), "%s: size=%zu\n", __func__, size); + + return size; +} + +static enum dma_status moxart_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct moxart_chan *ch = to_moxart_dma_chan(chan); + struct virt_dma_desc *vd; + struct moxart_desc *d; + enum dma_status ret; + unsigned long flags; + + /* + * dma_cookie_status() assigns initial residue value. + */ + ret = dma_cookie_status(chan, cookie, txstate); + + spin_lock_irqsave(&ch->vc.lock, flags); + vd = vchan_find_desc(&ch->vc, cookie); + if (vd) { + d = to_moxart_dma_desc(&vd->tx); + txstate->residue = moxart_dma_desc_size(d, 0); + } else if (ch->desc && ch->desc->vd.tx.cookie == cookie) { + txstate->residue = moxart_dma_desc_size_in_flight(ch); + } + spin_unlock_irqrestore(&ch->vc.lock, flags); + + if (ch->error) + return DMA_ERROR; + + return ret; +} + +static void moxart_dma_init(struct dma_device *dma, struct device *dev) +{ + dma->device_prep_slave_sg = moxart_prep_slave_sg; + dma->device_alloc_chan_resources = moxart_alloc_chan_resources; + dma->device_free_chan_resources = moxart_free_chan_resources; + dma->device_issue_pending = moxart_issue_pending; + dma->device_tx_status = moxart_tx_status; + dma->device_control = moxart_control; + dma->dev = dev; + + INIT_LIST_HEAD(&dma->channels); +} + +static irqreturn_t moxart_dma_interrupt(int irq, void *devid) +{ + struct moxart_dmadev *mc = devid; + struct moxart_chan *ch = &mc->slave_chans[0]; + unsigned int i; + unsigned long flags; + u32 ctrl; + + dev_dbg(chan2dev(&ch->vc.chan), "%s\n", __func__); + + for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) { + if (!ch->allocated) + continue; + + ctrl = readl(ch->base + REG_OFF_CTRL); + + dev_dbg(chan2dev(&ch->vc.chan), "%s: ch=%p ch->base=%p ctrl=%x\n", + __func__, ch, ch->base, ctrl); + + if (ctrl & APB_DMA_FIN_INT_STS) { + ctrl &= ~APB_DMA_FIN_INT_STS; + if (ch->desc) { + spin_lock_irqsave(&ch->vc.lock, flags); + if (++ch->sgidx < ch->desc->sglen) { + moxart_dma_start_sg(ch, ch->sgidx); + } else { + vchan_cookie_complete(&ch->desc->vd); + moxart_dma_start_desc(&ch->vc.chan); + } + spin_unlock_irqrestore(&ch->vc.lock, flags); + } + } + + if (ctrl & APB_DMA_ERR_INT_STS) { + ctrl &= ~APB_DMA_ERR_INT_STS; + ch->error = 1; + } + + writel(ctrl, ch->base + REG_OFF_CTRL); + } + + return IRQ_HANDLED; +} + +static int moxart_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct device_node *node = dev->of_node; + struct resource *res; + static void __iomem *dma_base_addr; + int ret, i; + unsigned int irq; + struct moxart_chan *ch; + struct moxart_dmadev *mdc; + + mdc = devm_kzalloc(dev, sizeof(*mdc), GFP_KERNEL); + if (!mdc) { + dev_err(dev, "can't allocate DMA container\n"); + return -ENOMEM; + } + + irq = irq_of_parse_and_map(node, 0); + if (irq == NO_IRQ) { + dev_err(dev, "no IRQ resource\n"); + return -EINVAL; + } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + dma_base_addr = devm_ioremap_resource(dev, res); + if (IS_ERR(dma_base_addr)) + return PTR_ERR(dma_base_addr); + + dma_cap_zero(mdc->dma_slave.cap_mask); + dma_cap_set(DMA_SLAVE, mdc->dma_slave.cap_mask); + dma_cap_set(DMA_PRIVATE, mdc->dma_slave.cap_mask); + + moxart_dma_init(&mdc->dma_slave, dev); + + ch = &mdc->slave_chans[0]; + for (i = 0; i < APB_DMA_MAX_CHANNEL; i++, ch++) { + ch->ch_num = i; + ch->base = dma_base_addr + i * REG_OFF_CHAN_SIZE; + ch->allocated = 0; + + ch->vc.desc_free = moxart_dma_desc_free; + vchan_init(&ch->vc, &mdc->dma_slave); + + dev_dbg(dev, "%s: chs[%d]: ch->ch_num=%u ch->base=%p\n", + __func__, i, ch->ch_num, ch->base); + } + + platform_set_drvdata(pdev, mdc); + + ret = devm_request_irq(dev, irq, moxart_dma_interrupt, 0, + "moxart-dma-engine", mdc); + if (ret) { + dev_err(dev, "devm_request_irq failed\n"); + return ret; + } + + ret = dma_async_device_register(&mdc->dma_slave); + if (ret) { + dev_err(dev, "dma_async_device_register failed\n"); + return ret; + } + + ret = of_dma_controller_register(node, moxart_of_xlate, mdc); + if (ret) { + dev_err(dev, "of_dma_controller_register failed\n"); + dma_async_device_unregister(&mdc->dma_slave); + return ret; + } + + dev_dbg(dev, "%s: IRQ=%u\n", __func__, irq); + + return 0; +} + +static int moxart_remove(struct platform_device *pdev) +{ + struct moxart_dmadev *m = platform_get_drvdata(pdev); + + dma_async_device_unregister(&m->dma_slave); + + if (pdev->dev.of_node) + of_dma_controller_free(pdev->dev.of_node); + + return 0; +} + +static const struct of_device_id moxart_dma_match[] = { + { .compatible = "moxa,moxart-dma" }, + { } +}; + +static struct platform_driver moxart_driver = { + .probe = moxart_probe, + .remove = moxart_remove, + .driver = { + .name = "moxart-dma-engine", + .owner = THIS_MODULE, + .of_match_table = moxart_dma_match, + }, +}; + +static int moxart_init(void) +{ + return platform_driver_register(&moxart_driver); +} +subsys_initcall(moxart_init); + +static void __exit moxart_exit(void) +{ + platform_driver_unregister(&moxart_driver); +} +module_exit(moxart_exit); + +MODULE_AUTHOR("Jonas Jensen <jonas.jensen@gmail.com>"); +MODULE_DESCRIPTION("MOXART DMA engine driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/mpc512x_dma.c b/drivers/dma/mpc512x_dma.c index 2fe43537733..2ad43738ac8 100644 --- a/drivers/dma/mpc512x_dma.c +++ b/drivers/dma/mpc512x_dma.c @@ -2,6 +2,7 @@ * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008. * Copyright (C) Semihalf 2009 * Copyright (C) Ilya Yanok, Emcraft Systems 2010 + * Copyright (C) Alexander Popov, Promcontroller 2014 * * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description * (defines, structures and comments) was taken from MPC5121 DMA driver @@ -29,8 +30,18 @@ */ /* - * This is initial version of MPC5121 DMA driver. Only memory to memory - * transfers are supported (tested using dmatest module). + * MPC512x and MPC8308 DMA driver. It supports + * memory to memory data transfers (tested using dmatest module) and + * data transfers between memory and peripheral I/O memory + * by means of slave scatter/gather with these limitations: + * - chunked transfers (described by s/g lists with more than one item) + * are refused as long as proper support for scatter/gather is missing; + * - transfers on MPC8308 always start from software as this SoC appears + * not to have external request lines for peripheral flow control; + * - only peripheral devices with 4-byte FIFO access register are supported; + * - minimal memory <-> I/O memory transfer chunk is 4 bytes and consequently + * source and destination addresses must be 4-byte aligned + * and transfer size must be aligned on (4 * maxburst) boundary; */ #include <linux/module.h> @@ -39,7 +50,9 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/slab.h> +#include <linux/of_address.h> #include <linux/of_device.h> +#include <linux/of_irq.h> #include <linux/of_platform.h> #include <linux/random.h> @@ -50,9 +63,17 @@ #define MPC_DMA_DESCRIPTORS 64 /* Macro definitions */ -#define MPC_DMA_CHANNELS 64 #define MPC_DMA_TCD_OFFSET 0x1000 +/* + * Maximum channel counts for individual hardware variants + * and the maximum channel count over all supported controllers, + * used for data structure size + */ +#define MPC8308_DMACHAN_MAX 16 +#define MPC512x_DMACHAN_MAX 64 +#define MPC_DMA_CHANNELS 64 + /* Arbitration mode of group and channel */ #define MPC_DMA_DMACR_EDCG (1 << 31) #define MPC_DMA_DMACR_ERGA (1 << 3) @@ -179,6 +200,7 @@ struct mpc_dma_desc { dma_addr_t tcd_paddr; int error; struct list_head node; + int will_access_peripheral; }; struct mpc_dma_chan { @@ -191,6 +213,12 @@ struct mpc_dma_chan { struct mpc_dma_tcd *tcd; dma_addr_t tcd_paddr; + /* Settings for access to peripheral FIFO */ + dma_addr_t src_per_paddr; + u32 src_tcd_nunits; + dma_addr_t dst_per_paddr; + u32 dst_tcd_nunits; + /* Lock for this structure */ spinlock_t lock; }; @@ -241,8 +269,23 @@ static void mpc_dma_execute(struct mpc_dma_chan *mchan) struct mpc_dma_desc *mdesc; int cid = mchan->chan.chan_id; - /* Move all queued descriptors to active list */ - list_splice_tail_init(&mchan->queued, &mchan->active); + while (!list_empty(&mchan->queued)) { + mdesc = list_first_entry(&mchan->queued, + struct mpc_dma_desc, node); + /* + * Grab either several mem-to-mem transfer descriptors + * or one peripheral transfer descriptor, + * don't mix mem-to-mem and peripheral transfer descriptors + * within the same 'active' list. + */ + if (mdesc->will_access_peripheral) { + if (list_empty(&mchan->active)) + list_move_tail(&mdesc->node, &mchan->active); + break; + } else { + list_move_tail(&mdesc->node, &mchan->active); + } + } /* Chain descriptors into one transaction */ list_for_each_entry(mdesc, &mchan->active, node) { @@ -268,7 +311,17 @@ static void mpc_dma_execute(struct mpc_dma_chan *mchan) if (first != prev) mdma->tcd[cid].e_sg = 1; - out_8(&mdma->regs->dmassrt, cid); + + if (mdma->is_mpc8308) { + /* MPC8308, no request lines, software initiated start */ + out_8(&mdma->regs->dmassrt, cid); + } else if (first->will_access_peripheral) { + /* Peripherals involved, start by external request signal */ + out_8(&mdma->regs->dmaserq, cid); + } else { + /* Memory to memory transfer, software initiated start */ + out_8(&mdma->regs->dmassrt, cid); + } } /* Handle interrupt on one half of DMA controller (32 channels) */ @@ -586,6 +639,7 @@ mpc_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, } mdesc->error = 0; + mdesc->will_access_peripheral = 0; tcd = mdesc->tcd; /* Prepare Transfer Control Descriptor for this transaction */ @@ -633,6 +687,193 @@ mpc_dma_prep_memcpy(struct dma_chan *chan, dma_addr_t dst, dma_addr_t src, return &mdesc->desc; } +static struct dma_async_tx_descriptor * +mpc_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct mpc_dma *mdma = dma_chan_to_mpc_dma(chan); + struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan); + struct mpc_dma_desc *mdesc = NULL; + dma_addr_t per_paddr; + u32 tcd_nunits; + struct mpc_dma_tcd *tcd; + unsigned long iflags; + struct scatterlist *sg; + size_t len; + int iter, i; + + /* Currently there is no proper support for scatter/gather */ + if (sg_len != 1) + return NULL; + + if (!is_slave_direction(direction)) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) { + spin_lock_irqsave(&mchan->lock, iflags); + + mdesc = list_first_entry(&mchan->free, + struct mpc_dma_desc, node); + if (!mdesc) { + spin_unlock_irqrestore(&mchan->lock, iflags); + /* Try to free completed descriptors */ + mpc_dma_process_completed(mdma); + return NULL; + } + + list_del(&mdesc->node); + + if (direction == DMA_DEV_TO_MEM) { + per_paddr = mchan->src_per_paddr; + tcd_nunits = mchan->src_tcd_nunits; + } else { + per_paddr = mchan->dst_per_paddr; + tcd_nunits = mchan->dst_tcd_nunits; + } + + spin_unlock_irqrestore(&mchan->lock, iflags); + + if (per_paddr == 0 || tcd_nunits == 0) + goto err_prep; + + mdesc->error = 0; + mdesc->will_access_peripheral = 1; + + /* Prepare Transfer Control Descriptor for this transaction */ + tcd = mdesc->tcd; + + memset(tcd, 0, sizeof(struct mpc_dma_tcd)); + + if (!IS_ALIGNED(sg_dma_address(sg), 4)) + goto err_prep; + + if (direction == DMA_DEV_TO_MEM) { + tcd->saddr = per_paddr; + tcd->daddr = sg_dma_address(sg); + tcd->soff = 0; + tcd->doff = 4; + } else { + tcd->saddr = sg_dma_address(sg); + tcd->daddr = per_paddr; + tcd->soff = 4; + tcd->doff = 0; + } + + tcd->ssize = MPC_DMA_TSIZE_4; + tcd->dsize = MPC_DMA_TSIZE_4; + + len = sg_dma_len(sg); + tcd->nbytes = tcd_nunits * 4; + if (!IS_ALIGNED(len, tcd->nbytes)) + goto err_prep; + + iter = len / tcd->nbytes; + if (iter >= 1 << 15) { + /* len is too big */ + goto err_prep; + } + /* citer_linkch contains the high bits of iter */ + tcd->biter = iter & 0x1ff; + tcd->biter_linkch = iter >> 9; + tcd->citer = tcd->biter; + tcd->citer_linkch = tcd->biter_linkch; + + tcd->e_sg = 0; + tcd->d_req = 1; + + /* Place descriptor in prepared list */ + spin_lock_irqsave(&mchan->lock, iflags); + list_add_tail(&mdesc->node, &mchan->prepared); + spin_unlock_irqrestore(&mchan->lock, iflags); + } + + return &mdesc->desc; + +err_prep: + /* Put the descriptor back */ + spin_lock_irqsave(&mchan->lock, iflags); + list_add_tail(&mdesc->node, &mchan->free); + spin_unlock_irqrestore(&mchan->lock, iflags); + + return NULL; +} + +static int mpc_dma_device_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct mpc_dma_chan *mchan; + struct mpc_dma *mdma; + struct dma_slave_config *cfg; + unsigned long flags; + + mchan = dma_chan_to_mpc_dma_chan(chan); + switch (cmd) { + case DMA_TERMINATE_ALL: + /* Disable channel requests */ + mdma = dma_chan_to_mpc_dma(chan); + + spin_lock_irqsave(&mchan->lock, flags); + + out_8(&mdma->regs->dmacerq, chan->chan_id); + list_splice_tail_init(&mchan->prepared, &mchan->free); + list_splice_tail_init(&mchan->queued, &mchan->free); + list_splice_tail_init(&mchan->active, &mchan->free); + + spin_unlock_irqrestore(&mchan->lock, flags); + + return 0; + + case DMA_SLAVE_CONFIG: + /* + * Software constraints: + * - only transfers between a peripheral device and + * memory are supported; + * - only peripheral devices with 4-byte FIFO access register + * are supported; + * - minimal transfer chunk is 4 bytes and consequently + * source and destination addresses must be 4-byte aligned + * and transfer size must be aligned on (4 * maxburst) + * boundary; + * - during the transfer RAM address is being incremented by + * the size of minimal transfer chunk; + * - peripheral port's address is constant during the transfer. + */ + + cfg = (void *)arg; + + if (cfg->src_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES || + cfg->dst_addr_width != DMA_SLAVE_BUSWIDTH_4_BYTES || + !IS_ALIGNED(cfg->src_addr, 4) || + !IS_ALIGNED(cfg->dst_addr, 4)) { + return -EINVAL; + } + + spin_lock_irqsave(&mchan->lock, flags); + + mchan->src_per_paddr = cfg->src_addr; + mchan->src_tcd_nunits = cfg->src_maxburst; + mchan->dst_per_paddr = cfg->dst_addr; + mchan->dst_tcd_nunits = cfg->dst_maxburst; + + /* Apply defaults */ + if (mchan->src_tcd_nunits == 0) + mchan->src_tcd_nunits = 1; + if (mchan->dst_tcd_nunits == 0) + mchan->dst_tcd_nunits = 1; + + spin_unlock_irqrestore(&mchan->lock, flags); + + return 0; + + default: + /* Unknown command */ + break; + } + + return -ENXIO; +} + static int mpc_dma_probe(struct platform_device *op) { struct device_node *dn = op->dev.of_node; @@ -647,13 +888,15 @@ static int mpc_dma_probe(struct platform_device *op) mdma = devm_kzalloc(dev, sizeof(struct mpc_dma), GFP_KERNEL); if (!mdma) { dev_err(dev, "Memory exhausted!\n"); - return -ENOMEM; + retval = -ENOMEM; + goto err; } mdma->irq = irq_of_parse_and_map(dn, 0); if (mdma->irq == NO_IRQ) { dev_err(dev, "Error mapping IRQ!\n"); - return -EINVAL; + retval = -EINVAL; + goto err; } if (of_device_is_compatible(dn, "fsl,mpc8308-dma")) { @@ -661,14 +904,15 @@ static int mpc_dma_probe(struct platform_device *op) mdma->irq2 = irq_of_parse_and_map(dn, 1); if (mdma->irq2 == NO_IRQ) { dev_err(dev, "Error mapping IRQ!\n"); - return -EINVAL; + retval = -EINVAL; + goto err_dispose1; } } retval = of_address_to_resource(dn, 0, &res); if (retval) { dev_err(dev, "Error parsing memory region!\n"); - return retval; + goto err_dispose2; } regs_start = res.start; @@ -676,31 +920,34 @@ static int mpc_dma_probe(struct platform_device *op) if (!devm_request_mem_region(dev, regs_start, regs_size, DRV_NAME)) { dev_err(dev, "Error requesting memory region!\n"); - return -EBUSY; + retval = -EBUSY; + goto err_dispose2; } mdma->regs = devm_ioremap(dev, regs_start, regs_size); if (!mdma->regs) { dev_err(dev, "Error mapping memory region!\n"); - return -ENOMEM; + retval = -ENOMEM; + goto err_dispose2; } mdma->tcd = (struct mpc_dma_tcd *)((u8 *)(mdma->regs) + MPC_DMA_TCD_OFFSET); - retval = devm_request_irq(dev, mdma->irq, &mpc_dma_irq, 0, DRV_NAME, - mdma); + retval = request_irq(mdma->irq, &mpc_dma_irq, 0, DRV_NAME, mdma); if (retval) { dev_err(dev, "Error requesting IRQ!\n"); - return -EINVAL; + retval = -EINVAL; + goto err_dispose2; } if (mdma->is_mpc8308) { - retval = devm_request_irq(dev, mdma->irq2, &mpc_dma_irq, 0, - DRV_NAME, mdma); + retval = request_irq(mdma->irq2, &mpc_dma_irq, 0, + DRV_NAME, mdma); if (retval) { dev_err(dev, "Error requesting IRQ2!\n"); - return -EINVAL; + retval = -EINVAL; + goto err_free1; } } @@ -708,18 +955,21 @@ static int mpc_dma_probe(struct platform_device *op) dma = &mdma->dma; dma->dev = dev; - if (!mdma->is_mpc8308) - dma->chancnt = MPC_DMA_CHANNELS; + if (mdma->is_mpc8308) + dma->chancnt = MPC8308_DMACHAN_MAX; else - dma->chancnt = 16; /* MPC8308 DMA has only 16 channels */ + dma->chancnt = MPC512x_DMACHAN_MAX; dma->device_alloc_chan_resources = mpc_dma_alloc_chan_resources; dma->device_free_chan_resources = mpc_dma_free_chan_resources; dma->device_issue_pending = mpc_dma_issue_pending; dma->device_tx_status = mpc_dma_tx_status; dma->device_prep_dma_memcpy = mpc_dma_prep_memcpy; + dma->device_prep_slave_sg = mpc_dma_prep_slave_sg; + dma->device_control = mpc_dma_device_control; INIT_LIST_HEAD(&dma->channels); dma_cap_set(DMA_MEMCPY, dma->cap_mask); + dma_cap_set(DMA_SLAVE, dma->cap_mask); for (i = 0; i < dma->chancnt; i++) { mchan = &mdma->channels[i]; @@ -745,7 +995,19 @@ static int mpc_dma_probe(struct platform_device *op) * - Round-robin group arbitration, * - Round-robin channel arbitration. */ - if (!mdma->is_mpc8308) { + if (mdma->is_mpc8308) { + /* MPC8308 has 16 channels and lacks some registers */ + out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_ERCA); + + /* enable snooping */ + out_be32(&mdma->regs->dmagpor, MPC_DMA_DMAGPOR_SNOOP_ENABLE); + /* Disable error interrupts */ + out_be32(&mdma->regs->dmaeeil, 0); + + /* Clear interrupts status */ + out_be32(&mdma->regs->dmaintl, 0xFFFF); + out_be32(&mdma->regs->dmaerrl, 0xFFFF); + } else { out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_EDCG | MPC_DMA_DMACR_ERGA | MPC_DMA_DMACR_ERCA); @@ -766,29 +1028,28 @@ static int mpc_dma_probe(struct platform_device *op) /* Route interrupts to IPIC */ out_be32(&mdma->regs->dmaihsa, 0); out_be32(&mdma->regs->dmailsa, 0); - } else { - /* MPC8308 has 16 channels and lacks some registers */ - out_be32(&mdma->regs->dmacr, MPC_DMA_DMACR_ERCA); - - /* enable snooping */ - out_be32(&mdma->regs->dmagpor, MPC_DMA_DMAGPOR_SNOOP_ENABLE); - /* Disable error interrupts */ - out_be32(&mdma->regs->dmaeeil, 0); - - /* Clear interrupts status */ - out_be32(&mdma->regs->dmaintl, 0xFFFF); - out_be32(&mdma->regs->dmaerrl, 0xFFFF); } /* Register DMA engine */ dev_set_drvdata(dev, mdma); retval = dma_async_device_register(dma); - if (retval) { - devm_free_irq(dev, mdma->irq, mdma); - irq_dispose_mapping(mdma->irq); - } + if (retval) + goto err_free2; return retval; + +err_free2: + if (mdma->is_mpc8308) + free_irq(mdma->irq2, mdma); +err_free1: + free_irq(mdma->irq, mdma); +err_dispose2: + if (mdma->is_mpc8308) + irq_dispose_mapping(mdma->irq2); +err_dispose1: + irq_dispose_mapping(mdma->irq); +err: + return retval; } static int mpc_dma_remove(struct platform_device *op) @@ -797,7 +1058,11 @@ static int mpc_dma_remove(struct platform_device *op) struct mpc_dma *mdma = dev_get_drvdata(dev); dma_async_device_unregister(&mdma->dma); - devm_free_irq(dev, mdma->irq, mdma); + if (mdma->is_mpc8308) { + free_irq(mdma->irq2, mdma); + irq_dispose_mapping(mdma->irq2); + } + free_irq(mdma->irq, mdma); irq_dispose_mapping(mdma->irq); return 0; @@ -805,6 +1070,7 @@ static int mpc_dma_remove(struct platform_device *op) static struct of_device_id mpc_dma_match[] = { { .compatible = "fsl,mpc5121-dma", }, + { .compatible = "fsl,mpc8308-dma", }, {}, }; diff --git a/drivers/dma/mv_xor.c b/drivers/dma/mv_xor.c index 536dcb8ba5f..394cbc5c93e 100644 --- a/drivers/dma/mv_xor.c +++ b/drivers/dma/mv_xor.c @@ -54,20 +54,6 @@ static void mv_desc_init(struct mv_xor_desc_slot *desc, unsigned long flags) hw_desc->desc_command = (1 << 31); } -static u32 mv_desc_get_dest_addr(struct mv_xor_desc_slot *desc) -{ - struct mv_xor_desc *hw_desc = desc->hw_desc; - return hw_desc->phy_dest_addr; -} - -static u32 mv_desc_get_src_addr(struct mv_xor_desc_slot *desc, - int src_idx) -{ - struct mv_xor_desc *hw_desc = desc->hw_desc; - return hw_desc->phy_src_addr[mv_phy_src_idx(src_idx)]; -} - - static void mv_desc_set_byte_count(struct mv_xor_desc_slot *desc, u32 byte_count) { @@ -205,12 +191,10 @@ static void mv_set_mode(struct mv_xor_chan *chan, static void mv_chan_activate(struct mv_xor_chan *chan) { - u32 activation; - dev_dbg(mv_chan_to_devp(chan), " activate chan.\n"); - activation = readl_relaxed(XOR_ACTIVATION(chan)); - activation |= 0x1; - writel_relaxed(activation, XOR_ACTIVATION(chan)); + + /* writel ensures all descriptors are flushed before activation */ + writel(BIT(0), XOR_ACTIVATION(chan)); } static char mv_chan_is_busy(struct mv_xor_chan *chan) @@ -278,42 +262,9 @@ mv_xor_run_tx_complete_actions(struct mv_xor_desc_slot *desc, desc->async_tx.callback( desc->async_tx.callback_param); - /* unmap dma addresses - * (unmap_single vs unmap_page?) - */ - if (desc->group_head && desc->unmap_len) { - struct mv_xor_desc_slot *unmap = desc->group_head; - struct device *dev = mv_chan_to_devp(mv_chan); - u32 len = unmap->unmap_len; - enum dma_ctrl_flags flags = desc->async_tx.flags; - u32 src_cnt; - dma_addr_t addr; - dma_addr_t dest; - - src_cnt = unmap->unmap_src_cnt; - dest = mv_desc_get_dest_addr(unmap); - if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - enum dma_data_direction dir; - - if (src_cnt > 1) /* is xor ? */ - dir = DMA_BIDIRECTIONAL; - else - dir = DMA_FROM_DEVICE; - dma_unmap_page(dev, dest, len, dir); - } - - if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - while (src_cnt--) { - addr = mv_desc_get_src_addr(unmap, - src_cnt); - if (addr == dest) - continue; - dma_unmap_page(dev, addr, len, - DMA_TO_DEVICE); - } - } + dma_descriptor_unmap(&desc->async_tx); + if (desc->group_head) desc->group_head = NULL; - } } /* run dependent operations */ @@ -544,8 +495,8 @@ mv_xor_tx_submit(struct dma_async_tx_descriptor *tx) if (!mv_can_chain(grp_start)) goto submit_done; - dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %x\n", - old_chain_tail->async_tx.phys); + dev_dbg(mv_chan_to_devp(mv_chan), "Append to last desc %pa\n", + &old_chain_tail->async_tx.phys); /* fix up the hardware chain */ mv_desc_set_next_desc(old_chain_tail, grp_start->async_tx.phys); @@ -574,7 +525,8 @@ submit_done: /* returns the number of allocated descriptors */ static int mv_xor_alloc_chan_resources(struct dma_chan *chan) { - char *hw_desc; + void *virt_desc; + dma_addr_t dma_desc; int idx; struct mv_xor_chan *mv_chan = to_mv_xor_chan(chan); struct mv_xor_desc_slot *slot = NULL; @@ -589,17 +541,16 @@ static int mv_xor_alloc_chan_resources(struct dma_chan *chan) " %d descriptor slots", idx); break; } - hw_desc = (char *) mv_chan->dma_desc_pool_virt; - slot->hw_desc = (void *) &hw_desc[idx * MV_XOR_SLOT_SIZE]; + virt_desc = mv_chan->dma_desc_pool_virt; + slot->hw_desc = virt_desc + idx * MV_XOR_SLOT_SIZE; dma_async_tx_descriptor_init(&slot->async_tx, chan); slot->async_tx.tx_submit = mv_xor_tx_submit; INIT_LIST_HEAD(&slot->chain_node); INIT_LIST_HEAD(&slot->slot_node); INIT_LIST_HEAD(&slot->tx_list); - hw_desc = (char *) mv_chan->dma_desc_pool; - slot->async_tx.phys = - (dma_addr_t) &hw_desc[idx * MV_XOR_SLOT_SIZE]; + dma_desc = mv_chan->dma_desc_pool; + slot->async_tx.phys = dma_desc + idx * MV_XOR_SLOT_SIZE; slot->idx = idx++; spin_lock_bh(&mv_chan->lock); @@ -629,8 +580,8 @@ mv_xor_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, int slot_cnt; dev_dbg(mv_chan_to_devp(mv_chan), - "%s dest: %x src %x len: %u flags: %ld\n", - __func__, dest, src, len, flags); + "%s dest: %pad src %pad len: %u flags: %ld\n", + __func__, &dest, &src, len, flags); if (unlikely(len < MV_XOR_MIN_BYTE_COUNT)) return NULL; @@ -673,8 +624,8 @@ mv_xor_prep_dma_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, BUG_ON(len > MV_XOR_MAX_BYTE_COUNT); dev_dbg(mv_chan_to_devp(mv_chan), - "%s src_cnt: %d len: dest %x %u flags: %ld\n", - __func__, src_cnt, len, dest, flags); + "%s src_cnt: %d len: %u dest %pad flags: %ld\n", + __func__, src_cnt, len, &dest, flags); spin_lock_bh(&mv_chan->lock); slot_cnt = mv_chan_xor_slot_count(len, src_cnt); @@ -749,7 +700,7 @@ static enum dma_status mv_xor_status(struct dma_chan *chan, enum dma_status ret; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS) { + if (ret == DMA_COMPLETE) { mv_xor_clean_completed_slots(mv_chan); return ret; } @@ -828,7 +779,6 @@ static void mv_xor_issue_pending(struct dma_chan *chan) /* * Perform a transaction to verify the HW works. */ -#define MV_XOR_TEST_SIZE 2000 static int mv_xor_memcpy_self_test(struct mv_xor_chan *mv_chan) { @@ -838,20 +788,21 @@ static int mv_xor_memcpy_self_test(struct mv_xor_chan *mv_chan) struct dma_chan *dma_chan; dma_cookie_t cookie; struct dma_async_tx_descriptor *tx; + struct dmaengine_unmap_data *unmap; int err = 0; - src = kmalloc(sizeof(u8) * MV_XOR_TEST_SIZE, GFP_KERNEL); + src = kmalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL); if (!src) return -ENOMEM; - dest = kzalloc(sizeof(u8) * MV_XOR_TEST_SIZE, GFP_KERNEL); + dest = kzalloc(sizeof(u8) * PAGE_SIZE, GFP_KERNEL); if (!dest) { kfree(src); return -ENOMEM; } /* Fill in src buffer */ - for (i = 0; i < MV_XOR_TEST_SIZE; i++) + for (i = 0; i < PAGE_SIZE; i++) ((u8 *) src)[i] = (u8)i; dma_chan = &mv_chan->dmachan; @@ -860,21 +811,33 @@ static int mv_xor_memcpy_self_test(struct mv_xor_chan *mv_chan) goto out; } - dest_dma = dma_map_single(dma_chan->device->dev, dest, - MV_XOR_TEST_SIZE, DMA_FROM_DEVICE); + unmap = dmaengine_get_unmap_data(dma_chan->device->dev, 2, GFP_KERNEL); + if (!unmap) { + err = -ENOMEM; + goto free_resources; + } - src_dma = dma_map_single(dma_chan->device->dev, src, - MV_XOR_TEST_SIZE, DMA_TO_DEVICE); + src_dma = dma_map_page(dma_chan->device->dev, virt_to_page(src), 0, + PAGE_SIZE, DMA_TO_DEVICE); + unmap->to_cnt = 1; + unmap->addr[0] = src_dma; + + dest_dma = dma_map_page(dma_chan->device->dev, virt_to_page(dest), 0, + PAGE_SIZE, DMA_FROM_DEVICE); + unmap->from_cnt = 1; + unmap->addr[1] = dest_dma; + + unmap->len = PAGE_SIZE; tx = mv_xor_prep_dma_memcpy(dma_chan, dest_dma, src_dma, - MV_XOR_TEST_SIZE, 0); + PAGE_SIZE, 0); cookie = mv_xor_tx_submit(tx); mv_xor_issue_pending(dma_chan); async_tx_ack(tx); msleep(1); if (mv_xor_status(dma_chan, cookie, NULL) != - DMA_SUCCESS) { + DMA_COMPLETE) { dev_err(dma_chan->device->dev, "Self-test copy timed out, disabling\n"); err = -ENODEV; @@ -882,8 +845,8 @@ static int mv_xor_memcpy_self_test(struct mv_xor_chan *mv_chan) } dma_sync_single_for_cpu(dma_chan->device->dev, dest_dma, - MV_XOR_TEST_SIZE, DMA_FROM_DEVICE); - if (memcmp(src, dest, MV_XOR_TEST_SIZE)) { + PAGE_SIZE, DMA_FROM_DEVICE); + if (memcmp(src, dest, PAGE_SIZE)) { dev_err(dma_chan->device->dev, "Self-test copy failed compare, disabling\n"); err = -ENODEV; @@ -891,6 +854,7 @@ static int mv_xor_memcpy_self_test(struct mv_xor_chan *mv_chan) } free_resources: + dmaengine_unmap_put(unmap); mv_xor_free_chan_resources(dma_chan); out: kfree(src); @@ -908,13 +872,15 @@ mv_xor_xor_self_test(struct mv_xor_chan *mv_chan) dma_addr_t dma_srcs[MV_XOR_NUM_SRC_TEST]; dma_addr_t dest_dma; struct dma_async_tx_descriptor *tx; + struct dmaengine_unmap_data *unmap; struct dma_chan *dma_chan; dma_cookie_t cookie; u8 cmp_byte = 0; u32 cmp_word; int err = 0; + int src_count = MV_XOR_NUM_SRC_TEST; - for (src_idx = 0; src_idx < MV_XOR_NUM_SRC_TEST; src_idx++) { + for (src_idx = 0; src_idx < src_count; src_idx++) { xor_srcs[src_idx] = alloc_page(GFP_KERNEL); if (!xor_srcs[src_idx]) { while (src_idx--) @@ -931,13 +897,13 @@ mv_xor_xor_self_test(struct mv_xor_chan *mv_chan) } /* Fill in src buffers */ - for (src_idx = 0; src_idx < MV_XOR_NUM_SRC_TEST; src_idx++) { + for (src_idx = 0; src_idx < src_count; src_idx++) { u8 *ptr = page_address(xor_srcs[src_idx]); for (i = 0; i < PAGE_SIZE; i++) ptr[i] = (1 << src_idx); } - for (src_idx = 0; src_idx < MV_XOR_NUM_SRC_TEST; src_idx++) + for (src_idx = 0; src_idx < src_count; src_idx++) cmp_byte ^= (u8) (1 << src_idx); cmp_word = (cmp_byte << 24) | (cmp_byte << 16) | @@ -951,16 +917,29 @@ mv_xor_xor_self_test(struct mv_xor_chan *mv_chan) goto out; } + unmap = dmaengine_get_unmap_data(dma_chan->device->dev, src_count + 1, + GFP_KERNEL); + if (!unmap) { + err = -ENOMEM; + goto free_resources; + } + /* test xor */ - dest_dma = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE, - DMA_FROM_DEVICE); + for (i = 0; i < src_count; i++) { + unmap->addr[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], + 0, PAGE_SIZE, DMA_TO_DEVICE); + dma_srcs[i] = unmap->addr[i]; + unmap->to_cnt++; + } - for (i = 0; i < MV_XOR_NUM_SRC_TEST; i++) - dma_srcs[i] = dma_map_page(dma_chan->device->dev, xor_srcs[i], - 0, PAGE_SIZE, DMA_TO_DEVICE); + unmap->addr[src_count] = dma_map_page(dma_chan->device->dev, dest, 0, PAGE_SIZE, + DMA_FROM_DEVICE); + dest_dma = unmap->addr[src_count]; + unmap->from_cnt = 1; + unmap->len = PAGE_SIZE; tx = mv_xor_prep_dma_xor(dma_chan, dest_dma, dma_srcs, - MV_XOR_NUM_SRC_TEST, PAGE_SIZE, 0); + src_count, PAGE_SIZE, 0); cookie = mv_xor_tx_submit(tx); mv_xor_issue_pending(dma_chan); @@ -968,7 +947,7 @@ mv_xor_xor_self_test(struct mv_xor_chan *mv_chan) msleep(8); if (mv_xor_status(dma_chan, cookie, NULL) != - DMA_SUCCESS) { + DMA_COMPLETE) { dev_err(dma_chan->device->dev, "Self-test xor timed out, disabling\n"); err = -ENODEV; @@ -989,9 +968,10 @@ mv_xor_xor_self_test(struct mv_xor_chan *mv_chan) } free_resources: + dmaengine_unmap_put(unmap); mv_xor_free_chan_resources(dma_chan); out: - src_idx = MV_XOR_NUM_SRC_TEST; + src_idx = src_count; while (src_idx--) __free_page(xor_srcs[src_idx]); __free_page(dest); @@ -1076,10 +1056,7 @@ mv_xor_channel_add(struct mv_xor_device *xordev, } mv_chan->mmr_base = xordev->xor_base; - if (!mv_chan->mmr_base) { - ret = -ENOMEM; - goto err_free_dma; - } + mv_chan->mmr_high_base = xordev->xor_high_base; tasklet_init(&mv_chan->irq_tasklet, mv_xor_tasklet, (unsigned long) mv_chan); @@ -1138,7 +1115,7 @@ static void mv_xor_conf_mbus_windows(struct mv_xor_device *xordev, const struct mbus_dram_target_info *dram) { - void __iomem *base = xordev->xor_base; + void __iomem *base = xordev->xor_high_base; u32 win_enable = 0; int i; @@ -1220,6 +1197,7 @@ static int mv_xor_probe(struct platform_device *pdev) int i = 0; for_each_child_of_node(pdev->dev.of_node, np) { + struct mv_xor_chan *chan; dma_cap_mask_t cap_mask; int irq; @@ -1237,21 +1215,21 @@ static int mv_xor_probe(struct platform_device *pdev) goto err_channel_add; } - xordev->channels[i] = - mv_xor_channel_add(xordev, pdev, i, - cap_mask, irq); - if (IS_ERR(xordev->channels[i])) { - ret = PTR_ERR(xordev->channels[i]); - xordev->channels[i] = NULL; + chan = mv_xor_channel_add(xordev, pdev, i, + cap_mask, irq); + if (IS_ERR(chan)) { + ret = PTR_ERR(chan); irq_dispose_mapping(irq); goto err_channel_add; } + xordev->channels[i] = chan; i++; } } else if (pdata && pdata->channels) { for (i = 0; i < MV_XOR_MAX_CHANNELS; i++) { struct mv_xor_channel_data *cd; + struct mv_xor_chan *chan; int irq; cd = &pdata->channels[i]; @@ -1266,13 +1244,14 @@ static int mv_xor_probe(struct platform_device *pdev) goto err_channel_add; } - xordev->channels[i] = - mv_xor_channel_add(xordev, pdev, i, - cd->cap_mask, irq); - if (IS_ERR(xordev->channels[i])) { - ret = PTR_ERR(xordev->channels[i]); + chan = mv_xor_channel_add(xordev, pdev, i, + cd->cap_mask, irq); + if (IS_ERR(chan)) { + ret = PTR_ERR(chan); goto err_channel_add; } + + xordev->channels[i] = chan; } } diff --git a/drivers/dma/mv_xor.h b/drivers/dma/mv_xor.h index 06b067f24c9..d0749229c87 100644 --- a/drivers/dma/mv_xor.h +++ b/drivers/dma/mv_xor.h @@ -34,13 +34,13 @@ #define XOR_OPERATION_MODE_MEMCPY 2 #define XOR_DESCRIPTOR_SWAP BIT(14) -#define XOR_CURR_DESC(chan) (chan->mmr_base + 0x210 + (chan->idx * 4)) -#define XOR_NEXT_DESC(chan) (chan->mmr_base + 0x200 + (chan->idx * 4)) -#define XOR_BYTE_COUNT(chan) (chan->mmr_base + 0x220 + (chan->idx * 4)) -#define XOR_DEST_POINTER(chan) (chan->mmr_base + 0x2B0 + (chan->idx * 4)) -#define XOR_BLOCK_SIZE(chan) (chan->mmr_base + 0x2C0 + (chan->idx * 4)) -#define XOR_INIT_VALUE_LOW(chan) (chan->mmr_base + 0x2E0) -#define XOR_INIT_VALUE_HIGH(chan) (chan->mmr_base + 0x2E4) +#define XOR_CURR_DESC(chan) (chan->mmr_high_base + 0x10 + (chan->idx * 4)) +#define XOR_NEXT_DESC(chan) (chan->mmr_high_base + 0x00 + (chan->idx * 4)) +#define XOR_BYTE_COUNT(chan) (chan->mmr_high_base + 0x20 + (chan->idx * 4)) +#define XOR_DEST_POINTER(chan) (chan->mmr_high_base + 0xB0 + (chan->idx * 4)) +#define XOR_BLOCK_SIZE(chan) (chan->mmr_high_base + 0xC0 + (chan->idx * 4)) +#define XOR_INIT_VALUE_LOW(chan) (chan->mmr_high_base + 0xE0) +#define XOR_INIT_VALUE_HIGH(chan) (chan->mmr_high_base + 0xE4) #define XOR_CONFIG(chan) (chan->mmr_base + 0x10 + (chan->idx * 4)) #define XOR_ACTIVATION(chan) (chan->mmr_base + 0x20 + (chan->idx * 4)) @@ -50,11 +50,11 @@ #define XOR_ERROR_ADDR(chan) (chan->mmr_base + 0x60) #define XOR_INTR_MASK_VALUE 0x3F5 -#define WINDOW_BASE(w) (0x250 + ((w) << 2)) -#define WINDOW_SIZE(w) (0x270 + ((w) << 2)) -#define WINDOW_REMAP_HIGH(w) (0x290 + ((w) << 2)) -#define WINDOW_BAR_ENABLE(chan) (0x240 + ((chan) << 2)) -#define WINDOW_OVERRIDE_CTRL(chan) (0x2A0 + ((chan) << 2)) +#define WINDOW_BASE(w) (0x50 + ((w) << 2)) +#define WINDOW_SIZE(w) (0x70 + ((w) << 2)) +#define WINDOW_REMAP_HIGH(w) (0x90 + ((w) << 2)) +#define WINDOW_BAR_ENABLE(chan) (0x40 + ((chan) << 2)) +#define WINDOW_OVERRIDE_CTRL(chan) (0xA0 + ((chan) << 2)) struct mv_xor_device { void __iomem *xor_base; @@ -82,6 +82,7 @@ struct mv_xor_chan { int pending; spinlock_t lock; /* protects the descriptor slot pool */ void __iomem *mmr_base; + void __iomem *mmr_high_base; unsigned int idx; int irq; enum dma_transaction_type current_type; diff --git a/drivers/dma/mxs-dma.c b/drivers/dma/mxs-dma.c index ccd13df841d..ead491346da 100644 --- a/drivers/dma/mxs-dma.c +++ b/drivers/dma/mxs-dma.c @@ -27,6 +27,7 @@ #include <linux/of.h> #include <linux/of_device.h> #include <linux/of_dma.h> +#include <linux/list.h> #include <asm/irq.h> @@ -57,6 +58,9 @@ (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x050 : 0x110) + (n) * 0x70) #define HW_APBHX_CHn_SEMA(d, n) \ (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x080 : 0x140) + (n) * 0x70) +#define HW_APBHX_CHn_BAR(d, n) \ + (((dma_is_apbh(d) && apbh_is_old(d)) ? 0x070 : 0x130) + (n) * 0x70) +#define HW_APBX_CHn_DEBUG1(d, n) (0x150 + (n) * 0x70) /* * ccw bits definitions @@ -115,7 +119,9 @@ struct mxs_dma_chan { int desc_count; enum dma_status status; unsigned int flags; + bool reset; #define MXS_DMA_SG_LOOP (1 << 0) +#define MXS_DMA_USE_SEMAPHORE (1 << 1) }; #define MXS_DMA_CHANNELS 16 @@ -201,12 +207,47 @@ static void mxs_dma_reset_chan(struct mxs_dma_chan *mxs_chan) struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; int chan_id = mxs_chan->chan.chan_id; - if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma)) + /* + * mxs dma channel resets can cause a channel stall. To recover from a + * channel stall, we have to reset the whole DMA engine. To avoid this, + * we use cyclic DMA with semaphores, that are enhanced in + * mxs_dma_int_handler. To reset the channel, we can simply stop writing + * into the semaphore counter. + */ + if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE && + mxs_chan->flags & MXS_DMA_SG_LOOP) { + mxs_chan->reset = true; + } else if (dma_is_apbh(mxs_dma) && apbh_is_old(mxs_dma)) { writel(1 << (chan_id + BP_APBH_CTRL0_RESET_CHANNEL), mxs_dma->base + HW_APBHX_CTRL0 + STMP_OFFSET_REG_SET); - else + } else { + unsigned long elapsed = 0; + const unsigned long max_wait = 50000; /* 50ms */ + void __iomem *reg_dbg1 = mxs_dma->base + + HW_APBX_CHn_DEBUG1(mxs_dma, chan_id); + + /* + * On i.MX28 APBX, the DMA channel can stop working if we reset + * the channel while it is in READ_FLUSH (0x08) state. + * We wait here until we leave the state. Then we trigger the + * reset. Waiting a maximum of 50ms, the kernel shouldn't crash + * because of this. + */ + while ((readl(reg_dbg1) & 0xf) == 0x8 && elapsed < max_wait) { + udelay(100); + elapsed += 100; + } + + if (elapsed >= max_wait) + dev_err(&mxs_chan->mxs_dma->pdev->dev, + "Failed waiting for the DMA channel %d to leave state READ_FLUSH, trying to reset channel in READ_FLUSH state now\n", + chan_id); + writel(1 << (chan_id + BP_APBHX_CHANNEL_CTRL_RESET_CHANNEL), mxs_dma->base + HW_APBHX_CHANNEL_CTRL + STMP_OFFSET_REG_SET); + } + + mxs_chan->status = DMA_COMPLETE; } static void mxs_dma_enable_chan(struct mxs_dma_chan *mxs_chan) @@ -219,12 +260,21 @@ static void mxs_dma_enable_chan(struct mxs_dma_chan *mxs_chan) mxs_dma->base + HW_APBHX_CHn_NXTCMDAR(mxs_dma, chan_id)); /* write 1 to SEMA to kick off the channel */ - writel(1, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id)); + if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE && + mxs_chan->flags & MXS_DMA_SG_LOOP) { + /* A cyclic DMA consists of at least 2 segments, so initialize + * the semaphore with 2 so we have enough time to add 1 to the + * semaphore if we need to */ + writel(2, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id)); + } else { + writel(1, mxs_dma->base + HW_APBHX_CHn_SEMA(mxs_dma, chan_id)); + } + mxs_chan->reset = false; } static void mxs_dma_disable_chan(struct mxs_dma_chan *mxs_chan) { - mxs_chan->status = DMA_SUCCESS; + mxs_chan->status = DMA_COMPLETE; } static void mxs_dma_pause_chan(struct mxs_dma_chan *mxs_chan) @@ -272,58 +322,88 @@ static void mxs_dma_tasklet(unsigned long data) mxs_chan->desc.callback(mxs_chan->desc.callback_param); } +static int mxs_dma_irq_to_chan(struct mxs_dma_engine *mxs_dma, int irq) +{ + int i; + + for (i = 0; i != mxs_dma->nr_channels; ++i) + if (mxs_dma->mxs_chans[i].chan_irq == irq) + return i; + + return -EINVAL; +} + static irqreturn_t mxs_dma_int_handler(int irq, void *dev_id) { struct mxs_dma_engine *mxs_dma = dev_id; - u32 stat1, stat2; + struct mxs_dma_chan *mxs_chan; + u32 completed; + u32 err; + int chan = mxs_dma_irq_to_chan(mxs_dma, irq); + + if (chan < 0) + return IRQ_NONE; /* completion status */ - stat1 = readl(mxs_dma->base + HW_APBHX_CTRL1); - stat1 &= MXS_DMA_CHANNELS_MASK; - writel(stat1, mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_CLR); + completed = readl(mxs_dma->base + HW_APBHX_CTRL1); + completed = (completed >> chan) & 0x1; + + /* Clear interrupt */ + writel((1 << chan), + mxs_dma->base + HW_APBHX_CTRL1 + STMP_OFFSET_REG_CLR); /* error status */ - stat2 = readl(mxs_dma->base + HW_APBHX_CTRL2); - writel(stat2, mxs_dma->base + HW_APBHX_CTRL2 + STMP_OFFSET_REG_CLR); + err = readl(mxs_dma->base + HW_APBHX_CTRL2); + err &= (1 << (MXS_DMA_CHANNELS + chan)) | (1 << chan); + + /* + * error status bit is in the upper 16 bits, error irq bit in the lower + * 16 bits. We transform it into a simpler error code: + * err: 0x00 = no error, 0x01 = TERMINATION, 0x02 = BUS_ERROR + */ + err = (err >> (MXS_DMA_CHANNELS + chan)) + (err >> chan); + + /* Clear error irq */ + writel((1 << chan), + mxs_dma->base + HW_APBHX_CTRL2 + STMP_OFFSET_REG_CLR); /* * When both completion and error of termination bits set at the * same time, we do not take it as an error. IOW, it only becomes - * an error we need to handle here in case of either it's (1) a bus - * error or (2) a termination error with no completion. + * an error we need to handle here in case of either it's a bus + * error or a termination error with no completion. 0x01 is termination + * error, so we can subtract err & completed to get the real error case. */ - stat2 = ((stat2 >> MXS_DMA_CHANNELS) & stat2) | /* (1) */ - (~(stat2 >> MXS_DMA_CHANNELS) & stat2 & ~stat1); /* (2) */ - - /* combine error and completion status for checking */ - stat1 = (stat2 << MXS_DMA_CHANNELS) | stat1; - while (stat1) { - int channel = fls(stat1) - 1; - struct mxs_dma_chan *mxs_chan = - &mxs_dma->mxs_chans[channel % MXS_DMA_CHANNELS]; - - if (channel >= MXS_DMA_CHANNELS) { - dev_dbg(mxs_dma->dma_device.dev, - "%s: error in channel %d\n", __func__, - channel - MXS_DMA_CHANNELS); - mxs_chan->status = DMA_ERROR; - mxs_dma_reset_chan(mxs_chan); - } else { - if (mxs_chan->flags & MXS_DMA_SG_LOOP) - mxs_chan->status = DMA_IN_PROGRESS; - else - mxs_chan->status = DMA_SUCCESS; - } + err -= err & completed; - stat1 &= ~(1 << channel); + mxs_chan = &mxs_dma->mxs_chans[chan]; - if (mxs_chan->status == DMA_SUCCESS) - dma_cookie_complete(&mxs_chan->desc); + if (err) { + dev_dbg(mxs_dma->dma_device.dev, + "%s: error in channel %d\n", __func__, + chan); + mxs_chan->status = DMA_ERROR; + mxs_dma_reset_chan(mxs_chan); + } else if (mxs_chan->status != DMA_COMPLETE) { + if (mxs_chan->flags & MXS_DMA_SG_LOOP) { + mxs_chan->status = DMA_IN_PROGRESS; + if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE) + writel(1, mxs_dma->base + + HW_APBHX_CHn_SEMA(mxs_dma, chan)); + } else { + mxs_chan->status = DMA_COMPLETE; + } + } - /* schedule tasklet on this channel */ - tasklet_schedule(&mxs_chan->tasklet); + if (mxs_chan->status == DMA_COMPLETE) { + if (mxs_chan->reset) + return IRQ_HANDLED; + dma_cookie_complete(&mxs_chan->desc); } + /* schedule tasklet on this channel */ + tasklet_schedule(&mxs_chan->tasklet); + return IRQ_HANDLED; } @@ -523,6 +603,7 @@ static struct dma_async_tx_descriptor *mxs_dma_prep_dma_cyclic( mxs_chan->status = DMA_IN_PROGRESS; mxs_chan->flags |= MXS_DMA_SG_LOOP; + mxs_chan->flags |= MXS_DMA_USE_SEMAPHORE; if (num_periods > NUM_CCW) { dev_err(mxs_dma->dma_device.dev, @@ -554,6 +635,7 @@ static struct dma_async_tx_descriptor *mxs_dma_prep_dma_cyclic( ccw->bits |= CCW_IRQ; ccw->bits |= CCW_HALT_ON_TERM; ccw->bits |= CCW_TERM_FLUSH; + ccw->bits |= CCW_DEC_SEM; ccw->bits |= BF_CCW(direction == DMA_DEV_TO_MEM ? MXS_DMA_CMD_WRITE : MXS_DMA_CMD_READ, COMMAND); @@ -599,8 +681,24 @@ static enum dma_status mxs_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan); + struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma; + u32 residue = 0; + + if (mxs_chan->status == DMA_IN_PROGRESS && + mxs_chan->flags & MXS_DMA_SG_LOOP) { + struct mxs_dma_ccw *last_ccw; + u32 bar; + + last_ccw = &mxs_chan->ccw[mxs_chan->desc_count - 1]; + residue = last_ccw->xfer_bytes + last_ccw->bufaddr; + + bar = readl(mxs_dma->base + + HW_APBHX_CHn_BAR(mxs_dma, chan->chan_id)); + residue -= bar; + } - dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, 0); + dma_set_tx_state(txstate, chan->completed_cookie, chan->cookie, + residue); return mxs_chan->status; } diff --git a/drivers/dma/of-dma.c b/drivers/dma/of-dma.c index 0b88dd3d05f..e8fe9dc455f 100644 --- a/drivers/dma/of-dma.c +++ b/drivers/dma/of-dma.c @@ -143,7 +143,7 @@ static int of_dma_match_channel(struct device_node *np, const char *name, * @np: device node to get DMA request from * @name: name of desired channel * - * Returns pointer to appropriate dma channel on success or NULL on error. + * Returns pointer to appropriate DMA channel on success or an error pointer. */ struct dma_chan *of_dma_request_slave_channel(struct device_node *np, const char *name) @@ -152,17 +152,18 @@ struct dma_chan *of_dma_request_slave_channel(struct device_node *np, struct of_dma *ofdma; struct dma_chan *chan; int count, i; + int ret_no_channel = -ENODEV; if (!np || !name) { pr_err("%s: not enough information provided\n", __func__); - return NULL; + return ERR_PTR(-ENODEV); } count = of_property_count_strings(np, "dma-names"); if (count < 0) { pr_err("%s: dma-names property of node '%s' missing or empty\n", __func__, np->full_name); - return NULL; + return ERR_PTR(-ENODEV); } for (i = 0; i < count; i++) { @@ -172,10 +173,12 @@ struct dma_chan *of_dma_request_slave_channel(struct device_node *np, mutex_lock(&of_dma_lock); ofdma = of_dma_find_controller(&dma_spec); - if (ofdma) + if (ofdma) { chan = ofdma->of_dma_xlate(&dma_spec, ofdma); - else + } else { + ret_no_channel = -EPROBE_DEFER; chan = NULL; + } mutex_unlock(&of_dma_lock); @@ -185,7 +188,7 @@ struct dma_chan *of_dma_request_slave_channel(struct device_node *np, return chan; } - return NULL; + return ERR_PTR(ret_no_channel); } /** diff --git a/drivers/dma/omap-dma.c b/drivers/dma/omap-dma.c index ec3fc4fd916..b19f04f4390 100644 --- a/drivers/dma/omap-dma.c +++ b/drivers/dma/omap-dma.c @@ -5,6 +5,7 @@ * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. */ +#include <linux/delay.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> #include <linux/err.h> @@ -26,11 +27,21 @@ struct omap_dmadev { spinlock_t lock; struct tasklet_struct task; struct list_head pending; + void __iomem *base; + const struct omap_dma_reg *reg_map; + struct omap_system_dma_plat_info *plat; + bool legacy; + spinlock_t irq_lock; + uint32_t irq_enable_mask; + struct omap_chan *lch_map[32]; }; struct omap_chan { struct virt_dma_chan vc; struct list_head node; + void __iomem *channel_base; + const struct omap_dma_reg *reg_map; + uint32_t ccr; struct dma_slave_config cfg; unsigned dma_sig; @@ -54,19 +65,93 @@ struct omap_desc { dma_addr_t dev_addr; int16_t fi; /* for OMAP_DMA_SYNC_PACKET */ - uint8_t es; /* OMAP_DMA_DATA_TYPE_xxx */ - uint8_t sync_mode; /* OMAP_DMA_SYNC_xxx */ - uint8_t sync_type; /* OMAP_DMA_xxx_SYNC* */ - uint8_t periph_port; /* Peripheral port */ + uint8_t es; /* CSDP_DATA_TYPE_xxx */ + uint32_t ccr; /* CCR value */ + uint16_t clnk_ctrl; /* CLNK_CTRL value */ + uint16_t cicr; /* CICR value */ + uint32_t csdp; /* CSDP value */ unsigned sglen; struct omap_sg sg[0]; }; +enum { + CCR_FS = BIT(5), + CCR_READ_PRIORITY = BIT(6), + CCR_ENABLE = BIT(7), + CCR_AUTO_INIT = BIT(8), /* OMAP1 only */ + CCR_REPEAT = BIT(9), /* OMAP1 only */ + CCR_OMAP31_DISABLE = BIT(10), /* OMAP1 only */ + CCR_SUSPEND_SENSITIVE = BIT(8), /* OMAP2+ only */ + CCR_RD_ACTIVE = BIT(9), /* OMAP2+ only */ + CCR_WR_ACTIVE = BIT(10), /* OMAP2+ only */ + CCR_SRC_AMODE_CONSTANT = 0 << 12, + CCR_SRC_AMODE_POSTINC = 1 << 12, + CCR_SRC_AMODE_SGLIDX = 2 << 12, + CCR_SRC_AMODE_DBLIDX = 3 << 12, + CCR_DST_AMODE_CONSTANT = 0 << 14, + CCR_DST_AMODE_POSTINC = 1 << 14, + CCR_DST_AMODE_SGLIDX = 2 << 14, + CCR_DST_AMODE_DBLIDX = 3 << 14, + CCR_CONSTANT_FILL = BIT(16), + CCR_TRANSPARENT_COPY = BIT(17), + CCR_BS = BIT(18), + CCR_SUPERVISOR = BIT(22), + CCR_PREFETCH = BIT(23), + CCR_TRIGGER_SRC = BIT(24), + CCR_BUFFERING_DISABLE = BIT(25), + CCR_WRITE_PRIORITY = BIT(26), + CCR_SYNC_ELEMENT = 0, + CCR_SYNC_FRAME = CCR_FS, + CCR_SYNC_BLOCK = CCR_BS, + CCR_SYNC_PACKET = CCR_BS | CCR_FS, + + CSDP_DATA_TYPE_8 = 0, + CSDP_DATA_TYPE_16 = 1, + CSDP_DATA_TYPE_32 = 2, + CSDP_SRC_PORT_EMIFF = 0 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_EMIFS = 1 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_OCP_T1 = 2 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_TIPB = 3 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_OCP_T2 = 4 << 2, /* OMAP1 only */ + CSDP_SRC_PORT_MPUI = 5 << 2, /* OMAP1 only */ + CSDP_SRC_PACKED = BIT(6), + CSDP_SRC_BURST_1 = 0 << 7, + CSDP_SRC_BURST_16 = 1 << 7, + CSDP_SRC_BURST_32 = 2 << 7, + CSDP_SRC_BURST_64 = 3 << 7, + CSDP_DST_PORT_EMIFF = 0 << 9, /* OMAP1 only */ + CSDP_DST_PORT_EMIFS = 1 << 9, /* OMAP1 only */ + CSDP_DST_PORT_OCP_T1 = 2 << 9, /* OMAP1 only */ + CSDP_DST_PORT_TIPB = 3 << 9, /* OMAP1 only */ + CSDP_DST_PORT_OCP_T2 = 4 << 9, /* OMAP1 only */ + CSDP_DST_PORT_MPUI = 5 << 9, /* OMAP1 only */ + CSDP_DST_PACKED = BIT(13), + CSDP_DST_BURST_1 = 0 << 14, + CSDP_DST_BURST_16 = 1 << 14, + CSDP_DST_BURST_32 = 2 << 14, + CSDP_DST_BURST_64 = 3 << 14, + + CICR_TOUT_IE = BIT(0), /* OMAP1 only */ + CICR_DROP_IE = BIT(1), + CICR_HALF_IE = BIT(2), + CICR_FRAME_IE = BIT(3), + CICR_LAST_IE = BIT(4), + CICR_BLOCK_IE = BIT(5), + CICR_PKT_IE = BIT(7), /* OMAP2+ only */ + CICR_TRANS_ERR_IE = BIT(8), /* OMAP2+ only */ + CICR_SUPERVISOR_ERR_IE = BIT(10), /* OMAP2+ only */ + CICR_MISALIGNED_ERR_IE = BIT(11), /* OMAP2+ only */ + CICR_DRAIN_IE = BIT(12), /* OMAP2+ only */ + CICR_SUPER_BLOCK_IE = BIT(14), /* OMAP2+ only */ + + CLNK_CTRL_ENABLE_LNK = BIT(15), +}; + static const unsigned es_bytes[] = { - [OMAP_DMA_DATA_TYPE_S8] = 1, - [OMAP_DMA_DATA_TYPE_S16] = 2, - [OMAP_DMA_DATA_TYPE_S32] = 4, + [CSDP_DATA_TYPE_8] = 1, + [CSDP_DATA_TYPE_16] = 2, + [CSDP_DATA_TYPE_32] = 4, }; static struct of_dma_filter_info omap_dma_info = { @@ -93,28 +178,214 @@ static void omap_dma_desc_free(struct virt_dma_desc *vd) kfree(container_of(vd, struct omap_desc, vd)); } +static void omap_dma_write(uint32_t val, unsigned type, void __iomem *addr) +{ + switch (type) { + case OMAP_DMA_REG_16BIT: + writew_relaxed(val, addr); + break; + case OMAP_DMA_REG_2X16BIT: + writew_relaxed(val, addr); + writew_relaxed(val >> 16, addr + 2); + break; + case OMAP_DMA_REG_32BIT: + writel_relaxed(val, addr); + break; + default: + WARN_ON(1); + } +} + +static unsigned omap_dma_read(unsigned type, void __iomem *addr) +{ + unsigned val; + + switch (type) { + case OMAP_DMA_REG_16BIT: + val = readw_relaxed(addr); + break; + case OMAP_DMA_REG_2X16BIT: + val = readw_relaxed(addr); + val |= readw_relaxed(addr + 2) << 16; + break; + case OMAP_DMA_REG_32BIT: + val = readl_relaxed(addr); + break; + default: + WARN_ON(1); + val = 0; + } + + return val; +} + +static void omap_dma_glbl_write(struct omap_dmadev *od, unsigned reg, unsigned val) +{ + const struct omap_dma_reg *r = od->reg_map + reg; + + WARN_ON(r->stride); + + omap_dma_write(val, r->type, od->base + r->offset); +} + +static unsigned omap_dma_glbl_read(struct omap_dmadev *od, unsigned reg) +{ + const struct omap_dma_reg *r = od->reg_map + reg; + + WARN_ON(r->stride); + + return omap_dma_read(r->type, od->base + r->offset); +} + +static void omap_dma_chan_write(struct omap_chan *c, unsigned reg, unsigned val) +{ + const struct omap_dma_reg *r = c->reg_map + reg; + + omap_dma_write(val, r->type, c->channel_base + r->offset); +} + +static unsigned omap_dma_chan_read(struct omap_chan *c, unsigned reg) +{ + const struct omap_dma_reg *r = c->reg_map + reg; + + return omap_dma_read(r->type, c->channel_base + r->offset); +} + +static void omap_dma_clear_csr(struct omap_chan *c) +{ + if (dma_omap1()) + omap_dma_chan_read(c, CSR); + else + omap_dma_chan_write(c, CSR, ~0); +} + +static unsigned omap_dma_get_csr(struct omap_chan *c) +{ + unsigned val = omap_dma_chan_read(c, CSR); + + if (!dma_omap1()) + omap_dma_chan_write(c, CSR, val); + + return val; +} + +static void omap_dma_assign(struct omap_dmadev *od, struct omap_chan *c, + unsigned lch) +{ + c->channel_base = od->base + od->plat->channel_stride * lch; + + od->lch_map[lch] = c; +} + +static void omap_dma_start(struct omap_chan *c, struct omap_desc *d) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + + if (__dma_omap15xx(od->plat->dma_attr)) + omap_dma_chan_write(c, CPC, 0); + else + omap_dma_chan_write(c, CDAC, 0); + + omap_dma_clear_csr(c); + + /* Enable interrupts */ + omap_dma_chan_write(c, CICR, d->cicr); + + /* Enable channel */ + omap_dma_chan_write(c, CCR, d->ccr | CCR_ENABLE); +} + +static void omap_dma_stop(struct omap_chan *c) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + uint32_t val; + + /* disable irq */ + omap_dma_chan_write(c, CICR, 0); + + omap_dma_clear_csr(c); + + val = omap_dma_chan_read(c, CCR); + if (od->plat->errata & DMA_ERRATA_i541 && val & CCR_TRIGGER_SRC) { + uint32_t sysconfig; + unsigned i; + + sysconfig = omap_dma_glbl_read(od, OCP_SYSCONFIG); + val = sysconfig & ~DMA_SYSCONFIG_MIDLEMODE_MASK; + val |= DMA_SYSCONFIG_MIDLEMODE(DMA_IDLEMODE_NO_IDLE); + omap_dma_glbl_write(od, OCP_SYSCONFIG, val); + + val = omap_dma_chan_read(c, CCR); + val &= ~CCR_ENABLE; + omap_dma_chan_write(c, CCR, val); + + /* Wait for sDMA FIFO to drain */ + for (i = 0; ; i++) { + val = omap_dma_chan_read(c, CCR); + if (!(val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE))) + break; + + if (i > 100) + break; + + udelay(5); + } + + if (val & (CCR_RD_ACTIVE | CCR_WR_ACTIVE)) + dev_err(c->vc.chan.device->dev, + "DMA drain did not complete on lch %d\n", + c->dma_ch); + + omap_dma_glbl_write(od, OCP_SYSCONFIG, sysconfig); + } else { + val &= ~CCR_ENABLE; + omap_dma_chan_write(c, CCR, val); + } + + mb(); + + if (!__dma_omap15xx(od->plat->dma_attr) && c->cyclic) { + val = omap_dma_chan_read(c, CLNK_CTRL); + + if (dma_omap1()) + val |= 1 << 14; /* set the STOP_LNK bit */ + else + val &= ~CLNK_CTRL_ENABLE_LNK; + + omap_dma_chan_write(c, CLNK_CTRL, val); + } +} + static void omap_dma_start_sg(struct omap_chan *c, struct omap_desc *d, unsigned idx) { struct omap_sg *sg = d->sg + idx; + unsigned cxsa, cxei, cxfi; - if (d->dir == DMA_DEV_TO_MEM) - omap_set_dma_dest_params(c->dma_ch, OMAP_DMA_PORT_EMIFF, - OMAP_DMA_AMODE_POST_INC, sg->addr, 0, 0); - else - omap_set_dma_src_params(c->dma_ch, OMAP_DMA_PORT_EMIFF, - OMAP_DMA_AMODE_POST_INC, sg->addr, 0, 0); + if (d->dir == DMA_DEV_TO_MEM) { + cxsa = CDSA; + cxei = CDEI; + cxfi = CDFI; + } else { + cxsa = CSSA; + cxei = CSEI; + cxfi = CSFI; + } - omap_set_dma_transfer_params(c->dma_ch, d->es, sg->en, sg->fn, - d->sync_mode, c->dma_sig, d->sync_type); + omap_dma_chan_write(c, cxsa, sg->addr); + omap_dma_chan_write(c, cxei, 0); + omap_dma_chan_write(c, cxfi, 0); + omap_dma_chan_write(c, CEN, sg->en); + omap_dma_chan_write(c, CFN, sg->fn); - omap_start_dma(c->dma_ch); + omap_dma_start(c, d); } static void omap_dma_start_desc(struct omap_chan *c) { struct virt_dma_desc *vd = vchan_next_desc(&c->vc); struct omap_desc *d; + unsigned cxsa, cxei, cxfi; if (!vd) { c->desc = NULL; @@ -126,12 +397,32 @@ static void omap_dma_start_desc(struct omap_chan *c) c->desc = d = to_omap_dma_desc(&vd->tx); c->sgidx = 0; - if (d->dir == DMA_DEV_TO_MEM) - omap_set_dma_src_params(c->dma_ch, d->periph_port, - OMAP_DMA_AMODE_CONSTANT, d->dev_addr, 0, d->fi); - else - omap_set_dma_dest_params(c->dma_ch, d->periph_port, - OMAP_DMA_AMODE_CONSTANT, d->dev_addr, 0, d->fi); + /* + * This provides the necessary barrier to ensure data held in + * DMA coherent memory is visible to the DMA engine prior to + * the transfer starting. + */ + mb(); + + omap_dma_chan_write(c, CCR, d->ccr); + if (dma_omap1()) + omap_dma_chan_write(c, CCR2, d->ccr >> 16); + + if (d->dir == DMA_DEV_TO_MEM) { + cxsa = CSSA; + cxei = CSEI; + cxfi = CSFI; + } else { + cxsa = CDSA; + cxei = CDEI; + cxfi = CDFI; + } + + omap_dma_chan_write(c, cxsa, d->dev_addr); + omap_dma_chan_write(c, cxei, 0); + omap_dma_chan_write(c, cxfi, d->fi); + omap_dma_chan_write(c, CSDP, d->csdp); + omap_dma_chan_write(c, CLNK_CTRL, d->clnk_ctrl); omap_dma_start_sg(c, d, 0); } @@ -186,24 +477,118 @@ static void omap_dma_sched(unsigned long data) } } +static irqreturn_t omap_dma_irq(int irq, void *devid) +{ + struct omap_dmadev *od = devid; + unsigned status, channel; + + spin_lock(&od->irq_lock); + + status = omap_dma_glbl_read(od, IRQSTATUS_L1); + status &= od->irq_enable_mask; + if (status == 0) { + spin_unlock(&od->irq_lock); + return IRQ_NONE; + } + + while ((channel = ffs(status)) != 0) { + unsigned mask, csr; + struct omap_chan *c; + + channel -= 1; + mask = BIT(channel); + status &= ~mask; + + c = od->lch_map[channel]; + if (c == NULL) { + /* This should never happen */ + dev_err(od->ddev.dev, "invalid channel %u\n", channel); + continue; + } + + csr = omap_dma_get_csr(c); + omap_dma_glbl_write(od, IRQSTATUS_L1, mask); + + omap_dma_callback(channel, csr, c); + } + + spin_unlock(&od->irq_lock); + + return IRQ_HANDLED; +} + static int omap_dma_alloc_chan_resources(struct dma_chan *chan) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); + int ret; - dev_info(c->vc.chan.device->dev, "allocating channel for %u\n", c->dma_sig); + if (od->legacy) { + ret = omap_request_dma(c->dma_sig, "DMA engine", + omap_dma_callback, c, &c->dma_ch); + } else { + ret = omap_request_dma(c->dma_sig, "DMA engine", NULL, NULL, + &c->dma_ch); + } + + dev_dbg(od->ddev.dev, "allocating channel %u for %u\n", + c->dma_ch, c->dma_sig); + + if (ret >= 0) { + omap_dma_assign(od, c, c->dma_ch); + + if (!od->legacy) { + unsigned val; + + spin_lock_irq(&od->irq_lock); + val = BIT(c->dma_ch); + omap_dma_glbl_write(od, IRQSTATUS_L1, val); + od->irq_enable_mask |= val; + omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); + + val = omap_dma_glbl_read(od, IRQENABLE_L0); + val &= ~BIT(c->dma_ch); + omap_dma_glbl_write(od, IRQENABLE_L0, val); + spin_unlock_irq(&od->irq_lock); + } + } + + if (dma_omap1()) { + if (__dma_omap16xx(od->plat->dma_attr)) { + c->ccr = CCR_OMAP31_DISABLE; + /* Duplicate what plat-omap/dma.c does */ + c->ccr |= c->dma_ch + 1; + } else { + c->ccr = c->dma_sig & 0x1f; + } + } else { + c->ccr = c->dma_sig & 0x1f; + c->ccr |= (c->dma_sig & ~0x1f) << 14; + } + if (od->plat->errata & DMA_ERRATA_IFRAME_BUFFERING) + c->ccr |= CCR_BUFFERING_DISABLE; - return omap_request_dma(c->dma_sig, "DMA engine", - omap_dma_callback, c, &c->dma_ch); + return ret; } static void omap_dma_free_chan_resources(struct dma_chan *chan) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); + if (!od->legacy) { + spin_lock_irq(&od->irq_lock); + od->irq_enable_mask &= ~BIT(c->dma_ch); + omap_dma_glbl_write(od, IRQENABLE_L1, od->irq_enable_mask); + spin_unlock_irq(&od->irq_lock); + } + + c->channel_base = NULL; + od->lch_map[c->dma_ch] = NULL; vchan_free_chan_resources(&c->vc); omap_free_dma(c->dma_ch); - dev_info(c->vc.chan.device->dev, "freeing channel for %u\n", c->dma_sig); + dev_dbg(od->ddev.dev, "freeing channel for %u\n", c->dma_sig); } static size_t omap_dma_sg_size(struct omap_sg *sg) @@ -239,6 +624,74 @@ static size_t omap_dma_desc_size_pos(struct omap_desc *d, dma_addr_t addr) return size; } +/* + * OMAP 3.2/3.3 erratum: sometimes 0 is returned if CSAC/CDAC is + * read before the DMA controller finished disabling the channel. + */ +static uint32_t omap_dma_chan_read_3_3(struct omap_chan *c, unsigned reg) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + uint32_t val; + + val = omap_dma_chan_read(c, reg); + if (val == 0 && od->plat->errata & DMA_ERRATA_3_3) + val = omap_dma_chan_read(c, reg); + + return val; +} + +static dma_addr_t omap_dma_get_src_pos(struct omap_chan *c) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + dma_addr_t addr, cdac; + + if (__dma_omap15xx(od->plat->dma_attr)) { + addr = omap_dma_chan_read(c, CPC); + } else { + addr = omap_dma_chan_read_3_3(c, CSAC); + cdac = omap_dma_chan_read_3_3(c, CDAC); + + /* + * CDAC == 0 indicates that the DMA transfer on the channel has + * not been started (no data has been transferred so far). + * Return the programmed source start address in this case. + */ + if (cdac == 0) + addr = omap_dma_chan_read(c, CSSA); + } + + if (dma_omap1()) + addr |= omap_dma_chan_read(c, CSSA) & 0xffff0000; + + return addr; +} + +static dma_addr_t omap_dma_get_dst_pos(struct omap_chan *c) +{ + struct omap_dmadev *od = to_omap_dma_dev(c->vc.chan.device); + dma_addr_t addr; + + if (__dma_omap15xx(od->plat->dma_attr)) { + addr = omap_dma_chan_read(c, CPC); + } else { + addr = omap_dma_chan_read_3_3(c, CDAC); + + /* + * CDAC == 0 indicates that the DMA transfer on the channel + * has not been started (no data has been transferred so + * far). Return the programmed destination start address in + * this case. + */ + if (addr == 0) + addr = omap_dma_chan_read(c, CDSA); + } + + if (dma_omap1()) + addr |= omap_dma_chan_read(c, CDSA) & 0xffff0000; + + return addr; +} + static enum dma_status omap_dma_tx_status(struct dma_chan *chan, dma_cookie_t cookie, struct dma_tx_state *txstate) { @@ -248,7 +701,7 @@ static enum dma_status omap_dma_tx_status(struct dma_chan *chan, unsigned long flags; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS || !txstate) + if (ret == DMA_COMPLETE || !txstate) return ret; spin_lock_irqsave(&c->vc.lock, flags); @@ -260,9 +713,9 @@ static enum dma_status omap_dma_tx_status(struct dma_chan *chan, dma_addr_t pos; if (d->dir == DMA_MEM_TO_DEV) - pos = omap_get_dma_src_pos(c->dma_ch); + pos = omap_dma_get_src_pos(c); else if (d->dir == DMA_DEV_TO_MEM) - pos = omap_get_dma_dst_pos(c->dma_ch); + pos = omap_dma_get_dst_pos(c); else pos = 0; @@ -304,24 +757,23 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( struct dma_chan *chan, struct scatterlist *sgl, unsigned sglen, enum dma_transfer_direction dir, unsigned long tx_flags, void *context) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); enum dma_slave_buswidth dev_width; struct scatterlist *sgent; struct omap_desc *d; dma_addr_t dev_addr; - unsigned i, j = 0, es, en, frame_bytes, sync_type; + unsigned i, j = 0, es, en, frame_bytes; u32 burst; if (dir == DMA_DEV_TO_MEM) { dev_addr = c->cfg.src_addr; dev_width = c->cfg.src_addr_width; burst = c->cfg.src_maxburst; - sync_type = OMAP_DMA_SRC_SYNC; } else if (dir == DMA_MEM_TO_DEV) { dev_addr = c->cfg.dst_addr; dev_width = c->cfg.dst_addr_width; burst = c->cfg.dst_maxburst; - sync_type = OMAP_DMA_DST_SYNC; } else { dev_err(chan->device->dev, "%s: bad direction?\n", __func__); return NULL; @@ -330,13 +782,13 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( /* Bus width translates to the element size (ES) */ switch (dev_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: - es = OMAP_DMA_DATA_TYPE_S8; + es = CSDP_DATA_TYPE_8; break; case DMA_SLAVE_BUSWIDTH_2_BYTES: - es = OMAP_DMA_DATA_TYPE_S16; + es = CSDP_DATA_TYPE_16; break; case DMA_SLAVE_BUSWIDTH_4_BYTES: - es = OMAP_DMA_DATA_TYPE_S32; + es = CSDP_DATA_TYPE_32; break; default: /* not reached */ return NULL; @@ -350,9 +802,31 @@ static struct dma_async_tx_descriptor *omap_dma_prep_slave_sg( d->dir = dir; d->dev_addr = dev_addr; d->es = es; - d->sync_mode = OMAP_DMA_SYNC_FRAME; - d->sync_type = sync_type; - d->periph_port = OMAP_DMA_PORT_TIPB; + + d->ccr = c->ccr | CCR_SYNC_FRAME; + if (dir == DMA_DEV_TO_MEM) + d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; + else + d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; + + d->cicr = CICR_DROP_IE | CICR_BLOCK_IE; + d->csdp = es; + + if (dma_omap1()) { + d->cicr |= CICR_TOUT_IE; + + if (dir == DMA_DEV_TO_MEM) + d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_TIPB; + else + d->csdp |= CSDP_DST_PORT_TIPB | CSDP_SRC_PORT_EMIFF; + } else { + if (dir == DMA_DEV_TO_MEM) + d->ccr |= CCR_TRIGGER_SRC; + + d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; + } + if (od->plat->errata & DMA_ERRATA_PARALLEL_CHANNELS) + d->clnk_ctrl = c->dma_ch; /* * Build our scatterlist entries: each contains the address, @@ -382,23 +856,22 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( size_t period_len, enum dma_transfer_direction dir, unsigned long flags, void *context) { + struct omap_dmadev *od = to_omap_dma_dev(chan->device); struct omap_chan *c = to_omap_dma_chan(chan); enum dma_slave_buswidth dev_width; struct omap_desc *d; dma_addr_t dev_addr; - unsigned es, sync_type; + unsigned es; u32 burst; if (dir == DMA_DEV_TO_MEM) { dev_addr = c->cfg.src_addr; dev_width = c->cfg.src_addr_width; burst = c->cfg.src_maxburst; - sync_type = OMAP_DMA_SRC_SYNC; } else if (dir == DMA_MEM_TO_DEV) { dev_addr = c->cfg.dst_addr; dev_width = c->cfg.dst_addr_width; burst = c->cfg.dst_maxburst; - sync_type = OMAP_DMA_DST_SYNC; } else { dev_err(chan->device->dev, "%s: bad direction?\n", __func__); return NULL; @@ -407,13 +880,13 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( /* Bus width translates to the element size (ES) */ switch (dev_width) { case DMA_SLAVE_BUSWIDTH_1_BYTE: - es = OMAP_DMA_DATA_TYPE_S8; + es = CSDP_DATA_TYPE_8; break; case DMA_SLAVE_BUSWIDTH_2_BYTES: - es = OMAP_DMA_DATA_TYPE_S16; + es = CSDP_DATA_TYPE_16; break; case DMA_SLAVE_BUSWIDTH_4_BYTES: - es = OMAP_DMA_DATA_TYPE_S32; + es = CSDP_DATA_TYPE_32; break; default: /* not reached */ return NULL; @@ -428,32 +901,51 @@ static struct dma_async_tx_descriptor *omap_dma_prep_dma_cyclic( d->dev_addr = dev_addr; d->fi = burst; d->es = es; - if (burst) - d->sync_mode = OMAP_DMA_SYNC_PACKET; - else - d->sync_mode = OMAP_DMA_SYNC_ELEMENT; - d->sync_type = sync_type; - d->periph_port = OMAP_DMA_PORT_MPUI; d->sg[0].addr = buf_addr; d->sg[0].en = period_len / es_bytes[es]; d->sg[0].fn = buf_len / period_len; d->sglen = 1; - if (!c->cyclic) { - c->cyclic = true; - omap_dma_link_lch(c->dma_ch, c->dma_ch); + d->ccr = c->ccr; + if (dir == DMA_DEV_TO_MEM) + d->ccr |= CCR_DST_AMODE_POSTINC | CCR_SRC_AMODE_CONSTANT; + else + d->ccr |= CCR_DST_AMODE_CONSTANT | CCR_SRC_AMODE_POSTINC; - if (flags & DMA_PREP_INTERRUPT) - omap_enable_dma_irq(c->dma_ch, OMAP_DMA_FRAME_IRQ); + d->cicr = CICR_DROP_IE; + if (flags & DMA_PREP_INTERRUPT) + d->cicr |= CICR_FRAME_IE; - omap_disable_dma_irq(c->dma_ch, OMAP_DMA_BLOCK_IRQ); - } + d->csdp = es; + + if (dma_omap1()) { + d->cicr |= CICR_TOUT_IE; + + if (dir == DMA_DEV_TO_MEM) + d->csdp |= CSDP_DST_PORT_EMIFF | CSDP_SRC_PORT_MPUI; + else + d->csdp |= CSDP_DST_PORT_MPUI | CSDP_SRC_PORT_EMIFF; + } else { + if (burst) + d->ccr |= CCR_SYNC_PACKET; + else + d->ccr |= CCR_SYNC_ELEMENT; + + if (dir == DMA_DEV_TO_MEM) + d->ccr |= CCR_TRIGGER_SRC; - if (dma_omap2plus()) { - omap_set_dma_src_burst_mode(c->dma_ch, OMAP_DMA_DATA_BURST_16); - omap_set_dma_dest_burst_mode(c->dma_ch, OMAP_DMA_DATA_BURST_16); + d->cicr |= CICR_MISALIGNED_ERR_IE | CICR_TRANS_ERR_IE; + + d->csdp |= CSDP_DST_BURST_64 | CSDP_SRC_BURST_64; } + if (__dma_omap15xx(od->plat->dma_attr)) + d->ccr |= CCR_AUTO_INIT | CCR_REPEAT; + else + d->clnk_ctrl = c->dma_ch | CLNK_CTRL_ENABLE_LNK; + + c->cyclic = true; + return vchan_tx_prep(&c->vc, &d->vd, flags); } @@ -483,20 +975,19 @@ static int omap_dma_terminate_all(struct omap_chan *c) /* * Stop DMA activity: we assume the callback will not be called - * after omap_stop_dma() returns (even if it does, it will see + * after omap_dma_stop() returns (even if it does, it will see * c->desc is NULL and exit.) */ if (c->desc) { c->desc = NULL; /* Avoid stopping the dma twice */ if (!c->paused) - omap_stop_dma(c->dma_ch); + omap_dma_stop(c); } if (c->cyclic) { c->cyclic = false; c->paused = false; - omap_dma_unlink_lch(c->dma_ch, c->dma_ch); } vchan_get_all_descriptors(&c->vc, &head); @@ -513,7 +1004,7 @@ static int omap_dma_pause(struct omap_chan *c) return -EINVAL; if (!c->paused) { - omap_stop_dma(c->dma_ch); + omap_dma_stop(c); c->paused = true; } @@ -527,7 +1018,7 @@ static int omap_dma_resume(struct omap_chan *c) return -EINVAL; if (c->paused) { - omap_start_dma(c->dma_ch); + omap_dma_start(c, c->desc); c->paused = false; } @@ -573,6 +1064,7 @@ static int omap_dma_chan_init(struct omap_dmadev *od, int dma_sig) if (!c) return -ENOMEM; + c->reg_map = od->reg_map; c->dma_sig = dma_sig; c->vc.desc_free = omap_dma_desc_free; vchan_init(&c->vc, &od->ddev); @@ -594,18 +1086,46 @@ static void omap_dma_free(struct omap_dmadev *od) tasklet_kill(&c->vc.task); kfree(c); } - kfree(od); +} + +#define OMAP_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES)) + +static int omap_dma_device_slave_caps(struct dma_chan *dchan, + struct dma_slave_caps *caps) +{ + caps->src_addr_widths = OMAP_DMA_BUSWIDTHS; + caps->dstn_addr_widths = OMAP_DMA_BUSWIDTHS; + caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + caps->cmd_pause = true; + caps->cmd_terminate = true; + caps->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; + + return 0; } static int omap_dma_probe(struct platform_device *pdev) { struct omap_dmadev *od; - int rc, i; + struct resource *res; + int rc, i, irq; - od = kzalloc(sizeof(*od), GFP_KERNEL); + od = devm_kzalloc(&pdev->dev, sizeof(*od), GFP_KERNEL); if (!od) return -ENOMEM; + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + od->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(od->base)) + return PTR_ERR(od->base); + + od->plat = omap_get_plat_info(); + if (!od->plat) + return -EPROBE_DEFER; + + od->reg_map = od->plat->reg_map; + dma_cap_set(DMA_SLAVE, od->ddev.cap_mask); dma_cap_set(DMA_CYCLIC, od->ddev.cap_mask); od->ddev.device_alloc_chan_resources = omap_dma_alloc_chan_resources; @@ -615,10 +1135,12 @@ static int omap_dma_probe(struct platform_device *pdev) od->ddev.device_prep_slave_sg = omap_dma_prep_slave_sg; od->ddev.device_prep_dma_cyclic = omap_dma_prep_dma_cyclic; od->ddev.device_control = omap_dma_control; + od->ddev.device_slave_caps = omap_dma_device_slave_caps; od->ddev.dev = &pdev->dev; INIT_LIST_HEAD(&od->ddev.channels); INIT_LIST_HEAD(&od->pending); spin_lock_init(&od->lock); + spin_lock_init(&od->irq_lock); tasklet_init(&od->task, omap_dma_sched, (unsigned long)od); @@ -630,6 +1152,21 @@ static int omap_dma_probe(struct platform_device *pdev) } } + irq = platform_get_irq(pdev, 1); + if (irq <= 0) { + dev_info(&pdev->dev, "failed to get L1 IRQ: %d\n", irq); + od->legacy = true; + } else { + /* Disable all interrupts */ + od->irq_enable_mask = 0; + omap_dma_glbl_write(od, IRQENABLE_L1, 0); + + rc = devm_request_irq(&pdev->dev, irq, omap_dma_irq, + IRQF_SHARED, "omap-dma-engine", od); + if (rc) + return rc; + } + rc = dma_async_device_register(&od->ddev); if (rc) { pr_warn("OMAP-DMA: failed to register slave DMA engine device: %d\n", @@ -666,6 +1203,12 @@ static int omap_dma_remove(struct platform_device *pdev) of_dma_controller_free(pdev->dev.of_node); dma_async_device_unregister(&od->ddev); + + if (!od->legacy) { + /* Disable all interrupts */ + omap_dma_glbl_write(od, IRQENABLE_L0, 0); + } + omap_dma_free(od); return 0; diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c index 61fdc54a3c8..9f9ca9fe5ce 100644 --- a/drivers/dma/pch_dma.c +++ b/drivers/dma/pch_dma.c @@ -21,6 +21,7 @@ #include <linux/dma-mapping.h> #include <linux/init.h> #include <linux/pci.h> +#include <linux/slab.h> #include <linux/interrupt.h> #include <linux/module.h> #include <linux/pch_dma.h> @@ -964,16 +965,16 @@ static void pch_dma_remove(struct pci_dev *pdev) if (pd) { dma_async_device_unregister(&pd->dma); + free_irq(pdev->irq, pd); + list_for_each_entry_safe(chan, _c, &pd->dma.channels, device_node) { pd_chan = to_pd_chan(chan); - tasklet_disable(&pd_chan->tasklet); tasklet_kill(&pd_chan->tasklet); } pci_pool_destroy(pd->pool); - free_irq(pdev->irq, pd); pci_iounmap(pdev, pd->membase); pci_release_regions(pdev); pci_disable_device(pdev); @@ -996,7 +997,7 @@ static void pch_dma_remove(struct pci_dev *pdev) #define PCI_DEVICE_ID_ML7831_DMA1_8CH 0x8810 #define PCI_DEVICE_ID_ML7831_DMA2_4CH 0x8815 -DEFINE_PCI_DEVICE_TABLE(pch_dma_id_table) = { +const struct pci_device_id pch_dma_id_table[] = { { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_8CH), 8 }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_EG20T_PCH_DMA_4CH), 4 }, { PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_DMA1_8CH), 8}, /* UART Video */ diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index a562d24d20b..73fa9b7a10a 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -543,7 +543,9 @@ struct dma_pl330_chan { /* DMA-Engine Channel */ struct dma_chan chan; - /* List of to be xfered descriptors */ + /* List of submitted descriptors */ + struct list_head submitted_list; + /* List of issued descriptors */ struct list_head work_list; /* List of completed descriptors */ struct list_head completed_list; @@ -578,12 +580,16 @@ struct dma_pl330_dmac { /* DMA-Engine Device */ struct dma_device ddma; + /* Holds info about sg limitations */ + struct device_dma_parameters dma_parms; + /* Pool of descriptors available for the DMAC's channels */ struct list_head desc_pool; /* To protect desc_pool manipulation */ spinlock_t pool_lock; /* Peripheral channels connected to this DMAC */ + unsigned int num_peripherals; struct dma_pl330_chan *peripherals; /* keep at end */ }; @@ -606,11 +612,6 @@ struct dma_pl330_desc { struct dma_pl330_chan *pchan; }; -struct dma_pl330_filter_args { - struct dma_pl330_dmac *pdmac; - unsigned int chan_id; -}; - static inline void _callback(struct pl330_req *r, enum pl330_op_err err) { if (r && r->xfer_cb) @@ -2268,6 +2269,8 @@ static void pl330_tasklet(unsigned long data) list_move_tail(&desc->node, &pch->dmac->desc_pool); } + dma_descriptor_unmap(&desc->txd); + if (callback) { spin_unlock_irqrestore(&pch->lock, flags); callback(callback_param); @@ -2296,16 +2299,6 @@ static void dma_pl330_rqcb(void *token, enum pl330_op_err err) tasklet_schedule(&pch->task); } -static bool pl330_dt_filter(struct dma_chan *chan, void *param) -{ - struct dma_pl330_filter_args *fargs = param; - - if (chan->device != &fargs->pdmac->ddma) - return false; - - return (chan->chan_id == fargs->chan_id); -} - bool pl330_filter(struct dma_chan *chan, void *param) { u8 *peri_id; @@ -2314,7 +2307,7 @@ bool pl330_filter(struct dma_chan *chan, void *param) return false; peri_id = chan->private; - return *peri_id == (unsigned)param; + return *peri_id == (unsigned long)param; } EXPORT_SYMBOL(pl330_filter); @@ -2323,23 +2316,16 @@ static struct dma_chan *of_dma_pl330_xlate(struct of_phandle_args *dma_spec, { int count = dma_spec->args_count; struct dma_pl330_dmac *pdmac = ofdma->of_dma_data; - struct dma_pl330_filter_args fargs; - dma_cap_mask_t cap; - - if (!pdmac) - return NULL; + unsigned int chan_id; if (count != 1) return NULL; - fargs.pdmac = pdmac; - fargs.chan_id = dma_spec->args[0]; - - dma_cap_zero(cap); - dma_cap_set(DMA_SLAVE, cap); - dma_cap_set(DMA_CYCLIC, cap); + chan_id = dma_spec->args[0]; + if (chan_id >= pdmac->num_peripherals) + return NULL; - return dma_request_channel(cap, pl330_dt_filter, &fargs); + return dma_get_slave_channel(&pdmac->peripherals[chan_id].chan); } static int pl330_alloc_chan_resources(struct dma_chan *chan) @@ -2383,6 +2369,11 @@ static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH); /* Mark all desc done */ + list_for_each_entry(desc, &pch->submitted_list, node) { + desc->status = FREE; + dma_cookie_complete(&desc->txd); + } + list_for_each_entry(desc, &pch->work_list , node) { desc->status = FREE; dma_cookie_complete(&desc->txd); @@ -2393,6 +2384,7 @@ static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned dma_cookie_complete(&desc->txd); } + list_splice_tail_init(&pch->submitted_list, &pdmac->desc_pool); list_splice_tail_init(&pch->work_list, &pdmac->desc_pool); list_splice_tail_init(&pch->completed_list, &pdmac->desc_pool); spin_unlock_irqrestore(&pch->lock, flags); @@ -2451,7 +2443,14 @@ pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie, static void pl330_issue_pending(struct dma_chan *chan) { - pl330_tasklet((unsigned long) to_pchan(chan)); + struct dma_pl330_chan *pch = to_pchan(chan); + unsigned long flags; + + spin_lock_irqsave(&pch->lock, flags); + list_splice_tail_init(&pch->submitted_list, &pch->work_list); + spin_unlock_irqrestore(&pch->lock, flags); + + pl330_tasklet((unsigned long)pch); } /* @@ -2478,11 +2477,11 @@ static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx) dma_cookie_assign(&desc->txd); - list_move_tail(&desc->node, &pch->work_list); + list_move_tail(&desc->node, &pch->submitted_list); } cookie = dma_cookie_assign(&last->txd); - list_add_tail(&last->node, &pch->work_list); + list_add_tail(&last->node, &pch->submitted_list); spin_unlock_irqrestore(&pch->lock, flags); return cookie; @@ -2490,12 +2489,9 @@ static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx) static inline void _init_desc(struct dma_pl330_desc *desc) { - desc->pchan = NULL; desc->req.x = &desc->px; desc->req.token = desc; desc->rqcfg.swap = SWAP_NO; - desc->rqcfg.privileged = 0; - desc->rqcfg.insnaccess = 0; desc->rqcfg.scctl = SCCTRL0; desc->rqcfg.dcctl = DCCTRL0; desc->req.cfg = &desc->rqcfg; @@ -2515,7 +2511,7 @@ static int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count) if (!pdmac) return 0; - desc = kmalloc(count * sizeof(*desc), flg); + desc = kcalloc(count, sizeof(*desc), flg); if (!desc) return 0; @@ -2885,6 +2881,7 @@ static int pl330_dma_device_slave_caps(struct dma_chan *dchan, caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); caps->cmd_pause = false; caps->cmd_terminate = true; + caps->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; return 0; } @@ -2903,6 +2900,10 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) pdat = dev_get_platdata(&adev->dev); + ret = dma_set_mask_and_coherent(&adev->dev, DMA_BIT_MASK(32)); + if (ret) + return ret; + /* Allocate a new DMAC and its Channels */ pdmac = devm_kzalloc(&adev->dev, sizeof(*pdmac), GFP_KERNEL); if (!pdmac) { @@ -2922,16 +2923,23 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) amba_set_drvdata(adev, pdmac); - irq = adev->irq[0]; - ret = request_irq(irq, pl330_irq_handler, 0, - dev_name(&adev->dev), pi); - if (ret) - return ret; + for (i = 0; i < AMBA_NR_IRQS; i++) { + irq = adev->irq[i]; + if (irq) { + ret = devm_request_irq(&adev->dev, irq, + pl330_irq_handler, 0, + dev_name(&adev->dev), pi); + if (ret) + return ret; + } else { + break; + } + } pi->pcfg.periph_id = adev->periphid; ret = pl330_add(pi); if (ret) - goto probe_err1; + return ret; INIT_LIST_HEAD(&pdmac->desc_pool); spin_lock_init(&pdmac->pool_lock); @@ -2949,6 +2957,8 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) else num_chan = max_t(int, pi->pcfg.num_peri, pi->pcfg.num_chan); + pdmac->num_peripherals = num_chan; + pdmac->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL); if (!pdmac->peripherals) { ret = -ENOMEM; @@ -2963,6 +2973,7 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) else pch->chan.private = adev->dev.of_node; + INIT_LIST_HEAD(&pch->submitted_list); INIT_LIST_HEAD(&pch->work_list); INIT_LIST_HEAD(&pch->completed_list); spin_lock_init(&pch->lock); @@ -3010,6 +3021,9 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) "unable to register DMA to the generic DT DMA helpers\n"); } } + + adev->dev.dma_parms = &pdmac->dma_parms; + /* * This is the limit for transfers with a buswidth of 1, larger * buswidths will have larger limits. @@ -3029,8 +3043,6 @@ pl330_probe(struct amba_device *adev, const struct amba_id *id) return 0; probe_err3: - amba_set_drvdata(adev, NULL); - /* Idle the DMAC */ list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels, chan.device_node) { @@ -3044,8 +3056,6 @@ probe_err3: } probe_err2: pl330_del(pi); -probe_err1: - free_irq(irq, pi); return ret; } @@ -3055,7 +3065,6 @@ static int pl330_remove(struct amba_device *adev) struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev); struct dma_pl330_chan *pch, *_p; struct pl330_info *pi; - int irq; if (!pdmac) return 0; @@ -3064,7 +3073,6 @@ static int pl330_remove(struct amba_device *adev) of_dma_controller_free(adev->dev.of_node); dma_async_device_unregister(&pdmac->ddma); - amba_set_drvdata(adev, NULL); /* Idle the DMAC */ list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels, @@ -3082,9 +3090,6 @@ static int pl330_remove(struct amba_device *adev) pl330_del(pi); - irq = adev->irq[0]; - free_irq(irq, pi); - return 0; } diff --git a/drivers/dma/ppc4xx/Makefile b/drivers/dma/ppc4xx/Makefile index b3d259b3e52..e7700985371 100644 --- a/drivers/dma/ppc4xx/Makefile +++ b/drivers/dma/ppc4xx/Makefile @@ -1 +1,3 @@ obj-$(CONFIG_AMCC_PPC440SPE_ADMA) += adma.o +obj-$(CONFIG_AMCC_PPC460EX_460GT_4CHAN_DMA) += ppc460ex_4chan_dma.o +obj-$(CONFIG_APM82181_ADMA) += apm82181-adma.o diff --git a/drivers/dma/ppc4xx/adma.c b/drivers/dma/ppc4xx/adma.c index 370ff826563..ce7a8d7564b 100644 --- a/drivers/dma/ppc4xx/adma.c +++ b/drivers/dma/ppc4xx/adma.c @@ -42,6 +42,8 @@ #include <linux/uaccess.h> #include <linux/proc_fs.h> #include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> #include <linux/of_platform.h> #include <asm/dcr.h> #include <asm/dcr-regs.h> @@ -531,29 +533,6 @@ static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, } /** - * ppc440spe_desc_init_memset - initialize the descriptor for MEMSET operation - */ -static void ppc440spe_desc_init_memset(struct ppc440spe_adma_desc_slot *desc, - int value, unsigned long flags) -{ - struct dma_cdb *hw_desc = desc->hw_desc; - - memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); - desc->hw_next = NULL; - desc->src_cnt = 1; - desc->dst_cnt = 1; - - if (flags & DMA_PREP_INTERRUPT) - set_bit(PPC440SPE_DESC_INT, &desc->flags); - else - clear_bit(PPC440SPE_DESC_INT, &desc->flags); - - hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value); - hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value); - hw_desc->opc = DMA_CDB_OPC_DFILL128; -} - -/** * ppc440spe_desc_set_src_addr - set source address into the descriptor */ static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, @@ -802,218 +781,6 @@ static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, } /** - * ppc440spe_desc_get_src_addr - extract the source address from the descriptor - */ -static u32 ppc440spe_desc_get_src_addr(struct ppc440spe_adma_desc_slot *desc, - struct ppc440spe_adma_chan *chan, int src_idx) -{ - struct dma_cdb *dma_hw_desc; - struct xor_cb *xor_hw_desc; - - switch (chan->device->id) { - case PPC440SPE_DMA0_ID: - case PPC440SPE_DMA1_ID: - dma_hw_desc = desc->hw_desc; - /* May have 0, 1, 2, or 3 sources */ - switch (dma_hw_desc->opc) { - case DMA_CDB_OPC_NO_OP: - case DMA_CDB_OPC_DFILL128: - return 0; - case DMA_CDB_OPC_DCHECK128: - if (unlikely(src_idx)) { - printk(KERN_ERR "%s: try to get %d source for" - " DCHECK128\n", __func__, src_idx); - BUG(); - } - return le32_to_cpu(dma_hw_desc->sg1l); - case DMA_CDB_OPC_MULTICAST: - case DMA_CDB_OPC_MV_SG1_SG2: - if (unlikely(src_idx > 2)) { - printk(KERN_ERR "%s: try to get %d source from" - " DMA descr\n", __func__, src_idx); - BUG(); - } - if (src_idx) { - if (le32_to_cpu(dma_hw_desc->sg1u) & - DMA_CUED_XOR_WIN_MSK) { - u8 region; - - if (src_idx == 1) - return le32_to_cpu( - dma_hw_desc->sg1l) + - desc->unmap_len; - - region = (le32_to_cpu( - dma_hw_desc->sg1u)) >> - DMA_CUED_REGION_OFF; - - region &= DMA_CUED_REGION_MSK; - switch (region) { - case DMA_RXOR123: - return le32_to_cpu( - dma_hw_desc->sg1l) + - (desc->unmap_len << 1); - case DMA_RXOR124: - return le32_to_cpu( - dma_hw_desc->sg1l) + - (desc->unmap_len * 3); - case DMA_RXOR125: - return le32_to_cpu( - dma_hw_desc->sg1l) + - (desc->unmap_len << 2); - default: - printk(KERN_ERR - "%s: try to" - " get src3 for region %02x" - "PPC440SPE_DESC_RXOR12?\n", - __func__, region); - BUG(); - } - } else { - printk(KERN_ERR - "%s: try to get %d" - " source for non-cued descr\n", - __func__, src_idx); - BUG(); - } - } - return le32_to_cpu(dma_hw_desc->sg1l); - default: - printk(KERN_ERR "%s: unknown OPC 0x%02x\n", - __func__, dma_hw_desc->opc); - BUG(); - } - return le32_to_cpu(dma_hw_desc->sg1l); - case PPC440SPE_XOR_ID: - /* May have up to 16 sources */ - xor_hw_desc = desc->hw_desc; - return xor_hw_desc->ops[src_idx].l; - } - return 0; -} - -/** - * ppc440spe_desc_get_dest_addr - extract the destination address from the - * descriptor - */ -static u32 ppc440spe_desc_get_dest_addr(struct ppc440spe_adma_desc_slot *desc, - struct ppc440spe_adma_chan *chan, int idx) -{ - struct dma_cdb *dma_hw_desc; - struct xor_cb *xor_hw_desc; - - switch (chan->device->id) { - case PPC440SPE_DMA0_ID: - case PPC440SPE_DMA1_ID: - dma_hw_desc = desc->hw_desc; - - if (likely(!idx)) - return le32_to_cpu(dma_hw_desc->sg2l); - return le32_to_cpu(dma_hw_desc->sg3l); - case PPC440SPE_XOR_ID: - xor_hw_desc = desc->hw_desc; - return xor_hw_desc->cbtal; - } - return 0; -} - -/** - * ppc440spe_desc_get_src_num - extract the number of source addresses from - * the descriptor - */ -static u32 ppc440spe_desc_get_src_num(struct ppc440spe_adma_desc_slot *desc, - struct ppc440spe_adma_chan *chan) -{ - struct dma_cdb *dma_hw_desc; - struct xor_cb *xor_hw_desc; - - switch (chan->device->id) { - case PPC440SPE_DMA0_ID: - case PPC440SPE_DMA1_ID: - dma_hw_desc = desc->hw_desc; - - switch (dma_hw_desc->opc) { - case DMA_CDB_OPC_NO_OP: - case DMA_CDB_OPC_DFILL128: - return 0; - case DMA_CDB_OPC_DCHECK128: - return 1; - case DMA_CDB_OPC_MV_SG1_SG2: - case DMA_CDB_OPC_MULTICAST: - /* - * Only for RXOR operations we have more than - * one source - */ - if (le32_to_cpu(dma_hw_desc->sg1u) & - DMA_CUED_XOR_WIN_MSK) { - /* RXOR op, there are 2 or 3 sources */ - if (((le32_to_cpu(dma_hw_desc->sg1u) >> - DMA_CUED_REGION_OFF) & - DMA_CUED_REGION_MSK) == DMA_RXOR12) { - /* RXOR 1-2 */ - return 2; - } else { - /* RXOR 1-2-3/1-2-4/1-2-5 */ - return 3; - } - } - return 1; - default: - printk(KERN_ERR "%s: unknown OPC 0x%02x\n", - __func__, dma_hw_desc->opc); - BUG(); - } - case PPC440SPE_XOR_ID: - /* up to 16 sources */ - xor_hw_desc = desc->hw_desc; - return xor_hw_desc->cbc & XOR_CDCR_OAC_MSK; - default: - BUG(); - } - return 0; -} - -/** - * ppc440spe_desc_get_dst_num - get the number of destination addresses in - * this descriptor - */ -static u32 ppc440spe_desc_get_dst_num(struct ppc440spe_adma_desc_slot *desc, - struct ppc440spe_adma_chan *chan) -{ - struct dma_cdb *dma_hw_desc; - - switch (chan->device->id) { - case PPC440SPE_DMA0_ID: - case PPC440SPE_DMA1_ID: - /* May be 1 or 2 destinations */ - dma_hw_desc = desc->hw_desc; - switch (dma_hw_desc->opc) { - case DMA_CDB_OPC_NO_OP: - case DMA_CDB_OPC_DCHECK128: - return 0; - case DMA_CDB_OPC_MV_SG1_SG2: - case DMA_CDB_OPC_DFILL128: - return 1; - case DMA_CDB_OPC_MULTICAST: - if (desc->dst_cnt == 2) - return 2; - else - return 1; - default: - printk(KERN_ERR "%s: unknown OPC 0x%02x\n", - __func__, dma_hw_desc->opc); - BUG(); - } - case PPC440SPE_XOR_ID: - /* Always only 1 destination */ - return 1; - default: - BUG(); - } - return 0; -} - -/** * ppc440spe_desc_get_link - get the address of the descriptor that * follows this one */ @@ -1705,43 +1472,6 @@ static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, } } -static void ppc440spe_adma_unmap(struct ppc440spe_adma_chan *chan, - struct ppc440spe_adma_desc_slot *desc) -{ - u32 src_cnt, dst_cnt; - dma_addr_t addr; - - /* - * get the number of sources & destination - * included in this descriptor and unmap - * them all - */ - src_cnt = ppc440spe_desc_get_src_num(desc, chan); - dst_cnt = ppc440spe_desc_get_dst_num(desc, chan); - - /* unmap destinations */ - if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - while (dst_cnt--) { - addr = ppc440spe_desc_get_dest_addr( - desc, chan, dst_cnt); - dma_unmap_page(chan->device->dev, - addr, desc->unmap_len, - DMA_FROM_DEVICE); - } - } - - /* unmap sources */ - if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - while (src_cnt--) { - addr = ppc440spe_desc_get_src_addr( - desc, chan, src_cnt); - dma_unmap_page(chan->device->dev, - addr, desc->unmap_len, - DMA_TO_DEVICE); - } - } -} - /** * ppc440spe_adma_run_tx_complete_actions - call functions to be called * upon completion @@ -1751,8 +1481,6 @@ static dma_cookie_t ppc440spe_adma_run_tx_complete_actions( struct ppc440spe_adma_chan *chan, dma_cookie_t cookie) { - int i; - BUG_ON(desc->async_tx.cookie < 0); if (desc->async_tx.cookie > 0) { cookie = desc->async_tx.cookie; @@ -1765,26 +1493,7 @@ static dma_cookie_t ppc440spe_adma_run_tx_complete_actions( desc->async_tx.callback( desc->async_tx.callback_param); - /* unmap dma addresses - * (unmap_single vs unmap_page?) - * - * actually, ppc's dma_unmap_page() functions are empty, so - * the following code is just for the sake of completeness - */ - if (chan && chan->needs_unmap && desc->group_head && - desc->unmap_len) { - struct ppc440spe_adma_desc_slot *unmap = - desc->group_head; - /* assume 1 slot per op always */ - u32 slot_count = unmap->slot_cnt; - - /* Run through the group list and unmap addresses */ - for (i = 0; i < slot_count; i++) { - BUG_ON(!unmap); - ppc440spe_adma_unmap(chan, unmap); - unmap = unmap->hw_next; - } - } + dma_descriptor_unmap(&desc->async_tx); } /* run dependent operations */ @@ -3891,7 +3600,7 @@ static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, ppc440spe_chan = to_ppc440spe_adma_chan(chan); ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; ppc440spe_adma_slot_cleanup(ppc440spe_chan); @@ -4164,7 +3873,7 @@ static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) ppc440spe_adma_prep_dma_interrupt; } pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " - "( %s%s%s%s%s%s%s)\n", + "( %s%s%s%s%s%s)\n", dev_name(adev->dev), dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", @@ -4405,6 +4114,7 @@ static int ppc440spe_adma_probe(struct platform_device *ofdev) regs = ioremap(res.start, resource_size(&res)); if (!regs) { dev_err(&ofdev->dev, "failed to ioremap regs!\n"); + ret = -ENOMEM; goto err_regs_alloc; } diff --git a/drivers/dma/ppc4xx/apm82181-adma.c b/drivers/dma/ppc4xx/apm82181-adma.c new file mode 100644 index 00000000000..c95e704b1d9 --- /dev/null +++ b/drivers/dma/ppc4xx/apm82181-adma.c @@ -0,0 +1,2201 @@ +/* + * Copyright(c) 2010 Applied Micro Circuits Corporation(AMCC). All rights reserved. + * + * Author: Tai Tri Nguyen <ttnguyen@appliedmicro.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ + +/* + * This driver supports the asynchrounous DMA copy and RAID engines available + * on the AppliedMicro APM82181 Processor. + * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) + * ADMA driver written by D.Williams. + */ +#define ADMA_DEBUG +#undef ADMA_DEBUG + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/async_tx.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/uaccess.h> +#include <linux/of_platform.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/proc_fs.h> +#include <linux/slab.h> +#include <asm/dcr.h> +#include <asm/dcr-regs.h> +#include <asm/apm82181-adma.h> +#include "../dmaengine.h" + +#define PPC4XX_EDMA "apm82181-adma: " +#ifdef ADMA_DEBUG +#define DBG(string, args...) \ + printk(PPC4XX_EDMA string ,##args) +#define INFO DBG("<%s> -- line %d\n",__func__,__LINE__); +#define ADMA_HEXDUMP(b, l) \ + print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, \ + 16, 1, (b), (l), false); +#else +#define DBG(string, args...) \ + {if (0) printk(KERN_INFO PPC4XX_EDMA string ,##args); 0; } +#define INFO DBG(""); +#define ADMA_HEXDUMP(b, l) \ + {if (0) print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, \ + 8, 1, (b), (l), false); 0;} +#endif + +#define MEM_HEXDUMP(b, l) \ + print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, \ + 16, 1, (b), (l), false); + +/* The list of channels exported by apm82181 ADMA */ +struct list_head +ppc_adma_chan_list = LIST_HEAD_INIT(ppc_adma_chan_list); + +/* This flag is set when want to refetch the xor chain in the interrupt + * handler + */ +static u32 do_xor_refetch = 0; + +/* Pointers to last submitted to DMA0/1/2/3 and XOR CDBs */ +static apm82181_desc_t *chan_last_sub[5]; +static apm82181_desc_t *chan_first_cdb[5]; + +/* Pointer to last linked and submitted xor CB */ +static apm82181_desc_t *xor_last_linked = NULL; +static apm82181_desc_t *xor_last_submit = NULL; + +/* /proc interface is used here to verify the h/w RAID 5 capabilities + */ +static struct proc_dir_entry *apm82181_proot; + +/* These are used in enable & check routines + */ +static u32 apm82181_xor_verified; +static u32 apm82181_memcpy_verified[4]; +static apm82181_ch_t *apm82181_dma_tchan[5]; +static struct completion apm82181_r5_test_comp; + +static inline int apm82181_chan_is_busy(apm82181_ch_t *chan); +#if 0 +static phys_addr_t fixup_bigphys_addr(phys_addr_t addr, phys_addr_t size) +{ + phys_addr_t page_4gb = 0; + + return (page_4gb | addr); +} +#endif +/** + * apm82181_adma_device_estimate - estimate the efficiency of processing + * the operation given on this channel. It's assumed that 'chan' is + * capable to process 'cap' type of operation. + * @chan: channel to use + * @cap: type of transaction + * @src_lst: array of source pointers + * @src_cnt: number of source operands + * @src_sz: size of each source operand + */ +int apm82181_adma_estimate (struct dma_chan *chan, + enum dma_transaction_type cap, struct page **src_lst, + int src_cnt, size_t src_sz) +{ + int ef = 1; + + /* channel idleness increases the priority */ + if (likely(ef) && + !apm82181_chan_is_busy(to_apm82181_adma_chan(chan))) + ef++; + else { + if(chan->chan_id !=APM82181_XOR_ID) + ef = -1; + } + return ef; +} + +/****************************************************************************** + * Command (Descriptor) Blocks low-level routines + ******************************************************************************/ +/** + * apm82181_desc_init_interrupt - initialize the descriptor for INTERRUPT + * pseudo operation + */ +static inline void apm82181_desc_init_interrupt (apm82181_desc_t *desc, + apm82181_ch_t *chan) +{ + xor_cb_t *p; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + BUG(); + break; + case APM82181_XOR_ID: + p = desc->hw_desc; + memset (desc->hw_desc, 0, sizeof(xor_cb_t)); + /* NOP with Command Block Complete Enable */ + p->cbc = XOR_CBCR_CBCE_BIT; + break; + default: + printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, + __FUNCTION__); + break; + } +} + +/** + * apm82181_desc_init_xor - initialize the descriptor for XOR operation + */ +static inline void apm82181_desc_init_xor(apm82181_desc_t *desc, int src_cnt, + unsigned long flags) +{ + xor_cb_t *hw_desc = desc->hw_desc; + + memset (desc->hw_desc, 0, sizeof(xor_cb_t)); + desc->hw_next = NULL; + desc->src_cnt = src_cnt; + desc->dst_cnt = 1; + + hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; + if (flags & DMA_PREP_INTERRUPT) + /* Enable interrupt on complete */ + hw_desc->cbc |= XOR_CBCR_CBCE_BIT; +} + +/** + * apm82181_desc_init_memcpy - initialize the descriptor for MEMCPY operation + */ +static inline void apm82181_desc_init_memcpy(apm82181_desc_t *desc, + unsigned long flags) +{ + dma_cdb_t *hw_desc = desc->hw_desc; + + memset(hw_desc, 0, sizeof(dma_cdb_t)); + desc->hw_next = NULL; + desc->src_cnt = 1; + desc->dst_cnt = 1; + + if (flags & DMA_PREP_INTERRUPT) + set_bit(APM82181_DESC_INT, &desc->flags); + else + clear_bit(APM82181_DESC_INT, &desc->flags); + /* dma configuration for running */ + hw_desc->ctrl.tm = 2; /* soft init mem-mem mode */ + hw_desc->ctrl.pw = 4; /* transfer width 128 bytes */ + hw_desc->ctrl.ben = 1;/* buffer enable */ + hw_desc->ctrl.sai = 1;/* increase source addr */ + hw_desc->ctrl.dai = 1;/* increase dest addr */ + hw_desc->ctrl.tce = 1;/* chan stops when TC is reached */ + hw_desc->ctrl.cp = 3; /* hinghest priority */ + hw_desc->ctrl.sl = 0; /* source is in PLB */ + hw_desc->ctrl.pl = 0; /* dest is in PLB */ + hw_desc->cnt.tcie = 0;/* no interrupt on init */ + hw_desc->cnt.etie = 0; /* enable error interrupt */ + hw_desc->cnt.eie = 1; /* enable error interrupt */ + hw_desc->cnt.link = 0;/* not link to next cdb */ + hw_desc->cnt.sgl = 0; + hw_desc->ctrl.ce =1; /* enable channel */ + hw_desc->ctrl.cie =1; /* enable int channel */ +} + +/** + * apm82181_desc_init_memset - initialize the descriptor for MEMSET operation + */ +static inline void apm82181_desc_init_memset(apm82181_desc_t *desc, int value, + unsigned long flags) +{ + //dma_cdb_t *hw_desc = desc->hw_desc; + + memset (desc->hw_desc, 0, sizeof(dma_cdb_t)); + desc->hw_next = NULL; + desc->src_cnt = 1; + desc->dst_cnt = 1; + + if (flags & DMA_PREP_INTERRUPT) + set_bit(APM82181_DESC_INT, &desc->flags); + else + clear_bit(APM82181_DESC_INT, &desc->flags); + +} + + + +/** + * apm82181_desc_set_src_addr - set source address into the descriptor + */ +static inline void apm82181_desc_set_src_addr( apm82181_desc_t *desc, + apm82181_ch_t *chan, int src_idx, dma_addr_t addr) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + dma_hw_desc = desc->hw_desc; + dma_hw_desc->src_hi = (u32)(addr >> 32); + dma_hw_desc->src_lo = (u32)addr; + break; + case APM82181_XOR_ID: + xor_hw_desc = desc->hw_desc; + xor_hw_desc->ops[src_idx].h = (u32)(addr >>32); + xor_hw_desc->ops[src_idx].l = (u32)addr; + break; + } +} + +static void apm82181_adma_set_src(apm82181_desc_t *sw_desc, + dma_addr_t addr, int index) +{ + apm82181_ch_t *chan = to_apm82181_adma_chan(sw_desc->async_tx.chan); + + sw_desc = sw_desc->group_head; + + if (likely(sw_desc)) + apm82181_desc_set_src_addr(sw_desc, chan, index, addr); +} + +/** + * apm82181_desc_set_dest_addr - set destination address into the descriptor + */ +static inline void apm82181_desc_set_dest_addr(apm82181_desc_t *desc, + apm82181_ch_t *chan, dma_addr_t addr, u32 index) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + dma_hw_desc = desc->hw_desc; + dma_hw_desc->dest_hi = (u32)(addr >> 32); + dma_hw_desc->dest_lo = (u32)addr; + break; + case APM82181_XOR_ID: + xor_hw_desc = desc->hw_desc; + xor_hw_desc->cbtah = (u32)(addr >> 32); + xor_hw_desc->cbtal |= (u32)addr; + break; + } +} + +static int plbdma_get_transfer_width(dma_cdb_t *dma_hw_desc) +{ + switch (dma_hw_desc->ctrl.pw){ + case 0: + return 1; /* unit: bytes */ + case 1: + return 2; + case 2: + return 4; + case 3: + return 8; + case 4: + return 16; + } + return 0; +} +/** + * apm82181_desc_set_byte_count - set number of data bytes involved + * into the operation + */ +static inline void apm82181_desc_set_byte_count(apm82181_desc_t *desc, + apm82181_ch_t *chan, size_t byte_count) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + int terminal_cnt, transfer_width = 0; + + DBG("<%s> byte_count %08x\n", __func__,byte_count); + switch (chan->device->id){ + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + dma_hw_desc = desc->hw_desc; + transfer_width = plbdma_get_transfer_width(dma_hw_desc); + terminal_cnt = byte_count/transfer_width; + dma_hw_desc->cnt.tc = terminal_cnt; + break; + case APM82181_XOR_ID: + xor_hw_desc = desc->hw_desc; + xor_hw_desc->cbbc = byte_count; + break; + } +} + +/** + * apm82181_xor_set_link - set link address in xor CB + */ +static inline void apm82181_xor_set_link (apm82181_desc_t *prev_desc, + apm82181_desc_t *next_desc) +{ + xor_cb_t *xor_hw_desc = prev_desc->hw_desc; + + if (unlikely(!next_desc || !(next_desc->phys))) { + printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", + __func__, next_desc, + next_desc ? next_desc->phys : 0); + BUG(); + } + DBG("<%s>:next_desc->phys %llx\n", __func__,next_desc->phys); + xor_hw_desc->cbs = 0; + xor_hw_desc->cblal = (u32)next_desc->phys; + xor_hw_desc->cblah = (u32)(next_desc->phys >> 32); + xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; +} + +/** + * apm82181_desc_set_link - set the address of descriptor following this + * descriptor in chain + */ +static inline void apm82181_desc_set_link(apm82181_ch_t *chan, + apm82181_desc_t *prev_desc, apm82181_desc_t *next_desc) +{ + unsigned long flags; + apm82181_desc_t *tail = next_desc; + + if (unlikely(!prev_desc || !next_desc || + (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { + /* If previous next is overwritten something is wrong. + * though we may refetch from append to initiate list + * processing; in this case - it's ok. + */ + printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " + "prev->hw_next=0x%p\n", __FUNCTION__, prev_desc, + next_desc, prev_desc ? prev_desc->hw_next : 0); + BUG(); + } + + local_irq_save(flags); + + /* do s/w chaining both for DMA and XOR descriptors */ + prev_desc->hw_next = next_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + break; + case APM82181_XOR_ID: + /* bind descriptor to the chain */ + while (tail->hw_next) + tail = tail->hw_next; + xor_last_linked = tail; + + if (prev_desc == xor_last_submit) + /* do not link to the last submitted CB */ + break; + apm82181_xor_set_link (prev_desc, next_desc); + break; + default: + BUG(); + } + + local_irq_restore(flags); +} + +/** + * apm82181_desc_get_src_addr - extract the source address from the descriptor + */ +static inline u32 apm82181_desc_get_src_addr(apm82181_desc_t *desc, + apm82181_ch_t *chan, int src_idx) +{ + dma_cdb_t *dma_hw_desc; + + dma_hw_desc = desc->hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + break; + default: + return 0; + } + /* May have 0, 1, 2, or 3 sources */ + return (dma_hw_desc->src_lo); +} + +/** + * apm82181_desc_get_dest_addr - extract the destination address from the + * descriptor + */ +static inline u32 apm82181_desc_get_dest_addr(apm82181_desc_t *desc, + apm82181_ch_t *chan, int idx) +{ + dma_cdb_t *dma_hw_desc; + + dma_hw_desc = desc->hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + break; + default: + return 0; + } + + /* May have 0, 1, 2, or 3 sources */ + return (dma_hw_desc->dest_lo); +} + +/** + * apm82181_desc_get_byte_count - extract the byte count from the descriptor + */ +static inline u32 apm82181_desc_get_byte_count(apm82181_desc_t *desc, + apm82181_ch_t *chan) +{ + dma_cdb_t *dma_hw_desc; + + dma_hw_desc = desc->hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + break; + default: + return 0; + } + /* May have 0, 1, 2, or 3 sources */ + //return (dma_hw_desc->cnt); +} + + +/** + * apm82181_desc_get_link - get the address of the descriptor that + * follows this one + */ +static inline u32 apm82181_desc_get_link(apm82181_desc_t *desc, + apm82181_ch_t *chan) +{ + if (!desc->hw_next) + return 0; + + return desc->hw_next->phys; +} + +/** + * apm82181_desc_is_aligned - check alignment + */ +static inline int apm82181_desc_is_aligned(apm82181_desc_t *desc, + int num_slots) +{ + return (desc->idx & (num_slots - 1)) ? 0 : 1; +} + + + +/****************************************************************************** + * ADMA channel low-level routines + ******************************************************************************/ + +static inline phys_addr_t apm82181_chan_get_current_descriptor(apm82181_ch_t *chan); +static inline void apm82181_chan_append(apm82181_ch_t *chan); + +/* + * apm82181_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine + */ +static inline void apm82181_adma_device_clear_eot_status (apm82181_ch_t *chan) +{ + u32 val ; + int idx = chan->device->id; + volatile xor_regs_t *xor_reg; + INFO; + switch (idx) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + val = mfdcr(DCR_DMA2P40_SR); + if(val & DMA_SR_RI(idx)){ + printk(KERN_ERR "Err occurred, DMA%d status: 0x%x\n", idx, val); + } + /* TC reached int, write back to clear */ + mtdcr(DCR_DMA2P40_SR, val); + break; + case APM82181_XOR_ID: + /* reset status bits to ack*/ + xor_reg = chan->device->xor_base; + + val = xor_reg->sr; + DBG("XOR engine status: 0x%08x\n", val); + xor_reg->sr = val; + + if (val & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { + if (val & XOR_IE_RPTIE_BIT) { + /* Read PLB Timeout Error. + * Try to resubmit the CB + */ + INFO; + xor_reg->cblalr = xor_reg->ccbalr; + xor_reg->crsr |= XOR_CRSR_XAE_BIT; + } else + printk (KERN_ERR "XOR ERR 0x%x status\n", val); + break; + } + + /* if the XORcore is idle, but there are unprocessed CBs + * then refetch the s/w chain here + */ + if (!(xor_reg->sr & XOR_SR_XCP_BIT) && do_xor_refetch) { + apm82181_chan_append(chan); + } + break; + } +} + +/* + * apm82181_chan_is_busy - get the channel status + */ + +static inline int apm82181_chan_is_busy(apm82181_ch_t *chan) +{ + int busy = 0; + volatile xor_regs_t *xor_reg = chan->device->xor_base; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + if(mfdcr(DCR_DMA2P40_SR) & DMA_SR_CB(chan->device->id)) + busy = 1; + else + busy = 0; + break; + case APM82181_XOR_ID: + /* use the special status bit for the XORcore + */ + busy = (xor_reg->sr & XOR_SR_XCP_BIT) ? 1 : 0; + break; + default: + BUG(); + } + + return busy; +} + +/** + * apm82181_dma_put_desc - put PLB DMA 0/1/2/3 descriptor to FIFO + */ +static inline void apm82181_dma_put_desc(apm82181_ch_t *chan, + apm82181_desc_t *desc) +{ + dma_cdb_t *cdb = desc->hw_desc; + u32 sg_cmd = 0; + + /* Enable TC interrupt */ + if(test_bit(APM82181_DESC_INT, &desc->flags)) + cdb->cnt.tcie = 1; + else + cdb->cnt.tcie = 0; + /* Not link to next cdb */ + cdb->sg_hi = 0xffffffff; + cdb->sg_lo = 0xffffffff; + + chan_last_sub[chan->device->id] = desc; + + /* Update new cdb addr */ + mtdcr(DCR_DMA2P40_SGHx(chan->device->id), (u32)(desc->phys >> 32)); + mtdcr(DCR_DMA2P40_SGLx(chan->device->id), (u32)desc->phys); + + INFO; + DBG("slot id: %d addr: %llx\n", desc->idx, desc->phys); + DBG("S/G addr H: %08x addr L: %08x\n", + mfdcr(DCR_DMA2P40_SGHx(chan->device->id)), + mfdcr(DCR_DMA2P40_SGLx(chan->device->id))); + ADMA_HEXDUMP(cdb, 96); + /* Enable S/G */ + sg_cmd |= (DMA_SGC_SSG(chan->device->id) | DMA_SGC_EM_ALL); + sg_cmd |= DMA_SGC_SGL(chan->device->id, 0); /* S/G addr in PLB */ + + mtdcr(DCR_DMA2P40_SGC, sg_cmd); + DBG("S/G addr H: %08x addr L: %08x\n", + mfdcr(DCR_DMA2P40_SGHx(chan->device->id)), + mfdcr(DCR_DMA2P40_SGLx(chan->device->id))); + /* need to use variable for logging current CDB */ + chan->current_cdb_addr = desc->phys; + +} + +/** + * apm82181_chan_append - update the h/w chain in the channel + */ +static inline void apm82181_chan_append(apm82181_ch_t *chan) +{ + apm82181_desc_t *iter; + volatile xor_regs_t *xor_reg; + phys_addr_t cur_desc; + xor_cb_t *xcb; + unsigned long flags; + INFO; + + local_irq_save(flags); + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + cur_desc = apm82181_chan_get_current_descriptor(chan); + DBG("current_desc %llx\n", cur_desc); + if (likely(cur_desc)) { + INFO; + iter = chan_last_sub[chan->device->id]; + BUG_ON(!iter); + } else { + INFO; + /* first peer */ + iter = chan_first_cdb[chan->device->id]; + BUG_ON(!iter); + INFO; + apm82181_dma_put_desc(chan, iter); + chan->hw_chain_inited = 1; + } + + /* is there something new to append */ + if (!iter->hw_next) + break; + + /* flush descriptors from the s/w queue to fifo */ + list_for_each_entry_continue(iter, &chan->chain, chain_node) { + apm82181_dma_put_desc(chan, iter); + if (!iter->hw_next) + break; + } + break; + case APM82181_XOR_ID: + /* update h/w links and refetch */ + if (!xor_last_submit->hw_next) + break; + xor_reg = chan->device->xor_base; + /* the last linked CDB has to generate an interrupt + * that we'd be able to append the next lists to h/w + * regardless of the XOR engine state at the moment of + * appending of these next lists + */ + xcb = xor_last_linked->hw_desc; + xcb->cbc |= XOR_CBCR_CBCE_BIT; + + if (!(xor_reg->sr & XOR_SR_XCP_BIT)) { + /* XORcore is idle. Refetch now */ + do_xor_refetch = 0; + apm82181_xor_set_link(xor_last_submit, + xor_last_submit->hw_next); + + xor_last_submit = xor_last_linked; + xor_reg->crsr |= XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT; + } else { + /* XORcore is running. Refetch later in the handler */ + do_xor_refetch = 1; + } + + break; + } + + local_irq_restore(flags); +} + +/** + * apm82181_chan_get_current_descriptor - get the currently executed descriptor + */ +static inline phys_addr_t apm82181_chan_get_current_descriptor(apm82181_ch_t *chan) +{ + phys_addr_t curr_cdb_addr; + volatile xor_regs_t *xor_reg; + int idx = chan->device->id; + + if (unlikely(!chan->hw_chain_inited)) + /* h/w descriptor chain is not initialized yet */ + return 0; + switch(idx){ + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + curr_cdb_addr = chan->current_cdb_addr; + break; + case APM82181_XOR_ID: + xor_reg = chan->device->xor_base; + curr_cdb_addr = (dma_addr_t)xor_reg->ccbahr; + curr_cdb_addr = (curr_cdb_addr << 32) | xor_reg->ccbalr; + break; + default: + BUG(); + } + return curr_cdb_addr; +} + + +/****************************************************************************** + * ADMA device level + ******************************************************************************/ + +static int apm82181_adma_alloc_chan_resources(struct dma_chan *chan); +static dma_cookie_t apm82181_adma_tx_submit( + struct dma_async_tx_descriptor *tx); + +static void apm82181_adma_set_dest( + apm82181_desc_t *tx, + dma_addr_t addr, int index); + +/** + * apm82181_get_group_entry - get group entry with index idx + * @tdesc: is the last allocated slot in the group. + */ +static inline apm82181_desc_t * +apm82181_get_group_entry ( apm82181_desc_t *tdesc, u32 entry_idx) +{ + apm82181_desc_t *iter = tdesc->group_head; + int i = 0; + + if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { + printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", + __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); + BUG(); + } + list_for_each_entry(iter, &tdesc->group_list, chain_node) { + if (i++ == entry_idx) + break; + } + return iter; +} + +/** + * apm82181_adma_free_slots - flags descriptor slots for reuse + * @slot: Slot to free + * Caller must hold &apm82181_chan->lock while calling this function + */ +static void apm82181_adma_free_slots(apm82181_desc_t *slot, + apm82181_ch_t *chan) +{ + int stride = slot->slots_per_op; + + while (stride--) { + /*async_tx_clear_ack(&slot->async_tx);*/ /* Don't need to clear. It is hack*/ + slot->slots_per_op = 0; + slot = list_entry(slot->slot_node.next, + apm82181_desc_t, + slot_node); + } +} + +static void +apm82181_adma_unmap(apm82181_ch_t *chan, apm82181_desc_t *desc) +{ + u32 src_cnt, dst_cnt; + dma_addr_t addr; + /* + * get the number of sources & destination + * included in this descriptor and unmap + * them all + */ + src_cnt = 1; + dst_cnt = 1; +} +/** + * apm82181_adma_run_tx_complete_actions - call functions to be called + * upon complete + */ +static dma_cookie_t apm82181_adma_run_tx_complete_actions( + apm82181_desc_t *desc, + apm82181_ch_t *chan, + dma_cookie_t cookie) +{ + int i; + //enum dma_data_direction dir; + INFO; + BUG_ON(desc->async_tx.cookie < 0); + if (desc->async_tx.cookie > 0) { + cookie = desc->async_tx.cookie; + desc->async_tx.cookie = 0; + + /* call the callback (must not sleep or submit new + * operations to this channel) + */ + if (desc->async_tx.callback) + desc->async_tx.callback( + desc->async_tx.callback_param); + + dma_descriptor_unmap(&desc->async_tx); + /* unmap dma addresses + * (unmap_single vs unmap_page?) + * + * actually, ppc's dma_unmap_page() functions are empty, so + * the following code is just for the sake of completeness + */ + if (chan && chan->needs_unmap && desc->group_head && + desc->unmap_len) { + apm82181_desc_t *unmap = desc->group_head; + /* assume 1 slot per op always */ + u32 slot_count = unmap->slot_cnt; + + /* Run through the group list and unmap addresses */ + for (i = 0; i < slot_count; i++) { + BUG_ON(!unmap); + apm82181_adma_unmap(chan, unmap); + unmap = unmap->hw_next; + } + desc->group_head = NULL; + } + } + + /* run dependent operations */ + dma_run_dependencies(&desc->async_tx); + + return cookie; +} + +/** + * apm82181_adma_clean_slot - clean up CDB slot (if ack is set) + */ +static int apm82181_adma_clean_slot(apm82181_desc_t *desc, + apm82181_ch_t *chan) +{ + /* the client is allowed to attach dependent operations + * until 'ack' is set + */ + if (!async_tx_test_ack(&desc->async_tx)) + return 0; + + /* leave the last descriptor in the chain + * so we can append to it + */ + if (list_is_last(&desc->chain_node, &chan->chain) || + desc->phys == apm82181_chan_get_current_descriptor(chan)) + return 1; + + dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", + desc->phys, desc->idx, desc->slots_per_op); + + list_del(&desc->chain_node); + apm82181_adma_free_slots(desc, chan); + return 0; +} + +/** + * __apm82181_adma_slot_cleanup - this is the common clean-up routine + * which runs through the channel CDBs list until reach the descriptor + * currently processed. When routine determines that all CDBs of group + * are completed then corresponding callbacks (if any) are called and slots + * are freed. + */ +static void __apm82181_adma_slot_cleanup(apm82181_ch_t *chan) +{ + apm82181_desc_t *iter, *_iter, *group_start = NULL; + dma_cookie_t cookie = 0; + phys_addr_t current_desc = apm82181_chan_get_current_descriptor(chan); + int busy = apm82181_chan_is_busy(chan); + int seen_current = 0, slot_cnt = 0, slots_per_op = 0; + + DBG("apm82181 adma%d: %s\n", + chan->device->id, __FUNCTION__); + DBG("current_desc %llx\n", current_desc); + + if (!current_desc) { + /* There were no transactions yet, so + * nothing to clean + */ + return; + } + + /* free completed slots from the chain starting with + * the oldest descriptor + */ + list_for_each_entry_safe(iter, _iter, &chan->chain, + chain_node) { + DBG(" cookie: %d slot: %d " + "busy: %d this_desc: %llx next_desc: %x cur: %llx ack: %d\n", + iter->async_tx.cookie, iter->idx, busy, iter->phys, + apm82181_desc_get_link(iter, chan), current_desc, + async_tx_test_ack(&iter->async_tx)); + prefetch(_iter); + prefetch(&_iter->async_tx); + + /* do not advance past the current descriptor loaded into the + * hardware channel,subsequent descriptors are either in process + * or have not been submitted + */ + if (seen_current) + break; + + /* stop the search if we reach the current descriptor and the + * channel is busy, or if it appears that the current descriptor + * needs to be re-read (i.e. has been appended to) + */ + if (iter->phys == current_desc) { + BUG_ON(seen_current++); + if (busy || apm82181_desc_get_link(iter, chan)) { + /* not all descriptors of the group have + * been completed; exit. + */ + break; + } + } + + /* detect the start of a group transaction */ + if (!slot_cnt && !slots_per_op) { + slot_cnt = iter->slot_cnt; + slots_per_op = iter->slots_per_op; + if (slot_cnt <= slots_per_op) { + slot_cnt = 0; + slots_per_op = 0; + } + } + + if (slot_cnt) { + if (!group_start) + group_start = iter; + slot_cnt -= slots_per_op; + } + + /* all the members of a group are complete */ + if (slots_per_op != 0 && slot_cnt == 0) { + apm82181_desc_t *grp_iter, *_grp_iter; + int end_of_chain = 0; + + /* clean up the group */ + slot_cnt = group_start->slot_cnt; + grp_iter = group_start; + list_for_each_entry_safe_from(grp_iter, _grp_iter, + &chan->chain, chain_node) { + + cookie = apm82181_adma_run_tx_complete_actions( + grp_iter, chan, cookie); + + slot_cnt -= slots_per_op; + end_of_chain = apm82181_adma_clean_slot( + grp_iter, chan); + if (end_of_chain && slot_cnt) { + /* Should wait for ZeroSum complete */ + if (cookie > 0) + chan->common.completed_cookie = cookie; + return; + } + + if (slot_cnt == 0 || end_of_chain) + break; + } + + /* the group should be complete at this point */ + BUG_ON(slot_cnt); + + slots_per_op = 0; + group_start = NULL; + if (end_of_chain) + break; + else + continue; + } else if (slots_per_op) /* wait for group completion */ + continue; + + cookie = apm82181_adma_run_tx_complete_actions(iter, chan, + cookie); + + if (apm82181_adma_clean_slot(iter, chan)) + break; + } + + BUG_ON(!seen_current); + + if (cookie > 0) { + chan->common.completed_cookie = cookie; + DBG("completed cookie %d\n", cookie); + } + +} + +/** + * apm82181_adma_tasklet - clean up watch-dog initiator + */ +static void apm82181_adma_tasklet (unsigned long data) +{ + apm82181_ch_t *chan = (apm82181_ch_t *) data; + spin_lock(&chan->lock); + INFO; + __apm82181_adma_slot_cleanup(chan); + spin_unlock(&chan->lock); +} + +/** + * apm82181_adma_slot_cleanup - clean up scheduled initiator + */ +static void apm82181_adma_slot_cleanup (apm82181_ch_t *chan) +{ + spin_lock_bh(&chan->lock); + __apm82181_adma_slot_cleanup(chan); + spin_unlock_bh(&chan->lock); +} + +/** + * apm82181_adma_alloc_slots - allocate free slots (if any) + */ +static apm82181_desc_t *apm82181_adma_alloc_slots( + apm82181_ch_t *chan, int num_slots, + int slots_per_op) +{ + apm82181_desc_t *iter = NULL, *_iter, *alloc_start = NULL; + struct list_head chain = LIST_HEAD_INIT(chain); + int slots_found, retry = 0; + + + BUG_ON(!num_slots || !slots_per_op); + /* start search from the last allocated descrtiptor + * if a contiguous allocation can not be found start searching + * from the beginning of the list + */ +retry: + slots_found = 0; + if (retry == 0) + iter = chan->last_used; + else + iter = list_entry(&chan->all_slots, apm82181_desc_t, + slot_node); + prefetch(iter); + DBG("---iter at %p idx %d\n ",iter,iter->idx); + list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, + slot_node) { + prefetch(_iter); + prefetch(&_iter->async_tx); + if (iter->slots_per_op) { + slots_found = 0; + continue; + } + + /* start the allocation if the slot is correctly aligned */ + if (!slots_found++) + alloc_start = iter; + if (slots_found == num_slots) { + apm82181_desc_t *alloc_tail = NULL; + apm82181_desc_t *last_used = NULL; + iter = alloc_start; + while (num_slots) { + int i; + + /* pre-ack all but the last descriptor */ + if (num_slots != slots_per_op) { + async_tx_ack(&iter->async_tx); + } + list_add_tail(&iter->chain_node, &chain); + alloc_tail = iter; + iter->async_tx.cookie = 0; + iter->hw_next = NULL; + iter->flags = 0; + iter->slot_cnt = num_slots; + for (i = 0; i < slots_per_op; i++) { + iter->slots_per_op = slots_per_op - i; + last_used = iter; + iter = list_entry(iter->slot_node.next, + apm82181_desc_t, + slot_node); + } + num_slots -= slots_per_op; + } + alloc_tail->group_head = alloc_start; + alloc_tail->async_tx.cookie = -EBUSY; + list_splice(&chain, &alloc_tail->group_list); + chan->last_used = last_used; + DBG("---slot allocated at %llx idx %d, hw_desc %p tx_ack %d\n", + alloc_tail->phys, alloc_tail->idx, alloc_tail->hw_desc, + async_tx_test_ack(&alloc_tail->async_tx)); + return alloc_tail; + } + } + if (!retry++) + goto retry; +#ifdef ADMA_DEBUG + static int empty_slot_cnt; + if(!(empty_slot_cnt%100)) + printk(KERN_INFO"No empty slots trying to free some\n"); + empty_slot_cnt++; +#endif + /* try to free some slots if the allocation fails */ + tasklet_schedule(&chan->irq_tasklet); + return NULL; +} + +/** + * apm82181_chan_xor_slot_count - get the number of slots necessary for + * XOR operation + */ +static inline int apm82181_chan_xor_slot_count(size_t len, int src_cnt, + int *slots_per_op) +{ + int slot_cnt; + + /* each XOR descriptor provides up to 16 source operands */ + slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; + + if (likely(len <= APM82181_ADMA_XOR_MAX_BYTE_COUNT)) + return slot_cnt; + + printk(KERN_ERR "%s: len %d > max %d !!\n", + __func__, len, APM82181_ADMA_XOR_MAX_BYTE_COUNT); + BUG(); + return slot_cnt; +} + +/** + * apm82181_desc_init_null_xor - initialize the descriptor for NULL XOR + * pseudo operation + */ +static inline void apm82181_desc_init_null_xor(apm82181_desc_t *desc) +{ + memset (desc->hw_desc, 0, sizeof(xor_cb_t)); + desc->hw_next = NULL; + desc->src_cnt = 0; + desc->dst_cnt = 1; +} +/** + * apm82181_chan_set_first_xor_descriptor - initi XORcore chain + */ +static inline void apm82181_chan_set_first_xor_descriptor(apm82181_ch_t *chan, + apm82181_desc_t *next_desc) +{ + volatile xor_regs_t *xor_reg; + + xor_reg = chan->device->xor_base; + + if (xor_reg->sr & XOR_SR_XCP_BIT) + printk(KERN_INFO "%s: Warn: XORcore is running " + "when try to set the first CDB!\n", + __func__); + + xor_last_submit = xor_last_linked = next_desc; + + xor_reg->crsr = XOR_CRSR_64BA_BIT; + + xor_reg->cblalr = next_desc->phys; + xor_reg->cblahr = 0; + xor_reg->cbcr |= XOR_CBCR_LNK_BIT; + + chan->hw_chain_inited = 1; +} +/** + * apm82181_chan_start_null_xor - initiate the first XOR operation (DMA engines + * use FIFOs (as opposite to chains used in XOR) so this is a XOR + * specific operation) + */ +static void apm82181_chan_start_null_xor(apm82181_ch_t *chan) +{ + apm82181_desc_t *sw_desc, *group_start; + dma_cookie_t cookie; + int slot_cnt, slots_per_op; + volatile xor_regs_t *xor_reg = chan->device->xor_base; + + dev_dbg(chan->device->common.dev, + "apm82181 adma%d: %s\n", chan->device->id, __func__); + INFO; + spin_lock_bh(&chan->lock); + slot_cnt = apm82181_chan_xor_slot_count(0, 2, &slots_per_op); + sw_desc = apm82181_adma_alloc_slots(chan, slot_cnt, slots_per_op); + if (sw_desc) { + INFO; + group_start = sw_desc->group_head; + list_splice_init(&sw_desc->group_list, &chan->chain); + async_tx_ack(&sw_desc->async_tx); + apm82181_desc_init_null_xor(group_start); + INFO; + + cookie = chan->common.cookie; + cookie++; + if (cookie <= 1) + cookie = 2; + + /* initialize the completed cookie to be less than + * the most recently used cookie + */ + chan->common.completed_cookie = cookie - 1; + chan->common.cookie = sw_desc->async_tx.cookie = cookie; + + /* channel should not be busy */ + BUG_ON(apm82181_chan_is_busy(chan)); + + /* set the descriptor address */ + apm82181_chan_set_first_xor_descriptor(chan, sw_desc); + + /* run the descriptor */ + xor_reg->crsr = XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT; + } else + printk(KERN_ERR "apm82181 adma%d" + " failed to allocate null descriptor\n", + chan->device->id); + spin_unlock_bh(&chan->lock); +} + +/** + * apm82181_adma_alloc_chan_resources - allocate pools for CDB slots + */ +static int apm82181_adma_alloc_chan_resources(struct dma_chan *chan) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *slot = NULL; + char *hw_desc; + int i, db_sz; + int init = apm82181_chan->slots_allocated ? 0 : 1; + + chan->chan_id = apm82181_chan->device->id; + + /* Allocate descriptor slots */ + i = apm82181_chan->slots_allocated; + if (apm82181_chan->device->id != APM82181_XOR_ID) + db_sz = sizeof (dma_cdb_t); + else + db_sz = sizeof (xor_cb_t); + + for (; i < (apm82181_chan->device->pool_size/db_sz); i++) { + slot = kzalloc(sizeof(apm82181_desc_t), GFP_KERNEL); + if (!slot) { + printk(KERN_INFO "APM82181/GT ADMA Channel only initialized" + " %d descriptor slots", i--); + break; + } + + hw_desc = (char *) apm82181_chan->device->dma_desc_pool_virt; + slot->hw_desc = (void *) &hw_desc[i * db_sz]; + dma_async_tx_descriptor_init(&slot->async_tx, chan); + slot->async_tx.tx_submit = apm82181_adma_tx_submit; + INIT_LIST_HEAD(&slot->chain_node); + INIT_LIST_HEAD(&slot->slot_node); + INIT_LIST_HEAD(&slot->group_list); + slot->phys = apm82181_chan->device->dma_desc_pool + i * db_sz; + slot->idx = i; + spin_lock_bh(&apm82181_chan->lock); + apm82181_chan->slots_allocated++; + list_add_tail(&slot->slot_node, &apm82181_chan->all_slots); + spin_unlock_bh(&apm82181_chan->lock); + } + + if (i && !apm82181_chan->last_used) { + apm82181_chan->last_used = + list_entry(apm82181_chan->all_slots.next, + apm82181_desc_t, + slot_node); + } + + printk("apm82181 adma%d: allocated %d descriptor slots\n", + apm82181_chan->device->id, i); + + /* initialize the channel and the chain with a null operation */ + if (init) { + switch (apm82181_chan->device->id) + { + apm82181_chan->hw_chain_inited = 0; + case APM82181_PDMA0_ID: + apm82181_dma_tchan[0] = apm82181_chan; + break; + case APM82181_PDMA1_ID: + apm82181_dma_tchan[1] = apm82181_chan; + break; + case APM82181_PDMA2_ID: + apm82181_dma_tchan[2] = apm82181_chan; + break; + case APM82181_PDMA3_ID: + apm82181_dma_tchan[3] = apm82181_chan; + break; + case APM82181_XOR_ID: + apm82181_dma_tchan[4] = apm82181_chan; + apm82181_chan_start_null_xor(apm82181_chan); + break; + default: + BUG(); + } + apm82181_chan->needs_unmap = 1; + } + + return (i > 0) ? i : -ENOMEM; +} + +/** + * apm82181_desc_assign_cookie - assign a cookie + */ +static dma_cookie_t apm82181_desc_assign_cookie(apm82181_ch_t *chan, + apm82181_desc_t *desc) +{ + dma_cookie_t cookie = chan->common.cookie; + cookie++; + if (cookie < 0) + cookie = 1; + chan->common.cookie = desc->async_tx.cookie = cookie; + return cookie; +} + + +/** + * apm82181_adma_check_threshold - append CDBs to h/w chain if threshold + * has been achieved + */ +static void apm82181_adma_check_threshold(apm82181_ch_t *chan) +{ + dev_dbg(chan->device->common.dev, "apm82181 adma%d: pending: %d\n", + chan->device->id, chan->pending); + INFO; + if (chan->pending >= APM82181_ADMA_THRESHOLD) { + chan->pending = 0; + apm82181_chan_append(chan); + } +} + +/** + * apm82181_adma_tx_submit - submit new descriptor group to the channel + * (it's not necessary that descriptors will be submitted to the h/w + * chains too right now) + */ +static dma_cookie_t apm82181_adma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + apm82181_desc_t *sw_desc = tx_to_apm82181_adma_slot(tx); + apm82181_ch_t *chan = to_apm82181_adma_chan(tx->chan); + apm82181_desc_t *group_start, *old_chain_tail; + int slot_cnt; + int slots_per_op; + dma_cookie_t cookie; + group_start = sw_desc->group_head; + slot_cnt = group_start->slot_cnt; + slots_per_op = group_start->slots_per_op; + INFO; + spin_lock_bh(&chan->lock); + cookie = apm82181_desc_assign_cookie(chan, sw_desc); + + if (unlikely(list_empty(&chan->chain))) { + /* first peer */ + list_splice_init(&sw_desc->group_list, &chan->chain); + chan_first_cdb[chan->device->id] = group_start; + } else { + /* isn't first peer, bind CDBs to chain */ + old_chain_tail = list_entry(chan->chain.prev, + apm82181_desc_t, chain_node); + list_splice_init(&sw_desc->group_list, + &old_chain_tail->chain_node); + /* fix up the hardware chain */ + apm82181_desc_set_link(chan, old_chain_tail, group_start); + } + + /* increment the pending count by the number of operations */ + chan->pending += slot_cnt / slots_per_op; + apm82181_adma_check_threshold(chan); + spin_unlock_bh(&chan->lock); + + DBG("apm82181 adma%d:cookie: %d slot: %d tx %p\n", + chan->device->id, sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); + return cookie; +} +/** + * apm82181_adma_prep_dma_xor - prepare CDB for a XOR operation + */ +static struct dma_async_tx_descriptor *apm82181_adma_prep_dma_xor( + struct dma_chan *chan, dma_addr_t dma_dest, + dma_addr_t *dma_src, unsigned int src_cnt, size_t len, + unsigned long flags) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + +#ifdef ADMA_DEBUG + printk("\n%s(%d):\n\tsrc: ", __func__, + apm82181_chan->device->id); + for (slot_cnt=0; slot_cnt < src_cnt; slot_cnt++) + printk("0x%llx ", dma_src[slot_cnt]); + printk("\n\tdst: 0x%llx\n", dma_dest); +#endif + if (unlikely(!len)) + return NULL; + BUG_ON(unlikely(len > APM82181_ADMA_XOR_MAX_BYTE_COUNT)); + + dev_dbg(apm82181_chan->device->common.dev, + "apm82181 adma%d: %s src_cnt: %d len: %u int_en: %d\n", + apm82181_chan->device->id, __func__, src_cnt, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + spin_lock_bh(&apm82181_chan->lock); + slot_cnt = apm82181_chan_xor_slot_count(len, src_cnt, &slots_per_op); + sw_desc = apm82181_adma_alloc_slots(apm82181_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + apm82181_desc_init_xor(group_start, src_cnt, flags); + apm82181_adma_set_dest(group_start, dma_dest, 0); + while (src_cnt--) + apm82181_adma_set_src(group_start, + dma_src[src_cnt], src_cnt); + apm82181_desc_set_byte_count(group_start, apm82181_chan, len); + sw_desc->unmap_len = len; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&apm82181_chan->lock); + + return sw_desc ? &sw_desc->async_tx : NULL; +} +/** + * apm82181_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation + */ +static struct dma_async_tx_descriptor *apm82181_adma_prep_dma_interrupt( + struct dma_chan *chan, unsigned long flags) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + + dev_dbg(apm82181_chan->device->common.dev, + "apm82181 adma%d: %s\n", apm82181_chan->device->id, + __FUNCTION__); + spin_lock_bh(&apm82181_chan->lock); + slot_cnt = slots_per_op = 1; + sw_desc = apm82181_adma_alloc_slots(apm82181_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + apm82181_desc_init_interrupt(group_start, apm82181_chan); + group_start->unmap_len = 0; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&apm82181_chan->lock); + + return sw_desc ? &sw_desc->async_tx : NULL; +} + +/** + * apm82181_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation + */ +static struct dma_async_tx_descriptor *apm82181_adma_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dma_dest, + dma_addr_t dma_src, size_t len, unsigned long flags) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + if (unlikely(!len)) + return NULL; + BUG_ON(unlikely(len > APM82181_ADMA_DMA_MAX_BYTE_COUNT)); + + spin_lock_bh(&apm82181_chan->lock); + + dev_dbg(apm82181_chan->device->common.dev, + "apm82181 adma%d: %s len: %u int_en %d \n", + apm82181_chan->device->id, __FUNCTION__, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + slot_cnt = slots_per_op = 1; + sw_desc = apm82181_adma_alloc_slots(apm82181_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + flags |= DMA_PREP_INTERRUPT; + apm82181_desc_init_memcpy(group_start, flags); + apm82181_adma_set_dest(group_start, dma_dest, 0); + apm82181_adma_set_src(group_start, dma_src, 0); + apm82181_desc_set_byte_count(group_start, apm82181_chan, len); + sw_desc->unmap_len = len; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&apm82181_chan->lock); + return sw_desc ? &sw_desc->async_tx : NULL; +} + + +/** + * apm82181_adma_set_dest - set destination address into descriptor + */ +static void apm82181_adma_set_dest(apm82181_desc_t *sw_desc, + dma_addr_t addr, int index) +{ + apm82181_ch_t *chan = to_apm82181_adma_chan(sw_desc->async_tx.chan); + BUG_ON(index >= sw_desc->dst_cnt); + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + /* to do: support transfers lengths > + * APM82181_ADMA_DMA/XOR_MAX_BYTE_COUNT + */ + apm82181_desc_set_dest_addr(sw_desc->group_head, + // chan, 0x8, addr, index); // Enabling HB bus + chan, addr, index); + break; + case APM82181_XOR_ID: + sw_desc = apm82181_get_group_entry(sw_desc, index); + apm82181_desc_set_dest_addr(sw_desc, chan, + addr, index); + break; + default: + BUG(); + } +} + + +/** + * apm82181_adma_free_chan_resources - free the resources allocated + */ +static void apm82181_adma_free_chan_resources(struct dma_chan *chan) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *iter, *_iter; + int in_use_descs = 0; + + apm82181_adma_slot_cleanup(apm82181_chan); + + spin_lock_bh(&apm82181_chan->lock); + list_for_each_entry_safe(iter, _iter, &apm82181_chan->chain, + chain_node) { + in_use_descs++; + list_del(&iter->chain_node); + } + list_for_each_entry_safe_reverse(iter, _iter, + &apm82181_chan->all_slots, slot_node) { + list_del(&iter->slot_node); + kfree(iter); + apm82181_chan->slots_allocated--; + } + apm82181_chan->last_used = NULL; + + dev_dbg(apm82181_chan->device->common.dev, + "apm82181 adma%d %s slots_allocated %d\n", + apm82181_chan->device->id, + __FUNCTION__, apm82181_chan->slots_allocated); + spin_unlock_bh(&apm82181_chan->lock); + + /* one is ok since we left it on there on purpose */ + if (in_use_descs > 1) + printk(KERN_ERR "GT: Freeing %d in use descriptors!\n", + in_use_descs - 1); +} + +/** + * apm82181_adma_tx_status - poll the status of an ADMA transaction + * @chan: ADMA channel handle + * @cookie: ADMA transaction identifier + */ +static enum dma_status apm82181_adma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + enum dma_status ret; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + apm82181_adma_slot_cleanup(apm82181_chan); + + return dma_cookie_status(chan, cookie, txstate); +} + +/** + * apm82181_adma_eot_handler - end of transfer interrupt handler + */ +static irqreturn_t apm82181_adma_eot_handler(int irq, void *data) +{ + apm82181_ch_t *chan = data; + + dev_dbg(chan->device->common.dev, + "apm82181 adma%d: %s\n", chan->device->id, __FUNCTION__); + INFO; + if(chan->device->id == APM82181_XOR_ID) + tasklet_schedule(&chan->irq_tasklet); + apm82181_adma_device_clear_eot_status(chan); + + return IRQ_HANDLED; +} + +/** + * apm82181_adma_err_handler - DMA error interrupt handler; + * do the same things as a eot handler + */ +#if 0 +static irqreturn_t apm82181_adma_err_handler(int irq, void *data) +{ + apm82181_ch_t *chan = data; + dev_dbg(chan->device->common.dev, + "apm82181 adma%d: %s\n", chan->device->id, __FUNCTION__); + tasklet_schedule(&chan->irq_tasklet); + apm82181_adma_device_clear_eot_status(chan); + + return IRQ_HANDLED; +} +#endif +/** + * apm82181_test_callback - called when test operation has been done + */ +static void apm82181_test_callback (void *unused) +{ + complete(&apm82181_r5_test_comp); +} + +/** + * apm82181_adma_issue_pending - flush all pending descriptors to h/w + */ +static void apm82181_adma_issue_pending(struct dma_chan *chan) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + + DBG("apm82181 adma%d: %s %d \n", apm82181_chan->device->id, + __FUNCTION__, apm82181_chan->pending); + if (apm82181_chan->pending) { + apm82181_chan->pending = 0; + apm82181_chan_append(apm82181_chan); + } +} + +static inline void xor_hw_init (apm82181_dev_t *adev) +{ + volatile xor_regs_t *xor_reg = adev->xor_base; + /* Reset XOR */ + xor_reg->crsr = XOR_CRSR_XASR_BIT; + xor_reg->crrr = XOR_CRSR_64BA_BIT; + + /* enable XOR engine interrupts */ + xor_reg->ier = XOR_IE_CBCIE_BIT | + XOR_IE_ICBIE_BIT | XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT; +} + +/* + * Per channel probe + */ +static int apm82181_dma_per_chan_probe(struct platform_device *ofdev) +{ + int ret = 0, irq; + const u32 *index, *dcr_regs, *pool_size; + apm82181_plb_dma_t *pdma; + apm82181_dev_t *adev; + apm82181_ch_t *chan; + struct device_node *np = ofdev->dev.of_node; + struct resource res; + int len; + + INFO; + pdma = dev_get_drvdata(ofdev->dev.parent); + BUG_ON(!pdma); + if ((adev = kzalloc(sizeof(*adev), GFP_KERNEL)) == NULL) { + printk("ERROR:No Free memory for allocating dma channels\n"); + ret = -ENOMEM; + goto out; + } + adev->dev = &ofdev->dev; + index = of_get_property(np, "cell-index", NULL); + if(!index) { + printk(KERN_ERR "adma-channel: Device node %s has missing or invalid " + "cell-index property\n", np->full_name); + goto err; + } + adev->id = (int)*index; + /* The XOR engine/PLB DMA 4 channels have different resources/pool_sizes */ + if (adev->id != APM82181_XOR_ID){ + dcr_regs = of_get_property(np, "dcr-reg", &len); + if (!dcr_regs || (len != 2 * sizeof(u32))) { + printk(KERN_ERR "plb_dma channel%d: Can't get DCR register base !", + adev->id); + goto err; + } + adev->dcr_base = dcr_regs[0]; + + pool_size = of_get_property(np, "pool_size", NULL); + if(!pool_size) { + printk(KERN_ERR "plb_dma channel%d: Device node %s has missing or " + "invalid pool_size property\n", adev->id, np->full_name); + goto err; + } + adev->pool_size = *pool_size; + } else { + if (of_address_to_resource(np, 0, &res)) { + printk(KERN_ERR "adma_xor channel%d %s: could not get resource address.\n", + adev->id,np->full_name); + goto err; + } + + DBG("XOR resource start = %llx end = %llx\n", res.start, res.end); + adev->xor_base = ioremap(res.start, res.end - res.start + 1); + if (!adev->xor_base){ + printk(KERN_ERR "XOR engine registers memory mapping failed.\n"); + goto err; + } + adev->pool_size = PAGE_SIZE << 1; + } + + adev->pdma = pdma; + adev->ofdev = ofdev; + dev_set_drvdata(&(ofdev->dev),adev); + + switch (adev->id){ + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + dma_cap_set(DMA_MEMCPY,adev->cap_mask); + break; + case APM82181_XOR_ID: + dma_cap_set(DMA_XOR,adev->cap_mask); + dma_cap_set(DMA_INTERRUPT,adev->cap_mask); + break; + default: + BUG(); + } + /* XOR h/w configuration */ + if(adev->id == APM82181_XOR_ID) + xor_hw_init(adev); + /* allocate coherent memory for hardware descriptors + * note: writecombine gives slightly better performance, but + * requires that we explicitly drain the write buffer + */ + if ((adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, + adev->pool_size, &adev->dma_desc_pool, GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_dma_alloc; + } + + adev->common.cap_mask = adev->cap_mask; + INIT_LIST_HEAD(&adev->common.channels); + /* set base routines */ + adev->common.device_alloc_chan_resources = + apm82181_adma_alloc_chan_resources; + adev->common.device_free_chan_resources = + apm82181_adma_free_chan_resources; + adev->common.device_tx_status = apm82181_adma_tx_status; + adev->common.device_issue_pending = apm82181_adma_issue_pending; + adev->common.dev = &ofdev->dev; + + /* set prep routines based on capability */ + if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { + adev->common.device_prep_dma_memcpy = + apm82181_adma_prep_dma_memcpy; + } + + if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { + adev->common.device_prep_dma_interrupt = + apm82181_adma_prep_dma_interrupt; + } + + if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { + adev->common.max_xor = XOR_MAX_OPS; + adev->common.device_prep_dma_xor = + apm82181_adma_prep_dma_xor; + } + + /* create a channel */ + if ((chan = kzalloc(sizeof(*chan), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_chan_alloc; + } + tasklet_init(&chan->irq_tasklet, apm82181_adma_tasklet, + (unsigned long)chan); + + irq = irq_of_parse_and_map(np, 0); + switch (adev->id){ + case 0: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-chan0", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + case 1: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-chan1", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + case 2: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-chan2", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + case 3: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-chan3", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + case 4: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-xor", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + default: + BUG(); + } + + spin_lock_init(&chan->lock); + chan->device = adev; + INIT_LIST_HEAD(&chan->chain); + INIT_LIST_HEAD(&chan->all_slots); + chan->common.device = &adev->common; + list_add_tail(&chan->common.device_node, &adev->common.channels); + adev->common.chancnt++; + + printk( "AMCC(R) APM82181 ADMA Engine found [%d]: " + "( capabilities: %s%s%s%s%s%s)\n", + adev->id, + dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq_xor " : "", + dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : + "", + dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", + dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : + "", + dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", + dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "int " : ""); + INFO; + ret = dma_async_device_register(&adev->common); + if (ret) { + dev_err(&ofdev->dev, "failed to register dma async device"); + goto err_irq; + } + INFO; + + goto out; +err_irq: + kfree(chan); +err_chan_alloc: + dma_free_coherent(&ofdev->dev, adev->pool_size, + adev->dma_desc_pool_virt, adev->dma_desc_pool); +err_dma_alloc: + if (adev->xor_base) + iounmap(adev->xor_base); +err: + kfree(adev); +out: + return ret; +} + +static struct of_device_id dma_4chan_match[] = +{ + { + .compatible = "amcc,apm82181-adma", + }, + {}, +}; + +static struct of_device_id dma_per_chan_match[] = { + {.compatible = "amcc,apm82181-dma-4channel",}, + {.compatible = "amcc,xor",}, + {}, +}; +/* + * apm82181_adma_probe - probe the asynch device + */ +static int apm82181_pdma_probe(struct platform_device *ofdev) +{ + int ret = 0; + apm82181_plb_dma_t *pdma; + + if ((pdma = kzalloc(sizeof(*pdma), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto out; + } + pdma->dev = &ofdev->dev; + pdma->ofdev = ofdev; + printk(PPC4XX_EDMA "Probing AMCC APM82181 ADMA engines...\n"); + + dev_set_drvdata(&(ofdev->dev),pdma); + of_platform_bus_probe(ofdev->dev.of_node, dma_per_chan_match,&ofdev->dev); + +out: + return ret; +} + +/* + * apm82181_test_xor - test are RAID-5 XOR capability enabled successfully. + * For this we just perform one DMA XOR operation with the 3 sources + * to a destination + */ +static int apm82181_test_xor (apm82181_ch_t *chan) +{ + apm82181_desc_t *sw_desc, *group_start; + struct page *pg_src[3], *pg_dest; + char *a; + dma_addr_t dma_src_addr[3]; + dma_addr_t dma_dst_addr; + int rval = -EFAULT, i; + int len = PAGE_SIZE, src_cnt = 3; + int slot_cnt, slots_per_op; + INFO; + printk("ADMA channel %d XOR testing\n",chan->device->id); + for(i = 0; i < 3; i++){ + pg_src[i] = alloc_page(GFP_KERNEL); + if (!pg_src[i]) + return -ENOMEM; + } + pg_dest = alloc_page(GFP_KERNEL); + if (!pg_dest) + return -ENOMEM; + /* Fill the test page with ones */ + memset(page_address(pg_src[0]), 0xDA, len); + memset(page_address(pg_src[1]), 0xDA, len); + memset(page_address(pg_src[2]), 0x00, len); + memset(page_address(pg_dest), 0xA5, len); + for(i = 0; i < 3; i++){ + a = page_address(pg_src[i]); + printk("The virtual addr of src %d =%x\n",i, (unsigned int)a); + MEM_HEXDUMP(a,50); + } + a = page_address(pg_dest); + printk("The virtual addr of dest=%x\n", (unsigned int)a); + MEM_HEXDUMP(a,50); + + for(i = 0; i < 3; i++){ + dma_src_addr[i] = dma_map_page(chan->device->dev, pg_src[i], 0, len, + DMA_BIDIRECTIONAL); + } + dma_dst_addr = dma_map_page(chan->device->dev, pg_dest, 0, len, + DMA_BIDIRECTIONAL); + printk("dma_src_addr[0]: %llx; dma_src_addr[1]: %llx;\n " + "dma_src_addr[2]: %llx; dma_dst_addr %llx, len: %x\n", dma_src_addr[0], + dma_src_addr[1], dma_src_addr[2], dma_dst_addr, len); + + spin_lock_bh(&chan->lock); + slot_cnt = apm82181_chan_xor_slot_count(len, src_cnt, &slots_per_op); + sw_desc = apm82181_adma_alloc_slots(chan, slot_cnt, slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + apm82181_desc_init_xor(group_start, src_cnt, DMA_PREP_INTERRUPT); + /* Setup addresses */ + while (src_cnt--) + apm82181_adma_set_src(group_start, + dma_src_addr[src_cnt], src_cnt); + apm82181_adma_set_dest(group_start, dma_dst_addr, 0); + apm82181_desc_set_byte_count(group_start, chan, len); + sw_desc->unmap_len = PAGE_SIZE; + } else { + rval = -EFAULT; + spin_unlock_bh(&chan->lock); + goto exit; + } + spin_unlock_bh(&chan->lock); + + printk("Submit CDB...\n"); + MEM_HEXDUMP(sw_desc->hw_desc, 96); + async_tx_ack(&sw_desc->async_tx); + sw_desc->async_tx.callback = apm82181_test_callback; + sw_desc->async_tx.callback_param = NULL; + + init_completion(&apm82181_r5_test_comp); + apm82181_adma_tx_submit(&sw_desc->async_tx); + apm82181_adma_issue_pending(&chan->common); + //wait_for_completion(&apm82181_r5_test_comp); + /* wait for a while so that dma transaction finishes */ + mdelay(100); + /* Now check if the test page zeroed */ + a = page_address(pg_dest); + /* XOR result at destination */ + MEM_HEXDUMP(a,50); + if ((*(u32*)a) == 0x00000000 && memcmp(a, a+4, PAGE_SIZE-4)==0) { + /* page dest XOR is corect as expected - RAID-5 enabled */ + rval = 0; + } else { + /* RAID-5 was not enabled */ + rval = -EINVAL; + } + +exit: + dma_unmap_page(chan->device->dev, dma_src_addr[0], PAGE_SIZE, DMA_BIDIRECTIONAL); + dma_unmap_page(chan->device->dev, dma_src_addr[1], PAGE_SIZE, DMA_BIDIRECTIONAL); + dma_unmap_page(chan->device->dev, dma_src_addr[2], PAGE_SIZE, DMA_BIDIRECTIONAL); + dma_unmap_page(chan->device->dev, dma_dst_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(pg_src[0]); + __free_page(pg_src[1]); + __free_page(pg_src[2]); + __free_page(pg_dest); + return rval; +} + + +/* + * apm82181_test_dma - test are RAID-5 capabilities enabled successfully. + * For this we just perform one WXOR operation with the same source + * and destination addresses, the GF-multiplier is 1; so if RAID-5 + o/of_platform_driver_unregister(&apm82181_pdma_driver); + * capabilities are enabled then we'll get src/dst filled with zero. + */ +static int apm82181_test_dma (apm82181_ch_t *chan) +{ + apm82181_desc_t *sw_desc; + struct page *pg_src, *pg_dest; + char *a, *d; + dma_addr_t dma_src_addr; + dma_addr_t dma_dst_addr; + int rval = -EFAULT; + int len = PAGE_SIZE; + + printk("PLB DMA channel %d memcpy testing\n",chan->device->id); + pg_src = alloc_page(GFP_KERNEL); + if (!pg_src) + return -ENOMEM; + pg_dest = alloc_page(GFP_KERNEL); + if (!pg_dest) + return -ENOMEM; + /* Fill the test page with ones */ + memset(page_address(pg_src), 0x77, len); + memset(page_address(pg_dest), 0xa5, len); + a = page_address(pg_src); + printk("The virtual addr of src =%x\n", (unsigned int)a); + MEM_HEXDUMP(a,50); + a = page_address(pg_dest); + printk("The virtual addr of dest=%x\n", (unsigned int)a); + MEM_HEXDUMP(a,50); + dma_src_addr = dma_map_page(chan->device->dev, pg_src, 0, len, + DMA_BIDIRECTIONAL); + dma_dst_addr = dma_map_page(chan->device->dev, pg_dest, 0, len, + DMA_BIDIRECTIONAL); + printk("dma_src_addr: %llx; dma_dst_addr %llx\n", dma_src_addr, dma_dst_addr); + + spin_lock_bh(&chan->lock); + sw_desc = apm82181_adma_alloc_slots(chan, 1, 1); + if (sw_desc) { + /* 1 src, 1 dst, int_ena */ + apm82181_desc_init_memcpy(sw_desc, DMA_PREP_INTERRUPT); + //apm82181_desc_init_memcpy(sw_desc, 0); + /* Setup adresses */ + apm82181_adma_set_src(sw_desc, dma_src_addr, 0); + apm82181_adma_set_dest(sw_desc, dma_dst_addr, 0); + apm82181_desc_set_byte_count(sw_desc, chan, len); + sw_desc->unmap_len = PAGE_SIZE; + } else { + rval = -EFAULT; + spin_unlock_bh(&chan->lock); + goto exit; + } + spin_unlock_bh(&chan->lock); + + printk("Submit CDB...\n"); + MEM_HEXDUMP(sw_desc->hw_desc, 96); + async_tx_ack(&sw_desc->async_tx); + sw_desc->async_tx.callback = apm82181_test_callback; + sw_desc->async_tx.callback_param = NULL; + + init_completion(&apm82181_r5_test_comp); + apm82181_adma_tx_submit(&sw_desc->async_tx); + apm82181_adma_issue_pending(&chan->common); + //wait_for_completion(&apm82181_r5_test_comp); + + a = page_address(pg_src); + d = page_address(pg_dest); + if (!memcmp(a, d, len)) { + rval = 0; + } else { + rval = -EINVAL; + } + + a = page_address(pg_src); + printk("\nAfter DMA done:"); + printk("\nsrc %x value:\n", (unsigned int)a); + MEM_HEXDUMP(a,96); + a = page_address(pg_dest); + printk("\ndest%x value:\n", (unsigned int)a); + MEM_HEXDUMP(a,96); + +exit: + __free_page(pg_src); + __free_page(pg_dest); + return rval; +} + +static struct platform_driver apm82181_pdma_driver = { + .driver = { + .name = "apm82181_plb_dma", + .owner = THIS_MODULE, + .of_match_table = dma_4chan_match, + }, + .probe = apm82181_pdma_probe, + //.remove = apm82181_pdma_remove, +}; +struct platform_driver apm82181_dma_per_chan_driver = { + .driver = { + .name = "apm82181-dma-4channel", + .owner = THIS_MODULE, + .of_match_table = dma_per_chan_match, + }, + .probe = apm82181_dma_per_chan_probe, +}; + +static int __init apm82181_adma_per_chan_init (void) +{ + int rval; + rval = platform_driver_register(&apm82181_dma_per_chan_driver); + return rval; +} + +static int __init apm82181_adma_init (void) +{ + int rval; + struct proc_dir_entry *p; + + rval = platform_driver_register(&apm82181_pdma_driver); + + return rval; +} + +#if 0 +static void __exit apm82181_adma_exit (void) +{ + of_unregister_platform_driver(&apm82181_pdma_driver); + return; +} +module_exit(apm82181_adma_exit); +#endif + +module_init(apm82181_adma_per_chan_init); +module_init(apm82181_adma_init); + +MODULE_AUTHOR("Tai Tri Nguyen<ttnguyen@appliedmicro.com>"); +MODULE_DESCRIPTION("APM82181 ADMA Engine Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/ppc4xx/ppc460ex_4chan_dma.c b/drivers/dma/ppc4xx/ppc460ex_4chan_dma.c new file mode 100644 index 00000000000..821e279e0b7 --- /dev/null +++ b/drivers/dma/ppc4xx/ppc460ex_4chan_dma.c @@ -0,0 +1,1110 @@ +/* + * Copyright(c) 2008 Applied Micro Circuits Corporation(AMCC). All rights reserved. + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ + + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/async_tx.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/of_platform.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/uaccess.h> +#include <linux/proc_fs.h> +#include <linux/slab.h> +#include <asm/dcr-regs.h> +#include <asm/dcr.h> +#include "ppc460ex_4chan_dma.h" + + + +#ifdef DEBUG_TEST +#define dma_pr printk +#else +#define dma_pr +#endif +#define TEST_SIZE 12 + + +ppc460ex_plb_dma_dev_t *adev; + + + +int ppc460ex_get_dma_channel(void) +{ + int i; + unsigned int status = 0; + status = mfdcr(DCR_DMA2P40_SR); + + for(i=0; i<MAX_PPC460EX_DMA_CHANNELS; i++) { + if ((status & (1 >> (20+i))) == 0) + return i; + } + return -ENODEV; +} + + +int ppc460ex_get_dma_status(void) +{ + return (mfdcr(DCR_DMA2P40_SR)); + +} + + +int ppc460ex_set_src_addr(int ch_id, phys_addr_t src_addr) +{ + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk("%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + +#ifdef PPC4xx_DMA_64BIT + mtdcr(DCR_DMA2P40_SAH0 + ch_id*8, src_addr >> 32); +#endif + mtdcr(DCR_DMA2P40_SAL0 + ch_id*8, (u32)src_addr); + + return DMA_STATUS_GOOD; +} + +int ppc460ex_set_dst_addr(int ch_id, phys_addr_t dst_addr) +{ + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + +#ifdef PPC4xx_DMA_64BIT + mtdcr(DCR_DMA2P40_DAH0 + ch_id*8, dst_addr >> 32); +#endif + mtdcr(DCR_DMA2P40_DAL0 + ch_id*8, (u32)dst_addr); + + return DMA_STATUS_GOOD; +} + + + +/* + * Sets the dma mode for single DMA transfers only. + * For scatter/gather transfers, the mode is passed to the + * alloc_dma_handle() function as one of the parameters. + * + * The mode is simply saved and used later. This allows + * the driver to call set_dma_mode() and set_dma_addr() in + * any order. + * + * Valid mode values are: + * + * DMA_MODE_READ peripheral to memory + * DMA_MODE_WRITE memory to peripheral + * DMA_MODE_MM memory to memory + * DMA_MODE_MM_DEVATSRC device-paced memory to memory, device at src + * DMA_MODE_MM_DEVATDST device-paced memory to memory, device at dst + */ +int ppc460ex_set_dma_mode(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, unsigned int mode) +{ + + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk("%s: bad channel %d\n", __FUNCTION__, dma_chan->chan_id); + return DMA_STATUS_BAD_CHANNEL; + } + + dma_chan->mode = mode; + return DMA_STATUS_GOOD; +} + + + + +/* + * Sets the DMA Count register. Note that 'count' is in bytes. + * However, the DMA Count register counts the number of "transfers", + * where each transfer is equal to the bus width. Thus, count + * MUST be a multiple of the bus width. + */ +void ppc460ex_set_dma_count(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, unsigned int count) +{ + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + +//#ifdef DEBUG_4xxDMA + + { + int error = 0; + switch (dma_chan->pwidth) { + case PW_8: + break; + case PW_16: + if (count & 0x1) + error = 1; + break; + case PW_32: + if (count & 0x3) + error = 1; + break; + case PW_64: + if (count & 0x7) + error = 1; + break; + + case PW_128: + if (count & 0xf) + error = 1; + break; + default: + printk("set_dma_count: invalid bus width: 0x%x\n", + dma_chan->pwidth); + return; + } + if (error) + printk + ("Warning: set_dma_count count 0x%x bus width %d\n", + count, dma_chan->pwidth); + } +//#endif + count = count >> dma_chan->shift; + //count = 10; + mtdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8), count); + +} + + + + +/* + * Enables the channel interrupt. + * + * If performing a scatter/gatter transfer, this function + * MUST be called before calling alloc_dma_handle() and building + * the sgl list. Otherwise, interrupts will not be enabled, if + * they were previously disabled. + */ +int ppc460ex_enable_dma_interrupt(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + dma_chan->int_enable = 1; + + + control = mfdcr(DCR_DMA2P40_CR0); + control |= DMA_CIE_ENABLE; /* Channel Interrupt Enable */ + mtdcr(DCR_DMA2P40_CR0, control); + + + +#if 1 + control = mfdcr(DCR_DMA2P40_CTC0); + control |= DMA_CTC_TCIE | DMA_CTC_ETIE| DMA_CTC_EIE; + mtdcr(DCR_DMA2P40_CTC0, control); + +#endif + + + return DMA_STATUS_GOOD; + +} + + +/* + * Disables the channel interrupt. + * + * If performing a scatter/gatter transfer, this function + * MUST be called before calling alloc_dma_handle() and building + * the sgl list. Otherwise, interrupts will not be disabled, if + * they were previously enabled. + */ +int ppc460ex_disable_dma_interrupt(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + dma_chan->int_enable = 0; + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + control &= ~DMA_CIE_ENABLE; /* Channel Interrupt Enable */ + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + + return DMA_STATUS_GOOD; +} + + +/* + * This function returns the channel configuration. + */ +int ppc460ex_get_channel_config(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, + ppc460ex_plb_dma_ch_t *p_dma_ch) +{ + unsigned int polarity; + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + memcpy(p_dma_ch, dma_chan, sizeof(ppc460ex_plb_dma_ch_t)); + + polarity = mfdcr(DCR_DMA2P40_POL); + + p_dma_ch->polarity = polarity & GET_DMA_POLARITY(ch_id); + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + + p_dma_ch->cp = GET_DMA_PRIORITY(control); + p_dma_ch->pwidth = GET_DMA_PW(control); + p_dma_ch->psc = GET_DMA_PSC(control); + p_dma_ch->pwc = GET_DMA_PWC(control); + p_dma_ch->phc = GET_DMA_PHC(control); + p_dma_ch->ce = GET_DMA_CE_ENABLE(control); + p_dma_ch->int_enable = GET_DMA_CIE_ENABLE(control); + p_dma_ch->shift = GET_DMA_PW(control); + p_dma_ch->pf = GET_DMA_PREFETCH(control); + + return DMA_STATUS_GOOD; + +} + +/* + * Sets the priority for the DMA channel dmanr. + * Since this is setup by the hardware init function, this function + * can be used to dynamically change the priority of a channel. + * + * Acceptable priorities: + * + * PRIORITY_LOW + * PRIORITY_MID_LOW + * PRIORITY_MID_HIGH + * PRIORITY_HIGH + * + */ +int ppc460ex_set_channel_priority(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, + unsigned int priority) +{ + unsigned int control; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + if ((priority != PRIORITY_LOW) && + (priority != PRIORITY_MID_LOW) && + (priority != PRIORITY_MID_HIGH) && (priority != PRIORITY_HIGH)) { + printk("%s:bad priority: 0x%x\n", __FUNCTION__, priority); + } + + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + control |= SET_DMA_PRIORITY(priority); + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + + return DMA_STATUS_GOOD; +} + +/* + * Returns the width of the peripheral attached to this channel. This assumes + * that someone who knows the hardware configuration, boot code or some other + * init code, already set the width. + * + * The return value is one of: + * PW_8 + * PW_16 + * PW_32 + * PW_64 + * + * The function returns 0 on error. + */ +unsigned int ppc460ex_get_peripheral_width(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + return (GET_DMA_PW(control)); +} + +/* + * Enables the burst on the channel (BTEN bit in the control/count register) + * Note: + * For scatter/gather dma, this function MUST be called before the + * ppc4xx_alloc_dma_handle() func as the chan count register is copied into the + * sgl list and used as each sgl element is added. + */ +int ppc460ex_enable_burst(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int ctc; + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + ctc = mfdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8)) | DMA_CTC_BTEN; + mtdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8), ctc); + return DMA_STATUS_GOOD; +} + + +/* + * Disables the burst on the channel (BTEN bit in the control/count register) + * Note: + * For scatter/gather dma, this function MUST be called before the + * ppc4xx_alloc_dma_handle() func as the chan count register is copied into the + * sgl list and used as each sgl element is added. + */ +int ppc460ex_disable_burst(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int ctc; + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + ctc = mfdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8)) &~ DMA_CTC_BTEN; + mtdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8), ctc); + return DMA_STATUS_GOOD; +} + + +/* + * Sets the burst size (number of peripheral widths) for the channel + * (BSIZ bits in the control/count register)) + * must be one of: + * DMA_CTC_BSIZ_2 + * DMA_CTC_BSIZ_4 + * DMA_CTC_BSIZ_8 + * DMA_CTC_BSIZ_16 + * Note: + * For scatter/gather dma, this function MUST be called before the + * ppc4xx_alloc_dma_handle() func as the chan count register is copied into the + * sgl list and used as each sgl element is added. + */ +int ppc460ex_set_burst_size(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, + unsigned int bsize) +{ + unsigned int ctc; + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + ctc = mfdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8)) &~ DMA_CTC_BSIZ_MSK; + ctc |= (bsize & DMA_CTC_BSIZ_MSK); + mtdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8), ctc); + return DMA_STATUS_GOOD; +} + +/* + * Returns the number of bytes left to be transferred. + * After a DMA transfer, this should return zero. + * Reading this while a DMA transfer is still in progress will return + * unpredictable results. + */ +int ppc460ex_get_dma_residue(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int count; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + count = mfdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8)); + count &= DMA_CTC_TC_MASK ; + + return (count << dma_chan->shift); + +} + + +/* + * Configures a DMA channel, including the peripheral bus width, if a + * peripheral is attached to the channel, the polarity of the DMAReq and + * DMAAck signals, etc. This information should really be setup by the boot + * code, since most likely the configuration won't change dynamically. + * If the kernel has to call this function, it's recommended that it's + * called from platform specific init code. The driver should not need to + * call this function. + */ +int ppc460ex_init_dma_channel(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, + ppc460ex_plb_dma_ch_t *p_init) +{ + unsigned int polarity; + uint32_t control = 0; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + + DMA_MODE_READ = (unsigned long) DMA_TD; /* Peripheral to Memory */ + DMA_MODE_WRITE = 0; /* Memory to Peripheral */ + + if (!p_init) { + printk("%s: NULL p_init\n", __FUNCTION__); + return DMA_STATUS_NULL_POINTER; + } + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } +#if DCR_DMA2P40_POL > 0 + polarity = mfdcr(DCR_DMA2P40_POL); +#else + polarity = 0; +#endif + + p_init->int_enable = 0; + p_init->buffer_enable = 1; + p_init->etd_output = 1; + p_init->tce_enable = 1; + p_init->pl = 0; + p_init->dai = 1; + p_init->sai = 1; + /* Duc Dang: make channel priority to 2, original is 3 */ + p_init->cp = 2; + p_init->pwidth = PW_8; + p_init->psc = 0; + p_init->pwc = 0; + p_init->phc = 0; + p_init->pf = 1; + + + /* Setup the control register based on the values passed to + * us in p_init. Then, over-write the control register with this + * new value. + */ +#if 0 + control |= SET_DMA_CONTROL; +#endif + control = SET_DMA_CONTROL; + /* clear all polarity signals and then "or" in new signal levels */ + +//PMB - Workaround + //control = 0x81A2CD80; + //control = 0x81A00180; + + + polarity &= ~GET_DMA_POLARITY(ch_id); + polarity |= p_init->polarity; + +#if DCR_DMA2P40_POL > 0 + mtdcr(DCR_DMA2P40_POL, polarity); +#endif + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + + /* save these values in our dma channel structure */ + //memcpy(dma_chan, p_init, sizeof(ppc460ex_plb_dma_ch_t)); + /* + * The peripheral width values written in the control register are: + * PW_8 0 + * PW_16 1 + * PW_32 2 + * PW_64 3 + * PW_128 4 + * + * Since the DMA count register takes the number of "transfers", + * we need to divide the count sent to us in certain + * functions by the appropriate number. It so happens that our + * right shift value is equal to the peripheral width value. + */ + dma_chan->shift = p_init->pwidth; + dma_chan->sai = p_init->sai; + dma_chan->dai = p_init->dai; + dma_chan->tce_enable = p_init->tce_enable; + dma_chan->mode = DMA_MODE_MM; + /* + * Save the control word for easy access. + */ + dma_chan->control = control; + mtdcr(DCR_DMA2P40_SR, 0xffffffff); + + + return DMA_STATUS_GOOD; +} + + +int ppc460ex_enable_dma(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + unsigned int status_bits[] = { DMA_CS0 | DMA_TS0 | DMA_CH0_ERR, + DMA_CS1 | DMA_TS1 | DMA_CH1_ERR}; + + if (dma_chan->in_use) { + printk("%s:enable_dma: channel %d in use\n", __FUNCTION__, ch_id); + return DMA_STATUS_CHANNEL_NOTFREE; + } + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + +#if 0 + if (dma_chan->mode == DMA_MODE_READ) { + /* peripheral to memory */ + ppc460ex_set_src_addr(ch_id, 0); + ppc460ex_set_dst_addr(ch_id, dma_chan->addr); + } else if (dma_chan->mode == DMA_MODE_WRITE) { + /* memory to peripheral */ + ppc460ex_set_src_addr(ch_id, dma_chan->addr); + ppc460ex_set_dst_addr(ch_id, 0); + } +#endif + /* for other xfer modes, the addresses are already set */ + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + control &= ~(DMA_TM_MASK | DMA_TD); /* clear all mode bits */ + if (dma_chan->mode == DMA_MODE_MM) { + /* software initiated memory to memory */ + control |= DMA_ETD_OUTPUT | DMA_TCE_ENABLE; + control |= DMA_MODE_MM; + if (dma_chan->dai) { + control |= DMA_DAI; + } + if (dma_chan->sai) { + control |= DMA_SAI; + } + } + + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + /* + * Clear the CS, TS, RI bits for the channel from DMASR. This + * has been observed to happen correctly only after the mode and + * ETD/DCE bits in DMACRx are set above. Must do this before + * enabling the channel. + */ + mtdcr(DCR_DMA2P40_SR, status_bits[ch_id]); + /* + * For device-paced transfers, Terminal Count Enable apparently + * must be on, and this must be turned on after the mode, etc. + * bits are cleared above (at least on Redwood-6). + */ + + if ((dma_chan->mode == DMA_MODE_MM_DEVATDST) || + (dma_chan->mode == DMA_MODE_MM_DEVATSRC)) + control |= DMA_TCE_ENABLE; + + /* + * Now enable the channel. + */ + + control |= (dma_chan->mode | DMA_CE_ENABLE); + control |= DMA_BEN; + //control = 0xc4effec0; + + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + dma_chan->in_use = 1; + return 0; + +} + + +void +ppc460ex_disable_dma(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (!dma_chan->in_use) { + printk("disable_dma: channel %d not in use\n", ch_id); + return; + } + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk("disable_dma: bad channel: %d\n", ch_id); + return; + } + + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + control &= ~DMA_CE_ENABLE; + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + + dma_chan->in_use = 0; +} + + + + +/* + * Clears the channel status bits + */ +int ppc460ex_clear_dma_status(unsigned int ch_id) +{ + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk("KERN_ERR %s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + mtdcr(DCR_DMA2P40_SR, ((u32)DMA_CH0_ERR | (u32)DMA_CS0 | (u32)DMA_TS0) >> ch_id); + return DMA_STATUS_GOOD; + +} + + +/** + * ppc460ex_dma_eot_handler - end of transfer interrupt handler + */ +irqreturn_t ppc460ex_4chan_dma_eot_handler(int irq, void *data) +{ + unsigned int data_read = 0; + unsigned int try_cnt = 0; + + //printk("transfer complete\n"); + data_read = mfdcr(DCR_DMA2P40_SR); + //printk("%s: status 0x%08x\n", __FUNCTION__, data_read); + + do{ + //while bit 3 TC done is 0 + data_read = mfdcr(DCR_DMA2P40_SR); + if (data_read & 0x00800000 ) {printk("test FAIL\n"); } //see if error bit is set + }while(((data_read & 0x80000000) != 0x80000000) && ++try_cnt <= 10);// TC is now 0 + + data_read = mfdcr(DCR_DMA2P40_SR); + while (data_read & 0x00000800){ //while channel is busy + data_read = mfdcr(DCR_DMA2P40_SR); + printk("%s: status for busy 0x%08x\n", __FUNCTION__, data_read); + } + mtdcr(DCR_DMA2P40_SR, 0xffffffff); + + + + return IRQ_HANDLED; +} + + + +static struct of_device_id dma_per_chan_match[] = { + { + .compatible = "amcc,dma-4channel", + }, + {}, +}; + + + + +#if 0 +/*** test code ***/ +static int ppc460ex_dma_memcpy_self_test(ppc460ex_plb_dma_dev_t *device, unsigned int dma_ch_id) +{ + ppc460ex_plb_dma_ch_t p_init; + int res = 0, i; + unsigned int control; + phys_addr_t *src; + phys_addr_t *dest; + + phys_addr_t *gap; + + phys_addr_t dma_dest, dma_src; + + src = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!src) + return -ENOMEM; + gap = kzalloc(200, GFP_KERNEL); + if (!gap) + return -ENOMEM; + + + + dest = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!dest) { + kfree(src); + return -ENOMEM; + } + + printk("src = 0x%08x\n", (unsigned int)src); + printk("gap = 0x%08x\n", (unsigned int)gap); + printk("dest = 0x%08x\n", (unsigned int)dest); + + /* Fill in src buffer */ + for (i = 0; i < TEST_SIZE; i++) + ((u8*)src)[i] = (u8)i; + + printk("dump src\n"); + DMA_HEXDUMP(src, TEST_SIZE); + DMA_HEXDUMP(dest, TEST_SIZE); +#if 1 + dma_src = dma_map_single(p_init.device->dev, src, TEST_SIZE, + DMA_TO_DEVICE); + dma_dest = dma_map_single(p_init.device->dev, dest, TEST_SIZE, + DMA_FROM_DEVICE); +#endif + printk("%s:channel = %d chan 0x%08x\n", __FUNCTION__, device->chan[dma_ch_id]->chan_id, + (unsigned int)(device->chan)); + + p_init.polarity = 0; + p_init.pwidth = PW_32; + p_init.in_use = 0; + p_init.sai = 1; + p_init.dai = 1; + res = ppc460ex_init_dma_channel(device, dma_ch_id, &p_init); + + if (res) { + printk("%32s: init_dma_channel return %d\n", + __FUNCTION__, res); + } + ppc460ex_clear_dma_status(dma_ch_id); + + ppc460ex_set_src_addr(dma_ch_id, dma_src); + ppc460ex_set_dst_addr(dma_ch_id, dma_dest); + + ppc460ex_set_dma_mode(device, dma_ch_id, DMA_MODE_MM); + ppc460ex_set_dma_count(device, dma_ch_id, TEST_SIZE); + + res = ppc460ex_enable_dma_interrupt(device, dma_ch_id); + if (res) { + printk("%32s: en/disable_dma_interrupt\n", + __FUNCTION__); + } + + + if (dma_ch_id == 0) + control = mfdcr(DCR_DMA2P40_CR0); + else if (dma_ch_id == 1) + control = mfdcr(DCR_DMA2P40_CR1); + + + control &= ~(SET_DMA_BEN(1)); + control &= ~(SET_DMA_PSC(3)); + control &= ~(SET_DMA_PWC(0x3f)); + control &= ~(SET_DMA_PHC(0x7)); + control &= ~(SET_DMA_PL(1)); + + + + if (dma_ch_id == 0) + mtdcr(DCR_DMA2P40_CR0, control); + else if (dma_ch_id == 1) + mtdcr(DCR_DMA2P40_CR1, control); + + + ppc460ex_enable_dma(device, dma_ch_id); + + + if (memcmp(src, dest, TEST_SIZE)) { + printk("Self-test copy failed compare, disabling\n"); + res = -ENODEV; + goto out; + } + + + return 0; + + out: kfree(src); + kfree(dest); + return res; + +} + + + +static int test1(void) +{ + void *src, *dest; + void *src1, *dest1; + int i; + unsigned int chan; + + src = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!src) + return -ENOMEM; + + dest = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!dest) { + kfree(src); + return -ENOMEM; + } + + src1 = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!src1) + return -ENOMEM; + + dest1 = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!dest1) { + kfree(src1); + return -ENOMEM; + } + + /* Fill in src buffer */ + for (i = 0; i < TEST_SIZE; i++) + ((u8*)src)[i] = (u8)i; + + /* Fill in src buffer */ + for (i = 0; i < TEST_SIZE; i++) + ((u8*)src1)[i] = (u8)0xaa; + +#ifdef DEBUG_TEST + DMA_HEXDUMP(src, TEST_SIZE); + DMA_HEXDUMP(dest, TEST_SIZE); + DMA_HEXDUMP(src1, TEST_SIZE); + DMA_HEXDUMP(dest1, TEST_SIZE); +#endif + chan = ppc460ex_get_dma_channel(); + +#ifdef ENABLE_SGL + test_sgdma_memcpy(src, dest, src1, dest1, TEST_SIZE, chan); +#endif + test_dma_memcpy(src, dest, TEST_SIZE, chan); + + + out: kfree(src); + kfree(dest); + kfree(src1); + kfree(dest1); + + return 0; + +} +#endif + + + +/******************************************************************************* + * Module Initialization Routine + ******************************************************************************* + */ +int ppc460ex_dma_per_chan_probe(struct platform_device *ofdev) +{ + int ret=0; + //ppc460ex_plb_dma_dev_t *adev; + ppc460ex_plb_dma_ch_t *new_chan; + int err; + + + + adev = dev_get_drvdata(ofdev->dev.parent); + BUG_ON(!adev); + /* create a device */ + if ((new_chan = kzalloc(sizeof(*new_chan), GFP_KERNEL)) == NULL) { + printk("ERROR:No Free memory for allocating dma channels\n"); + ret = -ENOMEM; + goto err; + } + + err = of_address_to_resource(ofdev->dev.of_node,0,&new_chan->reg); + if (err) { + printk("ERROR:Can't get %s property reg\n", __FUNCTION__); + goto err; + } + new_chan->device = adev; + new_chan->reg_base = ioremap(new_chan->reg.start,new_chan->reg.end - new_chan->reg.start + 1); +#if 1 + printk("PPC460ex PLB DMA engine @0x%02X_%08X size %d\n", + (u32)(new_chan->reg.start >> 32), + (u32)new_chan->reg.start, + (u32)(new_chan->reg.end - new_chan->reg.start + 1)); +#endif + + switch(new_chan->reg.start) { + case 0x100: + new_chan->chan_id = 0; + break; + case 0x108: + new_chan->chan_id = 1; + break; + case 0x110: + new_chan->chan_id = 2; + break; + case 0x118: + new_chan->chan_id = 3; + break; + } + new_chan->chan_id = ((new_chan->reg.start - 0x100)& 0xfff) >> 3; + printk("new_chan->chan_id 0x%x\n",new_chan->chan_id); + adev->chan[new_chan->chan_id] = new_chan; + printk("new_chan->chan->chan_id 0x%x\n",adev->chan[new_chan->chan_id]->chan_id); + //adev->chan[new_chan->chan_id]->reg_base = new_chan->reg_base; + + return 0; + + err: + return ret; + +} + +int ppc460ex_dma_4chan_probe(struct platform_device *ofdev) +{ + int ret=0, irq = 0; + //ppc460ex_plb_dma_dev_t *adev; + ppc460ex_plb_dma_ch_t *chan = NULL; + struct device_node *np = ofdev->dev.of_node; + + /* create a device */ + if ((adev = kzalloc(sizeof(*adev), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_adev_alloc; + } + adev->dev = &ofdev->dev; +#if !defined(CONFIG_APM821xx) + err = of_address_to_resource(np,0,&adev->reg); + if(err) { + printk(KERN_ERR"Can't get %s property 'reg'\n",ofdev->node->full_name); + } +#endif + printk(KERN_INFO"Probing AMCC DMA driver\n"); +#if !defined(CONFIG_APM821xx) + adev->reg_base = ioremap(adev->reg.start, adev->reg.end - adev->reg.start + 1); +#endif + +#if 1 + irq = of_irq_to_resource(np, 0, NULL); + if (irq >= 0) { + ret = request_irq(irq, ppc460ex_4chan_dma_eot_handler, + IRQF_DISABLED, "Peripheral DMA0-1", chan); + if (ret) { + ret = -EIO; + goto err_irq; + } + //irq = platform_get_irq(adev, 0); + /* only DMA engines have a separate err IRQ + * so it's Ok if irq < 0 in XOR case + */ + } else + ret = -ENXIO; + +#if !defined(CONFIG_APM821xx) + printk("PPC4xx PLB DMA engine @0x%02X_%08X size %d IRQ %d \n", + (u32)(adev->reg.start >> 32), + (u32)adev->reg.start, + (u32)(adev->reg.end - adev->reg.start + 1), + irq); +#else + printk("PPC4xx PLB DMA engine IRQ %d\n", irq); +#endif +#endif + dev_set_drvdata(&(ofdev->dev),adev); + of_platform_bus_probe(np,dma_per_chan_match,&ofdev->dev); + + + //ppc460ex_dma_memcpy_self_test(adev, 0); + //test1(); + + + return 0; + + +err_adev_alloc: + //release_mem_region(adev->reg.start, adev->reg.end - adev->reg.start); +err_irq: + kfree(chan); + + return ret; +} + + +static struct of_device_id dma_4chan_match[] = { + { + .compatible = "amcc,dma", + }, + {}, +}; + +struct platform_driver ppc460ex_dma_4chan_driver = { + .driver = { + .name = "plb_dma", + .owner = THIS_MODULE, + .of_match_table = dma_4chan_match, + }, + .probe = ppc460ex_dma_4chan_probe, +}; + +struct platform_driver ppc460ex_dma_per_chan_driver = { + .driver = { + .name = "dma-4channel", + .owner = THIS_MODULE, + .of_match_table = dma_per_chan_match, + }, + .probe = ppc460ex_dma_per_chan_probe, +}; + + +static int __init mod_init (void) +{ + printk("%s:%d\n", __FUNCTION__, __LINE__); + return platform_driver_register(&ppc460ex_dma_4chan_driver); + printk("here 2\n"); +} + +static void __exit mod_exit(void) +{ + platform_driver_unregister(&ppc460ex_dma_4chan_driver); +} + +static int __init ppc460ex_dma_per_chan_init (void) +{ + printk("%s:%d\n", __FUNCTION__, __LINE__); + return platform_driver_register(&ppc460ex_dma_per_chan_driver); + printk("here 3\n"); +} + +static void __exit ppc460ex_dma_per_chan_exit(void) +{ + platform_driver_unregister(&ppc460ex_dma_per_chan_driver); +} + +subsys_initcall(ppc460ex_dma_per_chan_init); +subsys_initcall(mod_init); + +//module_exit(mod_exit); + +//module_exit(ppc460ex_dma_per_chan_exit); + +MODULE_DESCRIPTION("AMCC PPC460EX 4 channel Engine Driver"); +MODULE_LICENSE("GPL"); + +EXPORT_SYMBOL_GPL(ppc460ex_get_dma_status); +EXPORT_SYMBOL_GPL(ppc460ex_set_src_addr); +EXPORT_SYMBOL_GPL(ppc460ex_set_dst_addr); +EXPORT_SYMBOL_GPL(ppc460ex_set_dma_mode); +EXPORT_SYMBOL_GPL(ppc460ex_set_dma_count); +EXPORT_SYMBOL_GPL(ppc460ex_enable_dma_interrupt); +EXPORT_SYMBOL_GPL(ppc460ex_init_dma_channel); +EXPORT_SYMBOL_GPL(ppc460ex_enable_dma); +EXPORT_SYMBOL_GPL(ppc460ex_disable_dma); +EXPORT_SYMBOL_GPL(ppc460ex_clear_dma_status); +EXPORT_SYMBOL_GPL(ppc460ex_get_dma_residue); +EXPORT_SYMBOL_GPL(ppc460ex_disable_dma_interrupt); +EXPORT_SYMBOL_GPL(ppc460ex_get_channel_config); +EXPORT_SYMBOL_GPL(ppc460ex_set_channel_priority); +EXPORT_SYMBOL_GPL(ppc460ex_get_peripheral_width); +EXPORT_SYMBOL_GPL(ppc460ex_enable_burst); +EXPORT_SYMBOL_GPL(ppc460ex_disable_burst); +EXPORT_SYMBOL_GPL(ppc460ex_set_burst_size); + +/************************************************************************/ diff --git a/drivers/dma/ppc4xx/ppc460ex_4chan_dma.h b/drivers/dma/ppc4xx/ppc460ex_4chan_dma.h new file mode 100644 index 00000000000..c9448f34de4 --- /dev/null +++ b/drivers/dma/ppc4xx/ppc460ex_4chan_dma.h @@ -0,0 +1,531 @@ + + +#include <linux/types.h> + + + + +#define DMA_HEXDUMP(b, l) \ + print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, \ + 16, 1, (b), (l), false); + + +#define MAX_PPC460EX_DMA_CHANNELS 4 + + +#define DCR_DMA0_BASE 0x200 +#define DCR_DMA1_BASE 0x208 +#define DCR_DMA2_BASE 0x210 +#define DCR_DMA3_BASE 0x218 +#define DCR_DMASR_BASE 0x220 + + + + + + +/* DMA Registers */ +#define DCR_DMA2P40_CR0 (DCR_DMA0_BASE + 0x0) /* DMA Channel Control 0 */ +#define DCR_DMA2P40_CTC0 (DCR_DMA0_BASE + 0x1) /* DMA Count 0 */ +#define DCR_DMA2P40_SAH0 (DCR_DMA0_BASE + 0x2) /* DMA Src Addr High 0 */ +#define DCR_DMA2P40_SAL0 (DCR_DMA0_BASE + 0x3) /* DMA Src Addr Low 0 */ +#define DCR_DMA2P40_DAH0 (DCR_DMA0_BASE + 0x4) /* DMA Dest Addr High 0 */ +#define DCR_DMA2P40_DAL0 (DCR_DMA0_BASE + 0x5) /* DMA Dest Addr Low 0 */ +#define DCR_DMA2P40_SGH0 (DCR_DMA0_BASE + 0x6) /* DMA SG Desc Addr High 0 */ +#define DCR_DMA2P40_SGL0 (DCR_DMA0_BASE + 0x7) /* DMA SG Desc Addr Low 0 */ + +#define DCR_DMA2P40_CR1 (DCR_DMA1_BASE + 0x0) /* DMA Channel Control 1 */ +#define DCR_DMA2P40_CTC1 (DCR_DMA1_BASE + 0x1) /* DMA Count 1 */ +#define DCR_DMA2P40_SAH1 (DCR_DMA1_BASE + 0x2) /* DMA Src Addr High 1 */ +#define DCR_DMA2P40_SAL1 (DCR_DMA1_BASE + 0x3) /* DMA Src Addr Low 1 */ +#define DCR_DMA2P40_DAH1 (DCR_DMA1_BASE + 0x4) /* DMA Dest Addr High 1 */ +#define DCR_DMA2P40_DAL1 (DCR_DMA1_BASE + 0x5) /* DMA Dest Addr Low 1 */ +#define DCR_DMA2P40_SGH1 (DCR_DMA1_BASE + 0x6) /* DMA SG Desc Addr High 1 */ +#define DCR_DMA2P40_SGL1 (DCR_DMA1_BASE + 0x7) /* DMA SG Desc Addr Low 1 */ + +#define DCR_DMA2P40_CR2 (DCR_DMA2_BASE + 0x0) /* DMA Channel Control 2 */ +#define DCR_DMA2P40_CTC2 (DCR_DMA2_BASE + 0x1) /* DMA Count 2 */ +#define DCR_DMA2P40_SAH2 (DCR_DMA2_BASE + 0x2) /* DMA Src Addr High 2 */ +#define DCR_DMA2P40_SAL2 (DCR_DMA2_BASE + 0x3) /* DMA Src Addr Low 2 */ +#define DCR_DMA2P40_DAH2 (DCR_DMA2_BASE + 0x4) /* DMA Dest Addr High 2 */ +#define DCR_DMA2P40_DAL2 (DCR_DMA2_BASE + 0x5) /* DMA Dest Addr Low 2 */ +#define DCR_DMA2P40_SGH2 (DCR_DMA2_BASE + 0x6) /* DMA SG Desc Addr High 2 */ +#define DCR_DMA2P40_SGL2 (DCR_DMA2_BASE + 0x7) /* DMA SG Desc Addr Low 2 */ + +#define DCR_DMA2P40_CR3 (DCR_DMA3_BASE + 0x0) /* DMA Channel Control 3 */ +#define DCR_DMA2P40_CTC3 (DCR_DMA3_BASE + 0x1) /* DMA Count 3 */ +#define DCR_DMA2P40_SAH3 (DCR_DMA3_BASE + 0x2) /* DMA Src Addr High 3 */ +#define DCR_DMA2P40_SAL3 (DCR_DMA3_BASE + 0x3) /* DMA Src Addr Low 3 */ +#define DCR_DMA2P40_DAH3 (DCR_DMA3_BASE + 0x4) /* DMA Dest Addr High 3 */ +#define DCR_DMA2P40_DAL3 (DCR_DMA3_BASE + 0x5) /* DMA Dest Addr Low 3 */ +#define DCR_DMA2P40_SGH3 (DCR_DMA3_BASE + 0x6) /* DMA SG Desc Addr High 3 */ +#define DCR_DMA2P40_SGL3 (DCR_DMA3_BASE + 0x7) /* DMA SG Desc Addr Low 3 */ + +#define DCR_DMA2P40_SR (DCR_DMASR_BASE + 0x0) /* DMA Status Register */ +#define DCR_DMA2P40_SGC (DCR_DMASR_BASE + 0x3) /* DMA Scatter/Gather Command */ +#define DCR_DMA2P40_SLP (DCR_DMASR_BASE + 0x5) /* DMA Sleep Register */ +#define DCR_DMA2P40_POL (DCR_DMASR_BASE + 0x6) /* DMA Polarity Register */ + + + +/* + * Function return status codes + * These values are used to indicate whether or not the function + * call was successful, or a bad/invalid parameter was passed. + */ +#define DMA_STATUS_GOOD 0 +#define DMA_STATUS_BAD_CHANNEL 1 +#define DMA_STATUS_BAD_HANDLE 2 +#define DMA_STATUS_BAD_MODE 3 +#define DMA_STATUS_NULL_POINTER 4 +#define DMA_STATUS_OUT_OF_MEMORY 5 +#define DMA_STATUS_SGL_LIST_EMPTY 6 +#define DMA_STATUS_GENERAL_ERROR 7 +#define DMA_STATUS_CHANNEL_NOTFREE 8 + +#define DMA_CHANNEL_BUSY 0x80000000 + +/* + * These indicate status as returned from the DMA Status Register. + */ +#define DMA_STATUS_NO_ERROR 0 +#define DMA_STATUS_CS 1 /* Count Status */ +#define DMA_STATUS_TS 2 /* Transfer Status */ +#define DMA_STATUS_DMA_ERROR 3 /* DMA Error Occurred */ +#define DMA_STATUS_DMA_BUSY 4 /* The channel is busy */ + +/* + * DMA Channel Control Registers + */ +#ifdef CONFIG_44x +#define PPC4xx_DMA_64BIT +#define DMA_CR_OFFSET 1 +#else +#define DMA_CR_OFFSET 0 +#endif + +#define DMA_CE_ENABLE (1<<31) /* DMA Channel Enable */ +#define SET_DMA_CE_ENABLE(x) (((x)&0x1)<<31) +#define GET_DMA_CE_ENABLE(x) (((x)&DMA_CE_ENABLE)>>31) + +#define DMA_CIE_ENABLE (1<<30) /* DMA Channel Interrupt Enable */ +#define SET_DMA_CIE_ENABLE(x) (((x)&0x1)<<30) +#define GET_DMA_CIE_ENABLE(x) (((x)&DMA_CIE_ENABLE)>>30) + +#define DMA_TD (1<<29) +#define SET_DMA_TD(x) (((x)&0x1)<<29) +#define GET_DMA_TD(x) (((x)&DMA_TD)>>29) + +#define DMA_PL (1<<28) /* Peripheral Location */ +#define SET_DMA_PL(x) (((x)&0x1)<<28) +#define GET_DMA_PL(x) (((x)&DMA_PL)>>28) + +#define EXTERNAL_PERIPHERAL 0 +#define INTERNAL_PERIPHERAL 1 + +#define SET_DMA_PW(x) (((x)&0x7)<<(26-DMA_CR_OFFSET)) /* Peripheral Width */ +#define DMA_PW_MASK SET_DMA_PW(7) +#define PW_8 0 +#define PW_16 1 +#define PW_32 2 +#define PW_64 3 +#define PW_128 4 + + +#define GET_DMA_PW(x) (((x)&DMA_PW_MASK)>>(26-DMA_CR_OFFSET)) + +#define DMA_DAI (1<<(25-DMA_CR_OFFSET)) /* Destination Address Increment */ +#define SET_DMA_DAI(x) (((x)&0x1)<<(25-DMA_CR_OFFSET)) + +#define DMA_SAI (1<<(24-DMA_CR_OFFSET)) /* Source Address Increment */ +#define SET_DMA_SAI(x) (((x)&0x1)<<(24-DMA_CR_OFFSET)) + +#define DMA_BEN (1<<(23-DMA_CR_OFFSET)) /* Buffer Enable */ +#define SET_DMA_BEN(x) (((x)&0x1)<<(23-DMA_CR_OFFSET)) + +#define SET_DMA_TM(x) (((x)&0x3)<<(21-DMA_CR_OFFSET)) /* Transfer Mode */ +#define DMA_TM_MASK SET_DMA_TM(3) +#define TM_PERIPHERAL 0 /* Peripheral */ +#define TM_RESERVED 1 /* Reserved */ +#define TM_S_MM 2 /* Memory to Memory */ +#define TM_D_MM 3 /* Device Paced Memory to Memory */ +#define GET_DMA_TM(x) (((x)&DMA_TM_MASK)>>(21-DMA_CR_OFFSET)) + +#define SET_DMA_PSC(x) (((x)&0x3)<<(19-DMA_CR_OFFSET)) /* Peripheral Setup Cycles */ +#define DMA_PSC_MASK SET_DMA_PSC(3) +#define GET_DMA_PSC(x) (((x)&DMA_PSC_MASK)>>(19-DMA_CR_OFFSET)) + +#define SET_DMA_PWC(x) (((x)&0x3F)<<(13-DMA_CR_OFFSET)) /* Peripheral Wait Cycles */ +#define DMA_PWC_MASK SET_DMA_PWC(0x3F) +#define GET_DMA_PWC(x) (((x)&DMA_PWC_MASK)>>(13-DMA_CR_OFFSET)) + +#define SET_DMA_PHC(x) (((x)&0x7)<<(10-DMA_CR_OFFSET)) /* Peripheral Hold Cycles */ +#define DMA_PHC_MASK SET_DMA_PHC(0x7) +#define GET_DMA_PHC(x) (((x)&DMA_PHC_MASK)>>(10-DMA_CR_OFFSET)) + +#define DMA_ETD_OUTPUT (1<<(9-DMA_CR_OFFSET)) /* EOT pin is a TC output */ +#define SET_DMA_ETD(x) (((x)&0x1)<<(9-DMA_CR_OFFSET)) + +#define DMA_TCE_ENABLE (1<<(8-DMA_CR_OFFSET)) +#define SET_DMA_TCE(x) (((x)&0x1)<<(8-DMA_CR_OFFSET)) + +#define DMA_DEC (1<<(2)) /* Address Decrement */ +#define SET_DMA_DEC(x) (((x)&0x1)<<2) +#define GET_DMA_DEC(x) (((x)&DMA_DEC)>>2) + + +/* + * Transfer Modes + * These modes are defined in a way that makes it possible to + * simply "or" in the value in the control register. + */ + +#define DMA_MODE_MM (SET_DMA_TM(TM_S_MM)) /* memory to memory */ + + /* Device-paced memory to memory, */ + /* device is at source address */ +#define DMA_MODE_MM_DEVATSRC (DMA_TD | SET_DMA_TM(TM_D_MM)) + + /* Device-paced memory to memory, */ + /* device is at destination address */ +#define DMA_MODE_MM_DEVATDST (SET_DMA_TM(TM_D_MM)) + +#define SGL_LIST_SIZE 16384 +#define DMA_PPC4xx_SIZE SGL_LIST_SIZE + +#define SET_DMA_PRIORITY(x) (((x)&0x3)<<(6-DMA_CR_OFFSET)) /* DMA Channel Priority */ +#define DMA_PRIORITY_MASK SET_DMA_PRIORITY(3) +#define PRIORITY_LOW 0 +#define PRIORITY_MID_LOW 1 +#define PRIORITY_MID_HIGH 2 +#define PRIORITY_HIGH 3 +#define GET_DMA_PRIORITY(x) (((x)&DMA_PRIORITY_MASK)>>(6-DMA_CR_OFFSET)) + + +#define SET_DMA_PREFETCH(x) (((x)&0x3)<<(4-DMA_CR_OFFSET)) /* Memory Read Prefetch */ +#define DMA_PREFETCH_MASK SET_DMA_PREFETCH(3) +#define PREFETCH_1 0 /* Prefetch 1 Double Word */ +#define PREFETCH_2 1 +#define PREFETCH_4 2 +#define GET_DMA_PREFETCH(x) (((x)&DMA_PREFETCH_MASK)>>(4-DMA_CR_OFFSET)) + +#define DMA_PCE (1<<(3-DMA_CR_OFFSET)) /* Parity Check Enable */ +#define SET_DMA_PCE(x) (((x)&0x1)<<(3-DMA_CR_OFFSET)) +#define GET_DMA_PCE(x) (((x)&DMA_PCE)>>(3-DMA_CR_OFFSET)) + +/* + * DMA Polarity Configuration Register + */ +#define DMAReq_ActiveLow(chan) (1<<(31-(chan*3))) +#define DMAAck_ActiveLow(chan) (1<<(30-(chan*3))) +#define EOT_ActiveLow(chan) (1<<(29-(chan*3))) /* End of Transfer */ + +/* + * DMA Sleep Mode Register + */ +#define SLEEP_MODE_ENABLE (1<<21) + +/* + * DMA Status Register + */ +#define DMA_CS0 (1<<31) /* Terminal Count has been reached */ +#define DMA_CS1 (1<<30) +#define DMA_CS2 (1<<29) +#define DMA_CS3 (1<<28) + +#define DMA_TS0 (1<<27) /* End of Transfer has been requested */ +#define DMA_TS1 (1<<26) +#define DMA_TS2 (1<<25) +#define DMA_TS3 (1<<24) + +#define DMA_CH0_ERR (1<<23) /* DMA Chanel 0 Error */ +#define DMA_CH1_ERR (1<<22) +#define DMA_CH2_ERR (1<<21) +#define DMA_CH3_ERR (1<<20) + +#define DMA_IN_DMA_REQ0 (1<<19) /* Internal DMA Request is pending */ +#define DMA_IN_DMA_REQ1 (1<<18) +#define DMA_IN_DMA_REQ2 (1<<17) +#define DMA_IN_DMA_REQ3 (1<<16) + +#define DMA_EXT_DMA_REQ0 (1<<15) /* External DMA Request is pending */ +#define DMA_EXT_DMA_REQ1 (1<<14) +#define DMA_EXT_DMA_REQ2 (1<<13) +#define DMA_EXT_DMA_REQ3 (1<<12) + +#define DMA_CH0_BUSY (1<<11) /* DMA Channel 0 Busy */ +#define DMA_CH1_BUSY (1<<10) +#define DMA_CH2_BUSY (1<<9) +#define DMA_CH3_BUSY (1<<8) + +#define DMA_SG0 (1<<7) /* DMA Channel 0 Scatter/Gather in progress */ +#define DMA_SG1 (1<<6) +#define DMA_SG2 (1<<5) +#define DMA_SG3 (1<<4) + +/* DMA Channel Count Register */ +#define DMA_CTC_TCIE (1<<29) /* Terminal Count Interrupt Enable */ +#define DMA_CTC_ETIE (1<<28) /* EOT Interupt Enable */ +#define DMA_CTC_EIE (1<<27) /* Error Interrupt Enable */ +#define DMA_CTC_BTEN (1<<23) /* Burst Enable/Disable bit */ +#define DMA_CTC_BSIZ_MSK (3<<21) /* Mask of the Burst size bits */ +#define DMA_CTC_BSIZ_2 (0) +#define DMA_CTC_BSIZ_4 (1<<21) +#define DMA_CTC_BSIZ_8 (2<<21) +#define DMA_CTC_BSIZ_16 (3<<21) +#define DMA_CTC_TC_MASK 0xFFFFF + +/* + * DMA SG Command Register + */ +#define SSG_ENABLE(chan) (1<<(31-chan)) /* Start Scatter Gather */ +#define SSG_MASK_ENABLE(chan) (1<<(15-chan)) /* Enable writing to SSG0 bit */ + + +/* + * DMA Scatter/Gather Descriptor Bit fields + */ +#define SG_LINK (1<<31) /* Link */ +#define SG_TCI_ENABLE (1<<29) /* Enable Terminal Count Interrupt */ +#define SG_ETI_ENABLE (1<<28) /* Enable End of Transfer Interrupt */ +#define SG_ERI_ENABLE (1<<27) /* Enable Error Interrupt */ +#define SG_COUNT_MASK 0xFFFF /* Count Field */ + +#define SET_DMA_CONTROL \ + (SET_DMA_CIE_ENABLE(p_init->int_enable) | /* interrupt enable */ \ + SET_DMA_BEN(p_init->buffer_enable) | /* buffer enable */\ + SET_DMA_ETD(p_init->etd_output) | /* end of transfer pin */ \ + SET_DMA_TCE(p_init->tce_enable) | /* terminal count enable */ \ + SET_DMA_PL(p_init->pl) | /* peripheral location */ \ + SET_DMA_DAI(p_init->dai) | /* dest addr increment */ \ + SET_DMA_SAI(p_init->sai) | /* src addr increment */ \ + SET_DMA_PRIORITY(p_init->cp) | /* channel priority */ \ + SET_DMA_PW(p_init->pwidth) | /* peripheral/bus width */ \ + SET_DMA_PSC(p_init->psc) | /* peripheral setup cycles */ \ + SET_DMA_PWC(p_init->pwc) | /* peripheral wait cycles */ \ + SET_DMA_PHC(p_init->phc) | /* peripheral hold cycles */ \ + SET_DMA_PREFETCH(p_init->pf) /* read prefetch */) + +#define GET_DMA_POLARITY(chan) (DMAReq_ActiveLow(chan) | DMAAck_ActiveLow(chan) | EOT_ActiveLow(chan)) + + +/** + * struct ppc460ex_dma_device - internal representation of an DMA device + * @pdev: Platform device + * @id: HW DMA Device selector + * @dma_desc_pool: base of DMA descriptor region (DMA address) + * @dma_desc_pool_virt: base of DMA descriptor region (CPU address) + * @common: embedded struct dma_device + */ +typedef struct ppc460ex_plb_dma_device { + //struct platform_device *pdev; + void __iomem *reg_base; + struct device *dev; + struct resource reg; /* Resource for register */ + int id; + struct ppc460ex_plb_dma_chan *chan[MAX_PPC460EX_DMA_CHANNELS]; + wait_queue_head_t queue; +} ppc460ex_plb_dma_dev_t; + +typedef uint32_t sgl_handle_t; +/** + * struct ppc460ex_dma_chan - internal representation of an ADMA channel + * @lock: serializes enqueue/dequeue operations to the slot pool + * @device: parent device + * @chain: device chain view of the descriptors + * @common: common dmaengine channel object members + * @all_slots: complete domain of slots usable by the channel + * @reg: Resource for register + * @pending: allows batching of hardware operations + * @completed_cookie: identifier for the most recently completed operation + * @slots_allocated: records the actual size of the descriptor slot pool + * @hw_chain_inited: h/w descriptor chain initialization flag + * @irq_tasklet: bottom half where ppc460ex_adma_slot_cleanup runs + * @needs_unmap: if buffers should not be unmapped upon final processing + */ +typedef struct ppc460ex_plb_dma_chan { + void __iomem *reg_base; + struct ppc460ex_plb_dma_device *device; + struct timer_list cleanup_watchdog; + struct resource reg; /* Resource for register */ + unsigned int chan_id; + struct tasklet_struct irq_tasklet; + sgl_handle_t *phandle; + unsigned short in_use; /* set when channel is being used, clr when + * available. + */ + /* + * Valid polarity settings: + * DMAReq_ActiveLow(n) + * DMAAck_ActiveLow(n) + * EOT_ActiveLow(n) + * + * n is 0 to max dma chans + */ + unsigned int polarity; + + char buffer_enable; /* Boolean: buffer enable */ + char tce_enable; /* Boolean: terminal count enable */ + char etd_output; /* Boolean: eot pin is a tc output */ + char pce; /* Boolean: parity check enable */ + + /* + * Peripheral location: + * INTERNAL_PERIPHERAL (UART0 on the 405GP) + * EXTERNAL_PERIPHERAL + */ + char pl; /* internal/external peripheral */ + + /* + * Valid pwidth settings: + * PW_8 + * PW_16 + * PW_32 + * PW_64 + */ + unsigned int pwidth; + + char dai; /* Boolean: dst address increment */ + char sai; /* Boolean: src address increment */ + + /* + * Valid psc settings: 0-3 + */ + unsigned int psc; /* Peripheral Setup Cycles */ + + /* + * Valid pwc settings: + * 0-63 + */ + unsigned int pwc; /* Peripheral Wait Cycles */ + + /* + * Valid phc settings: + * 0-7 + */ + unsigned int phc; /* Peripheral Hold Cycles */ + + /* + * Valid cp (channel priority) settings: + * PRIORITY_LOW + * PRIORITY_MID_LOW + * PRIORITY_MID_HIGH + * PRIORITY_HIGH + */ + unsigned int cp; /* channel priority */ + + /* + * Valid pf (memory read prefetch) settings: + * + * PREFETCH_1 + * PREFETCH_2 + * PREFETCH_4 + */ + unsigned int pf; /* memory read prefetch */ + + /* + * Boolean: channel interrupt enable + * NOTE: for sgl transfers, only the last descriptor will be setup to + * interrupt. + */ + char int_enable; + + char shift; /* easy access to byte_count shift, based on */ + /* the width of the channel */ + + uint32_t control; /* channel control word */ + + /* These variabled are used ONLY in single dma transfers */ + unsigned int mode; /* transfer mode */ + phys_addr_t addr; + char ce; /* channel enable */ + char int_on_final_sg;/* for scatter/gather - only interrupt on last sg */ + +} ppc460ex_plb_dma_ch_t; + +/* + * PPC44x DMA implementations have a slightly different + * descriptor layout. Probably moved about due to the + * change to 64-bit addresses and link pointer. I don't + * know why they didn't just leave control_count after + * the dst_addr. + */ +#ifdef PPC4xx_DMA_64BIT +typedef struct { + uint32_t control; + uint32_t control_count; + phys_addr_t src_addr; + phys_addr_t dst_addr; + phys_addr_t next; +} ppc_sgl_t; +#else +typedef struct { + uint32_t control; + phys_addr_t src_addr; + phys_addr_t dst_addr; + uint32_t control_count; + uint32_t next; +} ppc_sgl_t; +#endif + + + +typedef struct { + unsigned int ch_id; + uint32_t control; /* channel ctrl word; loaded from each descrptr */ + uint32_t sgl_control; /* LK, TCI, ETI, and ERI bits in sgl descriptor */ + dma_addr_t dma_addr; /* dma (physical) address of this list */ + dma_addr_t dummy; /*Dummy variable to allow quad word alignment*/ + ppc_sgl_t *phead; + dma_addr_t phead_dma; + ppc_sgl_t *ptail; + dma_addr_t ptail_dma; +} sgl_list_info_t; + +typedef struct { + phys_addr_t *src_addr; + phys_addr_t *dst_addr; + phys_addr_t dma_src_addr; + phys_addr_t dma_dst_addr; +} pci_alloc_desc_t; + +#define PPC460EX_DMA_SGXFR_COMPLETE(id) (!((1 << (11-id)) & mfdcr(DCR_DMA2P40_SR))) +#define PPC460EX_DMA_CHAN_BUSY(id) ( (1 << (11-id)) & mfdcr(DCR_DMA2P40_SR) ) +#define DMA_STATUS(id) (mfdcr(DCR_DMA2P40_SR)) +#define CLEAR_DMA_STATUS(id) (mtdcr(DCR_DMA2P40_SR, 0xFFFFFFFF)) +#define PPC460EX_DMA_SGSTAT_FREE(id) (!((1 << (7-id)) & mfdcr(DCR_DMA2P40_SR)) ) +#define PPC460EX_DMA_TC_REACHED(id) ( (1 << (31-id)) & mfdcr(DCR_DMA2P40_SR) ) +#define PPC460EX_DMA_CHAN_XFR_COMPLETE(id) ( (!PPC460EX_DMA_CHAN_BUSY(id)) && (PPC460EX_DMA_TC_REACHED(id)) ) +#define PPC460EX_DMA_CHAN_SGXFR_COMPLETE(id) ( (!PPC460EX_DMA_CHAN_BUSY(id)) && PPC460EX_DMA_SGSTAT_FREE(id) ) +#define PPC460EX_DMA_SG_IN_PROGRESS(id) ( (1 << (7-id)) | (1 << (11-id)) ) +#define PPC460EX_DMA_SG_OP_COMPLETE(id) ( (PPC460EX_DMA_SG_IN_PROGRESS(id) & DMA_STATUS(id) ) == 0) + +extern ppc460ex_plb_dma_dev_t *adev; +int ppc460ex_init_dma_channel(ppc460ex_plb_dma_dev_t *adev, + unsigned int ch_id, + ppc460ex_plb_dma_ch_t *p_init); + +int ppc460ex_set_src_addr(int ch_id, phys_addr_t src_addr); + +int ppc460ex_set_dst_addr(int ch_id, phys_addr_t dst_addr); + +int ppc460ex_set_dma_mode(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, unsigned int mode); + +void ppc460ex_set_dma_count(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, unsigned int count); + +int ppc460ex_enable_dma_interrupt(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id); + +int ppc460ex_enable_dma(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id); + +int ppc460ex_get_dma_channel(void); + +void ppc460ex_disable_dma(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id); + +int ppc460ex_clear_dma_status(unsigned int ch_id); + +#if 0 +extern int test_dma_memcpy(void *src, void *dst, unsigned int length, unsigned int dma_ch); + +extern int test_sgdma_memcpy(void *src, void *dst, void *src1, void *dst1, + unsigned int length, unsigned int dma_ch); +#endif diff --git a/drivers/dma/qcom_bam_dma.c b/drivers/dma/qcom_bam_dma.c new file mode 100644 index 00000000000..82c923146e4 --- /dev/null +++ b/drivers/dma/qcom_bam_dma.c @@ -0,0 +1,1111 @@ +/* + * Copyright (c) 2013-2014, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ +/* + * QCOM BAM DMA engine driver + * + * QCOM BAM DMA blocks are distributed amongst a number of the on-chip + * peripherals on the MSM 8x74. The configuration of the channels are dependent + * on the way they are hard wired to that specific peripheral. The peripheral + * device tree entries specify the configuration of each channel. + * + * The DMA controller requires the use of external memory for storage of the + * hardware descriptors for each channel. The descriptor FIFO is accessed as a + * circular buffer and operations are managed according to the offset within the + * FIFO. After pipe/channel reset, all of the pipe registers and internal state + * are back to defaults. + * + * During DMA operations, we write descriptors to the FIFO, being careful to + * handle wrapping and then write the last FIFO offset to that channel's + * P_EVNT_REG register to kick off the transaction. The P_SW_OFSTS register + * indicates the current FIFO offset that is being processed, so there is some + * indication of where the hardware is currently working. + */ + +#include <linux/kernel.h> +#include <linux/io.h> +#include <linux/init.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/interrupt.h> +#include <linux/dma-mapping.h> +#include <linux/scatterlist.h> +#include <linux/device.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/of_dma.h> +#include <linux/clk.h> +#include <linux/dmaengine.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +struct bam_desc_hw { + u32 addr; /* Buffer physical address */ + u16 size; /* Buffer size in bytes */ + u16 flags; +}; + +#define DESC_FLAG_INT BIT(15) +#define DESC_FLAG_EOT BIT(14) +#define DESC_FLAG_EOB BIT(13) + +struct bam_async_desc { + struct virt_dma_desc vd; + + u32 num_desc; + u32 xfer_len; + struct bam_desc_hw *curr_desc; + + enum dma_transfer_direction dir; + size_t length; + struct bam_desc_hw desc[0]; +}; + +#define BAM_CTRL 0x0000 +#define BAM_REVISION 0x0004 +#define BAM_SW_REVISION 0x0080 +#define BAM_NUM_PIPES 0x003C +#define BAM_TIMER 0x0040 +#define BAM_TIMER_CTRL 0x0044 +#define BAM_DESC_CNT_TRSHLD 0x0008 +#define BAM_IRQ_SRCS 0x000C +#define BAM_IRQ_SRCS_MSK 0x0010 +#define BAM_IRQ_SRCS_UNMASKED 0x0030 +#define BAM_IRQ_STTS 0x0014 +#define BAM_IRQ_CLR 0x0018 +#define BAM_IRQ_EN 0x001C +#define BAM_CNFG_BITS 0x007C +#define BAM_IRQ_SRCS_EE(ee) (0x0800 + ((ee) * 0x80)) +#define BAM_IRQ_SRCS_MSK_EE(ee) (0x0804 + ((ee) * 0x80)) +#define BAM_P_CTRL(pipe) (0x1000 + ((pipe) * 0x1000)) +#define BAM_P_RST(pipe) (0x1004 + ((pipe) * 0x1000)) +#define BAM_P_HALT(pipe) (0x1008 + ((pipe) * 0x1000)) +#define BAM_P_IRQ_STTS(pipe) (0x1010 + ((pipe) * 0x1000)) +#define BAM_P_IRQ_CLR(pipe) (0x1014 + ((pipe) * 0x1000)) +#define BAM_P_IRQ_EN(pipe) (0x1018 + ((pipe) * 0x1000)) +#define BAM_P_EVNT_DEST_ADDR(pipe) (0x182C + ((pipe) * 0x1000)) +#define BAM_P_EVNT_REG(pipe) (0x1818 + ((pipe) * 0x1000)) +#define BAM_P_SW_OFSTS(pipe) (0x1800 + ((pipe) * 0x1000)) +#define BAM_P_DATA_FIFO_ADDR(pipe) (0x1824 + ((pipe) * 0x1000)) +#define BAM_P_DESC_FIFO_ADDR(pipe) (0x181C + ((pipe) * 0x1000)) +#define BAM_P_EVNT_TRSHLD(pipe) (0x1828 + ((pipe) * 0x1000)) +#define BAM_P_FIFO_SIZES(pipe) (0x1820 + ((pipe) * 0x1000)) + +/* BAM CTRL */ +#define BAM_SW_RST BIT(0) +#define BAM_EN BIT(1) +#define BAM_EN_ACCUM BIT(4) +#define BAM_TESTBUS_SEL_SHIFT 5 +#define BAM_TESTBUS_SEL_MASK 0x3F +#define BAM_DESC_CACHE_SEL_SHIFT 13 +#define BAM_DESC_CACHE_SEL_MASK 0x3 +#define BAM_CACHED_DESC_STORE BIT(15) +#define IBC_DISABLE BIT(16) + +/* BAM REVISION */ +#define REVISION_SHIFT 0 +#define REVISION_MASK 0xFF +#define NUM_EES_SHIFT 8 +#define NUM_EES_MASK 0xF +#define CE_BUFFER_SIZE BIT(13) +#define AXI_ACTIVE BIT(14) +#define USE_VMIDMT BIT(15) +#define SECURED BIT(16) +#define BAM_HAS_NO_BYPASS BIT(17) +#define HIGH_FREQUENCY_BAM BIT(18) +#define INACTIV_TMRS_EXST BIT(19) +#define NUM_INACTIV_TMRS BIT(20) +#define DESC_CACHE_DEPTH_SHIFT 21 +#define DESC_CACHE_DEPTH_1 (0 << DESC_CACHE_DEPTH_SHIFT) +#define DESC_CACHE_DEPTH_2 (1 << DESC_CACHE_DEPTH_SHIFT) +#define DESC_CACHE_DEPTH_3 (2 << DESC_CACHE_DEPTH_SHIFT) +#define DESC_CACHE_DEPTH_4 (3 << DESC_CACHE_DEPTH_SHIFT) +#define CMD_DESC_EN BIT(23) +#define INACTIV_TMR_BASE_SHIFT 24 +#define INACTIV_TMR_BASE_MASK 0xFF + +/* BAM NUM PIPES */ +#define BAM_NUM_PIPES_SHIFT 0 +#define BAM_NUM_PIPES_MASK 0xFF +#define PERIPH_NON_PIPE_GRP_SHIFT 16 +#define PERIPH_NON_PIP_GRP_MASK 0xFF +#define BAM_NON_PIPE_GRP_SHIFT 24 +#define BAM_NON_PIPE_GRP_MASK 0xFF + +/* BAM CNFG BITS */ +#define BAM_PIPE_CNFG BIT(2) +#define BAM_FULL_PIPE BIT(11) +#define BAM_NO_EXT_P_RST BIT(12) +#define BAM_IBC_DISABLE BIT(13) +#define BAM_SB_CLK_REQ BIT(14) +#define BAM_PSM_CSW_REQ BIT(15) +#define BAM_PSM_P_RES BIT(16) +#define BAM_AU_P_RES BIT(17) +#define BAM_SI_P_RES BIT(18) +#define BAM_WB_P_RES BIT(19) +#define BAM_WB_BLK_CSW BIT(20) +#define BAM_WB_CSW_ACK_IDL BIT(21) +#define BAM_WB_RETR_SVPNT BIT(22) +#define BAM_WB_DSC_AVL_P_RST BIT(23) +#define BAM_REG_P_EN BIT(24) +#define BAM_PSM_P_HD_DATA BIT(25) +#define BAM_AU_ACCUMED BIT(26) +#define BAM_CMD_ENABLE BIT(27) + +#define BAM_CNFG_BITS_DEFAULT (BAM_PIPE_CNFG | \ + BAM_NO_EXT_P_RST | \ + BAM_IBC_DISABLE | \ + BAM_SB_CLK_REQ | \ + BAM_PSM_CSW_REQ | \ + BAM_PSM_P_RES | \ + BAM_AU_P_RES | \ + BAM_SI_P_RES | \ + BAM_WB_P_RES | \ + BAM_WB_BLK_CSW | \ + BAM_WB_CSW_ACK_IDL | \ + BAM_WB_RETR_SVPNT | \ + BAM_WB_DSC_AVL_P_RST | \ + BAM_REG_P_EN | \ + BAM_PSM_P_HD_DATA | \ + BAM_AU_ACCUMED | \ + BAM_CMD_ENABLE) + +/* PIPE CTRL */ +#define P_EN BIT(1) +#define P_DIRECTION BIT(3) +#define P_SYS_STRM BIT(4) +#define P_SYS_MODE BIT(5) +#define P_AUTO_EOB BIT(6) +#define P_AUTO_EOB_SEL_SHIFT 7 +#define P_AUTO_EOB_SEL_512 (0 << P_AUTO_EOB_SEL_SHIFT) +#define P_AUTO_EOB_SEL_256 (1 << P_AUTO_EOB_SEL_SHIFT) +#define P_AUTO_EOB_SEL_128 (2 << P_AUTO_EOB_SEL_SHIFT) +#define P_AUTO_EOB_SEL_64 (3 << P_AUTO_EOB_SEL_SHIFT) +#define P_PREFETCH_LIMIT_SHIFT 9 +#define P_PREFETCH_LIMIT_32 (0 << P_PREFETCH_LIMIT_SHIFT) +#define P_PREFETCH_LIMIT_16 (1 << P_PREFETCH_LIMIT_SHIFT) +#define P_PREFETCH_LIMIT_4 (2 << P_PREFETCH_LIMIT_SHIFT) +#define P_WRITE_NWD BIT(11) +#define P_LOCK_GROUP_SHIFT 16 +#define P_LOCK_GROUP_MASK 0x1F + +/* BAM_DESC_CNT_TRSHLD */ +#define CNT_TRSHLD 0xffff +#define DEFAULT_CNT_THRSHLD 0x4 + +/* BAM_IRQ_SRCS */ +#define BAM_IRQ BIT(31) +#define P_IRQ 0x7fffffff + +/* BAM_IRQ_SRCS_MSK */ +#define BAM_IRQ_MSK BAM_IRQ +#define P_IRQ_MSK P_IRQ + +/* BAM_IRQ_STTS */ +#define BAM_TIMER_IRQ BIT(4) +#define BAM_EMPTY_IRQ BIT(3) +#define BAM_ERROR_IRQ BIT(2) +#define BAM_HRESP_ERR_IRQ BIT(1) + +/* BAM_IRQ_CLR */ +#define BAM_TIMER_CLR BIT(4) +#define BAM_EMPTY_CLR BIT(3) +#define BAM_ERROR_CLR BIT(2) +#define BAM_HRESP_ERR_CLR BIT(1) + +/* BAM_IRQ_EN */ +#define BAM_TIMER_EN BIT(4) +#define BAM_EMPTY_EN BIT(3) +#define BAM_ERROR_EN BIT(2) +#define BAM_HRESP_ERR_EN BIT(1) + +/* BAM_P_IRQ_EN */ +#define P_PRCSD_DESC_EN BIT(0) +#define P_TIMER_EN BIT(1) +#define P_WAKE_EN BIT(2) +#define P_OUT_OF_DESC_EN BIT(3) +#define P_ERR_EN BIT(4) +#define P_TRNSFR_END_EN BIT(5) +#define P_DEFAULT_IRQS_EN (P_PRCSD_DESC_EN | P_ERR_EN | P_TRNSFR_END_EN) + +/* BAM_P_SW_OFSTS */ +#define P_SW_OFSTS_MASK 0xffff + +#define BAM_DESC_FIFO_SIZE SZ_32K +#define MAX_DESCRIPTORS (BAM_DESC_FIFO_SIZE / sizeof(struct bam_desc_hw) - 1) +#define BAM_MAX_DATA_SIZE (SZ_32K - 8) + +struct bam_chan { + struct virt_dma_chan vc; + + struct bam_device *bdev; + + /* configuration from device tree */ + u32 id; + + struct bam_async_desc *curr_txd; /* current running dma */ + + /* runtime configuration */ + struct dma_slave_config slave; + + /* fifo storage */ + struct bam_desc_hw *fifo_virt; + dma_addr_t fifo_phys; + + /* fifo markers */ + unsigned short head; /* start of active descriptor entries */ + unsigned short tail; /* end of active descriptor entries */ + + unsigned int initialized; /* is the channel hw initialized? */ + unsigned int paused; /* is the channel paused? */ + unsigned int reconfigure; /* new slave config? */ + + struct list_head node; +}; + +static inline struct bam_chan *to_bam_chan(struct dma_chan *common) +{ + return container_of(common, struct bam_chan, vc.chan); +} + +struct bam_device { + void __iomem *regs; + struct device *dev; + struct dma_device common; + struct device_dma_parameters dma_parms; + struct bam_chan *channels; + u32 num_channels; + + /* execution environment ID, from DT */ + u32 ee; + + struct clk *bamclk; + int irq; + + /* dma start transaction tasklet */ + struct tasklet_struct task; +}; + +/** + * bam_reset_channel - Reset individual BAM DMA channel + * @bchan: bam channel + * + * This function resets a specific BAM channel + */ +static void bam_reset_channel(struct bam_chan *bchan) +{ + struct bam_device *bdev = bchan->bdev; + + lockdep_assert_held(&bchan->vc.lock); + + /* reset channel */ + writel_relaxed(1, bdev->regs + BAM_P_RST(bchan->id)); + writel_relaxed(0, bdev->regs + BAM_P_RST(bchan->id)); + + /* don't allow cpu to reorder BAM register accesses done after this */ + wmb(); + + /* make sure hw is initialized when channel is used the first time */ + bchan->initialized = 0; +} + +/** + * bam_chan_init_hw - Initialize channel hardware + * @bchan: bam channel + * + * This function resets and initializes the BAM channel + */ +static void bam_chan_init_hw(struct bam_chan *bchan, + enum dma_transfer_direction dir) +{ + struct bam_device *bdev = bchan->bdev; + u32 val; + + /* Reset the channel to clear internal state of the FIFO */ + bam_reset_channel(bchan); + + /* + * write out 8 byte aligned address. We have enough space for this + * because we allocated 1 more descriptor (8 bytes) than we can use + */ + writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)), + bdev->regs + BAM_P_DESC_FIFO_ADDR(bchan->id)); + writel_relaxed(BAM_DESC_FIFO_SIZE, bdev->regs + + BAM_P_FIFO_SIZES(bchan->id)); + + /* enable the per pipe interrupts, enable EOT, ERR, and INT irqs */ + writel_relaxed(P_DEFAULT_IRQS_EN, bdev->regs + BAM_P_IRQ_EN(bchan->id)); + + /* unmask the specific pipe and EE combo */ + val = readl_relaxed(bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + val |= BIT(bchan->id); + writel_relaxed(val, bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + + /* don't allow cpu to reorder the channel enable done below */ + wmb(); + + /* set fixed direction and mode, then enable channel */ + val = P_EN | P_SYS_MODE; + if (dir == DMA_DEV_TO_MEM) + val |= P_DIRECTION; + + writel_relaxed(val, bdev->regs + BAM_P_CTRL(bchan->id)); + + bchan->initialized = 1; + + /* init FIFO pointers */ + bchan->head = 0; + bchan->tail = 0; +} + +/** + * bam_alloc_chan - Allocate channel resources for DMA channel. + * @chan: specified channel + * + * This function allocates the FIFO descriptor memory + */ +static int bam_alloc_chan(struct dma_chan *chan) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct bam_device *bdev = bchan->bdev; + + if (bchan->fifo_virt) + return 0; + + /* allocate FIFO descriptor space, but only if necessary */ + bchan->fifo_virt = dma_alloc_writecombine(bdev->dev, BAM_DESC_FIFO_SIZE, + &bchan->fifo_phys, GFP_KERNEL); + + if (!bchan->fifo_virt) { + dev_err(bdev->dev, "Failed to allocate desc fifo\n"); + return -ENOMEM; + } + + return 0; +} + +/** + * bam_free_chan - Frees dma resources associated with specific channel + * @chan: specified channel + * + * Free the allocated fifo descriptor memory and channel resources + * + */ +static void bam_free_chan(struct dma_chan *chan) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct bam_device *bdev = bchan->bdev; + u32 val; + unsigned long flags; + + vchan_free_chan_resources(to_virt_chan(chan)); + + if (bchan->curr_txd) { + dev_err(bchan->bdev->dev, "Cannot free busy channel\n"); + return; + } + + spin_lock_irqsave(&bchan->vc.lock, flags); + bam_reset_channel(bchan); + spin_unlock_irqrestore(&bchan->vc.lock, flags); + + dma_free_writecombine(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt, + bchan->fifo_phys); + bchan->fifo_virt = NULL; + + /* mask irq for pipe/channel */ + val = readl_relaxed(bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + val &= ~BIT(bchan->id); + writel_relaxed(val, bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + + /* disable irq */ + writel_relaxed(0, bdev->regs + BAM_P_IRQ_EN(bchan->id)); +} + +/** + * bam_slave_config - set slave configuration for channel + * @chan: dma channel + * @cfg: slave configuration + * + * Sets slave configuration for channel + * + */ +static void bam_slave_config(struct bam_chan *bchan, + struct dma_slave_config *cfg) +{ + memcpy(&bchan->slave, cfg, sizeof(*cfg)); + bchan->reconfigure = 1; +} + +/** + * bam_prep_slave_sg - Prep slave sg transaction + * + * @chan: dma channel + * @sgl: scatter gather list + * @sg_len: length of sg + * @direction: DMA transfer direction + * @flags: DMA flags + * @context: transfer context (unused) + */ +static struct dma_async_tx_descriptor *bam_prep_slave_sg(struct dma_chan *chan, + struct scatterlist *sgl, unsigned int sg_len, + enum dma_transfer_direction direction, unsigned long flags, + void *context) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct bam_device *bdev = bchan->bdev; + struct bam_async_desc *async_desc; + struct scatterlist *sg; + u32 i; + struct bam_desc_hw *desc; + unsigned int num_alloc = 0; + + + if (!is_slave_direction(direction)) { + dev_err(bdev->dev, "invalid dma direction\n"); + return NULL; + } + + /* calculate number of required entries */ + for_each_sg(sgl, sg, sg_len, i) + num_alloc += DIV_ROUND_UP(sg_dma_len(sg), BAM_MAX_DATA_SIZE); + + /* allocate enough room to accomodate the number of entries */ + async_desc = kzalloc(sizeof(*async_desc) + + (num_alloc * sizeof(struct bam_desc_hw)), GFP_NOWAIT); + + if (!async_desc) + goto err_out; + + async_desc->num_desc = num_alloc; + async_desc->curr_desc = async_desc->desc; + async_desc->dir = direction; + + /* fill in temporary descriptors */ + desc = async_desc->desc; + for_each_sg(sgl, sg, sg_len, i) { + unsigned int remainder = sg_dma_len(sg); + unsigned int curr_offset = 0; + + do { + desc->addr = sg_dma_address(sg) + curr_offset; + + if (remainder > BAM_MAX_DATA_SIZE) { + desc->size = BAM_MAX_DATA_SIZE; + remainder -= BAM_MAX_DATA_SIZE; + curr_offset += BAM_MAX_DATA_SIZE; + } else { + desc->size = remainder; + remainder = 0; + } + + async_desc->length += desc->size; + desc++; + } while (remainder > 0); + } + + return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags); + +err_out: + kfree(async_desc); + return NULL; +} + +/** + * bam_dma_terminate_all - terminate all transactions on a channel + * @bchan: bam dma channel + * + * Dequeues and frees all transactions + * No callbacks are done + * + */ +static void bam_dma_terminate_all(struct bam_chan *bchan) +{ + unsigned long flag; + LIST_HEAD(head); + + /* remove all transactions, including active transaction */ + spin_lock_irqsave(&bchan->vc.lock, flag); + if (bchan->curr_txd) { + list_add(&bchan->curr_txd->vd.node, &bchan->vc.desc_issued); + bchan->curr_txd = NULL; + } + + vchan_get_all_descriptors(&bchan->vc, &head); + spin_unlock_irqrestore(&bchan->vc.lock, flag); + + vchan_dma_desc_free_list(&bchan->vc, &head); +} + +/** + * bam_control - DMA device control + * @chan: dma channel + * @cmd: control cmd + * @arg: cmd argument + * + * Perform DMA control command + * + */ +static int bam_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct bam_device *bdev = bchan->bdev; + int ret = 0; + unsigned long flag; + + switch (cmd) { + case DMA_PAUSE: + spin_lock_irqsave(&bchan->vc.lock, flag); + writel_relaxed(1, bdev->regs + BAM_P_HALT(bchan->id)); + bchan->paused = 1; + spin_unlock_irqrestore(&bchan->vc.lock, flag); + break; + + case DMA_RESUME: + spin_lock_irqsave(&bchan->vc.lock, flag); + writel_relaxed(0, bdev->regs + BAM_P_HALT(bchan->id)); + bchan->paused = 0; + spin_unlock_irqrestore(&bchan->vc.lock, flag); + break; + + case DMA_TERMINATE_ALL: + bam_dma_terminate_all(bchan); + break; + + case DMA_SLAVE_CONFIG: + spin_lock_irqsave(&bchan->vc.lock, flag); + bam_slave_config(bchan, (struct dma_slave_config *)arg); + spin_unlock_irqrestore(&bchan->vc.lock, flag); + break; + + default: + ret = -ENXIO; + break; + } + + return ret; +} + +/** + * process_channel_irqs - processes the channel interrupts + * @bdev: bam controller + * + * This function processes the channel interrupts + * + */ +static u32 process_channel_irqs(struct bam_device *bdev) +{ + u32 i, srcs, pipe_stts; + unsigned long flags; + struct bam_async_desc *async_desc; + + srcs = readl_relaxed(bdev->regs + BAM_IRQ_SRCS_EE(bdev->ee)); + + /* return early if no pipe/channel interrupts are present */ + if (!(srcs & P_IRQ)) + return srcs; + + for (i = 0; i < bdev->num_channels; i++) { + struct bam_chan *bchan = &bdev->channels[i]; + + if (!(srcs & BIT(i))) + continue; + + /* clear pipe irq */ + pipe_stts = readl_relaxed(bdev->regs + + BAM_P_IRQ_STTS(i)); + + writel_relaxed(pipe_stts, bdev->regs + + BAM_P_IRQ_CLR(i)); + + spin_lock_irqsave(&bchan->vc.lock, flags); + async_desc = bchan->curr_txd; + + if (async_desc) { + async_desc->num_desc -= async_desc->xfer_len; + async_desc->curr_desc += async_desc->xfer_len; + bchan->curr_txd = NULL; + + /* manage FIFO */ + bchan->head += async_desc->xfer_len; + bchan->head %= MAX_DESCRIPTORS; + + /* + * if complete, process cookie. Otherwise + * push back to front of desc_issued so that + * it gets restarted by the tasklet + */ + if (!async_desc->num_desc) + vchan_cookie_complete(&async_desc->vd); + else + list_add(&async_desc->vd.node, + &bchan->vc.desc_issued); + } + + spin_unlock_irqrestore(&bchan->vc.lock, flags); + } + + return srcs; +} + +/** + * bam_dma_irq - irq handler for bam controller + * @irq: IRQ of interrupt + * @data: callback data + * + * IRQ handler for the bam controller + */ +static irqreturn_t bam_dma_irq(int irq, void *data) +{ + struct bam_device *bdev = data; + u32 clr_mask = 0, srcs = 0; + + srcs |= process_channel_irqs(bdev); + + /* kick off tasklet to start next dma transfer */ + if (srcs & P_IRQ) + tasklet_schedule(&bdev->task); + + if (srcs & BAM_IRQ) + clr_mask = readl_relaxed(bdev->regs + BAM_IRQ_STTS); + + /* don't allow reorder of the various accesses to the BAM registers */ + mb(); + + writel_relaxed(clr_mask, bdev->regs + BAM_IRQ_CLR); + + return IRQ_HANDLED; +} + +/** + * bam_tx_status - returns status of transaction + * @chan: dma channel + * @cookie: transaction cookie + * @txstate: DMA transaction state + * + * Return status of dma transaction + */ +static enum dma_status bam_tx_status(struct dma_chan *chan, dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + struct bam_chan *bchan = to_bam_chan(chan); + struct virt_dma_desc *vd; + int ret; + size_t residue = 0; + unsigned int i; + unsigned long flags; + + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + + if (!txstate) + return bchan->paused ? DMA_PAUSED : ret; + + spin_lock_irqsave(&bchan->vc.lock, flags); + vd = vchan_find_desc(&bchan->vc, cookie); + if (vd) + residue = container_of(vd, struct bam_async_desc, vd)->length; + else if (bchan->curr_txd && bchan->curr_txd->vd.tx.cookie == cookie) + for (i = 0; i < bchan->curr_txd->num_desc; i++) + residue += bchan->curr_txd->curr_desc[i].size; + + spin_unlock_irqrestore(&bchan->vc.lock, flags); + + dma_set_residue(txstate, residue); + + if (ret == DMA_IN_PROGRESS && bchan->paused) + ret = DMA_PAUSED; + + return ret; +} + +/** + * bam_apply_new_config + * @bchan: bam dma channel + * @dir: DMA direction + */ +static void bam_apply_new_config(struct bam_chan *bchan, + enum dma_transfer_direction dir) +{ + struct bam_device *bdev = bchan->bdev; + u32 maxburst; + + if (dir == DMA_DEV_TO_MEM) + maxburst = bchan->slave.src_maxburst; + else + maxburst = bchan->slave.dst_maxburst; + + writel_relaxed(maxburst, bdev->regs + BAM_DESC_CNT_TRSHLD); + + bchan->reconfigure = 0; +} + +/** + * bam_start_dma - start next transaction + * @bchan - bam dma channel + */ +static void bam_start_dma(struct bam_chan *bchan) +{ + struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc); + struct bam_device *bdev = bchan->bdev; + struct bam_async_desc *async_desc; + struct bam_desc_hw *desc; + struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt, + sizeof(struct bam_desc_hw)); + + lockdep_assert_held(&bchan->vc.lock); + + if (!vd) + return; + + list_del(&vd->node); + + async_desc = container_of(vd, struct bam_async_desc, vd); + bchan->curr_txd = async_desc; + + /* on first use, initialize the channel hardware */ + if (!bchan->initialized) + bam_chan_init_hw(bchan, async_desc->dir); + + /* apply new slave config changes, if necessary */ + if (bchan->reconfigure) + bam_apply_new_config(bchan, async_desc->dir); + + desc = bchan->curr_txd->curr_desc; + + if (async_desc->num_desc > MAX_DESCRIPTORS) + async_desc->xfer_len = MAX_DESCRIPTORS; + else + async_desc->xfer_len = async_desc->num_desc; + + /* set INT on last descriptor */ + desc[async_desc->xfer_len - 1].flags |= DESC_FLAG_INT; + + if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) { + u32 partial = MAX_DESCRIPTORS - bchan->tail; + + memcpy(&fifo[bchan->tail], desc, + partial * sizeof(struct bam_desc_hw)); + memcpy(fifo, &desc[partial], (async_desc->xfer_len - partial) * + sizeof(struct bam_desc_hw)); + } else { + memcpy(&fifo[bchan->tail], desc, + async_desc->xfer_len * sizeof(struct bam_desc_hw)); + } + + bchan->tail += async_desc->xfer_len; + bchan->tail %= MAX_DESCRIPTORS; + + /* ensure descriptor writes and dma start not reordered */ + wmb(); + writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw), + bdev->regs + BAM_P_EVNT_REG(bchan->id)); +} + +/** + * dma_tasklet - DMA IRQ tasklet + * @data: tasklet argument (bam controller structure) + * + * Sets up next DMA operation and then processes all completed transactions + */ +static void dma_tasklet(unsigned long data) +{ + struct bam_device *bdev = (struct bam_device *)data; + struct bam_chan *bchan; + unsigned long flags; + unsigned int i; + + /* go through the channels and kick off transactions */ + for (i = 0; i < bdev->num_channels; i++) { + bchan = &bdev->channels[i]; + spin_lock_irqsave(&bchan->vc.lock, flags); + + if (!list_empty(&bchan->vc.desc_issued) && !bchan->curr_txd) + bam_start_dma(bchan); + spin_unlock_irqrestore(&bchan->vc.lock, flags); + } +} + +/** + * bam_issue_pending - starts pending transactions + * @chan: dma channel + * + * Calls tasklet directly which in turn starts any pending transactions + */ +static void bam_issue_pending(struct dma_chan *chan) +{ + struct bam_chan *bchan = to_bam_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&bchan->vc.lock, flags); + + /* if work pending and idle, start a transaction */ + if (vchan_issue_pending(&bchan->vc) && !bchan->curr_txd) + bam_start_dma(bchan); + + spin_unlock_irqrestore(&bchan->vc.lock, flags); +} + +/** + * bam_dma_free_desc - free descriptor memory + * @vd: virtual descriptor + * + */ +static void bam_dma_free_desc(struct virt_dma_desc *vd) +{ + struct bam_async_desc *async_desc = container_of(vd, + struct bam_async_desc, vd); + + kfree(async_desc); +} + +static struct dma_chan *bam_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *of) +{ + struct bam_device *bdev = container_of(of->of_dma_data, + struct bam_device, common); + unsigned int request; + + if (dma_spec->args_count != 1) + return NULL; + + request = dma_spec->args[0]; + if (request >= bdev->num_channels) + return NULL; + + return dma_get_slave_channel(&(bdev->channels[request].vc.chan)); +} + +/** + * bam_init + * @bdev: bam device + * + * Initialization helper for global bam registers + */ +static int bam_init(struct bam_device *bdev) +{ + u32 val; + + /* read revision and configuration information */ + val = readl_relaxed(bdev->regs + BAM_REVISION) >> NUM_EES_SHIFT; + val &= NUM_EES_MASK; + + /* check that configured EE is within range */ + if (bdev->ee >= val) + return -EINVAL; + + val = readl_relaxed(bdev->regs + BAM_NUM_PIPES); + bdev->num_channels = val & BAM_NUM_PIPES_MASK; + + /* s/w reset bam */ + /* after reset all pipes are disabled and idle */ + val = readl_relaxed(bdev->regs + BAM_CTRL); + val |= BAM_SW_RST; + writel_relaxed(val, bdev->regs + BAM_CTRL); + val &= ~BAM_SW_RST; + writel_relaxed(val, bdev->regs + BAM_CTRL); + + /* make sure previous stores are visible before enabling BAM */ + wmb(); + + /* enable bam */ + val |= BAM_EN; + writel_relaxed(val, bdev->regs + BAM_CTRL); + + /* set descriptor threshhold, start with 4 bytes */ + writel_relaxed(DEFAULT_CNT_THRSHLD, bdev->regs + BAM_DESC_CNT_TRSHLD); + + /* Enable default set of h/w workarounds, ie all except BAM_FULL_PIPE */ + writel_relaxed(BAM_CNFG_BITS_DEFAULT, bdev->regs + BAM_CNFG_BITS); + + /* enable irqs for errors */ + writel_relaxed(BAM_ERROR_EN | BAM_HRESP_ERR_EN, + bdev->regs + BAM_IRQ_EN); + + /* unmask global bam interrupt */ + writel_relaxed(BAM_IRQ_MSK, bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + + return 0; +} + +static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan, + u32 index) +{ + bchan->id = index; + bchan->bdev = bdev; + + vchan_init(&bchan->vc, &bdev->common); + bchan->vc.desc_free = bam_dma_free_desc; +} + +static int bam_dma_probe(struct platform_device *pdev) +{ + struct bam_device *bdev; + struct resource *iores; + int ret, i; + + bdev = devm_kzalloc(&pdev->dev, sizeof(*bdev), GFP_KERNEL); + if (!bdev) + return -ENOMEM; + + bdev->dev = &pdev->dev; + + iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); + bdev->regs = devm_ioremap_resource(&pdev->dev, iores); + if (IS_ERR(bdev->regs)) + return PTR_ERR(bdev->regs); + + bdev->irq = platform_get_irq(pdev, 0); + if (bdev->irq < 0) + return bdev->irq; + + ret = of_property_read_u32(pdev->dev.of_node, "qcom,ee", &bdev->ee); + if (ret) { + dev_err(bdev->dev, "Execution environment unspecified\n"); + return ret; + } + + bdev->bamclk = devm_clk_get(bdev->dev, "bam_clk"); + if (IS_ERR(bdev->bamclk)) + return PTR_ERR(bdev->bamclk); + + ret = clk_prepare_enable(bdev->bamclk); + if (ret) { + dev_err(bdev->dev, "failed to prepare/enable clock\n"); + return ret; + } + + ret = bam_init(bdev); + if (ret) + goto err_disable_clk; + + tasklet_init(&bdev->task, dma_tasklet, (unsigned long)bdev); + + bdev->channels = devm_kcalloc(bdev->dev, bdev->num_channels, + sizeof(*bdev->channels), GFP_KERNEL); + + if (!bdev->channels) { + ret = -ENOMEM; + goto err_disable_clk; + } + + /* allocate and initialize channels */ + INIT_LIST_HEAD(&bdev->common.channels); + + for (i = 0; i < bdev->num_channels; i++) + bam_channel_init(bdev, &bdev->channels[i], i); + + ret = devm_request_irq(bdev->dev, bdev->irq, bam_dma_irq, + IRQF_TRIGGER_HIGH, "bam_dma", bdev); + if (ret) + goto err_disable_clk; + + /* set max dma segment size */ + bdev->common.dev = bdev->dev; + bdev->common.dev->dma_parms = &bdev->dma_parms; + ret = dma_set_max_seg_size(bdev->common.dev, BAM_MAX_DATA_SIZE); + if (ret) { + dev_err(bdev->dev, "cannot set maximum segment size\n"); + goto err_disable_clk; + } + + platform_set_drvdata(pdev, bdev); + + /* set capabilities */ + dma_cap_zero(bdev->common.cap_mask); + dma_cap_set(DMA_SLAVE, bdev->common.cap_mask); + + /* initialize dmaengine apis */ + bdev->common.device_alloc_chan_resources = bam_alloc_chan; + bdev->common.device_free_chan_resources = bam_free_chan; + bdev->common.device_prep_slave_sg = bam_prep_slave_sg; + bdev->common.device_control = bam_control; + bdev->common.device_issue_pending = bam_issue_pending; + bdev->common.device_tx_status = bam_tx_status; + bdev->common.dev = bdev->dev; + + ret = dma_async_device_register(&bdev->common); + if (ret) { + dev_err(bdev->dev, "failed to register dma async device\n"); + goto err_disable_clk; + } + + ret = of_dma_controller_register(pdev->dev.of_node, bam_dma_xlate, + &bdev->common); + if (ret) + goto err_unregister_dma; + + return 0; + +err_unregister_dma: + dma_async_device_unregister(&bdev->common); +err_disable_clk: + clk_disable_unprepare(bdev->bamclk); + return ret; +} + +static int bam_dma_remove(struct platform_device *pdev) +{ + struct bam_device *bdev = platform_get_drvdata(pdev); + u32 i; + + of_dma_controller_free(pdev->dev.of_node); + dma_async_device_unregister(&bdev->common); + + /* mask all interrupts for this execution environment */ + writel_relaxed(0, bdev->regs + BAM_IRQ_SRCS_MSK_EE(bdev->ee)); + + devm_free_irq(bdev->dev, bdev->irq, bdev); + + for (i = 0; i < bdev->num_channels; i++) { + bam_dma_terminate_all(&bdev->channels[i]); + tasklet_kill(&bdev->channels[i].vc.task); + + dma_free_writecombine(bdev->dev, BAM_DESC_FIFO_SIZE, + bdev->channels[i].fifo_virt, + bdev->channels[i].fifo_phys); + } + + tasklet_kill(&bdev->task); + + clk_disable_unprepare(bdev->bamclk); + + return 0; +} + +static const struct of_device_id bam_of_match[] = { + { .compatible = "qcom,bam-v1.4.0", }, + {} +}; +MODULE_DEVICE_TABLE(of, bam_of_match); + +static struct platform_driver bam_dma_driver = { + .probe = bam_dma_probe, + .remove = bam_dma_remove, + .driver = { + .name = "bam-dma-engine", + .owner = THIS_MODULE, + .of_match_table = bam_of_match, + }, +}; + +module_platform_driver(bam_dma_driver); + +MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>"); +MODULE_DESCRIPTION("QCOM BAM DMA engine driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/s3c24xx-dma.c b/drivers/dma/s3c24xx-dma.c new file mode 100644 index 00000000000..012520c9fd7 --- /dev/null +++ b/drivers/dma/s3c24xx-dma.c @@ -0,0 +1,1430 @@ +/* + * S3C24XX DMA handling + * + * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de> + * + * based on amba-pl08x.c + * + * Copyright (c) 2006 ARM Ltd. + * Copyright (c) 2010 ST-Ericsson SA + * + * Author: Peter Pearse <peter.pearse@arm.com> + * Author: Linus Walleij <linus.walleij@stericsson.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * The DMA controllers in S3C24XX SoCs have a varying number of DMA signals + * that can be routed to any of the 4 to 8 hardware-channels. + * + * Therefore on these DMA controllers the number of channels + * and the number of incoming DMA signals are two totally different things. + * It is usually not possible to theoretically handle all physical signals, + * so a multiplexing scheme with possible denial of use is necessary. + * + * Open items: + * - bursts + */ + +#include <linux/platform_device.h> +#include <linux/types.h> +#include <linux/dmaengine.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/clk.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/platform_data/dma-s3c24xx.h> + +#include "dmaengine.h" +#include "virt-dma.h" + +#define MAX_DMA_CHANNELS 8 + +#define S3C24XX_DISRC 0x00 +#define S3C24XX_DISRCC 0x04 +#define S3C24XX_DISRCC_INC_INCREMENT 0 +#define S3C24XX_DISRCC_INC_FIXED BIT(0) +#define S3C24XX_DISRCC_LOC_AHB 0 +#define S3C24XX_DISRCC_LOC_APB BIT(1) + +#define S3C24XX_DIDST 0x08 +#define S3C24XX_DIDSTC 0x0c +#define S3C24XX_DIDSTC_INC_INCREMENT 0 +#define S3C24XX_DIDSTC_INC_FIXED BIT(0) +#define S3C24XX_DIDSTC_LOC_AHB 0 +#define S3C24XX_DIDSTC_LOC_APB BIT(1) +#define S3C24XX_DIDSTC_INT_TC0 0 +#define S3C24XX_DIDSTC_INT_RELOAD BIT(2) + +#define S3C24XX_DCON 0x10 + +#define S3C24XX_DCON_TC_MASK 0xfffff +#define S3C24XX_DCON_DSZ_BYTE (0 << 20) +#define S3C24XX_DCON_DSZ_HALFWORD (1 << 20) +#define S3C24XX_DCON_DSZ_WORD (2 << 20) +#define S3C24XX_DCON_DSZ_MASK (3 << 20) +#define S3C24XX_DCON_DSZ_SHIFT 20 +#define S3C24XX_DCON_AUTORELOAD 0 +#define S3C24XX_DCON_NORELOAD BIT(22) +#define S3C24XX_DCON_HWTRIG BIT(23) +#define S3C24XX_DCON_HWSRC_SHIFT 24 +#define S3C24XX_DCON_SERV_SINGLE 0 +#define S3C24XX_DCON_SERV_WHOLE BIT(27) +#define S3C24XX_DCON_TSZ_UNIT 0 +#define S3C24XX_DCON_TSZ_BURST4 BIT(28) +#define S3C24XX_DCON_INT BIT(29) +#define S3C24XX_DCON_SYNC_PCLK 0 +#define S3C24XX_DCON_SYNC_HCLK BIT(30) +#define S3C24XX_DCON_DEMAND 0 +#define S3C24XX_DCON_HANDSHAKE BIT(31) + +#define S3C24XX_DSTAT 0x14 +#define S3C24XX_DSTAT_STAT_BUSY BIT(20) +#define S3C24XX_DSTAT_CURRTC_MASK 0xfffff + +#define S3C24XX_DMASKTRIG 0x20 +#define S3C24XX_DMASKTRIG_SWTRIG BIT(0) +#define S3C24XX_DMASKTRIG_ON BIT(1) +#define S3C24XX_DMASKTRIG_STOP BIT(2) + +#define S3C24XX_DMAREQSEL 0x24 +#define S3C24XX_DMAREQSEL_HW BIT(0) + +/* + * S3C2410, S3C2440 and S3C2442 SoCs cannot select any physical channel + * for a DMA source. Instead only specific channels are valid. + * All of these SoCs have 4 physical channels and the number of request + * source bits is 3. Additionally we also need 1 bit to mark the channel + * as valid. + * Therefore we separate the chansel element of the channel data into 4 + * parts of 4 bits each, to hold the information if the channel is valid + * and the hw request source to use. + * + * Example: + * SDI is valid on channels 0, 2 and 3 - with varying hw request sources. + * For it the chansel field would look like + * + * ((BIT(3) | 1) << 3 * 4) | // channel 3, with request source 1 + * ((BIT(3) | 2) << 2 * 4) | // channel 2, with request source 2 + * ((BIT(3) | 2) << 0 * 4) // channel 0, with request source 2 + */ +#define S3C24XX_CHANSEL_WIDTH 4 +#define S3C24XX_CHANSEL_VALID BIT(3) +#define S3C24XX_CHANSEL_REQ_MASK 7 + +/* + * struct soc_data - vendor-specific config parameters for individual SoCs + * @stride: spacing between the registers of each channel + * @has_reqsel: does the controller use the newer requestselection mechanism + * @has_clocks: are controllable dma-clocks present + */ +struct soc_data { + int stride; + bool has_reqsel; + bool has_clocks; +}; + +/* + * enum s3c24xx_dma_chan_state - holds the virtual channel states + * @S3C24XX_DMA_CHAN_IDLE: the channel is idle + * @S3C24XX_DMA_CHAN_RUNNING: the channel has allocated a physical transport + * channel and is running a transfer on it + * @S3C24XX_DMA_CHAN_WAITING: the channel is waiting for a physical transport + * channel to become available (only pertains to memcpy channels) + */ +enum s3c24xx_dma_chan_state { + S3C24XX_DMA_CHAN_IDLE, + S3C24XX_DMA_CHAN_RUNNING, + S3C24XX_DMA_CHAN_WAITING, +}; + +/* + * struct s3c24xx_sg - structure containing data per sg + * @src_addr: src address of sg + * @dst_addr: dst address of sg + * @len: transfer len in bytes + * @node: node for txd's dsg_list + */ +struct s3c24xx_sg { + dma_addr_t src_addr; + dma_addr_t dst_addr; + size_t len; + struct list_head node; +}; + +/* + * struct s3c24xx_txd - wrapper for struct dma_async_tx_descriptor + * @vd: virtual DMA descriptor + * @dsg_list: list of children sg's + * @at: sg currently being transfered + * @width: transfer width + * @disrcc: value for source control register + * @didstc: value for destination control register + * @dcon: base value for dcon register + * @cyclic: indicate cyclic transfer + */ +struct s3c24xx_txd { + struct virt_dma_desc vd; + struct list_head dsg_list; + struct list_head *at; + u8 width; + u32 disrcc; + u32 didstc; + u32 dcon; + bool cyclic; +}; + +struct s3c24xx_dma_chan; + +/* + * struct s3c24xx_dma_phy - holder for the physical channels + * @id: physical index to this channel + * @valid: does the channel have all required elements + * @base: virtual memory base (remapped) for the this channel + * @irq: interrupt for this channel + * @clk: clock for this channel + * @lock: a lock to use when altering an instance of this struct + * @serving: virtual channel currently being served by this physicalchannel + * @host: a pointer to the host (internal use) + */ +struct s3c24xx_dma_phy { + unsigned int id; + bool valid; + void __iomem *base; + int irq; + struct clk *clk; + spinlock_t lock; + struct s3c24xx_dma_chan *serving; + struct s3c24xx_dma_engine *host; +}; + +/* + * struct s3c24xx_dma_chan - this structure wraps a DMA ENGINE channel + * @id: the id of the channel + * @name: name of the channel + * @vc: wrappped virtual channel + * @phy: the physical channel utilized by this channel, if there is one + * @runtime_addr: address for RX/TX according to the runtime config + * @at: active transaction on this channel + * @lock: a lock for this channel data + * @host: a pointer to the host (internal use) + * @state: whether the channel is idle, running etc + * @slave: whether this channel is a device (slave) or for memcpy + */ +struct s3c24xx_dma_chan { + int id; + const char *name; + struct virt_dma_chan vc; + struct s3c24xx_dma_phy *phy; + struct dma_slave_config cfg; + struct s3c24xx_txd *at; + struct s3c24xx_dma_engine *host; + enum s3c24xx_dma_chan_state state; + bool slave; +}; + +/* + * struct s3c24xx_dma_engine - the local state holder for the S3C24XX + * @pdev: the corresponding platform device + * @pdata: platform data passed in from the platform/machine + * @base: virtual memory base (remapped) + * @slave: slave engine for this instance + * @memcpy: memcpy engine for this instance + * @phy_chans: array of data for the physical channels + */ +struct s3c24xx_dma_engine { + struct platform_device *pdev; + const struct s3c24xx_dma_platdata *pdata; + struct soc_data *sdata; + void __iomem *base; + struct dma_device slave; + struct dma_device memcpy; + struct s3c24xx_dma_phy *phy_chans; +}; + +/* + * Physical channel handling + */ + +/* + * Check whether a certain channel is busy or not. + */ +static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy) +{ + unsigned int val = readl(phy->base + S3C24XX_DSTAT); + return val & S3C24XX_DSTAT_STAT_BUSY; +} + +static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan, + struct s3c24xx_dma_phy *phy) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; + struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; + int phyvalid; + + /* every phy is valid for memcopy channels */ + if (!s3cchan->slave) + return true; + + /* On newer variants all phys can be used for all virtual channels */ + if (s3cdma->sdata->has_reqsel) + return true; + + phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH)); + return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false; +} + +/* + * Allocate a physical channel for a virtual channel + * + * Try to locate a physical channel to be used for this transfer. If all + * are taken return NULL and the requester will have to cope by using + * some fallback PIO mode or retrying later. + */ +static +struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; + struct s3c24xx_dma_channel *cdata; + struct s3c24xx_dma_phy *phy = NULL; + unsigned long flags; + int i; + int ret; + + if (s3cchan->slave) + cdata = &pdata->channels[s3cchan->id]; + + for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) { + phy = &s3cdma->phy_chans[i]; + + if (!phy->valid) + continue; + + if (!s3c24xx_dma_phy_valid(s3cchan, phy)) + continue; + + spin_lock_irqsave(&phy->lock, flags); + + if (!phy->serving) { + phy->serving = s3cchan; + spin_unlock_irqrestore(&phy->lock, flags); + break; + } + + spin_unlock_irqrestore(&phy->lock, flags); + } + + /* No physical channel available, cope with it */ + if (i == s3cdma->pdata->num_phy_channels) { + dev_warn(&s3cdma->pdev->dev, "no phy channel available\n"); + return NULL; + } + + /* start the phy clock */ + if (s3cdma->sdata->has_clocks) { + ret = clk_enable(phy->clk); + if (ret) { + dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n", + phy->id, ret); + phy->serving = NULL; + return NULL; + } + } + + return phy; +} + +/* + * Mark the physical channel as free. + * + * This drops the link between the physical and virtual channel. + */ +static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy) +{ + struct s3c24xx_dma_engine *s3cdma = phy->host; + + if (s3cdma->sdata->has_clocks) + clk_disable(phy->clk); + + phy->serving = NULL; +} + +/* + * Stops the channel by writing the stop bit. + * This should not be used for an on-going transfer, but as a method of + * shutting down a channel (eg, when it's no longer used) or terminating a + * transfer. + */ +static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy) +{ + writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG); +} + +/* + * Virtual channel handling + */ + +static inline +struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan) +{ + return container_of(chan, struct s3c24xx_dma_chan, vc.chan); +} + +static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_phy *phy = s3cchan->phy; + struct s3c24xx_txd *txd = s3cchan->at; + u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK; + + return tc * txd->width; +} + +static int s3c24xx_dma_set_runtime_config(struct s3c24xx_dma_chan *s3cchan, + struct dma_slave_config *config) +{ + if (!s3cchan->slave) + return -EINVAL; + + /* Reject definitely invalid configurations */ + if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES || + config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) + return -EINVAL; + + s3cchan->cfg = *config; + + return 0; +} + +/* + * Transfer handling + */ + +static inline +struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx) +{ + return container_of(tx, struct s3c24xx_txd, vd.tx); +} + +static struct s3c24xx_txd *s3c24xx_dma_get_txd(void) +{ + struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT); + + if (txd) { + INIT_LIST_HEAD(&txd->dsg_list); + txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD; + } + + return txd; +} + +static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd) +{ + struct s3c24xx_sg *dsg, *_dsg; + + list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) { + list_del(&dsg->node); + kfree(dsg); + } + + kfree(txd); +} + +static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan, + struct s3c24xx_txd *txd) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + struct s3c24xx_dma_phy *phy = s3cchan->phy; + const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; + struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node); + u32 dcon = txd->dcon; + u32 val; + + /* transfer-size and -count from len and width */ + switch (txd->width) { + case 1: + dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len; + break; + case 2: + dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2); + break; + case 4: + dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4); + break; + } + + if (s3cchan->slave) { + struct s3c24xx_dma_channel *cdata = + &pdata->channels[s3cchan->id]; + + if (s3cdma->sdata->has_reqsel) { + writel_relaxed((cdata->chansel << 1) | + S3C24XX_DMAREQSEL_HW, + phy->base + S3C24XX_DMAREQSEL); + } else { + int csel = cdata->chansel >> (phy->id * + S3C24XX_CHANSEL_WIDTH); + + csel &= S3C24XX_CHANSEL_REQ_MASK; + dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT; + dcon |= S3C24XX_DCON_HWTRIG; + } + } else { + if (s3cdma->sdata->has_reqsel) + writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL); + } + + writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC); + writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC); + writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST); + writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC); + writel_relaxed(dcon, phy->base + S3C24XX_DCON); + + val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG); + val &= ~S3C24XX_DMASKTRIG_STOP; + val |= S3C24XX_DMASKTRIG_ON; + + /* trigger the dma operation for memcpy transfers */ + if (!s3cchan->slave) + val |= S3C24XX_DMASKTRIG_SWTRIG; + + writel(val, phy->base + S3C24XX_DMASKTRIG); +} + +/* + * Set the initial DMA register values and start first sg. + */ +static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_phy *phy = s3cchan->phy; + struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc); + struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx); + + list_del(&txd->vd.node); + + s3cchan->at = txd; + + /* Wait for channel inactive */ + while (s3c24xx_dma_phy_busy(phy)) + cpu_relax(); + + /* point to the first element of the sg list */ + txd->at = txd->dsg_list.next; + s3c24xx_dma_start_next_sg(s3cchan, txd); +} + +static void s3c24xx_dma_free_txd_list(struct s3c24xx_dma_engine *s3cdma, + struct s3c24xx_dma_chan *s3cchan) +{ + LIST_HEAD(head); + + vchan_get_all_descriptors(&s3cchan->vc, &head); + vchan_dma_desc_free_list(&s3cchan->vc, &head); +} + +/* + * Try to allocate a physical channel. When successful, assign it to + * this virtual channel, and initiate the next descriptor. The + * virtual channel lock must be held at this point. + */ +static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + struct s3c24xx_dma_phy *phy; + + phy = s3c24xx_dma_get_phy(s3cchan); + if (!phy) { + dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n", + s3cchan->name); + s3cchan->state = S3C24XX_DMA_CHAN_WAITING; + return; + } + + dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n", + phy->id, s3cchan->name); + + s3cchan->phy = phy; + s3cchan->state = S3C24XX_DMA_CHAN_RUNNING; + + s3c24xx_dma_start_next_txd(s3cchan); +} + +static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy, + struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + + dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n", + phy->id, s3cchan->name); + + /* + * We do this without taking the lock; we're really only concerned + * about whether this pointer is NULL or not, and we're guaranteed + * that this will only be called when it _already_ is non-NULL. + */ + phy->serving = s3cchan; + s3cchan->phy = phy; + s3cchan->state = S3C24XX_DMA_CHAN_RUNNING; + s3c24xx_dma_start_next_txd(s3cchan); +} + +/* + * Free a physical DMA channel, potentially reallocating it to another + * virtual channel if we have any pending. + */ +static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan) +{ + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + struct s3c24xx_dma_chan *p, *next; + +retry: + next = NULL; + + /* Find a waiting virtual channel for the next transfer. */ + list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node) + if (p->state == S3C24XX_DMA_CHAN_WAITING) { + next = p; + break; + } + + if (!next) { + list_for_each_entry(p, &s3cdma->slave.channels, + vc.chan.device_node) + if (p->state == S3C24XX_DMA_CHAN_WAITING && + s3c24xx_dma_phy_valid(p, s3cchan->phy)) { + next = p; + break; + } + } + + /* Ensure that the physical channel is stopped */ + s3c24xx_dma_terminate_phy(s3cchan->phy); + + if (next) { + bool success; + + /* + * Eww. We know this isn't going to deadlock + * but lockdep probably doesn't. + */ + spin_lock(&next->vc.lock); + /* Re-check the state now that we have the lock */ + success = next->state == S3C24XX_DMA_CHAN_WAITING; + if (success) + s3c24xx_dma_phy_reassign_start(s3cchan->phy, next); + spin_unlock(&next->vc.lock); + + /* If the state changed, try to find another channel */ + if (!success) + goto retry; + } else { + /* No more jobs, so free up the physical channel */ + s3c24xx_dma_put_phy(s3cchan->phy); + } + + s3cchan->phy = NULL; + s3cchan->state = S3C24XX_DMA_CHAN_IDLE; +} + +static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd) +{ + struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx); + struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan); + + if (!s3cchan->slave) + dma_descriptor_unmap(&vd->tx); + + s3c24xx_dma_free_txd(txd); +} + +static irqreturn_t s3c24xx_dma_irq(int irq, void *data) +{ + struct s3c24xx_dma_phy *phy = data; + struct s3c24xx_dma_chan *s3cchan = phy->serving; + struct s3c24xx_txd *txd; + + dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id); + + /* + * Interrupts happen to notify the completion of a transfer and the + * channel should have moved into its stop state already on its own. + * Therefore interrupts on channels not bound to a virtual channel + * should never happen. Nevertheless send a terminate command to the + * channel if the unlikely case happens. + */ + if (unlikely(!s3cchan)) { + dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n", + phy->id); + + s3c24xx_dma_terminate_phy(phy); + + return IRQ_HANDLED; + } + + spin_lock(&s3cchan->vc.lock); + txd = s3cchan->at; + if (txd) { + /* when more sg's are in this txd, start the next one */ + if (!list_is_last(txd->at, &txd->dsg_list)) { + txd->at = txd->at->next; + if (txd->cyclic) + vchan_cyclic_callback(&txd->vd); + s3c24xx_dma_start_next_sg(s3cchan, txd); + } else if (!txd->cyclic) { + s3cchan->at = NULL; + vchan_cookie_complete(&txd->vd); + + /* + * And start the next descriptor (if any), + * otherwise free this channel. + */ + if (vchan_next_desc(&s3cchan->vc)) + s3c24xx_dma_start_next_txd(s3cchan); + else + s3c24xx_dma_phy_free(s3cchan); + } else { + vchan_cyclic_callback(&txd->vd); + + /* Cyclic: reset at beginning */ + txd->at = txd->dsg_list.next; + s3c24xx_dma_start_next_sg(s3cchan, txd); + } + } + spin_unlock(&s3cchan->vc.lock); + + return IRQ_HANDLED; +} + +/* + * The DMA ENGINE API + */ + +static int s3c24xx_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, + unsigned long arg) +{ + struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&s3cchan->vc.lock, flags); + + switch (cmd) { + case DMA_SLAVE_CONFIG: + ret = s3c24xx_dma_set_runtime_config(s3cchan, + (struct dma_slave_config *)arg); + break; + case DMA_TERMINATE_ALL: + if (!s3cchan->phy && !s3cchan->at) { + dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n", + s3cchan->id); + ret = -EINVAL; + break; + } + + s3cchan->state = S3C24XX_DMA_CHAN_IDLE; + + /* Mark physical channel as free */ + if (s3cchan->phy) + s3c24xx_dma_phy_free(s3cchan); + + /* Dequeue current job */ + if (s3cchan->at) { + s3c24xx_dma_desc_free(&s3cchan->at->vd); + s3cchan->at = NULL; + } + + /* Dequeue jobs not yet fired as well */ + s3c24xx_dma_free_txd_list(s3cdma, s3cchan); + break; + default: + /* Unknown command */ + ret = -ENXIO; + break; + } + + spin_unlock_irqrestore(&s3cchan->vc.lock, flags); + + return ret; +} + +static int s3c24xx_dma_alloc_chan_resources(struct dma_chan *chan) +{ + return 0; +} + +static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan) +{ + /* Ensure all queued descriptors are freed */ + vchan_free_chan_resources(to_virt_chan(chan)); +} + +static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan, + dma_cookie_t cookie, struct dma_tx_state *txstate) +{ + struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); + struct s3c24xx_txd *txd; + struct s3c24xx_sg *dsg; + struct virt_dma_desc *vd; + unsigned long flags; + enum dma_status ret; + size_t bytes = 0; + + spin_lock_irqsave(&s3cchan->vc.lock, flags); + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) { + spin_unlock_irqrestore(&s3cchan->vc.lock, flags); + return ret; + } + + /* + * There's no point calculating the residue if there's + * no txstate to store the value. + */ + if (!txstate) { + spin_unlock_irqrestore(&s3cchan->vc.lock, flags); + return ret; + } + + vd = vchan_find_desc(&s3cchan->vc, cookie); + if (vd) { + /* On the issued list, so hasn't been processed yet */ + txd = to_s3c24xx_txd(&vd->tx); + + list_for_each_entry(dsg, &txd->dsg_list, node) + bytes += dsg->len; + } else { + /* + * Currently running, so sum over the pending sg's and + * the currently active one. + */ + txd = s3cchan->at; + + dsg = list_entry(txd->at, struct s3c24xx_sg, node); + list_for_each_entry_from(dsg, &txd->dsg_list, node) + bytes += dsg->len; + + bytes += s3c24xx_dma_getbytes_chan(s3cchan); + } + spin_unlock_irqrestore(&s3cchan->vc.lock, flags); + + /* + * This cookie not complete yet + * Get number of bytes left in the active transactions and queue + */ + dma_set_residue(txstate, bytes); + + /* Whether waiting or running, we're in progress */ + return ret; +} + +/* + * Initialize a descriptor to be used by memcpy submit + */ +static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy( + struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, + size_t len, unsigned long flags) +{ + struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + struct s3c24xx_txd *txd; + struct s3c24xx_sg *dsg; + int src_mod, dest_mod; + + dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %d bytes from %s\n", + len, s3cchan->name); + + if ((len & S3C24XX_DCON_TC_MASK) != len) { + dev_err(&s3cdma->pdev->dev, "memcpy size %d to large\n", len); + return NULL; + } + + txd = s3c24xx_dma_get_txd(); + if (!txd) + return NULL; + + dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT); + if (!dsg) { + s3c24xx_dma_free_txd(txd); + return NULL; + } + list_add_tail(&dsg->node, &txd->dsg_list); + + dsg->src_addr = src; + dsg->dst_addr = dest; + dsg->len = len; + + /* + * Determine a suitable transfer width. + * The DMA controller cannot fetch/store information which is not + * naturally aligned on the bus, i.e., a 4 byte fetch must start at + * an address divisible by 4 - more generally addr % width must be 0. + */ + src_mod = src % 4; + dest_mod = dest % 4; + switch (len % 4) { + case 0: + txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1; + break; + case 2: + txd->width = ((src_mod == 2 || src_mod == 0) && + (dest_mod == 2 || dest_mod == 0)) ? 2 : 1; + break; + default: + txd->width = 1; + break; + } + + txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT; + txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT; + txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK | + S3C24XX_DCON_SERV_WHOLE; + + return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags); +} + +static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period, + enum dma_transfer_direction direction, unsigned long flags, + void *context) +{ + struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; + struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; + struct s3c24xx_txd *txd; + struct s3c24xx_sg *dsg; + unsigned sg_len; + dma_addr_t slave_addr; + u32 hwcfg = 0; + int i; + + dev_dbg(&s3cdma->pdev->dev, + "prepare cyclic transaction of %zu bytes with period %zu from %s\n", + size, period, s3cchan->name); + + if (!is_slave_direction(direction)) { + dev_err(&s3cdma->pdev->dev, + "direction %d unsupported\n", direction); + return NULL; + } + + txd = s3c24xx_dma_get_txd(); + if (!txd) + return NULL; + + txd->cyclic = 1; + + if (cdata->handshake) + txd->dcon |= S3C24XX_DCON_HANDSHAKE; + + switch (cdata->bus) { + case S3C24XX_DMA_APB: + txd->dcon |= S3C24XX_DCON_SYNC_PCLK; + hwcfg |= S3C24XX_DISRCC_LOC_APB; + break; + case S3C24XX_DMA_AHB: + txd->dcon |= S3C24XX_DCON_SYNC_HCLK; + hwcfg |= S3C24XX_DISRCC_LOC_AHB; + break; + } + + /* + * Always assume our peripheral desintation is a fixed + * address in memory. + */ + hwcfg |= S3C24XX_DISRCC_INC_FIXED; + + /* + * Individual dma operations are requested by the slave, + * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE). + */ + txd->dcon |= S3C24XX_DCON_SERV_SINGLE; + + if (direction == DMA_MEM_TO_DEV) { + txd->disrcc = S3C24XX_DISRCC_LOC_AHB | + S3C24XX_DISRCC_INC_INCREMENT; + txd->didstc = hwcfg; + slave_addr = s3cchan->cfg.dst_addr; + txd->width = s3cchan->cfg.dst_addr_width; + } else { + txd->disrcc = hwcfg; + txd->didstc = S3C24XX_DIDSTC_LOC_AHB | + S3C24XX_DIDSTC_INC_INCREMENT; + slave_addr = s3cchan->cfg.src_addr; + txd->width = s3cchan->cfg.src_addr_width; + } + + sg_len = size / period; + + for (i = 0; i < sg_len; i++) { + dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT); + if (!dsg) { + s3c24xx_dma_free_txd(txd); + return NULL; + } + list_add_tail(&dsg->node, &txd->dsg_list); + + dsg->len = period; + /* Check last period length */ + if (i == sg_len - 1) + dsg->len = size - period * i; + if (direction == DMA_MEM_TO_DEV) { + dsg->src_addr = addr + period * i; + dsg->dst_addr = slave_addr; + } else { /* DMA_DEV_TO_MEM */ + dsg->src_addr = slave_addr; + dsg->dst_addr = addr + period * i; + } + } + + return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags); +} + +static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); + struct s3c24xx_dma_engine *s3cdma = s3cchan->host; + const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata; + struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id]; + struct s3c24xx_txd *txd; + struct s3c24xx_sg *dsg; + struct scatterlist *sg; + dma_addr_t slave_addr; + u32 hwcfg = 0; + int tmp; + + dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n", + sg_dma_len(sgl), s3cchan->name); + + txd = s3c24xx_dma_get_txd(); + if (!txd) + return NULL; + + if (cdata->handshake) + txd->dcon |= S3C24XX_DCON_HANDSHAKE; + + switch (cdata->bus) { + case S3C24XX_DMA_APB: + txd->dcon |= S3C24XX_DCON_SYNC_PCLK; + hwcfg |= S3C24XX_DISRCC_LOC_APB; + break; + case S3C24XX_DMA_AHB: + txd->dcon |= S3C24XX_DCON_SYNC_HCLK; + hwcfg |= S3C24XX_DISRCC_LOC_AHB; + break; + } + + /* + * Always assume our peripheral desintation is a fixed + * address in memory. + */ + hwcfg |= S3C24XX_DISRCC_INC_FIXED; + + /* + * Individual dma operations are requested by the slave, + * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE). + */ + txd->dcon |= S3C24XX_DCON_SERV_SINGLE; + + if (direction == DMA_MEM_TO_DEV) { + txd->disrcc = S3C24XX_DISRCC_LOC_AHB | + S3C24XX_DISRCC_INC_INCREMENT; + txd->didstc = hwcfg; + slave_addr = s3cchan->cfg.dst_addr; + txd->width = s3cchan->cfg.dst_addr_width; + } else if (direction == DMA_DEV_TO_MEM) { + txd->disrcc = hwcfg; + txd->didstc = S3C24XX_DIDSTC_LOC_AHB | + S3C24XX_DIDSTC_INC_INCREMENT; + slave_addr = s3cchan->cfg.src_addr; + txd->width = s3cchan->cfg.src_addr_width; + } else { + s3c24xx_dma_free_txd(txd); + dev_err(&s3cdma->pdev->dev, + "direction %d unsupported\n", direction); + return NULL; + } + + for_each_sg(sgl, sg, sg_len, tmp) { + dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT); + if (!dsg) { + s3c24xx_dma_free_txd(txd); + return NULL; + } + list_add_tail(&dsg->node, &txd->dsg_list); + + dsg->len = sg_dma_len(sg); + if (direction == DMA_MEM_TO_DEV) { + dsg->src_addr = sg_dma_address(sg); + dsg->dst_addr = slave_addr; + } else { /* DMA_DEV_TO_MEM */ + dsg->src_addr = slave_addr; + dsg->dst_addr = sg_dma_address(sg); + } + } + + return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags); +} + +/* + * Slave transactions callback to the slave device to allow + * synchronization of slave DMA signals with the DMAC enable + */ +static void s3c24xx_dma_issue_pending(struct dma_chan *chan) +{ + struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan); + unsigned long flags; + + spin_lock_irqsave(&s3cchan->vc.lock, flags); + if (vchan_issue_pending(&s3cchan->vc)) { + if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING) + s3c24xx_dma_phy_alloc_and_start(s3cchan); + } + spin_unlock_irqrestore(&s3cchan->vc.lock, flags); +} + +/* + * Bringup and teardown + */ + +/* + * Initialise the DMAC memcpy/slave channels. + * Make a local wrapper to hold required data + */ +static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma, + struct dma_device *dmadev, unsigned int channels, bool slave) +{ + struct s3c24xx_dma_chan *chan; + int i; + + INIT_LIST_HEAD(&dmadev->channels); + + /* + * Register as many many memcpy as we have physical channels, + * we won't always be able to use all but the code will have + * to cope with that situation. + */ + for (i = 0; i < channels; i++) { + chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL); + if (!chan) { + dev_err(dmadev->dev, + "%s no memory for channel\n", __func__); + return -ENOMEM; + } + + chan->id = i; + chan->host = s3cdma; + chan->state = S3C24XX_DMA_CHAN_IDLE; + + if (slave) { + chan->slave = true; + chan->name = kasprintf(GFP_KERNEL, "slave%d", i); + if (!chan->name) + return -ENOMEM; + } else { + chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i); + if (!chan->name) + return -ENOMEM; + } + dev_dbg(dmadev->dev, + "initialize virtual channel \"%s\"\n", + chan->name); + + chan->vc.desc_free = s3c24xx_dma_desc_free; + vchan_init(&chan->vc, dmadev); + } + dev_info(dmadev->dev, "initialized %d virtual %s channels\n", + i, slave ? "slave" : "memcpy"); + return i; +} + +static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev) +{ + struct s3c24xx_dma_chan *chan = NULL; + struct s3c24xx_dma_chan *next; + + list_for_each_entry_safe(chan, + next, &dmadev->channels, vc.chan.device_node) + list_del(&chan->vc.chan.device_node); +} + +/* s3c2410, s3c2440 and s3c2442 have a 0x40 stride without separate clocks */ +static struct soc_data soc_s3c2410 = { + .stride = 0x40, + .has_reqsel = false, + .has_clocks = false, +}; + +/* s3c2412 and s3c2413 have a 0x40 stride and dmareqsel mechanism */ +static struct soc_data soc_s3c2412 = { + .stride = 0x40, + .has_reqsel = true, + .has_clocks = true, +}; + +/* s3c2443 and following have a 0x100 stride and dmareqsel mechanism */ +static struct soc_data soc_s3c2443 = { + .stride = 0x100, + .has_reqsel = true, + .has_clocks = true, +}; + +static struct platform_device_id s3c24xx_dma_driver_ids[] = { + { + .name = "s3c2410-dma", + .driver_data = (kernel_ulong_t)&soc_s3c2410, + }, { + .name = "s3c2412-dma", + .driver_data = (kernel_ulong_t)&soc_s3c2412, + }, { + .name = "s3c2443-dma", + .driver_data = (kernel_ulong_t)&soc_s3c2443, + }, + { }, +}; + +static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev) +{ + return (struct soc_data *) + platform_get_device_id(pdev)->driver_data; +} + +static int s3c24xx_dma_probe(struct platform_device *pdev) +{ + const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev); + struct s3c24xx_dma_engine *s3cdma; + struct soc_data *sdata; + struct resource *res; + int ret; + int i; + + if (!pdata) { + dev_err(&pdev->dev, "platform data missing\n"); + return -ENODEV; + } + + /* Basic sanity check */ + if (pdata->num_phy_channels > MAX_DMA_CHANNELS) { + dev_err(&pdev->dev, "to many dma channels %d, max %d\n", + pdata->num_phy_channels, MAX_DMA_CHANNELS); + return -EINVAL; + } + + sdata = s3c24xx_dma_get_soc_data(pdev); + if (!sdata) + return -EINVAL; + + s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL); + if (!s3cdma) + return -ENOMEM; + + s3cdma->pdev = pdev; + s3cdma->pdata = pdata; + s3cdma->sdata = sdata; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + s3cdma->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(s3cdma->base)) + return PTR_ERR(s3cdma->base); + + s3cdma->phy_chans = devm_kzalloc(&pdev->dev, + sizeof(struct s3c24xx_dma_phy) * + pdata->num_phy_channels, + GFP_KERNEL); + if (!s3cdma->phy_chans) + return -ENOMEM; + + /* aquire irqs and clocks for all physical channels */ + for (i = 0; i < pdata->num_phy_channels; i++) { + struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; + char clk_name[6]; + + phy->id = i; + phy->base = s3cdma->base + (i * sdata->stride); + phy->host = s3cdma; + + phy->irq = platform_get_irq(pdev, i); + if (phy->irq < 0) { + dev_err(&pdev->dev, "failed to get irq %d, err %d\n", + i, phy->irq); + continue; + } + + ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq, + 0, pdev->name, phy); + if (ret) { + dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n", + i, ret); + continue; + } + + if (sdata->has_clocks) { + sprintf(clk_name, "dma.%d", i); + phy->clk = devm_clk_get(&pdev->dev, clk_name); + if (IS_ERR(phy->clk) && sdata->has_clocks) { + dev_err(&pdev->dev, "unable to aquire clock for channel %d, error %lu", + i, PTR_ERR(phy->clk)); + continue; + } + + ret = clk_prepare(phy->clk); + if (ret) { + dev_err(&pdev->dev, "clock for phy %d failed, error %d\n", + i, ret); + continue; + } + } + + spin_lock_init(&phy->lock); + phy->valid = true; + + dev_dbg(&pdev->dev, "physical channel %d is %s\n", + i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE"); + } + + /* Initialize memcpy engine */ + dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask); + dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask); + s3cdma->memcpy.dev = &pdev->dev; + s3cdma->memcpy.device_alloc_chan_resources = + s3c24xx_dma_alloc_chan_resources; + s3cdma->memcpy.device_free_chan_resources = + s3c24xx_dma_free_chan_resources; + s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy; + s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status; + s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending; + s3cdma->memcpy.device_control = s3c24xx_dma_control; + + /* Initialize slave engine for SoC internal dedicated peripherals */ + dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask); + dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask); + dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask); + s3cdma->slave.dev = &pdev->dev; + s3cdma->slave.device_alloc_chan_resources = + s3c24xx_dma_alloc_chan_resources; + s3cdma->slave.device_free_chan_resources = + s3c24xx_dma_free_chan_resources; + s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status; + s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending; + s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg; + s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic; + s3cdma->slave.device_control = s3c24xx_dma_control; + + /* Register as many memcpy channels as there are physical channels */ + ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy, + pdata->num_phy_channels, false); + if (ret <= 0) { + dev_warn(&pdev->dev, + "%s failed to enumerate memcpy channels - %d\n", + __func__, ret); + goto err_memcpy; + } + + /* Register slave channels */ + ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave, + pdata->num_channels, true); + if (ret <= 0) { + dev_warn(&pdev->dev, + "%s failed to enumerate slave channels - %d\n", + __func__, ret); + goto err_slave; + } + + ret = dma_async_device_register(&s3cdma->memcpy); + if (ret) { + dev_warn(&pdev->dev, + "%s failed to register memcpy as an async device - %d\n", + __func__, ret); + goto err_memcpy_reg; + } + + ret = dma_async_device_register(&s3cdma->slave); + if (ret) { + dev_warn(&pdev->dev, + "%s failed to register slave as an async device - %d\n", + __func__, ret); + goto err_slave_reg; + } + + platform_set_drvdata(pdev, s3cdma); + dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n", + pdata->num_phy_channels); + + return 0; + +err_slave_reg: + dma_async_device_unregister(&s3cdma->memcpy); +err_memcpy_reg: + s3c24xx_dma_free_virtual_channels(&s3cdma->slave); +err_slave: + s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy); +err_memcpy: + if (sdata->has_clocks) + for (i = 0; i < pdata->num_phy_channels; i++) { + struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; + if (phy->valid) + clk_unprepare(phy->clk); + } + + return ret; +} + +static int s3c24xx_dma_remove(struct platform_device *pdev) +{ + const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev); + struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev); + struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev); + int i; + + dma_async_device_unregister(&s3cdma->slave); + dma_async_device_unregister(&s3cdma->memcpy); + + s3c24xx_dma_free_virtual_channels(&s3cdma->slave); + s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy); + + if (sdata->has_clocks) + for (i = 0; i < pdata->num_phy_channels; i++) { + struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i]; + if (phy->valid) + clk_unprepare(phy->clk); + } + + return 0; +} + +static struct platform_driver s3c24xx_dma_driver = { + .driver = { + .name = "s3c24xx-dma", + .owner = THIS_MODULE, + }, + .id_table = s3c24xx_dma_driver_ids, + .probe = s3c24xx_dma_probe, + .remove = s3c24xx_dma_remove, +}; + +module_platform_driver(s3c24xx_dma_driver); + +bool s3c24xx_dma_filter(struct dma_chan *chan, void *param) +{ + struct s3c24xx_dma_chan *s3cchan; + + if (chan->device->dev->driver != &s3c24xx_dma_driver.driver) + return false; + + s3cchan = to_s3c24xx_dma_chan(chan); + + return s3cchan->id == (int)param; +} +EXPORT_SYMBOL(s3c24xx_dma_filter); + +MODULE_DESCRIPTION("S3C24XX DMA Driver"); +MODULE_AUTHOR("Heiko Stuebner"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/dma/sa11x0-dma.c b/drivers/dma/sa11x0-dma.c index 461a91ab70b..5ebdfbc1051 100644 --- a/drivers/dma/sa11x0-dma.c +++ b/drivers/dma/sa11x0-dma.c @@ -113,11 +113,9 @@ struct sa11x0_dma_phy { struct sa11x0_dma_desc *txd_load; unsigned sg_done; struct sa11x0_dma_desc *txd_done; -#ifdef CONFIG_PM_SLEEP u32 dbs[2]; u32 dbt[2]; u32 dcsr; -#endif }; struct sa11x0_dma_dev { @@ -436,7 +434,7 @@ static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan, enum dma_status ret; ret = dma_cookie_status(&c->vc.chan, cookie, state); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; if (!state) @@ -984,7 +982,6 @@ static int sa11x0_dma_remove(struct platform_device *pdev) return 0; } -#ifdef CONFIG_PM_SLEEP static int sa11x0_dma_suspend(struct device *dev) { struct sa11x0_dma_dev *d = dev_get_drvdata(dev); @@ -1054,7 +1051,6 @@ static int sa11x0_dma_resume(struct device *dev) return 0; } -#endif static const struct dev_pm_ops sa11x0_dma_pm_ops = { .suspend_noirq = sa11x0_dma_suspend, diff --git a/drivers/dma/sh/Kconfig b/drivers/dma/sh/Kconfig index dadd9e010c0..0f719816c91 100644 --- a/drivers/dma/sh/Kconfig +++ b/drivers/dma/sh/Kconfig @@ -4,7 +4,7 @@ config SH_DMAE_BASE bool "Renesas SuperH DMA Engine support" - depends on (SUPERH && SH_DMA) || (ARM && ARCH_SHMOBILE) + depends on (SUPERH && SH_DMA) || ARCH_SHMOBILE || COMPILE_TEST depends on !SH_DMA_API default y select DMA_ENGINE @@ -29,6 +29,12 @@ config RCAR_HPB_DMAE help Enable support for the Renesas R-Car series DMA controllers. +config RCAR_AUDMAC_PP + tristate "Renesas R-Car Audio DMAC Peripheral Peripheral support" + depends on SH_DMAE_BASE + help + Enable support for the Renesas R-Car Audio DMAC Peripheral Peripheral controllers. + config SHDMA_R8A73A4 def_bool y depends on ARCH_R8A73A4 && SH_DMAE != n diff --git a/drivers/dma/sh/Makefile b/drivers/dma/sh/Makefile index e856af23b78..1ce88b28cfc 100644 --- a/drivers/dma/sh/Makefile +++ b/drivers/dma/sh/Makefile @@ -7,3 +7,4 @@ endif shdma-objs := $(shdma-y) obj-$(CONFIG_SUDMAC) += sudmac.o obj-$(CONFIG_RCAR_HPB_DMAE) += rcar-hpbdma.o +obj-$(CONFIG_RCAR_AUDMAC_PP) += rcar-audmapp.o diff --git a/drivers/dma/sh/rcar-audmapp.c b/drivers/dma/sh/rcar-audmapp.c new file mode 100644 index 00000000000..2de77289a2e --- /dev/null +++ b/drivers/dma/sh/rcar-audmapp.c @@ -0,0 +1,320 @@ +/* + * This is for Renesas R-Car Audio-DMAC-peri-peri. + * + * Copyright (C) 2014 Renesas Electronics Corporation + * Copyright (C) 2014 Kuninori Morimoto <kuninori.morimoto.gx@renesas.com> + * + * based on the drivers/dma/sh/shdma.c + * + * Copyright (C) 2011-2012 Guennadi Liakhovetski <g.liakhovetski@gmx.de> + * Copyright (C) 2009 Nobuhiro Iwamatsu <iwamatsu.nobuhiro@renesas.com> + * Copyright (C) 2009 Renesas Solutions, Inc. All rights reserved. + * Copyright (C) 2007 Freescale Semiconductor, Inc. All rights reserved. + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + */ +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/dmaengine.h> +#include <linux/platform_data/dma-rcar-audmapp.h> +#include <linux/platform_device.h> +#include <linux/shdma-base.h> + +/* + * DMA register + */ +#define PDMASAR 0x00 +#define PDMADAR 0x04 +#define PDMACHCR 0x0c + +/* PDMACHCR */ +#define PDMACHCR_DE (1 << 0) + +#define AUDMAPP_MAX_CHANNELS 29 + +/* Default MEMCPY transfer size = 2^2 = 4 bytes */ +#define LOG2_DEFAULT_XFER_SIZE 2 +#define AUDMAPP_SLAVE_NUMBER 256 +#define AUDMAPP_LEN_MAX (16 * 1024 * 1024) + +struct audmapp_chan { + struct shdma_chan shdma_chan; + struct audmapp_slave_config *config; + void __iomem *base; +}; + +struct audmapp_device { + struct shdma_dev shdma_dev; + struct audmapp_pdata *pdata; + struct device *dev; + void __iomem *chan_reg; +}; + +#define to_chan(chan) container_of(chan, struct audmapp_chan, shdma_chan) +#define to_dev(chan) container_of(chan->shdma_chan.dma_chan.device, \ + struct audmapp_device, shdma_dev.dma_dev) + +static void audmapp_write(struct audmapp_chan *auchan, u32 data, u32 reg) +{ + struct audmapp_device *audev = to_dev(auchan); + struct device *dev = audev->dev; + + dev_dbg(dev, "w %p : %08x\n", auchan->base + reg, data); + + iowrite32(data, auchan->base + reg); +} + +static u32 audmapp_read(struct audmapp_chan *auchan, u32 reg) +{ + return ioread32(auchan->base + reg); +} + +static void audmapp_halt(struct shdma_chan *schan) +{ + struct audmapp_chan *auchan = to_chan(schan); + int i; + + audmapp_write(auchan, 0, PDMACHCR); + + for (i = 0; i < 1024; i++) { + if (0 == audmapp_read(auchan, PDMACHCR)) + return; + udelay(1); + } +} + +static void audmapp_start_xfer(struct shdma_chan *schan, + struct shdma_desc *sdecs) +{ + struct audmapp_chan *auchan = to_chan(schan); + struct audmapp_device *audev = to_dev(auchan); + struct audmapp_slave_config *cfg = auchan->config; + struct device *dev = audev->dev; + u32 chcr = cfg->chcr | PDMACHCR_DE; + + dev_dbg(dev, "src/dst/chcr = %pad/%pad/%x\n", + &cfg->src, &cfg->dst, cfg->chcr); + + audmapp_write(auchan, cfg->src, PDMASAR); + audmapp_write(auchan, cfg->dst, PDMADAR); + audmapp_write(auchan, chcr, PDMACHCR); +} + +static struct audmapp_slave_config * +audmapp_find_slave(struct audmapp_chan *auchan, int slave_id) +{ + struct audmapp_device *audev = to_dev(auchan); + struct audmapp_pdata *pdata = audev->pdata; + struct audmapp_slave_config *cfg; + int i; + + if (slave_id >= AUDMAPP_SLAVE_NUMBER) + return NULL; + + for (i = 0, cfg = pdata->slave; i < pdata->slave_num; i++, cfg++) + if (cfg->slave_id == slave_id) + return cfg; + + return NULL; +} + +static int audmapp_set_slave(struct shdma_chan *schan, int slave_id, + dma_addr_t slave_addr, bool try) +{ + struct audmapp_chan *auchan = to_chan(schan); + struct audmapp_slave_config *cfg = + audmapp_find_slave(auchan, slave_id); + + if (!cfg) + return -ENODEV; + if (try) + return 0; + + auchan->config = cfg; + + return 0; +} + +static int audmapp_desc_setup(struct shdma_chan *schan, + struct shdma_desc *sdecs, + dma_addr_t src, dma_addr_t dst, size_t *len) +{ + struct audmapp_chan *auchan = to_chan(schan); + struct audmapp_slave_config *cfg = auchan->config; + + if (!cfg) + return -ENODEV; + + if (*len > (size_t)AUDMAPP_LEN_MAX) + *len = (size_t)AUDMAPP_LEN_MAX; + + return 0; +} + +static void audmapp_setup_xfer(struct shdma_chan *schan, + int slave_id) +{ +} + +static dma_addr_t audmapp_slave_addr(struct shdma_chan *schan) +{ + return 0; /* always fixed address */ +} + +static bool audmapp_channel_busy(struct shdma_chan *schan) +{ + struct audmapp_chan *auchan = to_chan(schan); + u32 chcr = audmapp_read(auchan, PDMACHCR); + + return chcr & ~PDMACHCR_DE; +} + +static bool audmapp_desc_completed(struct shdma_chan *schan, + struct shdma_desc *sdesc) +{ + return true; +} + +static struct shdma_desc *audmapp_embedded_desc(void *buf, int i) +{ + return &((struct shdma_desc *)buf)[i]; +} + +static const struct shdma_ops audmapp_shdma_ops = { + .halt_channel = audmapp_halt, + .desc_setup = audmapp_desc_setup, + .set_slave = audmapp_set_slave, + .start_xfer = audmapp_start_xfer, + .embedded_desc = audmapp_embedded_desc, + .setup_xfer = audmapp_setup_xfer, + .slave_addr = audmapp_slave_addr, + .channel_busy = audmapp_channel_busy, + .desc_completed = audmapp_desc_completed, +}; + +static int audmapp_chan_probe(struct platform_device *pdev, + struct audmapp_device *audev, int id) +{ + struct shdma_dev *sdev = &audev->shdma_dev; + struct audmapp_chan *auchan; + struct shdma_chan *schan; + struct device *dev = audev->dev; + + auchan = devm_kzalloc(dev, sizeof(*auchan), GFP_KERNEL); + if (!auchan) + return -ENOMEM; + + schan = &auchan->shdma_chan; + schan->max_xfer_len = AUDMAPP_LEN_MAX; + + shdma_chan_probe(sdev, schan, id); + + auchan->base = audev->chan_reg + 0x20 + (0x10 * id); + dev_dbg(dev, "%02d : %p / %p", id, auchan->base, audev->chan_reg); + + return 0; +} + +static void audmapp_chan_remove(struct audmapp_device *audev) +{ + struct dma_device *dma_dev = &audev->shdma_dev.dma_dev; + struct shdma_chan *schan; + int i; + + shdma_for_each_chan(schan, &audev->shdma_dev, i) { + BUG_ON(!schan); + shdma_chan_remove(schan); + } + dma_dev->chancnt = 0; +} + +static int audmapp_probe(struct platform_device *pdev) +{ + struct audmapp_pdata *pdata = pdev->dev.platform_data; + struct audmapp_device *audev; + struct shdma_dev *sdev; + struct dma_device *dma_dev; + struct resource *res; + int err, i; + + if (!pdata) + return -ENODEV; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + + audev = devm_kzalloc(&pdev->dev, sizeof(*audev), GFP_KERNEL); + if (!audev) + return -ENOMEM; + + audev->dev = &pdev->dev; + audev->pdata = pdata; + audev->chan_reg = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(audev->chan_reg)) + return PTR_ERR(audev->chan_reg); + + sdev = &audev->shdma_dev; + sdev->ops = &audmapp_shdma_ops; + sdev->desc_size = sizeof(struct shdma_desc); + + dma_dev = &sdev->dma_dev; + dma_dev->copy_align = LOG2_DEFAULT_XFER_SIZE; + dma_cap_set(DMA_SLAVE, dma_dev->cap_mask); + + err = shdma_init(&pdev->dev, sdev, AUDMAPP_MAX_CHANNELS); + if (err < 0) + return err; + + platform_set_drvdata(pdev, audev); + + /* Create DMA Channel */ + for (i = 0; i < AUDMAPP_MAX_CHANNELS; i++) { + err = audmapp_chan_probe(pdev, audev, i); + if (err) + goto chan_probe_err; + } + + err = dma_async_device_register(dma_dev); + if (err < 0) + goto chan_probe_err; + + return err; + +chan_probe_err: + audmapp_chan_remove(audev); + shdma_cleanup(sdev); + + return err; +} + +static int audmapp_remove(struct platform_device *pdev) +{ + struct audmapp_device *audev = platform_get_drvdata(pdev); + struct dma_device *dma_dev = &audev->shdma_dev.dma_dev; + + dma_async_device_unregister(dma_dev); + + audmapp_chan_remove(audev); + shdma_cleanup(&audev->shdma_dev); + + return 0; +} + +static struct platform_driver audmapp_driver = { + .probe = audmapp_probe, + .remove = audmapp_remove, + .driver = { + .owner = THIS_MODULE, + .name = "rcar-audmapp-engine", + }, +}; +module_platform_driver(audmapp_driver); + +MODULE_AUTHOR("Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>"); +MODULE_DESCRIPTION("Renesas R-Car Audio DMAC peri-peri driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/sh/rcar-hpbdma.c b/drivers/dma/sh/rcar-hpbdma.c index 45a520281ce..b212d9471ab 100644 --- a/drivers/dma/sh/rcar-hpbdma.c +++ b/drivers/dma/sh/rcar-hpbdma.c @@ -18,6 +18,7 @@ #include <linux/dmaengine.h> #include <linux/delay.h> +#include <linux/err.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/module.h> @@ -60,6 +61,7 @@ #define HPB_DMAE_DSTPR_DMSTP BIT(0) /* DMA status register (DSTSR) bits */ +#define HPB_DMAE_DSTSR_DQSTS BIT(2) #define HPB_DMAE_DSTSR_DMSTS BIT(0) /* DMA common registers */ @@ -93,6 +95,7 @@ struct hpb_dmae_chan { void __iomem *base; const struct hpb_dmae_slave_config *cfg; char dev_id[16]; /* unique name per DMAC of channel */ + dma_addr_t slave_addr; }; struct hpb_dmae_device { @@ -285,6 +288,9 @@ static void hpb_dmae_halt(struct shdma_chan *schan) ch_reg_write(chan, HPB_DMAE_DCMDR_DQEND, HPB_DMAE_DCMDR); ch_reg_write(chan, HPB_DMAE_DSTPR_DMSTP, HPB_DMAE_DSTPR); + + chan->plane_idx = 0; + chan->first_desc = true; } static const struct hpb_dmae_slave_config * @@ -384,7 +390,10 @@ static bool hpb_dmae_channel_busy(struct shdma_chan *schan) struct hpb_dmae_chan *chan = to_chan(schan); u32 dstsr = ch_reg_read(chan, HPB_DMAE_DSTSR); - return (dstsr & HPB_DMAE_DSTSR_DMSTS) == HPB_DMAE_DSTSR_DMSTS; + if (chan->xfer_mode == XFER_DOUBLE) + return dstsr & HPB_DMAE_DSTSR_DQSTS; + else + return dstsr & HPB_DMAE_DSTSR_DMSTS; } static int @@ -432,7 +441,6 @@ hpb_dmae_alloc_chan_resources(struct hpb_dmae_chan *hpb_chan, hpb_chan->xfer_mode = XFER_DOUBLE; } else { dev_err(hpb_chan->shdma_chan.dev, "DCR setting error"); - shdma_free_irq(&hpb_chan->shdma_chan); return -EINVAL; } @@ -446,7 +454,8 @@ hpb_dmae_alloc_chan_resources(struct hpb_dmae_chan *hpb_chan, return 0; } -static int hpb_dmae_set_slave(struct shdma_chan *schan, int slave_id, bool try) +static int hpb_dmae_set_slave(struct shdma_chan *schan, int slave_id, + dma_addr_t slave_addr, bool try) { struct hpb_dmae_chan *chan = to_chan(schan); const struct hpb_dmae_slave_config *sc = @@ -457,6 +466,7 @@ static int hpb_dmae_set_slave(struct shdma_chan *schan, int slave_id, bool try) if (try) return 0; chan->cfg = sc; + chan->slave_addr = slave_addr ? : sc->addr; return hpb_dmae_alloc_chan_resources(chan, sc); } @@ -468,7 +478,7 @@ static dma_addr_t hpb_dmae_slave_addr(struct shdma_chan *schan) { struct hpb_dmae_chan *chan = to_chan(schan); - return chan->cfg->addr; + return chan->slave_addr; } static struct shdma_desc *hpb_dmae_embedded_desc(void *buf, int i) @@ -508,6 +518,8 @@ static int hpb_dmae_chan_probe(struct hpb_dmae_device *hpbdev, int id) } schan = &new_hpb_chan->shdma_chan; + schan->max_xfer_len = HPB_DMA_TCR_MAX; + shdma_chan_probe(sdev, schan, id); if (pdev->id >= 0) @@ -614,7 +626,6 @@ static void hpb_dmae_chan_remove(struct hpb_dmae_device *hpbdev) shdma_for_each_chan(schan, &hpbdev->shdma_dev, i) { BUG_ON(!schan); - shdma_free_irq(schan); shdma_chan_remove(schan); } dma_dev->chancnt = 0; diff --git a/drivers/dma/sh/shdma-base.c b/drivers/dma/sh/shdma-base.c index d94ab592cc1..b35007e21e6 100644 --- a/drivers/dma/sh/shdma-base.c +++ b/drivers/dma/sh/shdma-base.c @@ -73,8 +73,7 @@ static void shdma_chan_xfer_ld_queue(struct shdma_chan *schan) static dma_cookie_t shdma_tx_submit(struct dma_async_tx_descriptor *tx) { struct shdma_desc *chunk, *c, *desc = - container_of(tx, struct shdma_desc, async_tx), - *last = desc; + container_of(tx, struct shdma_desc, async_tx); struct shdma_chan *schan = to_shdma_chan(tx->chan); dma_async_tx_callback callback = tx->callback; dma_cookie_t cookie; @@ -98,19 +97,20 @@ static dma_cookie_t shdma_tx_submit(struct dma_async_tx_descriptor *tx) &chunk->node == &schan->ld_free)) break; chunk->mark = DESC_SUBMITTED; - /* Callback goes to the last chunk */ - chunk->async_tx.callback = NULL; + if (chunk->chunks == 1) { + chunk->async_tx.callback = callback; + chunk->async_tx.callback_param = tx->callback_param; + } else { + /* Callback goes to the last chunk */ + chunk->async_tx.callback = NULL; + } chunk->cookie = cookie; list_move_tail(&chunk->node, &schan->ld_queue); - last = chunk; dev_dbg(schan->dev, "submit #%d@%p on %d\n", - tx->cookie, &last->async_tx, schan->id); + tx->cookie, &chunk->async_tx, schan->id); } - last->async_tx.callback = callback; - last->async_tx.callback_param = tx->callback_param; - if (power_up) { int ret; schan->pm_state = SHDMA_PM_BUSY; @@ -227,7 +227,7 @@ bool shdma_chan_filter(struct dma_chan *chan, void *arg) struct shdma_chan *schan = to_shdma_chan(chan); struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device); const struct shdma_ops *ops = sdev->ops; - int match = (int)arg; + int match = (long)arg; int ret; if (match < 0) @@ -304,6 +304,7 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all) dma_async_tx_callback callback = NULL; void *param = NULL; unsigned long flags; + LIST_HEAD(cyclic_list); spin_lock_irqsave(&schan->chan_lock, flags); list_for_each_entry_safe(desc, _desc, &schan->ld_queue, node) { @@ -369,10 +370,16 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all) if (((desc->mark == DESC_COMPLETED || desc->mark == DESC_WAITING) && async_tx_test_ack(&desc->async_tx)) || all) { - /* Remove from ld_queue list */ - desc->mark = DESC_IDLE; - list_move(&desc->node, &schan->ld_free); + if (all || !desc->cyclic) { + /* Remove from ld_queue list */ + desc->mark = DESC_IDLE; + list_move(&desc->node, &schan->ld_free); + } else { + /* reuse as cyclic */ + desc->mark = DESC_SUBMITTED; + list_move_tail(&desc->node, &cyclic_list); + } if (list_empty(&schan->ld_queue)) { dev_dbg(schan->dev, "Bring down channel %d\n", schan->id); @@ -389,6 +396,8 @@ static dma_async_tx_callback __ld_cleanup(struct shdma_chan *schan, bool all) */ schan->dma_chan.completed_cookie = schan->dma_chan.cookie; + list_splice_tail(&cyclic_list, &schan->ld_queue); + spin_unlock_irqrestore(&schan->chan_lock, flags); if (callback) @@ -491,8 +500,8 @@ static struct shdma_desc *shdma_add_desc(struct shdma_chan *schan, } dev_dbg(schan->dev, - "chaining (%u/%u)@%x -> %x with %p, cookie %d\n", - copy_size, *len, *src, *dst, &new->async_tx, + "chaining (%zu/%zu)@%pad -> %pad with %p, cookie %d\n", + copy_size, *len, src, dst, &new->async_tx, new->async_tx.cookie); new->mark = DESC_PREPARED; @@ -521,7 +530,7 @@ static struct shdma_desc *shdma_add_desc(struct shdma_chan *schan, */ static struct dma_async_tx_descriptor *shdma_prep_sg(struct shdma_chan *schan, struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr, - enum dma_transfer_direction direction, unsigned long flags) + enum dma_transfer_direction direction, unsigned long flags, bool cyclic) { struct scatterlist *sg; struct shdma_desc *first = NULL, *new = NULL /* compiler... */; @@ -555,8 +564,8 @@ static struct dma_async_tx_descriptor *shdma_prep_sg(struct shdma_chan *schan, goto err_get_desc; do { - dev_dbg(schan->dev, "Add SG #%d@%p[%d], dma %llx\n", - i, sg, len, (unsigned long long)sg_addr); + dev_dbg(schan->dev, "Add SG #%d@%p[%zu], dma %pad\n", + i, sg, len, &sg_addr); if (direction == DMA_DEV_TO_MEM) new = shdma_add_desc(schan, flags, @@ -569,7 +578,11 @@ static struct dma_async_tx_descriptor *shdma_prep_sg(struct shdma_chan *schan, if (!new) goto err_get_desc; - new->chunks = chunks--; + new->cyclic = cyclic; + if (cyclic) + new->chunks = 1; + else + new->chunks = chunks--; list_add_tail(&new->node, &tx_list); } while (len); } @@ -612,7 +625,8 @@ static struct dma_async_tx_descriptor *shdma_prep_memcpy( sg_dma_address(&sg) = dma_src; sg_dma_len(&sg) = len; - return shdma_prep_sg(schan, &sg, 1, &dma_dest, DMA_MEM_TO_MEM, flags); + return shdma_prep_sg(schan, &sg, 1, &dma_dest, DMA_MEM_TO_MEM, + flags, false); } static struct dma_async_tx_descriptor *shdma_prep_slave_sg( @@ -640,7 +654,58 @@ static struct dma_async_tx_descriptor *shdma_prep_slave_sg( slave_addr = ops->slave_addr(schan); return shdma_prep_sg(schan, sgl, sg_len, &slave_addr, - direction, flags); + direction, flags, false); +} + +#define SHDMA_MAX_SG_LEN 32 + +static struct dma_async_tx_descriptor *shdma_prep_dma_cyclic( + struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct shdma_chan *schan = to_shdma_chan(chan); + struct shdma_dev *sdev = to_shdma_dev(schan->dma_chan.device); + const struct shdma_ops *ops = sdev->ops; + unsigned int sg_len = buf_len / period_len; + int slave_id = schan->slave_id; + dma_addr_t slave_addr; + struct scatterlist sgl[SHDMA_MAX_SG_LEN]; + int i; + + if (!chan) + return NULL; + + BUG_ON(!schan->desc_num); + + if (sg_len > SHDMA_MAX_SG_LEN) { + dev_err(schan->dev, "sg length %d exceds limit %d", + sg_len, SHDMA_MAX_SG_LEN); + return NULL; + } + + /* Someone calling slave DMA on a generic channel? */ + if (slave_id < 0 || (buf_len < period_len)) { + dev_warn(schan->dev, + "%s: bad parameter: buf_len=%zu, period_len=%zu, id=%d\n", + __func__, buf_len, period_len, slave_id); + return NULL; + } + + slave_addr = ops->slave_addr(schan); + + sg_init_table(sgl, sg_len); + for (i = 0; i < sg_len; i++) { + dma_addr_t src = buf_addr + (period_len * i); + + sg_set_page(&sgl[i], pfn_to_page(PFN_DOWN(src)), period_len, + offset_in_page(src)); + sg_dma_address(&sgl[i]) = src; + sg_dma_len(&sgl[i]) = period_len; + } + + return shdma_prep_sg(schan, sgl, sg_len, &slave_addr, + direction, flags, true); } static int shdma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, @@ -724,7 +789,7 @@ static enum dma_status shdma_tx_status(struct dma_chan *chan, * If we don't find cookie on the queue, it has been aborted and we have * to report error */ - if (status != DMA_SUCCESS) { + if (status != DMA_COMPLETE) { struct shdma_desc *sdesc; status = DMA_ERROR; list_for_each_entry(sdesc, &schan->ld_queue, node) @@ -915,6 +980,7 @@ int shdma_init(struct device *dev, struct shdma_dev *sdev, /* Compulsory for DMA_SLAVE fields */ dma_dev->device_prep_slave_sg = shdma_prep_slave_sg; + dma_dev->device_prep_dma_cyclic = shdma_prep_dma_cyclic; dma_dev->device_control = shdma_control; dma_dev->dev = dev; diff --git a/drivers/dma/sh/shdma-of.c b/drivers/dma/sh/shdma-of.c index 06473a05fe4..b4ff9d3e56d 100644 --- a/drivers/dma/sh/shdma-of.c +++ b/drivers/dma/sh/shdma-of.c @@ -33,7 +33,8 @@ static struct dma_chan *shdma_of_xlate(struct of_phandle_args *dma_spec, /* Only slave DMA channels can be allocated via DT */ dma_cap_set(DMA_SLAVE, mask); - chan = dma_request_channel(mask, shdma_chan_filter, (void *)id); + chan = dma_request_channel(mask, shdma_chan_filter, + (void *)(uintptr_t)id); if (chan) to_shdma_chan(chan)->hw_req = id; diff --git a/drivers/dma/sh/shdmac.c b/drivers/dma/sh/shdmac.c index 1069e8869f2..146d5df926d 100644 --- a/drivers/dma/sh/shdmac.c +++ b/drivers/dma/sh/shdmac.c @@ -18,21 +18,22 @@ * */ +#include <linux/delay.h> +#include <linux/dmaengine.h> +#include <linux/err.h> #include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/kdebug.h> #include <linux/module.h> +#include <linux/notifier.h> #include <linux/of.h> #include <linux/of_device.h> -#include <linux/slab.h> -#include <linux/interrupt.h> -#include <linux/dmaengine.h> -#include <linux/delay.h> #include <linux/platform_device.h> #include <linux/pm_runtime.h> +#include <linux/rculist.h> #include <linux/sh_dma.h> -#include <linux/notifier.h> -#include <linux/kdebug.h> +#include <linux/slab.h> #include <linux/spinlock.h> -#include <linux/rculist.h> #include "../dmaengine.h" #include "shdma.h" @@ -443,6 +444,7 @@ static bool sh_dmae_reset(struct sh_dmae_device *shdev) return ret; } +#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARM) static irqreturn_t sh_dmae_err(int irq, void *data) { struct sh_dmae_device *shdev = data; @@ -453,6 +455,7 @@ static irqreturn_t sh_dmae_err(int irq, void *data) sh_dmae_reset(shdev); return IRQ_HANDLED; } +#endif static bool sh_dmae_desc_completed(struct shdma_chan *schan, struct shdma_desc *sdesc) @@ -637,7 +640,7 @@ static int sh_dmae_resume(struct device *dev) #define sh_dmae_resume NULL #endif -const struct dev_pm_ops sh_dmae_pm = { +static const struct dev_pm_ops sh_dmae_pm = { .suspend = sh_dmae_suspend, .resume = sh_dmae_resume, .runtime_suspend = sh_dmae_runtime_suspend, @@ -685,9 +688,12 @@ MODULE_DEVICE_TABLE(of, sh_dmae_of_match); static int sh_dmae_probe(struct platform_device *pdev) { const struct sh_dmae_pdata *pdata; - unsigned long irqflags = IRQF_DISABLED, - chan_flag[SH_DMAE_MAX_CHANNELS] = {}; - int errirq, chan_irq[SH_DMAE_MAX_CHANNELS]; + unsigned long chan_flag[SH_DMAE_MAX_CHANNELS] = {}; + int chan_irq[SH_DMAE_MAX_CHANNELS]; +#if defined(CONFIG_CPU_SH4) || defined(CONFIG_ARM) + unsigned long irqflags = 0; + int errirq; +#endif int err, i, irq_cnt = 0, irqres = 0, irq_cap = 0; struct sh_dmae_device *shdev; struct dma_device *dma_dev; @@ -838,7 +844,7 @@ static int sh_dmae_probe(struct platform_device *pdev) IORESOURCE_IRQ_SHAREABLE) chan_flag[irq_cnt] = IRQF_SHARED; else - chan_flag[irq_cnt] = IRQF_DISABLED; + chan_flag[irq_cnt] = 0; dev_dbg(&pdev->dev, "Found IRQ %d for channel %d\n", i, irq_cnt); diff --git a/drivers/dma/sh/sudmac.c b/drivers/dma/sh/sudmac.c index c7e9cdff070..3ce10390989 100644 --- a/drivers/dma/sh/sudmac.c +++ b/drivers/dma/sh/sudmac.c @@ -14,12 +14,13 @@ * published by the Free Software Foundation. */ +#include <linux/dmaengine.h> +#include <linux/err.h> #include <linux/init.h> -#include <linux/module.h> -#include <linux/slab.h> #include <linux/interrupt.h> -#include <linux/dmaengine.h> +#include <linux/module.h> #include <linux/platform_device.h> +#include <linux/slab.h> #include <linux/sudmac.h> struct sudmac_chan { @@ -178,8 +179,8 @@ static int sudmac_desc_setup(struct shdma_chan *schan, struct sudmac_chan *sc = to_chan(schan); struct sudmac_desc *sd = to_desc(sdesc); - dev_dbg(sc->shdma_chan.dev, "%s: src=%x, dst=%x, len=%d\n", - __func__, src, dst, *len); + dev_dbg(sc->shdma_chan.dev, "%s: src=%pad, dst=%pad, len=%zu\n", + __func__, &src, &dst, *len); if (*len > schan->max_xfer_len) *len = schan->max_xfer_len; diff --git a/drivers/dma/sirf-dma.c b/drivers/dma/sirf-dma.c index 6aec3ad814d..03f7820fa33 100644 --- a/drivers/dma/sirf-dma.c +++ b/drivers/dma/sirf-dma.c @@ -18,6 +18,7 @@ #include <linux/of_device.h> #include <linux/of_platform.h> #include <linux/clk.h> +#include <linux/of_dma.h> #include <linux/sirfsoc_dma.h> #include "dmaengine.h" @@ -640,6 +641,37 @@ bool sirfsoc_dma_filter_id(struct dma_chan *chan, void *chan_id) } EXPORT_SYMBOL(sirfsoc_dma_filter_id); +#define SIRFSOC_DMA_BUSWIDTHS \ + (BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) | \ + BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \ + BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) | \ + BIT(DMA_SLAVE_BUSWIDTH_8_BYTES)) + +static int sirfsoc_dma_device_slave_caps(struct dma_chan *dchan, + struct dma_slave_caps *caps) +{ + caps->src_addr_widths = SIRFSOC_DMA_BUSWIDTHS; + caps->dstn_addr_widths = SIRFSOC_DMA_BUSWIDTHS; + caps->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); + caps->cmd_pause = true; + caps->cmd_terminate = true; + + return 0; +} + +static struct dma_chan *of_dma_sirfsoc_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct sirfsoc_dma *sdma = ofdma->of_dma_data; + unsigned int request = dma_spec->args[0]; + + if (request >= SIRFSOC_DMA_CHANNELS) + return NULL; + + return dma_get_slave_channel(&sdma->channels[request].chan); +} + static int sirfsoc_dma_probe(struct platform_device *op) { struct device_node *dn = op->dev.of_node; @@ -712,6 +744,7 @@ static int sirfsoc_dma_probe(struct platform_device *op) dma->device_tx_status = sirfsoc_dma_tx_status; dma->device_prep_interleaved_dma = sirfsoc_dma_prep_interleaved; dma->device_prep_dma_cyclic = sirfsoc_dma_prep_cyclic; + dma->device_slave_caps = sirfsoc_dma_device_slave_caps; INIT_LIST_HEAD(&dma->channels); dma_cap_set(DMA_SLAVE, dma->cap_mask); @@ -744,11 +777,20 @@ static int sirfsoc_dma_probe(struct platform_device *op) if (ret) goto free_irq; + /* Device-tree DMA controller registration */ + ret = of_dma_controller_register(dn, of_dma_sirfsoc_xlate, sdma); + if (ret) { + dev_err(dev, "failed to register DMA controller\n"); + goto unreg_dma_dev; + } + pm_runtime_enable(&op->dev); dev_info(dev, "initialized SIRFSOC DMAC driver\n"); return 0; +unreg_dma_dev: + dma_async_device_unregister(dma); free_irq: free_irq(sdma->irq, sdma); irq_dispose: @@ -761,6 +803,7 @@ static int sirfsoc_dma_remove(struct platform_device *op) struct device *dev = &op->dev; struct sirfsoc_dma *sdma = dev_get_drvdata(dev); + of_dma_controller_free(op->dev.of_node); dma_async_device_unregister(&sdma->dma); free_irq(sdma->irq, sdma); irq_dispose_mapping(sdma->irq); diff --git a/drivers/dma/ste_dma40.c b/drivers/dma/ste_dma40.c index 82d2b97ad94..c7984459ede 100644 --- a/drivers/dma/ste_dma40.c +++ b/drivers/dma/ste_dma40.c @@ -14,6 +14,7 @@ #include <linux/platform_device.h> #include <linux/clk.h> #include <linux/delay.h> +#include <linux/log2.h> #include <linux/pm.h> #include <linux/pm_runtime.h> #include <linux/err.h> @@ -555,7 +556,6 @@ struct d40_gen_dmac { * later * @reg_val_backup_chan: Backup data for standard channel parameter registers. * @gcc_pwr_off_mask: Mask to maintain the channels that can be turned off. - * @initialized: true if the dma has been initialized * @gen_dmac: the struct for generic registers values to represent u8500/8540 * DMA controller */ @@ -593,7 +593,6 @@ struct d40_base { u32 reg_val_backup_v4[BACKUP_REGS_SZ_MAX]; u32 *reg_val_backup_chan; u16 gcc_pwr_off_mask; - bool initialized; struct d40_gen_dmac gen_dmac; }; @@ -1055,62 +1054,6 @@ static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len, return len; } - -#ifdef CONFIG_PM -static void dma40_backup(void __iomem *baseaddr, u32 *backup, - u32 *regaddr, int num, bool save) -{ - int i; - - for (i = 0; i < num; i++) { - void __iomem *addr = baseaddr + regaddr[i]; - - if (save) - backup[i] = readl_relaxed(addr); - else - writel_relaxed(backup[i], addr); - } -} - -static void d40_save_restore_registers(struct d40_base *base, bool save) -{ - int i; - - /* Save/Restore channel specific registers */ - for (i = 0; i < base->num_phy_chans; i++) { - void __iomem *addr; - int idx; - - if (base->phy_res[i].reserved) - continue; - - addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA; - idx = i * ARRAY_SIZE(d40_backup_regs_chan); - - dma40_backup(addr, &base->reg_val_backup_chan[idx], - d40_backup_regs_chan, - ARRAY_SIZE(d40_backup_regs_chan), - save); - } - - /* Save/Restore global registers */ - dma40_backup(base->virtbase, base->reg_val_backup, - d40_backup_regs, ARRAY_SIZE(d40_backup_regs), - save); - - /* Save/Restore registers only existing on dma40 v3 and later */ - if (base->gen_dmac.backup) - dma40_backup(base->virtbase, base->reg_val_backup_v4, - base->gen_dmac.backup, - base->gen_dmac.backup_size, - save); -} -#else -static void d40_save_restore_registers(struct d40_base *base, bool save) -{ -} -#endif - static int __d40_execute_command_phy(struct d40_chan *d40c, enum d40_command command) { @@ -1494,8 +1437,8 @@ static int d40_pause(struct d40_chan *d40c) if (!d40c->busy) return 0; - pm_runtime_get_sync(d40c->base->dev); spin_lock_irqsave(&d40c->lock, flags); + pm_runtime_get_sync(d40c->base->dev); res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); @@ -1640,6 +1583,7 @@ static void dma_tasklet(unsigned long data) struct d40_chan *d40c = (struct d40_chan *) data; struct d40_desc *d40d; unsigned long flags; + bool callback_active; dma_async_tx_callback callback; void *callback_param; @@ -1667,6 +1611,7 @@ static void dma_tasklet(unsigned long data) } /* Callback to client */ + callback_active = !!(d40d->txd.flags & DMA_PREP_INTERRUPT); callback = d40d->txd.callback; callback_param = d40d->txd.callback_param; @@ -1689,7 +1634,7 @@ static void dma_tasklet(unsigned long data) spin_unlock_irqrestore(&d40c->lock, flags); - if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT)) + if (callback_active && callback) callback(callback_param); return; @@ -2408,6 +2353,7 @@ static void d40_set_prio_realtime(struct d40_chan *d40c) #define D40_DT_FLAGS_DIR(flags) ((flags >> 1) & 0x1) #define D40_DT_FLAGS_BIG_ENDIAN(flags) ((flags >> 2) & 0x1) #define D40_DT_FLAGS_FIXED_CHAN(flags) ((flags >> 3) & 0x1) +#define D40_DT_FLAGS_HIGH_PRIO(flags) ((flags >> 4) & 0x1) static struct dma_chan *d40_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) @@ -2445,6 +2391,9 @@ static struct dma_chan *d40_xlate(struct of_phandle_args *dma_spec, cfg.use_fixed_channel = true; } + if (D40_DT_FLAGS_HIGH_PRIO(flags)) + cfg.high_priority = true; + return dma_request_channel(cap, stedma40_filter, &cfg); } @@ -2626,7 +2575,7 @@ static enum dma_status d40_tx_status(struct dma_chan *chan, } ret = dma_cookie_status(chan, cookie, txstate); - if (ret != DMA_SUCCESS) + if (ret != DMA_COMPLETE) dma_set_residue(txstate, stedma40_residue(chan)); if (d40_is_paused(d40c)) @@ -2796,8 +2745,8 @@ static int d40_set_runtime_config(struct dma_chan *chan, src_addr_width > DMA_SLAVE_BUSWIDTH_8_BYTES || dst_addr_width <= DMA_SLAVE_BUSWIDTH_UNDEFINED || dst_addr_width > DMA_SLAVE_BUSWIDTH_8_BYTES || - ((src_addr_width > 1) && (src_addr_width & 1)) || - ((dst_addr_width > 1) && (dst_addr_width & 1))) + !is_power_of_2(src_addr_width) || + !is_power_of_2(dst_addr_width)) return -EINVAL; cfg->src_info.data_width = src_addr_width; @@ -2991,18 +2940,88 @@ failure1: } /* Suspend resume functionality */ -#ifdef CONFIG_PM -static int dma40_pm_suspend(struct device *dev) +#ifdef CONFIG_PM_SLEEP +static int dma40_suspend(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct d40_base *base = platform_get_drvdata(pdev); - int ret = 0; + int ret; + + ret = pm_runtime_force_suspend(dev); + if (ret) + return ret; if (base->lcpa_regulator) ret = regulator_disable(base->lcpa_regulator); return ret; } +static int dma40_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct d40_base *base = platform_get_drvdata(pdev); + int ret = 0; + + if (base->lcpa_regulator) { + ret = regulator_enable(base->lcpa_regulator); + if (ret) + return ret; + } + + return pm_runtime_force_resume(dev); +} +#endif + +#ifdef CONFIG_PM +static void dma40_backup(void __iomem *baseaddr, u32 *backup, + u32 *regaddr, int num, bool save) +{ + int i; + + for (i = 0; i < num; i++) { + void __iomem *addr = baseaddr + regaddr[i]; + + if (save) + backup[i] = readl_relaxed(addr); + else + writel_relaxed(backup[i], addr); + } +} + +static void d40_save_restore_registers(struct d40_base *base, bool save) +{ + int i; + + /* Save/Restore channel specific registers */ + for (i = 0; i < base->num_phy_chans; i++) { + void __iomem *addr; + int idx; + + if (base->phy_res[i].reserved) + continue; + + addr = base->virtbase + D40_DREG_PCBASE + i * D40_DREG_PCDELTA; + idx = i * ARRAY_SIZE(d40_backup_regs_chan); + + dma40_backup(addr, &base->reg_val_backup_chan[idx], + d40_backup_regs_chan, + ARRAY_SIZE(d40_backup_regs_chan), + save); + } + + /* Save/Restore global registers */ + dma40_backup(base->virtbase, base->reg_val_backup, + d40_backup_regs, ARRAY_SIZE(d40_backup_regs), + save); + + /* Save/Restore registers only existing on dma40 v3 and later */ + if (base->gen_dmac.backup) + dma40_backup(base->virtbase, base->reg_val_backup_v4, + base->gen_dmac.backup, + base->gen_dmac.backup_size, + save); +} + static int dma40_runtime_suspend(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); @@ -3023,36 +3042,20 @@ static int dma40_runtime_resume(struct device *dev) struct platform_device *pdev = to_platform_device(dev); struct d40_base *base = platform_get_drvdata(pdev); - if (base->initialized) - d40_save_restore_registers(base, false); + d40_save_restore_registers(base, false); writel_relaxed(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC); return 0; } - -static int dma40_resume(struct device *dev) -{ - struct platform_device *pdev = to_platform_device(dev); - struct d40_base *base = platform_get_drvdata(pdev); - int ret = 0; - - if (base->lcpa_regulator) - ret = regulator_enable(base->lcpa_regulator); - - return ret; -} +#endif static const struct dev_pm_ops dma40_pm_ops = { - .suspend = dma40_pm_suspend, - .runtime_suspend = dma40_runtime_suspend, - .runtime_resume = dma40_runtime_resume, - .resume = dma40_resume, + SET_LATE_SYSTEM_SLEEP_PM_OPS(dma40_suspend, dma40_resume) + SET_PM_RUNTIME_PM_OPS(dma40_runtime_suspend, + dma40_runtime_resume, + NULL) }; -#define DMA40_PM_OPS (&dma40_pm_ops) -#else -#define DMA40_PM_OPS NULL -#endif /* Initialization functions. */ @@ -3638,12 +3641,6 @@ static int __init d40_probe(struct platform_device *pdev) goto failure; } - pm_runtime_irq_safe(base->dev); - pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY); - pm_runtime_use_autosuspend(base->dev); - pm_runtime_enable(base->dev); - pm_runtime_resume(base->dev); - if (base->plat_data->use_esram_lcla) { base->lcpa_regulator = regulator_get(base->dev, "lcla_esram"); @@ -3664,7 +3661,15 @@ static int __init d40_probe(struct platform_device *pdev) } } - base->initialized = true; + writel_relaxed(D40_DREG_GCC_ENABLE_ALL, base->virtbase + D40_DREG_GCC); + + pm_runtime_irq_safe(base->dev); + pm_runtime_set_autosuspend_delay(base->dev, DMA40_AUTOSUSPEND_DELAY); + pm_runtime_use_autosuspend(base->dev); + pm_runtime_mark_last_busy(base->dev); + pm_runtime_set_active(base->dev); + pm_runtime_enable(base->dev); + ret = d40_dmaengine_init(base, num_reserved_chans); if (ret) goto failure; @@ -3747,7 +3752,7 @@ static struct platform_driver d40_driver = { .driver = { .owner = THIS_MODULE, .name = D40_NAME, - .pm = DMA40_PM_OPS, + .pm = &dma40_pm_ops, .of_match_table = d40_match, }, }; diff --git a/drivers/dma/tegra20-apb-dma.c b/drivers/dma/tegra20-apb-dma.c index 5d4986e5f5f..03ad64ecaaf 100644 --- a/drivers/dma/tegra20-apb-dma.c +++ b/drivers/dma/tegra20-apb-dma.c @@ -1,7 +1,7 @@ /* * DMA driver for Nvidia's Tegra20 APB DMA controller. * - * Copyright (c) 2012, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2012-2013, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -29,11 +29,12 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/of_device.h> +#include <linux/of_dma.h> #include <linux/platform_device.h> #include <linux/pm.h> #include <linux/pm_runtime.h> +#include <linux/reset.h> #include <linux/slab.h> -#include <linux/clk/tegra.h> #include "dmaengine.h" @@ -99,6 +100,11 @@ #define TEGRA_APBDMA_APBSEQ_DATA_SWAP BIT(27) #define TEGRA_APBDMA_APBSEQ_WRAP_WORD_1 (1 << 16) +/* Tegra148 specific registers */ +#define TEGRA_APBDMA_CHAN_WCOUNT 0x20 + +#define TEGRA_APBDMA_CHAN_WORD_TRANSFER 0x24 + /* * If any burst is in flight and DMA paused then this is the time to complete * on-flight burst and update DMA status register. @@ -108,21 +114,22 @@ /* Channel base address offset from APBDMA base address */ #define TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET 0x1000 -/* DMA channel register space size */ -#define TEGRA_APBDMA_CHANNEL_REGISTER_SIZE 0x20 - struct tegra_dma; /* * tegra_dma_chip_data Tegra chip specific DMA data * @nr_channels: Number of channels available in the controller. + * @channel_reg_size: Channel register size/stride. * @max_dma_count: Maximum DMA transfer count supported by DMA controller. * @support_channel_pause: Support channel wise pause of dma. + * @support_separate_wcount_reg: Support separate word count register. */ struct tegra_dma_chip_data { int nr_channels; + int channel_reg_size; int max_dma_count; bool support_channel_pause; + bool support_separate_wcount_reg; }; /* DMA channel registers */ @@ -132,6 +139,7 @@ struct tegra_dma_channel_regs { unsigned long apb_ptr; unsigned long ahb_seq; unsigned long apb_seq; + unsigned long wcount; }; /* @@ -199,6 +207,7 @@ struct tegra_dma_channel { void *callback_param; /* Channel-slave specific configuration */ + unsigned int slave_id; struct dma_slave_config dma_sconfig; struct tegra_dma_channel_regs channel_reg; }; @@ -208,6 +217,7 @@ struct tegra_dma { struct dma_device dma_dev; struct device *dev; struct clk *dma_clk; + struct reset_control *rst; spinlock_t global_lock; void __iomem *base_addr; const struct tegra_dma_chip_data *chip_data; @@ -339,6 +349,8 @@ static int tegra_dma_slave_config(struct dma_chan *dc, } memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig)); + if (!tdc->slave_id) + tdc->slave_id = sconfig->slave_id; tdc->config_init = true; return 0; } @@ -421,6 +433,8 @@ static void tegra_dma_start(struct tegra_dma_channel *tdc, tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, ch_regs->apb_ptr); tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBSEQ, ch_regs->ahb_seq); tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, ch_regs->ahb_ptr); + if (tdc->tdma->chip_data->support_separate_wcount_reg) + tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, ch_regs->wcount); /* Start DMA */ tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, @@ -460,6 +474,9 @@ static void tegra_dma_configure_for_next(struct tegra_dma_channel *tdc, /* Safe to program new configuration */ tdc_write(tdc, TEGRA_APBDMA_CHAN_APBPTR, nsg_req->ch_regs.apb_ptr); tdc_write(tdc, TEGRA_APBDMA_CHAN_AHBPTR, nsg_req->ch_regs.ahb_ptr); + if (tdc->tdma->chip_data->support_separate_wcount_reg) + tdc_write(tdc, TEGRA_APBDMA_CHAN_WCOUNT, + nsg_req->ch_regs.wcount); tdc_write(tdc, TEGRA_APBDMA_CHAN_CSR, nsg_req->ch_regs.csr | TEGRA_APBDMA_CSR_ENB); nsg_req->configured = true; @@ -570,7 +587,7 @@ static void handle_once_dma_done(struct tegra_dma_channel *tdc, list_del(&sgreq->node); if (sgreq->last_sg) { - dma_desc->dma_status = DMA_SUCCESS; + dma_desc->dma_status = DMA_COMPLETE; dma_cookie_complete(&dma_desc->txd); if (!dma_desc->cb_count) list_add_tail(&dma_desc->cb_node, &tdc->cb_desc); @@ -713,6 +730,7 @@ static void tegra_dma_terminate_all(struct dma_chan *dc) struct tegra_dma_desc *dma_desc; unsigned long flags; unsigned long status; + unsigned long wcount; bool was_busy; spin_lock_irqsave(&tdc->lock, flags); @@ -733,6 +751,10 @@ static void tegra_dma_terminate_all(struct dma_chan *dc) tdc->isr_handler(tdc, true); status = tdc_read(tdc, TEGRA_APBDMA_CHAN_STATUS); } + if (tdc->tdma->chip_data->support_separate_wcount_reg) + wcount = tdc_read(tdc, TEGRA_APBDMA_CHAN_WORD_TRANSFER); + else + wcount = status; was_busy = tdc->busy; tegra_dma_stop(tdc); @@ -741,7 +763,7 @@ static void tegra_dma_terminate_all(struct dma_chan *dc) sgreq = list_first_entry(&tdc->pending_sg_req, typeof(*sgreq), node); sgreq->dma_desc->bytes_transferred += - get_current_xferred_count(tdc, sgreq, status); + get_current_xferred_count(tdc, sgreq, wcount); } tegra_dma_resume(tdc); @@ -768,7 +790,7 @@ static enum dma_status tegra_dma_tx_status(struct dma_chan *dc, unsigned int residual; ret = dma_cookie_status(dc, cookie, txstate); - if (ret == DMA_SUCCESS) + if (ret == DMA_COMPLETE) return ret; spin_lock_irqsave(&tdc->lock, flags); @@ -903,6 +925,17 @@ static int get_transfer_param(struct tegra_dma_channel *tdc, return -EINVAL; } +static void tegra_dma_prep_wcount(struct tegra_dma_channel *tdc, + struct tegra_dma_channel_regs *ch_regs, u32 len) +{ + u32 len_field = (len - 4) & 0xFFFC; + + if (tdc->tdma->chip_data->support_separate_wcount_reg) + ch_regs->wcount = len_field; + else + ch_regs->csr |= len_field; +} + static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( struct dma_chan *dc, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, @@ -941,7 +974,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( ahb_seq |= TEGRA_APBDMA_AHBSEQ_BUS_WIDTH_32; csr |= TEGRA_APBDMA_CSR_ONCE | TEGRA_APBDMA_CSR_FLOW; - csr |= tdc->dma_sconfig.slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; + csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; if (flags & DMA_PREP_INTERRUPT) csr |= TEGRA_APBDMA_CSR_IE_EOC; @@ -986,7 +1019,8 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( sg_req->ch_regs.apb_ptr = apb_ptr; sg_req->ch_regs.ahb_ptr = mem; - sg_req->ch_regs.csr = csr | ((len - 4) & 0xFFFC); + sg_req->ch_regs.csr = csr; + tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len); sg_req->ch_regs.apb_seq = apb_seq; sg_req->ch_regs.ahb_seq = ahb_seq; sg_req->configured = false; @@ -1018,7 +1052,7 @@ static struct dma_async_tx_descriptor *tegra_dma_prep_slave_sg( return &dma_desc->txd; } -struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( +static struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, unsigned long flags, void *context) @@ -1085,7 +1119,7 @@ struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( csr |= TEGRA_APBDMA_CSR_FLOW; if (flags & DMA_PREP_INTERRUPT) csr |= TEGRA_APBDMA_CSR_IE_EOC; - csr |= tdc->dma_sconfig.slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; + csr |= tdc->slave_id << TEGRA_APBDMA_CSR_REQ_SEL_SHIFT; apb_seq |= TEGRA_APBDMA_APBSEQ_WRAP_WORD_1; @@ -1115,7 +1149,8 @@ struct dma_async_tx_descriptor *tegra_dma_prep_dma_cyclic( ahb_seq |= get_burst_size(tdc, burst_size, slave_bw, len); sg_req->ch_regs.apb_ptr = apb_ptr; sg_req->ch_regs.ahb_ptr = mem; - sg_req->ch_regs.csr = csr | ((len - 4) & 0xFFFC); + sg_req->ch_regs.csr = csr; + tegra_dma_prep_wcount(tdc, &sg_req->ch_regs, len); sg_req->ch_regs.apb_seq = apb_seq; sg_req->ch_regs.ahb_seq = ahb_seq; sg_req->configured = false; @@ -1205,32 +1240,69 @@ static void tegra_dma_free_chan_resources(struct dma_chan *dc) kfree(sg_req); } clk_disable_unprepare(tdma->dma_clk); + + tdc->slave_id = 0; +} + +static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct tegra_dma *tdma = ofdma->of_dma_data; + struct dma_chan *chan; + struct tegra_dma_channel *tdc; + + chan = dma_get_any_slave_channel(&tdma->dma_dev); + if (!chan) + return NULL; + + tdc = to_tegra_dma_chan(chan); + tdc->slave_id = dma_spec->args[0]; + + return chan; } /* Tegra20 specific DMA controller information */ static const struct tegra_dma_chip_data tegra20_dma_chip_data = { .nr_channels = 16, + .channel_reg_size = 0x20, .max_dma_count = 1024UL * 64, .support_channel_pause = false, + .support_separate_wcount_reg = false, }; /* Tegra30 specific DMA controller information */ static const struct tegra_dma_chip_data tegra30_dma_chip_data = { .nr_channels = 32, + .channel_reg_size = 0x20, .max_dma_count = 1024UL * 64, .support_channel_pause = false, + .support_separate_wcount_reg = false, }; /* Tegra114 specific DMA controller information */ static const struct tegra_dma_chip_data tegra114_dma_chip_data = { .nr_channels = 32, + .channel_reg_size = 0x20, + .max_dma_count = 1024UL * 64, + .support_channel_pause = true, + .support_separate_wcount_reg = false, +}; + +/* Tegra148 specific DMA controller information */ +static const struct tegra_dma_chip_data tegra148_dma_chip_data = { + .nr_channels = 32, + .channel_reg_size = 0x40, .max_dma_count = 1024UL * 64, .support_channel_pause = true, + .support_separate_wcount_reg = true, }; static const struct of_device_id tegra_dma_of_match[] = { { + .compatible = "nvidia,tegra148-apbdma", + .data = &tegra148_dma_chip_data, + }, { .compatible = "nvidia,tegra114-apbdma", .data = &tegra114_dma_chip_data, }, { @@ -1282,6 +1354,12 @@ static int tegra_dma_probe(struct platform_device *pdev) return PTR_ERR(tdma->dma_clk); } + tdma->rst = devm_reset_control_get(&pdev->dev, "dma"); + if (IS_ERR(tdma->rst)) { + dev_err(&pdev->dev, "Error: Missing reset\n"); + return PTR_ERR(tdma->rst); + } + spin_lock_init(&tdma->global_lock); pm_runtime_enable(&pdev->dev); @@ -1302,9 +1380,9 @@ static int tegra_dma_probe(struct platform_device *pdev) } /* Reset DMA controller */ - tegra_periph_reset_assert(tdma->dma_clk); + reset_control_assert(tdma->rst); udelay(2); - tegra_periph_reset_deassert(tdma->dma_clk); + reset_control_deassert(tdma->rst); /* Enable global DMA registers */ tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE); @@ -1318,7 +1396,7 @@ static int tegra_dma_probe(struct platform_device *pdev) struct tegra_dma_channel *tdc = &tdma->channels[i]; tdc->chan_base_offset = TEGRA_APBDMA_CHANNEL_BASE_ADD_OFFSET + - i * TEGRA_APBDMA_CHANNEL_REGISTER_SIZE; + i * cdata->channel_reg_size; res = platform_get_resource(pdev, IORESOURCE_IRQ, i); if (!res) { @@ -1376,10 +1454,20 @@ static int tegra_dma_probe(struct platform_device *pdev) goto err_irq; } + ret = of_dma_controller_register(pdev->dev.of_node, + tegra_dma_of_xlate, tdma); + if (ret < 0) { + dev_err(&pdev->dev, + "Tegra20 APB DMA OF registration failed %d\n", ret); + goto err_unregister_dma_dev; + } + dev_info(&pdev->dev, "Tegra20 APB DMA driver register %d channels\n", cdata->nr_channels); return 0; +err_unregister_dma_dev: + dma_async_device_unregister(&tdma->dma_dev); err_irq: while (--i >= 0) { struct tegra_dma_channel *tdc = &tdma->channels[i]; diff --git a/drivers/dma/timb_dma.c b/drivers/dma/timb_dma.c index 28af214fce0..4506a7b4f97 100644 --- a/drivers/dma/timb_dma.c +++ b/drivers/dma/timb_dma.c @@ -154,38 +154,6 @@ static bool __td_dma_done_ack(struct timb_dma_chan *td_chan) return done; } -static void __td_unmap_desc(struct timb_dma_chan *td_chan, const u8 *dma_desc, - bool single) -{ - dma_addr_t addr; - int len; - - addr = (dma_desc[7] << 24) | (dma_desc[6] << 16) | (dma_desc[5] << 8) | - dma_desc[4]; - - len = (dma_desc[3] << 8) | dma_desc[2]; - - if (single) - dma_unmap_single(chan2dev(&td_chan->chan), addr, len, - DMA_TO_DEVICE); - else - dma_unmap_page(chan2dev(&td_chan->chan), addr, len, - DMA_TO_DEVICE); -} - -static void __td_unmap_descs(struct timb_dma_desc *td_desc, bool single) -{ - struct timb_dma_chan *td_chan = container_of(td_desc->txd.chan, - struct timb_dma_chan, chan); - u8 *descs; - - for (descs = td_desc->desc_list; ; descs += TIMB_DMA_DESC_SIZE) { - __td_unmap_desc(td_chan, descs, single); - if (descs[0] & 0x02) - break; - } -} - static int td_fill_desc(struct timb_dma_chan *td_chan, u8 *dma_desc, struct scatterlist *sg, bool last) { @@ -293,10 +261,7 @@ static void __td_finish(struct timb_dma_chan *td_chan) list_move(&td_desc->desc_node, &td_chan->free_list); - if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) - __td_unmap_descs(td_desc, - txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE); - + dma_descriptor_unmap(txd); /* * The API requires that no submissions are done from a * callback, so we don't need to drop the lock here diff --git a/drivers/dma/txx9dmac.c b/drivers/dma/txx9dmac.c index 71e8e775189..17686caf64d 100644 --- a/drivers/dma/txx9dmac.c +++ b/drivers/dma/txx9dmac.c @@ -406,7 +406,6 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc, dma_async_tx_callback callback; void *param; struct dma_async_tx_descriptor *txd = &desc->txd; - struct txx9dmac_slave *ds = dc->chan.private; dev_vdbg(chan2dev(&dc->chan), "descriptor %u %p complete\n", txd->cookie, desc); @@ -419,30 +418,7 @@ txx9dmac_descriptor_complete(struct txx9dmac_chan *dc, list_splice_init(&desc->tx_list, &dc->free_list); list_move(&desc->desc_node, &dc->free_list); - if (!ds) { - dma_addr_t dmaaddr; - if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - dmaaddr = is_dmac64(dc) ? - desc->hwdesc.DAR : desc->hwdesc32.DAR; - if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) - dma_unmap_single(chan2parent(&dc->chan), - dmaaddr, desc->len, DMA_FROM_DEVICE); - else - dma_unmap_page(chan2parent(&dc->chan), - dmaaddr, desc->len, DMA_FROM_DEVICE); - } - if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - dmaaddr = is_dmac64(dc) ? - desc->hwdesc.SAR : desc->hwdesc32.SAR; - if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) - dma_unmap_single(chan2parent(&dc->chan), - dmaaddr, desc->len, DMA_TO_DEVICE); - else - dma_unmap_page(chan2parent(&dc->chan), - dmaaddr, desc->len, DMA_TO_DEVICE); - } - } - + dma_descriptor_unmap(txd); /* * The API requires that no submissions are done from a * callback, so we don't need to drop the lock here @@ -962,8 +938,8 @@ txx9dmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie, enum dma_status ret; ret = dma_cookie_status(chan, cookie, txstate); - if (ret == DMA_SUCCESS) - return DMA_SUCCESS; + if (ret == DMA_COMPLETE) + return DMA_COMPLETE; spin_lock_bh(&dc->lock); txx9dmac_scan_descriptors(dc); diff --git a/drivers/dma/virt-dma.h b/drivers/dma/virt-dma.h index 85c19d63f9f..181b9526786 100644 --- a/drivers/dma/virt-dma.h +++ b/drivers/dma/virt-dma.h @@ -84,10 +84,12 @@ static inline bool vchan_issue_pending(struct virt_dma_chan *vc) static inline void vchan_cookie_complete(struct virt_dma_desc *vd) { struct virt_dma_chan *vc = to_virt_chan(vd->tx.chan); + dma_cookie_t cookie; + cookie = vd->tx.cookie; dma_cookie_complete(&vd->tx); dev_vdbg(vc->chan.device->dev, "txd %p[%x]: marked complete\n", - vd, vd->tx.cookie); + vd, cookie); list_add_tail(&vd->node, &vc->desc_completed); tasklet_schedule(&vc->task); diff --git a/drivers/dma/xilinx/Makefile b/drivers/dma/xilinx/Makefile new file mode 100644 index 00000000000..3c4e9f2fea2 --- /dev/null +++ b/drivers/dma/xilinx/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_XILINX_VDMA) += xilinx_vdma.o diff --git a/drivers/dma/xilinx/xilinx_vdma.c b/drivers/dma/xilinx/xilinx_vdma.c new file mode 100644 index 00000000000..42a13e8d460 --- /dev/null +++ b/drivers/dma/xilinx/xilinx_vdma.c @@ -0,0 +1,1379 @@ +/* + * DMA driver for Xilinx Video DMA Engine + * + * Copyright (C) 2010-2014 Xilinx, Inc. All rights reserved. + * + * Based on the Freescale DMA driver. + * + * Description: + * The AXI Video Direct Memory Access (AXI VDMA) core is a soft Xilinx IP + * core that provides high-bandwidth direct memory access between memory + * and AXI4-Stream type video target peripherals. The core provides efficient + * two dimensional DMA operations with independent asynchronous read (S2MM) + * and write (MM2S) channel operation. It can be configured to have either + * one channel or two channels. If configured as two channels, one is to + * transmit to the video device (MM2S) and another is to receive from the + * video device (S2MM). Initialization, status, interrupt and management + * registers are accessed through an AXI4-Lite slave interface. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + */ + +#include <linux/amba/xilinx_dma.h> +#include <linux/bitops.h> +#include <linux/dmapool.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_dma.h> +#include <linux/of_platform.h> +#include <linux/of_irq.h> +#include <linux/slab.h> + +#include "../dmaengine.h" + +/* Register/Descriptor Offsets */ +#define XILINX_VDMA_MM2S_CTRL_OFFSET 0x0000 +#define XILINX_VDMA_S2MM_CTRL_OFFSET 0x0030 +#define XILINX_VDMA_MM2S_DESC_OFFSET 0x0050 +#define XILINX_VDMA_S2MM_DESC_OFFSET 0x00a0 + +/* Control Registers */ +#define XILINX_VDMA_REG_DMACR 0x0000 +#define XILINX_VDMA_DMACR_DELAY_MAX 0xff +#define XILINX_VDMA_DMACR_DELAY_SHIFT 24 +#define XILINX_VDMA_DMACR_FRAME_COUNT_MAX 0xff +#define XILINX_VDMA_DMACR_FRAME_COUNT_SHIFT 16 +#define XILINX_VDMA_DMACR_ERR_IRQ BIT(14) +#define XILINX_VDMA_DMACR_DLY_CNT_IRQ BIT(13) +#define XILINX_VDMA_DMACR_FRM_CNT_IRQ BIT(12) +#define XILINX_VDMA_DMACR_MASTER_SHIFT 8 +#define XILINX_VDMA_DMACR_FSYNCSRC_SHIFT 5 +#define XILINX_VDMA_DMACR_FRAMECNT_EN BIT(4) +#define XILINX_VDMA_DMACR_GENLOCK_EN BIT(3) +#define XILINX_VDMA_DMACR_RESET BIT(2) +#define XILINX_VDMA_DMACR_CIRC_EN BIT(1) +#define XILINX_VDMA_DMACR_RUNSTOP BIT(0) +#define XILINX_VDMA_DMACR_FSYNCSRC_MASK GENMASK(6, 5) + +#define XILINX_VDMA_REG_DMASR 0x0004 +#define XILINX_VDMA_DMASR_EOL_LATE_ERR BIT(15) +#define XILINX_VDMA_DMASR_ERR_IRQ BIT(14) +#define XILINX_VDMA_DMASR_DLY_CNT_IRQ BIT(13) +#define XILINX_VDMA_DMASR_FRM_CNT_IRQ BIT(12) +#define XILINX_VDMA_DMASR_SOF_LATE_ERR BIT(11) +#define XILINX_VDMA_DMASR_SG_DEC_ERR BIT(10) +#define XILINX_VDMA_DMASR_SG_SLV_ERR BIT(9) +#define XILINX_VDMA_DMASR_EOF_EARLY_ERR BIT(8) +#define XILINX_VDMA_DMASR_SOF_EARLY_ERR BIT(7) +#define XILINX_VDMA_DMASR_DMA_DEC_ERR BIT(6) +#define XILINX_VDMA_DMASR_DMA_SLAVE_ERR BIT(5) +#define XILINX_VDMA_DMASR_DMA_INT_ERR BIT(4) +#define XILINX_VDMA_DMASR_IDLE BIT(1) +#define XILINX_VDMA_DMASR_HALTED BIT(0) +#define XILINX_VDMA_DMASR_DELAY_MASK GENMASK(31, 24) +#define XILINX_VDMA_DMASR_FRAME_COUNT_MASK GENMASK(23, 16) + +#define XILINX_VDMA_REG_CURDESC 0x0008 +#define XILINX_VDMA_REG_TAILDESC 0x0010 +#define XILINX_VDMA_REG_REG_INDEX 0x0014 +#define XILINX_VDMA_REG_FRMSTORE 0x0018 +#define XILINX_VDMA_REG_THRESHOLD 0x001c +#define XILINX_VDMA_REG_FRMPTR_STS 0x0024 +#define XILINX_VDMA_REG_PARK_PTR 0x0028 +#define XILINX_VDMA_PARK_PTR_WR_REF_SHIFT 8 +#define XILINX_VDMA_PARK_PTR_RD_REF_SHIFT 0 +#define XILINX_VDMA_REG_VDMA_VERSION 0x002c + +/* Register Direct Mode Registers */ +#define XILINX_VDMA_REG_VSIZE 0x0000 +#define XILINX_VDMA_REG_HSIZE 0x0004 + +#define XILINX_VDMA_REG_FRMDLY_STRIDE 0x0008 +#define XILINX_VDMA_FRMDLY_STRIDE_FRMDLY_SHIFT 24 +#define XILINX_VDMA_FRMDLY_STRIDE_STRIDE_SHIFT 0 + +#define XILINX_VDMA_REG_START_ADDRESS(n) (0x000c + 4 * (n)) + +/* HW specific definitions */ +#define XILINX_VDMA_MAX_CHANS_PER_DEVICE 0x2 + +#define XILINX_VDMA_DMAXR_ALL_IRQ_MASK \ + (XILINX_VDMA_DMASR_FRM_CNT_IRQ | \ + XILINX_VDMA_DMASR_DLY_CNT_IRQ | \ + XILINX_VDMA_DMASR_ERR_IRQ) + +#define XILINX_VDMA_DMASR_ALL_ERR_MASK \ + (XILINX_VDMA_DMASR_EOL_LATE_ERR | \ + XILINX_VDMA_DMASR_SOF_LATE_ERR | \ + XILINX_VDMA_DMASR_SG_DEC_ERR | \ + XILINX_VDMA_DMASR_SG_SLV_ERR | \ + XILINX_VDMA_DMASR_EOF_EARLY_ERR | \ + XILINX_VDMA_DMASR_SOF_EARLY_ERR | \ + XILINX_VDMA_DMASR_DMA_DEC_ERR | \ + XILINX_VDMA_DMASR_DMA_SLAVE_ERR | \ + XILINX_VDMA_DMASR_DMA_INT_ERR) + +/* + * Recoverable errors are DMA Internal error, SOF Early, EOF Early + * and SOF Late. They are only recoverable when C_FLUSH_ON_FSYNC + * is enabled in the h/w system. + */ +#define XILINX_VDMA_DMASR_ERR_RECOVER_MASK \ + (XILINX_VDMA_DMASR_SOF_LATE_ERR | \ + XILINX_VDMA_DMASR_EOF_EARLY_ERR | \ + XILINX_VDMA_DMASR_SOF_EARLY_ERR | \ + XILINX_VDMA_DMASR_DMA_INT_ERR) + +/* Axi VDMA Flush on Fsync bits */ +#define XILINX_VDMA_FLUSH_S2MM 3 +#define XILINX_VDMA_FLUSH_MM2S 2 +#define XILINX_VDMA_FLUSH_BOTH 1 + +/* Delay loop counter to prevent hardware failure */ +#define XILINX_VDMA_LOOP_COUNT 1000000 + +/** + * struct xilinx_vdma_desc_hw - Hardware Descriptor + * @next_desc: Next Descriptor Pointer @0x00 + * @pad1: Reserved @0x04 + * @buf_addr: Buffer address @0x08 + * @pad2: Reserved @0x0C + * @vsize: Vertical Size @0x10 + * @hsize: Horizontal Size @0x14 + * @stride: Number of bytes between the first + * pixels of each horizontal line @0x18 + */ +struct xilinx_vdma_desc_hw { + u32 next_desc; + u32 pad1; + u32 buf_addr; + u32 pad2; + u32 vsize; + u32 hsize; + u32 stride; +} __aligned(64); + +/** + * struct xilinx_vdma_tx_segment - Descriptor segment + * @hw: Hardware descriptor + * @node: Node in the descriptor segments list + * @phys: Physical address of segment + */ +struct xilinx_vdma_tx_segment { + struct xilinx_vdma_desc_hw hw; + struct list_head node; + dma_addr_t phys; +} __aligned(64); + +/** + * struct xilinx_vdma_tx_descriptor - Per Transaction structure + * @async_tx: Async transaction descriptor + * @segments: TX segments list + * @node: Node in the channel descriptors list + */ +struct xilinx_vdma_tx_descriptor { + struct dma_async_tx_descriptor async_tx; + struct list_head segments; + struct list_head node; +}; + +/** + * struct xilinx_vdma_chan - Driver specific VDMA channel structure + * @xdev: Driver specific device structure + * @ctrl_offset: Control registers offset + * @desc_offset: TX descriptor registers offset + * @lock: Descriptor operation lock + * @pending_list: Descriptors waiting + * @active_desc: Active descriptor + * @allocated_desc: Allocated descriptor + * @done_list: Complete descriptors + * @common: DMA common channel + * @desc_pool: Descriptors pool + * @dev: The dma device + * @irq: Channel IRQ + * @id: Channel ID + * @direction: Transfer direction + * @num_frms: Number of frames + * @has_sg: Support scatter transfers + * @genlock: Support genlock mode + * @err: Channel has errors + * @tasklet: Cleanup work after irq + * @config: Device configuration info + * @flush_on_fsync: Flush on Frame sync + */ +struct xilinx_vdma_chan { + struct xilinx_vdma_device *xdev; + u32 ctrl_offset; + u32 desc_offset; + spinlock_t lock; + struct list_head pending_list; + struct xilinx_vdma_tx_descriptor *active_desc; + struct xilinx_vdma_tx_descriptor *allocated_desc; + struct list_head done_list; + struct dma_chan common; + struct dma_pool *desc_pool; + struct device *dev; + int irq; + int id; + enum dma_transfer_direction direction; + int num_frms; + bool has_sg; + bool genlock; + bool err; + struct tasklet_struct tasklet; + struct xilinx_vdma_config config; + bool flush_on_fsync; +}; + +/** + * struct xilinx_vdma_device - VDMA device structure + * @regs: I/O mapped base address + * @dev: Device Structure + * @common: DMA device structure + * @chan: Driver specific VDMA channel + * @has_sg: Specifies whether Scatter-Gather is present or not + * @flush_on_fsync: Flush on frame sync + */ +struct xilinx_vdma_device { + void __iomem *regs; + struct device *dev; + struct dma_device common; + struct xilinx_vdma_chan *chan[XILINX_VDMA_MAX_CHANS_PER_DEVICE]; + bool has_sg; + u32 flush_on_fsync; +}; + +/* Macros */ +#define to_xilinx_chan(chan) \ + container_of(chan, struct xilinx_vdma_chan, common) +#define to_vdma_tx_descriptor(tx) \ + container_of(tx, struct xilinx_vdma_tx_descriptor, async_tx) + +/* IO accessors */ +static inline u32 vdma_read(struct xilinx_vdma_chan *chan, u32 reg) +{ + return ioread32(chan->xdev->regs + reg); +} + +static inline void vdma_write(struct xilinx_vdma_chan *chan, u32 reg, u32 value) +{ + iowrite32(value, chan->xdev->regs + reg); +} + +static inline void vdma_desc_write(struct xilinx_vdma_chan *chan, u32 reg, + u32 value) +{ + vdma_write(chan, chan->desc_offset + reg, value); +} + +static inline u32 vdma_ctrl_read(struct xilinx_vdma_chan *chan, u32 reg) +{ + return vdma_read(chan, chan->ctrl_offset + reg); +} + +static inline void vdma_ctrl_write(struct xilinx_vdma_chan *chan, u32 reg, + u32 value) +{ + vdma_write(chan, chan->ctrl_offset + reg, value); +} + +static inline void vdma_ctrl_clr(struct xilinx_vdma_chan *chan, u32 reg, + u32 clr) +{ + vdma_ctrl_write(chan, reg, vdma_ctrl_read(chan, reg) & ~clr); +} + +static inline void vdma_ctrl_set(struct xilinx_vdma_chan *chan, u32 reg, + u32 set) +{ + vdma_ctrl_write(chan, reg, vdma_ctrl_read(chan, reg) | set); +} + +/* ----------------------------------------------------------------------------- + * Descriptors and segments alloc and free + */ + +/** + * xilinx_vdma_alloc_tx_segment - Allocate transaction segment + * @chan: Driver specific VDMA channel + * + * Return: The allocated segment on success and NULL on failure. + */ +static struct xilinx_vdma_tx_segment * +xilinx_vdma_alloc_tx_segment(struct xilinx_vdma_chan *chan) +{ + struct xilinx_vdma_tx_segment *segment; + dma_addr_t phys; + + segment = dma_pool_alloc(chan->desc_pool, GFP_ATOMIC, &phys); + if (!segment) + return NULL; + + memset(segment, 0, sizeof(*segment)); + segment->phys = phys; + + return segment; +} + +/** + * xilinx_vdma_free_tx_segment - Free transaction segment + * @chan: Driver specific VDMA channel + * @segment: VDMA transaction segment + */ +static void xilinx_vdma_free_tx_segment(struct xilinx_vdma_chan *chan, + struct xilinx_vdma_tx_segment *segment) +{ + dma_pool_free(chan->desc_pool, segment, segment->phys); +} + +/** + * xilinx_vdma_tx_descriptor - Allocate transaction descriptor + * @chan: Driver specific VDMA channel + * + * Return: The allocated descriptor on success and NULL on failure. + */ +static struct xilinx_vdma_tx_descriptor * +xilinx_vdma_alloc_tx_descriptor(struct xilinx_vdma_chan *chan) +{ + struct xilinx_vdma_tx_descriptor *desc; + unsigned long flags; + + if (chan->allocated_desc) + return chan->allocated_desc; + + desc = kzalloc(sizeof(*desc), GFP_KERNEL); + if (!desc) + return NULL; + + spin_lock_irqsave(&chan->lock, flags); + chan->allocated_desc = desc; + spin_unlock_irqrestore(&chan->lock, flags); + + INIT_LIST_HEAD(&desc->segments); + + return desc; +} + +/** + * xilinx_vdma_free_tx_descriptor - Free transaction descriptor + * @chan: Driver specific VDMA channel + * @desc: VDMA transaction descriptor + */ +static void +xilinx_vdma_free_tx_descriptor(struct xilinx_vdma_chan *chan, + struct xilinx_vdma_tx_descriptor *desc) +{ + struct xilinx_vdma_tx_segment *segment, *next; + + if (!desc) + return; + + list_for_each_entry_safe(segment, next, &desc->segments, node) { + list_del(&segment->node); + xilinx_vdma_free_tx_segment(chan, segment); + } + + kfree(desc); +} + +/* Required functions */ + +/** + * xilinx_vdma_free_desc_list - Free descriptors list + * @chan: Driver specific VDMA channel + * @list: List to parse and delete the descriptor + */ +static void xilinx_vdma_free_desc_list(struct xilinx_vdma_chan *chan, + struct list_head *list) +{ + struct xilinx_vdma_tx_descriptor *desc, *next; + + list_for_each_entry_safe(desc, next, list, node) { + list_del(&desc->node); + xilinx_vdma_free_tx_descriptor(chan, desc); + } +} + +/** + * xilinx_vdma_free_descriptors - Free channel descriptors + * @chan: Driver specific VDMA channel + */ +static void xilinx_vdma_free_descriptors(struct xilinx_vdma_chan *chan) +{ + unsigned long flags; + + spin_lock_irqsave(&chan->lock, flags); + + xilinx_vdma_free_desc_list(chan, &chan->pending_list); + xilinx_vdma_free_desc_list(chan, &chan->done_list); + + xilinx_vdma_free_tx_descriptor(chan, chan->active_desc); + chan->active_desc = NULL; + + spin_unlock_irqrestore(&chan->lock, flags); +} + +/** + * xilinx_vdma_free_chan_resources - Free channel resources + * @dchan: DMA channel + */ +static void xilinx_vdma_free_chan_resources(struct dma_chan *dchan) +{ + struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan); + + dev_dbg(chan->dev, "Free all channel resources.\n"); + + xilinx_vdma_free_descriptors(chan); + dma_pool_destroy(chan->desc_pool); + chan->desc_pool = NULL; +} + +/** + * xilinx_vdma_chan_desc_cleanup - Clean channel descriptors + * @chan: Driver specific VDMA channel + */ +static void xilinx_vdma_chan_desc_cleanup(struct xilinx_vdma_chan *chan) +{ + struct xilinx_vdma_tx_descriptor *desc, *next; + unsigned long flags; + + spin_lock_irqsave(&chan->lock, flags); + + list_for_each_entry_safe(desc, next, &chan->done_list, node) { + dma_async_tx_callback callback; + void *callback_param; + + /* Remove from the list of running transactions */ + list_del(&desc->node); + + /* Run the link descriptor callback function */ + callback = desc->async_tx.callback; + callback_param = desc->async_tx.callback_param; + if (callback) { + spin_unlock_irqrestore(&chan->lock, flags); + callback(callback_param); + spin_lock_irqsave(&chan->lock, flags); + } + + /* Run any dependencies, then free the descriptor */ + dma_run_dependencies(&desc->async_tx); + xilinx_vdma_free_tx_descriptor(chan, desc); + } + + spin_unlock_irqrestore(&chan->lock, flags); +} + +/** + * xilinx_vdma_do_tasklet - Schedule completion tasklet + * @data: Pointer to the Xilinx VDMA channel structure + */ +static void xilinx_vdma_do_tasklet(unsigned long data) +{ + struct xilinx_vdma_chan *chan = (struct xilinx_vdma_chan *)data; + + xilinx_vdma_chan_desc_cleanup(chan); +} + +/** + * xilinx_vdma_alloc_chan_resources - Allocate channel resources + * @dchan: DMA channel + * + * Return: '0' on success and failure value on error + */ +static int xilinx_vdma_alloc_chan_resources(struct dma_chan *dchan) +{ + struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan); + + /* Has this channel already been allocated? */ + if (chan->desc_pool) + return 0; + + /* + * We need the descriptor to be aligned to 64bytes + * for meeting Xilinx VDMA specification requirement. + */ + chan->desc_pool = dma_pool_create("xilinx_vdma_desc_pool", + chan->dev, + sizeof(struct xilinx_vdma_tx_segment), + __alignof__(struct xilinx_vdma_tx_segment), 0); + if (!chan->desc_pool) { + dev_err(chan->dev, + "unable to allocate channel %d descriptor pool\n", + chan->id); + return -ENOMEM; + } + + dma_cookie_init(dchan); + return 0; +} + +/** + * xilinx_vdma_tx_status - Get VDMA transaction status + * @dchan: DMA channel + * @cookie: Transaction identifier + * @txstate: Transaction state + * + * Return: DMA transaction status + */ +static enum dma_status xilinx_vdma_tx_status(struct dma_chan *dchan, + dma_cookie_t cookie, + struct dma_tx_state *txstate) +{ + return dma_cookie_status(dchan, cookie, txstate); +} + +/** + * xilinx_vdma_is_running - Check if VDMA channel is running + * @chan: Driver specific VDMA channel + * + * Return: '1' if running, '0' if not. + */ +static bool xilinx_vdma_is_running(struct xilinx_vdma_chan *chan) +{ + return !(vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) & + XILINX_VDMA_DMASR_HALTED) && + (vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR) & + XILINX_VDMA_DMACR_RUNSTOP); +} + +/** + * xilinx_vdma_is_idle - Check if VDMA channel is idle + * @chan: Driver specific VDMA channel + * + * Return: '1' if idle, '0' if not. + */ +static bool xilinx_vdma_is_idle(struct xilinx_vdma_chan *chan) +{ + return vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) & + XILINX_VDMA_DMASR_IDLE; +} + +/** + * xilinx_vdma_halt - Halt VDMA channel + * @chan: Driver specific VDMA channel + */ +static void xilinx_vdma_halt(struct xilinx_vdma_chan *chan) +{ + int loop = XILINX_VDMA_LOOP_COUNT; + + vdma_ctrl_clr(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RUNSTOP); + + /* Wait for the hardware to halt */ + do { + if (vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) & + XILINX_VDMA_DMASR_HALTED) + break; + } while (loop--); + + if (!loop) { + dev_err(chan->dev, "Cannot stop channel %p: %x\n", + chan, vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR)); + chan->err = true; + } + + return; +} + +/** + * xilinx_vdma_start - Start VDMA channel + * @chan: Driver specific VDMA channel + */ +static void xilinx_vdma_start(struct xilinx_vdma_chan *chan) +{ + int loop = XILINX_VDMA_LOOP_COUNT; + + vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RUNSTOP); + + /* Wait for the hardware to start */ + do { + if (!(vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR) & + XILINX_VDMA_DMASR_HALTED)) + break; + } while (loop--); + + if (!loop) { + dev_err(chan->dev, "Cannot start channel %p: %x\n", + chan, vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR)); + + chan->err = true; + } + + return; +} + +/** + * xilinx_vdma_start_transfer - Starts VDMA transfer + * @chan: Driver specific channel struct pointer + */ +static void xilinx_vdma_start_transfer(struct xilinx_vdma_chan *chan) +{ + struct xilinx_vdma_config *config = &chan->config; + struct xilinx_vdma_tx_descriptor *desc; + unsigned long flags; + u32 reg; + struct xilinx_vdma_tx_segment *head, *tail = NULL; + + if (chan->err) + return; + + spin_lock_irqsave(&chan->lock, flags); + + /* There's already an active descriptor, bail out. */ + if (chan->active_desc) + goto out_unlock; + + if (list_empty(&chan->pending_list)) + goto out_unlock; + + desc = list_first_entry(&chan->pending_list, + struct xilinx_vdma_tx_descriptor, node); + + /* If it is SG mode and hardware is busy, cannot submit */ + if (chan->has_sg && xilinx_vdma_is_running(chan) && + !xilinx_vdma_is_idle(chan)) { + dev_dbg(chan->dev, "DMA controller still busy\n"); + goto out_unlock; + } + + /* + * If hardware is idle, then all descriptors on the running lists are + * done, start new transfers + */ + if (chan->has_sg) { + head = list_first_entry(&desc->segments, + struct xilinx_vdma_tx_segment, node); + tail = list_entry(desc->segments.prev, + struct xilinx_vdma_tx_segment, node); + + vdma_ctrl_write(chan, XILINX_VDMA_REG_CURDESC, head->phys); + } + + /* Configure the hardware using info in the config structure */ + reg = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR); + + if (config->frm_cnt_en) + reg |= XILINX_VDMA_DMACR_FRAMECNT_EN; + else + reg &= ~XILINX_VDMA_DMACR_FRAMECNT_EN; + + /* + * With SG, start with circular mode, so that BDs can be fetched. + * In direct register mode, if not parking, enable circular mode + */ + if (chan->has_sg || !config->park) + reg |= XILINX_VDMA_DMACR_CIRC_EN; + + if (config->park) + reg &= ~XILINX_VDMA_DMACR_CIRC_EN; + + vdma_ctrl_write(chan, XILINX_VDMA_REG_DMACR, reg); + + if (config->park && (config->park_frm >= 0) && + (config->park_frm < chan->num_frms)) { + if (chan->direction == DMA_MEM_TO_DEV) + vdma_write(chan, XILINX_VDMA_REG_PARK_PTR, + config->park_frm << + XILINX_VDMA_PARK_PTR_RD_REF_SHIFT); + else + vdma_write(chan, XILINX_VDMA_REG_PARK_PTR, + config->park_frm << + XILINX_VDMA_PARK_PTR_WR_REF_SHIFT); + } + + /* Start the hardware */ + xilinx_vdma_start(chan); + + if (chan->err) + goto out_unlock; + + /* Start the transfer */ + if (chan->has_sg) { + vdma_ctrl_write(chan, XILINX_VDMA_REG_TAILDESC, tail->phys); + } else { + struct xilinx_vdma_tx_segment *segment, *last = NULL; + int i = 0; + + list_for_each_entry(segment, &desc->segments, node) { + vdma_desc_write(chan, + XILINX_VDMA_REG_START_ADDRESS(i++), + segment->hw.buf_addr); + last = segment; + } + + if (!last) + goto out_unlock; + + /* HW expects these parameters to be same for one transaction */ + vdma_desc_write(chan, XILINX_VDMA_REG_HSIZE, last->hw.hsize); + vdma_desc_write(chan, XILINX_VDMA_REG_FRMDLY_STRIDE, + last->hw.stride); + vdma_desc_write(chan, XILINX_VDMA_REG_VSIZE, last->hw.vsize); + } + + list_del(&desc->node); + chan->active_desc = desc; + +out_unlock: + spin_unlock_irqrestore(&chan->lock, flags); +} + +/** + * xilinx_vdma_issue_pending - Issue pending transactions + * @dchan: DMA channel + */ +static void xilinx_vdma_issue_pending(struct dma_chan *dchan) +{ + struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan); + + xilinx_vdma_start_transfer(chan); +} + +/** + * xilinx_vdma_complete_descriptor - Mark the active descriptor as complete + * @chan : xilinx DMA channel + * + * CONTEXT: hardirq + */ +static void xilinx_vdma_complete_descriptor(struct xilinx_vdma_chan *chan) +{ + struct xilinx_vdma_tx_descriptor *desc; + unsigned long flags; + + spin_lock_irqsave(&chan->lock, flags); + + desc = chan->active_desc; + if (!desc) { + dev_dbg(chan->dev, "no running descriptors\n"); + goto out_unlock; + } + + dma_cookie_complete(&desc->async_tx); + list_add_tail(&desc->node, &chan->done_list); + + chan->active_desc = NULL; + +out_unlock: + spin_unlock_irqrestore(&chan->lock, flags); +} + +/** + * xilinx_vdma_reset - Reset VDMA channel + * @chan: Driver specific VDMA channel + * + * Return: '0' on success and failure value on error + */ +static int xilinx_vdma_reset(struct xilinx_vdma_chan *chan) +{ + int loop = XILINX_VDMA_LOOP_COUNT; + u32 tmp; + + vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR, XILINX_VDMA_DMACR_RESET); + + tmp = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR) & + XILINX_VDMA_DMACR_RESET; + + /* Wait for the hardware to finish reset */ + do { + tmp = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR) & + XILINX_VDMA_DMACR_RESET; + } while (loop-- && tmp); + + if (!loop) { + dev_err(chan->dev, "reset timeout, cr %x, sr %x\n", + vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR), + vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR)); + return -ETIMEDOUT; + } + + chan->err = false; + + return 0; +} + +/** + * xilinx_vdma_chan_reset - Reset VDMA channel and enable interrupts + * @chan: Driver specific VDMA channel + * + * Return: '0' on success and failure value on error + */ +static int xilinx_vdma_chan_reset(struct xilinx_vdma_chan *chan) +{ + int err; + + /* Reset VDMA */ + err = xilinx_vdma_reset(chan); + if (err) + return err; + + /* Enable interrupts */ + vdma_ctrl_set(chan, XILINX_VDMA_REG_DMACR, + XILINX_VDMA_DMAXR_ALL_IRQ_MASK); + + return 0; +} + +/** + * xilinx_vdma_irq_handler - VDMA Interrupt handler + * @irq: IRQ number + * @data: Pointer to the Xilinx VDMA channel structure + * + * Return: IRQ_HANDLED/IRQ_NONE + */ +static irqreturn_t xilinx_vdma_irq_handler(int irq, void *data) +{ + struct xilinx_vdma_chan *chan = data; + u32 status; + + /* Read the status and ack the interrupts. */ + status = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMASR); + if (!(status & XILINX_VDMA_DMAXR_ALL_IRQ_MASK)) + return IRQ_NONE; + + vdma_ctrl_write(chan, XILINX_VDMA_REG_DMASR, + status & XILINX_VDMA_DMAXR_ALL_IRQ_MASK); + + if (status & XILINX_VDMA_DMASR_ERR_IRQ) { + /* + * An error occurred. If C_FLUSH_ON_FSYNC is enabled and the + * error is recoverable, ignore it. Otherwise flag the error. + * + * Only recoverable errors can be cleared in the DMASR register, + * make sure not to write to other error bits to 1. + */ + u32 errors = status & XILINX_VDMA_DMASR_ALL_ERR_MASK; + vdma_ctrl_write(chan, XILINX_VDMA_REG_DMASR, + errors & XILINX_VDMA_DMASR_ERR_RECOVER_MASK); + + if (!chan->flush_on_fsync || + (errors & ~XILINX_VDMA_DMASR_ERR_RECOVER_MASK)) { + dev_err(chan->dev, + "Channel %p has errors %x, cdr %x tdr %x\n", + chan, errors, + vdma_ctrl_read(chan, XILINX_VDMA_REG_CURDESC), + vdma_ctrl_read(chan, XILINX_VDMA_REG_TAILDESC)); + chan->err = true; + } + } + + if (status & XILINX_VDMA_DMASR_DLY_CNT_IRQ) { + /* + * Device takes too long to do the transfer when user requires + * responsiveness. + */ + dev_dbg(chan->dev, "Inter-packet latency too long\n"); + } + + if (status & XILINX_VDMA_DMASR_FRM_CNT_IRQ) { + xilinx_vdma_complete_descriptor(chan); + xilinx_vdma_start_transfer(chan); + } + + tasklet_schedule(&chan->tasklet); + return IRQ_HANDLED; +} + +/** + * xilinx_vdma_tx_submit - Submit DMA transaction + * @tx: Async transaction descriptor + * + * Return: cookie value on success and failure value on error + */ +static dma_cookie_t xilinx_vdma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + struct xilinx_vdma_tx_descriptor *desc = to_vdma_tx_descriptor(tx); + struct xilinx_vdma_chan *chan = to_xilinx_chan(tx->chan); + dma_cookie_t cookie; + unsigned long flags; + int err; + + if (chan->err) { + /* + * If reset fails, need to hard reset the system. + * Channel is no longer functional + */ + err = xilinx_vdma_chan_reset(chan); + if (err < 0) + return err; + } + + spin_lock_irqsave(&chan->lock, flags); + + cookie = dma_cookie_assign(tx); + + /* Append the transaction to the pending transactions queue. */ + list_add_tail(&desc->node, &chan->pending_list); + + /* Free the allocated desc */ + chan->allocated_desc = NULL; + + spin_unlock_irqrestore(&chan->lock, flags); + + return cookie; +} + +/** + * xilinx_vdma_dma_prep_interleaved - prepare a descriptor for a + * DMA_SLAVE transaction + * @dchan: DMA channel + * @xt: Interleaved template pointer + * @flags: transfer ack flags + * + * Return: Async transaction descriptor on success and NULL on failure + */ +static struct dma_async_tx_descriptor * +xilinx_vdma_dma_prep_interleaved(struct dma_chan *dchan, + struct dma_interleaved_template *xt, + unsigned long flags) +{ + struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan); + struct xilinx_vdma_tx_descriptor *desc; + struct xilinx_vdma_tx_segment *segment, *prev = NULL; + struct xilinx_vdma_desc_hw *hw; + + if (!is_slave_direction(xt->dir)) + return NULL; + + if (!xt->numf || !xt->sgl[0].size) + return NULL; + + /* Allocate a transaction descriptor. */ + desc = xilinx_vdma_alloc_tx_descriptor(chan); + if (!desc) + return NULL; + + dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); + desc->async_tx.tx_submit = xilinx_vdma_tx_submit; + async_tx_ack(&desc->async_tx); + + /* Allocate the link descriptor from DMA pool */ + segment = xilinx_vdma_alloc_tx_segment(chan); + if (!segment) + goto error; + + /* Fill in the hardware descriptor */ + hw = &segment->hw; + hw->vsize = xt->numf; + hw->hsize = xt->sgl[0].size; + hw->stride = xt->sgl[0].icg << + XILINX_VDMA_FRMDLY_STRIDE_STRIDE_SHIFT; + hw->stride |= chan->config.frm_dly << + XILINX_VDMA_FRMDLY_STRIDE_FRMDLY_SHIFT; + + if (xt->dir != DMA_MEM_TO_DEV) + hw->buf_addr = xt->dst_start; + else + hw->buf_addr = xt->src_start; + + /* Link the previous next descriptor to current */ + prev = list_last_entry(&desc->segments, + struct xilinx_vdma_tx_segment, node); + prev->hw.next_desc = segment->phys; + + /* Insert the segment into the descriptor segments list. */ + list_add_tail(&segment->node, &desc->segments); + + prev = segment; + + /* Link the last hardware descriptor with the first. */ + segment = list_first_entry(&desc->segments, + struct xilinx_vdma_tx_segment, node); + prev->hw.next_desc = segment->phys; + + return &desc->async_tx; + +error: + xilinx_vdma_free_tx_descriptor(chan, desc); + return NULL; +} + +/** + * xilinx_vdma_terminate_all - Halt the channel and free descriptors + * @chan: Driver specific VDMA Channel pointer + */ +static void xilinx_vdma_terminate_all(struct xilinx_vdma_chan *chan) +{ + /* Halt the DMA engine */ + xilinx_vdma_halt(chan); + + /* Remove and free all of the descriptors in the lists */ + xilinx_vdma_free_descriptors(chan); +} + +/** + * xilinx_vdma_channel_set_config - Configure VDMA channel + * Run-time configuration for Axi VDMA, supports: + * . halt the channel + * . configure interrupt coalescing and inter-packet delay threshold + * . start/stop parking + * . enable genlock + * + * @dchan: DMA channel + * @cfg: VDMA device configuration pointer + * + * Return: '0' on success and failure value on error + */ +int xilinx_vdma_channel_set_config(struct dma_chan *dchan, + struct xilinx_vdma_config *cfg) +{ + struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan); + u32 dmacr; + + if (cfg->reset) + return xilinx_vdma_chan_reset(chan); + + dmacr = vdma_ctrl_read(chan, XILINX_VDMA_REG_DMACR); + + chan->config.frm_dly = cfg->frm_dly; + chan->config.park = cfg->park; + + /* genlock settings */ + chan->config.gen_lock = cfg->gen_lock; + chan->config.master = cfg->master; + + if (cfg->gen_lock && chan->genlock) { + dmacr |= XILINX_VDMA_DMACR_GENLOCK_EN; + dmacr |= cfg->master << XILINX_VDMA_DMACR_MASTER_SHIFT; + } + + chan->config.frm_cnt_en = cfg->frm_cnt_en; + if (cfg->park) + chan->config.park_frm = cfg->park_frm; + else + chan->config.park_frm = -1; + + chan->config.coalesc = cfg->coalesc; + chan->config.delay = cfg->delay; + + if (cfg->coalesc <= XILINX_VDMA_DMACR_FRAME_COUNT_MAX) { + dmacr |= cfg->coalesc << XILINX_VDMA_DMACR_FRAME_COUNT_SHIFT; + chan->config.coalesc = cfg->coalesc; + } + + if (cfg->delay <= XILINX_VDMA_DMACR_DELAY_MAX) { + dmacr |= cfg->delay << XILINX_VDMA_DMACR_DELAY_SHIFT; + chan->config.delay = cfg->delay; + } + + /* FSync Source selection */ + dmacr &= ~XILINX_VDMA_DMACR_FSYNCSRC_MASK; + dmacr |= cfg->ext_fsync << XILINX_VDMA_DMACR_FSYNCSRC_SHIFT; + + vdma_ctrl_write(chan, XILINX_VDMA_REG_DMACR, dmacr); + + return 0; +} +EXPORT_SYMBOL(xilinx_vdma_channel_set_config); + +/** + * xilinx_vdma_device_control - Configure DMA channel of the device + * @dchan: DMA Channel pointer + * @cmd: DMA control command + * @arg: Channel configuration + * + * Return: '0' on success and failure value on error + */ +static int xilinx_vdma_device_control(struct dma_chan *dchan, + enum dma_ctrl_cmd cmd, unsigned long arg) +{ + struct xilinx_vdma_chan *chan = to_xilinx_chan(dchan); + + if (cmd != DMA_TERMINATE_ALL) + return -ENXIO; + + xilinx_vdma_terminate_all(chan); + + return 0; +} + +/* ----------------------------------------------------------------------------- + * Probe and remove + */ + +/** + * xilinx_vdma_chan_remove - Per Channel remove function + * @chan: Driver specific VDMA channel + */ +static void xilinx_vdma_chan_remove(struct xilinx_vdma_chan *chan) +{ + /* Disable all interrupts */ + vdma_ctrl_clr(chan, XILINX_VDMA_REG_DMACR, + XILINX_VDMA_DMAXR_ALL_IRQ_MASK); + + if (chan->irq > 0) + free_irq(chan->irq, chan); + + tasklet_kill(&chan->tasklet); + + list_del(&chan->common.device_node); +} + +/** + * xilinx_vdma_chan_probe - Per Channel Probing + * It get channel features from the device tree entry and + * initialize special channel handling routines + * + * @xdev: Driver specific device structure + * @node: Device node + * + * Return: '0' on success and failure value on error + */ +static int xilinx_vdma_chan_probe(struct xilinx_vdma_device *xdev, + struct device_node *node) +{ + struct xilinx_vdma_chan *chan; + bool has_dre = false; + u32 value, width; + int err; + + /* Allocate and initialize the channel structure */ + chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL); + if (!chan) + return -ENOMEM; + + chan->dev = xdev->dev; + chan->xdev = xdev; + chan->has_sg = xdev->has_sg; + + spin_lock_init(&chan->lock); + INIT_LIST_HEAD(&chan->pending_list); + INIT_LIST_HEAD(&chan->done_list); + + /* Retrieve the channel properties from the device tree */ + has_dre = of_property_read_bool(node, "xlnx,include-dre"); + + chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode"); + + err = of_property_read_u32(node, "xlnx,datawidth", &value); + if (err) { + dev_err(xdev->dev, "missing xlnx,datawidth property\n"); + return err; + } + width = value >> 3; /* Convert bits to bytes */ + + /* If data width is greater than 8 bytes, DRE is not in hw */ + if (width > 8) + has_dre = false; + + if (!has_dre) + xdev->common.copy_align = fls(width - 1); + + if (of_device_is_compatible(node, "xlnx,axi-vdma-mm2s-channel")) { + chan->direction = DMA_MEM_TO_DEV; + chan->id = 0; + + chan->ctrl_offset = XILINX_VDMA_MM2S_CTRL_OFFSET; + chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET; + + if (xdev->flush_on_fsync == XILINX_VDMA_FLUSH_BOTH || + xdev->flush_on_fsync == XILINX_VDMA_FLUSH_MM2S) + chan->flush_on_fsync = true; + } else if (of_device_is_compatible(node, + "xlnx,axi-vdma-s2mm-channel")) { + chan->direction = DMA_DEV_TO_MEM; + chan->id = 1; + + chan->ctrl_offset = XILINX_VDMA_S2MM_CTRL_OFFSET; + chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET; + + if (xdev->flush_on_fsync == XILINX_VDMA_FLUSH_BOTH || + xdev->flush_on_fsync == XILINX_VDMA_FLUSH_S2MM) + chan->flush_on_fsync = true; + } else { + dev_err(xdev->dev, "Invalid channel compatible node\n"); + return -EINVAL; + } + + /* Request the interrupt */ + chan->irq = irq_of_parse_and_map(node, 0); + err = request_irq(chan->irq, xilinx_vdma_irq_handler, IRQF_SHARED, + "xilinx-vdma-controller", chan); + if (err) { + dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq); + return err; + } + + /* Initialize the tasklet */ + tasklet_init(&chan->tasklet, xilinx_vdma_do_tasklet, + (unsigned long)chan); + + /* + * Initialize the DMA channel and add it to the DMA engine channels + * list. + */ + chan->common.device = &xdev->common; + + list_add_tail(&chan->common.device_node, &xdev->common.channels); + xdev->chan[chan->id] = chan; + + /* Reset the channel */ + err = xilinx_vdma_chan_reset(chan); + if (err < 0) { + dev_err(xdev->dev, "Reset channel failed\n"); + return err; + } + + return 0; +} + +/** + * of_dma_xilinx_xlate - Translation function + * @dma_spec: Pointer to DMA specifier as found in the device tree + * @ofdma: Pointer to DMA controller data + * + * Return: DMA channel pointer on success and NULL on error + */ +static struct dma_chan *of_dma_xilinx_xlate(struct of_phandle_args *dma_spec, + struct of_dma *ofdma) +{ + struct xilinx_vdma_device *xdev = ofdma->of_dma_data; + int chan_id = dma_spec->args[0]; + + if (chan_id >= XILINX_VDMA_MAX_CHANS_PER_DEVICE) + return NULL; + + return dma_get_slave_channel(&xdev->chan[chan_id]->common); +} + +/** + * xilinx_vdma_probe - Driver probe function + * @pdev: Pointer to the platform_device structure + * + * Return: '0' on success and failure value on error + */ +static int xilinx_vdma_probe(struct platform_device *pdev) +{ + struct device_node *node = pdev->dev.of_node; + struct xilinx_vdma_device *xdev; + struct device_node *child; + struct resource *io; + u32 num_frames; + int i, err; + + /* Allocate and initialize the DMA engine structure */ + xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); + if (!xdev) + return -ENOMEM; + + xdev->dev = &pdev->dev; + + /* Request and map I/O memory */ + io = platform_get_resource(pdev, IORESOURCE_MEM, 0); + xdev->regs = devm_ioremap_resource(&pdev->dev, io); + if (IS_ERR(xdev->regs)) + return PTR_ERR(xdev->regs); + + /* Retrieve the DMA engine properties from the device tree */ + xdev->has_sg = of_property_read_bool(node, "xlnx,include-sg"); + + err = of_property_read_u32(node, "xlnx,num-fstores", &num_frames); + if (err < 0) { + dev_err(xdev->dev, "missing xlnx,num-fstores property\n"); + return err; + } + + err = of_property_read_u32(node, "xlnx,flush-fsync", + &xdev->flush_on_fsync); + if (err < 0) + dev_warn(xdev->dev, "missing xlnx,flush-fsync property\n"); + + /* Initialize the DMA engine */ + xdev->common.dev = &pdev->dev; + + INIT_LIST_HEAD(&xdev->common.channels); + dma_cap_set(DMA_SLAVE, xdev->common.cap_mask); + dma_cap_set(DMA_PRIVATE, xdev->common.cap_mask); + + xdev->common.device_alloc_chan_resources = + xilinx_vdma_alloc_chan_resources; + xdev->common.device_free_chan_resources = + xilinx_vdma_free_chan_resources; + xdev->common.device_prep_interleaved_dma = + xilinx_vdma_dma_prep_interleaved; + xdev->common.device_control = xilinx_vdma_device_control; + xdev->common.device_tx_status = xilinx_vdma_tx_status; + xdev->common.device_issue_pending = xilinx_vdma_issue_pending; + + platform_set_drvdata(pdev, xdev); + + /* Initialize the channels */ + for_each_child_of_node(node, child) { + err = xilinx_vdma_chan_probe(xdev, child); + if (err < 0) + goto error; + } + + for (i = 0; i < XILINX_VDMA_MAX_CHANS_PER_DEVICE; i++) + if (xdev->chan[i]) + xdev->chan[i]->num_frms = num_frames; + + /* Register the DMA engine with the core */ + dma_async_device_register(&xdev->common); + + err = of_dma_controller_register(node, of_dma_xilinx_xlate, + xdev); + if (err < 0) { + dev_err(&pdev->dev, "Unable to register DMA to DT\n"); + dma_async_device_unregister(&xdev->common); + goto error; + } + + dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n"); + + return 0; + +error: + for (i = 0; i < XILINX_VDMA_MAX_CHANS_PER_DEVICE; i++) + if (xdev->chan[i]) + xilinx_vdma_chan_remove(xdev->chan[i]); + + return err; +} + +/** + * xilinx_vdma_remove - Driver remove function + * @pdev: Pointer to the platform_device structure + * + * Return: Always '0' + */ +static int xilinx_vdma_remove(struct platform_device *pdev) +{ + struct xilinx_vdma_device *xdev = platform_get_drvdata(pdev); + int i; + + of_dma_controller_free(pdev->dev.of_node); + + dma_async_device_unregister(&xdev->common); + + for (i = 0; i < XILINX_VDMA_MAX_CHANS_PER_DEVICE; i++) + if (xdev->chan[i]) + xilinx_vdma_chan_remove(xdev->chan[i]); + + return 0; +} + +static const struct of_device_id xilinx_vdma_of_ids[] = { + { .compatible = "xlnx,axi-vdma-1.00.a",}, + {} +}; + +static struct platform_driver xilinx_vdma_driver = { + .driver = { + .name = "xilinx-vdma", + .owner = THIS_MODULE, + .of_match_table = xilinx_vdma_of_ids, + }, + .probe = xilinx_vdma_probe, + .remove = xilinx_vdma_remove, +}; + +module_platform_driver(xilinx_vdma_driver); + +MODULE_AUTHOR("Xilinx, Inc."); +MODULE_DESCRIPTION("Xilinx VDMA driver"); +MODULE_LICENSE("GPL v2"); |