diff options
Diffstat (limited to 'drivers/dma/at_hdmac.c')
| -rw-r--r-- | drivers/dma/at_hdmac.c | 1013 |
1 files changed, 762 insertions, 251 deletions
diff --git a/drivers/dma/at_hdmac.c b/drivers/dma/at_hdmac.c index a0f3e6a06e0..c13a3bb0f59 100644 --- a/drivers/dma/at_hdmac.c +++ b/drivers/dma/at_hdmac.c @@ -9,12 +9,12 @@ * (at your option) any later version. * * - * This supports the Atmel AHB DMA Controller, - * - * The driver has currently been tested with the Atmel AT91SAM9RL - * and AT91SAM9G45 series. + * This supports the Atmel AHB DMA Controller found in several Atmel SoCs. + * The only Atmel DMA Controller that is not covered by this driver is the one + * found on AT91SAM9263. */ +#include <dt-bindings/dma/at91.h> #include <linux/clk.h> #include <linux/dmaengine.h> #include <linux/dma-mapping.h> @@ -23,8 +23,12 @@ #include <linux/module.h> #include <linux/platform_device.h> #include <linux/slab.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/of_dma.h> #include "at_hdmac_regs.h" +#include "dmaengine.h" /* * Glossary @@ -36,9 +40,8 @@ */ #define ATC_DEFAULT_CFG (ATC_FIFOCFG_HALFFIFO) -#define ATC_DEFAULT_CTRLA (0) -#define ATC_DEFAULT_CTRLB (ATC_SIF(0) \ - |ATC_DIF(1)) +#define ATC_DEFAULT_CTRLB (ATC_SIF(AT_DMA_MEM_IF) \ + |ATC_DIF(AT_DMA_MEM_IF)) /* * Initial number of descriptors to allocate for each channel. This could @@ -52,6 +55,7 @@ MODULE_PARM_DESC(init_nr_desc_per_channel, /* prototypes */ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx); +static void atc_issue_pending(struct dma_chan *chan); /*----------------------------------------------------------------------*/ @@ -107,10 +111,11 @@ static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) { struct at_desc *desc, *_desc; struct at_desc *ret = NULL; + unsigned long flags; unsigned int i = 0; LIST_HEAD(tmp_list); - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { i++; if (async_tx_test_ack(&desc->txd)) { @@ -121,7 +126,7 @@ static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) dev_dbg(chan2dev(&atchan->chan_common), "desc %p not ACKed\n", desc); } - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); dev_vdbg(chan2dev(&atchan->chan_common), "scanned %u descriptors on freelist\n", i); @@ -129,9 +134,9 @@ static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) if (!ret) { ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); if (ret) { - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); atchan->descs_allocated++; - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } else { dev_err(chan2dev(&atchan->chan_common), "not enough descriptors available\n"); @@ -150,8 +155,9 @@ static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) { if (desc) { struct at_desc *child; + unsigned long flags; - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); list_for_each_entry(child, &desc->tx_list, desc_node) dev_vdbg(chan2dev(&atchan->chan_common), "moving child desc %p to freelist\n", @@ -160,29 +166,31 @@ static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) dev_vdbg(chan2dev(&atchan->chan_common), "moving desc %p to freelist\n", desc); list_add(&desc->desc_node, &atchan->free_list); - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } } /** - * atc_assign_cookie - compute and assign new cookie - * @atchan: channel we work on - * @desc: descriptor to asign cookie for + * atc_desc_chain - build chain adding a descriptor + * @first: address of first descriptor of the chain + * @prev: address of previous descriptor of the chain + * @desc: descriptor to queue * - * Called with atchan->lock held and bh disabled + * Called from prep_* functions */ -static dma_cookie_t -atc_assign_cookie(struct at_dma_chan *atchan, struct at_desc *desc) +static void atc_desc_chain(struct at_desc **first, struct at_desc **prev, + struct at_desc *desc) { - dma_cookie_t cookie = atchan->chan_common.cookie; - - if (++cookie < 0) - cookie = 1; - - atchan->chan_common.cookie = cookie; - desc->txd.cookie = cookie; - - return cookie; + if (!(*first)) { + *first = desc; + } else { + /* inform the HW lli about chaining */ + (*prev)->lli.dscr = desc->txd.phys; + /* insert the link descriptor to the LD ring */ + list_add_tail(&desc->desc_node, + &(*first)->tx_list); + } + *prev = desc; } /** @@ -214,10 +222,6 @@ static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) vdbg_dump_regs(atchan); - /* clear any pending interrupt */ - while (dma_readl(atdma, EBCISR)) - cpu_relax(); - channel_writel(atchan, SADDR, 0); channel_writel(atchan, DADDR, 0); channel_writel(atchan, CTRLA, 0); @@ -228,6 +232,95 @@ static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) vdbg_dump_regs(atchan); } +/* + * atc_get_current_descriptors - + * locate the descriptor which equal to physical address in DSCR + * @atchan: the channel we want to start + * @dscr_addr: physical descriptor address in DSCR + */ +static struct at_desc *atc_get_current_descriptors(struct at_dma_chan *atchan, + u32 dscr_addr) +{ + struct at_desc *desc, *_desc, *child, *desc_cur = NULL; + + list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { + if (desc->lli.dscr == dscr_addr) { + desc_cur = desc; + break; + } + + list_for_each_entry(child, &desc->tx_list, desc_node) { + if (child->lli.dscr == dscr_addr) { + desc_cur = child; + break; + } + } + } + + return desc_cur; +} + +/* + * atc_get_bytes_left - + * Get the number of bytes residue in dma buffer, + * @chan: the channel we want to start + */ +static int atc_get_bytes_left(struct dma_chan *chan) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_dma *atdma = to_at_dma(chan->device); + int chan_id = atchan->chan_common.chan_id; + struct at_desc *desc_first = atc_first_active(atchan); + struct at_desc *desc_cur; + int ret = 0, count = 0; + + /* + * Initialize necessary values in the first time. + * remain_desc record remain desc length. + */ + if (atchan->remain_desc == 0) + /* First descriptor embedds the transaction length */ + atchan->remain_desc = desc_first->len; + + /* + * This happens when current descriptor transfer complete. + * The residual buffer size should reduce current descriptor length. + */ + if (unlikely(test_bit(ATC_IS_BTC, &atchan->status))) { + clear_bit(ATC_IS_BTC, &atchan->status); + desc_cur = atc_get_current_descriptors(atchan, + channel_readl(atchan, DSCR)); + if (!desc_cur) { + ret = -EINVAL; + goto out; + } + atchan->remain_desc -= (desc_cur->lli.ctrla & ATC_BTSIZE_MAX) + << (desc_first->tx_width); + if (atchan->remain_desc < 0) { + ret = -EINVAL; + goto out; + } else { + ret = atchan->remain_desc; + } + } else { + /* + * Get residual bytes when current + * descriptor transfer in progress. + */ + count = (channel_readl(atchan, CTRLA) & ATC_BTSIZE_MAX) + << (desc_first->tx_width); + ret = atchan->remain_desc - count; + } + /* + * Check fifo empty. + */ + if (!(dma_readl(atdma, CHSR) & AT_DMA_EMPT(chan_id))) + atc_issue_pending(chan); + +out: + return ret; +} + /** * atc_chain_complete - finish work for one transaction chain * @atchan: channel we work on @@ -237,53 +330,34 @@ static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) static void atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) { - dma_async_tx_callback callback; - void *param; struct dma_async_tx_descriptor *txd = &desc->txd; dev_vdbg(chan2dev(&atchan->chan_common), "descriptor %u complete\n", txd->cookie); - atchan->completed_cookie = txd->cookie; - callback = txd->callback; - param = txd->callback_param; + /* mark the descriptor as complete for non cyclic cases only */ + if (!atc_chan_is_cyclic(atchan)) + dma_cookie_complete(txd); /* move children to free_list */ list_splice_init(&desc->tx_list, &atchan->free_list); /* move myself to free_list */ list_move(&desc->desc_node, &atchan->free_list); - /* unmap dma addresses */ - if (!atchan->chan_common.private) { - struct device *parent = chan2parent(&atchan->chan_common); - if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) { - if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE) - dma_unmap_single(parent, - desc->lli.daddr, - desc->len, DMA_FROM_DEVICE); - else - dma_unmap_page(parent, - desc->lli.daddr, - desc->len, DMA_FROM_DEVICE); - } - if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) { - if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE) - dma_unmap_single(parent, - desc->lli.saddr, - desc->len, DMA_TO_DEVICE); - else - dma_unmap_page(parent, - desc->lli.saddr, - desc->len, DMA_TO_DEVICE); - } - } + dma_descriptor_unmap(txd); + /* for cyclic transfers, + * no need to replay callback function while stopping */ + if (!atc_chan_is_cyclic(atchan)) { + dma_async_tx_callback callback = txd->callback; + void *param = txd->callback_param; - /* - * The API requires that no submissions are done from a - * callback, so we don't need to drop the lock here - */ - if (callback) - callback(param); + /* + * The API requires that no submissions are done from a + * callback, so we don't need to drop the lock here + */ + if (callback) + callback(param); + } dma_run_dependencies(txd); } @@ -304,8 +378,6 @@ static void atc_complete_all(struct at_dma_chan *atchan) dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); - BUG_ON(atc_chan_is_enabled(atchan)); - /* * Submit queued descriptors ASAP, i.e. before we go through * the completed ones. @@ -322,37 +394,6 @@ static void atc_complete_all(struct at_dma_chan *atchan) } /** - * atc_cleanup_descriptors - cleanup up finished descriptors in active_list - * @atchan: channel to be cleaned up - * - * Called with atchan->lock held and bh disabled - */ -static void atc_cleanup_descriptors(struct at_dma_chan *atchan) -{ - struct at_desc *desc, *_desc; - struct at_desc *child; - - dev_vdbg(chan2dev(&atchan->chan_common), "cleanup descriptors\n"); - - list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { - if (!(desc->lli.ctrla & ATC_DONE)) - /* This one is currently in progress */ - return; - - list_for_each_entry(child, &desc->tx_list, desc_node) - if (!(child->lli.ctrla & ATC_DONE)) - /* Currently in progress */ - return; - - /* - * No descriptors so far seem to be in progress, i.e. - * this chain must be done. - */ - atc_chain_complete(atchan, desc); - } -} - -/** * atc_advance_work - at the end of a transaction, move forward * @atchan: channel where the transaction ended * @@ -362,6 +403,9 @@ static void atc_advance_work(struct at_dma_chan *atchan) { dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); + if (atc_chan_is_enabled(atchan)) + return; + if (list_empty(&atchan->active_list) || list_is_singular(&atchan->active_list)) { atc_complete_all(atchan); @@ -419,27 +463,43 @@ static void atc_handle_error(struct at_dma_chan *atchan) atc_chain_complete(atchan, bad_desc); } +/** + * atc_handle_cyclic - at the end of a period, run callback function + * @atchan: channel used for cyclic operations + * + * Called with atchan->lock held and bh disabled + */ +static void atc_handle_cyclic(struct at_dma_chan *atchan) +{ + struct at_desc *first = atc_first_active(atchan); + struct dma_async_tx_descriptor *txd = &first->txd; + dma_async_tx_callback callback = txd->callback; + void *param = txd->callback_param; + + dev_vdbg(chan2dev(&atchan->chan_common), + "new cyclic period llp 0x%08x\n", + channel_readl(atchan, DSCR)); + + if (callback) + callback(param); +} /*-- IRQ & Tasklet ---------------------------------------------------*/ static void atc_tasklet(unsigned long data) { struct at_dma_chan *atchan = (struct at_dma_chan *)data; + unsigned long flags; - /* Channel cannot be enabled here */ - if (atc_chan_is_enabled(atchan)) { - dev_err(chan2dev(&atchan->chan_common), - "BUG: channel enabled in tasklet\n"); - return; - } - - spin_lock(&atchan->lock); - if (test_and_clear_bit(0, &atchan->error_status)) + spin_lock_irqsave(&atchan->lock, flags); + if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) atc_handle_error(atchan); + else if (atc_chan_is_cyclic(atchan)) + atc_handle_cyclic(atchan); else atc_advance_work(atchan); - spin_unlock(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); } static irqreturn_t at_dma_interrupt(int irq, void *dev_id) @@ -464,13 +524,16 @@ static irqreturn_t at_dma_interrupt(int irq, void *dev_id) for (i = 0; i < atdma->dma_common.chancnt; i++) { atchan = &atdma->chan[i]; - if (pending & (AT_DMA_CBTC(i) | AT_DMA_ERR(i))) { + if (pending & (AT_DMA_BTC(i) | AT_DMA_ERR(i))) { if (pending & AT_DMA_ERR(i)) { /* Disable channel on AHB error */ - dma_writel(atdma, CHDR, atchan->mask); + dma_writel(atdma, CHDR, + AT_DMA_RES(i) | atchan->mask); /* Give information to tasklet */ - set_bit(0, &atchan->error_status); + set_bit(ATC_IS_ERROR, &atchan->status); } + if (pending & AT_DMA_BTC(i)) + set_bit(ATC_IS_BTC, &atchan->status); tasklet_schedule(&atchan->tasklet); ret = IRQ_HANDLED; } @@ -497,9 +560,10 @@ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx) struct at_desc *desc = txd_to_at_desc(tx); struct at_dma_chan *atchan = to_at_dma_chan(tx->chan); dma_cookie_t cookie; + unsigned long flags; - spin_lock_bh(&atchan->lock); - cookie = atc_assign_cookie(atchan, desc); + spin_lock_irqsave(&atchan->lock, flags); + cookie = dma_cookie_assign(tx); if (list_empty(&atchan->active_list)) { dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n", @@ -512,7 +576,7 @@ static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx) list_add_tail(&desc->desc_node, &atchan->queue); } - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); return cookie; } @@ -548,8 +612,7 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, return NULL; } - ctrla = ATC_DEFAULT_CTRLA; - ctrlb = ATC_DEFAULT_CTRLB + ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN | ATC_SRC_ADDR_MODE_INCR | ATC_DST_ADDR_MODE_INCR | ATC_FC_MEM2MEM; @@ -559,13 +622,13 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, * of the most common optimization. */ if (!((src | dest | len) & 3)) { - ctrla |= ATC_SRC_WIDTH_WORD | ATC_DST_WIDTH_WORD; + ctrla = ATC_SRC_WIDTH_WORD | ATC_DST_WIDTH_WORD; src_width = dst_width = 2; } else if (!((src | dest | len) & 1)) { - ctrla |= ATC_SRC_WIDTH_HALFWORD | ATC_DST_WIDTH_HALFWORD; + ctrla = ATC_SRC_WIDTH_HALFWORD | ATC_DST_WIDTH_HALFWORD; src_width = dst_width = 1; } else { - ctrla |= ATC_SRC_WIDTH_BYTE | ATC_DST_WIDTH_BYTE; + ctrla = ATC_SRC_WIDTH_BYTE | ATC_DST_WIDTH_BYTE; src_width = dst_width = 0; } @@ -583,28 +646,19 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, desc->lli.ctrlb = ctrlb; desc->txd.cookie = 0; - async_tx_ack(&desc->txd); - if (!first) { - first = desc; - } else { - /* inform the HW lli about chaining */ - prev->lli.dscr = desc->txd.phys; - /* insert the link descriptor to the LD ring */ - list_add_tail(&desc->desc_node, - &first->tx_list); - } - prev = desc; + atc_desc_chain(&first, &prev, desc); } /* First descriptor of the chain embedds additional information */ first->txd.cookie = -EBUSY; first->len = len; + first->tx_width = src_width; /* set end-of-link to the last link descriptor of list*/ set_desc_eol(desc); - desc->txd.flags = flags; /* client is in control of this ack */ + first->txd.flags = flags; /* client is in control of this ack */ return &first->txd; @@ -621,14 +675,16 @@ err_desc_get: * @sg_len: number of entries in @scatterlist * @direction: DMA direction * @flags: tx descriptor status flags + * @context: transaction context (ignored) */ static struct dma_async_tx_descriptor * atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, - unsigned int sg_len, enum dma_data_direction direction, - unsigned long flags) + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma_slave *atslave = chan->private; + struct dma_slave_config *sconfig = &atchan->dma_sconfig; struct at_desc *first = NULL; struct at_desc *prev = NULL; u32 ctrla; @@ -640,27 +696,29 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, struct scatterlist *sg; size_t total_len = 0; - dev_vdbg(chan2dev(chan), "prep_slave_sg: %s f0x%lx\n", - direction == DMA_TO_DEVICE ? "TO DEVICE" : "FROM DEVICE", + dev_vdbg(chan2dev(chan), "prep_slave_sg (%d): %s f0x%lx\n", + sg_len, + direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE", flags); if (unlikely(!atslave || !sg_len)) { - dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n"); + dev_dbg(chan2dev(chan), "prep_slave_sg: sg length is zero!\n"); return NULL; } - reg_width = atslave->reg_width; - - ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla; - ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN; + ctrla = ATC_SCSIZE(sconfig->src_maxburst) + | ATC_DCSIZE(sconfig->dst_maxburst); + ctrlb = ATC_IEN; switch (direction) { - case DMA_TO_DEVICE: + case DMA_MEM_TO_DEV: + reg_width = convert_buswidth(sconfig->dst_addr_width); ctrla |= ATC_DST_WIDTH(reg_width); ctrlb |= ATC_DST_ADDR_MODE_FIXED | ATC_SRC_ADDR_MODE_INCR - | ATC_FC_MEM2PER; - reg = atslave->tx_reg; + | ATC_FC_MEM2PER + | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if); + reg = sconfig->dst_addr; for_each_sg(sgl, sg, sg_len, i) { struct at_desc *desc; u32 len; @@ -670,8 +728,13 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!desc) goto err_desc_get; - mem = sg_phys(sg); + mem = sg_dma_address(sg); len = sg_dma_len(sg); + if (unlikely(!len)) { + dev_dbg(chan2dev(chan), + "prep_slave_sg: sg(%d) data length is zero\n", i); + goto err; + } mem_width = 2; if (unlikely(mem & 3 || len & 3)) mem_width = 0; @@ -683,26 +746,19 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, | len >> mem_width; desc->lli.ctrlb = ctrlb; - if (!first) { - first = desc; - } else { - /* inform the HW lli about chaining */ - prev->lli.dscr = desc->txd.phys; - /* insert the link descriptor to the LD ring */ - list_add_tail(&desc->desc_node, - &first->tx_list); - } - prev = desc; + atc_desc_chain(&first, &prev, desc); total_len += len; } break; - case DMA_FROM_DEVICE: + case DMA_DEV_TO_MEM: + reg_width = convert_buswidth(sconfig->src_addr_width); ctrla |= ATC_SRC_WIDTH(reg_width); ctrlb |= ATC_DST_ADDR_MODE_INCR | ATC_SRC_ADDR_MODE_FIXED - | ATC_FC_PER2MEM; + | ATC_FC_PER2MEM + | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if); - reg = atslave->rx_reg; + reg = sconfig->src_addr; for_each_sg(sgl, sg, sg_len, i) { struct at_desc *desc; u32 len; @@ -712,8 +768,13 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, if (!desc) goto err_desc_get; - mem = sg_phys(sg); + mem = sg_dma_address(sg); len = sg_dma_len(sg); + if (unlikely(!len)) { + dev_dbg(chan2dev(chan), + "prep_slave_sg: sg(%d) data length is zero\n", i); + goto err; + } mem_width = 2; if (unlikely(mem & 3 || len & 3)) mem_width = 0; @@ -722,19 +783,10 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, desc->lli.daddr = mem; desc->lli.ctrla = ctrla | ATC_DST_WIDTH(mem_width) - | len >> mem_width; + | len >> reg_width; desc->lli.ctrlb = ctrlb; - if (!first) { - first = desc; - } else { - /* inform the HW lli about chaining */ - prev->lli.dscr = desc->txd.phys; - /* insert the link descriptor to the LD ring */ - list_add_tail(&desc->desc_node, - &first->tx_list); - } - prev = desc; + atc_desc_chain(&first, &prev, desc); total_len += len; } break; @@ -748,53 +800,259 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, /* First descriptor of the chain embedds additional information */ first->txd.cookie = -EBUSY; first->len = total_len; + first->tx_width = reg_width; - /* last link descriptor of list is responsible of flags */ - prev->txd.flags = flags; /* client is in control of this ack */ + /* first link descriptor of list is responsible of flags */ + first->txd.flags = flags; /* client is in control of this ack */ return &first->txd; err_desc_get: dev_err(chan2dev(chan), "not enough descriptors available\n"); +err: atc_desc_put(atchan, first); return NULL; } +/** + * atc_dma_cyclic_check_values + * Check for too big/unaligned periods and unaligned DMA buffer + */ +static int +atc_dma_cyclic_check_values(unsigned int reg_width, dma_addr_t buf_addr, + size_t period_len) +{ + if (period_len > (ATC_BTSIZE_MAX << reg_width)) + goto err_out; + if (unlikely(period_len & ((1 << reg_width) - 1))) + goto err_out; + if (unlikely(buf_addr & ((1 << reg_width) - 1))) + goto err_out; + + return 0; + +err_out: + return -EINVAL; +} + +/** + * atc_dma_cyclic_fill_desc - Fill one period descriptor + */ +static int +atc_dma_cyclic_fill_desc(struct dma_chan *chan, struct at_desc *desc, + unsigned int period_index, dma_addr_t buf_addr, + unsigned int reg_width, size_t period_len, + enum dma_transfer_direction direction) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct dma_slave_config *sconfig = &atchan->dma_sconfig; + u32 ctrla; + + /* prepare common CRTLA value */ + ctrla = ATC_SCSIZE(sconfig->src_maxburst) + | ATC_DCSIZE(sconfig->dst_maxburst) + | ATC_DST_WIDTH(reg_width) + | ATC_SRC_WIDTH(reg_width) + | period_len >> reg_width; + + switch (direction) { + case DMA_MEM_TO_DEV: + desc->lli.saddr = buf_addr + (period_len * period_index); + desc->lli.daddr = sconfig->dst_addr; + desc->lli.ctrla = ctrla; + desc->lli.ctrlb = ATC_DST_ADDR_MODE_FIXED + | ATC_SRC_ADDR_MODE_INCR + | ATC_FC_MEM2PER + | ATC_SIF(atchan->mem_if) + | ATC_DIF(atchan->per_if); + break; + + case DMA_DEV_TO_MEM: + desc->lli.saddr = sconfig->src_addr; + desc->lli.daddr = buf_addr + (period_len * period_index); + desc->lli.ctrla = ctrla; + desc->lli.ctrlb = ATC_DST_ADDR_MODE_INCR + | ATC_SRC_ADDR_MODE_FIXED + | ATC_FC_PER2MEM + | ATC_SIF(atchan->per_if) + | ATC_DIF(atchan->mem_if); + break; + + default: + return -EINVAL; + } + + return 0; +} + +/** + * atc_prep_dma_cyclic - prepare the cyclic DMA transfer + * @chan: the DMA channel to prepare + * @buf_addr: physical DMA address where the buffer starts + * @buf_len: total number of bytes for the entire buffer + * @period_len: number of bytes for each period + * @direction: transfer direction, to or from device + * @flags: tx descriptor status flags + * @context: transfer context (ignored) + */ +static struct dma_async_tx_descriptor * +atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, + size_t period_len, enum dma_transfer_direction direction, + unsigned long flags, void *context) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + struct at_dma_slave *atslave = chan->private; + struct dma_slave_config *sconfig = &atchan->dma_sconfig; + struct at_desc *first = NULL; + struct at_desc *prev = NULL; + unsigned long was_cyclic; + unsigned int reg_width; + unsigned int periods = buf_len / period_len; + unsigned int i; + + dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@0x%08x - %d (%d/%d)\n", + direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE", + buf_addr, + periods, buf_len, period_len); + + if (unlikely(!atslave || !buf_len || !period_len)) { + dev_dbg(chan2dev(chan), "prep_dma_cyclic: length is zero!\n"); + return NULL; + } + + was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status); + if (was_cyclic) { + dev_dbg(chan2dev(chan), "prep_dma_cyclic: channel in use!\n"); + return NULL; + } + + if (unlikely(!is_slave_direction(direction))) + goto err_out; + + if (sconfig->direction == DMA_MEM_TO_DEV) + reg_width = convert_buswidth(sconfig->dst_addr_width); + else + reg_width = convert_buswidth(sconfig->src_addr_width); + + /* Check for too big/unaligned periods and unaligned DMA buffer */ + if (atc_dma_cyclic_check_values(reg_width, buf_addr, period_len)) + goto err_out; + + /* build cyclic linked list */ + for (i = 0; i < periods; i++) { + struct at_desc *desc; + + desc = atc_desc_get(atchan); + if (!desc) + goto err_desc_get; + + if (atc_dma_cyclic_fill_desc(chan, desc, i, buf_addr, + reg_width, period_len, direction)) + goto err_desc_get; + + atc_desc_chain(&first, &prev, desc); + } + + /* lets make a cyclic list */ + prev->lli.dscr = first->txd.phys; + + /* First descriptor of the chain embedds additional information */ + first->txd.cookie = -EBUSY; + first->len = buf_len; + first->tx_width = reg_width; + + return &first->txd; + +err_desc_get: + dev_err(chan2dev(chan), "not enough descriptors available\n"); + atc_desc_put(atchan, first); +err_out: + clear_bit(ATC_IS_CYCLIC, &atchan->status); + return NULL; +} + +static int set_runtime_config(struct dma_chan *chan, + struct dma_slave_config *sconfig) +{ + struct at_dma_chan *atchan = to_at_dma_chan(chan); + + /* Check if it is chan is configured for slave transfers */ + if (!chan->private) + return -EINVAL; + + memcpy(&atchan->dma_sconfig, sconfig, sizeof(*sconfig)); + + convert_burst(&atchan->dma_sconfig.src_maxburst); + convert_burst(&atchan->dma_sconfig.dst_maxburst); + + return 0; +} + + static int atc_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg) { struct at_dma_chan *atchan = to_at_dma_chan(chan); struct at_dma *atdma = to_at_dma(chan->device); - struct at_desc *desc, *_desc; + int chan_id = atchan->chan_common.chan_id; + unsigned long flags; + LIST_HEAD(list); - /* Only supports DMA_TERMINATE_ALL */ - if (cmd != DMA_TERMINATE_ALL) - return -ENXIO; + dev_vdbg(chan2dev(chan), "atc_control (%d)\n", cmd); - /* - * This is only called when something went wrong elsewhere, so - * we don't really care about the data. Just disable the - * channel. We still have to poll the channel enable bit due - * to AHB/HSB limitations. - */ - spin_lock_bh(&atchan->lock); + if (cmd == DMA_PAUSE) { + spin_lock_irqsave(&atchan->lock, flags); - dma_writel(atdma, CHDR, atchan->mask); + dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id)); + set_bit(ATC_IS_PAUSED, &atchan->status); - /* confirm that this channel is disabled */ - while (dma_readl(atdma, CHSR) & atchan->mask) - cpu_relax(); + spin_unlock_irqrestore(&atchan->lock, flags); + } else if (cmd == DMA_RESUME) { + if (!atc_chan_is_paused(atchan)) + return 0; - /* active_list entries will end up before queued entries */ - list_splice_init(&atchan->queue, &list); - list_splice_init(&atchan->active_list, &list); + spin_lock_irqsave(&atchan->lock, flags); - /* Flush all pending and queued descriptors */ - list_for_each_entry_safe(desc, _desc, &list, desc_node) - atc_chain_complete(atchan, desc); + dma_writel(atdma, CHDR, AT_DMA_RES(chan_id)); + clear_bit(ATC_IS_PAUSED, &atchan->status); - spin_unlock_bh(&atchan->lock); + spin_unlock_irqrestore(&atchan->lock, flags); + } else if (cmd == DMA_TERMINATE_ALL) { + struct at_desc *desc, *_desc; + /* + * This is only called when something went wrong elsewhere, so + * we don't really care about the data. Just disable the + * channel. We still have to poll the channel enable bit due + * to AHB/HSB limitations. + */ + spin_lock_irqsave(&atchan->lock, flags); + + /* disabling channel: must also remove suspend state */ + dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); + + /* confirm that this channel is disabled */ + while (dma_readl(atdma, CHSR) & atchan->mask) + cpu_relax(); + + /* active_list entries will end up before queued entries */ + list_splice_init(&atchan->queue, &list); + list_splice_init(&atchan->active_list, &list); + + /* Flush all pending and queued descriptors */ + list_for_each_entry_safe(desc, _desc, &list, desc_node) + atc_chain_complete(atchan, desc); + + clear_bit(ATC_IS_PAUSED, &atchan->status); + /* if channel dedicated to cyclic operations, free it */ + clear_bit(ATC_IS_CYCLIC, &atchan->status); + + spin_unlock_irqrestore(&atchan->lock, flags); + } else if (cmd == DMA_SLAVE_CONFIG) { + return set_runtime_config(chan, (struct dma_slave_config *)arg); + } else { + return -ENXIO; + } return 0; } @@ -815,31 +1073,36 @@ atc_tx_status(struct dma_chan *chan, struct dma_tx_state *txstate) { struct at_dma_chan *atchan = to_at_dma_chan(chan); - dma_cookie_t last_used; - dma_cookie_t last_complete; + unsigned long flags; enum dma_status ret; + int bytes = 0; - spin_lock_bh(&atchan->lock); + ret = dma_cookie_status(chan, cookie, txstate); + if (ret == DMA_COMPLETE) + return ret; + /* + * There's no point calculating the residue if there's + * no txstate to store the value. + */ + if (!txstate) + return DMA_ERROR; - last_complete = atchan->completed_cookie; - last_used = chan->cookie; + spin_lock_irqsave(&atchan->lock, flags); - ret = dma_async_is_complete(cookie, last_complete, last_used); - if (ret != DMA_SUCCESS) { - atc_cleanup_descriptors(atchan); + /* Get number of bytes left in the active transactions */ + bytes = atc_get_bytes_left(chan); - last_complete = atchan->completed_cookie; - last_used = chan->cookie; + spin_unlock_irqrestore(&atchan->lock, flags); - ret = dma_async_is_complete(cookie, last_complete, last_used); + if (unlikely(bytes < 0)) { + dev_vdbg(chan2dev(chan), "get residual bytes error\n"); + return DMA_ERROR; + } else { + dma_set_residue(txstate, bytes); } - spin_unlock_bh(&atchan->lock); - - dma_set_tx_state(txstate, last_complete, last_used, 0); - dev_vdbg(chan2dev(chan), "tx_status: %d (d%d, u%d)\n", - cookie, last_complete ? last_complete : 0, - last_used ? last_used : 0); + dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %d\n", + ret, cookie, bytes); return ret; } @@ -851,14 +1114,17 @@ atc_tx_status(struct dma_chan *chan, static void atc_issue_pending(struct dma_chan *chan) { struct at_dma_chan *atchan = to_at_dma_chan(chan); + unsigned long flags; dev_vdbg(chan2dev(chan), "issue_pending\n"); - if (!atc_chan_is_enabled(atchan)) { - spin_lock_bh(&atchan->lock); - atc_advance_work(atchan); - spin_unlock_bh(&atchan->lock); - } + /* Not needed for cyclic transfers */ + if (atc_chan_is_cyclic(atchan)) + return; + + spin_lock_irqsave(&atchan->lock, flags); + atc_advance_work(atchan); + spin_unlock_irqrestore(&atchan->lock, flags); } /** @@ -874,6 +1140,7 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) struct at_dma *atdma = to_at_dma(chan->device); struct at_desc *desc; struct at_dma_slave *atslave; + unsigned long flags; int i; u32 cfg; LIST_HEAD(tmp_list); @@ -896,7 +1163,7 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) */ BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev); - /* if cfg configuration specified take it instad of default */ + /* if cfg configuration specified take it instead of default */ if (atslave->cfg) cfg = atslave->cfg; } @@ -917,11 +1184,12 @@ static int atc_alloc_chan_resources(struct dma_chan *chan) list_add_tail(&desc->desc_node, &tmp_list); } - spin_lock_bh(&atchan->lock); + spin_lock_irqsave(&atchan->lock, flags); atchan->descs_allocated = i; + atchan->remain_desc = 0; list_splice(&tmp_list, &atchan->free_list); - atchan->completed_cookie = chan->cookie = 1; - spin_unlock_bh(&atchan->lock); + dma_cookie_init(chan); + spin_unlock_irqrestore(&atchan->lock, flags); /* channel parameters */ channel_writel(atchan, CFG, cfg); @@ -960,13 +1228,143 @@ static void atc_free_chan_resources(struct dma_chan *chan) } list_splice_init(&atchan->free_list, &list); atchan->descs_allocated = 0; + atchan->status = 0; + atchan->remain_desc = 0; dev_vdbg(chan2dev(chan), "free_chan_resources: done\n"); } +#ifdef CONFIG_OF +static bool at_dma_filter(struct dma_chan *chan, void *slave) +{ + struct at_dma_slave *atslave = slave; + + if (atslave->dma_dev == chan->device->dev) { + chan->private = atslave; + return true; + } else { + return false; + } +} + +static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *of_dma) +{ + struct dma_chan *chan; + struct at_dma_chan *atchan; + struct at_dma_slave *atslave; + dma_cap_mask_t mask; + unsigned int per_id; + struct platform_device *dmac_pdev; + + if (dma_spec->args_count != 2) + return NULL; + + dmac_pdev = of_find_device_by_node(dma_spec->np); + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + atslave = devm_kzalloc(&dmac_pdev->dev, sizeof(*atslave), GFP_KERNEL); + if (!atslave) + return NULL; + + atslave->cfg = ATC_DST_H2SEL_HW | ATC_SRC_H2SEL_HW; + /* + * We can fill both SRC_PER and DST_PER, one of these fields will be + * ignored depending on DMA transfer direction. + */ + per_id = dma_spec->args[1] & AT91_DMA_CFG_PER_ID_MASK; + atslave->cfg |= ATC_DST_PER_MSB(per_id) | ATC_DST_PER(per_id) + | ATC_SRC_PER_MSB(per_id) | ATC_SRC_PER(per_id); + /* + * We have to translate the value we get from the device tree since + * the half FIFO configuration value had to be 0 to keep backward + * compatibility. + */ + switch (dma_spec->args[1] & AT91_DMA_CFG_FIFOCFG_MASK) { + case AT91_DMA_CFG_FIFOCFG_ALAP: + atslave->cfg |= ATC_FIFOCFG_LARGESTBURST; + break; + case AT91_DMA_CFG_FIFOCFG_ASAP: + atslave->cfg |= ATC_FIFOCFG_ENOUGHSPACE; + break; + case AT91_DMA_CFG_FIFOCFG_HALF: + default: + atslave->cfg |= ATC_FIFOCFG_HALFFIFO; + } + atslave->dma_dev = &dmac_pdev->dev; + + chan = dma_request_channel(mask, at_dma_filter, atslave); + if (!chan) + return NULL; + + atchan = to_at_dma_chan(chan); + atchan->per_if = dma_spec->args[0] & 0xff; + atchan->mem_if = (dma_spec->args[0] >> 16) & 0xff; + + return chan; +} +#else +static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec, + struct of_dma *of_dma) +{ + return NULL; +} +#endif /*-- Module Management -----------------------------------------------*/ +/* cap_mask is a multi-u32 bitfield, fill it with proper C code. */ +static struct at_dma_platform_data at91sam9rl_config = { + .nr_channels = 2, +}; +static struct at_dma_platform_data at91sam9g45_config = { + .nr_channels = 8, +}; + +#if defined(CONFIG_OF) +static const struct of_device_id atmel_dma_dt_ids[] = { + { + .compatible = "atmel,at91sam9rl-dma", + .data = &at91sam9rl_config, + }, { + .compatible = "atmel,at91sam9g45-dma", + .data = &at91sam9g45_config, + }, { + /* sentinel */ + } +}; + +MODULE_DEVICE_TABLE(of, atmel_dma_dt_ids); +#endif + +static const struct platform_device_id atdma_devtypes[] = { + { + .name = "at91sam9rl_dma", + .driver_data = (unsigned long) &at91sam9rl_config, + }, { + .name = "at91sam9g45_dma", + .driver_data = (unsigned long) &at91sam9g45_config, + }, { + /* sentinel */ + } +}; + +static inline const struct at_dma_platform_data * __init at_dma_get_driver_data( + struct platform_device *pdev) +{ + if (pdev->dev.of_node) { + const struct of_device_id *match; + match = of_match_node(atmel_dma_dt_ids, pdev->dev.of_node); + if (match == NULL) + return NULL; + return match->data; + } + return (struct at_dma_platform_data *) + platform_get_device_id(pdev)->driver_data; +} + /** * at_dma_off - disable DMA controller * @atdma: the Atmel HDAMC device @@ -985,18 +1383,23 @@ static void at_dma_off(struct at_dma *atdma) static int __init at_dma_probe(struct platform_device *pdev) { - struct at_dma_platform_data *pdata; struct resource *io; struct at_dma *atdma; size_t size; int irq; int err; int i; + const struct at_dma_platform_data *plat_dat; - /* get DMA Controller parameters from platform */ - pdata = pdev->dev.platform_data; - if (!pdata || pdata->nr_channels > AT_DMA_MAX_NR_CHANNELS) - return -EINVAL; + /* setup platform data for each SoC */ + dma_cap_set(DMA_MEMCPY, at91sam9rl_config.cap_mask); + dma_cap_set(DMA_MEMCPY, at91sam9g45_config.cap_mask); + dma_cap_set(DMA_SLAVE, at91sam9g45_config.cap_mask); + + /* get DMA parameters from controller type */ + plat_dat = at_dma_get_driver_data(pdev); + if (!plat_dat) + return -ENODEV; io = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (!io) @@ -1007,16 +1410,16 @@ static int __init at_dma_probe(struct platform_device *pdev) return irq; size = sizeof(struct at_dma); - size += pdata->nr_channels * sizeof(struct at_dma_chan); + size += plat_dat->nr_channels * sizeof(struct at_dma_chan); atdma = kzalloc(size, GFP_KERNEL); if (!atdma) return -ENOMEM; - /* discover transaction capabilites from the platform data */ - atdma->dma_common.cap_mask = pdata->cap_mask; - atdma->all_chan_mask = (1 << pdata->nr_channels) - 1; + /* discover transaction capabilities */ + atdma->dma_common.cap_mask = plat_dat->cap_mask; + atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1; - size = io->end - io->start + 1; + size = resource_size(io); if (!request_mem_region(io->start, size, pdev->dev.driver->name)) { err = -EBUSY; goto err_kfree; @@ -1033,7 +1436,9 @@ static int __init at_dma_probe(struct platform_device *pdev) err = PTR_ERR(atdma->clk); goto err_clk; } - clk_enable(atdma->clk); + err = clk_prepare_enable(atdma->clk); + if (err) + goto err_clk_prepare; /* force dma off, just in case */ at_dma_off(atdma); @@ -1060,12 +1465,13 @@ static int __init at_dma_probe(struct platform_device *pdev) /* initialize channels related values */ INIT_LIST_HEAD(&atdma->dma_common.channels); - for (i = 0; i < pdata->nr_channels; i++, atdma->dma_common.chancnt++) { + for (i = 0; i < plat_dat->nr_channels; i++) { struct at_dma_chan *atchan = &atdma->chan[i]; + atchan->mem_if = AT_DMA_MEM_IF; + atchan->per_if = AT_DMA_PER_IF; atchan->chan_common.device = &atdma->dma_common; - atchan->chan_common.cookie = atchan->completed_cookie = 1; - atchan->chan_common.chan_id = i; + dma_cookie_init(&atchan->chan_common); list_add_tail(&atchan->chan_common.device_node, &atdma->dma_common.channels); @@ -1079,7 +1485,7 @@ static int __init at_dma_probe(struct platform_device *pdev) tasklet_init(&atchan->tasklet, atc_tasklet, (unsigned long)atchan); - atc_enable_irq(atchan); + atc_enable_chan_irq(atdma, i); } /* set base routines */ @@ -1095,6 +1501,9 @@ static int __init at_dma_probe(struct platform_device *pdev) if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) { atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg; + /* controller can do slave DMA: can trigger cyclic transfers */ + dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask); + atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic; atdma->dma_common.device_control = atc_control; } @@ -1103,17 +1512,34 @@ static int __init at_dma_probe(struct platform_device *pdev) dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s), %d channels\n", dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "", dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask) ? "slave " : "", - atdma->dma_common.chancnt); + plat_dat->nr_channels); dma_async_device_register(&atdma->dma_common); + /* + * Do not return an error if the dmac node is not present in order to + * not break the existing way of requesting channel with + * dma_request_channel(). + */ + if (pdev->dev.of_node) { + err = of_dma_controller_register(pdev->dev.of_node, + at_dma_xlate, atdma); + if (err) { + dev_err(&pdev->dev, "could not register of_dma_controller\n"); + goto err_of_dma_controller_register; + } + } + return 0; +err_of_dma_controller_register: + dma_async_device_unregister(&atdma->dma_common); + dma_pool_destroy(atdma->dma_desc_pool); err_pool_create: - platform_set_drvdata(pdev, NULL); free_irq(platform_get_irq(pdev, 0), atdma); err_irq: - clk_disable(atdma->clk); + clk_disable_unprepare(atdma->clk); +err_clk_prepare: clk_put(atdma->clk); err_clk: iounmap(atdma->regs); @@ -1125,7 +1551,7 @@ err_kfree: return err; } -static int __exit at_dma_remove(struct platform_device *pdev) +static int at_dma_remove(struct platform_device *pdev) { struct at_dma *atdma = platform_get_drvdata(pdev); struct dma_chan *chan, *_chan; @@ -1135,7 +1561,6 @@ static int __exit at_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&atdma->dma_common); dma_pool_destroy(atdma->dma_desc_pool); - platform_set_drvdata(pdev, NULL); free_irq(platform_get_irq(pdev, 0), atdma); list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, @@ -1143,21 +1568,20 @@ static int __exit at_dma_remove(struct platform_device *pdev) struct at_dma_chan *atchan = to_at_dma_chan(chan); /* Disable interrupts */ - atc_disable_irq(atchan); - tasklet_disable(&atchan->tasklet); + atc_disable_chan_irq(atdma, chan->chan_id); tasklet_kill(&atchan->tasklet); list_del(&chan->device_node); } - clk_disable(atdma->clk); + clk_disable_unprepare(atdma->clk); clk_put(atdma->clk); iounmap(atdma->regs); atdma->regs = NULL; io = platform_get_resource(pdev, IORESOURCE_MEM, 0); - release_mem_region(io->start, io->end - io->start + 1); + release_mem_region(io->start, resource_size(io)); kfree(atdma); @@ -1169,40 +1593,127 @@ static void at_dma_shutdown(struct platform_device *pdev) struct at_dma *atdma = platform_get_drvdata(pdev); at_dma_off(platform_get_drvdata(pdev)); - clk_disable(atdma->clk); + clk_disable_unprepare(atdma->clk); +} + +static int at_dma_prepare(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct at_dma *atdma = platform_get_drvdata(pdev); + struct dma_chan *chan, *_chan; + + list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + device_node) { + struct at_dma_chan *atchan = to_at_dma_chan(chan); + /* wait for transaction completion (except in cyclic case) */ + if (atc_chan_is_enabled(atchan) && !atc_chan_is_cyclic(atchan)) + return -EAGAIN; + } + return 0; +} + +static void atc_suspend_cyclic(struct at_dma_chan *atchan) +{ + struct dma_chan *chan = &atchan->chan_common; + + /* Channel should be paused by user + * do it anyway even if it is not done already */ + if (!atc_chan_is_paused(atchan)) { + dev_warn(chan2dev(chan), + "cyclic channel not paused, should be done by channel user\n"); + atc_control(chan, DMA_PAUSE, 0); + } + + /* now preserve additional data for cyclic operations */ + /* next descriptor address in the cyclic list */ + atchan->save_dscr = channel_readl(atchan, DSCR); + + vdbg_dump_regs(atchan); } static int at_dma_suspend_noirq(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct at_dma *atdma = platform_get_drvdata(pdev); + struct dma_chan *chan, *_chan; - at_dma_off(platform_get_drvdata(pdev)); - clk_disable(atdma->clk); + /* preserve data */ + list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + device_node) { + struct at_dma_chan *atchan = to_at_dma_chan(chan); + + if (atc_chan_is_cyclic(atchan)) + atc_suspend_cyclic(atchan); + atchan->save_cfg = channel_readl(atchan, CFG); + } + atdma->save_imr = dma_readl(atdma, EBCIMR); + + /* disable DMA controller */ + at_dma_off(atdma); + clk_disable_unprepare(atdma->clk); return 0; } +static void atc_resume_cyclic(struct at_dma_chan *atchan) +{ + struct at_dma *atdma = to_at_dma(atchan->chan_common.device); + + /* restore channel status for cyclic descriptors list: + * next descriptor in the cyclic list at the time of suspend */ + channel_writel(atchan, SADDR, 0); + channel_writel(atchan, DADDR, 0); + channel_writel(atchan, CTRLA, 0); + channel_writel(atchan, CTRLB, 0); + channel_writel(atchan, DSCR, atchan->save_dscr); + dma_writel(atdma, CHER, atchan->mask); + + /* channel pause status should be removed by channel user + * We cannot take the initiative to do it here */ + + vdbg_dump_regs(atchan); +} + static int at_dma_resume_noirq(struct device *dev) { struct platform_device *pdev = to_platform_device(dev); struct at_dma *atdma = platform_get_drvdata(pdev); + struct dma_chan *chan, *_chan; - clk_enable(atdma->clk); + /* bring back DMA controller */ + clk_prepare_enable(atdma->clk); dma_writel(atdma, EN, AT_DMA_ENABLE); + + /* clear any pending interrupt */ + while (dma_readl(atdma, EBCISR)) + cpu_relax(); + + /* restore saved data */ + dma_writel(atdma, EBCIER, atdma->save_imr); + list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels, + device_node) { + struct at_dma_chan *atchan = to_at_dma_chan(chan); + + channel_writel(atchan, CFG, atchan->save_cfg); + if (atc_chan_is_cyclic(atchan)) + atc_resume_cyclic(atchan); + } return 0; } static const struct dev_pm_ops at_dma_dev_pm_ops = { + .prepare = at_dma_prepare, .suspend_noirq = at_dma_suspend_noirq, .resume_noirq = at_dma_resume_noirq, }; static struct platform_driver at_dma_driver = { - .remove = __exit_p(at_dma_remove), + .remove = at_dma_remove, .shutdown = at_dma_shutdown, + .id_table = atdma_devtypes, .driver = { .name = "at_hdmac", .pm = &at_dma_dev_pm_ops, + .of_match_table = of_match_ptr(atmel_dma_dt_ids), }, }; @@ -1210,7 +1721,7 @@ static int __init at_dma_init(void) { return platform_driver_probe(&at_dma_driver, at_dma_probe); } -module_init(at_dma_init); +subsys_initcall(at_dma_init); static void __exit at_dma_exit(void) { |