diff options
Diffstat (limited to 'drivers/dma/shdma.c')
-rw-r--r-- | drivers/dma/shdma.c | 842 |
1 files changed, 579 insertions, 263 deletions
diff --git a/drivers/dma/shdma.c b/drivers/dma/shdma.c index d10cc899c46..7cc31b3f40d 100644 --- a/drivers/dma/shdma.c +++ b/drivers/dma/shdma.c @@ -19,13 +19,16 @@ #include <linux/init.h> #include <linux/module.h> +#include <linux/slab.h> #include <linux/interrupt.h> #include <linux/dmaengine.h> #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/platform_device.h> -#include <cpu/dma.h> -#include <asm/dma-sh.h> +#include <linux/pm_runtime.h> + +#include <asm/dmaengine.h> + #include "shdma.h" /* DMA descriptor control */ @@ -38,33 +41,32 @@ enum sh_dmae_desc_status { }; #define NR_DESCS_PER_CHANNEL 32 -/* - * Define the default configuration for dual address memory-memory transfer. - * The 0x400 value represents auto-request, external->external. - * - * And this driver set 4byte burst mode. - * If you want to change mode, you need to change RS_DEFAULT of value. - * (ex 1byte burst mode -> (RS_DUAL & ~TS_32) - */ -#define RS_DEFAULT (RS_DUAL) +/* Default MEMCPY transfer size = 2^2 = 4 bytes */ +#define LOG2_DEFAULT_XFER_SIZE 2 + +/* A bitmask with bits enough for enum sh_dmae_slave_chan_id */ +static unsigned long sh_dmae_slave_used[BITS_TO_LONGS(SHDMA_SLAVE_NUMBER)]; static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all); -#define SH_DMAC_CHAN_BASE(id) (dma_base_addr[id]) static void sh_dmae_writel(struct sh_dmae_chan *sh_dc, u32 data, u32 reg) { - ctrl_outl(data, (SH_DMAC_CHAN_BASE(sh_dc->id) + reg)); + __raw_writel(data, sh_dc->base + reg / sizeof(u32)); } static u32 sh_dmae_readl(struct sh_dmae_chan *sh_dc, u32 reg) { - return ctrl_inl((SH_DMAC_CHAN_BASE(sh_dc->id) + reg)); + return __raw_readl(sh_dc->base + reg / sizeof(u32)); } -static void dmae_init(struct sh_dmae_chan *sh_chan) +static u16 dmaor_read(struct sh_dmae_device *shdev) { - u32 chcr = RS_DEFAULT; /* default is DUAL mode */ - sh_dmae_writel(sh_chan, chcr, CHCR); + return __raw_readw(shdev->chan_reg + DMAOR / sizeof(u32)); +} + +static void dmaor_write(struct sh_dmae_device *shdev, u16 data) +{ + __raw_writew(data, shdev->chan_reg + DMAOR / sizeof(u32)); } /* @@ -72,50 +74,75 @@ static void dmae_init(struct sh_dmae_chan *sh_chan) * * SH7780 has two DMAOR register */ -static void sh_dmae_ctl_stop(int id) +static void sh_dmae_ctl_stop(struct sh_dmae_device *shdev) { - unsigned short dmaor = dmaor_read_reg(id); + unsigned short dmaor = dmaor_read(shdev); - dmaor &= ~(DMAOR_NMIF | DMAOR_AE); - dmaor_write_reg(id, dmaor); + dmaor_write(shdev, dmaor & ~(DMAOR_NMIF | DMAOR_AE | DMAOR_DME)); } -static int sh_dmae_rst(int id) +static int sh_dmae_rst(struct sh_dmae_device *shdev) { unsigned short dmaor; - sh_dmae_ctl_stop(id); - dmaor = dmaor_read_reg(id) | DMAOR_INIT; + sh_dmae_ctl_stop(shdev); + dmaor = dmaor_read(shdev) | shdev->pdata->dmaor_init; - dmaor_write_reg(id, dmaor); - if (dmaor_read_reg(id) & (DMAOR_AE | DMAOR_NMIF)) { - pr_warning(KERN_ERR "dma-sh: Can't initialize DMAOR.\n"); + dmaor_write(shdev, dmaor); + if (dmaor_read(shdev) & (DMAOR_AE | DMAOR_NMIF)) { + pr_warning("dma-sh: Can't initialize DMAOR.\n"); return -EINVAL; } return 0; } -static int dmae_is_busy(struct sh_dmae_chan *sh_chan) +static bool dmae_is_busy(struct sh_dmae_chan *sh_chan) { u32 chcr = sh_dmae_readl(sh_chan, CHCR); - if (chcr & CHCR_DE) { - if (!(chcr & CHCR_TE)) - return -EBUSY; /* working */ - } - return 0; /* waiting */ + + if ((chcr & (CHCR_DE | CHCR_TE)) == CHCR_DE) + return true; /* working */ + + return false; /* waiting */ } -static inline unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan) +static unsigned int calc_xmit_shift(struct sh_dmae_chan *sh_chan, u32 chcr) { - u32 chcr = sh_dmae_readl(sh_chan, CHCR); - return ts_shift[(chcr & CHCR_TS_MASK) >> CHCR_TS_SHIFT]; + struct sh_dmae_device *shdev = container_of(sh_chan->common.device, + struct sh_dmae_device, common); + struct sh_dmae_pdata *pdata = shdev->pdata; + int cnt = ((chcr & pdata->ts_low_mask) >> pdata->ts_low_shift) | + ((chcr & pdata->ts_high_mask) >> pdata->ts_high_shift); + + if (cnt >= pdata->ts_shift_num) + cnt = 0; + + return pdata->ts_shift[cnt]; +} + +static u32 log2size_to_chcr(struct sh_dmae_chan *sh_chan, int l2size) +{ + struct sh_dmae_device *shdev = container_of(sh_chan->common.device, + struct sh_dmae_device, common); + struct sh_dmae_pdata *pdata = shdev->pdata; + int i; + + for (i = 0; i < pdata->ts_shift_num; i++) + if (pdata->ts_shift[i] == l2size) + break; + + if (i == pdata->ts_shift_num) + i = 0; + + return ((i << pdata->ts_low_shift) & pdata->ts_low_mask) | + ((i << pdata->ts_high_shift) & pdata->ts_high_mask); } static void dmae_set_reg(struct sh_dmae_chan *sh_chan, struct sh_dmae_regs *hw) { sh_dmae_writel(sh_chan, hw->sar, SAR); sh_dmae_writel(sh_chan, hw->dar, DAR); - sh_dmae_writel(sh_chan, hw->tcr >> calc_xmit_shift(sh_chan), TCR); + sh_dmae_writel(sh_chan, hw->tcr >> sh_chan->xmit_shift, TCR); } static void dmae_start(struct sh_dmae_chan *sh_chan) @@ -123,7 +150,7 @@ static void dmae_start(struct sh_dmae_chan *sh_chan) u32 chcr = sh_dmae_readl(sh_chan, CHCR); chcr |= CHCR_DE | CHCR_IE; - sh_dmae_writel(sh_chan, chcr, CHCR); + sh_dmae_writel(sh_chan, chcr & ~CHCR_TE, CHCR); } static void dmae_halt(struct sh_dmae_chan *sh_chan) @@ -134,55 +161,44 @@ static void dmae_halt(struct sh_dmae_chan *sh_chan) sh_dmae_writel(sh_chan, chcr, CHCR); } +static void dmae_init(struct sh_dmae_chan *sh_chan) +{ + /* + * Default configuration for dual address memory-memory transfer. + * 0x400 represents auto-request. + */ + u32 chcr = DM_INC | SM_INC | 0x400 | log2size_to_chcr(sh_chan, + LOG2_DEFAULT_XFER_SIZE); + sh_chan->xmit_shift = calc_xmit_shift(sh_chan, chcr); + sh_dmae_writel(sh_chan, chcr, CHCR); +} + static int dmae_set_chcr(struct sh_dmae_chan *sh_chan, u32 val) { - int ret = dmae_is_busy(sh_chan); /* When DMA was working, can not set data to CHCR */ - if (ret) - return ret; + if (dmae_is_busy(sh_chan)) + return -EBUSY; + sh_chan->xmit_shift = calc_xmit_shift(sh_chan, val); sh_dmae_writel(sh_chan, val, CHCR); + return 0; } -#define DMARS1_ADDR 0x04 -#define DMARS2_ADDR 0x08 -#define DMARS_SHIFT 8 -#define DMARS_CHAN_MSK 0x01 static int dmae_set_dmars(struct sh_dmae_chan *sh_chan, u16 val) { - u32 addr; - int shift = 0; - int ret = dmae_is_busy(sh_chan); - if (ret) - return ret; - - if (sh_chan->id & DMARS_CHAN_MSK) - shift = DMARS_SHIFT; - - switch (sh_chan->id) { - /* DMARS0 */ - case 0: - case 1: - addr = SH_DMARS_BASE; - break; - /* DMARS1 */ - case 2: - case 3: - addr = (SH_DMARS_BASE + DMARS1_ADDR); - break; - /* DMARS2 */ - case 4: - case 5: - addr = (SH_DMARS_BASE + DMARS2_ADDR); - break; - default: - return -EINVAL; - } + struct sh_dmae_device *shdev = container_of(sh_chan->common.device, + struct sh_dmae_device, common); + struct sh_dmae_pdata *pdata = shdev->pdata; + struct sh_dmae_channel *chan_pdata = &pdata->channel[sh_chan->id]; + u16 __iomem *addr = shdev->dmars + chan_pdata->dmars / sizeof(u16); + int shift = chan_pdata->dmars_bit; + + if (dmae_is_busy(sh_chan)) + return -EBUSY; - ctrl_outw((val << shift) | - (ctrl_inw(addr) & (shift ? 0xFF00 : 0x00FF)), - addr); + __raw_writew((__raw_readw(addr) & (0xff00 >> shift)) | (val << shift), + addr); return 0; } @@ -250,10 +266,54 @@ static struct sh_desc *sh_dmae_get_desc(struct sh_dmae_chan *sh_chan) return NULL; } +static struct sh_dmae_slave_config *sh_dmae_find_slave( + struct sh_dmae_chan *sh_chan, enum sh_dmae_slave_chan_id slave_id) +{ + struct dma_device *dma_dev = sh_chan->common.device; + struct sh_dmae_device *shdev = container_of(dma_dev, + struct sh_dmae_device, common); + struct sh_dmae_pdata *pdata = shdev->pdata; + int i; + + if ((unsigned)slave_id >= SHDMA_SLAVE_NUMBER) + return NULL; + + for (i = 0; i < pdata->slave_num; i++) + if (pdata->slave[i].slave_id == slave_id) + return pdata->slave + i; + + return NULL; +} + static int sh_dmae_alloc_chan_resources(struct dma_chan *chan) { struct sh_dmae_chan *sh_chan = to_sh_chan(chan); struct sh_desc *desc; + struct sh_dmae_slave *param = chan->private; + + pm_runtime_get_sync(sh_chan->dev); + + /* + * This relies on the guarantee from dmaengine that alloc_chan_resources + * never runs concurrently with itself or free_chan_resources. + */ + if (param) { + struct sh_dmae_slave_config *cfg; + + cfg = sh_dmae_find_slave(sh_chan, param->slave_id); + if (!cfg) + return -EINVAL; + + if (test_and_set_bit(param->slave_id, sh_dmae_slave_used)) + return -EBUSY; + + param->config = cfg; + + dmae_set_dmars(sh_chan, cfg->mid_rid); + dmae_set_chcr(sh_chan, cfg->chcr); + } else if ((sh_dmae_readl(sh_chan, CHCR) & 0xf00) != 0x400) { + dmae_init(sh_chan); + } spin_lock_bh(&sh_chan->desc_lock); while (sh_chan->descs_allocated < NR_DESCS_PER_CHANNEL) { @@ -274,6 +334,9 @@ static int sh_dmae_alloc_chan_resources(struct dma_chan *chan) } spin_unlock_bh(&sh_chan->desc_lock); + if (!sh_chan->descs_allocated) + pm_runtime_put(sh_chan->dev); + return sh_chan->descs_allocated; } @@ -285,11 +348,20 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan) struct sh_dmae_chan *sh_chan = to_sh_chan(chan); struct sh_desc *desc, *_desc; LIST_HEAD(list); + int descs = sh_chan->descs_allocated; + + dmae_halt(sh_chan); /* Prepared and not submitted descriptors can still be on the queue */ if (!list_empty(&sh_chan->ld_queue)) sh_dmae_chan_ld_cleanup(sh_chan, true); + if (chan->private) { + /* The caller is holding dma_list_mutex */ + struct sh_dmae_slave *param = chan->private; + clear_bit(param->slave_id, sh_dmae_slave_used); + } + spin_lock_bh(&sh_chan->desc_lock); list_splice_init(&sh_chan->ld_free, &list); @@ -297,27 +369,104 @@ static void sh_dmae_free_chan_resources(struct dma_chan *chan) spin_unlock_bh(&sh_chan->desc_lock); + if (descs > 0) + pm_runtime_put(sh_chan->dev); + list_for_each_entry_safe(desc, _desc, &list, node) kfree(desc); } -static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( - struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, - size_t len, unsigned long flags) +/** + * sh_dmae_add_desc - get, set up and return one transfer descriptor + * @sh_chan: DMA channel + * @flags: DMA transfer flags + * @dest: destination DMA address, incremented when direction equals + * DMA_FROM_DEVICE or DMA_BIDIRECTIONAL + * @src: source DMA address, incremented when direction equals + * DMA_TO_DEVICE or DMA_BIDIRECTIONAL + * @len: DMA transfer length + * @first: if NULL, set to the current descriptor and cookie set to -EBUSY + * @direction: needed for slave DMA to decide which address to keep constant, + * equals DMA_BIDIRECTIONAL for MEMCPY + * Returns 0 or an error + * Locks: called with desc_lock held + */ +static struct sh_desc *sh_dmae_add_desc(struct sh_dmae_chan *sh_chan, + unsigned long flags, dma_addr_t *dest, dma_addr_t *src, size_t *len, + struct sh_desc **first, enum dma_data_direction direction) { - struct sh_dmae_chan *sh_chan; - struct sh_desc *first = NULL, *prev = NULL, *new; + struct sh_desc *new; size_t copy_size; - LIST_HEAD(tx_list); - int chunks = (len + SH_DMA_TCR_MAX) / (SH_DMA_TCR_MAX + 1); - if (!chan) + if (!*len) return NULL; - if (!len) + /* Allocate the link descriptor from the free list */ + new = sh_dmae_get_desc(sh_chan); + if (!new) { + dev_err(sh_chan->dev, "No free link descriptor available\n"); return NULL; + } - sh_chan = to_sh_chan(chan); + copy_size = min(*len, (size_t)SH_DMA_TCR_MAX + 1); + + new->hw.sar = *src; + new->hw.dar = *dest; + new->hw.tcr = copy_size; + + if (!*first) { + /* First desc */ + new->async_tx.cookie = -EBUSY; + *first = new; + } else { + /* Other desc - invisible to the user */ + new->async_tx.cookie = -EINVAL; + } + + dev_dbg(sh_chan->dev, + "chaining (%u/%u)@%x -> %x with %p, cookie %d, shift %d\n", + copy_size, *len, *src, *dest, &new->async_tx, + new->async_tx.cookie, sh_chan->xmit_shift); + + new->mark = DESC_PREPARED; + new->async_tx.flags = flags; + new->direction = direction; + + *len -= copy_size; + if (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE) + *src += copy_size; + if (direction == DMA_BIDIRECTIONAL || direction == DMA_FROM_DEVICE) + *dest += copy_size; + + return new; +} + +/* + * sh_dmae_prep_sg - prepare transfer descriptors from an SG list + * + * Common routine for public (MEMCPY) and slave DMA. The MEMCPY case is also + * converted to scatter-gather to guarantee consistent locking and a correct + * list manipulation. For slave DMA direction carries the usual meaning, and, + * logically, the SG list is RAM and the addr variable contains slave address, + * e.g., the FIFO I/O register. For MEMCPY direction equals DMA_BIDIRECTIONAL + * and the SG list contains only one element and points at the source buffer. + */ +static struct dma_async_tx_descriptor *sh_dmae_prep_sg(struct sh_dmae_chan *sh_chan, + struct scatterlist *sgl, unsigned int sg_len, dma_addr_t *addr, + enum dma_data_direction direction, unsigned long flags) +{ + struct scatterlist *sg; + struct sh_desc *first = NULL, *new = NULL /* compiler... */; + LIST_HEAD(tx_list); + int chunks = 0; + int i; + + if (!sg_len) + return NULL; + + for_each_sg(sgl, sg, sg_len, i) + chunks += (sg_dma_len(sg) + SH_DMA_TCR_MAX) / + (SH_DMA_TCR_MAX + 1); /* Have to lock the whole loop to protect against concurrent release */ spin_lock_bh(&sh_chan->desc_lock); @@ -333,49 +482,32 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( * only during this function, then they are immediately spliced * back onto the free list in form of a chain */ - do { - /* Allocate the link descriptor from the free list */ - new = sh_dmae_get_desc(sh_chan); - if (!new) { - dev_err(sh_chan->dev, - "No free memory for link descriptor\n"); - list_for_each_entry(new, &tx_list, node) - new->mark = DESC_IDLE; - list_splice(&tx_list, &sh_chan->ld_free); - spin_unlock_bh(&sh_chan->desc_lock); - return NULL; - } - - copy_size = min(len, (size_t)SH_DMA_TCR_MAX + 1); - - new->hw.sar = dma_src; - new->hw.dar = dma_dest; - new->hw.tcr = copy_size; - if (!first) { - /* First desc */ - new->async_tx.cookie = -EBUSY; - first = new; - } else { - /* Other desc - invisible to the user */ - new->async_tx.cookie = -EINVAL; - } - - dev_dbg(sh_chan->dev, - "chaining %u of %u with %p, dst %x, cookie %d\n", - copy_size, len, &new->async_tx, dma_dest, - new->async_tx.cookie); - - new->mark = DESC_PREPARED; - new->async_tx.flags = flags; - new->chunks = chunks--; - - prev = new; - len -= copy_size; - dma_src += copy_size; - dma_dest += copy_size; - /* Insert the link descriptor to the LD ring */ - list_add_tail(&new->node, &tx_list); - } while (len); + for_each_sg(sgl, sg, sg_len, i) { + dma_addr_t sg_addr = sg_dma_address(sg); + size_t len = sg_dma_len(sg); + + if (!len) + goto err_get_desc; + + do { + dev_dbg(sh_chan->dev, "Add SG #%d@%p[%d], dma %llx\n", + i, sg, len, (unsigned long long)sg_addr); + + if (direction == DMA_FROM_DEVICE) + new = sh_dmae_add_desc(sh_chan, flags, + &sg_addr, addr, &len, &first, + direction); + else + new = sh_dmae_add_desc(sh_chan, flags, + addr, &sg_addr, &len, &first, + direction); + if (!new) + goto err_get_desc; + + new->chunks = chunks--; + list_add_tail(&new->node, &tx_list); + } while (len); + } if (new != first) new->async_tx.cookie = -ENOSPC; @@ -386,6 +518,90 @@ static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( spin_unlock_bh(&sh_chan->desc_lock); return &first->async_tx; + +err_get_desc: + list_for_each_entry(new, &tx_list, node) + new->mark = DESC_IDLE; + list_splice(&tx_list, &sh_chan->ld_free); + + spin_unlock_bh(&sh_chan->desc_lock); + + return NULL; +} + +static struct dma_async_tx_descriptor *sh_dmae_prep_memcpy( + struct dma_chan *chan, dma_addr_t dma_dest, dma_addr_t dma_src, + size_t len, unsigned long flags) +{ + struct sh_dmae_chan *sh_chan; + struct scatterlist sg; + + if (!chan || !len) + return NULL; + + chan->private = NULL; + + sh_chan = to_sh_chan(chan); + + sg_init_table(&sg, 1); + sg_set_page(&sg, pfn_to_page(PFN_DOWN(dma_src)), len, + offset_in_page(dma_src)); + sg_dma_address(&sg) = dma_src; + sg_dma_len(&sg) = len; + + return sh_dmae_prep_sg(sh_chan, &sg, 1, &dma_dest, DMA_BIDIRECTIONAL, + flags); +} + +static struct dma_async_tx_descriptor *sh_dmae_prep_slave_sg( + struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, + enum dma_data_direction direction, unsigned long flags) +{ + struct sh_dmae_slave *param; + struct sh_dmae_chan *sh_chan; + + if (!chan) + return NULL; + + sh_chan = to_sh_chan(chan); + param = chan->private; + + /* Someone calling slave DMA on a public channel? */ + if (!param || !sg_len) { + dev_warn(sh_chan->dev, "%s: bad parameter: %p, %d, %d\n", + __func__, param, sg_len, param ? param->slave_id : -1); + return NULL; + } + + /* + * if (param != NULL), this is a successfully requested slave channel, + * therefore param->config != NULL too. + */ + return sh_dmae_prep_sg(sh_chan, sgl, sg_len, ¶m->config->addr, + direction, flags); +} + +static void sh_dmae_terminate_all(struct dma_chan *chan) +{ + struct sh_dmae_chan *sh_chan = to_sh_chan(chan); + + if (!chan) + return; + + dmae_halt(sh_chan); + + spin_lock_bh(&sh_chan->desc_lock); + if (!list_empty(&sh_chan->ld_queue)) { + /* Record partial transfer */ + struct sh_desc *desc = list_entry(sh_chan->ld_queue.next, + struct sh_desc, node); + desc->partial = (desc->hw.tcr - sh_dmae_readl(sh_chan, TCR)) << + sh_chan->xmit_shift; + + } + spin_unlock_bh(&sh_chan->desc_lock); + + sh_dmae_chan_ld_cleanup(sh_chan, true); } static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) @@ -419,7 +635,11 @@ static dma_async_tx_callback __ld_cleanup(struct sh_dmae_chan *sh_chan, bool all cookie = tx->cookie; if (desc->mark == DESC_COMPLETED && desc->chunks == 1) { - BUG_ON(sh_chan->completed_cookie != desc->cookie - 1); + if (sh_chan->completed_cookie != desc->cookie - 1) + dev_dbg(sh_chan->dev, + "Completing cookie %d, expected %d\n", + desc->cookie, + sh_chan->completed_cookie + 1); sh_chan->completed_cookie = desc->cookie; } @@ -483,7 +703,7 @@ static void sh_dmae_chan_ld_cleanup(struct sh_dmae_chan *sh_chan, bool all) static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) { - struct sh_desc *sd; + struct sh_desc *desc; spin_lock_bh(&sh_chan->desc_lock); /* DMA work check */ @@ -492,11 +712,14 @@ static void sh_chan_xfer_ld_queue(struct sh_dmae_chan *sh_chan) return; } - /* Find the first un-transfer desciptor */ - list_for_each_entry(sd, &sh_chan->ld_queue, node) - if (sd->mark == DESC_SUBMITTED) { + /* Find the first not transferred desciptor */ + list_for_each_entry(desc, &sh_chan->ld_queue, node) + if (desc->mark == DESC_SUBMITTED) { + dev_dbg(sh_chan->dev, "Queue #%d to %d: %u@%x -> %x\n", + desc->async_tx.cookie, sh_chan->id, + desc->hw.tcr, desc->hw.sar, desc->hw.dar); /* Get the ld start address from ld_queue */ - dmae_set_reg(sh_chan, &sd->hw); + dmae_set_reg(sh_chan, &desc->hw); dmae_start(sh_chan); break; } @@ -518,6 +741,7 @@ static enum dma_status sh_dmae_is_complete(struct dma_chan *chan, struct sh_dmae_chan *sh_chan = to_sh_chan(chan); dma_cookie_t last_used; dma_cookie_t last_complete; + enum dma_status status; sh_dmae_chan_ld_cleanup(sh_chan, false); @@ -531,7 +755,27 @@ static enum dma_status sh_dmae_is_complete(struct dma_chan *chan, if (used) *used = last_used; - return dma_async_is_complete(cookie, last_complete, last_used); + spin_lock_bh(&sh_chan->desc_lock); + + status = dma_async_is_complete(cookie, last_complete, last_used); + + /* + * If we don't find cookie on the queue, it has been aborted and we have + * to report error + */ + if (status != DMA_SUCCESS) { + struct sh_desc *desc; + status = DMA_ERROR; + list_for_each_entry(desc, &sh_chan->ld_queue, node) + if (desc->cookie == cookie) { + status = DMA_IN_PROGRESS; + break; + } + } + + spin_unlock_bh(&sh_chan->desc_lock); + + return status; } static irqreturn_t sh_dmae_interrupt(int irq, void *data) @@ -554,40 +798,32 @@ static irqreturn_t sh_dmae_interrupt(int irq, void *data) #if defined(CONFIG_CPU_SH4) static irqreturn_t sh_dmae_err(int irq, void *data) { - int err = 0; struct sh_dmae_device *shdev = (struct sh_dmae_device *)data; + int i; - /* IRQ Multi */ - if (shdev->pdata.mode & SHDMA_MIX_IRQ) { - int cnt = 0; - switch (irq) { -#if defined(DMTE6_IRQ) && defined(DMAE1_IRQ) - case DMTE6_IRQ: - cnt++; -#endif - case DMTE0_IRQ: - if (dmaor_read_reg(cnt) & (DMAOR_NMIF | DMAOR_AE)) { - disable_irq(irq); - return IRQ_HANDLED; + /* halt the dma controller */ + sh_dmae_ctl_stop(shdev); + + /* We cannot detect, which channel caused the error, have to reset all */ + for (i = 0; i < SH_DMAC_MAX_CHANNELS; i++) { + struct sh_dmae_chan *sh_chan = shdev->chan[i]; + if (sh_chan) { + struct sh_desc *desc; + /* Stop the channel */ + dmae_halt(sh_chan); + /* Complete all */ + list_for_each_entry(desc, &sh_chan->ld_queue, node) { + struct dma_async_tx_descriptor *tx = &desc->async_tx; + desc->mark = DESC_IDLE; + if (tx->callback) + tx->callback(tx->callback_param); } - default: - return IRQ_NONE; - } - } else { - /* reset dma controller */ - err = sh_dmae_rst(0); - if (err) - return err; -#ifdef SH_DMAC_BASE1 - if (shdev->pdata.mode & SHDMA_DMAOR1) { - err = sh_dmae_rst(1); - if (err) - return err; + list_splice_init(&sh_chan->ld_queue, &sh_chan->ld_free); } -#endif - disable_irq(irq); - return IRQ_HANDLED; } + sh_dmae_rst(shdev); + + return IRQ_HANDLED; } #endif @@ -596,11 +832,14 @@ static void dmae_do_tasklet(unsigned long data) struct sh_dmae_chan *sh_chan = (struct sh_dmae_chan *)data; struct sh_desc *desc; u32 sar_buf = sh_dmae_readl(sh_chan, SAR); + u32 dar_buf = sh_dmae_readl(sh_chan, DAR); spin_lock(&sh_chan->desc_lock); list_for_each_entry(desc, &sh_chan->ld_queue, node) { - if ((desc->hw.sar + desc->hw.tcr) == sar_buf && - desc->mark == DESC_SUBMITTED) { + if (desc->mark == DESC_SUBMITTED && + ((desc->direction == DMA_FROM_DEVICE && + (desc->hw.dar + desc->hw.tcr) == dar_buf) || + (desc->hw.sar + desc->hw.tcr) == sar_buf)) { dev_dbg(sh_chan->dev, "done #%d@%p dst %u\n", desc->async_tx.cookie, &desc->async_tx, desc->hw.dar); @@ -615,19 +854,12 @@ static void dmae_do_tasklet(unsigned long data) sh_dmae_chan_ld_cleanup(sh_chan, false); } -static unsigned int get_dmae_irq(unsigned int id) -{ - unsigned int irq = 0; - if (id < ARRAY_SIZE(dmte_irq_map)) - irq = dmte_irq_map[id]; - return irq; -} - -static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id) +static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id, + int irq, unsigned long flags) { int err; - unsigned int irq = get_dmae_irq(id); - unsigned long irqflags = IRQF_DISABLED; + struct sh_dmae_channel *chan_pdata = &shdev->pdata->channel[id]; + struct platform_device *pdev = to_platform_device(shdev->common.dev); struct sh_dmae_chan *new_sh_chan; /* alloc channel */ @@ -638,8 +870,13 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id) return -ENOMEM; } + /* copy struct dma_device */ + new_sh_chan->common.device = &shdev->common; + new_sh_chan->dev = shdev->common.dev; new_sh_chan->id = id; + new_sh_chan->irq = irq; + new_sh_chan->base = shdev->chan_reg + chan_pdata->offset / sizeof(u32); /* Init DMA tasklet */ tasklet_init(&new_sh_chan->tasklet, dmae_do_tasklet, @@ -654,29 +891,20 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id) INIT_LIST_HEAD(&new_sh_chan->ld_queue); INIT_LIST_HEAD(&new_sh_chan->ld_free); - /* copy struct dma_device */ - new_sh_chan->common.device = &shdev->common; - /* Add the channel to DMA device channel list */ list_add_tail(&new_sh_chan->common.device_node, &shdev->common.channels); shdev->common.chancnt++; - if (shdev->pdata.mode & SHDMA_MIX_IRQ) { - irqflags = IRQF_SHARED; -#if defined(DMTE6_IRQ) - if (irq >= DMTE6_IRQ) - irq = DMTE6_IRQ; - else -#endif - irq = DMTE0_IRQ; - } - - snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), - "sh-dmae%d", new_sh_chan->id); + if (pdev->id >= 0) + snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), + "sh-dmae%d.%d", pdev->id, new_sh_chan->id); + else + snprintf(new_sh_chan->dev_id, sizeof(new_sh_chan->dev_id), + "sh-dma%d", new_sh_chan->id); /* set up channel irq */ - err = request_irq(irq, &sh_dmae_interrupt, irqflags, + err = request_irq(irq, &sh_dmae_interrupt, flags, new_sh_chan->dev_id, new_sh_chan); if (err) { dev_err(shdev->common.dev, "DMA channel %d request_irq error " @@ -684,11 +912,6 @@ static int __devinit sh_dmae_chan_probe(struct sh_dmae_device *shdev, int id) goto err_no_irq; } - /* CHCR register control function */ - new_sh_chan->set_chcr = dmae_set_chcr; - /* DMARS register control function */ - new_sh_chan->set_dmars = dmae_set_dmars; - shdev->chan[id] = new_sh_chan; return 0; @@ -705,12 +928,12 @@ static void sh_dmae_chan_remove(struct sh_dmae_device *shdev) for (i = shdev->common.chancnt - 1 ; i >= 0 ; i--) { if (shdev->chan[i]) { - struct sh_dmae_chan *shchan = shdev->chan[i]; - if (!(shdev->pdata.mode & SHDMA_MIX_IRQ)) - free_irq(dmte_irq_map[i], shchan); + struct sh_dmae_chan *sh_chan = shdev->chan[i]; - list_del(&shchan->common.device_node); - kfree(shchan); + free_irq(sh_chan->irq, sh_chan); + + list_del(&sh_chan->common.device_node); + kfree(sh_chan); shdev->chan[i] = NULL; } } @@ -719,85 +942,164 @@ static void sh_dmae_chan_remove(struct sh_dmae_device *shdev) static int __init sh_dmae_probe(struct platform_device *pdev) { - int err = 0, cnt, ecnt; - unsigned long irqflags = IRQF_DISABLED; -#if defined(CONFIG_CPU_SH4) - int eirq[] = { DMAE0_IRQ, -#if defined(DMAE1_IRQ) - DMAE1_IRQ -#endif - }; -#endif + struct sh_dmae_pdata *pdata = pdev->dev.platform_data; + unsigned long irqflags = IRQF_DISABLED, + chan_flag[SH_DMAC_MAX_CHANNELS] = {}; + int errirq, chan_irq[SH_DMAC_MAX_CHANNELS]; + int err, i, irq_cnt = 0, irqres = 0; struct sh_dmae_device *shdev; + struct resource *chan, *dmars, *errirq_res, *chanirq_res; /* get platform data */ - if (!pdev->dev.platform_data) + if (!pdata || !pdata->channel_num) + return -ENODEV; + + chan = platform_get_resource(pdev, IORESOURCE_MEM, 0); + /* DMARS area is optional, if absent, this controller cannot do slave DMA */ + dmars = platform_get_resource(pdev, IORESOURCE_MEM, 1); + /* + * IRQ resources: + * 1. there always must be at least one IRQ IO-resource. On SH4 it is + * the error IRQ, in which case it is the only IRQ in this resource: + * start == end. If it is the only IRQ resource, all channels also + * use the same IRQ. + * 2. DMA channel IRQ resources can be specified one per resource or in + * ranges (start != end) + * 3. iff all events (channels and, optionally, error) on this + * controller use the same IRQ, only one IRQ resource can be + * specified, otherwise there must be one IRQ per channel, even if + * some of them are equal + * 4. if all IRQs on this controller are equal or if some specific IRQs + * specify IORESOURCE_IRQ_SHAREABLE in their resources, they will be + * requested with the IRQF_SHARED flag + */ + errirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); + if (!chan || !errirq_res) return -ENODEV; + if (!request_mem_region(chan->start, resource_size(chan), pdev->name)) { + dev_err(&pdev->dev, "DMAC register region already claimed\n"); + return -EBUSY; + } + + if (dmars && !request_mem_region(dmars->start, resource_size(dmars), pdev->name)) { + dev_err(&pdev->dev, "DMAC DMARS region already claimed\n"); + err = -EBUSY; + goto ermrdmars; + } + + err = -ENOMEM; shdev = kzalloc(sizeof(struct sh_dmae_device), GFP_KERNEL); if (!shdev) { - dev_err(&pdev->dev, "No enough memory\n"); - return -ENOMEM; + dev_err(&pdev->dev, "Not enough memory\n"); + goto ealloc; + } + + shdev->chan_reg = ioremap(chan->start, resource_size(chan)); + if (!shdev->chan_reg) + goto emapchan; + if (dmars) { + shdev->dmars = ioremap(dmars->start, resource_size(dmars)); + if (!shdev->dmars) + goto emapdmars; } /* platform data */ - memcpy(&shdev->pdata, pdev->dev.platform_data, - sizeof(struct sh_dmae_pdata)); + shdev->pdata = pdata; + + pm_runtime_enable(&pdev->dev); + pm_runtime_get_sync(&pdev->dev); /* reset dma controller */ - err = sh_dmae_rst(0); + err = sh_dmae_rst(shdev); if (err) goto rst_err; - /* SH7780/85/23 has DMAOR1 */ - if (shdev->pdata.mode & SHDMA_DMAOR1) { - err = sh_dmae_rst(1); - if (err) - goto rst_err; - } - INIT_LIST_HEAD(&shdev->common.channels); dma_cap_set(DMA_MEMCPY, shdev->common.cap_mask); + if (dmars) + dma_cap_set(DMA_SLAVE, shdev->common.cap_mask); + shdev->common.device_alloc_chan_resources = sh_dmae_alloc_chan_resources; shdev->common.device_free_chan_resources = sh_dmae_free_chan_resources; shdev->common.device_prep_dma_memcpy = sh_dmae_prep_memcpy; shdev->common.device_is_tx_complete = sh_dmae_is_complete; shdev->common.device_issue_pending = sh_dmae_memcpy_issue_pending; + + /* Compulsory for DMA_SLAVE fields */ + shdev->common.device_prep_slave_sg = sh_dmae_prep_slave_sg; + shdev->common.device_terminate_all = sh_dmae_terminate_all; + shdev->common.dev = &pdev->dev; /* Default transfer size of 32 bytes requires 32-byte alignment */ - shdev->common.copy_align = 5; + shdev->common.copy_align = LOG2_DEFAULT_XFER_SIZE; #if defined(CONFIG_CPU_SH4) - /* Non Mix IRQ mode SH7722/SH7730 etc... */ - if (shdev->pdata.mode & SHDMA_MIX_IRQ) { + chanirq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 1); + + if (!chanirq_res) + chanirq_res = errirq_res; + else + irqres++; + + if (chanirq_res == errirq_res || + (errirq_res->flags & IORESOURCE_BITS) == IORESOURCE_IRQ_SHAREABLE) irqflags = IRQF_SHARED; - eirq[0] = DMTE0_IRQ; -#if defined(DMTE6_IRQ) && defined(DMAE1_IRQ) - eirq[1] = DMTE6_IRQ; -#endif + + errirq = errirq_res->start; + + err = request_irq(errirq, sh_dmae_err, irqflags, + "DMAC Address Error", shdev); + if (err) { + dev_err(&pdev->dev, + "DMA failed requesting irq #%d, error %d\n", + errirq, err); + goto eirq_err; } - for (ecnt = 0 ; ecnt < ARRAY_SIZE(eirq); ecnt++) { - err = request_irq(eirq[ecnt], sh_dmae_err, irqflags, - "DMAC Address Error", shdev); - if (err) { - dev_err(&pdev->dev, "DMA device request_irq" - "error (irq %d) with return %d\n", - eirq[ecnt], err); - goto eirq_err; +#else + chanirq_res = errirq_res; +#endif /* CONFIG_CPU_SH4 */ + + if (chanirq_res->start == chanirq_res->end && + !platform_get_resource(pdev, IORESOURCE_IRQ, 1)) { + /* Special case - all multiplexed */ + for (; irq_cnt < pdata->channel_num; irq_cnt++) { + chan_irq[irq_cnt] = chanirq_res->start; + chan_flag[irq_cnt] = IRQF_SHARED; } + } else { + do { + for (i = chanirq_res->start; i <= chanirq_res->end; i++) { + if ((errirq_res->flags & IORESOURCE_BITS) == + IORESOURCE_IRQ_SHAREABLE) + chan_flag[irq_cnt] = IRQF_SHARED; + else + chan_flag[irq_cnt] = IRQF_DISABLED; + dev_dbg(&pdev->dev, + "Found IRQ %d for channel %d\n", + i, irq_cnt); + chan_irq[irq_cnt++] = i; + } + chanirq_res = platform_get_resource(pdev, + IORESOURCE_IRQ, ++irqres); + } while (irq_cnt < pdata->channel_num && chanirq_res); } -#endif /* CONFIG_CPU_SH4 */ + + if (irq_cnt < pdata->channel_num) + goto eirqres; /* Create DMA Channel */ - for (cnt = 0 ; cnt < MAX_DMA_CHANNELS ; cnt++) { - err = sh_dmae_chan_probe(shdev, cnt); + for (i = 0; i < pdata->channel_num; i++) { + err = sh_dmae_chan_probe(shdev, i, chan_irq[i], chan_flag[i]); if (err) goto chan_probe_err; } + pm_runtime_put(&pdev->dev); + platform_set_drvdata(pdev, shdev); dma_async_device_register(&shdev->common); @@ -805,13 +1107,24 @@ static int __init sh_dmae_probe(struct platform_device *pdev) chan_probe_err: sh_dmae_chan_remove(shdev); - +eirqres: +#if defined(CONFIG_CPU_SH4) + free_irq(errirq, shdev); eirq_err: - for (ecnt-- ; ecnt >= 0; ecnt--) - free_irq(eirq[ecnt], shdev); - +#endif rst_err: + pm_runtime_put(&pdev->dev); + if (dmars) + iounmap(shdev->dmars); +emapdmars: + iounmap(shdev->chan_reg); +emapchan: kfree(shdev); +ealloc: + if (dmars) + release_mem_region(dmars->start, resource_size(dmars)); +ermrdmars: + release_mem_region(chan->start, resource_size(chan)); return err; } @@ -819,36 +1132,39 @@ rst_err: static int __exit sh_dmae_remove(struct platform_device *pdev) { struct sh_dmae_device *shdev = platform_get_drvdata(pdev); + struct resource *res; + int errirq = platform_get_irq(pdev, 0); dma_async_device_unregister(&shdev->common); - if (shdev->pdata.mode & SHDMA_MIX_IRQ) { - free_irq(DMTE0_IRQ, shdev); -#if defined(DMTE6_IRQ) - free_irq(DMTE6_IRQ, shdev); -#endif - } + if (errirq > 0) + free_irq(errirq, shdev); /* channel data remove */ sh_dmae_chan_remove(shdev); - if (!(shdev->pdata.mode & SHDMA_MIX_IRQ)) { - free_irq(DMAE0_IRQ, shdev); -#if defined(DMAE1_IRQ) - free_irq(DMAE1_IRQ, shdev); -#endif - } + pm_runtime_disable(&pdev->dev); + + if (shdev->dmars) + iounmap(shdev->dmars); + iounmap(shdev->chan_reg); + kfree(shdev); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (res) + release_mem_region(res->start, resource_size(res)); + res = platform_get_resource(pdev, IORESOURCE_MEM, 1); + if (res) + release_mem_region(res->start, resource_size(res)); + return 0; } static void sh_dmae_shutdown(struct platform_device *pdev) { struct sh_dmae_device *shdev = platform_get_drvdata(pdev); - sh_dmae_ctl_stop(0); - if (shdev->pdata.mode & SHDMA_DMAOR1) - sh_dmae_ctl_stop(1); + sh_dmae_ctl_stop(shdev); } static struct platform_driver sh_dmae_driver = { |