diff options
Diffstat (limited to 'drivers/dma')
| -rw-r--r-- | drivers/dma/Kconfig | 40 | ||||
| -rw-r--r-- | drivers/dma/Makefile | 4 | ||||
| -rw-r--r-- | drivers/dma/apm82181-adma.c | 2433 | ||||
| -rw-r--r-- | drivers/dma/dmaengine.c | 11 | ||||
| -rw-r--r-- | drivers/dma/ppc460ex-adma.c | 5409 | ||||
| -rw-r--r-- | drivers/dma/ppc460ex-plbadma.c | 2009 | ||||
| -rwxr-xr-x | drivers/dma/ppc460ex_4chan_dma.c | 1103 | ||||
| -rwxr-xr-x | drivers/dma/ppc460ex_4chan_dma.h | 531 | ||||
| -rwxr-xr-x | drivers/dma/ppc460ex_4chan_sgdma.c | 1003 |
9 files changed, 12535 insertions, 8 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig index b401dadad4a..3d1b57d4c1b 100644 --- a/drivers/dma/Kconfig +++ b/drivers/dma/Kconfig @@ -111,6 +111,46 @@ config SH_DMAE help Enable support for the Renesas SuperH DMA controllers. +config AMCC_PPC460EX_460GT_ADMA + tristate "AMCC PPC460EX/GT ADMA support" + depends on 460EX || 460GT + select ASYNC_CORE + select DMA_ENGINE + select ARCH_HAS_ASYNC_TX_FIND_CHANNEL + default y + ---help--- + Enable support for the AMCC PPC440SPe RAID engines. + +config AMCC_PPC460EX_460GT_4CHAN_DMA + tristate "AMCC PPC460EX PPC460GT PLB DMA support" + depends on 460EX || 460GT || APM82181 + select DMA_ENGINE + default y + +config AMCC_PPC460EX_460GT_PLB_ADMA + tristate "AMCC PPC460EX/GT 4Channel PLB ADMA support" + depends on 460EX || 460GT + select ASYNC_CORE + select DMA_ENGINE + select ARCH_HAS_ASYNC_TX_FIND_CHANNEL + default y + ---help--- + Enable support for the AMCC PPC460Ex PLB engines. + +config APM82181_ADMA + tristate "APM82181 Asynchonous DMA support" + depends on APM82181 + select ASYNC_CORE + select ASYNC_TX_DMA + select DMA_ENGINE + select ARCH_HAS_ASYNC_TX_FIND_CHANNEL + default y + ---help--- + Enable support for the APM82181 Asynchonous DMA engines. + +config ARCH_HAS_ASYNC_TX_FIND_CHANNEL + bool + config DMA_ENGINE bool diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile index eca71ba78ae..3637a70241f 100644 --- a/drivers/dma/Makefile +++ b/drivers/dma/Makefile @@ -3,6 +3,9 @@ obj-$(CONFIG_NET_DMA) += iovlock.o obj-$(CONFIG_DMATEST) += dmatest.o obj-$(CONFIG_INTEL_IOATDMA) += ioat/ obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o +obj-$(CONFIG_AMCC_PPC460EX_460GT_ADMA) += ppc460ex-adma.o +obj-$(CONFIG_AMCC_PPC460EX_460GT_4CHAN_DMA) += ppc460ex_4chan_dma.o ppc460ex_4chan_sgdma.o +obj-$(CONFIG_APM82181_ADMA) += apm82181-adma.o obj-$(CONFIG_FSL_DMA) += fsldma.o obj-$(CONFIG_MV_XOR) += mv_xor.o obj-$(CONFIG_DW_DMAC) += dw_dmac.o @@ -10,3 +13,4 @@ obj-$(CONFIG_AT_HDMAC) += at_hdmac.o obj-$(CONFIG_MX3_IPU) += ipu/ obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o obj-$(CONFIG_SH_DMAE) += shdma.o +#amcc_ppc460ex_4chan_dma-objs := ppc460ex_4chan_dma.o ppc460ex_4chan_sgdma.o diff --git a/drivers/dma/apm82181-adma.c b/drivers/dma/apm82181-adma.c new file mode 100644 index 00000000000..5800ca15e56 --- /dev/null +++ b/drivers/dma/apm82181-adma.c @@ -0,0 +1,2433 @@ +/* + * Copyright(c) 2010 Applied Micro Circuits Corporation(AMCC). All rights reserved. + * + * Author: Tai Tri Nguyen <ttnguyen@appliedmicro.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ + +/* + * This driver supports the asynchrounous DMA copy and RAID engines available + * on the AppliedMicro APM82181 Processor. + * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) + * ADMA driver written by D.Williams. + */ +#define ADMA_DEBUG +#undef ADMA_DEBUG + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/async_tx.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/uaccess.h> +#include <linux/of_platform.h> +#include <linux/proc_fs.h> +#include <asm/dcr.h> +#include <asm/dcr-regs.h> +#include <asm/apm82181-adma.h> + +#define PPC4XX_EDMA "apm82181-adma: " +#ifdef ADMA_DEBUG +#define DBG(string, args...) \ + printk(PPC4XX_EDMA string ,##args) +#define INFO DBG("<%s> -- line %d\n",__func__,__LINE__); +#define ADMA_HEXDUMP(b, l) \ + print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, \ + 16, 1, (b), (l), false); +#else +#define DBG(string, args...) \ + {if (0) printk(KERN_INFO PPC4XX_EDMA string ,##args); 0; } +#define INFO DBG(""); +#define ADMA_HEXDUMP(b, l) \ + {if (0) print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, \ + 8, 1, (b), (l), false); 0;} +#endif + +#define MEM_HEXDUMP(b, l) \ + print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, \ + 16, 1, (b), (l), false); + +/* The list of channels exported by apm82181 ADMA */ +struct list_head +ppc_adma_chan_list = LIST_HEAD_INIT(ppc_adma_chan_list); + +/* This flag is set when want to refetch the xor chain in the interrupt + * handler + */ +static u32 do_xor_refetch = 0; + +/* Pointers to last submitted to DMA0/1/2/3 and XOR CDBs */ +static apm82181_desc_t *chan_last_sub[5]; +static apm82181_desc_t *chan_first_cdb[5]; + +/* Pointer to last linked and submitted xor CB */ +static apm82181_desc_t *xor_last_linked = NULL; +static apm82181_desc_t *xor_last_submit = NULL; + +/* /proc interface is used here to verify the h/w RAID 5 capabilities + */ +static struct proc_dir_entry *apm82181_proot; + +/* These are used in enable & check routines + */ +static u32 apm82181_xor_verified; +static u32 apm82181_memcpy_verified[4]; +static apm82181_ch_t *apm82181_dma_tchan[5]; +static struct completion apm82181_r5_test_comp; + +static inline int apm82181_chan_is_busy(apm82181_ch_t *chan); +#if 0 +static phys_addr_t fixup_bigphys_addr(phys_addr_t addr, phys_addr_t size) +{ + phys_addr_t page_4gb = 0; + + return (page_4gb | addr); +} +#endif +/** + * apm82181_adma_device_estimate - estimate the efficiency of processing + * the operation given on this channel. It's assumed that 'chan' is + * capable to process 'cap' type of operation. + * @chan: channel to use + * @cap: type of transaction + * @src_lst: array of source pointers + * @src_cnt: number of source operands + * @src_sz: size of each source operand + */ +int apm82181_adma_estimate (struct dma_chan *chan, + enum dma_transaction_type cap, struct page **src_lst, + int src_cnt, size_t src_sz) +{ + int ef = 1; + + /* channel idleness increases the priority */ + if (likely(ef) && + !apm82181_chan_is_busy(to_apm82181_adma_chan(chan))) + ef++; + else { + if(chan->chan_id !=APM82181_XOR_ID) + ef = -1; + } + return ef; +} + +/****************************************************************************** + * Command (Descriptor) Blocks low-level routines + ******************************************************************************/ +/** + * apm82181_desc_init_interrupt - initialize the descriptor for INTERRUPT + * pseudo operation + */ +static inline void apm82181_desc_init_interrupt (apm82181_desc_t *desc, + apm82181_ch_t *chan) +{ + xor_cb_t *p; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + BUG(); + break; + case APM82181_XOR_ID: + p = desc->hw_desc; + memset (desc->hw_desc, 0, sizeof(xor_cb_t)); + /* NOP with Command Block Complete Enable */ + p->cbc = XOR_CBCR_CBCE_BIT; + break; + default: + printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, + __FUNCTION__); + break; + } +} + +/** + * apm82181_desc_init_xor - initialize the descriptor for XOR operation + */ +static inline void apm82181_desc_init_xor(apm82181_desc_t *desc, int src_cnt, + unsigned long flags) +{ + xor_cb_t *hw_desc = desc->hw_desc; + + memset (desc->hw_desc, 0, sizeof(xor_cb_t)); + desc->hw_next = NULL; + desc->src_cnt = src_cnt; + desc->dst_cnt = 1; + + hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; + if (flags & DMA_PREP_INTERRUPT) + /* Enable interrupt on complete */ + hw_desc->cbc |= XOR_CBCR_CBCE_BIT; +} + +/** + * apm82181_desc_init_memcpy - initialize the descriptor for MEMCPY operation + */ +static inline void apm82181_desc_init_memcpy(apm82181_desc_t *desc, + unsigned long flags) +{ + dma_cdb_t *hw_desc = desc->hw_desc; + + memset(hw_desc, 0, sizeof(dma_cdb_t)); + desc->hw_next = NULL; + desc->src_cnt = 1; + desc->dst_cnt = 1; + + if (flags & DMA_PREP_INTERRUPT) + set_bit(APM82181_DESC_INT, &desc->flags); + else + clear_bit(APM82181_DESC_INT, &desc->flags); + /* dma configuration for running */ + hw_desc->ctrl.tm = 2; /* soft init mem-mem mode */ + hw_desc->ctrl.pw = 4; /* transfer width 128 bytes */ + hw_desc->ctrl.ben = 1;/* buffer enable */ + hw_desc->ctrl.sai = 1;/* increase source addr */ + hw_desc->ctrl.dai = 1;/* increase dest addr */ + hw_desc->ctrl.tce = 1;/* chan stops when TC is reached */ + hw_desc->ctrl.cp = 3; /* hinghest priority */ + hw_desc->ctrl.sl = 0; /* source is in PLB */ + hw_desc->ctrl.pl = 0; /* dest is in PLB */ + hw_desc->cnt.tcie = 0;/* no interrupt on init */ + hw_desc->cnt.etie = 0; /* enable error interrupt */ + hw_desc->cnt.eie = 1; /* enable error interrupt */ + hw_desc->cnt.link = 0;/* not link to next cdb */ + hw_desc->cnt.sgl = 0; + hw_desc->ctrl.ce =1; /* enable channel */ + hw_desc->ctrl.cie =1; /* enable int channel */ +} + +/** + * apm82181_desc_init_memset - initialize the descriptor for MEMSET operation + */ +static inline void apm82181_desc_init_memset(apm82181_desc_t *desc, int value, + unsigned long flags) +{ + //dma_cdb_t *hw_desc = desc->hw_desc; + + memset (desc->hw_desc, 0, sizeof(dma_cdb_t)); + desc->hw_next = NULL; + desc->src_cnt = 1; + desc->dst_cnt = 1; + + if (flags & DMA_PREP_INTERRUPT) + set_bit(APM82181_DESC_INT, &desc->flags); + else + clear_bit(APM82181_DESC_INT, &desc->flags); + +} + + + +/** + * apm82181_desc_set_src_addr - set source address into the descriptor + */ +static inline void apm82181_desc_set_src_addr( apm82181_desc_t *desc, + apm82181_ch_t *chan, int src_idx, dma_addr_t addr) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + dma_hw_desc = desc->hw_desc; + dma_hw_desc->src_hi = (u32)(addr >> 32); + dma_hw_desc->src_lo = (u32)addr; + break; + case APM82181_XOR_ID: + xor_hw_desc = desc->hw_desc; + xor_hw_desc->ops[src_idx].h = (u32)(addr >>32); + xor_hw_desc->ops[src_idx].l = (u32)addr; + break; + } +} + +static void apm82181_adma_set_src(apm82181_desc_t *sw_desc, + dma_addr_t addr, int index) +{ + apm82181_ch_t *chan = to_apm82181_adma_chan(sw_desc->async_tx.chan); + + sw_desc = sw_desc->group_head; + + if (likely(sw_desc)) + apm82181_desc_set_src_addr(sw_desc, chan, index, addr); +} + +/** + * apm82181_desc_set_dest_addr - set destination address into the descriptor + */ +static inline void apm82181_desc_set_dest_addr(apm82181_desc_t *desc, + apm82181_ch_t *chan, dma_addr_t addr, u32 index) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + dma_hw_desc = desc->hw_desc; + dma_hw_desc->dest_hi = (u32)(addr >> 32); + dma_hw_desc->dest_lo = (u32)addr; + break; + case APM82181_XOR_ID: + xor_hw_desc = desc->hw_desc; + xor_hw_desc->cbtah = (u32)(addr >> 32); + xor_hw_desc->cbtal |= (u32)addr; + break; + } +} + +static int plbdma_get_transfer_width(dma_cdb_t *dma_hw_desc) +{ + switch (dma_hw_desc->ctrl.pw){ + case 0: + return 1; /* unit: bytes */ + case 1: + return 2; + case 2: + return 4; + case 3: + return 8; + case 4: + return 16; + } + return 0; +} +/** + * apm82181_desc_set_byte_count - set number of data bytes involved + * into the operation + */ +static inline void apm82181_desc_set_byte_count(apm82181_desc_t *desc, + apm82181_ch_t *chan, size_t byte_count) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + int terminal_cnt, transfer_width = 0; + + DBG("<%s> byte_count %08x\n", __func__,byte_count); + switch (chan->device->id){ + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + dma_hw_desc = desc->hw_desc; + transfer_width = plbdma_get_transfer_width(dma_hw_desc); + terminal_cnt = byte_count/transfer_width; + dma_hw_desc->cnt.tc = terminal_cnt; + break; + case APM82181_XOR_ID: + xor_hw_desc = desc->hw_desc; + xor_hw_desc->cbbc = byte_count; + break; + } +} + +/** + * apm82181_xor_set_link - set link address in xor CB + */ +static inline void apm82181_xor_set_link (apm82181_desc_t *prev_desc, + apm82181_desc_t *next_desc) +{ + xor_cb_t *xor_hw_desc = prev_desc->hw_desc; + + if (unlikely(!next_desc || !(next_desc->phys))) { + printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", + __func__, next_desc, + next_desc ? next_desc->phys : 0); + BUG(); + } + DBG("<%s>:next_desc->phys %llx\n", __func__,next_desc->phys); + xor_hw_desc->cbs = 0; + xor_hw_desc->cblal = (u32)next_desc->phys; + xor_hw_desc->cblah = (u32)(next_desc->phys >> 32); + xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; +} + +/** + * apm82181_desc_set_link - set the address of descriptor following this + * descriptor in chain + */ +static inline void apm82181_desc_set_link(apm82181_ch_t *chan, + apm82181_desc_t *prev_desc, apm82181_desc_t *next_desc) +{ + unsigned long flags; + apm82181_desc_t *tail = next_desc; + + if (unlikely(!prev_desc || !next_desc || + (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { + /* If previous next is overwritten something is wrong. + * though we may refetch from append to initiate list + * processing; in this case - it's ok. + */ + printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " + "prev->hw_next=0x%p\n", __FUNCTION__, prev_desc, + next_desc, prev_desc ? prev_desc->hw_next : 0); + BUG(); + } + + local_irq_save(flags); + + /* do s/w chaining both for DMA and XOR descriptors */ + prev_desc->hw_next = next_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + break; + case APM82181_XOR_ID: + /* bind descriptor to the chain */ + while (tail->hw_next) + tail = tail->hw_next; + xor_last_linked = tail; + + if (prev_desc == xor_last_submit) + /* do not link to the last submitted CB */ + break; + apm82181_xor_set_link (prev_desc, next_desc); + break; + default: + BUG(); + } + + local_irq_restore(flags); +} + +/** + * apm82181_desc_get_src_addr - extract the source address from the descriptor + */ +static inline u32 apm82181_desc_get_src_addr(apm82181_desc_t *desc, + apm82181_ch_t *chan, int src_idx) +{ + dma_cdb_t *dma_hw_desc; + + dma_hw_desc = desc->hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + break; + default: + return 0; + } + /* May have 0, 1, 2, or 3 sources */ + return (dma_hw_desc->src_lo); +} + +/** + * apm82181_desc_get_dest_addr - extract the destination address from the + * descriptor + */ +static inline u32 apm82181_desc_get_dest_addr(apm82181_desc_t *desc, + apm82181_ch_t *chan, int idx) +{ + dma_cdb_t *dma_hw_desc; + + dma_hw_desc = desc->hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + break; + default: + return 0; + } + + /* May have 0, 1, 2, or 3 sources */ + return (dma_hw_desc->dest_lo); +} + +/** + * apm82181_desc_get_byte_count - extract the byte count from the descriptor + */ +static inline u32 apm82181_desc_get_byte_count(apm82181_desc_t *desc, + apm82181_ch_t *chan) +{ + dma_cdb_t *dma_hw_desc; + + dma_hw_desc = desc->hw_desc; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + break; + default: + return 0; + } + /* May have 0, 1, 2, or 3 sources */ + //return (dma_hw_desc->cnt); +} + + +/** + * apm82181_desc_get_link - get the address of the descriptor that + * follows this one + */ +static inline u32 apm82181_desc_get_link(apm82181_desc_t *desc, + apm82181_ch_t *chan) +{ + if (!desc->hw_next) + return 0; + + return desc->hw_next->phys; +} + +/** + * apm82181_desc_is_aligned - check alignment + */ +static inline int apm82181_desc_is_aligned(apm82181_desc_t *desc, + int num_slots) +{ + return (desc->idx & (num_slots - 1)) ? 0 : 1; +} + + + +/****************************************************************************** + * ADMA channel low-level routines + ******************************************************************************/ + +static inline phys_addr_t apm82181_chan_get_current_descriptor(apm82181_ch_t *chan); +static inline void apm82181_chan_append(apm82181_ch_t *chan); + +/* + * apm82181_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine + */ +static inline void apm82181_adma_device_clear_eot_status (apm82181_ch_t *chan) +{ + u32 val ; + int idx = chan->device->id; + volatile xor_regs_t *xor_reg; + INFO; + switch (idx) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + val = mfdcr(DCR_DMA2P40_SR); + if(val & DMA_SR_RI(idx)){ + printk(KERN_ERR "Err occurred, DMA%d status: 0x%x\n", idx, val); + } + /* TC reached int, write back to clear */ + mtdcr(DCR_DMA2P40_SR, val); + break; + case APM82181_XOR_ID: + /* reset status bits to ack*/ + xor_reg = chan->device->xor_base; + + val = xor_reg->sr; + DBG("XOR engine status: 0x%08x\n", val); + xor_reg->sr = val; + + if (val & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { + if (val & XOR_IE_RPTIE_BIT) { + /* Read PLB Timeout Error. + * Try to resubmit the CB + */ + INFO; + xor_reg->cblalr = xor_reg->ccbalr; + xor_reg->crsr |= XOR_CRSR_XAE_BIT; + } else + printk (KERN_ERR "XOR ERR 0x%x status\n", val); + break; + } + + /* if the XORcore is idle, but there are unprocessed CBs + * then refetch the s/w chain here + */ + if (!(xor_reg->sr & XOR_SR_XCP_BIT) && do_xor_refetch) { + apm82181_chan_append(chan); + } + break; + } +} + +/* + * apm82181_chan_is_busy - get the channel status + */ + +static inline int apm82181_chan_is_busy(apm82181_ch_t *chan) +{ + int busy = 0; + volatile xor_regs_t *xor_reg = chan->device->xor_base; + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + if(mfdcr(DCR_DMA2P40_SR) & DMA_SR_CB(chan->device->id)) + busy = 1; + else + busy = 0; + break; + case APM82181_XOR_ID: + /* use the special status bit for the XORcore + */ + busy = (xor_reg->sr & XOR_SR_XCP_BIT) ? 1 : 0; + break; + default: + BUG(); + } + + return busy; +} + +/** + * apm82181_dma_put_desc - put PLB DMA 0/1/2/3 descriptor to FIFO + */ +static inline void apm82181_dma_put_desc(apm82181_ch_t *chan, + apm82181_desc_t *desc) +{ + dma_cdb_t *cdb = desc->hw_desc; + u32 sg_cmd = 0; + + /* Enable TC interrupt */ + if(test_bit(APM82181_DESC_INT, &desc->flags)) + cdb->cnt.tcie = 1; + else + cdb->cnt.tcie = 0; + /* Not link to next cdb */ + cdb->sg_hi = 0xffffffff; + cdb->sg_lo = 0xffffffff; + + chan_last_sub[chan->device->id] = desc; + + /* Update new cdb addr */ + mtdcr(DCR_DMA2P40_SGHx(chan->device->id), (u32)(desc->phys >> 32)); + mtdcr(DCR_DMA2P40_SGLx(chan->device->id), (u32)desc->phys); + + INFO; + DBG("slot id: %d addr: %llx\n", desc->idx, desc->phys); + DBG("S/G addr H: %08x addr L: %08x\n", + mfdcr(DCR_DMA2P40_SGHx(chan->device->id)), + mfdcr(DCR_DMA2P40_SGLx(chan->device->id))); + ADMA_HEXDUMP(cdb, 96); + /* Enable S/G */ + sg_cmd |= (DMA_SGC_SSG(chan->device->id) | DMA_SGC_EM_ALL); + sg_cmd |= DMA_SGC_SGL(chan->device->id, 0); /* S/G addr in PLB */ + + mtdcr(DCR_DMA2P40_SGC, sg_cmd); + DBG("S/G addr H: %08x addr L: %08x\n", + mfdcr(DCR_DMA2P40_SGHx(chan->device->id)), + mfdcr(DCR_DMA2P40_SGLx(chan->device->id))); + /* need to use variable for logging current CDB */ + chan->current_cdb_addr = desc->phys; + +} + +/** + * apm82181_chan_append - update the h/w chain in the channel + */ +static inline void apm82181_chan_append(apm82181_ch_t *chan) +{ + apm82181_desc_t *iter; + volatile xor_regs_t *xor_reg; + phys_addr_t cur_desc; + xor_cb_t *xcb; + unsigned long flags; + INFO; + + local_irq_save(flags); + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + cur_desc = apm82181_chan_get_current_descriptor(chan); + DBG("current_desc %llx\n", cur_desc); + if (likely(cur_desc)) { + INFO; + iter = chan_last_sub[chan->device->id]; + BUG_ON(!iter); + } else { + INFO; + /* first peer */ + iter = chan_first_cdb[chan->device->id]; + BUG_ON(!iter); + INFO; + apm82181_dma_put_desc(chan, iter); + chan->hw_chain_inited = 1; + } + + /* is there something new to append */ + if (!iter->hw_next) + break; + + /* flush descriptors from the s/w queue to fifo */ + list_for_each_entry_continue(iter, &chan->chain, chain_node) { + apm82181_dma_put_desc(chan, iter); + if (!iter->hw_next) + break; + } + break; + case APM82181_XOR_ID: + /* update h/w links and refetch */ + if (!xor_last_submit->hw_next) + break; + xor_reg = chan->device->xor_base; + /* the last linked CDB has to generate an interrupt + * that we'd be able to append the next lists to h/w + * regardless of the XOR engine state at the moment of + * appending of these next lists + */ + xcb = xor_last_linked->hw_desc; + xcb->cbc |= XOR_CBCR_CBCE_BIT; + + if (!(xor_reg->sr & XOR_SR_XCP_BIT)) { + /* XORcore is idle. Refetch now */ + do_xor_refetch = 0; + apm82181_xor_set_link(xor_last_submit, + xor_last_submit->hw_next); + + xor_last_submit = xor_last_linked; + xor_reg->crsr |= XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT; + } else { + /* XORcore is running. Refetch later in the handler */ + do_xor_refetch = 1; + } + + break; + } + + local_irq_restore(flags); +} + +/** + * apm82181_chan_get_current_descriptor - get the currently executed descriptor + */ +static inline phys_addr_t apm82181_chan_get_current_descriptor(apm82181_ch_t *chan) +{ + phys_addr_t curr_cdb_addr; + volatile xor_regs_t *xor_reg; + int idx = chan->device->id; + + if (unlikely(!chan->hw_chain_inited)) + /* h/w descriptor chain is not initialized yet */ + return 0; + switch(idx){ + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + curr_cdb_addr = chan->current_cdb_addr; + break; + case APM82181_XOR_ID: + xor_reg = chan->device->xor_base; + curr_cdb_addr = (dma_addr_t)xor_reg->ccbahr; + curr_cdb_addr = (curr_cdb_addr << 32) | xor_reg->ccbalr; + break; + default: + BUG(); + } + return curr_cdb_addr; +} + + +/****************************************************************************** + * ADMA device level + ******************************************************************************/ + +static int apm82181_adma_alloc_chan_resources(struct dma_chan *chan); +static dma_cookie_t apm82181_adma_tx_submit( + struct dma_async_tx_descriptor *tx); + +static void apm82181_adma_set_dest( + apm82181_desc_t *tx, + dma_addr_t addr, int index); + +/** + * apm82181_get_group_entry - get group entry with index idx + * @tdesc: is the last allocated slot in the group. + */ +static inline apm82181_desc_t * +apm82181_get_group_entry ( apm82181_desc_t *tdesc, u32 entry_idx) +{ + apm82181_desc_t *iter = tdesc->group_head; + int i = 0; + + if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { + printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", + __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); + BUG(); + } + list_for_each_entry(iter, &tdesc->group_list, chain_node) { + if (i++ == entry_idx) + break; + } + return iter; +} + +/** + * apm82181_adma_free_slots - flags descriptor slots for reuse + * @slot: Slot to free + * Caller must hold &apm82181_chan->lock while calling this function + */ +static void apm82181_adma_free_slots(apm82181_desc_t *slot, + apm82181_ch_t *chan) +{ + int stride = slot->slots_per_op; + + while (stride--) { + /*async_tx_clear_ack(&slot->async_tx);*/ /* Don't need to clear. It is hack*/ + slot->slots_per_op = 0; + slot = list_entry(slot->slot_node.next, + apm82181_desc_t, + slot_node); + } +} + +static void +apm82181_adma_unmap(apm82181_ch_t *chan, apm82181_desc_t *desc) +{ + u32 src_cnt, dst_cnt; + dma_addr_t addr; + /* + * get the number of sources & destination + * included in this descriptor and unmap + * them all + */ + src_cnt = 1; + dst_cnt = 1; + + /* unmap destinations */ + if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { + while (dst_cnt--) { + addr = apm82181_desc_get_dest_addr( + desc, chan, dst_cnt); + dma_unmap_page(&chan->device->ofdev->dev, + addr, desc->unmap_len, + DMA_FROM_DEVICE); + } + } + + /* unmap sources */ + if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { + while (src_cnt--) { + addr = apm82181_desc_get_src_addr( + desc, chan, src_cnt); + dma_unmap_page(&chan->device->ofdev->dev, + addr, desc->unmap_len, + DMA_TO_DEVICE); + } + } + +} +/** + * apm82181_adma_run_tx_complete_actions - call functions to be called + * upon complete + */ +static dma_cookie_t apm82181_adma_run_tx_complete_actions( + apm82181_desc_t *desc, + apm82181_ch_t *chan, + dma_cookie_t cookie) +{ + int i; + //enum dma_data_direction dir; + INFO; + BUG_ON(desc->async_tx.cookie < 0); + if (desc->async_tx.cookie > 0) { + cookie = desc->async_tx.cookie; + desc->async_tx.cookie = 0; + + /* call the callback (must not sleep or submit new + * operations to this channel) + */ + if (desc->async_tx.callback) + desc->async_tx.callback( + desc->async_tx.callback_param); + + /* unmap dma addresses + * (unmap_single vs unmap_page?) + * + * actually, ppc's dma_unmap_page() functions are empty, so + * the following code is just for the sake of completeness + */ + if (chan && chan->needs_unmap && desc->group_head && + desc->unmap_len) { + apm82181_desc_t *unmap = desc->group_head; + /* assume 1 slot per op always */ + u32 slot_count = unmap->slot_cnt; + + /* Run through the group list and unmap addresses */ + for (i = 0; i < slot_count; i++) { + BUG_ON(!unmap); + apm82181_adma_unmap(chan, unmap); + unmap = unmap->hw_next; + } + desc->group_head = NULL; + } + } + + /* run dependent operations */ + dma_run_dependencies(&desc->async_tx); + + return cookie; +} + +/** + * apm82181_adma_clean_slot - clean up CDB slot (if ack is set) + */ +static int apm82181_adma_clean_slot(apm82181_desc_t *desc, + apm82181_ch_t *chan) +{ + /* the client is allowed to attach dependent operations + * until 'ack' is set + */ + if (!async_tx_test_ack(&desc->async_tx)) + return 0; + + /* leave the last descriptor in the chain + * so we can append to it + */ + if (list_is_last(&desc->chain_node, &chan->chain) || + desc->phys == apm82181_chan_get_current_descriptor(chan)) + return 1; + + dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", + desc->phys, desc->idx, desc->slots_per_op); + + list_del(&desc->chain_node); + apm82181_adma_free_slots(desc, chan); + return 0; +} + +/** + * __apm82181_adma_slot_cleanup - this is the common clean-up routine + * which runs through the channel CDBs list until reach the descriptor + * currently processed. When routine determines that all CDBs of group + * are completed then corresponding callbacks (if any) are called and slots + * are freed. + */ +static void __apm82181_adma_slot_cleanup(apm82181_ch_t *chan) +{ + apm82181_desc_t *iter, *_iter, *group_start = NULL; + dma_cookie_t cookie = 0; + phys_addr_t current_desc = apm82181_chan_get_current_descriptor(chan); + int busy = apm82181_chan_is_busy(chan); + int seen_current = 0, slot_cnt = 0, slots_per_op = 0; + + DBG("apm82181 adma%d: %s\n", + chan->device->id, __FUNCTION__); + DBG("current_desc %llx\n", current_desc); + + if (!current_desc) { + /* There were no transactions yet, so + * nothing to clean + */ + return; + } + + /* free completed slots from the chain starting with + * the oldest descriptor + */ + list_for_each_entry_safe(iter, _iter, &chan->chain, + chain_node) { + DBG(" cookie: %d slot: %d " + "busy: %d this_desc: %llx next_desc: %x cur: %llx ack: %d\n", + iter->async_tx.cookie, iter->idx, busy, iter->phys, + apm82181_desc_get_link(iter, chan), current_desc, + async_tx_test_ack(&iter->async_tx)); + prefetch(_iter); + prefetch(&_iter->async_tx); + + /* do not advance past the current descriptor loaded into the + * hardware channel,subsequent descriptors are either in process + * or have not been submitted + */ + if (seen_current) + break; + + /* stop the search if we reach the current descriptor and the + * channel is busy, or if it appears that the current descriptor + * needs to be re-read (i.e. has been appended to) + */ + if (iter->phys == current_desc) { + BUG_ON(seen_current++); + if (busy || apm82181_desc_get_link(iter, chan)) { + /* not all descriptors of the group have + * been completed; exit. + */ + break; + } + } + + /* detect the start of a group transaction */ + if (!slot_cnt && !slots_per_op) { + slot_cnt = iter->slot_cnt; + slots_per_op = iter->slots_per_op; + if (slot_cnt <= slots_per_op) { + slot_cnt = 0; + slots_per_op = 0; + } + } + + if (slot_cnt) { + if (!group_start) + group_start = iter; + slot_cnt -= slots_per_op; + } + + /* all the members of a group are complete */ + if (slots_per_op != 0 && slot_cnt == 0) { + apm82181_desc_t *grp_iter, *_grp_iter; + int end_of_chain = 0; + + /* clean up the group */ + slot_cnt = group_start->slot_cnt; + grp_iter = group_start; + list_for_each_entry_safe_from(grp_iter, _grp_iter, + &chan->chain, chain_node) { + + cookie = apm82181_adma_run_tx_complete_actions( + grp_iter, chan, cookie); + + slot_cnt -= slots_per_op; + end_of_chain = apm82181_adma_clean_slot( + grp_iter, chan); + if (end_of_chain && slot_cnt) { + /* Should wait for ZeroSum complete */ + if (cookie > 0) + chan->completed_cookie = cookie; + return; + } + + if (slot_cnt == 0 || end_of_chain) + break; + } + + /* the group should be complete at this point */ + BUG_ON(slot_cnt); + + slots_per_op = 0; + group_start = NULL; + if (end_of_chain) + break; + else + continue; + } else if (slots_per_op) /* wait for group completion */ + continue; + + cookie = apm82181_adma_run_tx_complete_actions(iter, chan, + cookie); + + if (apm82181_adma_clean_slot(iter, chan)) + break; + } + + BUG_ON(!seen_current); + + if (cookie > 0) { + chan->completed_cookie = cookie; + DBG("completed cookie %d\n", cookie); + } + +} + +/** + * apm82181_adma_tasklet - clean up watch-dog initiator + */ +static void apm82181_adma_tasklet (unsigned long data) +{ + apm82181_ch_t *chan = (apm82181_ch_t *) data; + spin_lock(&chan->lock); + INFO; + __apm82181_adma_slot_cleanup(chan); + spin_unlock(&chan->lock); +} + +/** + * apm82181_adma_slot_cleanup - clean up scheduled initiator + */ +static void apm82181_adma_slot_cleanup (apm82181_ch_t *chan) +{ + spin_lock_bh(&chan->lock); + __apm82181_adma_slot_cleanup(chan); + spin_unlock_bh(&chan->lock); +} + +/** + * apm82181_adma_alloc_slots - allocate free slots (if any) + */ +static apm82181_desc_t *apm82181_adma_alloc_slots( + apm82181_ch_t *chan, int num_slots, + int slots_per_op) +{ + apm82181_desc_t *iter = NULL, *_iter, *alloc_start = NULL; + struct list_head chain = LIST_HEAD_INIT(chain); + int slots_found, retry = 0; + + + BUG_ON(!num_slots || !slots_per_op); + /* start search from the last allocated descrtiptor + * if a contiguous allocation can not be found start searching + * from the beginning of the list + */ +retry: + slots_found = 0; + if (retry == 0) + iter = chan->last_used; + else + iter = list_entry(&chan->all_slots, apm82181_desc_t, + slot_node); + prefetch(iter); + DBG("---iter at %p idx %d\n ",iter,iter->idx); + list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, + slot_node) { + prefetch(_iter); + prefetch(&_iter->async_tx); + if (iter->slots_per_op) { + slots_found = 0; + continue; + } + + /* start the allocation if the slot is correctly aligned */ + if (!slots_found++) + alloc_start = iter; + if (slots_found == num_slots) { + apm82181_desc_t *alloc_tail = NULL; + apm82181_desc_t *last_used = NULL; + iter = alloc_start; + while (num_slots) { + int i; + + /* pre-ack all but the last descriptor */ + if (num_slots != slots_per_op) { + async_tx_ack(&iter->async_tx); + } + list_add_tail(&iter->chain_node, &chain); + alloc_tail = iter; + iter->async_tx.cookie = 0; + iter->hw_next = NULL; + iter->flags = 0; + iter->slot_cnt = num_slots; + for (i = 0; i < slots_per_op; i++) { + iter->slots_per_op = slots_per_op - i; + last_used = iter; + iter = list_entry(iter->slot_node.next, + apm82181_desc_t, + slot_node); + } + num_slots -= slots_per_op; + } + alloc_tail->group_head = alloc_start; + alloc_tail->async_tx.cookie = -EBUSY; + list_splice(&chain, &alloc_tail->group_list); + chan->last_used = last_used; + DBG("---slot allocated at %llx idx %d, hw_desc %p tx_ack %d\n", + alloc_tail->phys, alloc_tail->idx, alloc_tail->hw_desc, + async_tx_test_ack(&alloc_tail->async_tx)); + return alloc_tail; + } + } + if (!retry++) + goto retry; +#ifdef ADMA_DEBUG + static int empty_slot_cnt; + if(!(empty_slot_cnt%100)) + printk(KERN_INFO"No empty slots trying to free some\n"); + empty_slot_cnt++; +#endif + /* try to free some slots if the allocation fails */ + tasklet_schedule(&chan->irq_tasklet); + return NULL; +} + +/** + * apm82181_chan_xor_slot_count - get the number of slots necessary for + * XOR operation + */ +static inline int apm82181_chan_xor_slot_count(size_t len, int src_cnt, + int *slots_per_op) +{ + int slot_cnt; + + /* each XOR descriptor provides up to 16 source operands */ + slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; + + if (likely(len <= APM82181_ADMA_XOR_MAX_BYTE_COUNT)) + return slot_cnt; + + printk(KERN_ERR "%s: len %d > max %d !!\n", + __func__, len, APM82181_ADMA_XOR_MAX_BYTE_COUNT); + BUG(); + return slot_cnt; +} + +/** + * apm82181_desc_init_null_xor - initialize the descriptor for NULL XOR + * pseudo operation + */ +static inline void apm82181_desc_init_null_xor(apm82181_desc_t *desc) +{ + memset (desc->hw_desc, 0, sizeof(xor_cb_t)); + desc->hw_next = NULL; + desc->src_cnt = 0; + desc->dst_cnt = 1; +} +/** + * apm82181_chan_set_first_xor_descriptor - initi XORcore chain + */ +static inline void apm82181_chan_set_first_xor_descriptor(apm82181_ch_t *chan, + apm82181_desc_t *next_desc) +{ + volatile xor_regs_t *xor_reg; + + xor_reg = chan->device->xor_base; + + if (xor_reg->sr & XOR_SR_XCP_BIT) + printk(KERN_INFO "%s: Warn: XORcore is running " + "when try to set the first CDB!\n", + __func__); + + xor_last_submit = xor_last_linked = next_desc; + + xor_reg->crsr = XOR_CRSR_64BA_BIT; + + xor_reg->cblalr = next_desc->phys; + xor_reg->cblahr = 0; + xor_reg->cbcr |= XOR_CBCR_LNK_BIT; + + chan->hw_chain_inited = 1; +} +/** + * apm82181_chan_start_null_xor - initiate the first XOR operation (DMA engines + * use FIFOs (as opposite to chains used in XOR) so this is a XOR + * specific operation) + */ +static void apm82181_chan_start_null_xor(apm82181_ch_t *chan) +{ + apm82181_desc_t *sw_desc, *group_start; + dma_cookie_t cookie; + int slot_cnt, slots_per_op; + volatile xor_regs_t *xor_reg = chan->device->xor_base; + + dev_dbg(chan->device->common.dev, + "apm82181 adma%d: %s\n", chan->device->id, __func__); + INFO; + spin_lock_bh(&chan->lock); + slot_cnt = apm82181_chan_xor_slot_count(0, 2, &slots_per_op); + sw_desc = apm82181_adma_alloc_slots(chan, slot_cnt, slots_per_op); + if (sw_desc) { + INFO; + group_start = sw_desc->group_head; + list_splice_init(&sw_desc->group_list, &chan->chain); + async_tx_ack(&sw_desc->async_tx); + apm82181_desc_init_null_xor(group_start); + INFO; + + cookie = chan->common.cookie; + cookie++; + if (cookie <= 1) + cookie = 2; + + /* initialize the completed cookie to be less than + * the most recently used cookie + */ + chan->completed_cookie = cookie - 1; + chan->common.cookie = sw_desc->async_tx.cookie = cookie; + + /* channel should not be busy */ + BUG_ON(apm82181_chan_is_busy(chan)); + + /* set the descriptor address */ + apm82181_chan_set_first_xor_descriptor(chan, sw_desc); + + /* run the descriptor */ + xor_reg->crsr = XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT; + } else + printk(KERN_ERR "apm82181 adma%d" + " failed to allocate null descriptor\n", + chan->device->id); + spin_unlock_bh(&chan->lock); +} + +/** + * apm82181_adma_alloc_chan_resources - allocate pools for CDB slots + */ +static int apm82181_adma_alloc_chan_resources(struct dma_chan *chan) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *slot = NULL; + char *hw_desc; + int i, db_sz; + int init = apm82181_chan->slots_allocated ? 0 : 1; + + chan->chan_id = apm82181_chan->device->id; + + /* Allocate descriptor slots */ + i = apm82181_chan->slots_allocated; + if (apm82181_chan->device->id != APM82181_XOR_ID) + db_sz = sizeof (dma_cdb_t); + else + db_sz = sizeof (xor_cb_t); + + for (; i < (apm82181_chan->device->pool_size/db_sz); i++) { + slot = kzalloc(sizeof(apm82181_desc_t), GFP_KERNEL); + if (!slot) { + printk(KERN_INFO "APM82181/GT ADMA Channel only initialized" + " %d descriptor slots", i--); + break; + } + + hw_desc = (char *) apm82181_chan->device->dma_desc_pool_virt; + slot->hw_desc = (void *) &hw_desc[i * db_sz]; + dma_async_tx_descriptor_init(&slot->async_tx, chan); + slot->async_tx.tx_submit = apm82181_adma_tx_submit; + INIT_LIST_HEAD(&slot->chain_node); + INIT_LIST_HEAD(&slot->slot_node); + INIT_LIST_HEAD(&slot->group_list); + slot->phys = apm82181_chan->device->dma_desc_pool + i * db_sz; + slot->idx = i; + spin_lock_bh(&apm82181_chan->lock); + apm82181_chan->slots_allocated++; + list_add_tail(&slot->slot_node, &apm82181_chan->all_slots); + spin_unlock_bh(&apm82181_chan->lock); + } + + if (i && !apm82181_chan->last_used) { + apm82181_chan->last_used = + list_entry(apm82181_chan->all_slots.next, + apm82181_desc_t, + slot_node); + } + + printk("apm82181 adma%d: allocated %d descriptor slots\n", + apm82181_chan->device->id, i); + + /* initialize the channel and the chain with a null operation */ + if (init) { + switch (apm82181_chan->device->id) + { + apm82181_chan->hw_chain_inited = 0; + case APM82181_PDMA0_ID: + apm82181_dma_tchan[0] = apm82181_chan; + break; + case APM82181_PDMA1_ID: + apm82181_dma_tchan[1] = apm82181_chan; + break; + case APM82181_PDMA2_ID: + apm82181_dma_tchan[2] = apm82181_chan; + break; + case APM82181_PDMA3_ID: + apm82181_dma_tchan[3] = apm82181_chan; + break; + case APM82181_XOR_ID: + apm82181_dma_tchan[4] = apm82181_chan; + apm82181_chan_start_null_xor(apm82181_chan); + break; + default: + BUG(); + } + apm82181_chan->needs_unmap = 1; + } + + return (i > 0) ? i : -ENOMEM; +} + +/** + * apm82181_desc_assign_cookie - assign a cookie + */ +static dma_cookie_t apm82181_desc_assign_cookie(apm82181_ch_t *chan, + apm82181_desc_t *desc) +{ + dma_cookie_t cookie = chan->common.cookie; + cookie++; + if (cookie < 0) + cookie = 1; + chan->common.cookie = desc->async_tx.cookie = cookie; + return cookie; +} + + +/** + * apm82181_adma_check_threshold - append CDBs to h/w chain if threshold + * has been achieved + */ +static void apm82181_adma_check_threshold(apm82181_ch_t *chan) +{ + dev_dbg(chan->device->common.dev, "apm82181 adma%d: pending: %d\n", + chan->device->id, chan->pending); + INFO; + if (chan->pending >= APM82181_ADMA_THRESHOLD) { + chan->pending = 0; + apm82181_chan_append(chan); + } +} + +/** + * apm82181_adma_tx_submit - submit new descriptor group to the channel + * (it's not necessary that descriptors will be submitted to the h/w + * chains too right now) + */ +static dma_cookie_t apm82181_adma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + apm82181_desc_t *sw_desc = tx_to_apm82181_adma_slot(tx); + apm82181_ch_t *chan = to_apm82181_adma_chan(tx->chan); + apm82181_desc_t *group_start, *old_chain_tail; + int slot_cnt; + int slots_per_op; + dma_cookie_t cookie; + group_start = sw_desc->group_head; + slot_cnt = group_start->slot_cnt; + slots_per_op = group_start->slots_per_op; + INFO; + spin_lock_bh(&chan->lock); + cookie = apm82181_desc_assign_cookie(chan, sw_desc); + + if (unlikely(list_empty(&chan->chain))) { + /* first peer */ + list_splice_init(&sw_desc->group_list, &chan->chain); + chan_first_cdb[chan->device->id] = group_start; + } else { + /* isn't first peer, bind CDBs to chain */ + old_chain_tail = list_entry(chan->chain.prev, + apm82181_desc_t, chain_node); + list_splice_init(&sw_desc->group_list, + &old_chain_tail->chain_node); + /* fix up the hardware chain */ + apm82181_desc_set_link(chan, old_chain_tail, group_start); + } + + /* increment the pending count by the number of operations */ + chan->pending += slot_cnt / slots_per_op; + apm82181_adma_check_threshold(chan); + spin_unlock_bh(&chan->lock); + + DBG("apm82181 adma%d:cookie: %d slot: %d tx %p\n", + chan->device->id, sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); + return cookie; +} +/** + * apm82181_adma_prep_dma_xor - prepare CDB for a XOR operation + */ +static struct dma_async_tx_descriptor *apm82181_adma_prep_dma_xor( + struct dma_chan *chan, dma_addr_t dma_dest, + dma_addr_t *dma_src, unsigned int src_cnt, size_t len, + unsigned long flags) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + +#ifdef ADMA_DEBUG + printk("\n%s(%d):\n\tsrc: ", __func__, + apm82181_chan->device->id); + for (slot_cnt=0; slot_cnt < src_cnt; slot_cnt++) + printk("0x%llx ", dma_src[slot_cnt]); + printk("\n\tdst: 0x%llx\n", dma_dest); +#endif + if (unlikely(!len)) + return NULL; + BUG_ON(unlikely(len > APM82181_ADMA_XOR_MAX_BYTE_COUNT)); + + dev_dbg(apm82181_chan->device->common.dev, + "apm82181 adma%d: %s src_cnt: %d len: %u int_en: %d\n", + apm82181_chan->device->id, __func__, src_cnt, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + spin_lock_bh(&apm82181_chan->lock); + slot_cnt = apm82181_chan_xor_slot_count(len, src_cnt, &slots_per_op); + sw_desc = apm82181_adma_alloc_slots(apm82181_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + apm82181_desc_init_xor(group_start, src_cnt, flags); + apm82181_adma_set_dest(group_start, dma_dest, 0); + while (src_cnt--) + apm82181_adma_set_src(group_start, + dma_src[src_cnt], src_cnt); + apm82181_desc_set_byte_count(group_start, apm82181_chan, len); + sw_desc->unmap_len = len; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&apm82181_chan->lock); + + return sw_desc ? &sw_desc->async_tx : NULL; +} +/** + * apm82181_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation + */ +static struct dma_async_tx_descriptor *apm82181_adma_prep_dma_interrupt( + struct dma_chan *chan, unsigned long flags) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + + dev_dbg(apm82181_chan->device->common.dev, + "apm82181 adma%d: %s\n", apm82181_chan->device->id, + __FUNCTION__); + spin_lock_bh(&apm82181_chan->lock); + slot_cnt = slots_per_op = 1; + sw_desc = apm82181_adma_alloc_slots(apm82181_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + apm82181_desc_init_interrupt(group_start, apm82181_chan); + group_start->unmap_len = 0; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&apm82181_chan->lock); + + return sw_desc ? &sw_desc->async_tx : NULL; +} + +/** + * apm82181_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation + */ +static struct dma_async_tx_descriptor *apm82181_adma_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dma_dest, + dma_addr_t dma_src, size_t len, unsigned long flags) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + if (unlikely(!len)) + return NULL; + BUG_ON(unlikely(len > APM82181_ADMA_DMA_MAX_BYTE_COUNT)); + + spin_lock_bh(&apm82181_chan->lock); + + dev_dbg(apm82181_chan->device->common.dev, + "apm82181 adma%d: %s len: %u int_en %d \n", + apm82181_chan->device->id, __FUNCTION__, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + slot_cnt = slots_per_op = 1; + sw_desc = apm82181_adma_alloc_slots(apm82181_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + flags |= DMA_PREP_INTERRUPT; + apm82181_desc_init_memcpy(group_start, flags); + apm82181_adma_set_dest(group_start, dma_dest, 0); + apm82181_adma_set_src(group_start, dma_src, 0); + apm82181_desc_set_byte_count(group_start, apm82181_chan, len); + sw_desc->unmap_len = len; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&apm82181_chan->lock); + return sw_desc ? &sw_desc->async_tx : NULL; +} + +/** + * apm82181_adma_prep_dma_memset - prepare CDB for a MEMSET operation + */ +static struct dma_async_tx_descriptor *apm82181_adma_prep_dma_memset( + struct dma_chan *chan, dma_addr_t dma_dest, int value, + size_t len, unsigned long flags) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + if (unlikely(!len)) + return NULL; + BUG_ON(unlikely(len > APM82181_ADMA_DMA_MAX_BYTE_COUNT)); + + spin_lock_bh(&apm82181_chan->lock); + + dev_dbg(apm82181_chan->device->common.dev, + "apm82181 adma%d: %s cal: %u len: %u int_en %d\n", + apm82181_chan->device->id, __FUNCTION__, value, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + slot_cnt = slots_per_op = 1; + sw_desc = apm82181_adma_alloc_slots(apm82181_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + apm82181_desc_init_memset(group_start, value, flags); + apm82181_adma_set_dest(group_start, dma_dest, 0); + apm82181_desc_set_byte_count(group_start, apm82181_chan, len); + sw_desc->unmap_len = len; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&apm82181_chan->lock); + + return sw_desc ? &sw_desc->async_tx : NULL; +} + + +/** + * apm82181_adma_set_dest - set destination address into descriptor + */ +static void apm82181_adma_set_dest(apm82181_desc_t *sw_desc, + dma_addr_t addr, int index) +{ + apm82181_ch_t *chan = to_apm82181_adma_chan(sw_desc->async_tx.chan); + BUG_ON(index >= sw_desc->dst_cnt); + + switch (chan->device->id) { + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + /* to do: support transfers lengths > + * APM82181_ADMA_DMA/XOR_MAX_BYTE_COUNT + */ + apm82181_desc_set_dest_addr(sw_desc->group_head, + // chan, 0x8, addr, index); // Enabling HB bus + chan, addr, index); + break; + case APM82181_XOR_ID: + sw_desc = apm82181_get_group_entry(sw_desc, index); + apm82181_desc_set_dest_addr(sw_desc, chan, + addr, index); + break; + default: + BUG(); + } +} + + +/** + * apm82181_adma_free_chan_resources - free the resources allocated + */ +static void apm82181_adma_free_chan_resources(struct dma_chan *chan) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + apm82181_desc_t *iter, *_iter; + int in_use_descs = 0; + + apm82181_adma_slot_cleanup(apm82181_chan); + + spin_lock_bh(&apm82181_chan->lock); + list_for_each_entry_safe(iter, _iter, &apm82181_chan->chain, + chain_node) { + in_use_descs++; + list_del(&iter->chain_node); + } + list_for_each_entry_safe_reverse(iter, _iter, + &apm82181_chan->all_slots, slot_node) { + list_del(&iter->slot_node); + kfree(iter); + apm82181_chan->slots_allocated--; + } + apm82181_chan->last_used = NULL; + + dev_dbg(apm82181_chan->device->common.dev, + "apm82181 adma%d %s slots_allocated %d\n", + apm82181_chan->device->id, + __FUNCTION__, apm82181_chan->slots_allocated); + spin_unlock_bh(&apm82181_chan->lock); + + /* one is ok since we left it on there on purpose */ + if (in_use_descs > 1) + printk(KERN_ERR "GT: Freeing %d in use descriptors!\n", + in_use_descs - 1); +} + +/** + * apm82181_adma_is_complete - poll the status of an ADMA transaction + * @chan: ADMA channel handle + * @cookie: ADMA transaction identifier + */ +static enum dma_status apm82181_adma_is_complete(struct dma_chan *chan, + dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + dma_cookie_t last_used; + dma_cookie_t last_complete; + enum dma_status ret; + + last_used = chan->cookie; + last_complete = apm82181_chan->completed_cookie; + + if (done) + *done= last_complete; + if (used) + *used = last_used; + + ret = dma_async_is_complete(cookie, last_complete, last_used); + if (ret == DMA_SUCCESS) + return ret; + + apm82181_adma_slot_cleanup(apm82181_chan); + + last_used = chan->cookie; + last_complete = apm82181_chan->completed_cookie; + + if (done) + *done= last_complete; + if (used) + *used = last_used; + + return dma_async_is_complete(cookie, last_complete, last_used); +} + +/** + * apm82181_adma_eot_handler - end of transfer interrupt handler + */ +static irqreturn_t apm82181_adma_eot_handler(int irq, void *data) +{ + apm82181_ch_t *chan = data; + + dev_dbg(chan->device->common.dev, + "apm82181 adma%d: %s\n", chan->device->id, __FUNCTION__); + INFO; + if(chan->device->id == APM82181_XOR_ID) + tasklet_schedule(&chan->irq_tasklet); + apm82181_adma_device_clear_eot_status(chan); + + return IRQ_HANDLED; +} + +/** + * apm82181_adma_err_handler - DMA error interrupt handler; + * do the same things as a eot handler + */ +#if 0 +static irqreturn_t apm82181_adma_err_handler(int irq, void *data) +{ + apm82181_ch_t *chan = data; + dev_dbg(chan->device->common.dev, + "apm82181 adma%d: %s\n", chan->device->id, __FUNCTION__); + tasklet_schedule(&chan->irq_tasklet); + apm82181_adma_device_clear_eot_status(chan); + + return IRQ_HANDLED; +} +#endif +/** + * apm82181_test_callback - called when test operation has been done + */ +static void apm82181_test_callback (void *unused) +{ + complete(&apm82181_r5_test_comp); +} + +/** + * apm82181_adma_issue_pending - flush all pending descriptors to h/w + */ +static void apm82181_adma_issue_pending(struct dma_chan *chan) +{ + apm82181_ch_t *apm82181_chan = to_apm82181_adma_chan(chan); + + DBG("apm82181 adma%d: %s %d \n", apm82181_chan->device->id, + __FUNCTION__, apm82181_chan->pending); + if (apm82181_chan->pending) { + apm82181_chan->pending = 0; + apm82181_chan_append(apm82181_chan); + } +} + +/** + * apm82181_adma_remove - remove the asynch device + */ +static int __devexit apm82181_pdma_remove(struct platform_device *dev) +{ + apm82181_dev_t *device = platform_get_drvdata(dev); + struct dma_chan *chan, *_chan; + struct ppc_dma_chan_ref *ref, *_ref; + apm82181_ch_t *apm82181_chan; + int i; + + dma_async_device_unregister(&device->common); + + for (i = 0; i < 3; i++) { + u32 irq; + irq = platform_get_irq(dev, i); + free_irq(irq, device); + } + + + do { + struct resource *res; + res = platform_get_resource(dev, IORESOURCE_MEM, 0); + release_mem_region(res->start, res->end - res->start); + } while (0); + + list_for_each_entry_safe(chan, _chan, &device->common.channels, + device_node) { + apm82181_chan = to_apm82181_adma_chan(chan); + list_del(&chan->device_node); + kfree(apm82181_chan); + } + + list_for_each_entry_safe(ref, _ref, &ppc_adma_chan_list, node) { + list_del(&ref->node); + kfree(ref); + } + + kfree(device); + + return 0; +} + +static inline void xor_hw_init (apm82181_dev_t *adev) +{ + volatile xor_regs_t *xor_reg = adev->xor_base; + /* Reset XOR */ + xor_reg->crsr = XOR_CRSR_XASR_BIT; + xor_reg->crrr = XOR_CRSR_64BA_BIT; + + /* enable XOR engine interrupts */ + xor_reg->ier = XOR_IE_CBCIE_BIT | + XOR_IE_ICBIE_BIT | XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT; +} + +/* + * Per channel probe + */ +static int __devinit apm82181_dma_per_chan_probe(struct of_device *ofdev, + const struct of_device_id *match) +{ + int ret = 0, irq; + const u32 *index, *dcr_regs, *pool_size; + apm82181_plb_dma_t *pdma; + apm82181_dev_t *adev; + apm82181_ch_t *chan; + struct ppc_dma_chan_ref *ref; + struct device_node *np = ofdev->node; + struct resource res; + int len; + + INFO; + pdma = dev_get_drvdata(ofdev->dev.parent); + BUG_ON(!pdma); + if ((adev = kzalloc(sizeof(*adev), GFP_KERNEL)) == NULL) { + printk("ERROR:No Free memory for allocating dma channels\n"); + ret = -ENOMEM; + goto out; + } + adev->dev = &ofdev->dev; + index = of_get_property(np, "cell-index", NULL); + if(!index) { + printk(KERN_ERR "adma-channel: Device node %s has missing or invalid " + "cell-index property\n", np->full_name); + goto err; + } + adev->id = (int)*index; + /* The XOR engine/PLB DMA 4 channels have different resources/pool_sizes */ + if (adev->id != APM82181_XOR_ID){ + dcr_regs = of_get_property(np, "dcr-reg", &len); + if (!dcr_regs || (len != 2 * sizeof(u32))) { + printk(KERN_ERR "plb_dma channel%d: Can't get DCR register base !", + adev->id); + goto err; + } + adev->dcr_base = dcr_regs[0]; + + pool_size = of_get_property(np, "pool_size", NULL); + if(!pool_size) { + printk(KERN_ERR "plb_dma channel%d: Device node %s has missing or " + "invalid pool_size property\n", adev->id, np->full_name); + goto err; + } + adev->pool_size = *pool_size; + } else { + if (of_address_to_resource(np, 0, &res)) { + printk(KERN_ERR "adma_xor channel%d %s: could not get resource address.\n", + adev->id,np->full_name); + goto err; + } + + DBG("XOR resource start = %llx end = %llx\n", res.start, res.end); + adev->xor_base = ioremap(res.start, res.end - res.start + 1); + if (!adev->xor_base){ + printk(KERN_ERR "XOR engine registers memory mapping failed.\n"); + goto err; + } + adev->pool_size = PAGE_SIZE << 1; + } + + adev->pdma = pdma; + adev->ofdev = ofdev; + dev_set_drvdata(&(ofdev->dev),adev); + + switch (adev->id){ + case APM82181_PDMA0_ID: + case APM82181_PDMA1_ID: + case APM82181_PDMA2_ID: + case APM82181_PDMA3_ID: + dma_cap_set(DMA_MEMCPY,adev->cap_mask); + break; + case APM82181_XOR_ID: + dma_cap_set(DMA_XOR,adev->cap_mask); + dma_cap_set(DMA_INTERRUPT,adev->cap_mask); + break; + default: + BUG(); + } + /* XOR h/w configuration */ + if(adev->id == APM82181_XOR_ID) + xor_hw_init(adev); + /* allocate coherent memory for hardware descriptors + * note: writecombine gives slightly better performance, but + * requires that we explicitly drain the write buffer + */ + if ((adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, + adev->pool_size, &adev->dma_desc_pool, GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_dma_alloc; + } + + adev->common.cap_mask = adev->cap_mask; + INIT_LIST_HEAD(&adev->common.channels); + /* set base routines */ + adev->common.device_alloc_chan_resources = + apm82181_adma_alloc_chan_resources; + adev->common.device_free_chan_resources = + apm82181_adma_free_chan_resources; + adev->common.device_is_tx_complete = apm82181_adma_is_complete; + adev->common.device_issue_pending = apm82181_adma_issue_pending; + adev->common.dev = &ofdev->dev; + + /* set prep routines based on capability */ + if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { + adev->common.device_prep_dma_memcpy = + apm82181_adma_prep_dma_memcpy; + } + if (dma_has_cap(DMA_MEMSET, adev->common.cap_mask)) { + adev->common.device_prep_dma_memset = + apm82181_adma_prep_dma_memset; + } + + if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { + adev->common.device_prep_dma_interrupt = + apm82181_adma_prep_dma_interrupt; + } + + if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { + adev->common.max_xor = XOR_MAX_OPS; + adev->common.device_prep_dma_xor = + apm82181_adma_prep_dma_xor; + } + + /* create a channel */ + if ((chan = kzalloc(sizeof(*chan), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_chan_alloc; + } + tasklet_init(&chan->irq_tasklet, apm82181_adma_tasklet, + (unsigned long)chan); + + irq = irq_of_parse_and_map(np, 0); + switch (adev->id){ + case 0: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-chan0", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + case 1: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-chan1", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + case 2: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-chan2", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + case 3: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-chan3", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + case 4: + if (irq >= 0) { + ret = request_irq(irq, apm82181_adma_eot_handler, + IRQF_DISABLED, "adma-xor", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + break; + default: + BUG(); + } + + spin_lock_init(&chan->lock); + chan->device = adev; + INIT_LIST_HEAD(&chan->chain); + INIT_LIST_HEAD(&chan->all_slots); + chan->common.device = &adev->common; + list_add_tail(&chan->common.device_node, &adev->common.channels); + adev->common.chancnt++; + + printk( "AMCC(R) APM82181 ADMA Engine found [%d]: " + "( capabilities: %s%s%s%s%s%s%s)\n", + adev->id, + dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq_xor " : "", + dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : + "", + dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", + dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : + "", + dma_has_cap(DMA_MEMSET, adev->common.cap_mask) ? "memset " : "", + dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", + dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "int " : ""); + INFO; + ret = dma_async_device_register(&adev->common); + if (ret) { + dev_err(&ofdev->dev, "failed to register dma async device"); + goto err_irq; + } + INFO; + ref = kmalloc(sizeof(*ref), GFP_KERNEL); + if (ref) { + INFO; + ref->chan = &chan->common; + INIT_LIST_HEAD(&ref->node); + list_add_tail(&ref->node, &ppc_adma_chan_list); + } else + dev_warn(&ofdev->dev, "failed to allocate channel reference!\n"); + + goto out; +err_irq: + kfree(chan); +err_chan_alloc: + dma_free_coherent(&ofdev->dev, adev->pool_size, + adev->dma_desc_pool_virt, adev->dma_desc_pool); +err_dma_alloc: + if (adev->xor_base) + iounmap(adev->xor_base); +err: + kfree(adev); +out: + return ret; +} + +static struct of_device_id dma_4chan_match[] = +{ + { + .compatible = "amcc,apm82181-adma", + }, + {}, +}; + +static struct of_device_id dma_per_chan_match[] = { + {.compatible = "amcc,apm82181-dma-4channel",}, + {.compatible = "amcc,xor",}, + {}, +}; +/* + * apm82181_adma_probe - probe the asynch device + */ +static int __devinit apm82181_pdma_probe(struct of_device *ofdev, + const struct of_device_id *match) +{ + int ret = 0; + apm82181_plb_dma_t *pdma; + + if ((pdma = kzalloc(sizeof(*pdma), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto out; + } + pdma->dev = &ofdev->dev; + pdma->ofdev = ofdev; + printk(PPC4XX_EDMA "Probing AMCC APM82181 ADMA engines...\n"); + + dev_set_drvdata(&(ofdev->dev),pdma); + of_platform_bus_probe(ofdev->node, dma_per_chan_match,&ofdev->dev); + +out: + return ret; +} + +/* + * apm82181_test_xor - test are RAID-5 XOR capability enabled successfully. + * For this we just perform one DMA XOR operation with the 3 sources + * to a destination + */ +static int apm82181_test_xor (apm82181_ch_t *chan) +{ + apm82181_desc_t *sw_desc, *group_start; + struct page *pg_src[3], *pg_dest; + char *a; + dma_addr_t dma_src_addr[3]; + dma_addr_t dma_dst_addr; + int rval = -EFAULT, i; + int len = PAGE_SIZE, src_cnt = 3; + int slot_cnt, slots_per_op; + INFO; + printk("ADMA channel %d XOR testing\n",chan->device->id); + for(i = 0; i < 3; i++){ + pg_src[i] = alloc_page(GFP_KERNEL); + if (!pg_src[i]) + return -ENOMEM; + } + pg_dest = alloc_page(GFP_KERNEL); + if (!pg_dest) + return -ENOMEM; + /* Fill the test page with ones */ + memset(page_address(pg_src[0]), 0xDA, len); + memset(page_address(pg_src[1]), 0xDA, len); + memset(page_address(pg_src[2]), 0x00, len); + memset(page_address(pg_dest), 0xA5, len); + for(i = 0; i < 3; i++){ + a = page_address(pg_src[i]); + printk("The virtual addr of src %d =%x\n",i, (unsigned int)a); + MEM_HEXDUMP(a,50); + } + a = page_address(pg_dest); + printk("The virtual addr of dest=%x\n", (unsigned int)a); + MEM_HEXDUMP(a,50); + + for(i = 0; i < 3; i++){ + dma_src_addr[i] = dma_map_page(chan->device->dev, pg_src[i], 0, len, + DMA_BIDIRECTIONAL); + } + dma_dst_addr = dma_map_page(chan->device->dev, pg_dest, 0, len, + DMA_BIDIRECTIONAL); + printk("dma_src_addr[0]: %llx; dma_src_addr[1]: %llx;\n " + "dma_src_addr[2]: %llx; dma_dst_addr %llx, len: %x\n", dma_src_addr[0], + dma_src_addr[1], dma_src_addr[2], dma_dst_addr, len); + + spin_lock_bh(&chan->lock); + slot_cnt = apm82181_chan_xor_slot_count(len, src_cnt, &slots_per_op); + sw_desc = apm82181_adma_alloc_slots(chan, slot_cnt, slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + apm82181_desc_init_xor(group_start, src_cnt, DMA_PREP_INTERRUPT); + /* Setup addresses */ + while (src_cnt--) + apm82181_adma_set_src(group_start, + dma_src_addr[src_cnt], src_cnt); + apm82181_adma_set_dest(group_start, dma_dst_addr, 0); + apm82181_desc_set_byte_count(group_start, chan, len); + sw_desc->unmap_len = PAGE_SIZE; + } else { + rval = -EFAULT; + spin_unlock_bh(&chan->lock); + goto exit; + } + spin_unlock_bh(&chan->lock); + + printk("Submit CDB...\n"); + MEM_HEXDUMP(sw_desc->hw_desc, 96); + async_tx_ack(&sw_desc->async_tx); + sw_desc->async_tx.callback = apm82181_test_callback; + sw_desc->async_tx.callback_param = NULL; + + init_completion(&apm82181_r5_test_comp); + apm82181_adma_tx_submit(&sw_desc->async_tx); + apm82181_adma_issue_pending(&chan->common); + //wait_for_completion(&apm82181_r5_test_comp); + /* wait for a while so that dma transaction finishes */ + mdelay(100); + /* Now check if the test page zeroed */ + a = page_address(pg_dest); + /* XOR result at destination */ + MEM_HEXDUMP(a,50); + if ((*(u32*)a) == 0x00000000 && memcmp(a, a+4, PAGE_SIZE-4)==0) { + /* page dest XOR is corect as expected - RAID-5 enabled */ + rval = 0; + } else { + /* RAID-5 was not enabled */ + rval = -EINVAL; + } + +exit: + dma_unmap_page(chan->device->dev, dma_src_addr[0], PAGE_SIZE, DMA_BIDIRECTIONAL); + dma_unmap_page(chan->device->dev, dma_src_addr[1], PAGE_SIZE, DMA_BIDIRECTIONAL); + dma_unmap_page(chan->device->dev, dma_src_addr[2], PAGE_SIZE, DMA_BIDIRECTIONAL); + dma_unmap_page(chan->device->dev, dma_dst_addr, PAGE_SIZE, DMA_BIDIRECTIONAL); + __free_page(pg_src[0]); + __free_page(pg_src[1]); + __free_page(pg_src[2]); + __free_page(pg_dest); + return rval; +} + + +/* + * apm82181_test_dma - test are RAID-5 capabilities enabled successfully. + * For this we just perform one WXOR operation with the same source + * and destination addresses, the GF-multiplier is 1; so if RAID-5 + o/of_platform_driver_unregister(&apm82181_pdma_driver); + * capabilities are enabled then we'll get src/dst filled with zero. + */ +static int apm82181_test_dma (apm82181_ch_t *chan) +{ + apm82181_desc_t *sw_desc; + struct page *pg_src, *pg_dest; + char *a, *d; + dma_addr_t dma_src_addr; + dma_addr_t dma_dst_addr; + int rval = -EFAULT; + int len = PAGE_SIZE; + + printk("PLB DMA channel %d memcpy testing\n",chan->device->id); + pg_src = alloc_page(GFP_KERNEL); + if (!pg_src) + return -ENOMEM; + pg_dest = alloc_page(GFP_KERNEL); + if (!pg_dest) + return -ENOMEM; + /* Fill the test page with ones */ + memset(page_address(pg_src), 0x77, len); + memset(page_address(pg_dest), 0xa5, len); + a = page_address(pg_src); + printk("The virtual addr of src =%x\n", (unsigned int)a); + MEM_HEXDUMP(a,50); + a = page_address(pg_dest); + printk("The virtual addr of dest=%x\n", (unsigned int)a); + MEM_HEXDUMP(a,50); + dma_src_addr = dma_map_page(chan->device->dev, pg_src, 0, len, + DMA_BIDIRECTIONAL); + dma_dst_addr = dma_map_page(chan->device->dev, pg_dest, 0, len, + DMA_BIDIRECTIONAL); + printk("dma_src_addr: %llx; dma_dst_addr %llx\n", dma_src_addr, dma_dst_addr); + + spin_lock_bh(&chan->lock); + sw_desc = apm82181_adma_alloc_slots(chan, 1, 1); + if (sw_desc) { + /* 1 src, 1 dst, int_ena */ + apm82181_desc_init_memcpy(sw_desc, DMA_PREP_INTERRUPT); + //apm82181_desc_init_memcpy(sw_desc, 0); + /* Setup adresses */ + apm82181_adma_set_src(sw_desc, dma_src_addr, 0); + apm82181_adma_set_dest(sw_desc, dma_dst_addr, 0); + apm82181_desc_set_byte_count(sw_desc, chan, len); + sw_desc->unmap_len = PAGE_SIZE; + } else { + rval = -EFAULT; + spin_unlock_bh(&chan->lock); + goto exit; + } + spin_unlock_bh(&chan->lock); + + printk("Submit CDB...\n"); + MEM_HEXDUMP(sw_desc->hw_desc, 96); + async_tx_ack(&sw_desc->async_tx); + sw_desc->async_tx.callback = apm82181_test_callback; + sw_desc->async_tx.callback_param = NULL; + + init_completion(&apm82181_r5_test_comp); + apm82181_adma_tx_submit(&sw_desc->async_tx); + apm82181_adma_issue_pending(&chan->common); + //wait_for_completion(&apm82181_r5_test_comp); + + a = page_address(pg_src); + d = page_address(pg_dest); + if (!memcmp(a, d, len)) { + rval = 0; + } else { + rval = -EINVAL; + } + + a = page_address(pg_src); + printk("\nAfter DMA done:"); + printk("\nsrc %x value:\n", (unsigned int)a); + MEM_HEXDUMP(a,96); + a = page_address(pg_dest); + printk("\ndest%x value:\n", (unsigned int)a); + MEM_HEXDUMP(a,96); + +exit: + __free_page(pg_src); + __free_page(pg_dest); + return rval; +} + +static struct of_platform_driver apm82181_pdma_driver = { + .name = "apm82181_plb_dma", + .match_table = dma_4chan_match, + + .probe = apm82181_pdma_probe, + //.remove = apm82181_pdma_remove, +}; +struct of_platform_driver apm82181_dma_per_chan_driver = { + .name = "apm82181-dma-4channel", + .match_table = dma_per_chan_match, + .probe = apm82181_dma_per_chan_probe, +}; + +static int apm82181_xor_read (char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + char *p = page; + + p += sprintf(p, "%s\n", + apm82181_xor_verified ? + "APM82181 ASYNC XOR capability are VERIFIED.\n" : + "APM82181 ASYNC XOR capability are NOT VERIFIED.\n"); + + return p - page; +} + +static int apm82181_xor_write (struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + if(count != 2) + return -EFAULT; + /* Verify does it really work now */ + if (!apm82181_test_xor(apm82181_dma_tchan[4])) { + /* APM82181 RAID-5 XOR has been activated successfully */; + printk("APM82181 ADMA XOR engine has been verified " + "successfully\n"); + apm82181_xor_verified = 1; + } else { + /* APM82181 RAID-5 memcpy hasn't been activated! */; + printk("APM82181 ADMA XOR engine hasn't been " + "verified yet\n"); + apm82181_xor_verified = 0; + } + + return count; +} + +static int apm82181_dma_read (char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + int i; + + printk("APM82181 ASYNC MEMCPY capability\n"); + for(i = 0; i < 4; i++){ + printk("\tPLB DMA channel %d: %s ", i, + apm82181_memcpy_verified[i] ? + "VERIFIED.\n" : "NOT VERIFIED.\n"); + } + return 0; +} + +static int apm82181_dma_write (struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + /* e.g. 0xffffffff */ + char tmp[2]; + u32 val; + + if(count != 2) + return -EFAULT; + + if (copy_from_user(tmp, buffer, count)) + return -EFAULT; + val = simple_strtoul(tmp, NULL, 10); /* decimal base */ + if(!(val == 0 || val == 1 || val == 2 || val == 3 )) { + printk("Error! Wrong channel id, please choose 1 valid id [0/1/2/3]\n"); + return -EFAULT; + } + + /* Verify does it really work now */ + if (!apm82181_test_dma(apm82181_dma_tchan[val])) { + /* APM82181 RAID-5 memcpy has been activated successfully */; + printk("APM82181 PLBDMA MEMCPY channel %d has been verified " + "successfully\n", val); + apm82181_memcpy_verified[val] = 1; + } else { + /* APM82181 RAID-5 memcpy hasn't been activated! */; + printk("APM82181 PLBDMA MEMCPY channel %d hasn't been " + "verified yet\n", val); + apm82181_memcpy_verified[val] = 0; + } + + return count; +} + +static int __init apm82181_adma_per_chan_init (void) +{ + int rval; + rval = of_register_platform_driver(&apm82181_dma_per_chan_driver); + return rval; +} + +static int __init apm82181_adma_init (void) +{ + int rval; + struct proc_dir_entry *p; + + rval = of_register_platform_driver(&apm82181_pdma_driver); + + if (rval == 0) { + /* Create /proc entries */ + apm82181_proot = proc_mkdir(APM82181_DMA_PROC_ROOT, NULL); + if (!apm82181_proot) { + printk(KERN_ERR "%s: failed to create %s proc " + "directory\n",__FUNCTION__,APM82181_DMA_PROC_ROOT); + /* User will not be able to enable h/w RAID-6 */ + return rval; + } + + /* ADMA MEMCPY verification entry */ + p = create_proc_entry("adma_memcopy_test", 0, apm82181_proot); + if (p) { + p->read_proc = apm82181_dma_read; + p->write_proc = apm82181_dma_write; + } + /* ADMA XOR capability verification entry */ + p = create_proc_entry("adma_xor_test", 0, apm82181_proot); + if (p) { + p->read_proc = apm82181_xor_read; + p->write_proc = apm82181_xor_write; + } + } + return rval; +} + +#if 0 +static void __exit apm82181_adma_exit (void) +{ + of_unregister_platform_driver(&apm82181_pdma_driver); + return; +} +module_exit(apm82181_adma_exit); +#endif + +module_init(apm82181_adma_per_chan_init); +module_init(apm82181_adma_init); + +MODULE_AUTHOR("Tai Tri Nguyen<ttnguyen@appliedmicro.com>"); +MODULE_DESCRIPTION("APM82181 ADMA Engine Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 8f99354082c..0177c3ff6ff 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -680,6 +680,7 @@ int dma_async_device_register(struct dma_device *device) struct dma_chan* chan; atomic_t *idr_ref; + printk( "--------------- %s: %i-------------------------\n",__FUNCTION__,__LINE__); if (!device) return -ENODEV; @@ -858,14 +859,10 @@ dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, dma_addr_t dma_dest, dma_src; dma_cookie_t cookie; int cpu; - unsigned long flags; dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE); - flags = DMA_CTRL_ACK | - DMA_COMPL_SRC_UNMAP_SINGLE | - DMA_COMPL_DEST_UNMAP_SINGLE; - tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); + tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, DMA_CTRL_ACK); if (!tx) { dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); @@ -907,12 +904,10 @@ dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, dma_addr_t dma_dest, dma_src; dma_cookie_t cookie; int cpu; - unsigned long flags; dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE); - flags = DMA_CTRL_ACK | DMA_COMPL_SRC_UNMAP_SINGLE; - tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, flags); + tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, DMA_CTRL_ACK); if (!tx) { dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); diff --git a/drivers/dma/ppc460ex-adma.c b/drivers/dma/ppc460ex-adma.c new file mode 100644 index 00000000000..2ef1e9d6052 --- /dev/null +++ b/drivers/dma/ppc460ex-adma.c @@ -0,0 +1,5409 @@ +/* + * Copyright(c) 2006 DENX Engineering. All rights reserved. + * + * Author: Yuri Tikhonov <yur@emcraft.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ + +/* + * This driver supports the asynchrounous DMA copy and RAID engines available + * on the AMCC PPC460ex Processors. + * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) + * ADMA driver written by D.Williams. + */ + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/async_tx.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/uaccess.h> +#include <linux/of_platform.h> +#include <linux/proc_fs.h> +#include <asm/dcr.h> +#include <asm/dcr-regs.h> +#include <asm/ppc460ex_adma.h> +#include <asm/ppc460ex_xor.h> +#include <asm/ppc4xx_ocm.h> +/* The list of channels exported by ppc460ex ADMA */ +struct list_head +ppc_adma_chan_list = LIST_HEAD_INIT(ppc_adma_chan_list); + +/* This flag is set when want to refetch the xor chain in the interrupt + * handler + */ +static u32 do_xor_refetch = 0; + +/* Pointers to last submitted to DMA0, DMA1 CDBs */ +static ppc460ex_desc_t *chan_last_sub[3]; +static ppc460ex_desc_t *chan_first_cdb[3]; + +/* Pointer to last linked and submitted xor CB */ +static ppc460ex_desc_t *xor_last_linked = NULL; +static ppc460ex_desc_t *xor_last_submit = NULL; + +/* This array is used in data-check operations for storing a pattern */ +static char ppc460ex_qword[16]; + +/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up + * the block size in transactions, then we do not allow to activate more than + * only one RXOR transactions simultaneously. So use this var to store + * the information about is RXOR currently active (PPC460EX_RXOR_RUN bit is + * set) or not (PPC460EX_RXOR_RUN is clear). + */ +static unsigned long ppc460ex_rxor_state; + +/* /proc interface is used here to enable the h/w RAID-6 capabilities + */ +static struct proc_dir_entry *ppc460ex_proot; +static struct proc_dir_entry *ppc460ex_pqroot; + +/* These are used in enable & check routines + */ +static u32 ppc460ex_r6_enabled; +static u32 ppc460ex_r5_enabled; +static ppc460ex_ch_t *ppc460ex_r6_tchan; +static ppc460ex_ch_t *ppc460ex_r5_tchan; +static struct completion ppc460ex_r6_test_comp; +static struct completion ppc460ex_r5_test_comp; + +static int ppc460ex_adma_dma2rxor_prep_src (ppc460ex_desc_t *desc, + ppc460ex_rxor_cursor_t *cursor, int index, + int src_cnt, u32 addr); +static void ppc460ex_adma_dma2rxor_set_src (ppc460ex_desc_t *desc, + int index, dma_addr_t addr); +static void ppc460ex_adma_dma2rxor_set_mult (ppc460ex_desc_t *desc, + int index, u8 mult); +#if 1 +static inline void pr_dma(int x, char *str) +{ + if(mfdcr(0x60)) { + printk("<%s> Line:%d\n",str,x); + } +} +#else +static inline void pr_dma(int x, char *str) +{ +} +#endif +phys_addr_t fixup_bigphys_addr(phys_addr_t addr, phys_addr_t size) +{ + phys_addr_t page_4gb = 0; + + return (page_4gb | addr); +} +/*********************************************************************** + * HW specific initialization + * ********************************************************************/ +static u64 ppc460ex_adma_dmamask = DMA_32BIT_MASK; + +/* DMA and XOR platform devices' resources */ + +static struct resource ppc460ex_dma_1_resources[] = { + { + .flags = IORESOURCE_MEM, + }, + { + .start = DMA1_CS_FIFO_NEED_SERVICE_IRQ, + .end = DMA1_CS_FIFO_NEED_SERVICE_IRQ, + .flags = IORESOURCE_IRQ + }, + { + .start = DMA_ERROR_IRQ, + .end = DMA_ERROR_IRQ, + .flags = IORESOURCE_IRQ + } +}; + + +/* DMA and XOR platform devices' data */ + +/* DMA0,1 engines use FIFO to maintain CDBs, so we + * should allocate the pool accordingly to size of this + * FIFO. Thus, the pool size depends on the FIFO depth: + * how much CDBs pointers FIFO may contaun then so much + * CDBs we should provide in pool. + * That is + * CDB size = 32B; + * CDBs number = (DMA0_FIFO_SIZE >> 3); + * Pool size = CDBs number * CDB size = + * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. + * + * As far as the XOR engine is concerned, it does not + * use FIFOs but uses linked list. So there is no dependency + * between pool size to allocate and the engine configuration. + */ + +static struct ppc460ex_adma_platform_data ppc460ex_dma_1_data = { + .hw_id = PPC460EX_DMA1_ID, + .pool_size = DMA1_FIFO_SIZE << 2, +}; + +/* DMA and XOR platform devices definitions */ +#if 1 +static struct platform_device ppc460ex_dma_1_channel = { + .name = "PPC460EX-ADMA", + .id = PPC460EX_DMA1_ID, + .num_resources = ARRAY_SIZE(ppc460ex_dma_1_resources), + .resource = ppc460ex_dma_1_resources, + .dev = { + .dma_mask = &ppc460ex_adma_dmamask, + .coherent_dma_mask = DMA_64BIT_MASK, + .platform_data = (void *) &ppc460ex_dma_1_data, + }, +}; +#endif + +/* + * Init DMA0/1 and XOR engines; allocate memory for DMAx FIFOs; set platform_device + * memory resources addresses + */ +static void ppc460ex_configure_raid_devices(void) +{ + void *fifo_buf; + volatile i2o_regs_t *i2o_reg; + volatile dma_regs_t *dma_reg1; + /* + * volatile dma_regs_t *dma_reg0, *dma_reg1; + volatile xor_regs_t *xor_reg; + */ + u32 mask; + + /* + * Map registers and allocate fifo buffer + */ + if (!(i2o_reg = ioremap(I2O_MMAP_BASE, I2O_MMAP_SIZE))) { + printk(KERN_ERR "I2O registers mapping failed.\n"); + return; + } + if (!(dma_reg1 = ioremap(DMA1_MMAP_BASE, DMA_MMAP_SIZE))) { + printk(KERN_ERR "DMA1 registers mapping failed.\n"); + goto err1; + } + + /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share + * the base address of FIFO memory space. + * Actually we need twice more physical memory than programmed in the + * <fsiz> register (because there are two FIFOs foreach DMA: CP and CS) + */ + fifo_buf = kmalloc(( DMA1_FIFO_SIZE)<<1, GFP_KERNEL); + if (!fifo_buf) { + printk(KERN_ERR "DMA FIFO buffer allocating failed.\n"); + goto err2; + } + + /* + * Configure h/w + */ + /* Reset I2O/DMA */ + SDR_WRITE(SDR0_SRST0, SDR0_SRST_I2ODMA); + SDR_WRITE(SDR0_SRST0, 0); + + + /* Setup the base address of mmaped registers */ + mtdcr(DCRN_I2O0_IBAH, (u32)(I2O_MMAP_BASE >> 32)); + mtdcr(DCRN_I2O0_IBAL, (u32)(I2O_MMAP_BASE) | I2O_REG_ENABLE); + + /* SetUp FIFO memory space base address */ + out_le32(&i2o_reg->ifbah, 0); + out_le32(&i2o_reg->ifbal, ((u32)__pa(fifo_buf))); + + /* set zero FIFO size for I2O, so the whole fifo_buf is used by DMAs. + * DMA0_FIFO_SIZE is defined in bytes, <fsiz> - in number of CDB pointers (8byte). + * DMA FIFO Length = CSlength + CPlength, where + * CSlength = CPlength = (fsiz + 1) * 8. + */ + out_le32(&i2o_reg->ifsiz, 0); + out_le32(&dma_reg1->fsiz, DMA_FIFO_ENABLE | ((DMA1_FIFO_SIZE>>3) - 2)); + /* Configure DMA engine */ + out_le32(&dma_reg1->cfg, DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN); + + /* Clear Status */ + out_le32(&dma_reg1->dsts, ~0); + + /* + * Prepare WXOR/RXOR (finally it is being enabled via /proc interface of + * the ppc460ex ADMA driver) + */ + /* Set HB alias */ + mtdcr(DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); + + /* Set: + * - LL transaction passing limit to 1; + * - Memory controller cycle limit to 1; + * - Galois Polynomial to 0x14d (default) + */ + mtdcr(DCRN_MQ0_CFBHL, 0x88a68000 | (1 << MQ0_CFBHL_TPLM) | + (1 << MQ0_CFBHL_HBCL) | + (PPC460EX_DEFAULT_POLY << MQ0_CFBHL_POLY)); + + /* Unmask 'CS FIFO Attention' interrupts and + * enable generating interrupts on errors + */ + mask = in_le32(&i2o_reg->iopim) & ~( + I2O_IOPIM_P0SNE | I2O_IOPIM_P1SNE | + I2O_IOPIM_P0EM | I2O_IOPIM_P1EM); + out_le32(&i2o_reg->iopim, mask); + + /* enable XOR engine interrupts */ + + /* + * Unmap I2O registers + */ + iounmap(i2o_reg); + printk("<%s> line %d\n", __FUNCTION__, __LINE__); + + /* Configure MQ as follows: + * MQ: 0x80001C80. This means + * - AddrAck First Request, + * - Read Passing Limit = 1, + * - Read Passing Enable, + * - Read Flow Through Enable, + * - MCIF Cycle Limit = 1. + */ +#if 1 + mdelay(1000); + mask = (1 << MQ_CF1_AAFR) | ((1 & MQ_CF1_RPLM_MSK) << MQ_CF1_RPLM) | + (1 << MQ_CF1_RPEN) | (1 << MQ_CF1_RFTE) | + ((1 & MQ_CF1_WRCL_MSK) << MQ_CF1_WRCL); + mtdcr(DCRN_MQ0_CF1H, mask); + mtdcr(DCRN_MQ0_CF1L, mask); +#endif + printk("<%s> line %d\n", __FUNCTION__, __LINE__); + + /* Configure PLB as follows: + * PLB: 0xDF000000. This means + * - Priority level 00 fair priority, + * - Priority level 01 fair priority, + * - Priority level 11 fair priority, + * - High Bus Utilization enabled, + * - 4 Deep read pipe, + * - 2 Deep write pipe. + */ + mask = (1 << PLB_ACR_PPM0) | (1 << PLB_ACR_PPM1) | (1 << PLB_ACR_PPM3) | + (1 << PLB_ACR_HBU) | ((3 & PLB_ACR_RDP_MSK) << PLB_ACR_RDP) | + (1 << PLB_ACR_WRP); + mtdcr(DCRN_PLB0_ACR, mask); + mtdcr(DCRN_PLB1_ACR, mask); + printk("<%s> line %d\n", __FUNCTION__, __LINE__); + + /* + * Set resource addresses + */ + + ppc460ex_dma_1_channel.resource[0].start = (resource_size_t)(dma_reg1); + ppc460ex_dma_1_channel.resource[0].end = + ppc460ex_dma_1_channel.resource[0].start+DMA_MMAP_SIZE; + printk( " ppc460ex_dma_1_channel.resource[0].start=0x%lx \n", + ppc460ex_dma_1_channel.resource[0].start); + printk("<%s> line %d dma_reg1=0x%lx \n", __FUNCTION__, __LINE__,dma_reg1); + + + printk("<%s> line %d\n", __FUNCTION__, __LINE__); + return; +err2: + iounmap(dma_reg1); +err1: + iounmap(i2o_reg); + return; +} +#if 1 +static struct platform_device *ppc460ex_devs[] __initdata = { +/* &ppc460ex_dma_0_channel, */ + &ppc460ex_dma_1_channel, + /*&ppc460ex_xor_channel, */ +}; +#endif + +/****************************************************************************** + * Command (Descriptor) Blocks low-level routines + ******************************************************************************/ +/** + * ppc460ex_desc_init_interrupt - initialize the descriptor for INTERRUPT + * pseudo operation + */ +static inline void ppc460ex_desc_init_interrupt (ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan) +{ + xor_cb_t *p; + + switch (chan->device->id) { + case PPC460EX_XOR_ID: + p = desc->hw_desc; + memset (desc->hw_desc, 0, sizeof(xor_cb_t)); + /* NOP with Command Block Complete Enable */ + p->cbc = XOR_CBCR_CBCE_BIT; + break; + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + memset (desc->hw_desc, 0, sizeof(dma_cdb_t)); + /* NOP with interrupt */ + set_bit(PPC460EX_DESC_INT, &desc->flags); + break; + default: + printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, + __FUNCTION__); + break; + } +} + +/** + * ppc460ex_desc_init_null_xor - initialize the descriptor for NULL XOR + * pseudo operation + */ +static inline void ppc460ex_desc_init_null_xor(ppc460ex_desc_t *desc) +{ + memset (desc->hw_desc, 0, sizeof(xor_cb_t)); + desc->hw_next = NULL; + desc->src_cnt = 0; + desc->dst_cnt = 1; +} + +/** + * ppc460ex_desc_init_pqxor_xor - initialize the descriptor for PQ_XOR + * operation in DMA2 controller + */ +static inline void ppc460ex_desc_init_dma2rxor(ppc460ex_desc_t *desc, + int dst_cnt, int src_cnt, unsigned long flags) +{ + xor_cb_t *hw_desc = desc->hw_desc; + + memset (desc->hw_desc, 0, sizeof(xor_cb_t)); + desc->hw_next = NULL; + desc->src_cnt = src_cnt; + desc->dst_cnt = dst_cnt; + memset (desc->reverse_flags, 0, sizeof (desc->reverse_flags)); + desc->descs_per_op = 0; + + hw_desc->cbc = XOR_CBCR_TGT_BIT; + if (flags & DMA_PREP_INTERRUPT) + /* Enable interrupt on complete */ + hw_desc->cbc |= XOR_CBCR_CBCE_BIT; +} + +/** + * ppc460ex_desc_init_pq - initialize the descriptor for PQ_XOR operation + */ +static inline void ppc460ex_desc_init_pq(ppc460ex_desc_t *desc, + int dst_cnt, int src_cnt, unsigned long flags, + unsigned long op) +{ + dma_cdb_t *hw_desc; + ppc460ex_desc_t *iter; + u8 dopc; + + + /* Common initialization of a PQ descriptors chain */ + + set_bits(op, &desc->flags); + desc->src_cnt = src_cnt; + desc->dst_cnt = dst_cnt; + + dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? + DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; + + list_for_each_entry(iter, &desc->group_list, chain_node) { + hw_desc = iter->hw_desc; + memset (iter->hw_desc, 0, sizeof(dma_cdb_t)); + + if (likely(!list_is_last(&iter->chain_node, + &desc->group_list))) { + /* set 'next' pointer */ + iter->hw_next = list_entry(iter->chain_node.next, + ppc460ex_desc_t, chain_node); + clear_bit(PPC460EX_DESC_INT, &iter->flags); + } else { + /* this is the last descriptor. + * this slot will be pasted from ADMA level + * each time it wants to configure parameters + * of the transaction (src, dst, ...) + */ + iter->hw_next = NULL; + if (flags & DMA_PREP_INTERRUPT) + set_bit(PPC460EX_DESC_INT, &iter->flags); + else + clear_bit(PPC460EX_DESC_INT, &iter->flags); + } + } + + /* Set OPS depending on WXOR/RXOR type of operation */ + if (!test_bit(PPC460EX_DESC_RXOR, &desc->flags)) { + /* This is a WXOR only chain: + * - first descriptors are for zeroing destinations + * if PPC460EX_ZERO_P/Q set; + * - descriptors remained are for GF-XOR operations. + */ + iter = list_first_entry(&desc->group_list, + ppc460ex_desc_t, chain_node); + + if (test_bit(PPC460EX_ZERO_P, &desc->flags)) { + hw_desc = iter->hw_desc; + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + iter = list_first_entry(&iter->chain_node, + ppc460ex_desc_t, chain_node); + } + + if (test_bit(PPC460EX_ZERO_Q, &desc->flags)) { + hw_desc = iter->hw_desc; + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + iter = list_first_entry(&iter->chain_node, + ppc460ex_desc_t, chain_node); + } + + list_for_each_entry_from(iter, &desc->group_list, chain_node) { + hw_desc = iter->hw_desc; + hw_desc->opc = dopc; + } + } else { + /* This is either RXOR-only or mixed RXOR/WXOR */ + + /* The first 1 or 2 slots in chain are always RXOR, + * if need to calculate P & Q, then there are two + * RXOR slots; if only P or only Q, then there is one + */ + iter = list_first_entry(&desc->group_list, + ppc460ex_desc_t, chain_node); + hw_desc = iter->hw_desc; + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + + if (desc->dst_cnt == DMA_DEST_MAX_NUM) { + iter = list_first_entry(&iter->chain_node, + ppc460ex_desc_t, chain_node); + hw_desc = iter->hw_desc; + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + } + + /* The remain descs (if any) are WXORs */ + if (test_bit(PPC460EX_DESC_WXOR, &desc->flags)) { + iter = list_first_entry(&iter->chain_node, + ppc460ex_desc_t, chain_node); + list_for_each_entry_from(iter, &desc->group_list, + chain_node) { + hw_desc = iter->hw_desc; + hw_desc->opc = dopc; + } + } + } +} +void ppc460ex_desc_init_xor(ppc460ex_desc_t *desc, + int dst_cnt, int src_cnt, unsigned long flags, + unsigned long op) +{ + dma_cdb_t *hw_desc; + ppc460ex_desc_t *iter; + u8 dopc; + + + /* Common initialization of a PQ descriptors chain */ + + set_bits(op, &desc->flags); + desc->src_cnt = src_cnt; + desc->dst_cnt = dst_cnt; + + dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? + DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; + + list_for_each_entry(iter, &desc->group_list, chain_node) { + hw_desc = iter->hw_desc; + memset (iter->hw_desc, 0, sizeof(dma_cdb_t)); + + if (likely(!list_is_last(&iter->chain_node, + &desc->group_list))) { + /* set 'next' pointer */ + iter->hw_next = list_entry(iter->chain_node.next, + ppc460ex_desc_t, chain_node); + clear_bit(PPC460EX_DESC_INT, &iter->flags); + } else { + /* this is the last descriptor. + * this slot will be pasted from ADMA level + * each time it wants to configure parameters + * of the transaction (src, dst, ...) + */ + iter->hw_next = NULL; + if (flags & DMA_PREP_INTERRUPT) + set_bit(PPC460EX_DESC_INT, &iter->flags); + else + clear_bit(PPC460EX_DESC_INT, &iter->flags); + } + } + + /* Set OPS depending on WXOR/RXOR type of operation */ + if (!test_bit(PPC460EX_DESC_RXOR, &desc->flags)) { + /* This is a WXOR only chain: + * - first descriptors are for zeroing destinations + * if PPC460EX_ZERO_P/Q set; + * - descriptors remained are for GF-XOR operations. + */ + iter = list_first_entry(&desc->group_list, + ppc460ex_desc_t, chain_node); + + if (test_bit(PPC460EX_ZERO_P, &desc->flags)) { + hw_desc = iter->hw_desc; + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + iter = list_first_entry(&iter->chain_node, + ppc460ex_desc_t, chain_node); + } + + + list_for_each_entry_from(iter, &desc->group_list, chain_node) { + hw_desc = iter->hw_desc; + hw_desc->opc = dopc; + } + } else { + /* This is either RXOR-only or mixed RXOR/WXOR */ + + /* The first 1 or 2 slots in chain are always RXOR, + * if need to calculate P & Q, then there are two + * RXOR slots; if only P or only Q, then there is one + */ + iter = list_first_entry(&desc->group_list, + ppc460ex_desc_t, chain_node); + hw_desc = iter->hw_desc; + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + + if (desc->dst_cnt == DMA_DEST_MAX_NUM) { + iter = list_first_entry(&iter->chain_node, + ppc460ex_desc_t, chain_node); + hw_desc = iter->hw_desc; + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + } + + /* The remain descs (if any) are WXORs */ + if (test_bit(PPC460EX_DESC_WXOR, &desc->flags)) { + iter = list_first_entry(&iter->chain_node, + ppc460ex_desc_t, chain_node); + list_for_each_entry_from(iter, &desc->group_list, + chain_node) { + hw_desc = iter->hw_desc; + hw_desc->opc = dopc; + } + } + } +} +/** + * ppc460ex_desc_init_dma01_xor - initialize the descriptor for P_XOR operation + */ +static inline void ppc460ex_desc_init_dma01_xor(ppc460ex_desc_t *desc, + int dst_cnt, int src_cnt, unsigned long flags, + unsigned long op) +{ + dma_cdb_t *hw_desc; + ppc460ex_desc_t *iter; + + /* Common initialization of a XOR descriptors chain */ + + set_bits(op, &desc->flags); + desc->src_cnt = src_cnt; + desc->dst_cnt = dst_cnt; + + list_for_each_entry(iter, &desc->group_list, chain_node) { + hw_desc = iter->hw_desc; + memset (iter->hw_desc, 0, sizeof(dma_cdb_t)); + + if (likely(!list_is_last(&iter->chain_node, + &desc->group_list))) { + /* set 'next' pointer */ + iter->hw_next = list_entry(iter->chain_node.next, + ppc460ex_desc_t, chain_node); + clear_bit(PPC460EX_DESC_INT, &iter->flags); + } else { + /* this is the last descriptor. + * this slot will be pasted from ADMA level + * each time it wants to configure parameters + * of the transaction (src, dst, ...) + */ + iter->hw_next = NULL; + if (flags & DMA_PREP_INTERRUPT) + set_bit(PPC460EX_DESC_INT, &iter->flags); + else + clear_bit(PPC460EX_DESC_INT, &iter->flags); + } + } + + /* Set OPS depending on WXOR/RXOR type of operation */ + if (!test_bit(PPC460EX_DESC_RXOR, &desc->flags)) { + /* This is a WXOR only chain: + * - first <dst_cnt> descriptors are for zeroing destinations + * if PPC460EX_ZERO_P is set; + * - descriptors remained are for GF-XOR operations. + */ + iter = list_first_entry(&desc->group_list, + ppc460ex_desc_t, chain_node); + + if (dst_cnt && test_bit(PPC460EX_ZERO_P, + &desc->flags)) { + /* MV_SG1_SG2 to zero P or Q if this is + * just PQ_XOR operation and MV_SG1_SG2 + * if only Q has to be calculated + */ + hw_desc = iter->hw_desc; + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + iter = list_first_entry(&iter->chain_node, + ppc460ex_desc_t, chain_node); + } + list_for_each_entry(iter, &desc->group_list, chain_node) { + hw_desc = iter->hw_desc; + if (desc->dst_cnt == DMA_DEST_MAX_NUM) + hw_desc->opc = DMA_CDB_OPC_MULTICAST; + else + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + } + } else { + /* This is either RXOR-only or mixed RXOR/WXOR + * The first slot in chain is always RXOR, + * the slots remained (if there are) are WXOR + */ + list_for_each_entry(iter, &desc->group_list, chain_node) { + hw_desc = iter->hw_desc; + /* No DMA_CDB_OPC_MULTICAST option for RXOR */ + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; + } + } +} + +/** + * ppc460ex_desc_init_pqzero_sum - initialize the descriptor for PQ_VAL + * operation + */ +static inline void ppc460ex_desc_init_pqzero_sum(ppc460ex_desc_t *desc, + int dst_cnt, int src_cnt) +{ + dma_cdb_t *hw_desc; + ppc460ex_desc_t *iter; + int i = 0; + + /* initialize each descriptor in chain */ + list_for_each_entry(iter, &desc->group_list, chain_node) { + hw_desc = iter->hw_desc; + memset (iter->hw_desc, 0, sizeof(dma_cdb_t)); + + /* This is a PQ_VAL operation: + * - first <dst_cnt> descriptors are for GF-XOR operations; + * - <dst_cnt> descriptors remained are for checking the result. + */ + if (i++ < src_cnt) + /* MV_SG1_SG2 if only Q is being verified + * MULTICAST if both P and Q are being verified + */ + hw_desc->opc = (dst_cnt == DMA_DEST_MAX_NUM) ? + DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; + else + /* DMA_CDB_OPC_DCHECK128 operation */ + hw_desc->opc = DMA_CDB_OPC_DCHECK128; + + if (likely(!list_is_last(&iter->chain_node, + &desc->group_list))) { + /* set 'next' pointer */ + iter->hw_next = list_entry(iter->chain_node.next, + ppc460ex_desc_t, chain_node); + } else { + /* this is the last descriptor. + * this slot will be pasted from ADMA level + * each time it wants to configure parameters + * of the transaction (src, dst, ...) + */ + iter->hw_next = NULL; + /* always enable interrupt generating since we get + * the status of pqzero from the handler + */ + set_bit(PPC460EX_DESC_INT, &iter->flags); + } + } + desc->src_cnt = src_cnt; + desc->dst_cnt = dst_cnt; +} + +/** + * ppc460ex_desc_init_memcpy - initialize the descriptor for MEMCPY operation + */ +static inline void ppc460ex_desc_init_memcpy(ppc460ex_desc_t *desc, + unsigned long flags) +{ + dma_cdb_t *hw_desc = desc->hw_desc; + + memset (desc->hw_desc, 0, sizeof(dma_cdb_t)); + desc->hw_next = NULL; + desc->src_cnt = 1; + desc->dst_cnt = 1; + + if (flags & DMA_PREP_INTERRUPT) + set_bit(PPC460EX_DESC_INT, &desc->flags); + else + clear_bit(PPC460EX_DESC_INT, &desc->flags); + + hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; +} + +/** + * ppc460ex_desc_init_memset - initialize the descriptor for MEMSET operation + */ +static inline void ppc460ex_desc_init_memset(ppc460ex_desc_t *desc, int value, + unsigned long flags) +{ + dma_cdb_t *hw_desc = desc->hw_desc; + + memset (desc->hw_desc, 0, sizeof(dma_cdb_t)); + desc->hw_next = NULL; + desc->src_cnt = 1; + desc->dst_cnt = 1; + + if (flags & DMA_PREP_INTERRUPT) + set_bit(PPC460EX_DESC_INT, &desc->flags); + else + clear_bit(PPC460EX_DESC_INT, &desc->flags); + + hw_desc->sg1u = hw_desc->sg1l = cpu_to_le32((u32)value); + hw_desc->sg3u = hw_desc->sg3l = cpu_to_le32((u32)value); + hw_desc->opc = DMA_CDB_OPC_DFILL128; +} + +/** + * ppc460ex_desc_set_src_addr - set source address into the descriptor + */ +static inline void ppc460ex_desc_set_src_addr( ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan, int src_idx, + dma_addr_t addrh, dma_addr_t addrl) +{ + dma_cdb_t *dma_hw_desc; + phys_addr_t addr64, tmplow, tmphi; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + if (!addrh) { + addr64 = fixup_bigphys_addr(addrl, sizeof(phys_addr_t)); + tmphi = (addr64 >> 32); + tmplow = (addr64 & 0xFFFFFFFF); + } else { + tmphi = addrh; + tmplow = addrl; + } + dma_hw_desc = desc->hw_desc; + dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); + dma_hw_desc->sg1u = cpu_to_le32((u32)tmphi); + break; + } +} + +/** + * ppc460ex_desc_set_src_mult - set source address mult into the descriptor + */ +static inline void ppc460ex_desc_set_src_mult( ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan, u32 mult_index, int sg_index, + unsigned char mult_value) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + u32 *psgu; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + dma_hw_desc = desc->hw_desc; + + switch(sg_index){ + /* for RXOR operations set multiplier + * into source cued address + */ + case DMA_CDB_SG_SRC: + psgu = &dma_hw_desc->sg1u; + break; + /* for WXOR operations set multiplier + * into destination cued address(es) + */ + case DMA_CDB_SG_DST1: + psgu = &dma_hw_desc->sg2u; + break; + case DMA_CDB_SG_DST2: + psgu = &dma_hw_desc->sg3u; + break; + default: + BUG(); + } + + *psgu |= cpu_to_le32(mult_value << mult_index); + if(mfdcr(0x60) == 0xfee8) { + printk("Line--%d mult_value = 0x%x mult_index=0x%x *psgu=0x%x\n",__LINE__, mult_value,mult_index,*psgu); + } + *psgu |= cpu_to_le32( 1 << mult_index); + if(mfdcr(0x60) == 0xfee8) { + printk("Line--%d mult_value = 0x%x mult_index=0x%x *psgu=0x%x\n",__LINE__, mult_value,mult_index,*psgu); + } + break; + case PPC460EX_XOR_ID: + xor_hw_desc = desc->hw_desc; + break; + default: + BUG(); + } +} + +/** + * ppc460ex_desc_set_dest_addr - set destination address into the descriptor + */ +static inline void ppc460ex_desc_set_dest_addr(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan, + dma_addr_t addrh, dma_addr_t addrl, + u32 dst_idx) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + phys_addr_t addr64, tmphi, tmplow; + u32 *psgu, *psgl; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + if (!addrh) { + addr64 = fixup_bigphys_addr(addrl, sizeof(phys_addr_t)); + tmphi = (addr64 >> 32); + tmplow = (addr64 & 0xFFFFFFFF); + } else { + tmphi = addrh; + tmplow = addrl; + } + dma_hw_desc = desc->hw_desc; + + psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; + psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; + + *psgl = cpu_to_le32((u32)tmplow); + *psgu |= cpu_to_le32((u32)tmphi); + break; + case PPC460EX_XOR_ID: + xor_hw_desc = desc->hw_desc; + xor_hw_desc->cbtal = addrl; + xor_hw_desc->cbtah = 0; + break; + } +} + +/** + * ppc460ex_desc_set_byte_count - set number of data bytes involved + * into the operation + */ +static inline void ppc460ex_desc_set_byte_count(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan, u32 byte_count) +{ + dma_cdb_t *dma_hw_desc; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + dma_hw_desc = desc->hw_desc; + dma_hw_desc->cnt = cpu_to_le32(byte_count); + break; + } +} + +/** + * ppc460ex_desc_set_rxor_block_size - set RXOR block size + */ +static inline void ppc460ex_desc_set_rxor_block_size(u32 byte_count) +{ + /* assume that byte_count is aligned on the 512-boundary; + * thus write it directly to the register (bits 23:31 are + * reserved there). + */ + mtdcr(DCRN_MQ0_CF2H, byte_count); +} + +/** + * ppc460ex_desc_set_dcheck - set CHECK pattern + */ +static inline void ppc460ex_desc_set_dcheck(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan, u8 *qword) +{ + dma_cdb_t *dma_hw_desc; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + dma_hw_desc = desc->hw_desc; + out_le32(&dma_hw_desc->sg3l, qword[0]); + out_le32(&dma_hw_desc->sg3u, qword[4]); + out_le32(&dma_hw_desc->sg2l, qword[8]); + out_le32(&dma_hw_desc->sg2u, qword[12]); + break; + default: + BUG(); + } +} + +/** + * ppc460ex_xor_set_link - set link address in xor CB + */ +static inline void ppc460ex_xor_set_link (ppc460ex_desc_t *prev_desc, + ppc460ex_desc_t *next_desc) +{ + xor_cb_t *xor_hw_desc = prev_desc->hw_desc; + + if (unlikely(!next_desc || !(next_desc->phys))) { + printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%x\n", + __FUNCTION__, next_desc, + next_desc ? next_desc->phys : 0); + BUG(); + } + + xor_hw_desc->cbs = 0; + xor_hw_desc->cblal = next_desc->phys; + xor_hw_desc->cblah = 0; + xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; +} + +/** + * ppc460ex_desc_set_link - set the address of descriptor following this + * descriptor in chain + */ +static inline void ppc460ex_desc_set_link(ppc460ex_ch_t *chan, + ppc460ex_desc_t *prev_desc, ppc460ex_desc_t *next_desc) +{ + unsigned long flags; + ppc460ex_desc_t *tail = next_desc; + + if (unlikely(!prev_desc || !next_desc || + (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { + /* If previous next is overwritten something is wrong. + * though we may refetch from append to initiate list + * processing; in this case - it's ok. + */ + printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " + "prev->hw_next=0x%p\n", __FUNCTION__, prev_desc, + next_desc, prev_desc ? prev_desc->hw_next : 0); + BUG(); + } + + local_irq_save(flags); + + /* do s/w chaining both for DMA and XOR descriptors */ + prev_desc->hw_next = next_desc; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + break; + case PPC460EX_XOR_ID: + /* bind descriptor to the chain */ + while (tail->hw_next) + tail = tail->hw_next; + xor_last_linked = tail; + + if (prev_desc == xor_last_submit) + /* do not link to the last submitted CB */ + break; + ppc460ex_xor_set_link (prev_desc, next_desc); + break; + } + + local_irq_restore(flags); +} + +/** + * ppc460ex_desc_get_src_addr - extract the source address from the descriptor + */ +static inline u32 ppc460ex_desc_get_src_addr(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan, int src_idx) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + dma_hw_desc = desc->hw_desc; + /* May have 0, 1, 2, or 3 sources */ + switch (dma_hw_desc->opc) { + case DMA_CDB_OPC_NO_OP: + case DMA_CDB_OPC_DFILL128: + return 0; + case DMA_CDB_OPC_DCHECK128: + if (unlikely(src_idx)) { + printk(KERN_ERR "%s: try to get %d source for" + " DCHECK128\n", __FUNCTION__, src_idx); + BUG(); + } + return le32_to_cpu(dma_hw_desc->sg1l); + case DMA_CDB_OPC_MULTICAST: + case DMA_CDB_OPC_MV_SG1_SG2: + if (unlikely(src_idx > 2)) { + printk(KERN_ERR "%s: try to get %d source from" + " DMA descr\n", __FUNCTION__, src_idx); + BUG(); + } + if (src_idx) { + if (le32_to_cpu(dma_hw_desc->sg1u) & + DMA_CUED_XOR_WIN_MSK) { + u8 region; + + if (src_idx == 1) + return le32_to_cpu( + dma_hw_desc->sg1l) + + desc->unmap_len; + + region = (le32_to_cpu( + dma_hw_desc->sg1u)) >> + DMA_CUED_REGION_OFF; + + region &= DMA_CUED_REGION_MSK; + switch (region) { + case DMA_RXOR123: + return le32_to_cpu( + dma_hw_desc->sg1l) + + (desc->unmap_len << 1); + case DMA_RXOR124: + return le32_to_cpu( + dma_hw_desc->sg1l) + + (desc->unmap_len * 3); + case DMA_RXOR125: + return le32_to_cpu( + dma_hw_desc->sg1l) + + (desc->unmap_len << 2); + default: + printk (KERN_ERR + "%s: try to" + " get src3 for region %02x" + "PPC460EX_DESC_RXOR12?\n", + __FUNCTION__, region); + BUG(); + } + } else { + printk(KERN_ERR + "%s: try to get %d" + " source for non-cued descr\n", + __FUNCTION__, src_idx); + BUG(); + } + } + return le32_to_cpu(dma_hw_desc->sg1l); + default: + printk(KERN_ERR "%s: unknown OPC 0x%02x\n", + __FUNCTION__, dma_hw_desc->opc); + BUG(); + } + return le32_to_cpu(dma_hw_desc->sg1l); + case PPC460EX_XOR_ID: + /* May have up to 16 sources */ + xor_hw_desc = desc->hw_desc; + return xor_hw_desc->ops[src_idx].l; + } + return 0; +} + +/** + * ppc460ex_desc_get_dest_addr - extract the destination address from the + * descriptor + */ +static inline u32 ppc460ex_desc_get_dest_addr(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan, int idx) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + dma_hw_desc = desc->hw_desc; + + if (likely(!idx)) + return le32_to_cpu(dma_hw_desc->sg2l); + return le32_to_cpu(dma_hw_desc->sg3l); + case PPC460EX_XOR_ID: + xor_hw_desc = desc->hw_desc; + return xor_hw_desc->cbtal; + } + return 0; +} + +/** + * ppc460ex_desc_get_byte_count - extract the byte count from the descriptor + */ +static inline u32 ppc460ex_desc_get_byte_count(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + dma_hw_desc = desc->hw_desc; + return le32_to_cpu(dma_hw_desc->cnt); + case PPC460EX_XOR_ID: + xor_hw_desc = desc->hw_desc; + return xor_hw_desc->cbbc; + default: + BUG(); + } + return 0; +} + +/** + * ppc460ex_desc_get_src_num - extract the number of source addresses from + * the descriptor + */ +static inline u32 ppc460ex_desc_get_src_num(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan) +{ + dma_cdb_t *dma_hw_desc; + xor_cb_t *xor_hw_desc; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + dma_hw_desc = desc->hw_desc; + + switch (dma_hw_desc->opc) { + case DMA_CDB_OPC_NO_OP: + case DMA_CDB_OPC_DFILL128: + return 0; + case DMA_CDB_OPC_DCHECK128: + return 1; + case DMA_CDB_OPC_MV_SG1_SG2: + case DMA_CDB_OPC_MULTICAST: + /* + * Only for RXOR operations we have more than + * one source + */ + if (le32_to_cpu(dma_hw_desc->sg1u) & + DMA_CUED_XOR_WIN_MSK) { + /* RXOR op, there are 2 or 3 sources */ + if (((le32_to_cpu(dma_hw_desc->sg1u) >> + DMA_CUED_REGION_OFF) & + DMA_CUED_REGION_MSK) == DMA_RXOR12) { + /* RXOR 1-2 */ + return 2; + } else { + /* RXOR 1-2-3/1-2-4/1-2-5 */ + return 3; + } + } + return 1; + default: + printk(KERN_ERR "%s: unknown OPC 0x%02x\n", + __FUNCTION__, dma_hw_desc->opc); + BUG(); + } + case PPC460EX_XOR_ID: + /* up to 16 sources */ + xor_hw_desc = desc->hw_desc; + return (xor_hw_desc->cbc & XOR_CDCR_OAC_MSK); + default: + BUG(); + } + return 0; +} + +/** + * ppc460ex_desc_get_dst_num - get the number of destination addresses in + * this descriptor + */ +static inline u32 ppc460ex_desc_get_dst_num(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan) +{ + dma_cdb_t *dma_hw_desc; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + /* May be 1 or 2 destinations */ + dma_hw_desc = desc->hw_desc; + switch (dma_hw_desc->opc) { + case DMA_CDB_OPC_NO_OP: + case DMA_CDB_OPC_DCHECK128: + return 0; + case DMA_CDB_OPC_MV_SG1_SG2: + case DMA_CDB_OPC_DFILL128: + return 1; + case DMA_CDB_OPC_MULTICAST: + return 2; + default: + printk(KERN_ERR "%s: unknown OPC 0x%02x\n", + __FUNCTION__, dma_hw_desc->opc); + BUG(); + } + case PPC460EX_XOR_ID: + /* Always only 1 destination */ + return 1; + default: + BUG(); + } + return 0; +} + +/** + * ppc460ex_desc_get_link - get the address of the descriptor that + * follows this one + */ +static inline u32 ppc460ex_desc_get_link(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan) +{ + if (!desc->hw_next) + return 0; + + return desc->hw_next->phys; +} + +/** + * ppc460ex_desc_is_aligned - check alignment + */ +static inline int ppc460ex_desc_is_aligned(ppc460ex_desc_t *desc, + int num_slots) +{ + return (desc->idx & (num_slots - 1)) ? 0 : 1; +} + +/** + * ppc460ex_chan_xor_slot_count - get the number of slots necessary for + * XOR operation + */ +static inline int ppc460ex_chan_xor_slot_count(size_t len, int src_cnt, + int *slots_per_op) +{ + int slot_cnt; + + /* each XOR descriptor provides up to 16 source operands */ + slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; + + if (likely(len <= PPC460EX_ADMA_XOR_MAX_BYTE_COUNT)) + return slot_cnt; + + printk(KERN_ERR "%s: len %d > max %d !!\n", + __FUNCTION__, len, PPC460EX_ADMA_XOR_MAX_BYTE_COUNT); + BUG(); + return slot_cnt; +} + +/** + */ +static inline int ppc460ex_chan_pqxor_slot_count (dma_addr_t *srcs, + int src_cnt, size_t len) +{ + int order = 0; + int state = 0; + int addr_count = 0; + int i; + for (i=1; i<src_cnt; i++) { + char *cur_addr = (char *)srcs[i]; + char *old_addr = (char *)srcs[i-1]; + switch (state) { + case 0: + if (cur_addr == old_addr + len) { + /* direct RXOR */ + order = 1; + state = 1; + if (i == src_cnt-1) { + addr_count++; + } + } else if (old_addr == cur_addr + len) { + /* reverse RXOR */ + order = -1; + state = 1; + if (i == src_cnt-1) { + addr_count++; + } + } else { + state = 3; + } + break; + case 1: + if (i == src_cnt-2 || (order == -1 + && cur_addr != old_addr - len)) { + order = 0; + state = 0; + addr_count++; + } else if (cur_addr == old_addr + len*order) { + state = 2; + if (i == src_cnt-1) { + addr_count++; + } + } else if (cur_addr == old_addr + 2*len) { + state = 2; + if (i == src_cnt-1) { + addr_count++; + } + } else if (cur_addr == old_addr + 3*len) { + state = 2; + if (i == src_cnt-1) { + addr_count++; + } + } else { + order = 0; + state = 0; + addr_count++; + } + break; + case 2: + order = 0; + state = 0; + addr_count++; + break; + } + if (state == 3) break; + } + if (src_cnt <= 1 || (state != 1 && state != 2)) { + /* FIXME. return 0 here and check for this when called. */ + BUG (); + } + + return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; +} + + +/****************************************************************************** + * ADMA channel low-level routines + ******************************************************************************/ + +static inline u32 ppc460ex_chan_get_current_descriptor(ppc460ex_ch_t *chan); +static inline void ppc460ex_chan_append(ppc460ex_ch_t *chan); + +/** + * ppc460ex_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine + */ +static inline void ppc460ex_adma_device_clear_eot_status (ppc460ex_ch_t *chan) +{ + volatile dma_regs_t *dma_reg; + u8 *p = chan->device->dma_desc_pool_virt; + dma_cdb_t *cdb; + u32 rv, hv, i; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + /* read FIFO to ack */ + //dma_reg = (dma_regs_t *)chan->device->pdev->resource[0].start; + dma_reg = (dma_regs_t *)chan->device->res[0].start; + while (rv = in_le32(&dma_reg->csfpl)) { + + if ( chan->device->desc_memory == ADMA_DESC_MEM_OCM) + hv = in_le32(&dma_reg->csfph);/* clear the upper bits too */ + + i = rv & DMA_CDB_ADDR_MSK; + cdb = (dma_cdb_t *)&p[i - + (u32)chan->device->dma_desc_pool]; + + /* Clear opcode to ack. This is necessary for + * ZeroSum operations only + */ + cdb->opc = 0; + + if (test_bit(PPC460EX_RXOR_RUN, + &ppc460ex_rxor_state)) { + /* probably this is a completed RXOR op, + * get pointer to CDB using the fact that + * physical and virtual addresses of CDB + * in pools have the same offsets + */ + if (le32_to_cpu(cdb->sg1u) & + DMA_CUED_XOR_BASE) { + /* this is a RXOR */ + clear_bit(PPC460EX_RXOR_RUN, + &ppc460ex_rxor_state); + } + } + + if (rv & DMA_CDB_STATUS_MSK) { + /* ZeroSum check failed + */ + ppc460ex_desc_t *iter; + dma_addr_t phys = rv & ~DMA_CDB_MSK; + + /* + * Update the status of corresponding + * descriptor. + */ + list_for_each_entry(iter, &chan->chain, + chain_node) { + if (iter->phys == phys) + break; + } + /* + * if cannot find the corresponding + * slot it's a bug + */ + BUG_ON (&iter->chain_node == &chan->chain); + + if (iter->xor_check_result) + *iter->xor_check_result |= + rv & DMA_CDB_STATUS_MSK; + } + } + + rv = in_le32(&dma_reg->dsts); + if (rv) { + printk("DMA%d err status: 0x%x\n", chan->device->id, + rv); + /* write back to clear */ + out_le32(&dma_reg->dsts, rv); + } + break; + } +} + + +/** + * ppc460ex_chan_is_busy - get the channel status + */ +static inline int ppc460ex_chan_is_busy(ppc460ex_ch_t *chan) +{ + int busy = 0; + volatile xor_regs_t *xor_reg; + volatile dma_regs_t *dma_reg; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + dma_reg = (dma_regs_t *)chan->device->res[0].start; + /* if command FIFO's head and tail pointers are equal and + * status tail is the same as command, then channel is free + */ + if (dma_reg->cpfhp != dma_reg->cpftp || + dma_reg->cpftp != dma_reg->csftp) + busy = 1; + break; + case PPC460EX_XOR_ID: + /* use the gtcial status bit for the XORcore + */ + busy = (xor_reg->sr & XOR_SR_XCP_BIT) ? 1 : 0; + break; + } + + return busy; +} + +/** + * ppc460ex_chan_set_first_xor_descriptor - initi XORcore chain + */ +static inline void ppc460ex_chan_set_first_xor_descriptor(ppc460ex_ch_t *chan, + ppc460ex_desc_t *next_desc) +{ + volatile xor_regs_t *xor_reg; + + //xor_reg = (xor_regs_t *)chan->device->pdev->resource[0].start; + + if (xor_reg->sr & XOR_SR_XCP_BIT) + printk(KERN_INFO "%s: Warn: XORcore is running " + "when try to set the first CDB!\n", + __FUNCTION__); + + xor_last_submit = xor_last_linked = next_desc; + + xor_reg->crsr = XOR_CRSR_64BA_BIT; + + xor_reg->cblalr = next_desc->phys; + xor_reg->cblahr = 0; + xor_reg->cbcr |= XOR_CBCR_LNK_BIT; + + chan->hw_chain_inited = 1; +} + +/** + * ppc460ex_dma_put_desc - put DMA0,1 descriptor to FIFO + */ +static void ppc460ex_dma_put_desc(ppc460ex_ch_t *chan, + ppc460ex_desc_t *desc) +{ + u32 pcdb; + volatile dma_regs_t *dma_reg = + dma_reg = (dma_regs_t *)chan->device->res[0].start; + + pcdb = desc->phys; + if (!test_bit(PPC460EX_DESC_INT, &desc->flags)) + pcdb |= DMA_CDB_NO_INT; + if ( chan->device->desc_memory == ADMA_DESC_MEM_OCM) + pcdb |= DMA_CDB_64B_ADDR; /* 64 bit */ + chan_last_sub[chan->device->id] = desc; + out_le32 (&dma_reg->cpfpl, pcdb); + + if ( chan->device->desc_memory == ADMA_DESC_MEM_OCM) + out_le32 (&dma_reg->cpfph, 0x4); //upper bits +} + +/** + * ppc460ex_chan_append - update the h/w chain in the channel + */ +static inline void ppc460ex_chan_append(ppc460ex_ch_t *chan) +{ + volatile dma_regs_t *dma_reg; + ppc460ex_desc_t *iter; + u32 cur_desc; + unsigned long flags; + + local_irq_save(flags); + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + //dma_reg = (dma_regs_t *)chan->device->pdev->resource[0].start; + //dma_reg = (dma_regs_t *)chan->device->odev->dev.resource[0].start; + dma_reg = (dma_regs_t *)chan->device->res[0].start; + cur_desc = ppc460ex_chan_get_current_descriptor(chan); + + if (likely(cur_desc)) { + iter = chan_last_sub[chan->device->id]; + BUG_ON(!iter); + } else { + /* first peer */ + iter = chan_first_cdb[chan->device->id]; + BUG_ON(!iter); + ppc460ex_dma_put_desc(chan, iter); + chan->hw_chain_inited = 1; + } + + /* is there something new to append */ + if (!iter->hw_next) + goto out; + + /* flush descriptors from the s/w queue to fifo */ + list_for_each_entry_continue(iter, &chan->chain, chain_node) { + ppc460ex_dma_put_desc(chan, iter); + if (!iter->hw_next) + break; + } + break; + } +out: + local_irq_restore(flags); +} + +/** + * ppc460ex_chan_get_current_descriptor - get the currently executed descriptor + */ +static inline u32 ppc460ex_chan_get_current_descriptor(ppc460ex_ch_t *chan) +{ + volatile dma_regs_t *dma_reg; + volatile xor_regs_t *xor_reg; + + if (unlikely(!chan->hw_chain_inited)) + /* h/w descriptor chain is not initialized yet */ + return 0; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + //dma_reg = (dma_regs_t *)chan->device->pdev->resource[0].start; + dma_reg = (dma_regs_t *)chan->device->res[0].start; + return (le32_to_cpu(dma_reg->acpl)) & (~DMA_CDB_MSK); + case PPC460EX_XOR_ID: + //xor_reg = (xor_regs_t *)chan->device->pdev->resource[0].start; + return xor_reg->ccbalr; + } + return 0; +} + +/** + * ppc460ex_chan_run - enable the channel + */ +static inline void ppc460ex_chan_run(ppc460ex_ch_t *chan) +{ + volatile xor_regs_t *xor_reg; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + /* DMAs are always enabled, do nothing */ + break; + case PPC460EX_XOR_ID: + /* drain write buffer */ + + /* fetch descriptor pointed to in <link> */ + xor_reg->crsr = XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT; + break; + } +} + + +/****************************************************************************** + * ADMA device level + ******************************************************************************/ + +static void ppc460ex_chan_start_null_xor(ppc460ex_ch_t *chan); +static int ppc460ex_adma_alloc_chan_resources(struct dma_chan *chan); +static dma_cookie_t ppc460ex_adma_tx_submit( + struct dma_async_tx_descriptor *tx); + +static void ppc460ex_adma_set_dest( + ppc460ex_desc_t *tx, + dma_addr_t addr, int index); +static void ppc460ex_adma_memcpy_xor_set_src( + ppc460ex_desc_t *tx, + dma_addr_t addr, int index); + +static void ppc460ex_adma_dma01_xor_set_dest( + ppc460ex_desc_t *tx, + dma_addr_t addr, int index); +static void ppc460ex_adma_dma01_xor_set_src( + ppc460ex_desc_t *tx, + dma_addr_t addr, int index); +static void ppc460ex_adma_dma01_xor_set_src_mult( + ppc460ex_desc_t *tx, + unsigned char mult, int index); + +static void ppc460ex_adma_pqxor_set_dest( + ppc460ex_desc_t *tx, + dma_addr_t *paddr, unsigned long flags); +static void ppc460ex_adma_pqxor_set_src( + ppc460ex_desc_t *tx, + dma_addr_t addr, int index); +static void ppc460ex_adma_pqxor_set_src_mult ( + ppc460ex_desc_t *sw_desc, + unsigned char mult, int index,int dst_pos); + +static void ppc460ex_adma_pqzero_sum_set_dest ( + ppc460ex_desc_t *sw_desc, + dma_addr_t paddr, dma_addr_t qaddr); +static void ppc460ex_adma_mq_zero_sum_set_dest ( + ppc460ex_desc_t *sw_desc, + dma_addr_t addr); +static void ppc460ex_adma_pqzero_sum_set_src( + ppc460ex_desc_t *tx, + dma_addr_t addr, int index); +static void ppc460ex_adma_pqzero_sum_set_src_mult( + ppc460ex_desc_t *tx, + unsigned char mult, int index, int dst_pos); + +static void ppc460ex_adma_dma2rxor_set_dest ( + ppc460ex_desc_t *tx, + dma_addr_t addr, int index); + +void ppc460ex_adma_xor_set_dest(ppc460ex_desc_t *sw_desc, + dma_addr_t *addrs, unsigned long flags); +void ppc460ex_adma_xor_set_src( + ppc460ex_desc_t *sw_desc, + dma_addr_t addr, + int index); +void ppc460ex_adma_xor_set_src_mult ( + ppc460ex_desc_t *sw_desc, + unsigned char mult, int index,int dst_pos); +/** + * ppc460ex_can_rxor - check if the operands may be processed with RXOR + */ +static int ppc460ex_can_rxor (struct page **srcs, int src_cnt, size_t len) +{ + int i, order = 0, state = 0; + + if (unlikely(!(src_cnt > 1))) + return 0; + + for (i=1; i<src_cnt; i++) { + char *cur_addr = page_address (srcs[i]); + char *old_addr = page_address (srcs[i-1]); + switch (state) { + case 0: + if (cur_addr == old_addr + len) { + /* direct RXOR */ + order = 1; + state = 1; + } else + if (old_addr == cur_addr + len) { + /* reverse RXOR */ + order = -1; + state = 1; + } else + goto out; + break; + case 1: + if ((i == src_cnt-2) || + (order == -1 && cur_addr != old_addr - len)) { + order = 0; + state = 0; + } else + if ((cur_addr == old_addr + len*order) || + (cur_addr == old_addr + 2*len) || + (cur_addr == old_addr + 3*len)) { + state = 2; + } else { + order = 0; + state = 0; + } + break; + case 2: + order = 0; + state = 0; + break; + } + } + +out: + if (state == 1 || state == 2) + return 1; + + return 0; +} + +/** + * ppc460ex_adma_device_estimate - estimate the efficiency of processing + * the operation given on this channel. It's assumed that 'chan' is + * capable to process 'cap' type of operation. + * @chan: channel to use + * @cap: type of transaction + * @src_lst: array of source pointers + * @src_cnt: number of source operands + * @src_sz: size of each source operand + */ +int ppc460ex_adma_estimate (struct dma_chan *chan, + enum dma_transaction_type cap, struct page **src_lst, + int src_cnt, size_t src_sz) +{ + int ef = 1; + + if (cap == DMA_PQ || cap == DMA_PQ_VAL) { + /* If RAID-6 capabilities were not activated don't try + * to use them + */ + if (unlikely(!ppc460ex_r6_enabled)) + return -1; + } + if(cap == DMA_XOR) { + if (unlikely(!ppc460ex_r5_enabled)) + return -1; + } + /* in the current implementation of ppc460ex ADMA driver it + * makes sense to pick out only pqxor case, because it may be + * processed: + * (1) either using Biskup method on DMA2; + * (2) or on DMA0/1. + * Thus we give a favour to (1) if the sources are suitable; + * else let it be processed on one of the DMA0/1 engines. + */ + if (cap == DMA_PQ && chan->chan_id == PPC460EX_XOR_ID) { + if (ppc460ex_can_rxor(src_lst, src_cnt, src_sz)) + ef = 3; /* override (dma0/1 + idle) */ + else + ef = 0; /* can't process on DMA2 if !rxor */ + } + + /* channel idleness increases the priority */ + if (likely(ef) && + !ppc460ex_chan_is_busy(to_ppc460ex_adma_chan(chan))) + ef++; + + return ef; +} + +/** + * ppc460ex_get_group_entry - get group entry with index idx + * @tdesc: is the last allocated slot in the group. + */ +static inline ppc460ex_desc_t * +ppc460ex_get_group_entry ( ppc460ex_desc_t *tdesc, u32 entry_idx) +{ + ppc460ex_desc_t *iter = tdesc->group_head; + int i = 0; + + if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { + printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", + __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); + BUG(); + } + list_for_each_entry(iter, &tdesc->group_list, chain_node) { + if (i++ == entry_idx) + break; + } + return iter; +} + +/** + * ppc460ex_adma_free_slots - flags descriptor slots for reuse + * @slot: Slot to free + * Caller must hold &ppc460ex_chan->lock while calling this function + */ +static void ppc460ex_adma_free_slots(ppc460ex_desc_t *slot, + ppc460ex_ch_t *chan) +{ + int stride = slot->slots_per_op; + + while (stride--) { + /*async_tx_clear_ack(&slot->async_tx);*/ /* Don't need to clear. It is hack*/ + slot->slots_per_op = 0; + slot = list_entry(slot->slot_node.next, + ppc460ex_desc_t, + slot_node); + } +} + +static void +ppc460ex_adma_unmap(ppc460ex_ch_t *chan, ppc460ex_desc_t *desc) +{ + u32 src_cnt, dst_cnt; + dma_addr_t addr; + /* + * get the number of sources & destination + * included in this descriptor and unmap + * them all + */ + src_cnt = ppc460ex_desc_get_src_num(desc, chan); + dst_cnt = ppc460ex_desc_get_dst_num(desc, chan); + + /* unmap destinations */ + if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { + while (dst_cnt--) { + addr = ppc460ex_desc_get_dest_addr( + desc, chan, dst_cnt); + dma_unmap_page(&chan->device->odev->dev, + addr, desc->unmap_len, + DMA_FROM_DEVICE); + } + } + + /* unmap sources */ + if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { + while (src_cnt--) { + addr = ppc460ex_desc_get_src_addr( + desc, chan, src_cnt); + dma_unmap_page(&chan->device->odev->dev, + addr, desc->unmap_len, + DMA_TO_DEVICE); + } + } + +} +/** + * ppc460ex_adma_run_tx_complete_actions - call functions to be called + * upon complete + */ +static dma_cookie_t ppc460ex_adma_run_tx_complete_actions( + ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan, + dma_cookie_t cookie) +{ + int i; + + BUG_ON(desc->async_tx.cookie < 0); + if (desc->async_tx.cookie > 0) { + cookie = desc->async_tx.cookie; + desc->async_tx.cookie = 0; + + /* call the callback (must not sleep or submit new + * operations to this channel) + */ + if (desc->async_tx.callback) + desc->async_tx.callback( + desc->async_tx.callback_param); + + /* unmap dma addresses + * (unmap_single vs unmap_page?) + * + * actually, ppc's dma_unmap_page() functions are empty, so + * the following code is just for the sake of completeness + */ + if (chan && chan->needs_unmap && desc->group_head && + desc->unmap_len) { + ppc460ex_desc_t *unmap = desc->group_head; + /* assume 1 slot per op always */ + u32 slot_count = unmap->slot_cnt; + + /* Run through the group list and unmap addresses */ + for (i = 0; i < slot_count; i++) { + BUG_ON(!unmap); + ppc460ex_adma_unmap(chan, unmap); + unmap = unmap->hw_next; + } + //desc->group_head = NULL; + } + } + + /* run dependent operations */ + dma_run_dependencies(&desc->async_tx); + + return cookie; +} + +/** + * ppc460ex_adma_clean_slot - clean up CDB slot (if ack is set) + */ +static int ppc460ex_adma_clean_slot(ppc460ex_desc_t *desc, + ppc460ex_ch_t *chan) +{ + /* the client is allowed to attach dependent operations + * until 'ack' is set + */ + if (!async_tx_test_ack(&desc->async_tx)) + return 0; + + /* leave the last descriptor in the chain + * so we can append to it + */ + if (list_is_last(&desc->chain_node, &chan->chain) || + desc->phys == ppc460ex_chan_get_current_descriptor(chan)) + return 1; + + if (chan->device->id != PPC460EX_XOR_ID) { + /* our DMA interrupt handler clears opc field of + * each processed descriptor. For all types of + * operations except for ZeroSum we do not actually + * need ack from the interrupt handler. ZeroSum is a + * gtcial case since the result of this operation + * is available from the handler only, so if we see + * such type of descriptor (which is unprocessed yet) + * then leave it in chain. + */ + dma_cdb_t *cdb = desc->hw_desc; + if (cdb->opc == DMA_CDB_OPC_DCHECK128) + return 1; + } + + dev_dbg(chan->device->common.dev, "\tfree slot %x: %d stride: %d\n", + desc->phys, desc->idx, desc->slots_per_op); + + list_del(&desc->chain_node); + ppc460ex_adma_free_slots(desc, chan); + return 0; +} + +/** + * #define DEBUG 1__ppc460ex_adma_slot_cleanup - this is the common clean-up routine + * which runs through the channel CDBs list until reach the descriptor + * currently processed. When routine determines that all CDBs of group + * are completed then corresponding callbacks (if any) are called and slots + * are freed. + */ +static void __ppc460ex_adma_slot_cleanup(ppc460ex_ch_t *chan) +{ + ppc460ex_desc_t *iter, *_iter, *group_start = NULL; + dma_cookie_t cookie = 0; + u32 current_desc = ppc460ex_chan_get_current_descriptor(chan); + int busy = ppc460ex_chan_is_busy(chan); + int seen_current = 0, slot_cnt = 0, slots_per_op = 0; + + dev_dbg(chan->device->common.dev, "ppc460ex adma%d: %s\n", + chan->device->id, __FUNCTION__); + + if (!current_desc) { + /* There were no transactions yet, so + * nothing to clean + */ + return; + } + + /* free completed slots from the chain starting with + * the oldest descriptor + */ + list_for_each_entry_safe(iter, _iter, &chan->chain, + chain_node) { + dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " + "busy: %d this_desc: %#x next_desc: %#x cur: %#x ack: %d\n", + iter->async_tx.cookie, iter->idx, busy, iter->phys, + ppc460ex_desc_get_link(iter, chan), current_desc, + async_tx_test_ack(&iter->async_tx)); + prefetch(_iter); + prefetch(&_iter->async_tx); + + /* do not advance past the current descriptor loaded into the + * hardware channel,subsequent descriptors are either in process + * or have not been submitted + */ + if (seen_current) + break; + + /* stop the search if we reach the current descriptor and the + * channel is busy, or if it appears that the current descriptor + * needs to be re-read (i.e. has been appended to) + */ + if (iter->phys == current_desc) { + BUG_ON(seen_current++); + if (busy || ppc460ex_desc_get_link(iter, chan)) { + /* not all descriptors of the group have + * been completed; exit. + */ + break; + } + } + + /* detect the start of a group transaction */ + if (!slot_cnt && !slots_per_op) { + slot_cnt = iter->slot_cnt; + slots_per_op = iter->slots_per_op; + if (slot_cnt <= slots_per_op) { + slot_cnt = 0; + slots_per_op = 0; + } + } + + if (slot_cnt) { + if (!group_start) + group_start = iter; + slot_cnt -= slots_per_op; + } + + /* all the members of a group are complete */ + if (slots_per_op != 0 && slot_cnt == 0) { + ppc460ex_desc_t *grp_iter, *_grp_iter; + int end_of_chain = 0; + + /* clean up the group */ + slot_cnt = group_start->slot_cnt; + grp_iter = group_start; + list_for_each_entry_safe_from(grp_iter, _grp_iter, + &chan->chain, chain_node) { + + cookie = ppc460ex_adma_run_tx_complete_actions( + grp_iter, chan, cookie); + + slot_cnt -= slots_per_op; + end_of_chain = ppc460ex_adma_clean_slot( + grp_iter, chan); + if (end_of_chain && slot_cnt) { + /* Should wait for ZeroSum complete */ + if (cookie > 0) + chan->completed_cookie = cookie; + return; + } + + if (slot_cnt == 0 || end_of_chain) + break; + } + + /* the group should be complete at this point */ + BUG_ON(slot_cnt); + + slots_per_op = 0; + group_start = NULL; + if (end_of_chain) + break; + else + continue; + } else if (slots_per_op) /* wait for group completion */ + continue; + + cookie = ppc460ex_adma_run_tx_complete_actions(iter, chan, + cookie); + + if (ppc460ex_adma_clean_slot(iter, chan)) + break; + } + + BUG_ON(!seen_current); + + if (cookie > 0) { + chan->completed_cookie = cookie; + pr_debug("\tcompleted cookie %d\n", cookie); +#ifdef DEBUG_ADMA + static int tcnt=0; + if(tcnt%100 == 0) + printk("\t <%s> completed cookie %d\n",__FUNCTION__, cookie); +#endif + } + +} + +/** + * ppc460ex_adma_tasklet - clean up watch-dog initiator + */ +static void ppc460ex_adma_tasklet (unsigned long data) +{ + ppc460ex_ch_t *chan = (ppc460ex_ch_t *) data; + __ppc460ex_adma_slot_cleanup(chan); +} + +/** + * ppc460ex_adma_slot_cleanup - clean up scheduled initiator + */ +static void ppc460ex_adma_slot_cleanup (ppc460ex_ch_t *chan) +{ + spin_lock_bh(&chan->lock); + __ppc460ex_adma_slot_cleanup(chan); + spin_unlock_bh(&chan->lock); +} + +/** + * ppc460ex_adma_alloc_slots - allocate free slots (if any) + */ +ppc460ex_desc_t *ppc460ex_adma_alloc_slots( + ppc460ex_ch_t *chan, int num_slots, + int slots_per_op) +{ + ppc460ex_desc_t *iter = NULL, *_iter, *alloc_start = NULL; + struct list_head chain = LIST_HEAD_INIT(chain); + int slots_found, retry = 0; + + + BUG_ON(!num_slots || !slots_per_op); + /* start search from the last allocated descrtiptor + * if a contiguous allocation can not be found start searching + * from the beginning of the list + */ +retry: + slots_found = 0; + if (retry == 0) + iter = chan->last_used; + else + iter = list_entry(&chan->all_slots, ppc460ex_desc_t, + slot_node); + list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, + slot_node) { + prefetch(_iter); + prefetch(&_iter->async_tx); + if (iter->slots_per_op) { + slots_found = 0; + continue; + } + + /* start the allocation if the slot is correctly aligned */ + if (!slots_found++) + alloc_start = iter; + if (slots_found == num_slots) { + ppc460ex_desc_t *alloc_tail = NULL; + ppc460ex_desc_t *last_used = NULL; + iter = alloc_start; + while (num_slots) { + int i; + + /* pre-ack all but the last descriptor */ + if (num_slots != slots_per_op) { + async_tx_ack(&iter->async_tx); + } + + list_add_tail(&iter->chain_node, &chain); + alloc_tail = iter; + iter->async_tx.cookie = 0; + iter->hw_next = NULL; + iter->flags = 0; + iter->slot_cnt = num_slots; + iter->xor_check_result = NULL; + for (i = 0; i < slots_per_op; i++) { + iter->slots_per_op = slots_per_op - i; + last_used = iter; + iter = list_entry(iter->slot_node.next, + ppc460ex_desc_t, + slot_node); + } + num_slots -= slots_per_op; + } + alloc_tail->group_head = alloc_start; + alloc_tail->async_tx.cookie = -EBUSY; + list_splice(&chain, &alloc_tail->group_list); + chan->last_used = last_used; + return alloc_tail; + } + } + if (!retry++) + goto retry; + static empty_slot_cnt; + if(!(empty_slot_cnt%100)) + dev_dbg(chan->device->common.dev, + "No empty slots trying to free some\n"); + empty_slot_cnt++; + /* try to free some slots if the allocation fails */ + tasklet_schedule(&chan->irq_tasklet); + return NULL; +} + +/** + * ppc460ex_adma_alloc_chan_resources - allocate pools for CDB slots + */ +static int ppc460ex_adma_alloc_chan_resources(struct dma_chan *chan) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_desc_t *slot = NULL; + char *hw_desc; + int i, db_sz; + int init = ppc460ex_chan->slots_allocated ? 0 : 1; + ppc460ex_aplat_t *plat_data; + + chan->chan_id = ppc460ex_chan->device->id; + plat_data = ppc460ex_chan->device->odev->dev.platform_data; + + /* Allocate descriptor slots */ + i = ppc460ex_chan->slots_allocated; + if (ppc460ex_chan->device->id != PPC460EX_XOR_ID) + db_sz = sizeof (dma_cdb_t); + else + db_sz = sizeof (xor_cb_t); + + for (; i < (plat_data->pool_size/db_sz); i++) { + slot = kzalloc(sizeof(ppc460ex_desc_t), GFP_KERNEL); + if (!slot) { + printk(KERN_INFO "GT ADMA Channel only initialized" + " %d descriptor slots", i--); + break; + } + + hw_desc = (char *) ppc460ex_chan->device->dma_desc_pool_virt; + slot->hw_desc = (void *) &hw_desc[i * db_sz]; + dma_async_tx_descriptor_init(&slot->async_tx, chan); + slot->async_tx.tx_submit = ppc460ex_adma_tx_submit; + INIT_LIST_HEAD(&slot->chain_node); + INIT_LIST_HEAD(&slot->slot_node); + INIT_LIST_HEAD(&slot->group_list); + hw_desc = (char *) ppc460ex_chan->device->dma_desc_pool; + slot->phys = (dma_addr_t) &hw_desc[i * db_sz]; + slot->idx = i; + spin_lock_bh(&ppc460ex_chan->lock); + ppc460ex_chan->slots_allocated++; + list_add_tail(&slot->slot_node, &ppc460ex_chan->all_slots); + spin_unlock_bh(&ppc460ex_chan->lock); + } + + if (i && !ppc460ex_chan->last_used) { + ppc460ex_chan->last_used = + list_entry(ppc460ex_chan->all_slots.next, + ppc460ex_desc_t, + slot_node); + } + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: allocated %d descriptor slots\n", + ppc460ex_chan->device->id, i); + + /* initialize the channel and the chain with a null operation */ + if (init) { + switch (ppc460ex_chan->device->id) + { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + ppc460ex_chan->hw_chain_inited = 0; + /* Use WXOR for self-testing */ + if (!ppc460ex_r5_tchan) + ppc460ex_r5_tchan = ppc460ex_chan; + if (!ppc460ex_r6_tchan) + ppc460ex_r6_tchan = ppc460ex_chan; + break; + case PPC460EX_XOR_ID: + ppc460ex_chan_start_null_xor(ppc460ex_chan); + break; + default: + BUG(); + } + ppc460ex_chan->needs_unmap = 1; + } + + return (i > 0) ? i : -ENOMEM; +} + +/** + * ppc460ex_desc_assign_cookie - assign a cookie + */ +static dma_cookie_t ppc460ex_desc_assign_cookie(ppc460ex_ch_t *chan, + ppc460ex_desc_t *desc) +{ + dma_cookie_t cookie = chan->common.cookie; + cookie++; + if (cookie < 0) + cookie = 1; + chan->common.cookie = desc->async_tx.cookie = cookie; + return cookie; +} + +/** + * ppc460ex_rxor_set_region_data - + */ +static void ppc460ex_rxor_set_region (ppc460ex_desc_t *desc, + u8 xor_arg_no, u32 mask) +{ + xor_cb_t *xcb = desc->hw_desc; + + xcb->ops [xor_arg_no].h |= mask; +} + +/** + * ppc460ex_rxor_set_src - + */ +static void ppc460ex_rxor_set_src (ppc460ex_desc_t *desc, + u8 xor_arg_no, dma_addr_t addr) +{ + xor_cb_t *xcb = desc->hw_desc; + + xcb->ops [xor_arg_no].h |= DMA_CUED_XOR_BASE; + xcb->ops [xor_arg_no].l = addr; +} + +/** + * ppc460ex_rxor_set_mult - + */ +static void ppc460ex_rxor_set_mult (ppc460ex_desc_t *desc, + u8 xor_arg_no, u8 idx, u8 mult) +{ + xor_cb_t *xcb = desc->hw_desc; + + xcb->ops [xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); +} + +/** + * ppc460ex_wxor_set_base + */ +static void ppc460ex_wxor_set_base (ppc460ex_desc_t *desc) +{ + xor_cb_t *xcb = desc->hw_desc; + + xcb->cbtah = DMA_CUED_XOR_BASE; + xcb->cbtah |= (1 << DMA_CUED_MULT1_OFF); +} + +/** + * ppc460ex_adma_check_threshold - append CDBs to h/w chain if threshold + * has been achieved + */ +static void ppc460ex_adma_check_threshold(ppc460ex_ch_t *chan) +{ + dev_dbg(chan->device->common.dev, "ppc460ex adma%d: pending: %d\n", + chan->device->id, chan->pending); + + if (chan->pending >= PPC460EX_ADMA_THRESHOLD) { + chan->pending = 0; + ppc460ex_chan_append(chan); + } +} + +/** + * ppc460ex_adma_tx_submit - submit new descriptor group to the channel + * (it's not necessary that descriptors will be submitted to the h/w + * chains too right now) + */ +static dma_cookie_t ppc460ex_adma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + ppc460ex_desc_t *sw_desc = tx_to_ppc460ex_adma_slot(tx); + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(tx->chan); + ppc460ex_desc_t *group_start, *old_chain_tail; + int slot_cnt; + int slots_per_op; + dma_cookie_t cookie; + + group_start = sw_desc->group_head; + slot_cnt = group_start->slot_cnt; + slots_per_op = group_start->slots_per_op; + + spin_lock_bh(&chan->lock); + + cookie = ppc460ex_desc_assign_cookie(chan, sw_desc); + + if (unlikely(list_empty(&chan->chain))) { + /* first peer */ + list_splice_init(&sw_desc->group_list, &chan->chain); + chan_first_cdb[chan->device->id] = group_start; + } else { + /* isn't first peer, bind CDBs to chain */ + old_chain_tail = list_entry(chan->chain.prev, + ppc460ex_desc_t, chain_node); + list_splice_init(&sw_desc->group_list, + &old_chain_tail->chain_node); + /* fix up the hardware chain */ + ppc460ex_desc_set_link(chan, old_chain_tail, group_start); + } + + /* increment the pending count by the number of operations */ + chan->pending += slot_cnt / slots_per_op; + ppc460ex_adma_check_threshold(chan); + spin_unlock_bh(&chan->lock); + + dev_dbg(chan->device->common.dev, + "ppc460ex adma%d: %s cookie: %d slot: %d tx %p\n", + chan->device->id,__FUNCTION__, + sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); + return cookie; +} + +/** + * ppc460ex_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_interrupt( + struct dma_chan *chan, unsigned long flags) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s\n", ppc460ex_chan->device->id, + __FUNCTION__); + + spin_lock_bh(&ppc460ex_chan->lock); + slot_cnt = slots_per_op = 1; + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + ppc460ex_desc_init_interrupt(group_start, ppc460ex_chan); + group_start->unmap_len = 0; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&ppc460ex_chan->lock); + + return sw_desc ? &sw_desc->async_tx : NULL; +} + +/** + * ppc460ex_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dma_dest, + dma_addr_t dma_src, size_t len, unsigned long flags) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + if (unlikely(!len)) + return NULL; + BUG_ON(unlikely(len > PPC460EX_ADMA_DMA_MAX_BYTE_COUNT)); + + spin_lock_bh(&ppc460ex_chan->lock); + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s len: %u int_en %d \n", + ppc460ex_chan->device->id, __FUNCTION__, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + slot_cnt = slots_per_op = 1; + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + ppc460ex_desc_init_memcpy(group_start, flags); + ppc460ex_adma_set_dest(group_start, dma_dest, 0); + ppc460ex_adma_memcpy_xor_set_src(group_start, dma_src, 0); + ppc460ex_desc_set_byte_count(group_start, ppc460ex_chan, len); + sw_desc->unmap_len = len; + sw_desc->async_tx.flags = flags; + if(mfdcr(0x60) == 0xfee8) { + printk("Byte Count = 0x%x\n",len); + printk("src= 0x%x\n",dma_src); + printk("Dest = 0x%x\n",dma_dest); + } + } + spin_unlock_bh(&ppc460ex_chan->lock); + return sw_desc ? &sw_desc->async_tx : NULL; +} + +/** + * ppc460ex_adma_prep_dma_memset - prepare CDB for a MEMSET operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_memset( + struct dma_chan *chan, dma_addr_t dma_dest, int value, + size_t len, unsigned long flags) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + if (unlikely(!len)) + return NULL; + BUG_ON(unlikely(len > PPC460EX_ADMA_DMA_MAX_BYTE_COUNT)); + + spin_lock_bh(&ppc460ex_chan->lock); + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s cal: %u len: %u int_en %d\n", + ppc460ex_chan->device->id, __FUNCTION__, value, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + slot_cnt = slots_per_op = 1; + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + ppc460ex_desc_init_memset(group_start, value, flags); + ppc460ex_adma_set_dest(group_start, dma_dest, 0); + ppc460ex_desc_set_byte_count(group_start, ppc460ex_chan, len); + sw_desc->unmap_len = len; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&ppc460ex_chan->lock); + + return sw_desc ? &sw_desc->async_tx : NULL; +} + + +static inline void ppc460ex_desc_set_xor_src_cnt (ppc460ex_desc_t *desc, + int src_cnt); +static void ppc460ex_init_rxor_cursor (ppc460ex_rxor_cursor_t *cursor); + +/** + * ppc460ex_adma_init_dma2rxor_slot - + */ +static void ppc460ex_adma_init_dma2rxor_slot (ppc460ex_desc_t *desc, + dma_addr_t *src, int src_cnt) +{ + int i; + /* initialize CDB */ + for (i=0; i<src_cnt; i++) { + ppc460ex_adma_dma2rxor_prep_src(desc, + &desc->rxor_cursor, + i, desc->src_cnt, + (u32)src[i]); + } +} +#if 1 +static inline ppc460ex_desc_t *ppc460ex_dma_prep_pq( + ppc460ex_ch_t *ppc460ex_chan, + dma_addr_t *dst, unsigned int dst_cnt, + dma_addr_t *src, unsigned int src_cnt, unsigned char *scf, + size_t len, unsigned long flags) +{ + int slot_cnt; + ppc460ex_desc_t *sw_desc = NULL, *iter; + unsigned long op = 0; + unsigned char mult = 1; + + /* select operations WXOR/RXOR depending on the + * source addresses of operators and the number + * of destinations (RXOR support only Q-parity calculations) + */ + set_bit(PPC460EX_DESC_WXOR, &op); + if (!test_and_set_bit(PPC460EX_RXOR_RUN, &ppc460ex_rxor_state)) { + /* no active RXOR; + * do RXOR if: + * - destination os only one, + * - there are more than 1 source, + * - len is aligned on 512-byte boundary, + * - source addresses fit to one of 4 possible regions. + */ + if (dst_cnt == 1 && src_cnt > 1 && + !(len & ~MQ0_CF2H_RXOR_BS_MASK) && + (src[0] + len) == src[1]) { + /* may do RXOR R1 R2 */ + set_bit(PPC460EX_DESC_RXOR, &op); + if (src_cnt != 2) { + /* may try to enhance region of RXOR */ + if ((src[1] + len) == src[2]) { + /* do RXOR R1 R2 R3 */ + set_bit(PPC460EX_DESC_RXOR123, + &op); + } else if ((src[1] + len * 2) == src[2]) { + /* do RXOR R1 R2 R4 */ + set_bit(PPC460EX_DESC_RXOR124, &op); + } else if ((src[1] + len * 3) == src[2]) { + /* do RXOR R1 R2 R5 */ + set_bit(PPC460EX_DESC_RXOR125, + &op); + } else { + /* do RXOR R1 R2 */ + set_bit(PPC460EX_DESC_RXOR12, + &op); + } + } else { + /* do RXOR R1 R2 */ + set_bit(PPC460EX_DESC_RXOR12, &op); + } + } + + if (!test_bit(PPC460EX_DESC_RXOR, &op)) { + /* can not do this operation with RXOR */ + clear_bit(PPC460EX_RXOR_RUN, + &ppc460ex_rxor_state); + } else { + /* can do; set block size right now */ + ppc460ex_desc_set_rxor_block_size(len); + } + } + + /* Number of necessary slots depends on operation type selected */ + if (!test_bit(PPC460EX_DESC_RXOR, &op)) { + /* This is a WXOR only chain. Need descriptors for each + * source to GF-XOR them with WXOR, and need descriptors + * for each destination to zero them with WXOR + */ + slot_cnt = src_cnt; + + if (flags & DMA_PREP_ZERO_P) { + slot_cnt++; + set_bit(PPC460EX_ZERO_P, &op); + } + if (flags & DMA_PREP_ZERO_Q) { + slot_cnt++; + set_bit(PPC460EX_ZERO_Q, &op); + } + } else { + /* Need 1/2 descriptor for RXOR operation, and + * need (src_cnt - (2 or 3)) for WXOR of sources + * remained (if any) + */ + slot_cnt = dst_cnt; + + if (flags & DMA_PREP_ZERO_P) + set_bit(PPC460EX_ZERO_P, &op); + if (flags & DMA_PREP_ZERO_Q) + set_bit(PPC460EX_ZERO_Q, &op); + + if (test_bit(PPC460EX_DESC_RXOR12, &op)) + slot_cnt += src_cnt - 2; + else + slot_cnt += src_cnt - 3; + + /* Thus we have either RXOR only chain or + * mixed RXOR/WXOR + */ + if (slot_cnt == dst_cnt) { + /* RXOR only chain */ + clear_bit(PPC460EX_DESC_WXOR, &op); + } + } + + spin_lock_bh(&ppc460ex_chan->lock); + /* for both RXOR/WXOR each descriptor occupies one slot */ + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, 1); + if (sw_desc) { + ppc460ex_desc_init_pq(sw_desc, dst_cnt, src_cnt, + flags, op); + + /* setup dst/src/mult */ + ppc460ex_adma_pqxor_set_dest(sw_desc, + dst, flags); + while(src_cnt--) { + ppc460ex_adma_pqxor_set_src(sw_desc, + src[src_cnt], src_cnt); + if ((flags & DMA_PREP_HAVE_Q ) && !scf) { + mult = scf[src_cnt]; + ppc460ex_adma_pqxor_set_src_mult(sw_desc, + scf[src_cnt], src_cnt, dst_cnt -1 ); + } + } + + /* Setup byte count foreach slot just allocated */ + sw_desc->async_tx.flags = flags; + list_for_each_entry(iter, &sw_desc->group_list, + chain_node) { + if(mfdcr(0x60) == 0xfee8) { + printk("Byte Count = 0x%x\n",len); + printk("src[0]= 0x%x\n",src[0]); + printk("src[1]= 0x%x\n",src[1]); + printk("src[2]= 0x%x\n",src[2]); + printk("Dest = 0x%x\n",dst); + } + ppc460ex_desc_set_byte_count(iter, + ppc460ex_chan, len); + iter->unmap_len = len; + } + } + spin_unlock_bh(&ppc460ex_chan->lock); + + return sw_desc; +} + +#endif +static inline ppc460ex_desc_t *ppc460ex_dma_prep_xor( + ppc460ex_ch_t *ppc460ex_chan, + dma_addr_t *dst, unsigned int dst_cnt, + dma_addr_t *src, unsigned int src_cnt, unsigned char *scf, + size_t len, unsigned long flags) +{ + int slot_cnt; + ppc460ex_desc_t *sw_desc = NULL, *iter; + unsigned long op = 0; + + /* select operations WXOR/RXOR depending on the + * source addresses of operators and the number + * of destinations (RXOR support only Q-parity calculations) + */ + set_bit(PPC460EX_DESC_WXOR, &op); + if (!test_and_set_bit(PPC460EX_RXOR_RUN, &ppc460ex_rxor_state)) { + /* no active RXOR; + * do RXOR if: + * - destination os only one, + * - there are more than 1 source, + * - len is aligned on 512-byte boundary, + * - source addresses fit to one of 4 possible regions. + */ + if (dst_cnt == 2 && src_cnt > 1 && + !(len & ~MQ0_CF2H_RXOR_BS_MASK) && + (src[0] + len) == src[1]) { + /* may do RXOR R1 R2 */ + set_bit(PPC460EX_DESC_RXOR, &op); + if (src_cnt != 2) { + /* may try to enhance region of RXOR */ + if ((src[1] + len) == src[2]) { + /* do RXOR R1 R2 R3 */ + set_bit(PPC460EX_DESC_RXOR123, + &op); + } else if ((src[1] + len * 2) == src[2]) { + /* do RXOR R1 R2 R4 */ + set_bit(PPC460EX_DESC_RXOR124, &op); + } else if ((src[1] + len * 3) == src[2]) { + /* do RXOR R1 R2 R5 */ + set_bit(PPC460EX_DESC_RXOR125, + &op); + } else { + /* do RXOR R1 R2 */ + set_bit(PPC460EX_DESC_RXOR12, + &op); + } + } else { + /* do RXOR R1 R2 */ + set_bit(PPC460EX_DESC_RXOR12, &op); + } + } + + if (!test_bit(PPC460EX_DESC_RXOR, &op)) { + /* can not do this operation with RXOR */ + clear_bit(PPC460EX_RXOR_RUN, + &ppc460ex_rxor_state); + } else { + /* can do; set block size right now */ + ppc460ex_desc_set_rxor_block_size(len); + } + } + + /* Number of necessary slots depends on operation type selected */ + if (!test_bit(PPC460EX_DESC_RXOR, &op)) { + /* This is a WXOR only chain. Need descriptors for each + * source to GF-XOR them with WXOR, and need descriptors + * for each destination to zero them with WXOR + */ + slot_cnt = src_cnt; + + if (flags & DMA_PREP_ZERO_P) { + slot_cnt++; + set_bit(PPC460EX_ZERO_P, &op); + } + } else { + /* Need 1/2 descriptor for RXOR operation, and + * need (src_cnt - (2 or 3)) for WXOR of sources + * remained (if any) + */ + slot_cnt = dst_cnt; + + if (flags & DMA_PREP_ZERO_P) + set_bit(PPC460EX_ZERO_P, &op); + + if (test_bit(PPC460EX_DESC_RXOR12, &op)) + slot_cnt += src_cnt - 2; + else + slot_cnt += src_cnt - 3; + + /* Thus we have either RXOR only chain or + * mixed RXOR/WXOR + */ + if (slot_cnt == dst_cnt) { + /* RXOR only chain */ + clear_bit(PPC460EX_DESC_WXOR, &op); + } + } + + spin_lock_bh(&ppc460ex_chan->lock); + /* for both RXOR/WXOR each descriptor occupies one slot */ + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, 1); + if (sw_desc) { + ppc460ex_desc_init_xor(sw_desc, dst_cnt, src_cnt, + flags, op); + + /* setup dst/src/mult */ + ppc460ex_adma_xor_set_dest(sw_desc, + dst, flags); + while(src_cnt--) { + ppc460ex_adma_xor_set_src(sw_desc, + src[src_cnt], src_cnt); + } + + /* Setup byte count foreach slot just allocated */ + sw_desc->async_tx.flags = flags; + list_for_each_entry(iter, &sw_desc->group_list, + chain_node) { + if(mfdcr(0x60) == 0xfee8) { + printk("Byte Count = 0x%x\n",len); + printk("src[0]= 0x%x\n",src[0]); + printk("src[1]= 0x%x\n",src[1]); + printk("src[2]= 0x%x\n",src[2]); + printk("Dest = 0x%x\n",dst); + } + ppc460ex_desc_set_byte_count(iter, + ppc460ex_chan, len); + iter->unmap_len = len; + } + } + spin_unlock_bh(&ppc460ex_chan->lock); + + return sw_desc; +} +static inline ppc460ex_desc_t *ppc460ex_dma01_prep_xor ( + ppc460ex_ch_t *ppc460ex_chan, + dma_addr_t dst, dma_addr_t *src, unsigned int src_cnt, + size_t len, unsigned long flags) +{ + int slot_cnt; + ppc460ex_desc_t *sw_desc = NULL, *iter; + unsigned long op = 0; + unsigned int dst_cnt = 1; /*Marri */ + /* select operations WXOR/RXOR depending on the + * source addresses of operators and the number + * of destinations (RXOR support only Q-parity calculations) + */ + set_bit(PPC460EX_DESC_WXOR, &op); + if (!test_and_set_bit(PPC460EX_RXOR_RUN, &ppc460ex_rxor_state)) { + /* no active RXOR; + * do RXOR if: + * - destination os only one, + * - there are more than 1 source, + * - len is aligned on 512-byte boundary, + * - source addresses fit to one of 4 possible regions. + */ + if (dst_cnt == 3 && src_cnt > 1 && /* Marri dstcnt == 3 never comes here */ + !(len & ~MQ0_CF2H_RXOR_BS_MASK) && + (src[0] - len) == src[1]) { /* Marri */ + /* may do RXOR R1 R2 */ + set_bit(PPC460EX_DESC_RXOR, &op); + if (src_cnt != 2) { + /* may try to enhance region of RXOR */ + if ((src[1] - len) == src[2]) { + /* do RXOR R1 R2 R3 */ + set_bit(PPC460EX_DESC_RXOR123, + &op); + } else if ((src[1] - len * 2) == src[2]) { + /* do RXOR R1 R2 R4 */ + set_bit(PPC460EX_DESC_RXOR124, &op); + } else if ((src[1] - len * 3) == src[2]) { + /* do RXOR R1 R2 R5 */ + set_bit(PPC460EX_DESC_RXOR125, + &op); + } else { + /* do RXOR R1 R2 */ + set_bit(PPC460EX_DESC_RXOR12, + &op); + } + } else { + /* do RXOR R1 R2 */ + set_bit(PPC460EX_DESC_RXOR12, &op); + } + } + + if (!test_bit(PPC460EX_DESC_RXOR, &op)) { + /* can not do this operation with RXOR */ + clear_bit(PPC460EX_RXOR_RUN, + &ppc460ex_rxor_state); + } else { + /* can do; set block size right now */ + ppc460ex_desc_set_rxor_block_size(len); + } + } + + /* Number of necessary slots depends on operation type selected */ + if (!test_bit(PPC460EX_DESC_RXOR, &op)) { + /* This is a WXOR only chain. Need descriptors for each + * source to GF-XOR them with WXOR, and need descriptors + * for each destination to zero them with WXOR + */ + slot_cnt = src_cnt; + +#if 1 + if (flags & DMA_PREP_ZERO_P) { + slot_cnt += dst_cnt; + set_bit(PPC460EX_ZERO_P, &op); + } +#endif /* RAID-6 stuff*/ + } else { + /* Need 1 descriptor for RXOR operation, and + * need (src_cnt - (2 or 3)) for WXOR of sources + * remained (if any) + * Thus we have 1 CDB for RXOR, let the set_dst + * function think that this is just a zeroing descriptor + * and skip it when walking through the chain. + * So set PPC460EX_ZERO_P. + */ + set_bit(PPC460EX_ZERO_P, &op); + + if (test_bit(PPC460EX_DESC_RXOR12, &op)) { + slot_cnt = src_cnt - 1; + } else { + slot_cnt = src_cnt - 2; + } + + /* Thus we have either RXOR only chain or + * mixed RXOR/WXOR + */ + if (slot_cnt == 1) { + /* RXOR only chain */ + clear_bit(PPC460EX_DESC_WXOR, &op); + } + } + + spin_lock_bh(&ppc460ex_chan->lock); + /* for both RXOR/WXOR each descriptor occupies one slot */ + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, 1); + if (sw_desc) { + ppc460ex_desc_init_dma01_xor(sw_desc, dst_cnt, src_cnt, + flags, op); + + /* setup dst/src/mult */ + while(dst_cnt--) + ppc460ex_adma_dma01_xor_set_dest(sw_desc, + dst, dst_cnt); + //dst[dst_cnt], dst_cnt); /*marri */ + while(src_cnt--) { + ppc460ex_adma_dma01_xor_set_src(sw_desc, + src[src_cnt], src_cnt); + ppc460ex_adma_dma01_xor_set_src_mult(sw_desc, + 1, src_cnt);/* Marri forcing RAID-5*/ + /*scf[src_cnt], src_cnt);*/ + } + + /* Setup byte count foreach slot just allocated */ + sw_desc->async_tx.flags = flags; + list_for_each_entry(iter, &sw_desc->group_list, + chain_node) { + if(mfdcr(0x60) == 0xfee8) { + printk("Byte Count = 0x%x\n",len); + printk("src[0]= 0x%x\n",src[0]); + printk("src[1]= 0x%x\n",src[1]); + printk("src[2]= 0x%x\n",src[2]); + printk("Dest = 0x%x\n",dst); + } + ppc460ex_desc_set_byte_count(iter, + ppc460ex_chan, len); + iter->unmap_len = len; + } + } + + spin_unlock_bh(&ppc460ex_chan->lock); + + return sw_desc; +} +#if 1 +/** + * ppc460ex_adma_prep_dma_pq- prepare CDB (group) for a GF-XOR operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_pq( + struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, + unsigned int src_cnt, unsigned char *scf, + size_t len, unsigned long flags) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_desc_t *sw_desc = NULL; + int dst_cnt = 0; + + BUG_ON(!len); + BUG_ON(unlikely(len > PPC460EX_ADMA_XOR_MAX_BYTE_COUNT)); + BUG_ON(!src_cnt); + + if (flags & DMA_PREP_HAVE_P) { + BUG_ON(!dst[0]); + dst_cnt++; + } else + BUG_ON(flags & DMA_PREP_ZERO_P); + if (flags & DMA_PREP_HAVE_Q) { + BUG_ON(!dst[1]); + dst_cnt++; + } else + BUG_ON(flags & DMA_PREP_ZERO_Q); + BUG_ON(!dst_cnt); + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s src_cnt: %d len: %u int_en: %d\n", + ppc460ex_chan->device->id, __FUNCTION__, src_cnt, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + switch (ppc460ex_chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + sw_desc = ppc460ex_dma_prep_pq(ppc460ex_chan, + dst, dst_cnt, src, src_cnt, scf, + len, flags); + break; + + } + + return sw_desc ? &sw_desc->async_tx : NULL; +} +#endif +/** + * ppc460ex_adma_prep_dma_mq_xor - prepare CDB (group) for a GF-XOR operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_mq_xor( + struct dma_chan *chan, dma_addr_t dst, + dma_addr_t *src, unsigned int src_cnt, + size_t len, unsigned long flags) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_desc_t *sw_desc = NULL; + int dst_cnt = 1; + + BUG_ON(!len); + BUG_ON(unlikely(len > PPC460EX_ADMA_XOR_MAX_BYTE_COUNT)); + BUG_ON(!src_cnt ); + +// printk("<%s> line %d\n",__FUNCTION__,__LINE__); + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s src_cnt: %d len: %u int_en: %d\n", + ppc460ex_chan->device->id, __FUNCTION__, src_cnt, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + switch (ppc460ex_chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: +#if 0 + sw_desc = ppc460ex_dma01_prep_xor (ppc460ex_chan, + dst, src, src_cnt, + len, flags); +#else + sw_desc = ppc460ex_dma_prep_xor(ppc460ex_chan, + &dst, dst_cnt, src, src_cnt, 0, + len, flags); +#endif + break; + + } + + return sw_desc ? &sw_desc->async_tx : NULL; +} + +/** + * ppc460ex_adma_prep_dma_pqzero_sum - prepare CDB group for + * a PQ_VAL operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_pqzero_sum( + struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, + unsigned char *scf, size_t len, + u32 *pqres, unsigned long flags) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_desc_t *sw_desc, *iter; + dma_addr_t pdest, qdest; + int slot_cnt, slots_per_op, idst, dst_cnt; + + if (flags & DMA_PREP_HAVE_P) + pdest = src[src_cnt]; + else + pdest = 0; + if (flags & DMA_PREP_HAVE_Q) + qdest = src[src_cnt+1]; + else + qdest = 0; + + BUG_ON(src_cnt < 3 ); + + /* Always use WXOR for P/Q calculations (two destinations). + * Need two extra slots to verify results are zero. Since src_cnt + * is the size of the src[] buffer (which includes destination + * pointers at the first and/or second positions) then the number + * of actual sources should be reduced by DMA_DEST_MAX_NUM (2). + */ + idst = dst_cnt = (pdest && qdest) ? 2 : 1; + + slot_cnt = src_cnt + dst_cnt; + slots_per_op = 1; + + spin_lock_bh(&ppc460ex_chan->lock); + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + ppc460ex_desc_init_pqzero_sum(sw_desc, dst_cnt, src_cnt); + + /* Setup byte count foreach slot just allocated */ + sw_desc->async_tx.flags = flags; + list_for_each_entry(iter, &sw_desc->group_list, chain_node) { + ppc460ex_desc_set_byte_count(iter, ppc460ex_chan, + len); + iter->unmap_len = len; + } + + /* Setup destinations for P/Q ops */ + ppc460ex_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); + + /* Setup sources and mults for P/Q ops */ + while (src_cnt--) { + ppc460ex_adma_pqzero_sum_set_src (sw_desc, + src[src_cnt], src_cnt); + /* Setup mults for Q-check only; in case of P - + * keep the default 0 (==1) + */ + if (qdest) + ppc460ex_adma_pqzero_sum_set_src_mult (sw_desc, + scf[src_cnt], src_cnt,dst_cnt - 1 ); + } + + /* Setup zero QWORDs into DCHECK CDBs */ + idst = dst_cnt; + list_for_each_entry_reverse(iter, &sw_desc->group_list, + chain_node) { + /* + * The last CDB corresponds to P-parity check + * (if any), the one before last CDB corresponds + * Q-parity check + */ + if (idst == DMA_DEST_MAX_NUM) { + if (idst == dst_cnt) { + set_bit(PPC460EX_DESC_QCHECK, + &iter->flags); + } else { + set_bit(PPC460EX_DESC_PCHECK, + &iter->flags); + } + } else { + if (qdest) { + set_bit(PPC460EX_DESC_QCHECK, + &iter->flags); + } else { + set_bit(PPC460EX_DESC_PCHECK, + &iter->flags); + } + } + iter->xor_check_result = pqres; + /* + * set it to zero, if check fail then result will + * be updated + */ + *iter->xor_check_result = 0; + ppc460ex_desc_set_dcheck(iter, ppc460ex_chan, + ppc460ex_qword); + if (!(--dst_cnt)) + break; + } + } + spin_unlock_bh(&ppc460ex_chan->lock); + return sw_desc ? &sw_desc->async_tx : NULL; +} +/** + * ppc460ex_adma_prep_dma_mq_zero_sum - prepare CDB group for + * a PQ_VAL operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_mq_zero_sum( + struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, + size_t len, u32 *presult, unsigned long flags) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_desc_t *sw_desc, *iter; + int slot_cnt, slots_per_op, idst, dst_cnt; + + BUG_ON(src_cnt < 3 || !src[0]); + + /* Always use WXOR for P/Q calculations (two destinations). + * Need two extra slots to verify results are zero. Since src_cnt + * is the size of the src[] buffer (which includes destination + * pointers at the first and/or second positions) then the number + * of actual sources should be reduced by DMA_DEST_MAX_NUM (2). + */ + idst = dst_cnt = 1; + slot_cnt = src_cnt ; + src_cnt -= dst_cnt; + slots_per_op = 1; + + spin_lock_bh(&ppc460ex_chan->lock); + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + ppc460ex_desc_init_pqzero_sum(sw_desc, dst_cnt, src_cnt); + + /* Setup byte count foreach slot just allocated */ + sw_desc->async_tx.flags = flags; + list_for_each_entry(iter, &sw_desc->group_list, chain_node) { + ppc460ex_desc_set_byte_count(iter, ppc460ex_chan, + len); + iter->unmap_len = len; + } + + /* Setup destinations for P/Q ops */ + + ppc460ex_adma_mq_zero_sum_set_dest(sw_desc, src[0]); + + /* Setup sources and mults for P/Q ops */ + src = &src[dst_cnt]; + while (src_cnt-- ) { + ppc460ex_adma_pqzero_sum_set_src (sw_desc, + src[src_cnt ], src_cnt); + } + + /* Setup zero QWORDs into DCHECK CDBs */ + idst = dst_cnt; + list_for_each_entry_reverse(iter, &sw_desc->group_list, + chain_node) { + /* + * The last CDB corresponds to P-parity check + * (if any), the one before last CDB corresponds + * Q-parity check + */ + iter->xor_check_result = presult; + /* + * set it to zero, if check fail then result will + * be updated + */ + *iter->xor_check_result = 0; + ppc460ex_desc_set_dcheck(iter, ppc460ex_chan, + ppc460ex_qword); + if (!(--dst_cnt)) + break; + } + } + spin_unlock_bh(&ppc460ex_chan->lock); + return sw_desc ? &sw_desc->async_tx : NULL; +} + +static void ppc460ex_adma_pq_zero_op(ppc460ex_desc_t *iter, + ppc460ex_ch_t *chan, dma_addr_t addr) +{ + /* To clear destinations update the descriptor + * (P or Q depending on index) as follows: + * addr is destination (0 corresponds to SG2): + */ + ppc460ex_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); + + /* ... and the addr is source: */ + ppc460ex_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); + + /* addr is always SG2 then the mult is always DST1 */ + //ppc460ex_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, + // DMA_CDB_SG_DST1, 1); +} +/** + * ppc460ex_adma_set_dest - set destination address into descriptor + */ +static inline void ppc460ex_adma_set_dest(ppc460ex_desc_t *sw_desc, + dma_addr_t addr, int index) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + BUG_ON(index >= sw_desc->dst_cnt); + + switch (chan->device->id) { + case PPC460EX_DMA1_ID: + /* to do: support transfers lengths > + * PPC460EX_ADMA_DMA/XOR_MAX_BYTE_COUNT + */ + ppc460ex_desc_set_dest_addr(sw_desc->group_head, + // chan, 0x8, addr, index); // Enabling HB bus + chan, 0, addr, index); + break; + case PPC460EX_XOR_ID: + sw_desc = ppc460ex_get_group_entry(sw_desc, index); + ppc460ex_desc_set_dest_addr(sw_desc, + chan, 0, addr, index); + break; + } +} + + +static void ppc460ex_adma_dma2rxor_set_dest ( + ppc460ex_desc_t *sw_desc, + dma_addr_t addr, int index) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + ppc460ex_desc_t *iter; + int i; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + BUG(); + break; + case PPC460EX_XOR_ID: + iter = ppc460ex_get_group_entry(sw_desc, + sw_desc->descs_per_op*index); + for (i=0;i<sw_desc->descs_per_op;i++) { + ppc460ex_desc_set_dest_addr(iter, + chan, 0, addr, index); + if (i) ppc460ex_wxor_set_base (iter); + iter = list_entry (iter->chain_node.next, + ppc460ex_desc_t, chain_node); + } + break; + } +} + +/** + * ppc460ex_adma_pq_xor_set_dest - set destination address into descriptor + * for the PQXOR operation + */ +static void ppc460ex_adma_pqxor_set_dest(ppc460ex_desc_t *sw_desc, + dma_addr_t *addrs, unsigned long flags) +{ + ppc460ex_desc_t *iter; + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + dma_addr_t paddr, qaddr; + dma_addr_t addr = 0, ppath, qpath; + int index = 0, i; + + if (flags & DMA_PREP_HAVE_P) + paddr = addrs[0]; + else + paddr = 0; + + if (flags & DMA_PREP_HAVE_Q) + qaddr = addrs[1]; + else + qaddr = 0; + + if (!paddr || !qaddr) + addr = paddr ? paddr : qaddr; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + /* walk through the WXOR source list and set P/Q-destinations + * for each slot: + */ + if (!test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags)) { + /* This is WXOR-only chain; may have 1/2 zero descs */ + if (test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) + index++; + if (test_bit(PPC460EX_ZERO_Q, &sw_desc->flags)) + index++; + + iter = ppc460ex_get_group_entry(sw_desc, index); + if (addr) { + /* one destination */ + list_for_each_entry_from(iter, + &sw_desc->group_list, chain_node) + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_BASE, addr, 0); + } else { + /* two destinations */ + list_for_each_entry_from(iter, + &sw_desc->group_list, chain_node) { + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_BASE, paddr, 0); + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_BASE, qaddr, 1); + } + } + + if (index) { + /* To clear destinations update the descriptor + * (1st,2nd, or both depending on flags) + */ + index = 0; + if (test_bit(PPC460EX_ZERO_P, + &sw_desc->flags)) { + iter = ppc460ex_get_group_entry( + sw_desc, index++); + ppc460ex_adma_pq_zero_op(iter, chan, + paddr); + } + + if (test_bit(PPC460EX_ZERO_Q, + &sw_desc->flags)) { + iter = ppc460ex_get_group_entry( + sw_desc, index++); + ppc460ex_adma_pq_zero_op(iter, chan, + qaddr); + } + + return; + } + } else { + /* This is RXOR-only or RXOR/WXOR mixed chain */ + + /* If we want to include destination into calculations, + * then make dest addresses cued with mult=1 (XOR). + */ + ppath = test_bit(PPC460EX_ZERO_P, &sw_desc->flags) ? + DMA_CUED_XOR_HB : + DMA_CUED_XOR_BASE | + (1 << DMA_CUED_MULT1_OFF); + qpath = test_bit(PPC460EX_ZERO_Q, &sw_desc->flags) ? + DMA_CUED_XOR_HB : + DMA_CUED_XOR_BASE | + (1 << DMA_CUED_MULT1_OFF); + + /* Setup destination(s) in RXOR slot(s) */ + iter = ppc460ex_get_group_entry (sw_desc, index++); + ppc460ex_desc_set_dest_addr(iter, chan, + paddr ? ppath : qpath, + paddr ? paddr : qaddr, 0); + if (!addr) { + /* two destinations */ + iter = ppc460ex_get_group_entry (sw_desc, + index++); + ppc460ex_desc_set_dest_addr(iter, chan, + qpath, qaddr, 0); + } + + if (test_bit(PPC460EX_DESC_WXOR, &sw_desc->flags)) { + /* Setup destination(s) in remaining WXOR + * slots + */ + iter = ppc460ex_get_group_entry(sw_desc, + index); + if (addr) { + /* one destination */ + list_for_each_entry_from(iter, + &sw_desc->group_list, + chain_node) + ppc460ex_desc_set_dest_addr( + iter, chan, + DMA_CUED_XOR_BASE, + addr, 0); + + } else { + /* two destinations */ + list_for_each_entry_from(iter, + &sw_desc->group_list, + chain_node) { + ppc460ex_desc_set_dest_addr( + iter, chan, + DMA_CUED_XOR_BASE, + paddr, 0); + ppc460ex_desc_set_dest_addr( + iter, chan, + DMA_CUED_XOR_BASE, + qaddr, 1); + } + } + } + + } + break; + + case PPC460EX_XOR_ID: + /* DMA2 descriptors have only 1 destination, so there are + * two chains - one for each dest. + * If we want to include destination into calculations, + * then make dest addresses cued with mult=1 (XOR). + */ + ppath = test_bit(PPC460EX_ZERO_P, &sw_desc->flags) ? + DMA_CUED_XOR_HB : + DMA_CUED_XOR_BASE | + (1 << DMA_CUED_MULT1_OFF); + + qpath = test_bit(PPC460EX_ZERO_Q, &sw_desc->flags) ? + DMA_CUED_XOR_HB : + DMA_CUED_XOR_BASE | + (1 << DMA_CUED_MULT1_OFF); + + iter = ppc460ex_get_group_entry (sw_desc, 0); + for (i=0; i<sw_desc->descs_per_op; i++) { + ppc460ex_desc_set_dest_addr(iter, chan, + paddr ? ppath : qpath, + paddr ? paddr : qaddr, 0); + iter = list_entry (iter->chain_node.next, + ppc460ex_desc_t, chain_node); + } + + if (!addr) { + /* Two destinations; setup Q here */ + iter = ppc460ex_get_group_entry (sw_desc, + sw_desc->descs_per_op); + for (i=0; i<sw_desc->descs_per_op; i++) { + ppc460ex_desc_set_dest_addr(iter, + chan, qpath, qaddr, 0); + iter = list_entry (iter->chain_node.next, + ppc460ex_desc_t, chain_node); + } + } + + break; + } +} +void ppc460ex_adma_xor_set_dest(ppc460ex_desc_t *sw_desc, + dma_addr_t *addrs, unsigned long flags) +{ + ppc460ex_desc_t *iter; + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + dma_addr_t paddr, qaddr; + dma_addr_t addr = 0, ppath, qpath; + int index = 0; + + paddr = addrs[0]; + + + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + /* walk through the WXOR source list and set P/Q-destinations + * for each slot: + */ + if (!test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags)) { + /* This is WXOR-only chain; may have 1/2 zero descs */ + if (test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) + index++; + + iter = ppc460ex_get_group_entry(sw_desc, index); + if (paddr) { + /* one destination */ + list_for_each_entry_from(iter, + &sw_desc->group_list, chain_node) + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_BASE, paddr, 0); + } else { + BUG(); + /* two destinations */ + list_for_each_entry_from(iter, + &sw_desc->group_list, chain_node) { + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_BASE, paddr, 0); + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_BASE, qaddr, 1); + } + } + + if (index) { + /* To clear destinations update the descriptor + * (1st,2nd, or both depending on flags) + */ + index = 0; + if (test_bit(PPC460EX_ZERO_P, + &sw_desc->flags)) { + iter = ppc460ex_get_group_entry( + sw_desc, index++); + ppc460ex_adma_pq_zero_op(iter, chan, + paddr); + } + + return; + } + } else { + /* This is RXOR-only or RXOR/WXOR mixed chain */ + + /* If we want to include destination into calculations, + * then make dest addresses cued with mult=1 (XOR). + */ + ppath = test_bit(PPC460EX_ZERO_P, &sw_desc->flags) ? + DMA_CUED_XOR_HB : + DMA_CUED_XOR_BASE | + (1 << DMA_CUED_MULT1_OFF); + + /* Setup destination(s) in RXOR slot(s) */ + iter = ppc460ex_get_group_entry (sw_desc, index++); + ppc460ex_desc_set_dest_addr(iter, chan, + paddr ? ppath : qpath, + paddr ? paddr : qaddr, 0); + if (!addr) { + /* two destinations */ + iter = ppc460ex_get_group_entry (sw_desc, + index++); + ppc460ex_desc_set_dest_addr(iter, chan, + qpath, qaddr, 0); + } + + if (test_bit(PPC460EX_DESC_WXOR, &sw_desc->flags)) { + /* Setup destination(s) in remaining WXOR + * slots + */ + iter = ppc460ex_get_group_entry(sw_desc, + index); + if (addr) { + /* one destination */ + list_for_each_entry_from(iter, + &sw_desc->group_list, + chain_node) + ppc460ex_desc_set_dest_addr( + iter, chan, + DMA_CUED_XOR_BASE, + addr, 0); + + } else { + /* two destinations */ + list_for_each_entry_from(iter, + &sw_desc->group_list, + chain_node) { + ppc460ex_desc_set_dest_addr( + iter, chan, + DMA_CUED_XOR_BASE, + paddr, 0); + ppc460ex_desc_set_dest_addr( + iter, chan, + DMA_CUED_XOR_BASE, + qaddr, 1); + } + } + } + + } + break; + + } +} +/** + * ppc460ex_adma_dma01_xor_set_dest - set destination address into descriptor + * for the PQXOR operation + */ +static void ppc460ex_adma_dma01_xor_set_dest(ppc460ex_desc_t *sw_desc, + dma_addr_t addr, int index) +{ + ppc460ex_desc_t *iter; + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + + BUG_ON(index >= sw_desc->dst_cnt); + BUG_ON(test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags) && index); + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + /* walk through the WXOR source list and set P/Q-destinations + * for each slot: + */ + if (test_bit(PPC460EX_DESC_WXOR, &sw_desc->flags)) { + /* If this is RXOR/WXOR chain then dst_cnt == 1 + * and first WXOR descriptor is the second in RXOR/WXOR + * chain + */ + //if (!test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) { + if (!test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) { + iter = ppc460ex_get_group_entry(sw_desc, 0); + } else { + iter = ppc460ex_get_group_entry(sw_desc, + sw_desc->dst_cnt); + } + list_for_each_entry_from(iter, &sw_desc->group_list, + chain_node) { + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_BASE, addr, index); + } + if (!test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags) && + test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) { + /* In a WXOR-only case we probably has had + * a reasonable data at P/Q addresses, so + * the first operation in chain will be + * zeroing P/Q dest: + * WXOR (Q, 1*Q) -> 0. + * + * To do this (clear) update the descriptor + * (P or Q depending on index) as follows: + * addr is destination (0 corresponds to SG2): + */ + iter = ppc460ex_get_group_entry (sw_desc, + index); + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_BASE, addr, 0); + /* ... and the addr is source: */ + ppc460ex_desc_set_src_addr(iter, chan, 0, + DMA_CUED_XOR_HB, addr); + /* addr is always SG2 then the mult is always + DST1 */ + ppc460ex_desc_set_src_mult(iter, chan, + DMA_CUED_MULT1_OFF, DMA_CDB_SG_DST1, 1); + } + } + + if (test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags)) { + /* + * setup Q-destination for RXOR slot ( + * it shall be a HB address) + */ + iter = ppc460ex_get_group_entry (sw_desc, index); + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_HB, addr, 0); + } + break; + case PPC460EX_XOR_ID: + iter = ppc460ex_get_group_entry (sw_desc, index); + ppc460ex_desc_set_dest_addr(iter, chan, 0, addr, 0); + break; + } +} + +/** + * ppc460ex_adma_pq_zero_sum_set_dest - set destination address into descriptor + * for the PQ_VAL operation + */ +static void ppc460ex_adma_mq_zero_sum_set_dest ( + ppc460ex_desc_t *sw_desc, + dma_addr_t addr) +{ + ppc460ex_desc_t *iter, *end; + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + + + /* walk through the WXOR source list and set P/Q-destinations + * for each slot + */ + end = ppc460ex_get_group_entry(sw_desc, sw_desc->src_cnt); + list_for_each_entry(iter, &sw_desc->group_list, chain_node) { + if (unlikely(iter == end)) + break; + ppc460ex_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, + addr, 0); + } + /* The descriptors remain are DATACHECK. These have no need in + * destination. Actually, these destination are used there + * as a sources for check operation. So, set addr ass source. + */ + end = ppc460ex_get_group_entry(sw_desc, sw_desc->src_cnt ); + BUG_ON(!end); + ppc460ex_desc_set_src_addr(end, chan, 0, 0, addr); +} +static void ppc460ex_adma_pqzero_sum_set_dest ( + ppc460ex_desc_t *sw_desc, + dma_addr_t paddr, dma_addr_t qaddr) +{ + ppc460ex_desc_t *iter, *end; + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + dma_addr_t addr = 0; + + + /* walk through the WXOR source list and set P/Q-destinations + * for each slot + */ + end = ppc460ex_get_group_entry(sw_desc, sw_desc->src_cnt); + if(paddr && qaddr) { + list_for_each_entry(iter, &sw_desc->group_list, chain_node) { + if (unlikely(iter == end)) + break; + ppc460ex_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, + paddr, 0); + ppc460ex_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, + qaddr, 1); + } + } else { + /* one destination */ + addr = paddr ? paddr : qaddr; + list_for_each_entry(iter, &sw_desc->group_list, chain_node) { + if (unlikely(iter == end)) + break; + ppc460ex_desc_set_dest_addr(iter, chan, + DMA_CUED_XOR_BASE, addr, 0); + } + } + /* The descriptors remain are DATACHECK. These have no need in + * destination. Actually, these destination are used there + * as a sources for check operation. So, set addr ass source. + */ + end = ppc460ex_get_group_entry(sw_desc, sw_desc->src_cnt ); + ppc460ex_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); + if (!addr) { + end = ppc460ex_get_group_entry(sw_desc, sw_desc->src_cnt + 1); + ppc460ex_desc_set_src_addr(end, chan, 0, 0, qaddr); + } +} + +/** + * ppc460ex_desc_set_xor_src_cnt (ppc460ex_desc_t *desc, int src_cnt) + */ +static inline void ppc460ex_desc_set_xor_src_cnt (ppc460ex_desc_t *desc, + int src_cnt) +{ + xor_cb_t *hw_desc = desc->hw_desc; + hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; + hw_desc->cbc |= src_cnt; +} + +/** + * ppc460ex_adma_pqxor_set_src - set source address into descriptor + */ +static void ppc460ex_adma_pqxor_set_src( + ppc460ex_desc_t *sw_desc, + dma_addr_t addr, + int index) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + dma_addr_t haddr = 0; + ppc460ex_desc_t *iter; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain + */ + if (test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags)) { + /* RXOR-only or RXOR/WXOR operation */ + int iskip = test_bit(PPC460EX_DESC_RXOR12, + &sw_desc->flags) ? 2 : 3; + + if (index == 0) { + /* 1st slot (RXOR) */ + /* setup sources region (R1-2-3, R1-2-4, + or R1-2-5)*/ + if (test_bit(PPC460EX_DESC_RXOR12, + &sw_desc->flags)) + haddr = DMA_RXOR12 << + DMA_CUED_REGION_OFF; + else if (test_bit(PPC460EX_DESC_RXOR123, + &sw_desc->flags)) + haddr = DMA_RXOR123 << + DMA_CUED_REGION_OFF; + else if (test_bit(PPC460EX_DESC_RXOR124, + &sw_desc->flags)) + haddr = DMA_RXOR124 << + DMA_CUED_REGION_OFF; + else if (test_bit(PPC460EX_DESC_RXOR125, + &sw_desc->flags)) + haddr = DMA_RXOR125 << + DMA_CUED_REGION_OFF; + else + BUG(); + haddr |= DMA_CUED_XOR_BASE; + sw_desc = sw_desc->group_head; + } else if (index < iskip) { + /* 1st slot (RXOR) + * shall actually set source address only once + * instead of first <iskip> + */ + sw_desc = NULL; + } else { + /* second and next slots (WXOR); + * skip first slot with RXOR + */ + haddr = DMA_CUED_XOR_HB; + sw_desc = ppc460ex_get_group_entry(sw_desc, + index - iskip + 1); + } + } else { + /* WXOR-only operation; + * skip first slots with destinations + */ + haddr = DMA_CUED_XOR_HB; + if (!test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) + sw_desc = ppc460ex_get_group_entry(sw_desc, + index); + else + sw_desc = ppc460ex_get_group_entry(sw_desc, + sw_desc->dst_cnt + index); + } + + if (likely(sw_desc)) + ppc460ex_desc_set_src_addr(sw_desc, chan, index, haddr, + addr); + break; + case PPC460EX_XOR_ID: + /* DMA2 may do Biskup + */ + iter = sw_desc->group_head; + if (iter->dst_cnt == 2) { + /* both P & Q calculations required; set Q src here */ + ppc460ex_adma_dma2rxor_set_src(iter, index, addr); + /* this is for P. Actually sw_desc already points + * to the second CDB though. + */ + iter = ppc460ex_get_group_entry(sw_desc, + sw_desc->descs_per_op); + } + ppc460ex_adma_dma2rxor_set_src(iter, index, addr); + break; + } +} +void ppc460ex_adma_xor_set_src( + ppc460ex_desc_t *sw_desc, + dma_addr_t addr, + int index) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + dma_addr_t haddr = 0; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain + */ + if (test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags)) { + /* RXOR-only or RXOR/WXOR operation */ + int iskip = test_bit(PPC460EX_DESC_RXOR12, + &sw_desc->flags) ? 2 : 3; + + if (index == 0) { + /* 1st slot (RXOR) */ + /* setup sources region (R1-2-3, R1-2-4, + or R1-2-5)*/ + if (test_bit(PPC460EX_DESC_RXOR12, + &sw_desc->flags)) + haddr = DMA_RXOR12 << + DMA_CUED_REGION_OFF; + else if (test_bit(PPC460EX_DESC_RXOR123, + &sw_desc->flags)) + haddr = DMA_RXOR123 << + DMA_CUED_REGION_OFF; + else if (test_bit(PPC460EX_DESC_RXOR124, + &sw_desc->flags)) + haddr = DMA_RXOR124 << + DMA_CUED_REGION_OFF; + else if (test_bit(PPC460EX_DESC_RXOR125, + &sw_desc->flags)) + haddr = DMA_RXOR125 << + DMA_CUED_REGION_OFF; + else + BUG(); + haddr |= DMA_CUED_XOR_BASE; + sw_desc = sw_desc->group_head; + } else if (index < iskip) { + /* 1st slot (RXOR) + * shall actually set source address only once + * instead of first <iskip> + */ + sw_desc = NULL; + } else { + /* second and next slots (WXOR); + * skip first slot with RXOR + */ + haddr = DMA_CUED_XOR_HB; + sw_desc = ppc460ex_get_group_entry(sw_desc, + index - iskip + 1); + } + } else { + /* WXOR-only operation; + * skip first slots with destinations + */ + haddr = DMA_CUED_XOR_HB; + if (!test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) + sw_desc = ppc460ex_get_group_entry(sw_desc, + index); + else + sw_desc = ppc460ex_get_group_entry(sw_desc, + sw_desc->dst_cnt + index); + } + + if (likely(sw_desc)) + ppc460ex_desc_set_src_addr(sw_desc, chan, index, haddr, + addr); + break; + } +} +/** + * ppc460ex_adma_dma01_xor_set_src - set source address into descriptor + */ +static void ppc460ex_adma_dma01_xor_set_src( + ppc460ex_desc_t *sw_desc, + dma_addr_t addr, + int index) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + dma_addr_t haddr = 0; + ppc460ex_desc_t *iter; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain + */ + if (test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags)) { + /* RXOR-only or RXOR/WXOR operation */ + int iskip = test_bit(PPC460EX_DESC_RXOR12, + &sw_desc->flags) ? 2 : 3; + + if (index == 0) { + /* 1st slot (RXOR) */ + /* setup sources region (R1-2-3, R1-2-4, + or R1-2-5)*/ + if (test_bit(PPC460EX_DESC_RXOR12, + &sw_desc->flags)) + haddr = DMA_RXOR12 << + DMA_CUED_REGION_OFF; + else if (test_bit(PPC460EX_DESC_RXOR123, + &sw_desc->flags)) + haddr = DMA_RXOR123 << + DMA_CUED_REGION_OFF; + else if (test_bit(PPC460EX_DESC_RXOR124, + &sw_desc->flags)) + haddr = DMA_RXOR124 << + DMA_CUED_REGION_OFF; + else if (test_bit(PPC460EX_DESC_RXOR125, + &sw_desc->flags)) + haddr = DMA_RXOR125 << + DMA_CUED_REGION_OFF; + else + BUG(); + haddr |= DMA_CUED_XOR_BASE; + sw_desc = sw_desc->group_head; + } else if (index < iskip) { + /* 1st slot (RXOR) + * shall actually set source address only once + * instead of first <iskip> + */ + sw_desc = NULL; + } else { + /* second and next slots (WXOR); + * skip first slot with RXOR + */ + haddr = DMA_CUED_XOR_HB; + sw_desc = ppc460ex_get_group_entry(sw_desc, + index - iskip + 1); + } + } else { + /* WXOR-only operation; + * skip first slots with destinations + */ + haddr = DMA_CUED_XOR_HB; + if (!test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) { + sw_desc = ppc460ex_get_group_entry(sw_desc, + index); + } else { + sw_desc = ppc460ex_get_group_entry(sw_desc, + sw_desc->dst_cnt + index); + } + } + + if (likely(sw_desc)) { + ppc460ex_desc_set_src_addr(sw_desc, chan, index, haddr, + addr); + } + break; + case PPC460EX_XOR_ID: + /* DMA2 may do Biskup + */ + iter = sw_desc->group_head; + if (iter->dst_cnt == 2) { + /* both P & Q calculations required; set Q src here */ + ppc460ex_adma_dma2rxor_set_src(iter, index, addr); + /* this is for P. Actually sw_desc already points + * to the second CDB though. + */ + iter = ppc460ex_get_group_entry(sw_desc, + sw_desc->descs_per_op); + } + ppc460ex_adma_dma2rxor_set_src(iter, index, addr); + break; + } +} + +/** + * ppc460ex_adma_pqzero_sum_set_src - set source address into descriptor + */ +static void ppc460ex_adma_pqzero_sum_set_src( + ppc460ex_desc_t *sw_desc, + dma_addr_t addr, + int index) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + dma_addr_t haddr = DMA_CUED_XOR_HB; + + sw_desc = ppc460ex_get_group_entry(sw_desc, index); + + if (likely(sw_desc)) + ppc460ex_desc_set_src_addr(sw_desc, chan, index, haddr, addr); +} + +/** + * ppc460ex_adma_memcpy_xor_set_src - set source address into descriptor + */ +static inline void ppc460ex_adma_memcpy_xor_set_src( + ppc460ex_desc_t *sw_desc, + dma_addr_t addr, + int index) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + + sw_desc = sw_desc->group_head; + + if (likely(sw_desc)) + //ppc460ex_desc_set_src_addr(sw_desc, chan, index, 0x8, addr); // Enabling HB bus + ppc460ex_desc_set_src_addr(sw_desc, chan, index, 0, addr); +} + +/** + * ppc460ex_adma_dma2rxor_inc_addr - + */ +static void ppc460ex_adma_dma2rxor_inc_addr (ppc460ex_desc_t *desc, + ppc460ex_rxor_cursor_t *cursor, int index, int src_cnt) +{ + cursor->addr_count++; + if (index == src_cnt-1) { + ppc460ex_desc_set_xor_src_cnt (desc, + cursor->addr_count); + if (cursor->desc_count) { + ppc460ex_wxor_set_base (desc); + } + } else if (cursor->addr_count == XOR_MAX_OPS) { + ppc460ex_desc_set_xor_src_cnt (desc, + cursor->addr_count); + if (cursor->desc_count) { + ppc460ex_wxor_set_base (desc); + } + cursor->addr_count = 0; + cursor->desc_count++; + } +} + +/** + * ppc460ex_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB + */ +static int ppc460ex_adma_dma2rxor_prep_src (ppc460ex_desc_t *hdesc, + ppc460ex_rxor_cursor_t *cursor, int index, + int src_cnt, u32 addr) +{ + int rval = 0; + u32 sign; + ppc460ex_desc_t *desc = hdesc; + int i; + + for (i=0;i<cursor->desc_count;i++) { + desc = list_entry (hdesc->chain_node.next, ppc460ex_desc_t, + chain_node); + } + + switch (cursor->state) { + case 0: + if (addr == cursor->addrl + cursor->len ) { + /* direct RXOR */ + cursor->state = 1; + cursor->xor_count++; + if (index == src_cnt-1) { + ppc460ex_rxor_set_region (desc, + cursor->addr_count, + DMA_RXOR12 << + DMA_CUED_REGION_OFF); + ppc460ex_adma_dma2rxor_inc_addr ( + desc, cursor, index, src_cnt); + } + } else if (cursor->addrl == addr + cursor->len) { + /* reverse RXOR */ + cursor->state = 1; + cursor->xor_count++; + set_bit (cursor->addr_count, + &desc->reverse_flags[0]); + if (index == src_cnt-1) { + ppc460ex_rxor_set_region (desc, + cursor->addr_count, + DMA_RXOR12 << + DMA_CUED_REGION_OFF); + ppc460ex_adma_dma2rxor_inc_addr ( + desc, cursor, index, src_cnt); + } + } else { + printk (KERN_ERR "Cannot build " + "DMA2 RXOR command block.\n"); + BUG (); + } + break; + case 1: + sign = test_bit (cursor->addr_count, + desc->reverse_flags) + ? -1 : 1; + if (index == src_cnt-2 || (sign == -1 + && addr != cursor->addrl - 2*cursor->len)) { + cursor->state = 0; + cursor->xor_count = 1; + cursor->addrl = addr; + ppc460ex_rxor_set_region (desc, + cursor->addr_count, + DMA_RXOR12 << DMA_CUED_REGION_OFF); + ppc460ex_adma_dma2rxor_inc_addr ( + desc, cursor, index, src_cnt); + } else if (addr == cursor->addrl + 2*sign*cursor->len) { + cursor->state = 2; + cursor->xor_count = 0; + ppc460ex_rxor_set_region (desc, + cursor->addr_count, + DMA_RXOR123 << DMA_CUED_REGION_OFF); + if (index == src_cnt-1) { + ppc460ex_adma_dma2rxor_inc_addr ( + desc, cursor, index, src_cnt); + } + } else if (addr == cursor->addrl + 3*cursor->len) { + cursor->state = 2; + cursor->xor_count = 0; + ppc460ex_rxor_set_region (desc, + cursor->addr_count, + DMA_RXOR124 << DMA_CUED_REGION_OFF); + if (index == src_cnt-1) { + ppc460ex_adma_dma2rxor_inc_addr ( + desc, cursor, index, src_cnt); + } + } else if (addr == cursor->addrl + 4*cursor->len) { + cursor->state = 2; + cursor->xor_count = 0; + ppc460ex_rxor_set_region (desc, + cursor->addr_count, + DMA_RXOR125 << DMA_CUED_REGION_OFF); + if (index == src_cnt-1) { + ppc460ex_adma_dma2rxor_inc_addr ( + desc, cursor, index, src_cnt); + } + } else { + cursor->state = 0; + cursor->xor_count = 1; + cursor->addrl = addr; + ppc460ex_rxor_set_region (desc, + cursor->addr_count, + DMA_RXOR12 << DMA_CUED_REGION_OFF); + ppc460ex_adma_dma2rxor_inc_addr ( + desc, cursor, index, src_cnt); + } + break; + case 2: + cursor->state = 0; + cursor->addrl = addr; + cursor->xor_count++; + if (index) { + ppc460ex_adma_dma2rxor_inc_addr ( + desc, cursor, index, src_cnt); + } + break; + } + + return rval; +} + +/** + * ppc460ex_adma_dma2rxor_set_src - set RXOR source address; it's assumed that + * ppc460ex_adma_dma2rxor_prep_src() has already done prior this call + */ +static void ppc460ex_adma_dma2rxor_set_src (ppc460ex_desc_t *desc, + int index, dma_addr_t addr) +{ + xor_cb_t *xcb = desc->hw_desc; + int k = 0, op = 0, lop = 0; + + /* get the RXOR operand which corresponds to index addr */ + while (op <= index) { + lop = op; + if (k == XOR_MAX_OPS) { + k = 0; + desc = list_entry (desc->chain_node.next, + ppc460ex_desc_t, chain_node); + xcb = desc->hw_desc; + + } + if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == + (DMA_RXOR12 << DMA_CUED_REGION_OFF)) + op += 2; + else + op += 3; + } + + if (test_bit(/*PPC460EX_DESC_RXOR_REV*/k-1, desc->reverse_flags)) { + /* reverse operand order; put last op in RXOR group */ + if (index == op - 1) + ppc460ex_rxor_set_src(desc, k - 1, addr); + } else { + /* direct operand order; put first op in RXOR group */ + if (index == lop) + ppc460ex_rxor_set_src(desc, k - 1, addr); + } +} + +/** + * ppc460ex_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that + * ppc460ex_adma_dma2rxor_prep_src() has already done prior this call + */ +static void ppc460ex_adma_dma2rxor_set_mult (ppc460ex_desc_t *desc, + int index, u8 mult) +{ + xor_cb_t *xcb = desc->hw_desc; + int k = 0, op = 0, lop = 0; + + /* get the RXOR operand which corresponds to index mult */ + while (op <= index) { + lop = op; + if (k == XOR_MAX_OPS) { + k = 0; + desc = list_entry (desc->chain_node.next, + ppc460ex_desc_t, chain_node); + xcb = desc->hw_desc; + + } + if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == + (DMA_RXOR12 << DMA_CUED_REGION_OFF)) + op += 2; + else + op += 3; + } + + if (test_bit(/*PPC460EX_DESC_RXOR_REV*/k-1, desc->reverse_flags)) { + /* reverse order */ + ppc460ex_rxor_set_mult(desc, k - 1, op - index - 1, mult); + } else { + /* direct order */ + ppc460ex_rxor_set_mult(desc, k - 1, index - lop, mult); + } +} + +/** + * ppc460ex_init_rxor_cursor - + */ +static void ppc460ex_init_rxor_cursor (ppc460ex_rxor_cursor_t *cursor) +{ + memset (cursor, 0, sizeof (ppc460ex_rxor_cursor_t)); + cursor->state = 2; +} + +/** + * ppc460ex_adma_pqxor_set_src_mult - set multiplication coefficient into + * descriptor for the PQXOR operation + */ +static void ppc460ex_adma_pqxor_set_src_mult ( + ppc460ex_desc_t *sw_desc, + unsigned char mult, int index,int dst_pos) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + u32 mult_idx, mult_dst; + ppc460ex_desc_t *iter=NULL, *iter1=NULL; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + if (test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags)) { + int region = test_bit(PPC460EX_DESC_RXOR12, + &sw_desc->flags) ? 2 : 3; + + if (index < region) { + /* RXOR multipliers */ + + iter = ppc460ex_get_group_entry(sw_desc, + sw_desc->dst_cnt - 1); + if (sw_desc->dst_cnt == 2) + iter1 = ppc460ex_get_group_entry(sw_desc, 0); + mult_idx = DMA_CUED_MULT1_OFF + (index << 3); + mult_dst = DMA_CDB_SG_SRC; + } else { + /* WXOR multiplier */ + iter = ppc460ex_get_group_entry(sw_desc, + index - region + 1); + mult_idx = DMA_CUED_MULT1_OFF; + mult_dst = dst_pos ? DMA_CDB_SG_DST2 : + DMA_CDB_SG_DST1; + } + } else { + int znum = 0; + + /* WXOR-only; + * skip first slots with destinations (if ZERO_DST has + * place) + */ + if (test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) + znum++; + if (test_bit(PPC460EX_ZERO_Q, &sw_desc->flags)) + znum++; + iter = ppc460ex_get_group_entry(sw_desc, index + znum); + mult_idx = DMA_CUED_MULT1_OFF; + mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; + } + + if (likely(sw_desc)) + ppc460ex_desc_set_src_mult(iter, chan, + mult_idx, mult_dst, mult); + break; + case PPC460EX_XOR_ID: + iter = sw_desc->group_head; + if (iter->dst_cnt == 2) { + /* both P & Q calculations required; set Q mult here */ + ppc460ex_adma_dma2rxor_set_mult(iter, index, mult); + /* this is for P. Actually sw_desc already points + * to the second CDB though. + */ + mult = 1; + iter = ppc460ex_get_group_entry(sw_desc, + sw_desc->descs_per_op); + } + ppc460ex_adma_dma2rxor_set_mult(iter, index, mult); + break; + } +} +void ppc460ex_adma_xor_set_src_mult ( + ppc460ex_desc_t *sw_desc, + unsigned char mult, int index,int dst_pos) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + u32 mult_idx, mult_dst; + ppc460ex_desc_t *iter=NULL, *iter1=NULL; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + if (test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags)) { + int region = test_bit(PPC460EX_DESC_RXOR12, + &sw_desc->flags) ? 2 : 3; + + if (index < region) { + /* RXOR multipliers */ + + iter = ppc460ex_get_group_entry(sw_desc, + sw_desc->dst_cnt - 1); + if (sw_desc->dst_cnt == 2) + iter1 = ppc460ex_get_group_entry(sw_desc, 0); + mult_idx = DMA_CUED_MULT1_OFF + (index << 3); + mult_dst = DMA_CDB_SG_SRC; + } else { + /* WXOR multiplier */ + iter = ppc460ex_get_group_entry(sw_desc, + index - region + 1); + mult_idx = DMA_CUED_MULT1_OFF; + mult_dst = dst_pos ? DMA_CDB_SG_DST2 : + DMA_CDB_SG_DST1; + } + } else { + int znum = 0; + + /* WXOR-only; + * skip first slots with destinations (if ZERO_DST has + * place) + */ + if (test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) + znum++; + if (test_bit(PPC460EX_ZERO_Q, &sw_desc->flags)) + znum++; + iter = ppc460ex_get_group_entry(sw_desc, index + znum); + mult_idx = DMA_CUED_MULT1_OFF; + mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; + } + + if (likely(sw_desc)) + ppc460ex_desc_set_src_mult(iter, chan, + mult_idx, mult_dst, mult); + break; + } +} +/** + * ppc460ex_adma_pqxor_set_src_mult - set multiplication coefficient into + * descriptor for the PQXOR operation + */ +static void ppc460ex_adma_dma01_xor_set_src_mult ( + ppc460ex_desc_t *sw_desc, + unsigned char mult, int index) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + u32 mult_idx, mult_dst; + + switch (chan->device->id) { + case PPC460EX_DMA0_ID: + case PPC460EX_DMA1_ID: + if (test_bit(PPC460EX_DESC_RXOR, &sw_desc->flags)) { + int region = test_bit(PPC460EX_DESC_RXOR12, + &sw_desc->flags) ? 2 : 3; + + if (index < region) { + /* RXOR multipliers */ + sw_desc = ppc460ex_get_group_entry(sw_desc, 0); + mult_idx = DMA_CUED_MULT1_OFF + (index << 3); + mult_dst = DMA_CDB_SG_SRC; + } else { + /* WXOR multiplier */ + sw_desc = ppc460ex_get_group_entry(sw_desc, + index - region + 1); + mult_idx = DMA_CUED_MULT1_OFF; + mult_dst = DMA_CDB_SG_DST1; + } + } else { + /* WXOR-only; + * skip first slots with destinations (if ZERO_DST has + * place) + */ + if (!test_bit(PPC460EX_ZERO_P, &sw_desc->flags)) { + sw_desc = ppc460ex_get_group_entry(sw_desc, + index); + } else { + sw_desc = ppc460ex_get_group_entry(sw_desc, + sw_desc->dst_cnt + index); + } + mult_idx = DMA_CUED_MULT1_OFF; + mult_dst = DMA_CDB_SG_DST1; + } + + if (likely(sw_desc)) { + ppc460ex_desc_set_src_mult(sw_desc, chan, + mult_idx, mult_dst, mult); + } + break; + } +} + +/** + * ppc460ex_adma_pqzero_sum_set_src_mult - set multiplication coefficient + * into descriptor for the PQ_VAL operation + */ +static void ppc460ex_adma_pqzero_sum_set_src_mult ( + ppc460ex_desc_t *sw_desc, + unsigned char mult, int index, int dst_pos) +{ + ppc460ex_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + u32 mult_idx, mult_dst; + + /* set mult for sources only */ + BUG_ON(index >= sw_desc->src_cnt); + + /* get pointed slot */ + sw_desc = ppc460ex_get_group_entry(sw_desc, index); + + mult_idx = DMA_CUED_MULT1_OFF; + mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; + + if (likely(sw_desc)) + ppc460ex_desc_set_src_mult(sw_desc, chan, mult_idx, mult_dst, + mult); +} +#if 0 +/** + * ppc460ex_adma_dependency_added - schedule clean-up + */ +static void ppc460ex_adma_dependency_added(struct dma_chan *chan) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + tasklet_schedule(&ppc460ex_chan->irq_tasklet); +} +#endif + +/** + * ppc460ex_adma_free_chan_resources - free the resources allocated + */ +static void ppc460ex_adma_free_chan_resources(struct dma_chan *chan) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_desc_t *iter, *_iter; + int in_use_descs = 0; + + ppc460ex_adma_slot_cleanup(ppc460ex_chan); + + spin_lock_bh(&ppc460ex_chan->lock); + list_for_each_entry_safe(iter, _iter, &ppc460ex_chan->chain, + chain_node) { + in_use_descs++; + list_del(&iter->chain_node); + } + list_for_each_entry_safe_reverse(iter, _iter, + &ppc460ex_chan->all_slots, slot_node) { + list_del(&iter->slot_node); + kfree(iter); + ppc460ex_chan->slots_allocated--; + } + ppc460ex_chan->last_used = NULL; + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d %s slots_allocated %d\n", + ppc460ex_chan->device->id, + __FUNCTION__, ppc460ex_chan->slots_allocated); + spin_unlock_bh(&ppc460ex_chan->lock); + + /* one is ok since we left it on there on purpose */ + if (in_use_descs > 1) + printk(KERN_ERR "GT: Freeing %d in use descriptors!\n", + in_use_descs - 1); +} + +/** + * ppc460ex_adma_is_complete - poll the status of an ADMA transaction + * @chan: ADMA channel handle + * @cookie: ADMA transaction identifier + */ +static enum dma_status ppc460ex_adma_is_complete(struct dma_chan *chan, + dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + dma_cookie_t last_used; + dma_cookie_t last_complete; + enum dma_status ret; + + //printk( "--------------- %s: %i-------------------------\n",__FUNCTION__,__LINE__); + last_used = chan->cookie; + last_complete = ppc460ex_chan->completed_cookie; + + if (done) + *done= last_complete; + if (used) + *used = last_used; + + ret = dma_async_is_complete(cookie, last_complete, last_used); + if (ret == DMA_SUCCESS) + return ret; + + ppc460ex_adma_slot_cleanup(ppc460ex_chan); + + last_used = chan->cookie; + last_complete = ppc460ex_chan->completed_cookie; + + if (done) + *done= last_complete; + if (used) + *used = last_used; + + return dma_async_is_complete(cookie, last_complete, last_used); +} + +/** + * ppc460ex_adma_eot_handler - end of transfer interrupt handler + */ +static irqreturn_t ppc460ex_adma_eot_handler(int irq, void *data) +{ + ppc460ex_ch_t *chan = data; + + dev_dbg(chan->device->common.dev, + "ppc460ex adma%d: %s\n", chan->device->id, __FUNCTION__); + + tasklet_schedule(&chan->irq_tasklet); + ppc460ex_adma_device_clear_eot_status(chan); + + return IRQ_HANDLED; +} + +/** + * ppc460ex_adma_err_handler - DMA error interrupt handler; + * do the same things as a eot handler + */ +static irqreturn_t ppc460ex_adma_err_handler(int irq, void *data) +{ + ppc460ex_ch_t *chan = data; + dev_dbg(chan->device->common.dev, + "ppc460ex adma%d: %s\n", chan->device->id, __FUNCTION__); + tasklet_schedule(&chan->irq_tasklet); + ppc460ex_adma_device_clear_eot_status(chan); + + return IRQ_HANDLED; +} + +static void ppc460ex_test_rad6_callback (void *unused) +{ + complete(&ppc460ex_r6_test_comp); +} +/** + * ppc460ex_test_callback - called when test operation has been done + */ +static void ppc460ex_test_callback (void *unused) +{ + complete(&ppc460ex_r5_test_comp); +} + +/** + * ppc460ex_adma_issue_pending - flush all pending descriptors to h/w + */ +static void ppc460ex_adma_issue_pending(struct dma_chan *chan) +{ + ppc460ex_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); +#if 0 + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s %d \n", ppc460ex_chan->device->id, + __FUNCTION__, ppc460ex_chan->pending); +#endif + + if (ppc460ex_chan->pending) { + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s %d \n", ppc460ex_chan->device->id, + __FUNCTION__, ppc460ex_chan->pending); + ppc460ex_chan->pending = 0; + ppc460ex_chan_append(ppc460ex_chan); + } +} + +/** + * ppc460ex_adma_remove - remove the asynch device + */ +static int __devexit ppc460ex_adma_remove(struct platform_device *dev) +{ + ppc460ex_dev_t *device = platform_get_drvdata(dev); + struct dma_chan *chan, *_chan; + struct ppc_dma_chan_ref *ref, *_ref; + ppc460ex_ch_t *ppc460ex_chan; + int i; + ppc460ex_aplat_t *plat_data = dev->dev.platform_data; + + dma_async_device_unregister(&device->common); + + for (i = 0; i < 3; i++) { + u32 irq; + irq = platform_get_irq(dev, i); + free_irq(irq, device); + } + + if ( (ppc460ex_chan) && (ppc460ex_chan->device->desc_memory == ADMA_DESC_MEM_OCM)) + ocm_free(device->dma_desc_pool_virt); + else + dma_free_coherent(&dev->dev, plat_data->pool_size, + device->dma_desc_pool_virt, device->dma_desc_pool); + + + do { + struct resource *res; + res = platform_get_resource(dev, IORESOURCE_MEM, 0); + release_mem_region(res->start, res->end - res->start); + } while (0); + + list_for_each_entry_safe(chan, _chan, &device->common.channels, + device_node) { + ppc460ex_chan = to_ppc460ex_adma_chan(chan); + list_del(&chan->device_node); + kfree(ppc460ex_chan); + } + + list_for_each_entry_safe(ref, _ref, &ppc_adma_chan_list, node) { + list_del(&ref->node); + kfree(ref); + } + + kfree(device); + + return 0; +} + +/** + * ppc460ex_adma_probe - probe the asynch device + */ +static int __devinit ppc460ex_adma_probe(struct of_device *ofdev, + const struct of_device_id *match) +{ + struct resource *res; + int ret=0, irq; + ppc460ex_dev_t *adev; + ppc460ex_ch_t *chan; + ppc460ex_aplat_t *plat_data; + struct ppc_dma_chan_ref *ref; + const char *str_prop; + + printk("Improved ADMA - 08312009\n"); + plat_data = &ppc460ex_dma_1_data; + if ((adev = kzalloc(sizeof(*adev), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_adev_alloc; + } + adev->res[0].start = ppc460ex_dma_1_channel.resource[0].start; + adev->id = ppc460ex_dma_1_channel.id; + printk("adev->res[0].start=0x%x\n",adev->res[0].start); + + printk( " adev->id = 0x%x ppc460ex_dma_1_channel.resource[0].start=0x%x \n", + adev->id,ppc460ex_dma_1_channel.resource[0].start); + + /* allocate coherent memory for hardware descriptors + * note: writecombine gives slightly better performance, but + * requires that we explicitly drain the write buffer + */ + str_prop = of_get_property(ofdev->node, "descriptor-memory", NULL); + if (str_prop && (!strcmp(str_prop,"ocm") || !strcmp(str_prop,"OCM"))) { + printk(KERN_INFO + " descriptor-memory = %s\n", str_prop); + adev->dma_desc_pool_virt = ocm_alloc(&adev->dma_desc_pool, DMA1_FIFO_SIZE << 2, 4, + OCM_NON_CACHED, "ADMA_descriptors"); + adev->desc_memory = ADMA_DESC_MEM_OCM; + } else { + + if ((adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, + DMA1_FIFO_SIZE << 2, &adev->dma_desc_pool, GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_dma_alloc; + } + adev->desc_memory = 0; + } + if (adev->dma_desc_pool_virt == NULL) { + ret = -ENOMEM; + goto err_dma_alloc; + } + dev_dbg(&ofdev->dev, "%s: allocted descriptor pool virt %p phys %p\n", + __FUNCTION__, adev->dma_desc_pool_virt, + (void *) adev->dma_desc_pool); + + adev->id = PPC460EX_DMA1_ID; + /* create the DMA capability MASK . This used to come from resources structure*/ + dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); + dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); + dma_cap_set(DMA_MEMSET, adev->common.cap_mask); + dma_cap_set(DMA_PQ, adev->common.cap_mask); + //dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); + dma_cap_set(DMA_XOR, adev->common.cap_mask);/* Marri RAID-5 */ + dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); + adev->odev = ofdev; + dev_set_drvdata(&(ofdev->dev), adev); + + INIT_LIST_HEAD(&adev->common.channels); + + /* set base routines */ + adev->common.device_alloc_chan_resources = + ppc460ex_adma_alloc_chan_resources; + adev->common.device_free_chan_resources = + ppc460ex_adma_free_chan_resources; + adev->common.device_is_tx_complete = ppc460ex_adma_is_complete; + adev->common.device_issue_pending = ppc460ex_adma_issue_pending; + adev->common.dev = &ofdev->dev; + + /* set prep routines based on capability */ +#if 1 + if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { + adev->common.device_prep_dma_memcpy = + ppc460ex_adma_prep_dma_memcpy; + } +#endif + if (dma_has_cap(DMA_MEMSET, adev->common.cap_mask)) { + adev->common.device_prep_dma_memset = + ppc460ex_adma_prep_dma_memset; + } +#if 1 + if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { + adev->common.max_xor = XOR_MAX_OPS; + adev->common.device_prep_dma_xor = ppc460ex_adma_prep_dma_mq_xor; + } +#endif +#if 1 + if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { + adev->common.max_xor = XOR_MAX_OPS; + adev->common.device_prep_dma_xor_val = ppc460ex_adma_prep_dma_mq_zero_sum; + } +#endif + if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { + switch (adev->id) { + case PPC460EX_DMA1_ID: + adev->common.max_pq = DMA1_FIFO_SIZE / + sizeof(dma_cdb_t); + break; + } + adev->common.device_prep_dma_pq = + ppc460ex_adma_prep_dma_pq; + + } + if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { + switch (adev->id) { + case PPC460EX_DMA1_ID: + adev->common.max_pq = DMA1_FIFO_SIZE / + sizeof(dma_cdb_t); + break; + } + adev->common.device_prep_dma_pq_val = + ppc460ex_adma_prep_dma_pqzero_sum; + } + + if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { + adev->common.device_prep_dma_interrupt = + ppc460ex_adma_prep_dma_interrupt; + } + + /* create a channel */ + if ((chan = kzalloc(sizeof(*chan), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_chan_alloc; + } + + tasklet_init(&chan->irq_tasklet, ppc460ex_adma_tasklet, + (unsigned long)chan); + irq = irq_of_parse_and_map(ofdev->node, 0); + printk("<%s> irq=0x%x\n",__FUNCTION__, irq); + if (irq >= 0) { + ret = request_irq(irq, ppc460ex_adma_eot_handler, + IRQF_DISABLED, "adma-compl", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + + irq = irq_of_parse_and_map(ofdev->node, 2); + printk("<%s> irq=0x%x\n",__FUNCTION__, irq); + if (irq >= 0) { + ret = request_irq(irq, ppc460ex_adma_err_handler, + IRQF_DISABLED, "adma-err", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + } + } else + ret = -ENXIO; + + chan->device = adev; + + /* pass the platform data */ + chan->device->odev->dev.platform_data = &ppc460ex_dma_1_data; + spin_lock_init(&chan->lock); +#if 0 + init_timer(&chan->cleanup_watchdog); + chan->cleanup_watchdog.data = (unsigned long) chan; + chan->cleanup_watchdog.function = ppc460ex_adma_tasklet; +#endif + INIT_LIST_HEAD(&chan->chain); + INIT_LIST_HEAD(&chan->all_slots); + chan->common.device = &adev->common; + list_add_tail(&chan->common.device_node, &adev->common.channels); + + dev_dbg(&ofdev->dev, "AMCC(R) PPC460 ADMA Engine found [%d]: " + "( %s%s%s%s%s%s%s%s)\n", + adev->id, + dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq_xor " : "", + dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_zero_sum " : + "", + dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", + dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_zero_sum " : + "", + dma_has_cap(DMA_MEMSET, adev->common.cap_mask) ? "memset " : "", + dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", + dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "int " : ""); + + dma_async_device_register(&adev->common); + ref = kmalloc(sizeof(*ref), GFP_KERNEL); + printk("<%s> ret=0x%x\n", __FUNCTION__,ret); + if (ref) { + ref->chan = &chan->common; + INIT_LIST_HEAD(&ref->node); + list_add_tail(&ref->node, &ppc_adma_chan_list); + } else + printk(KERN_WARNING "%s: failed to allocate channel reference!\n", + __FUNCTION__); + goto out; + +err_irq: + free_irq(irq,&adev->id); +err_chan_alloc: + kfree(chan); +err_dma_alloc: + if ( chan->device->desc_memory == ADMA_DESC_MEM_OCM) + ocm_free(adev->dma_desc_pool_virt ); + else + dma_free_coherent(&ofdev->dev,DMA1_FIFO_SIZE << 2, + adev->dma_desc_pool_virt, adev->dma_desc_pool); +err_adev_alloc: + kfree(adev); + release_mem_region(res->start, res->end - res->start); +out: + return ret; +} + +/** + * ppc460ex_chan_start_null_xor - initiate the first XOR operation (DMA engines + * use FIFOs (as opposite to chains used in XOR) so this is a XOR + * gtcific operation) + */ +static void ppc460ex_chan_start_null_xor(ppc460ex_ch_t *chan) +{ + ppc460ex_desc_t *sw_desc, *group_start; + dma_cookie_t cookie; + int slot_cnt, slots_per_op; + + dev_dbg(chan->device->common.dev, + "ppc460ex adma%d: %s\n", chan->device->id, __FUNCTION__); + + spin_lock_bh(&chan->lock); + slot_cnt = ppc460ex_chan_xor_slot_count(0, 2, &slots_per_op); + sw_desc = ppc460ex_adma_alloc_slots(chan, slot_cnt, slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + list_splice_init(&sw_desc->group_list, &chan->chain); + async_tx_ack(&sw_desc->async_tx); + ppc460ex_desc_init_null_xor(group_start); + + cookie = chan->common.cookie; + cookie++; + if (cookie <= 1) + cookie = 2; + + /* initialize the completed cookie to be less than + * the most recently used cookie + */ + chan->completed_cookie = cookie - 1; + chan->common.cookie = sw_desc->async_tx.cookie = cookie; + + /* channel should not be busy */ + BUG_ON(ppc460ex_chan_is_busy(chan)); + + /* set the descriptor address */ + ppc460ex_chan_set_first_xor_descriptor(chan, sw_desc); + + /* run the descriptor */ + ppc460ex_chan_run(chan); + } else + printk(KERN_ERR "ppc460ex adma%d" + " failed to allocate null descriptor\n", + chan->device->id); + spin_unlock_bh(&chan->lock); +} + +/** + * ppc460ex_test_raid6 - test are RAID-6 capabilities enabled successfully. + * For this we just perform one WXOR operation with the same source + * and destination addresses, the GF-multiplier is 1; so if RAID-6 + o/of_platform_driver_unregister(&ppc460ex_adma_driver); + * capabilities are enabled then we'll get src/dst filled with zero. + */ +static int ppc460ex_test_raid6 (ppc460ex_ch_t *chan) +{ + ppc460ex_desc_t *sw_desc, *iter; + struct page *pg; + char *a; + dma_addr_t dma_addr, addrs[2];; + unsigned long op = 0; + int rval = 0; + + if (!ppc460ex_r6_tchan) + return -1; + /*FIXME*/ + + set_bit(PPC460EX_DESC_WXOR, &op); + + pg = alloc_page(GFP_KERNEL); + if (!pg) + return -ENOMEM; + + spin_lock_bh(&chan->lock); + sw_desc = ppc460ex_adma_alloc_slots(chan, 1, 1); + if (sw_desc) { + /* 1 src, 1 dsr, int_ena, WXOR */ + ppc460ex_desc_init_pq(sw_desc, 1, 1, 1, op); + list_for_each_entry(iter, &sw_desc->group_list, chain_node) { + ppc460ex_desc_set_byte_count(iter, chan, PAGE_SIZE); + iter->unmap_len = PAGE_SIZE; + } + } else { + rval = -EFAULT; + spin_unlock_bh(&chan->lock); + goto exit; + } + spin_unlock_bh(&chan->lock); + + /* Fill the test page with ones */ + memset(page_address(pg), 0xFF, PAGE_SIZE); +#if 0 + int i = 0; + char *pg_addr = page_address(pg); + //for(i=0;i < PAGE_SIZE; i+=64) + for(i=0;i < 1000; i+=64) + printk("addr = 0x%x data = 0x%x\n",pg_addr + i,*(pg_addr+i)); +#endif + dma_addr = dma_map_page(&chan->device->odev->dev, pg, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + + /* Setup adresses */ + ppc460ex_adma_pqxor_set_src(sw_desc, dma_addr, 0); + ppc460ex_adma_pqxor_set_src_mult(sw_desc, 1, 0,0); + addrs[0] = dma_addr; + addrs[1] = 0; + ppc460ex_adma_pqxor_set_dest(sw_desc, addrs, DMA_PREP_HAVE_P); + + async_tx_ack(&sw_desc->async_tx); + sw_desc->async_tx.callback = ppc460ex_test_rad6_callback; + sw_desc->async_tx.callback_param = NULL; + + init_completion(&ppc460ex_r6_test_comp); + + ppc460ex_adma_tx_submit(&sw_desc->async_tx); + ppc460ex_adma_issue_pending(&chan->common); + + wait_for_completion(&ppc460ex_r6_test_comp); + + /* Now check is the test page zeroed */ + a = page_address(pg); +#if 0 + i = 0; + for(i=0;i < PAGE_SIZE; i+=64) + printk("addr = 0x%x data = 0x%x\n",a + i,*(a+i)); +#endif + if ((*(u32*)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4)==0) { + /* page is zero - RAID-6 enabled */ + rval = 0; + } else { + /* RAID-6 was not enabled */ + rval = -EINVAL; + } + //printk(__LINE__,__FUNCTION__); +exit: + __free_page(pg); + return rval; +} +/** + * ppc460ex_test_raid5 - test are RAID-5 capabilities enabled successfully. + * For this we just perform one WXOR operation with the same source + * and destination addresses, the GF-multiplier is 1; so if RAID-5 + o/of_platform_driver_unregister(&ppc460ex_adma_driver); + * capabilities are enabled then we'll get src/dst filled with zero. + */ +static int ppc460ex_test_raid5 (ppc460ex_ch_t *chan) +{ + ppc460ex_desc_t *sw_desc, *iter; + struct page *pg; + char *a; + dma_addr_t dma_addr; + unsigned long op = 0; + int rval = 0; + + if (!ppc460ex_r5_tchan) + return -1; + /*FIXME*/ + + set_bit(PPC460EX_DESC_WXOR, &op); + + pg = alloc_page(GFP_KERNEL); + if (!pg) + return -ENOMEM; + + spin_lock_bh(&chan->lock); + sw_desc = ppc460ex_adma_alloc_slots(chan, 1, 1); + if (sw_desc) { + /* 1 src, 1 dsr, int_ena, WXOR */ + //ppc460ex_desc_init_pqxor(sw_desc, 1, 1, 1, op); + ppc460ex_desc_init_dma01_xor(sw_desc, 1, 1, 1, op); + list_for_each_entry(iter, &sw_desc->group_list, chain_node) { + ppc460ex_desc_set_byte_count(iter, chan, PAGE_SIZE); + iter->unmap_len = PAGE_SIZE; + } + } else { + rval = -EFAULT; + spin_unlock_bh(&chan->lock); + goto exit; + } + spin_unlock_bh(&chan->lock); + + /* Fill the test page with ones */ + memset(page_address(pg), 0xFF, PAGE_SIZE); +#if 0 + int i = 0; + char *pg_addr = page_address(pg); + for(i=0;i < PAGE_SIZE; i+=64) + printk("addr = 0x%x data = 0x%x\n",pg_addr + i,*(pg_addr+i)); +#endif + //dma_addr = dma_map_page(&chan->device->common, pg, 0, PAGE_SIZE, + dma_addr = dma_map_page(&chan->device->odev->dev, pg, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + + /* Setup adresses */ + ppc460ex_adma_dma01_xor_set_src(sw_desc, dma_addr, 0); + ppc460ex_adma_dma01_xor_set_src_mult(sw_desc, 1, 0); + ppc460ex_adma_dma01_xor_set_dest(sw_desc, dma_addr, 0); + + async_tx_ack(&sw_desc->async_tx); + sw_desc->async_tx.callback = ppc460ex_test_callback; + sw_desc->async_tx.callback_param = NULL; + + init_completion(&ppc460ex_r5_test_comp); + + ppc460ex_adma_tx_submit(&sw_desc->async_tx); + ppc460ex_adma_issue_pending(&chan->common); + + wait_for_completion(&ppc460ex_r5_test_comp); + + /*Make sure cache is flushed to memory*/ + dma_addr = dma_map_page(&chan->device->odev->dev, pg, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + /* Now check is the test page zeroed */ + a = page_address(pg); +#if 0 + i = 0; + for(i=0;i < PAGE_SIZE; i+=64) + printk("addr = 0x%x data = 0x%x\n",a + i,*(a+i)); +#endif + if ((*(u32*)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4)==0) { + /* page is zero - RAID-5 enabled */ + rval = 0; + } else { + /* RAID-5 was not enabled */ + rval = -EINVAL; + } + pr_dma(__LINE__,__FUNCTION__); +exit: + __free_page(pg); + return rval; +} +#if 1 +static struct of_device_id adma_match[] = +{ + { + .compatible = "amcc,adma", + }, + {}, +}; +static struct of_platform_driver ppc460ex_adma_driver = { + .name = "adma", + .match_table = adma_match, + + .probe = ppc460ex_adma_probe, + .remove = ppc460ex_adma_remove, +}; +#else +static struct platform_driver ppc460ex_adma_driver= { + .probe = ppc460ex_adma_probe, + .remove = ppc460ex_adma_remove, + .driver = { + .owner = "marri", + .name = "PPC460EX-ADMA", + }, +}; +#endif + +/** + * /proc interface + */ +static int ppc460ex_poly_read (char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + char *p = page; + u32 reg; + +#ifdef CONFIG_440SP + /* 440SP has fixed polynomial */ + reg = 0x4d; +#else + reg = mfdcr(DCRN_MQ0_CFBHL); + reg >>= MQ0_CFBHL_POLY; + reg &= 0xFF; +#endif + + p += sprintf (p, "PPC460EX RAID-5 driver uses 0x1%02x polynomial.\n", + reg); + + return p - page; +} + +static int ppc460ex_poly_write (struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + /* e.g., 0x14D or 0x11d */ + char tmp[6]; + unsigned long val, rval; + +#ifdef CONFIG_440SP + /* 440SP use default 0x14D polynomial only */ + return -EINVAL; +#endif + + if (!count || count > 6) + return -EINVAL; + + if (copy_from_user(tmp, buffer, count)) + return -EFAULT; + + tmp[count] = 0; + val = simple_strtoul(tmp, NULL, 16); + + if (val & ~0x1FF) + return -EINVAL; + + val &= 0xFF; + rval = mfdcr(DCRN_MQ0_CFBHL); + rval &= ~(0xFF << MQ0_CFBHL_POLY); + rval |= val << MQ0_CFBHL_POLY; + mtdcr(DCRN_MQ0_CFBHL, rval); + + return count; +} + +static int ppc460ex_r6ena_read (char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + char *p = page; + + p += sprintf(p, "%s\n", + ppc460ex_r6_enabled ? + "PPC460Ex RAID-6 capabilities are ENABLED.\n" : + "PPC460Ex RAID-6 capabilities are DISABLED.\n"); + + return p - page; +} + +static int ppc460ex_r6ena_write (struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + /* e.g. 0xffffffff */ + char tmp[11]; + unsigned long val; + + if (!count || count > 11) + return -EINVAL; + + if (!ppc460ex_r6_tchan) + return -EFAULT; + + if (copy_from_user(tmp, buffer, count)) + return -EFAULT; + + /* Write a key */ + val = simple_strtoul(tmp, NULL, 16); + mtdcr(DCRN_MQ0_XORBA, val); + isync(); + + /* Verify does it really work now */ + if (ppc460ex_test_raid6(ppc460ex_r6_tchan) == 0) { + /* PPC440SP(e) RAID-6 has been activated successfully */; + printk(KERN_INFO "PPC460Ex RAID-6 has been activated " + "successfully\n"); + ppc460ex_r6_enabled = 0; + ppc460ex_r5_enabled = 0; + } else { + /* PPC440SP(e) RAID-6 hasn't been activated! Error key ? */; + printk(KERN_INFO "PPC460Ex RAID-6 hasn't been activated!" + " Error key ?\n"); + ppc460ex_r6_enabled = 0; + } + + return count; +} +static int ppc460ex_r5ena_read (char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + char *p = page; + + p += sprintf(p, "%s\n", + ppc460ex_r5_enabled ? + "PPC460Ex RAID-r5 capabilities are ENABLED.\n" : + "PPC460Ex RAID-r5 capabilities are DISABLED.\n"); + + return p - page; +} + +static int ppc460ex_r5ena_write (struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + /* e.g. 0xffffffff */ + char tmp[11]; + unsigned long val; + + if (!count /*|| count > 11*/) + return -EINVAL; + + if (!ppc460ex_r5_tchan) + return -EFAULT; + + if (copy_from_user(tmp, buffer, count)) + return -EFAULT; + + /* Write a key */ + val = simple_strtoul(tmp, NULL, 16); + mtdcr(DCRN_MQ0_XORBA, val); + isync(); + + /* Verify does it really work now */ + if (ppc460ex_test_raid5(ppc460ex_r5_tchan) == 0) { + /* PPC440SP(e) RAID-6 has been activated successfully */; + printk(KERN_INFO "PPC460Ex RAID-5 has been activated " + "successfully\n"); + ppc460ex_r5_enabled = 1; + ppc460ex_r6_enabled = 0; + } else { + /* PPC440SP(e) RAID-6 hasn't been activated! Error key ? */; + printk(KERN_INFO "PPC460Ex RAID-5 hasn't been activated!" + " Error key ?\n"); + ppc460ex_r5_enabled = 0; + } + + return count; +} + +static int __init ppc460ex_adma_init (void) +{ + int rval; + struct proc_dir_entry *p; + + ppc460ex_configure_raid_devices(); + rval = of_register_platform_driver(&ppc460ex_adma_driver); + //rval = platform_driver_register(&ppc460ex_adma_driver); + + if (rval == 0) { + /* Create /proc entries */ + ppc460ex_proot = proc_mkdir(PPC460EX_R5_PROC_ROOT, NULL); + if (!ppc460ex_proot) { + printk(KERN_ERR "%s: failed to create %s proc " + "directory\n",__FUNCTION__,PPC460EX_R5_PROC_ROOT); + /* User will not be able to enable h/w RAID-6 */ + return rval; + } + + /* GF polynome to use */ + p = create_proc_entry("poly", 0, ppc460ex_proot); + if (p) { + p->read_proc = ppc460ex_poly_read; + p->write_proc = ppc460ex_poly_write; + } + + /* RAID-6 h/w enable entry */ + p = create_proc_entry("enable", 0, ppc460ex_proot); + if (p) { + p->read_proc = ppc460ex_r5ena_read; + p->write_proc = ppc460ex_r5ena_write; + } + /* Create /proc entries */ + ppc460ex_pqroot = proc_mkdir(PPC460EX_R6_PROC_ROOT, NULL); + if (!ppc460ex_pqroot) { + printk(KERN_ERR "%s: failed to create %s proc " + "directory\n",__FUNCTION__,PPC460EX_R6_PROC_ROOT); + /* User will not be able to enable h/w RAID-6 */ + return rval; + } + + /* GF polynome to use */ + p = create_proc_entry("poly", 0, ppc460ex_pqroot); + if (p) { + p->read_proc = ppc460ex_poly_read; + p->write_proc = ppc460ex_poly_write; + } + + /* RAID-6 h/w enable entry */ + p = create_proc_entry("enable", 0, ppc460ex_pqroot); + if (p) { + p->read_proc = ppc460ex_r6ena_read; + p->write_proc = ppc460ex_r6ena_write; + } + } + return rval; +} + +#if 0 +static void __exit ppc460ex_adma_exit (void) +{ + of_unregister_platform_driver(&ppc460ex_adma_driver); + return; +} +module_exit(ppc460ex_adma_exit); +#endif + +module_init(ppc460ex_adma_init); + +MODULE_AUTHOR("Tirumala Marri<tmarri@amcc.com>"); +MODULE_DESCRIPTION("PPC460EX ADMA Engine Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/ppc460ex-plbadma.c b/drivers/dma/ppc460ex-plbadma.c new file mode 100644 index 00000000000..44467211cba --- /dev/null +++ b/drivers/dma/ppc460ex-plbadma.c @@ -0,0 +1,2009 @@ +/* + * Copyright(c) 2006 DENX Engineering. All rights reserved. + * + * Author: Tirumala Marr <tmarri@amcc.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ + +/* + * This driver supports the asynchrounous DMA copy and RAID engines available + * on the AMCC PPC460ex Processors. + * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) + * ADMA driver written by D.Williams. + */ + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/async_tx.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/uaccess.h> +#include <linux/of_platform.h> +#include <linux/proc_fs.h> +#include <asm/dcr.h> +#include <asm/dcr-regs.h> +#include <asm/ppc460ex_plb_adma.h> +#include <asm/ppc460ex_xor.h> +#define PPC44x_SRAM_ADDR 0x00000000400048000ULL +//#define PPC44x_SRAM_SIZE 0x10000 /* 64 Kb*/ +#define PPC44x_SRAM_SIZE 0x8000 /* 32 Kb*/ +//#define CONFIG_ADMA_SRAM 1 + +/* The list of channels exported by ppc460ex ADMA */ +struct list_head +ppc_adma_p_chan_list = LIST_HEAD_INIT(ppc_adma_p_chan_list); + +/* This flag is set when want to refetch the xor chain in the interrupt + * handler + */ +static u32 do_xor_refetch = 0; + +/* Pointers to last submitted to DMA0, DMA1 CDBs */ +static ppc460ex_p_desc_t *chan_last_sub[4]; +static ppc460ex_p_desc_t *chan_first_cdb[4]; + +/* Pointer to last linked and submitted xor CB */ +static ppc460ex_p_desc_t *xor_last_linked = NULL; +static ppc460ex_p_desc_t *xor_last_submit = NULL; + + +/* Since RXOR operations use the common register (MQ0_CF2H) for setting-up + * the block size in transactions, then we do not allow to activate more than + * only one RXOR transactions simultaneously. So use this var to store + * the information about is RXOR currently active (PPC460EX_RXOR_RUN bit is + * set) or not (PPC460EX_RXOR_RUN is clear). + */ + +/* /proc interface is used here to enable the h/w RAID-6 capabilities + */ +static struct proc_dir_entry *ppc460ex_proot; + +/* These are used in enable & check routines + */ +static u32 ppc460ex_r6_enabled; +static u32 ppc460ex_r5_enabled; +static ppc460ex_p_ch_t *ppc460ex_r6_tchan; +static ppc460ex_p_ch_t *ppc460ex_dma_tchan; +static struct completion ppc460ex_r6_test_comp; +static struct completion ppc460ex_r5_test_comp; + +#if 1 +static inline void pr_dma(int x, char *str) +{ + if(mfdcr(0x60)) { + printk("<%s> Line:%d\n",str,x); + } +} +#else +static inline void pr_dma(int x, char *str) +{ +} +#endif +static phys_addr_t fixup_bigphys_addr(phys_addr_t addr, phys_addr_t size) +{ + phys_addr_t page_4gb = 0; + + return (page_4gb | addr); +} + + +/****************************************************************************** + * Command (Descriptor) Blocks low-level routines + ******************************************************************************/ +/** + * ppc460ex_desc_init_interrupt - initialize the descriptor for INTERRUPT + * pseudo operation + */ +static inline void ppc460ex_desc_init_interrupt (ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan) +{ + u32 base = 0; + dma_cdb_t *hw_desc; + + + hw_desc = desc->hw_desc; + + + memset (desc->hw_desc, 0, sizeof(dma_cdb_t)); + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + default: + printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, + __FUNCTION__); + BUG(); + break; + } + hw_desc->ctrl = mfdcr(base + DCR_DMA2P40_CTC0); + set_bit(PPC460EX_DESC_INT, &desc->flags); + set_bit(DMA_CIE_ENABLE,hw_desc->ctrl); +} + +/** + * ppc460ex_desc_init_memcpy - initialize the descriptor for MEMCPY operation + */ +static inline void ppc460ex_desc_init_memcpy(ppc460ex_p_desc_t *desc, + unsigned long flags) +{ + + memset (desc->hw_desc, 0, sizeof(dma_cdb_t)); + desc->hw_next = NULL; + desc->src_cnt = 1; + desc->dst_cnt = 1; + + if (flags & DMA_PREP_INTERRUPT) + set_bit(PPC460EX_DESC_INT, &desc->flags); + else + clear_bit(PPC460EX_DESC_INT, &desc->flags); + +} + +/** + * ppc460ex_desc_init_memset - initialize the descriptor for MEMSET operation + */ +static inline void ppc460ex_desc_init_memset(ppc460ex_p_desc_t *desc, int value, + unsigned long flags) +{ + dma_cdb_t *hw_desc = desc->hw_desc; + + memset (desc->hw_desc, 0, sizeof(dma_cdb_t)); + desc->hw_next = NULL; + desc->src_cnt = 1; + desc->dst_cnt = 1; + + if (flags & DMA_PREP_INTERRUPT) + set_bit(PPC460EX_DESC_INT, &desc->flags); + else + clear_bit(PPC460EX_DESC_INT, &desc->flags); + +} + +/** + * ppc460ex_desc_set_src_addr - set source address into the descriptor + */ +static inline void ppc460ex_desc_set_src_addr( ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan, + dma_addr_t addrh, dma_addr_t addrl) +{ + dma_cdb_t *dma_hw_desc; + phys_addr_t addr64, tmplow, tmphi; + u32 base = 0; + + dma_hw_desc = desc->hw_desc; + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + default: + BUG(); + } + if (!addrh) { + addr64 = fixup_bigphys_addr(addrl, sizeof(phys_addr_t)); + tmphi = (addr64 >> 32); + tmplow = (addr64 & 0xFFFFFFFF); + } else { + tmphi = addrh; + tmplow = addrl; + } + dma_hw_desc->src_hi = tmphi; + dma_hw_desc->src_lo = tmplow; +} + + +/** + * ppc460ex_desc_set_dest_addr - set destination address into the descriptor + */ +static inline void ppc460ex_desc_set_dest_addr(ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan, + dma_addr_t addrh, dma_addr_t addrl) +{ + dma_cdb_t *dma_hw_desc; + phys_addr_t addr64, tmphi, tmplow; + + dma_hw_desc = desc->hw_desc; + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + case PPC460EX_PDMA1_ID: + case PPC460EX_PDMA2_ID: + case PPC460EX_PDMA3_ID: + break; + default : + BUG(); + } + + if (!addrh) { + addr64 = fixup_bigphys_addr(addrl, sizeof(phys_addr_t)); + tmphi = (addr64 >> 32); + tmplow = (addr64 & 0xFFFFFFFF); + } else { + tmphi = addrh; + tmplow = addrl; + } + dma_hw_desc->dest_hi = tmphi; + dma_hw_desc->dest_lo = tmplow; +} + +/** + * ppc460ex_desc_set_byte_count - set number of data bytes involved + * into the operation + */ +static inline void ppc460ex_desc_set_byte_count(ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan, u32 byte_count) +{ + dma_cdb_t *dma_hw_desc; + u32 base = 0; + u32 count = 0; + u32 error = 0; + + dma_hw_desc = desc->hw_desc; + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + } + switch (chan->pwidth) { + case PW_8: + break; + case PW_16: + if (count & 0x1) + error = 1; + break; + case PW_32: + if (count & 0x3) + error = 1; + break; + case PW_64: + if (count & 0x7) + error = 1; + break; + + case PW_128: + if (count & 0xf) + error = 1; + break; + default: + printk("set_dma_count: invalid bus width: 0x%x\n", + chan->pwidth); + return; + } + if (error) + printk + ("Warning: set_dma_count count 0x%x bus width %d\n", + count, chan->pwidth); + + count = count >> chan->shift; + dma_hw_desc->cnt = count; + + +} + +/** + * ppc460ex_desc_set_link - set the address of descriptor following this + * descriptor in chain + */ +static inline void ppc460ex_desc_set_link(ppc460ex_p_ch_t *chan, + ppc460ex_p_desc_t *prev_desc, ppc460ex_p_desc_t *next_desc) +{ + unsigned long flags; + ppc460ex_p_desc_t *tail = next_desc; + + if (unlikely(!prev_desc || !next_desc || + (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { + /* If previous next is overwritten something is wrong. + * though we may refetch from append to initiate list + * processing; in this case - it's ok. + */ + printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " + "prev->hw_next=0x%p\n", __FUNCTION__, prev_desc, + next_desc, prev_desc ? prev_desc->hw_next : 0); + BUG(); + } + + local_irq_save(flags); + + /* do s/w chaining both for DMA and XOR descriptors */ + prev_desc->hw_next = next_desc; + + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + case PPC460EX_PDMA1_ID: + case PPC460EX_PDMA2_ID: + case PPC460EX_PDMA3_ID: + break; + default: + BUG(); + } + + local_irq_restore(flags); +} + +/** + * ppc460ex_desc_get_src_addr - extract the source address from the descriptor + */ +static inline u32 ppc460ex_desc_get_src_addr(ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan, int src_idx) +{ + dma_cdb_t *dma_hw_desc; + u32 base; + + dma_hw_desc = desc->hw_desc; + + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + default: + return 0; + } + /* May have 0, 1, 2, or 3 sources */ + return (dma_hw_desc->src_lo); +} + +/** + * ppc460ex_desc_get_dest_addr - extract the destination address from the + * descriptor + */ +static inline u32 ppc460ex_desc_get_dest_addr(ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan, int idx) +{ + dma_cdb_t *dma_hw_desc; + u32 base; + + dma_hw_desc = desc->hw_desc; + + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + default: + return 0; + } + + /* May have 0, 1, 2, or 3 sources */ + return (dma_hw_desc->dest_lo); +} + +/** + * ppc460ex_desc_get_byte_count - extract the byte count from the descriptor + */ +static inline u32 ppc460ex_desc_get_byte_count(ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan) +{ + dma_cdb_t *dma_hw_desc; + u32 base; + + dma_hw_desc = desc->hw_desc; + + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + default: + return 0; + } + /* May have 0, 1, 2, or 3 sources */ + return (dma_hw_desc->cnt); +} + + +/** + * ppc460ex_desc_get_link - get the address of the descriptor that + * follows this one + */ +static inline u32 ppc460ex_desc_get_link(ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan) +{ + if (!desc->hw_next) + return 0; + + return desc->hw_next->phys; +} + +/** + * ppc460ex_desc_is_aligned - check alignment + */ +static inline int ppc460ex_desc_is_aligned(ppc460ex_p_desc_t *desc, + int num_slots) +{ + return (desc->idx & (num_slots - 1)) ? 0 : 1; +} + + + +/****************************************************************************** + * ADMA channel low-level routines + ******************************************************************************/ + +static inline u32 ppc460ex_chan_get_current_descriptor(ppc460ex_p_ch_t *chan); +static inline void ppc460ex_chan_append(ppc460ex_p_ch_t *chan); + +/* + * ppc460ex_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine + */ +static inline void ppc460ex_adma_device_clear_eot_status (ppc460ex_p_ch_t *chan) +{ + u8 *p = chan->dma_desc_pool_virt; + dma_cdb_t *cdb; + u32 rv ; + u32 base; + + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + + rv = mfdcr(base + DCR_DMA2P40_CR0) & ((DMA_CH0_ERR >> chan->chan_id)); + if (rv) { + printk("DMA%d err status: 0x%x\n", chan->device->id, + rv); + /* write back to clear */ + mtdcr(base + DCR_DMA2P40_CR0, rv); + } + break; + default: + break; + } + +} + +/* + * ppc460ex_chan_is_busy - get the channel status + */ + +static inline int ppc460ex_chan_is_busy(ppc460ex_p_ch_t *chan) +{ + int busy = 0; + u32 base = 0; + + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + default: + BUG(); + } + if(mfdcr((DCR_DMA2P40_SR) & 0x00000800)) + busy = 1; + else + busy = 0; + + return busy; +} + +/** + * ppc460ex_dma_put_desc - put DMA0,1 descriptor to FIFO + */ +static inline void ppc460ex_dma_put_desc(ppc460ex_p_ch_t *chan, + ppc460ex_p_desc_t *desc) +{ + unsigned int control; + u32 sg_cmd; + u32 sg_hi; + u32 sg_lo; + u32 base = 0; + + sg_lo = desc->phys; + + control |= (chan->mode | DMA_CE_ENABLE); + control |= DMA_BEN; + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + default: + BUG(); + } + chan->in_use = 1; + sg_cmd = mfdcr(DCR_DMA2P40_SGC); + sg_cmd = sg_cmd | SSG_ENABLE(chan->chan_id); + sg_cmd = sg_cmd & 0xF0FFFFFF; + mtdcr(base + DCR_DMA2P40_SGL0, sg_lo); +#ifdef PPC4xx_DMA_64BIT + mtdcr(base + DCR_DMA2P40_SGH0, sg_hi); +#endif + mtdcr(DCR_DMA2P40_SGC,sg_cmd); +} + +/** + * ppc460ex_chan_append - update the h/w chain in the channel + */ +static inline void ppc460ex_chan_append(ppc460ex_p_ch_t *chan) +{ + ppc460ex_p_desc_t *iter; + u32 cur_desc; + unsigned long flags; + + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + case PPC460EX_PDMA1_ID: + case PPC460EX_PDMA2_ID: + case PPC460EX_PDMA3_ID: + cur_desc = ppc460ex_chan_get_current_descriptor(chan); + + if (likely(cur_desc)) { + iter = chan_last_sub[chan->device->id]; + BUG_ON(!iter); + } else { + /* first peer */ + iter = chan_first_cdb[chan->device->id]; + BUG_ON(!iter); + ppc460ex_dma_put_desc(chan, iter); + chan->hw_chain_inited = 1; + } + + /* is there something new to append */ + if (!iter->hw_next) + return; + + /* flush descriptors from the s/w queue to fifo */ + list_for_each_entry_continue(iter, &chan->chain, chain_node) { + ppc460ex_dma_put_desc(chan, iter); + if (!iter->hw_next) + break; + } + break; + default: + BUG(); + } +} + +/** + * ppc460ex_chan_get_current_descriptor - get the currently executed descriptor + */ +static inline u32 ppc460ex_chan_get_current_descriptor(ppc460ex_p_ch_t *chan) +{ + u32 base; + + + if (unlikely(!chan->hw_chain_inited)) + /* h/w descriptor chain is not initialized yet */ + return 0; + + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + base = DCR_DMA0_BASE; + break; + case PPC460EX_PDMA1_ID: + base = DCR_DMA1_BASE; + break; + case PPC460EX_PDMA2_ID: + base = DCR_DMA2_BASE; + break; + case PPC460EX_PDMA3_ID: + base = DCR_DMA3_BASE; + break; + default: + BUG(); + } + + return (mfdcr(base + DCR_DMA2P40_SGH0)); +} + + +/****************************************************************************** + * ADMA device level + ******************************************************************************/ + +static void ppc460ex_chan_start_null_xor(ppc460ex_p_ch_t *chan); +static int ppc460ex_adma_alloc_chan_resources(struct dma_chan *chan); +static dma_cookie_t ppc460ex_adma_tx_submit( + struct dma_async_tx_descriptor *tx); + +static void ppc460ex_adma_set_dest( + ppc460ex_p_desc_t *tx, + dma_addr_t addr, int index); + + + +/** + * ppc460ex_adma_device_estimate - estimate the efficiency of processing + * the operation given on this channel. It's assumed that 'chan' is + * capable to process 'cap' type of operation. + * @chan: channel to use + * @cap: type of transaction + * @src_lst: array of source pointers + * @src_cnt: number of source operands + * @src_sz: size of each source operand + */ +int ppc460ex_adma_p_estimate (struct dma_chan *chan, + enum dma_transaction_type cap, struct page **src_lst, + int src_cnt, size_t src_sz) +{ + int ef = 1; + + if (cap == DMA_PQ || cap == DMA_PQ_ZERO_SUM) { + /* If RAID-6 capabilities were not activated don't try + * to use them + */ + if (unlikely(!ppc460ex_r6_enabled)) + return -1; + } + /* channel idleness increases the priority */ + if (likely(ef) && + !ppc460ex_chan_is_busy(to_ppc460ex_adma_chan(chan))) + ef++; + + return ef; +} + +/** + * ppc460ex_get_group_entry - get group entry with index idx + * @tdesc: is the last allocated slot in the group. + */ +static inline ppc460ex_p_desc_t * +ppc460ex_get_group_entry ( ppc460ex_p_desc_t *tdesc, u32 entry_idx) +{ + ppc460ex_p_desc_t *iter = tdesc->group_head; + int i = 0; + + if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { + printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", + __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); + BUG(); + } + list_for_each_entry(iter, &tdesc->group_list, chain_node) { + if (i++ == entry_idx) + break; + } + return iter; +} + +/** + * ppc460ex_adma_free_slots - flags descriptor slots for reuse + * @slot: Slot to free + * Caller must hold &ppc460ex_chan->lock while calling this function + */ +static void ppc460ex_adma_free_slots(ppc460ex_p_desc_t *slot, + ppc460ex_p_ch_t *chan) +{ + int stride = slot->slots_per_op; + + while (stride--) { + /*async_tx_clear_ack(&slot->async_tx);*/ /* Don't need to clear. It is hack*/ + slot->slots_per_op = 0; + slot = list_entry(slot->slot_node.next, + ppc460ex_p_desc_t, + slot_node); + } +} + +static void +ppc460ex_adma_unmap(ppc460ex_p_ch_t *chan, ppc460ex_p_desc_t *desc) +{ + u32 src_cnt, dst_cnt; + dma_addr_t addr; + /* + * get the number of sources & destination + * included in this descriptor and unmap + * them all + */ + src_cnt = 1; + dst_cnt = 1; + + /* unmap destinations */ + if (!(desc->async_tx.flags & DMA_COMPL_SKIP_DEST_UNMAP)) { + while (dst_cnt--) { + addr = ppc460ex_desc_get_dest_addr( + desc, chan, dst_cnt); + dma_unmap_page(&chan->device->odev->dev, + addr, desc->unmap_len, + DMA_FROM_DEVICE); + } + } + + /* unmap sources */ + if (!(desc->async_tx.flags & DMA_COMPL_SKIP_SRC_UNMAP)) { + while (src_cnt--) { + addr = ppc460ex_desc_get_src_addr( + desc, chan, src_cnt); + dma_unmap_page(&chan->device->odev->dev, + addr, desc->unmap_len, + DMA_TO_DEVICE); + } + } + +} +/** + * ppc460ex_adma_run_tx_complete_actions - call functions to be called + * upon complete + */ +static dma_cookie_t ppc460ex_adma_run_tx_complete_actions( + ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan, + dma_cookie_t cookie) +{ + int i; + enum dma_data_direction dir; + + BUG_ON(desc->async_tx.cookie < 0); + if (desc->async_tx.cookie > 0) { + cookie = desc->async_tx.cookie; + desc->async_tx.cookie = 0; + + /* call the callback (must not sleep or submit new + * operations to this channel) + */ + if (desc->async_tx.callback) + desc->async_tx.callback( + desc->async_tx.callback_param); + + /* unmap dma addresses + * (unmap_single vs unmap_page?) + * + * actually, ppc's dma_unmap_page() functions are empty, so + * the following code is just for the sake of completeness + */ + if (chan && chan->needs_unmap && desc->group_head && + desc->unmap_len) { + ppc460ex_p_desc_t *unmap = desc->group_head; + /* assume 1 slot per op always */ + u32 slot_count = unmap->slot_cnt; + + /* Run through the group list and unmap addresses */ + for (i = 0; i < slot_count; i++) { + BUG_ON(!unmap); + ppc460ex_adma_unmap(chan, unmap); + unmap = unmap->hw_next; + } + desc->group_head = NULL; + } + } + + /* run dependent operations */ + dma_run_dependencies(&desc->async_tx); + + return cookie; +} + +/** + * ppc460ex_adma_clean_slot - clean up CDB slot (if ack is set) + */ +static int ppc460ex_adma_clean_slot(ppc460ex_p_desc_t *desc, + ppc460ex_p_ch_t *chan) +{ + /* the client is allowed to attach dependent operations + * until 'ack' is set + */ + if (!async_tx_test_ack(&desc->async_tx)) + return 0; + + /* leave the last descriptor in the chain + * so we can append to it + */ + if (list_is_last(&desc->chain_node, &chan->chain) || + desc->phys == ppc460ex_chan_get_current_descriptor(chan)) + return 1; + + dev_dbg(chan->device->common.dev, "\tfree slot %x: %d stride: %d\n", + desc->phys, desc->idx, desc->slots_per_op); + + list_del(&desc->chain_node); + ppc460ex_adma_free_slots(desc, chan); + return 0; +} + +/** + * #define DEBUG 1__ppc460ex_adma_slot_cleanup - this is the common clean-up routine + * which runs through the channel CDBs list until reach the descriptor + * currently processed. When routine determines that all CDBs of group + * are completed then corresponding callbacks (if any) are called and slots + * are freed. + */ +static void __ppc460ex_adma_slot_cleanup(ppc460ex_p_ch_t *chan) +{ + ppc460ex_p_desc_t *iter, *_iter, *group_start = NULL; + dma_cookie_t cookie = 0; + u32 current_desc = ppc460ex_chan_get_current_descriptor(chan); + int busy = ppc460ex_chan_is_busy(chan); + int seen_current = 0, slot_cnt = 0, slots_per_op = 0; + + dev_dbg(chan->device->common.dev, "ppc460ex adma%d: %s\n", + chan->device->id, __FUNCTION__); + + if (!current_desc) { + /* There were no transactions yet, so + * nothing to clean + */ + return; + } + + /* free completed slots from the chain starting with + * the oldest descriptor + */ + list_for_each_entry_safe(iter, _iter, &chan->chain, + chain_node) { + dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " + "busy: %d this_desc: %#x next_desc: %#x cur: %#x ack: %d\n", + iter->async_tx.cookie, iter->idx, busy, iter->phys, + ppc460ex_desc_get_link(iter, chan), current_desc, + async_tx_test_ack(&iter->async_tx)); + prefetch(_iter); + prefetch(&_iter->async_tx); + + /* do not advance past the current descriptor loaded into the + * hardware channel,subsequent descriptors are either in process + * or have not been submitted + */ + if (seen_current) + break; + + /* stop the search if we reach the current descriptor and the + * channel is busy, or if it appears that the current descriptor + * needs to be re-read (i.e. has been appended to) + */ + if (iter->phys == current_desc) { + BUG_ON(seen_current++); + if (busy || ppc460ex_desc_get_link(iter, chan)) { + /* not all descriptors of the group have + * been completed; exit. + */ + break; + } + } + + /* detect the start of a group transaction */ + if (!slot_cnt && !slots_per_op) { + slot_cnt = iter->slot_cnt; + slots_per_op = iter->slots_per_op; + if (slot_cnt <= slots_per_op) { + slot_cnt = 0; + slots_per_op = 0; + } + } + + if (slot_cnt) { + if (!group_start) + group_start = iter; + slot_cnt -= slots_per_op; + } + + /* all the members of a group are complete */ + if (slots_per_op != 0 && slot_cnt == 0) { + ppc460ex_p_desc_t *grp_iter, *_grp_iter; + int end_of_chain = 0; + + /* clean up the group */ + slot_cnt = group_start->slot_cnt; + grp_iter = group_start; + list_for_each_entry_safe_from(grp_iter, _grp_iter, + &chan->chain, chain_node) { + + cookie = ppc460ex_adma_run_tx_complete_actions( + grp_iter, chan, cookie); + + slot_cnt -= slots_per_op; + end_of_chain = ppc460ex_adma_clean_slot( + grp_iter, chan); + if (end_of_chain && slot_cnt) { + /* Should wait for ZeroSum complete */ + if (cookie > 0) + chan->completed_cookie = cookie; + return; + } + + if (slot_cnt == 0 || end_of_chain) + break; + } + + /* the group should be complete at this point */ + BUG_ON(slot_cnt); + + slots_per_op = 0; + group_start = NULL; + if (end_of_chain) + break; + else + continue; + } else if (slots_per_op) /* wait for group completion */ + continue; + + cookie = ppc460ex_adma_run_tx_complete_actions(iter, chan, + cookie); + + if (ppc460ex_adma_clean_slot(iter, chan)) + break; + } + + BUG_ON(!seen_current); + + if (cookie > 0) { + chan->completed_cookie = cookie; + pr_debug("\tcompleted cookie %d\n", cookie); + } + +} + +/** + * ppc460ex_adma_tasklet - clean up watch-dog initiator + */ +static void ppc460ex_adma_tasklet (unsigned long data) +{ + ppc460ex_p_ch_t *chan = (ppc460ex_p_ch_t *) data; + __ppc460ex_adma_slot_cleanup(chan); +} + +/** + * ppc460ex_adma_slot_cleanup - clean up scheduled initiator + */ +static void ppc460ex_adma_slot_cleanup (ppc460ex_p_ch_t *chan) +{ + spin_lock_bh(&chan->lock); + __ppc460ex_adma_slot_cleanup(chan); + spin_unlock_bh(&chan->lock); +} + +/** + * ppc460ex_adma_alloc_slots - allocate free slots (if any) + */ +static ppc460ex_p_desc_t *ppc460ex_adma_alloc_slots( + ppc460ex_p_ch_t *chan, int num_slots, + int slots_per_op) +{ + ppc460ex_p_desc_t *iter = NULL, *_iter, *alloc_start = NULL; + struct list_head chain = LIST_HEAD_INIT(chain); + int slots_found, retry = 0; + + + BUG_ON(!num_slots || !slots_per_op); + /* start search from the last allocated descrtiptor + * if a contiguous allocation can not be found start searching + * from the beginning of the list + */ +retry: + slots_found = 0; + if (retry == 0) + iter = chan->last_used; + else + iter = list_entry(&chan->all_slots, ppc460ex_p_desc_t, + slot_node); + prefetch(iter); + list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, + slot_node) { + prefetch(_iter); + prefetch(&_iter->async_tx); + if (iter->slots_per_op) { + slots_found = 0; + continue; + } + + /* start the allocation if the slot is correctly aligned */ + if (!slots_found++) + alloc_start = iter; + if (slots_found == num_slots) { + ppc460ex_p_desc_t *alloc_tail = NULL; + ppc460ex_p_desc_t *last_used = NULL; + iter = alloc_start; + while (num_slots) { + int i; + + /* pre-ack all but the last descriptor */ + if (num_slots != slots_per_op) { + async_tx_ack(&iter->async_tx); + } +#if 0 + else + /* Don't need to clear. It is hack*/ + async_tx_clear_ack(&iter->async_tx); +#endif + + list_add_tail(&iter->chain_node, &chain); + alloc_tail = iter; + iter->async_tx.cookie = 0; + iter->hw_next = NULL; + iter->flags = 0; + iter->slot_cnt = num_slots; + for (i = 0; i < slots_per_op; i++) { + iter->slots_per_op = slots_per_op - i; + last_used = iter; + iter = list_entry(iter->slot_node.next, + ppc460ex_p_desc_t, + slot_node); + } + num_slots -= slots_per_op; + } + alloc_tail->group_head = alloc_start; + alloc_tail->async_tx.cookie = -EBUSY; + list_splice(&chain, &alloc_tail->group_list); + chan->last_used = last_used; + return alloc_tail; + } + } + if (!retry++) + goto retry; + static empty_slot_cnt; + if(!(empty_slot_cnt%100)) + printk(KERN_INFO"No empty slots trying to free some\n"); + empty_slot_cnt++; + /* try to free some slots if the allocation fails */ + tasklet_schedule(&chan->irq_tasklet); + return NULL; +} + +/** + * ppc460ex_adma_alloc_chan_resources - allocate pools for CDB slots + */ +static int ppc460ex_adma_alloc_chan_resources(struct dma_chan *chan) +{ + ppc460ex_p_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_p_desc_t *slot = NULL; + char *hw_desc; + int i, db_sz; + int init = ppc460ex_chan->slots_allocated ? 0 : 1; + int pool_size = DMA_FIFO_SIZE * DMA_CDB_SIZE; + + chan->chan_id = ppc460ex_chan->device->id; + + /* Allocate descriptor slots */ + i = ppc460ex_chan->slots_allocated; + db_sz = sizeof (dma_cdb_t); + + for (; i < (pool_size/db_sz); i++) { + slot = kzalloc(sizeof(ppc460ex_p_desc_t), GFP_KERNEL); + if (!slot) { + printk(KERN_INFO "GT ADMA Channel only initialized" + " %d descriptor slots", i--); + break; + } + + hw_desc = (char *) ppc460ex_chan->dma_desc_pool_virt; + slot->hw_desc = (void *) &hw_desc[i * db_sz]; + dma_async_tx_descriptor_init(&slot->async_tx, chan); + slot->async_tx.tx_submit = ppc460ex_adma_tx_submit; + INIT_LIST_HEAD(&slot->chain_node); + INIT_LIST_HEAD(&slot->slot_node); + INIT_LIST_HEAD(&slot->group_list); + hw_desc = (char *) ppc460ex_chan->dma_desc_pool; + slot->phys = (dma_addr_t) &hw_desc[i * db_sz]; + slot->idx = i; + spin_lock_bh(&ppc460ex_chan->lock); + ppc460ex_chan->slots_allocated++; + list_add_tail(&slot->slot_node, &ppc460ex_chan->all_slots); + spin_unlock_bh(&ppc460ex_chan->lock); + } + + if (i && !ppc460ex_chan->last_used) { + ppc460ex_chan->last_used = + list_entry(ppc460ex_chan->all_slots.next, + ppc460ex_p_desc_t, + slot_node); + } + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: allocated %d descriptor slots\n", + ppc460ex_chan->device->id, i); + + /* initialize the channel and the chain with a null operation */ + if (init) { + switch (ppc460ex_chan->chan_id) + { + case PPC460EX_PDMA0_ID: + case PPC460EX_PDMA1_ID: + ppc460ex_chan->hw_chain_inited = 0; + /* Use WXOR for self-testing */ + if (!ppc460ex_dma_tchan) + ppc460ex_dma_tchan = ppc460ex_chan; + if (!ppc460ex_r6_tchan) + ppc460ex_r6_tchan = ppc460ex_chan; + break; + default: + BUG(); + } + ppc460ex_chan->needs_unmap = 1; + } + + return (i > 0) ? i : -ENOMEM; +} + +/** + * ppc460ex_desc_assign_cookie - assign a cookie + */ +static dma_cookie_t ppc460ex_desc_assign_cookie(ppc460ex_p_ch_t *chan, + ppc460ex_p_desc_t *desc) +{ + dma_cookie_t cookie = chan->common.cookie; + cookie++; + if (cookie < 0) + cookie = 1; + chan->common.cookie = desc->async_tx.cookie = cookie; + return cookie; +} + + +/** + * ppc460ex_adma_check_threshold - append CDBs to h/w chain if threshold + * has been achieved + */ +static void ppc460ex_adma_check_threshold(ppc460ex_p_ch_t *chan) +{ + dev_dbg(chan->device->common.dev, "ppc460ex adma%d: pending: %d\n", + chan->device->id, chan->pending); + + if (chan->pending >= PPC460EX_ADMA_THRESHOLD) { + chan->pending = 0; + ppc460ex_chan_append(chan); + } +} + +/** + * ppc460ex_adma_tx_submit - submit new descriptor group to the channel + * (it's not necessary that descriptors will be submitted to the h/w + * chains too right now) + */ +static dma_cookie_t ppc460ex_adma_tx_submit(struct dma_async_tx_descriptor *tx) +{ + ppc460ex_p_desc_t *sw_desc = tx_to_ppc460ex_adma_slot(tx); + ppc460ex_p_ch_t *chan = to_ppc460ex_adma_chan(tx->chan); + ppc460ex_p_desc_t *group_start, *old_chain_tail; + int slot_cnt; + int slots_per_op; + dma_cookie_t cookie; + + group_start = sw_desc->group_head; + slot_cnt = group_start->slot_cnt; + slots_per_op = group_start->slots_per_op; + + spin_lock_bh(&chan->lock); + + cookie = ppc460ex_desc_assign_cookie(chan, sw_desc); + + if (unlikely(list_empty(&chan->chain))) { + /* first peer */ + list_splice_init(&sw_desc->group_list, &chan->chain); + chan_first_cdb[chan->device->id] = group_start; + } else { + /* isn't first peer, bind CDBs to chain */ + old_chain_tail = list_entry(chan->chain.prev, + ppc460ex_p_desc_t, chain_node); + list_splice_init(&sw_desc->group_list, + &old_chain_tail->chain_node); + /* fix up the hardware chain */ + ppc460ex_desc_set_link(chan, old_chain_tail, group_start); + } + + /* increment the pending count by the number of operations */ + chan->pending += slot_cnt / slots_per_op; + ppc460ex_adma_check_threshold(chan); + spin_unlock_bh(&chan->lock); + + dev_dbg(chan->device->common.dev, + "ppc460ex adma%d: %s cookie: %d slot: %d tx %p\n", + chan->device->id,__FUNCTION__, + sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); + return cookie; +} + +/** + * ppc460ex_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_interrupt( + struct dma_chan *chan, unsigned long flags) +{ + ppc460ex_p_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_p_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s\n", ppc460ex_chan->device->id, + __FUNCTION__); + + spin_lock_bh(&ppc460ex_chan->lock); + slot_cnt = slots_per_op = 1; + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + ppc460ex_desc_init_interrupt(group_start, ppc460ex_chan); + group_start->unmap_len = 0; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&ppc460ex_chan->lock); + + return sw_desc ? &sw_desc->async_tx : NULL; +} + +/** + * ppc460ex_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_memcpy( + struct dma_chan *chan, dma_addr_t dma_dest, + dma_addr_t dma_src, size_t len, unsigned long flags) +{ + ppc460ex_p_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_p_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + if (unlikely(!len)) + return NULL; + BUG_ON(unlikely(len > PPC460EX_ADMA_DMA_MAX_BYTE_COUNT)); + + spin_lock_bh(&ppc460ex_chan->lock); + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s len: %u int_en %d \n", + ppc460ex_chan->device->id, __FUNCTION__, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + slot_cnt = slots_per_op = 1; + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + prefetch(group_start); + ppc460ex_desc_init_memcpy(group_start, flags); + ppc460ex_desc_set_dest_addr(sw_desc->group_head, chan, dma_dest, 0); + ppc460ex_desc_set_src_addr(sw_desc->group_head, chan, dma_src, 0); + ppc460ex_desc_set_byte_count(group_start, ppc460ex_chan, len); + sw_desc->unmap_len = len; + sw_desc->async_tx.flags = flags; + if(mfdcr(0x60) == 0xfee8) { + printk("Byte Count = 0x%x\n",len); + printk("src= 0x%x\n",dma_src); + printk("Dest = 0x%x\n",dma_dest); + } + } + spin_unlock_bh(&ppc460ex_chan->lock); + return sw_desc ? &sw_desc->async_tx : NULL; +} + +/** + * ppc460ex_adma_prep_dma_memset - prepare CDB for a MEMSET operation + */ +static struct dma_async_tx_descriptor *ppc460ex_adma_prep_dma_memset( + struct dma_chan *chan, dma_addr_t dma_dest, int value, + size_t len, unsigned long flags) +{ + ppc460ex_p_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_p_desc_t *sw_desc, *group_start; + int slot_cnt, slots_per_op; + if (unlikely(!len)) + return NULL; + BUG_ON(unlikely(len > PPC460EX_ADMA_DMA_MAX_BYTE_COUNT)); + + spin_lock_bh(&ppc460ex_chan->lock); + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s cal: %u len: %u int_en %d\n", + ppc460ex_chan->device->id, __FUNCTION__, value, len, + flags & DMA_PREP_INTERRUPT ? 1 : 0); + + slot_cnt = slots_per_op = 1; + sw_desc = ppc460ex_adma_alloc_slots(ppc460ex_chan, slot_cnt, + slots_per_op); + if (sw_desc) { + group_start = sw_desc->group_head; + ppc460ex_desc_init_memset(group_start, value, flags); + ppc460ex_adma_set_dest(group_start, dma_dest, 0); + ppc460ex_desc_set_byte_count(group_start, ppc460ex_chan, len); + sw_desc->unmap_len = len; + sw_desc->async_tx.flags = flags; + } + spin_unlock_bh(&ppc460ex_chan->lock); + + return sw_desc ? &sw_desc->async_tx : NULL; +} + + +/** + * ppc460ex_adma_set_dest - set destination address into descriptor + */ +static void ppc460ex_adma_set_dest(ppc460ex_p_desc_t *sw_desc, + dma_addr_t addr, int index) +{ + ppc460ex_p_ch_t *chan = to_ppc460ex_adma_chan(sw_desc->async_tx.chan); + BUG_ON(index >= sw_desc->dst_cnt); + + switch (chan->chan_id) { + case PPC460EX_PDMA0_ID: + case PPC460EX_PDMA1_ID: + case PPC460EX_PDMA2_ID: + case PPC460EX_PDMA3_ID: + /* to do: support transfers lengths > + * PPC460EX_ADMA_DMA/XOR_MAX_BYTE_COUNT + */ + ppc460ex_desc_set_dest_addr(sw_desc->group_head, + // chan, 0x8, addr, index); // Enabling HB bus + chan, 0, addr); + break; + default: + BUG(); + } +} + + + + +/** + * ppc460ex_adma_free_chan_resources - free the resources allocated + */ +static void ppc460ex_adma_free_chan_resources(struct dma_chan *chan) +{ + ppc460ex_p_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + ppc460ex_p_desc_t *iter, *_iter; + int in_use_descs = 0; + + ppc460ex_adma_slot_cleanup(ppc460ex_chan); + + spin_lock_bh(&ppc460ex_chan->lock); + list_for_each_entry_safe(iter, _iter, &ppc460ex_chan->chain, + chain_node) { + in_use_descs++; + list_del(&iter->chain_node); + } + list_for_each_entry_safe_reverse(iter, _iter, + &ppc460ex_chan->all_slots, slot_node) { + list_del(&iter->slot_node); + kfree(iter); + ppc460ex_chan->slots_allocated--; + } + ppc460ex_chan->last_used = NULL; + + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d %s slots_allocated %d\n", + ppc460ex_chan->device->id, + __FUNCTION__, ppc460ex_chan->slots_allocated); + spin_unlock_bh(&ppc460ex_chan->lock); + + /* one is ok since we left it on there on purpose */ + if (in_use_descs > 1) + printk(KERN_ERR "GT: Freeing %d in use descriptors!\n", + in_use_descs - 1); +} + +/** + * ppc460ex_adma_is_complete - poll the status of an ADMA transaction + * @chan: ADMA channel handle + * @cookie: ADMA transaction identifier + */ +static enum dma_status ppc460ex_adma_is_complete(struct dma_chan *chan, + dma_cookie_t cookie, dma_cookie_t *done, dma_cookie_t *used) +{ + ppc460ex_p_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + dma_cookie_t last_used; + dma_cookie_t last_complete; + enum dma_status ret; + + last_used = chan->cookie; + last_complete = ppc460ex_chan->completed_cookie; + + if (done) + *done= last_complete; + if (used) + *used = last_used; + + ret = dma_async_is_complete(cookie, last_complete, last_used); + if (ret == DMA_SUCCESS) + return ret; + + ppc460ex_adma_slot_cleanup(ppc460ex_chan); + + last_used = chan->cookie; + last_complete = ppc460ex_chan->completed_cookie; + + if (done) + *done= last_complete; + if (used) + *used = last_used; + + return dma_async_is_complete(cookie, last_complete, last_used); +} + +/** + * ppc460ex_adma_eot_handler - end of transfer interrupt handler + */ +static irqreturn_t ppc460ex_adma_eot_handler(int irq, void *data) +{ + ppc460ex_p_ch_t *chan = data; + + dev_dbg(chan->device->common.dev, + "ppc460ex adma%d: %s\n", chan->device->id, __FUNCTION__); + + tasklet_schedule(&chan->irq_tasklet); + ppc460ex_adma_device_clear_eot_status(chan); + + return IRQ_HANDLED; +} + +/** + * ppc460ex_adma_err_handler - DMA error interrupt handler; + * do the same things as a eot handler + */ +static irqreturn_t ppc460ex_adma_err_handler(int irq, void *data) +{ + ppc460ex_p_ch_t *chan = data; + dev_dbg(chan->device->common.dev, + "ppc460ex adma%d: %s\n", chan->device->id, __FUNCTION__); + tasklet_schedule(&chan->irq_tasklet); + ppc460ex_adma_device_clear_eot_status(chan); + + return IRQ_HANDLED; +} + +static void ppc460ex_test_rad6_callback (void *unused) +{ + complete(&ppc460ex_r6_test_comp); +} +/** + * ppc460ex_test_callback - called when test operation has been done + */ +static void ppc460ex_test_callback (void *unused) +{ + complete(&ppc460ex_r5_test_comp); +} + +/** + * ppc460ex_adma_issue_pending - flush all pending descriptors to h/w + */ +static void ppc460ex_adma_issue_pending(struct dma_chan *chan) +{ + ppc460ex_p_ch_t *ppc460ex_chan = to_ppc460ex_adma_chan(chan); + + if (ppc460ex_chan->pending) { + dev_dbg(ppc460ex_chan->device->common.dev, + "ppc460ex adma%d: %s %d \n", ppc460ex_chan->device->id, + __FUNCTION__, ppc460ex_chan->pending); + ppc460ex_chan->pending = 0; + ppc460ex_chan_append(ppc460ex_chan); + } +} + +/** + * ppc460ex_adma_remove - remove the asynch device + */ +static int __devexit ppc460ex_pdma_remove(struct platform_device *dev) +{ + ppc460ex_p_dev_t *device = platform_get_drvdata(dev); + struct dma_chan *chan, *_chan; + struct ppc_dma_chan_ref *ref, *_ref; + ppc460ex_p_ch_t *ppc460ex_chan; + int i; + + dma_async_device_unregister(&device->common); + + for (i = 0; i < 3; i++) { + u32 irq; + irq = platform_get_irq(dev, i); + free_irq(irq, device); + } + + + do { + struct resource *res; + res = platform_get_resource(dev, IORESOURCE_MEM, 0); + release_mem_region(res->start, res->end - res->start); + } while (0); + + list_for_each_entry_safe(chan, _chan, &device->common.channels, + device_node) { + ppc460ex_chan = to_ppc460ex_adma_chan(chan); + list_del(&chan->device_node); + kfree(ppc460ex_chan); + } + + list_for_each_entry_safe(ref, _ref, &ppc_adma_p_chan_list, node) { + list_del(&ref->node); + kfree(ref); + } + + kfree(device); + + return 0; +} +/* + * Per channel probe + */ +int __devinit ppc460ex_dma_per_chan_probe(struct of_device *ofdev, + const struct of_device_id *match) +{ + int ret=0; + ppc460ex_p_dev_t *adev; + ppc460ex_p_ch_t *new_chan; + int err; + + adev = dev_get_drvdata(ofdev->dev.parent); + BUG_ON(!adev); + if ((new_chan = kzalloc(sizeof(*new_chan), GFP_KERNEL)) == NULL) { + printk("ERROR:No Free memory for allocating dma channels\n"); + ret = -ENOMEM; + goto err; + } + err = of_address_to_resource(ofdev->node,0,&new_chan->reg); + if (err) { + printk("ERROR:Can't get %s property reg\n", __FUNCTION__); + goto err; + } + new_chan->device = &ofdev->dev; + new_chan->reg_base = ioremap(new_chan->reg.start, + new_chan->reg.end - new_chan->reg.start + 1); + if ((new_chan->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, + DMA_FIFO_SIZE << 2, &new_chan->dma_desc_pool, GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_dma_alloc; + } + new_chan->chan_id = ((new_chan->reg.start - 0x200)& 0xfff) >> 3; + adev->chan[new_chan->chan_id] = new_chan; + + return 0; +err: + return ret; +err_dma_alloc: +err_chan_alloc: + kfree(new_chan); + return ret; +} +static struct of_device_id dma_4chan_match[] = +{ + { + .compatible = "amcc,dma", + }, + {}, +}; +static struct of_device_id dma_per_chan_match[] = { + { + .compatible = "amcc,dma-4channel", + }, + {}, +}; +/** + * ppc460ex_adma_probe - probe the asynch device + */ +//static int __devinit ppc460ex_adma_probe(struct platform_device *pdev) +static int __devinit ppc460ex_pdma_probe(struct of_device *ofdev, + const struct of_device_id *match) +{ + struct resource *res; + int ret=0, irq; + ppc460ex_p_dev_t *adev; + ppc460ex_p_ch_t *chan; + struct ppc_dma_chan_ref *ref; + + + if ((adev = kzalloc(sizeof(*adev), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_adev_alloc; + } + adev->dev = &ofdev->dev; + adev->id = PPC460EX_PDMA0_ID; + /* create the DMA capability MASK . This used to come from resources structure*/ + dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); + dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); + dma_cap_set(DMA_MEMSET, adev->common.cap_mask); + adev->odev = ofdev; + dev_set_drvdata(&(ofdev->dev), adev); + + INIT_LIST_HEAD(&adev->common.channels); + + /* set base routines */ + adev->common.device_alloc_chan_resources = + ppc460ex_adma_alloc_chan_resources; + adev->common.device_free_chan_resources = + ppc460ex_adma_free_chan_resources; + adev->common.device_is_tx_complete = ppc460ex_adma_is_complete; + adev->common.device_issue_pending = ppc460ex_adma_issue_pending; + adev->common.dev = &ofdev->dev; + + /* set prep routines based on capability */ + if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { + adev->common.device_prep_dma_memcpy = + ppc460ex_adma_prep_dma_memcpy; + } + if (dma_has_cap(DMA_MEMSET, adev->common.cap_mask)) { + adev->common.device_prep_dma_memset = + ppc460ex_adma_prep_dma_memset; + } + + if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { + adev->common.device_prep_dma_interrupt = + ppc460ex_adma_prep_dma_interrupt; + } + + /* create a channel */ + if ((chan = kzalloc(sizeof(*chan), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_chan_alloc; + } + + tasklet_init(&chan->irq_tasklet, ppc460ex_adma_tasklet, + (unsigned long)chan); + irq = irq_of_parse_and_map(ofdev->node, 0); + printk("<%s> irq=0x%x\n",__FUNCTION__, irq); + if (irq >= 0) { + ret = request_irq(irq, ppc460ex_adma_eot_handler, + IRQF_DISABLED, "adma-chan0", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + + irq = irq_of_parse_and_map(ofdev->node, 1); + printk("<%s> irq=0x%x\n",__FUNCTION__, irq); + if (irq >= 0) { + ret = request_irq(irq, ppc460ex_adma_err_handler, + IRQF_DISABLED, "adma-chan-1", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + irq = irq_of_parse_and_map(ofdev->node, 2); + printk("<%s> irq=0x%x\n",__FUNCTION__, irq); + if (irq >= 0) { + ret = request_irq(irq, ppc460ex_adma_err_handler, + IRQF_DISABLED, "adma-chan2", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + irq = irq_of_parse_and_map(ofdev->node, 3); + printk("<%s> irq=0x%x\n",__FUNCTION__, irq); + if (irq >= 0) { + ret = request_irq(irq, ppc460ex_adma_err_handler, + IRQF_DISABLED, "adma-chan3", chan); + if (ret) { + printk("Failed to request IRQ %d\n",irq); + ret = -EIO; + goto err_irq; + } + + + } + + + } + + } + } else + ret = -ENXIO; + + chan->device = adev; + /* pass the platform data */ + spin_lock_init(&chan->lock); +#if 0 + init_timer(&chan->cleanup_watchdog); + chan->cleanup_watchdog.data = (unsigned long) chan; + chan->cleanup_watchdog.function = ppc460ex_adma_tasklet; +#endif + INIT_LIST_HEAD(&chan->chain); + INIT_LIST_HEAD(&chan->all_slots); + chan->common.device = &adev->common; + list_add_tail(&chan->common.device_node, &adev->common.channels); + + dev_dbg(&ofdev->dev, "AMCC(R) PPC440SP(E) ADMA Engine found [%d]: " + "( %s%s%s%s%s%s%s%s%s%s)\n", + adev->id, + dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq_xor " : "", + dma_has_cap(DMA_PQ_UPDATE, adev->common.cap_mask) ? "pq_update " : "", + dma_has_cap(DMA_PQ_ZERO_SUM, adev->common.cap_mask) ? "pq_zero_sum " : + "", + dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", + dma_has_cap(DMA_DUAL_XOR, adev->common.cap_mask) ? "dual_xor " : "", + dma_has_cap(DMA_ZERO_SUM, adev->common.cap_mask) ? "xor_zero_sum " : + "", + dma_has_cap(DMA_MEMSET, adev->common.cap_mask) ? "memset " : "", + dma_has_cap(DMA_MEMCPY_CRC32C, adev->common.cap_mask) ? "memcpy+crc " + : "", + dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", + dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "int " : ""); + + of_platform_bus_probe(ofdev->node, dma_per_chan_match,&ofdev->dev); + dma_async_device_register(&adev->common); + ref = kmalloc(sizeof(*ref), GFP_KERNEL); + printk("<%s> ret=0x%x\n", __FUNCTION__,ret); + if (ref) { + ref->chan = &chan->common; + INIT_LIST_HEAD(&ref->node); + list_add_tail(&ref->node, &ppc_adma_p_chan_list); + } else + printk(KERN_WARNING "%s: failed to allocate channel reference!\n", + __FUNCTION__); + goto out; + +err: + ret = ret; +err_irq: + kfree(chan); +err_chan_alloc: +err_dma_alloc: + kfree(adev); +err_adev_alloc: + release_mem_region(res->start, res->end - res->start); +out: + return ret; +} + +/** + * ppc460ex_test_dma - test are RAID-5 capabilities enabled successfully. + * For this we just perform one WXOR operation with the same source + * and destination addresses, the GF-multiplier is 1; so if RAID-5 + o/of_platform_driver_unregister(&ppc460ex_pdma_driver); + * capabilities are enabled then we'll get src/dst filled with zero. + */ +static int ppc460ex_test_dma (ppc460ex_p_ch_t *chan) +{ + ppc460ex_p_desc_t *sw_desc, *iter; + struct page *pg; + char *a; + dma_addr_t dma_addr; + unsigned long op = 0; + int rval = 0; + + if (!ppc460ex_dma_tchan) + return -1; + /*FIXME*/ + + pg = alloc_page(GFP_KERNEL); + if (!pg) + return -ENOMEM; + + spin_lock_bh(&chan->lock); + sw_desc = ppc460ex_adma_alloc_slots(chan, 1, 1); + if (sw_desc) { + /* 1 src, 1 dsr, int_ena */ + ppc460ex_desc_init_memcpy(sw_desc,0); + list_for_each_entry(iter, &sw_desc->group_list, chain_node) { + ppc460ex_desc_set_byte_count(iter, chan, PAGE_SIZE); + iter->unmap_len = PAGE_SIZE; + } + } else { + rval = -EFAULT; + spin_unlock_bh(&chan->lock); + goto exit; + } + spin_unlock_bh(&chan->lock); + + /* Fill the test page with ones */ + memset(page_address(pg), 0xFF, PAGE_SIZE); + int i = 0; + char *pg_addr = page_address(pg); +#if 0 + for(i=0;i < PAGE_SIZE; i+=64) + printk("addr = 0x%x data = 0x%x\n",pg_addr + i,*(pg_addr+i)); +#endif + //dma_addr = dma_map_page(&chan->device->common, pg, 0, PAGE_SIZE, + dma_addr = dma_map_page(&chan->device->odev->dev, pg, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + + /* Setup adresses */ + ppc460ex_desc_set_src_addr(sw_desc, chan, dma_addr, 0); + ppc460ex_desc_set_dest_addr(sw_desc, chan, dma_addr, 0); + + async_tx_ack(&sw_desc->async_tx); + sw_desc->async_tx.callback = ppc460ex_test_callback; + sw_desc->async_tx.callback_param = NULL; + + init_completion(&ppc460ex_r5_test_comp); + + ppc460ex_adma_tx_submit(&sw_desc->async_tx); + ppc460ex_adma_issue_pending(&chan->common); + + wait_for_completion(&ppc460ex_r5_test_comp); + + /*Make sure cache is flushed to memory*/ + dma_addr = dma_map_page(&chan->device->odev->dev, pg, 0, PAGE_SIZE, + DMA_BIDIRECTIONAL); + /* Now check is the test page zeroed */ + a = page_address(pg); +#if 0 + i = 0; + for(i=0;i < PAGE_SIZE; i+=64) + printk("addr = 0x%x data = 0x%x\n",a + i,*(a+i)); +#endif + if ((*(u32*)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4)==0) { + /* page is zero - RAID-5 enabled */ + rval = 0; + } else { + /* RAID-5 was not enabled */ + rval = -EINVAL; + } + pr_dma(__LINE__,__FUNCTION__); +exit: + __free_page(pg); + return rval; +} + + +static struct of_platform_driver ppc460ex_pdma_driver = { + .name = "plb_dma", + .match_table = dma_4chan_match, + + .probe = ppc460ex_pdma_probe, + .remove = ppc460ex_pdma_remove, +}; +struct of_platform_driver ppc460ex_dma_per_chan_driver = { + .name = "dma-4channel", + .match_table = dma_per_chan_match, + .probe = ppc460ex_dma_per_chan_probe, +}; + +static int ppc460ex_dma_read (char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + char *p = page; + + p += sprintf(p, "%s\n", + ppc460ex_r5_enabled ? + "PPC460Ex RAID-r5 capabilities are ENABLED.\n" : + "PPC460Ex RAID-r5 capabilities are DISABLED.\n"); + + return p - page; +} + +static int ppc460ex_dma_write (struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + /* e.g. 0xffffffff */ + char tmp[11]; + unsigned long val; + + if (!count || count > 11) + return -EINVAL; + + if (!ppc460ex_dma_tchan) + return -EFAULT; + + if (copy_from_user(tmp, buffer, count)) + return -EFAULT; + + /* Write a key */ + val = simple_strtoul(tmp, NULL, 16); + if(!strcmp(val,"copy")) + printk("Testing copy feature"); + /* Verify does it really work now */ + if (ppc460ex_test_dma(ppc460ex_dma_tchan) == 0) { + /* PPC440SP(e) RAID-6 has been activated successfully */; + printk(KERN_INFO "PPC460Ex RAID-5 has been activated " + "successfully\n"); + ppc460ex_r5_enabled = 1; + ppc460ex_r6_enabled = 1; + } else { + /* PPC440SP(e) RAID-6 hasn't been activated! Error key ? */; + printk(KERN_INFO "PPC460Ex RAID-5 hasn't been activated!" + " Error key ?\n"); + ppc460ex_r5_enabled = 0; + } + + return count; +} + +static int __init ppc460ex_adma_init (void) +{ + int rval; + struct proc_dir_entry *p; + + rval = of_register_platform_driver(&ppc460ex_pdma_driver); + + if (rval == 0) { + /* Create /proc entries */ + ppc460ex_proot = proc_mkdir(PPC460EX_DMA_PROC_ROOT, NULL); + if (!ppc460ex_proot) { + printk(KERN_ERR "%s: failed to create %s proc " + "directory\n",__FUNCTION__,PPC460EX_DMA_PROC_ROOT); + /* User will not be able to enable h/w RAID-6 */ + return rval; + } + + /* RAID-6 h/w enable entry */ + p = create_proc_entry("enable", 0, ppc460ex_proot); + if (p) { + p->read_proc = ppc460ex_dma_read; + p->write_proc = ppc460ex_dma_write; + } + } + return rval; +} + +#if 0 +static void __exit ppc460ex_adma_exit (void) +{ + of_unregister_platform_driver(&ppc460ex_pdma_driver); + return; +} +module_exit(ppc460ex_adma_exit); +#endif + +module_init(ppc460ex_adma_init); + +MODULE_AUTHOR("Tirumala Marri<tmarri@amcc.com>"); +MODULE_DESCRIPTION("PPC460EX ADMA Engine Driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/dma/ppc460ex_4chan_dma.c b/drivers/dma/ppc460ex_4chan_dma.c new file mode 100755 index 00000000000..cb2888d5213 --- /dev/null +++ b/drivers/dma/ppc460ex_4chan_dma.c @@ -0,0 +1,1103 @@ +/* + * Copyright(c) 2008 Applied Micro Circuits Corporation(AMCC). All rights reserved. + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ + + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/async_tx.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/of_platform.h> +#include <linux/uaccess.h> +#include <linux/proc_fs.h> +#include <asm/dcr-regs.h> +#include <asm/dcr.h> +#include "ppc460ex_4chan_dma.h" + + + +#ifdef DEBUG_TEST +#define dma_pr printk +#else +#define dma_pr +#endif +#define TEST_SIZE 12 + + +ppc460ex_plb_dma_dev_t *adev; + + + +int ppc460ex_get_dma_channel(void) +{ + int i; + unsigned int status = 0; + status = mfdcr(DCR_DMA2P40_SR); + + for(i=0; i<MAX_PPC460EX_DMA_CHANNELS; i++) { + if ((status & (1 >> (20+i))) == 0) + return i; + } + return -ENODEV; +} + + +int ppc460ex_get_dma_status(void) +{ + return (mfdcr(DCR_DMA2P40_SR)); + +} + + +int ppc460ex_set_src_addr(int ch_id, phys_addr_t src_addr) +{ + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk("%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + +#ifdef PPC4xx_DMA_64BIT + mtdcr(DCR_DMA2P40_SAH0 + ch_id*8, src_addr >> 32); +#endif + mtdcr(DCR_DMA2P40_SAL0 + ch_id*8, (u32)src_addr); + + return DMA_STATUS_GOOD; +} + +int ppc460ex_set_dst_addr(int ch_id, phys_addr_t dst_addr) +{ + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + +#ifdef PPC4xx_DMA_64BIT + mtdcr(DCR_DMA2P40_DAH0 + ch_id*8, dst_addr >> 32); +#endif + mtdcr(DCR_DMA2P40_DAL0 + ch_id*8, (u32)dst_addr); + + return DMA_STATUS_GOOD; +} + + + +/* + * Sets the dma mode for single DMA transfers only. + * For scatter/gather transfers, the mode is passed to the + * alloc_dma_handle() function as one of the parameters. + * + * The mode is simply saved and used later. This allows + * the driver to call set_dma_mode() and set_dma_addr() in + * any order. + * + * Valid mode values are: + * + * DMA_MODE_READ peripheral to memory + * DMA_MODE_WRITE memory to peripheral + * DMA_MODE_MM memory to memory + * DMA_MODE_MM_DEVATSRC device-paced memory to memory, device at src + * DMA_MODE_MM_DEVATDST device-paced memory to memory, device at dst + */ +int ppc460ex_set_dma_mode(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, unsigned int mode) +{ + + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk("%s: bad channel %d\n", __FUNCTION__, dma_chan->chan_id); + return DMA_STATUS_BAD_CHANNEL; + } + + dma_chan->mode = mode; + return DMA_STATUS_GOOD; +} + + + + +/* + * Sets the DMA Count register. Note that 'count' is in bytes. + * However, the DMA Count register counts the number of "transfers", + * where each transfer is equal to the bus width. Thus, count + * MUST be a multiple of the bus width. + */ +void ppc460ex_set_dma_count(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, unsigned int count) +{ + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + +//#ifdef DEBUG_4xxDMA + + { + int error = 0; + switch (dma_chan->pwidth) { + case PW_8: + break; + case PW_16: + if (count & 0x1) + error = 1; + break; + case PW_32: + if (count & 0x3) + error = 1; + break; + case PW_64: + if (count & 0x7) + error = 1; + break; + + case PW_128: + if (count & 0xf) + error = 1; + break; + default: + printk("set_dma_count: invalid bus width: 0x%x\n", + dma_chan->pwidth); + return; + } + if (error) + printk + ("Warning: set_dma_count count 0x%x bus width %d\n", + count, dma_chan->pwidth); + } +//#endif + count = count >> dma_chan->shift; + //count = 10; + mtdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8), count); + +} + + + + +/* + * Enables the channel interrupt. + * + * If performing a scatter/gatter transfer, this function + * MUST be called before calling alloc_dma_handle() and building + * the sgl list. Otherwise, interrupts will not be enabled, if + * they were previously disabled. + */ +int ppc460ex_enable_dma_interrupt(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + dma_chan->int_enable = 1; + + + control = mfdcr(DCR_DMA2P40_CR0); + control |= DMA_CIE_ENABLE; /* Channel Interrupt Enable */ + mtdcr(DCR_DMA2P40_CR0, control); + + + +#if 1 + control = mfdcr(DCR_DMA2P40_CTC0); + control |= DMA_CTC_TCIE | DMA_CTC_ETIE| DMA_CTC_EIE; + mtdcr(DCR_DMA2P40_CTC0, control); + +#endif + + + return DMA_STATUS_GOOD; + +} + + +/* + * Disables the channel interrupt. + * + * If performing a scatter/gatter transfer, this function + * MUST be called before calling alloc_dma_handle() and building + * the sgl list. Otherwise, interrupts will not be disabled, if + * they were previously enabled. + */ +int ppc460ex_disable_dma_interrupt(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + dma_chan->int_enable = 0; + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + control &= ~DMA_CIE_ENABLE; /* Channel Interrupt Enable */ + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + + return DMA_STATUS_GOOD; +} + + +/* + * This function returns the channel configuration. + */ +int ppc460ex_get_channel_config(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, + ppc460ex_plb_dma_ch_t *p_dma_ch) +{ + unsigned int polarity; + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + memcpy(p_dma_ch, dma_chan, sizeof(ppc460ex_plb_dma_ch_t)); + + polarity = mfdcr(DCR_DMA2P40_POL); + + p_dma_ch->polarity = polarity & GET_DMA_POLARITY(ch_id); + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + + p_dma_ch->cp = GET_DMA_PRIORITY(control); + p_dma_ch->pwidth = GET_DMA_PW(control); + p_dma_ch->psc = GET_DMA_PSC(control); + p_dma_ch->pwc = GET_DMA_PWC(control); + p_dma_ch->phc = GET_DMA_PHC(control); + p_dma_ch->ce = GET_DMA_CE_ENABLE(control); + p_dma_ch->int_enable = GET_DMA_CIE_ENABLE(control); + p_dma_ch->shift = GET_DMA_PW(control); + p_dma_ch->pf = GET_DMA_PREFETCH(control); + + return DMA_STATUS_GOOD; + +} + +/* + * Sets the priority for the DMA channel dmanr. + * Since this is setup by the hardware init function, this function + * can be used to dynamically change the priority of a channel. + * + * Acceptable priorities: + * + * PRIORITY_LOW + * PRIORITY_MID_LOW + * PRIORITY_MID_HIGH + * PRIORITY_HIGH + * + */ +int ppc460ex_set_channel_priority(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, + unsigned int priority) +{ + unsigned int control; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + if ((priority != PRIORITY_LOW) && + (priority != PRIORITY_MID_LOW) && + (priority != PRIORITY_MID_HIGH) && (priority != PRIORITY_HIGH)) { + printk("%s:bad priority: 0x%x\n", __FUNCTION__, priority); + } + + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + control |= SET_DMA_PRIORITY(priority); + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + + return DMA_STATUS_GOOD; +} + +/* + * Returns the width of the peripheral attached to this channel. This assumes + * that someone who knows the hardware configuration, boot code or some other + * init code, already set the width. + * + * The return value is one of: + * PW_8 + * PW_16 + * PW_32 + * PW_64 + * + * The function returns 0 on error. + */ +unsigned int ppc460ex_get_peripheral_width(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + return (GET_DMA_PW(control)); +} + +/* + * Enables the burst on the channel (BTEN bit in the control/count register) + * Note: + * For scatter/gather dma, this function MUST be called before the + * ppc4xx_alloc_dma_handle() func as the chan count register is copied into the + * sgl list and used as each sgl element is added. + */ +int ppc460ex_enable_burst(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int ctc; + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + ctc = mfdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8)) | DMA_CTC_BTEN; + mtdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8), ctc); + return DMA_STATUS_GOOD; +} + + +/* + * Disables the burst on the channel (BTEN bit in the control/count register) + * Note: + * For scatter/gather dma, this function MUST be called before the + * ppc4xx_alloc_dma_handle() func as the chan count register is copied into the + * sgl list and used as each sgl element is added. + */ +int ppc460ex_disable_burst(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int ctc; + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + ctc = mfdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8)) &~ DMA_CTC_BTEN; + mtdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8), ctc); + return DMA_STATUS_GOOD; +} + + +/* + * Sets the burst size (number of peripheral widths) for the channel + * (BSIZ bits in the control/count register)) + * must be one of: + * DMA_CTC_BSIZ_2 + * DMA_CTC_BSIZ_4 + * DMA_CTC_BSIZ_8 + * DMA_CTC_BSIZ_16 + * Note: + * For scatter/gather dma, this function MUST be called before the + * ppc4xx_alloc_dma_handle() func as the chan count register is copied into the + * sgl list and used as each sgl element is added. + */ +int ppc460ex_set_burst_size(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, + unsigned int bsize) +{ + unsigned int ctc; + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + ctc = mfdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8)) &~ DMA_CTC_BSIZ_MSK; + ctc |= (bsize & DMA_CTC_BSIZ_MSK); + mtdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8), ctc); + return DMA_STATUS_GOOD; +} + +/* + * Returns the number of bytes left to be transferred. + * After a DMA transfer, this should return zero. + * Reading this while a DMA transfer is still in progress will return + * unpredictable results. + */ +int ppc460ex_get_dma_residue(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int count; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + count = mfdcr(DCR_DMA2P40_CTC0 + (ch_id * 0x8)); + count &= DMA_CTC_TC_MASK ; + + return (count << dma_chan->shift); + +} + + +/* + * Configures a DMA channel, including the peripheral bus width, if a + * peripheral is attached to the channel, the polarity of the DMAReq and + * DMAAck signals, etc. This information should really be setup by the boot + * code, since most likely the configuration won't change dynamically. + * If the kernel has to call this function, it's recommended that it's + * called from platform specific init code. The driver should not need to + * call this function. + */ +int ppc460ex_init_dma_channel(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, + ppc460ex_plb_dma_ch_t *p_init) +{ + unsigned int polarity; + uint32_t control = 0; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + + DMA_MODE_READ = (unsigned long) DMA_TD; /* Peripheral to Memory */ + DMA_MODE_WRITE = 0; /* Memory to Peripheral */ + + if (!p_init) { + printk("%s: NULL p_init\n", __FUNCTION__); + return DMA_STATUS_NULL_POINTER; + } + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } +#if DCR_DMA2P40_POL > 0 + polarity = mfdcr(DCR_DMA2P40_POL); +#else + polarity = 0; +#endif + + p_init->int_enable = 0; + p_init->buffer_enable = 1; + p_init->etd_output = 1; + p_init->tce_enable = 1; + p_init->pl = 0; + p_init->dai = 1; + p_init->sai = 1; + /* Duc Dang: make channel priority to 2, original is 3 */ + p_init->cp = 2; + p_init->pwidth = PW_8; + p_init->psc = 0; + p_init->pwc = 0; + p_init->phc = 0; + p_init->pf = 1; + + + /* Setup the control register based on the values passed to + * us in p_init. Then, over-write the control register with this + * new value. + */ +#if 0 + control |= SET_DMA_CONTROL; +#endif + control = SET_DMA_CONTROL; + /* clear all polarity signals and then "or" in new signal levels */ + +//PMB - Workaround + //control = 0x81A2CD80; + //control = 0x81A00180; + + + polarity &= ~GET_DMA_POLARITY(ch_id); + polarity |= p_init->polarity; + +#if DCR_DMA2P40_POL > 0 + mtdcr(DCR_DMA2P40_POL, polarity); +#endif + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + + /* save these values in our dma channel structure */ + //memcpy(dma_chan, p_init, sizeof(ppc460ex_plb_dma_ch_t)); + /* + * The peripheral width values written in the control register are: + * PW_8 0 + * PW_16 1 + * PW_32 2 + * PW_64 3 + * PW_128 4 + * + * Since the DMA count register takes the number of "transfers", + * we need to divide the count sent to us in certain + * functions by the appropriate number. It so happens that our + * right shift value is equal to the peripheral width value. + */ + dma_chan->shift = p_init->pwidth; + dma_chan->sai = p_init->sai; + dma_chan->dai = p_init->dai; + dma_chan->tce_enable = p_init->tce_enable; + dma_chan->mode = DMA_MODE_MM; + /* + * Save the control word for easy access. + */ + dma_chan->control = control; + mtdcr(DCR_DMA2P40_SR, 0xffffffff); + + + return DMA_STATUS_GOOD; +} + + +int ppc460ex_enable_dma(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + unsigned int status_bits[] = { DMA_CS0 | DMA_TS0 | DMA_CH0_ERR, + DMA_CS1 | DMA_TS1 | DMA_CH1_ERR}; + + if (dma_chan->in_use) { + printk("%s:enable_dma: channel %d in use\n", __FUNCTION__, ch_id); + return DMA_STATUS_CHANNEL_NOTFREE; + } + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk(KERN_ERR "%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + +#if 0 + if (dma_chan->mode == DMA_MODE_READ) { + /* peripheral to memory */ + ppc460ex_set_src_addr(ch_id, 0); + ppc460ex_set_dst_addr(ch_id, dma_chan->addr); + } else if (dma_chan->mode == DMA_MODE_WRITE) { + /* memory to peripheral */ + ppc460ex_set_src_addr(ch_id, dma_chan->addr); + ppc460ex_set_dst_addr(ch_id, 0); + } +#endif + /* for other xfer modes, the addresses are already set */ + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + control &= ~(DMA_TM_MASK | DMA_TD); /* clear all mode bits */ + if (dma_chan->mode == DMA_MODE_MM) { + /* software initiated memory to memory */ + control |= DMA_ETD_OUTPUT | DMA_TCE_ENABLE; + control |= DMA_MODE_MM; + if (dma_chan->dai) { + control |= DMA_DAI; + } + if (dma_chan->sai) { + control |= DMA_SAI; + } + } + + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + /* + * Clear the CS, TS, RI bits for the channel from DMASR. This + * has been observed to happen correctly only after the mode and + * ETD/DCE bits in DMACRx are set above. Must do this before + * enabling the channel. + */ + mtdcr(DCR_DMA2P40_SR, status_bits[ch_id]); + /* + * For device-paced transfers, Terminal Count Enable apparently + * must be on, and this must be turned on after the mode, etc. + * bits are cleared above (at least on Redwood-6). + */ + + if ((dma_chan->mode == DMA_MODE_MM_DEVATDST) || + (dma_chan->mode == DMA_MODE_MM_DEVATSRC)) + control |= DMA_TCE_ENABLE; + + /* + * Now enable the channel. + */ + + control |= (dma_chan->mode | DMA_CE_ENABLE); + control |= DMA_BEN; + //control = 0xc4effec0; + + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + dma_chan->in_use = 1; + return 0; + +} + + +void +ppc460ex_disable_dma(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id) +{ + unsigned int control; + ppc460ex_plb_dma_ch_t *dma_chan = adev->chan[ch_id]; + + if (!dma_chan->in_use) { + printk("disable_dma: channel %d not in use\n", ch_id); + return; + } + + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk("disable_dma: bad channel: %d\n", ch_id); + return; + } + + control = mfdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8)); + control &= ~DMA_CE_ENABLE; + mtdcr(DCR_DMA2P40_CR0 + (ch_id * 0x8), control); + + dma_chan->in_use = 0; +} + + + + +/* + * Clears the channel status bits + */ +int ppc460ex_clear_dma_status(unsigned int ch_id) +{ + if (ch_id >= MAX_PPC460EX_DMA_CHANNELS) { + printk("KERN_ERR %s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + mtdcr(DCR_DMA2P40_SR, ((u32)DMA_CH0_ERR | (u32)DMA_CS0 | (u32)DMA_TS0) >> ch_id); + return DMA_STATUS_GOOD; + +} + + +/** + * ppc460ex_dma_eot_handler - end of transfer interrupt handler + */ +irqreturn_t ppc460ex_4chan_dma_eot_handler(int irq, void *data) +{ + unsigned int data_read = 0; + unsigned int try_cnt = 0; + + //printk("transfer complete\n"); + data_read = mfdcr(DCR_DMA2P40_SR); + //printk("%s: status 0x%08x\n", __FUNCTION__, data_read); + + do{ + //while bit 3 TC done is 0 + data_read = mfdcr(DCR_DMA2P40_SR); + if (data_read & 0x00800000 ) {printk("test FAIL\n"); } //see if error bit is set + }while(((data_read & 0x80000000) != 0x80000000) && ++try_cnt <= 10);// TC is now 0 + + data_read = mfdcr(DCR_DMA2P40_SR); + while (data_read & 0x00000800){ //while channel is busy + data_read = mfdcr(DCR_DMA2P40_SR); + printk("%s: status for busy 0x%08x\n", __FUNCTION__, data_read); + } + mtdcr(DCR_DMA2P40_SR, 0xffffffff); + + + + return IRQ_HANDLED; +} + + + +static struct of_device_id dma_per_chan_match[] = { + { + .compatible = "amcc,dma-4channel", + }, + {}, +}; + + + + +#if 0 +/*** test code ***/ +static int ppc460ex_dma_memcpy_self_test(ppc460ex_plb_dma_dev_t *device, unsigned int dma_ch_id) +{ + ppc460ex_plb_dma_ch_t p_init; + int res = 0, i; + unsigned int control; + phys_addr_t *src; + phys_addr_t *dest; + + phys_addr_t *gap; + + phys_addr_t dma_dest, dma_src; + + src = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!src) + return -ENOMEM; + gap = kzalloc(200, GFP_KERNEL); + if (!gap) + return -ENOMEM; + + + + dest = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!dest) { + kfree(src); + return -ENOMEM; + } + + printk("src = 0x%08x\n", (unsigned int)src); + printk("gap = 0x%08x\n", (unsigned int)gap); + printk("dest = 0x%08x\n", (unsigned int)dest); + + /* Fill in src buffer */ + for (i = 0; i < TEST_SIZE; i++) + ((u8*)src)[i] = (u8)i; + + printk("dump src\n"); + DMA_HEXDUMP(src, TEST_SIZE); + DMA_HEXDUMP(dest, TEST_SIZE); +#if 1 + dma_src = dma_map_single(p_init.device->dev, src, TEST_SIZE, + DMA_TO_DEVICE); + dma_dest = dma_map_single(p_init.device->dev, dest, TEST_SIZE, + DMA_FROM_DEVICE); +#endif + printk("%s:channel = %d chan 0x%08x\n", __FUNCTION__, device->chan[dma_ch_id]->chan_id, + (unsigned int)(device->chan)); + + p_init.polarity = 0; + p_init.pwidth = PW_32; + p_init.in_use = 0; + p_init.sai = 1; + p_init.dai = 1; + res = ppc460ex_init_dma_channel(device, dma_ch_id, &p_init); + + if (res) { + printk("%32s: init_dma_channel return %d\n", + __FUNCTION__, res); + } + ppc460ex_clear_dma_status(dma_ch_id); + + ppc460ex_set_src_addr(dma_ch_id, dma_src); + ppc460ex_set_dst_addr(dma_ch_id, dma_dest); + + ppc460ex_set_dma_mode(device, dma_ch_id, DMA_MODE_MM); + ppc460ex_set_dma_count(device, dma_ch_id, TEST_SIZE); + + res = ppc460ex_enable_dma_interrupt(device, dma_ch_id); + if (res) { + printk("%32s: en/disable_dma_interrupt\n", + __FUNCTION__); + } + + + if (dma_ch_id == 0) + control = mfdcr(DCR_DMA2P40_CR0); + else if (dma_ch_id == 1) + control = mfdcr(DCR_DMA2P40_CR1); + + + control &= ~(SET_DMA_BEN(1)); + control &= ~(SET_DMA_PSC(3)); + control &= ~(SET_DMA_PWC(0x3f)); + control &= ~(SET_DMA_PHC(0x7)); + control &= ~(SET_DMA_PL(1)); + + + + if (dma_ch_id == 0) + mtdcr(DCR_DMA2P40_CR0, control); + else if (dma_ch_id == 1) + mtdcr(DCR_DMA2P40_CR1, control); + + + ppc460ex_enable_dma(device, dma_ch_id); + + + if (memcmp(src, dest, TEST_SIZE)) { + printk("Self-test copy failed compare, disabling\n"); + res = -ENODEV; + goto out; + } + + + return 0; + + out: kfree(src); + kfree(dest); + return res; + +} + + + +static int test1(void) +{ + void *src, *dest; + void *src1, *dest1; + int i; + unsigned int chan; + + src = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!src) + return -ENOMEM; + + dest = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!dest) { + kfree(src); + return -ENOMEM; + } + + src1 = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!src1) + return -ENOMEM; + + dest1 = kzalloc(TEST_SIZE, GFP_KERNEL); + if (!dest1) { + kfree(src1); + return -ENOMEM; + } + + /* Fill in src buffer */ + for (i = 0; i < TEST_SIZE; i++) + ((u8*)src)[i] = (u8)i; + + /* Fill in src buffer */ + for (i = 0; i < TEST_SIZE; i++) + ((u8*)src1)[i] = (u8)0xaa; + +#ifdef DEBUG_TEST + DMA_HEXDUMP(src, TEST_SIZE); + DMA_HEXDUMP(dest, TEST_SIZE); + DMA_HEXDUMP(src1, TEST_SIZE); + DMA_HEXDUMP(dest1, TEST_SIZE); +#endif + chan = ppc460ex_get_dma_channel(); + +#ifdef ENABLE_SGL + test_sgdma_memcpy(src, dest, src1, dest1, TEST_SIZE, chan); +#endif + test_dma_memcpy(src, dest, TEST_SIZE, chan); + + + out: kfree(src); + kfree(dest); + kfree(src1); + kfree(dest1); + + return 0; + +} +#endif + + + +/******************************************************************************* + * Module Initialization Routine + ******************************************************************************* + */ +int __devinit ppc460ex_dma_per_chan_probe(struct of_device *ofdev, + const struct of_device_id *match) +{ + int ret=0; + //ppc460ex_plb_dma_dev_t *adev; + ppc460ex_plb_dma_ch_t *new_chan; + int err; + + + + adev = dev_get_drvdata(ofdev->dev.parent); + BUG_ON(!adev); + /* create a device */ + if ((new_chan = kzalloc(sizeof(*new_chan), GFP_KERNEL)) == NULL) { + printk("ERROR:No Free memory for allocating dma channels\n"); + ret = -ENOMEM; + goto err; + } + + err = of_address_to_resource(ofdev->node,0,&new_chan->reg); + if (err) { + printk("ERROR:Can't get %s property reg\n", __FUNCTION__); + goto err; + } + new_chan->device = adev; + new_chan->reg_base = ioremap(new_chan->reg.start,new_chan->reg.end - new_chan->reg.start + 1); +#if 1 + printk("PPC460ex PLB DMA engine @0x%02X_%08X size %d\n", + (u32)(new_chan->reg.start >> 32), + (u32)new_chan->reg.start, + (u32)(new_chan->reg.end - new_chan->reg.start + 1)); +#endif + + switch(new_chan->reg.start) { + case 0x100: + new_chan->chan_id = 0; + break; + case 0x108: + new_chan->chan_id = 1; + break; + case 0x110: + new_chan->chan_id = 2; + break; + case 0x118: + new_chan->chan_id = 3; + break; + } + new_chan->chan_id = ((new_chan->reg.start - 0x100)& 0xfff) >> 3; + printk("new_chan->chan_id 0x%x\n",new_chan->chan_id); + adev->chan[new_chan->chan_id] = new_chan; + printk("new_chan->chan->chan_id 0x%x\n",adev->chan[new_chan->chan_id]->chan_id); + //adev->chan[new_chan->chan_id]->reg_base = new_chan->reg_base; + + return 0; + + err: + return ret; + +} + +int __devinit ppc460ex_dma_4chan_probe(struct of_device *ofdev, + const struct of_device_id *match) +{ + int ret=0, irq = 0; + //ppc460ex_plb_dma_dev_t *adev; + ppc460ex_plb_dma_ch_t *chan = NULL; + + + /* create a device */ + if ((adev = kzalloc(sizeof(*adev), GFP_KERNEL)) == NULL) { + ret = -ENOMEM; + goto err_adev_alloc; + } + adev->dev = &ofdev->dev; +#if !defined(CONFIG_APM82181) + err = of_address_to_resource(ofdev->node,0,&adev->reg); + if(err) { + printk(KERN_ERR"Can't get %s property 'reg'\n",ofdev->node->full_name); + } +#endif + printk(KERN_INFO"Probing AMCC DMA driver\n"); +#if !defined(CONFIG_APM82181) + adev->reg_base = ioremap(adev->reg.start, adev->reg.end - adev->reg.start + 1); +#endif + +#if 1 + irq = of_irq_to_resource(ofdev->node, 0, NULL); + if (irq >= 0) { + ret = request_irq(irq, ppc460ex_4chan_dma_eot_handler, + IRQF_DISABLED, "Peripheral DMA0/1", chan); + if (ret) { + ret = -EIO; + goto err_irq; + } + //irq = platform_get_irq(adev, 0); + /* only DMA engines have a separate err IRQ + * so it's Ok if irq < 0 in XOR case + */ + } else + ret = -ENXIO; + +#if !defined(CONFIG_APM82181) + printk("PPC4xx PLB DMA engine @0x%02X_%08X size %d IRQ %d \n", + (u32)(adev->reg.start >> 32), + (u32)adev->reg.start, + (u32)(adev->reg.end - adev->reg.start + 1), + irq); +#else + printk("PPC4xx PLB DMA engine IRQ %d\n", irq); +#endif +#endif + dev_set_drvdata(&(ofdev->dev),adev); + of_platform_bus_probe(ofdev->node,dma_per_chan_match,&ofdev->dev); + + + //ppc460ex_dma_memcpy_self_test(adev, 0); + //test1(); + + + return 0; + + +err_adev_alloc: + //release_mem_region(adev->reg.start, adev->reg.end - adev->reg.start); +err_irq: + kfree(chan); + + return ret; +} + + +static struct of_device_id dma_4chan_match[] = { + { + .compatible = "amcc,dma", + }, + {}, +}; + +struct of_platform_driver ppc460ex_dma_4chan_driver = { + .name = "plb_dma", + .match_table = dma_4chan_match, + .probe = ppc460ex_dma_4chan_probe, +}; + +struct of_platform_driver ppc460ex_dma_per_chan_driver = { + .name = "dma-4channel", + .match_table = dma_per_chan_match, + .probe = ppc460ex_dma_per_chan_probe, +}; + + +static int __init mod_init (void) +{ + printk("%s:%d\n", __FUNCTION__, __LINE__); + return of_register_platform_driver(&ppc460ex_dma_4chan_driver); + printk("here 2\n"); +} + +static void __exit mod_exit(void) +{ + of_unregister_platform_driver(&ppc460ex_dma_4chan_driver); +} + +static int __init ppc460ex_dma_per_chan_init (void) +{ + printk("%s:%d\n", __FUNCTION__, __LINE__); + return of_register_platform_driver(&ppc460ex_dma_per_chan_driver); + printk("here 3\n"); +} + +static void __exit ppc460ex_dma_per_chan_exit(void) +{ + of_unregister_platform_driver(&ppc460ex_dma_per_chan_driver); +} + +subsys_initcall(ppc460ex_dma_per_chan_init); +subsys_initcall(mod_init); + +//module_exit(mod_exit); + +//module_exit(ppc460ex_dma_per_chan_exit); + +MODULE_DESCRIPTION("AMCC PPC460EX 4 channel Engine Driver"); +MODULE_LICENSE("GPL"); + +EXPORT_SYMBOL_GPL(ppc460ex_get_dma_status); +EXPORT_SYMBOL_GPL(ppc460ex_set_src_addr); +EXPORT_SYMBOL_GPL(ppc460ex_set_dst_addr); +EXPORT_SYMBOL_GPL(ppc460ex_set_dma_mode); +EXPORT_SYMBOL_GPL(ppc460ex_set_dma_count); +EXPORT_SYMBOL_GPL(ppc460ex_enable_dma_interrupt); +EXPORT_SYMBOL_GPL(ppc460ex_init_dma_channel); +EXPORT_SYMBOL_GPL(ppc460ex_enable_dma); +EXPORT_SYMBOL_GPL(ppc460ex_disable_dma); +EXPORT_SYMBOL_GPL(ppc460ex_clear_dma_status); +EXPORT_SYMBOL_GPL(ppc460ex_get_dma_residue); +EXPORT_SYMBOL_GPL(ppc460ex_disable_dma_interrupt); +EXPORT_SYMBOL_GPL(ppc460ex_get_channel_config); +EXPORT_SYMBOL_GPL(ppc460ex_set_channel_priority); +EXPORT_SYMBOL_GPL(ppc460ex_get_peripheral_width); +EXPORT_SYMBOL_GPL(ppc460ex_enable_burst); +EXPORT_SYMBOL_GPL(ppc460ex_disable_burst); +EXPORT_SYMBOL_GPL(ppc460ex_set_burst_size); + +/************************************************************************/ diff --git a/drivers/dma/ppc460ex_4chan_dma.h b/drivers/dma/ppc460ex_4chan_dma.h new file mode 100755 index 00000000000..c9448f34de4 --- /dev/null +++ b/drivers/dma/ppc460ex_4chan_dma.h @@ -0,0 +1,531 @@ + + +#include <linux/types.h> + + + + +#define DMA_HEXDUMP(b, l) \ + print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, \ + 16, 1, (b), (l), false); + + +#define MAX_PPC460EX_DMA_CHANNELS 4 + + +#define DCR_DMA0_BASE 0x200 +#define DCR_DMA1_BASE 0x208 +#define DCR_DMA2_BASE 0x210 +#define DCR_DMA3_BASE 0x218 +#define DCR_DMASR_BASE 0x220 + + + + + + +/* DMA Registers */ +#define DCR_DMA2P40_CR0 (DCR_DMA0_BASE + 0x0) /* DMA Channel Control 0 */ +#define DCR_DMA2P40_CTC0 (DCR_DMA0_BASE + 0x1) /* DMA Count 0 */ +#define DCR_DMA2P40_SAH0 (DCR_DMA0_BASE + 0x2) /* DMA Src Addr High 0 */ +#define DCR_DMA2P40_SAL0 (DCR_DMA0_BASE + 0x3) /* DMA Src Addr Low 0 */ +#define DCR_DMA2P40_DAH0 (DCR_DMA0_BASE + 0x4) /* DMA Dest Addr High 0 */ +#define DCR_DMA2P40_DAL0 (DCR_DMA0_BASE + 0x5) /* DMA Dest Addr Low 0 */ +#define DCR_DMA2P40_SGH0 (DCR_DMA0_BASE + 0x6) /* DMA SG Desc Addr High 0 */ +#define DCR_DMA2P40_SGL0 (DCR_DMA0_BASE + 0x7) /* DMA SG Desc Addr Low 0 */ + +#define DCR_DMA2P40_CR1 (DCR_DMA1_BASE + 0x0) /* DMA Channel Control 1 */ +#define DCR_DMA2P40_CTC1 (DCR_DMA1_BASE + 0x1) /* DMA Count 1 */ +#define DCR_DMA2P40_SAH1 (DCR_DMA1_BASE + 0x2) /* DMA Src Addr High 1 */ +#define DCR_DMA2P40_SAL1 (DCR_DMA1_BASE + 0x3) /* DMA Src Addr Low 1 */ +#define DCR_DMA2P40_DAH1 (DCR_DMA1_BASE + 0x4) /* DMA Dest Addr High 1 */ +#define DCR_DMA2P40_DAL1 (DCR_DMA1_BASE + 0x5) /* DMA Dest Addr Low 1 */ +#define DCR_DMA2P40_SGH1 (DCR_DMA1_BASE + 0x6) /* DMA SG Desc Addr High 1 */ +#define DCR_DMA2P40_SGL1 (DCR_DMA1_BASE + 0x7) /* DMA SG Desc Addr Low 1 */ + +#define DCR_DMA2P40_CR2 (DCR_DMA2_BASE + 0x0) /* DMA Channel Control 2 */ +#define DCR_DMA2P40_CTC2 (DCR_DMA2_BASE + 0x1) /* DMA Count 2 */ +#define DCR_DMA2P40_SAH2 (DCR_DMA2_BASE + 0x2) /* DMA Src Addr High 2 */ +#define DCR_DMA2P40_SAL2 (DCR_DMA2_BASE + 0x3) /* DMA Src Addr Low 2 */ +#define DCR_DMA2P40_DAH2 (DCR_DMA2_BASE + 0x4) /* DMA Dest Addr High 2 */ +#define DCR_DMA2P40_DAL2 (DCR_DMA2_BASE + 0x5) /* DMA Dest Addr Low 2 */ +#define DCR_DMA2P40_SGH2 (DCR_DMA2_BASE + 0x6) /* DMA SG Desc Addr High 2 */ +#define DCR_DMA2P40_SGL2 (DCR_DMA2_BASE + 0x7) /* DMA SG Desc Addr Low 2 */ + +#define DCR_DMA2P40_CR3 (DCR_DMA3_BASE + 0x0) /* DMA Channel Control 3 */ +#define DCR_DMA2P40_CTC3 (DCR_DMA3_BASE + 0x1) /* DMA Count 3 */ +#define DCR_DMA2P40_SAH3 (DCR_DMA3_BASE + 0x2) /* DMA Src Addr High 3 */ +#define DCR_DMA2P40_SAL3 (DCR_DMA3_BASE + 0x3) /* DMA Src Addr Low 3 */ +#define DCR_DMA2P40_DAH3 (DCR_DMA3_BASE + 0x4) /* DMA Dest Addr High 3 */ +#define DCR_DMA2P40_DAL3 (DCR_DMA3_BASE + 0x5) /* DMA Dest Addr Low 3 */ +#define DCR_DMA2P40_SGH3 (DCR_DMA3_BASE + 0x6) /* DMA SG Desc Addr High 3 */ +#define DCR_DMA2P40_SGL3 (DCR_DMA3_BASE + 0x7) /* DMA SG Desc Addr Low 3 */ + +#define DCR_DMA2P40_SR (DCR_DMASR_BASE + 0x0) /* DMA Status Register */ +#define DCR_DMA2P40_SGC (DCR_DMASR_BASE + 0x3) /* DMA Scatter/Gather Command */ +#define DCR_DMA2P40_SLP (DCR_DMASR_BASE + 0x5) /* DMA Sleep Register */ +#define DCR_DMA2P40_POL (DCR_DMASR_BASE + 0x6) /* DMA Polarity Register */ + + + +/* + * Function return status codes + * These values are used to indicate whether or not the function + * call was successful, or a bad/invalid parameter was passed. + */ +#define DMA_STATUS_GOOD 0 +#define DMA_STATUS_BAD_CHANNEL 1 +#define DMA_STATUS_BAD_HANDLE 2 +#define DMA_STATUS_BAD_MODE 3 +#define DMA_STATUS_NULL_POINTER 4 +#define DMA_STATUS_OUT_OF_MEMORY 5 +#define DMA_STATUS_SGL_LIST_EMPTY 6 +#define DMA_STATUS_GENERAL_ERROR 7 +#define DMA_STATUS_CHANNEL_NOTFREE 8 + +#define DMA_CHANNEL_BUSY 0x80000000 + +/* + * These indicate status as returned from the DMA Status Register. + */ +#define DMA_STATUS_NO_ERROR 0 +#define DMA_STATUS_CS 1 /* Count Status */ +#define DMA_STATUS_TS 2 /* Transfer Status */ +#define DMA_STATUS_DMA_ERROR 3 /* DMA Error Occurred */ +#define DMA_STATUS_DMA_BUSY 4 /* The channel is busy */ + +/* + * DMA Channel Control Registers + */ +#ifdef CONFIG_44x +#define PPC4xx_DMA_64BIT +#define DMA_CR_OFFSET 1 +#else +#define DMA_CR_OFFSET 0 +#endif + +#define DMA_CE_ENABLE (1<<31) /* DMA Channel Enable */ +#define SET_DMA_CE_ENABLE(x) (((x)&0x1)<<31) +#define GET_DMA_CE_ENABLE(x) (((x)&DMA_CE_ENABLE)>>31) + +#define DMA_CIE_ENABLE (1<<30) /* DMA Channel Interrupt Enable */ +#define SET_DMA_CIE_ENABLE(x) (((x)&0x1)<<30) +#define GET_DMA_CIE_ENABLE(x) (((x)&DMA_CIE_ENABLE)>>30) + +#define DMA_TD (1<<29) +#define SET_DMA_TD(x) (((x)&0x1)<<29) +#define GET_DMA_TD(x) (((x)&DMA_TD)>>29) + +#define DMA_PL (1<<28) /* Peripheral Location */ +#define SET_DMA_PL(x) (((x)&0x1)<<28) +#define GET_DMA_PL(x) (((x)&DMA_PL)>>28) + +#define EXTERNAL_PERIPHERAL 0 +#define INTERNAL_PERIPHERAL 1 + +#define SET_DMA_PW(x) (((x)&0x7)<<(26-DMA_CR_OFFSET)) /* Peripheral Width */ +#define DMA_PW_MASK SET_DMA_PW(7) +#define PW_8 0 +#define PW_16 1 +#define PW_32 2 +#define PW_64 3 +#define PW_128 4 + + +#define GET_DMA_PW(x) (((x)&DMA_PW_MASK)>>(26-DMA_CR_OFFSET)) + +#define DMA_DAI (1<<(25-DMA_CR_OFFSET)) /* Destination Address Increment */ +#define SET_DMA_DAI(x) (((x)&0x1)<<(25-DMA_CR_OFFSET)) + +#define DMA_SAI (1<<(24-DMA_CR_OFFSET)) /* Source Address Increment */ +#define SET_DMA_SAI(x) (((x)&0x1)<<(24-DMA_CR_OFFSET)) + +#define DMA_BEN (1<<(23-DMA_CR_OFFSET)) /* Buffer Enable */ +#define SET_DMA_BEN(x) (((x)&0x1)<<(23-DMA_CR_OFFSET)) + +#define SET_DMA_TM(x) (((x)&0x3)<<(21-DMA_CR_OFFSET)) /* Transfer Mode */ +#define DMA_TM_MASK SET_DMA_TM(3) +#define TM_PERIPHERAL 0 /* Peripheral */ +#define TM_RESERVED 1 /* Reserved */ +#define TM_S_MM 2 /* Memory to Memory */ +#define TM_D_MM 3 /* Device Paced Memory to Memory */ +#define GET_DMA_TM(x) (((x)&DMA_TM_MASK)>>(21-DMA_CR_OFFSET)) + +#define SET_DMA_PSC(x) (((x)&0x3)<<(19-DMA_CR_OFFSET)) /* Peripheral Setup Cycles */ +#define DMA_PSC_MASK SET_DMA_PSC(3) +#define GET_DMA_PSC(x) (((x)&DMA_PSC_MASK)>>(19-DMA_CR_OFFSET)) + +#define SET_DMA_PWC(x) (((x)&0x3F)<<(13-DMA_CR_OFFSET)) /* Peripheral Wait Cycles */ +#define DMA_PWC_MASK SET_DMA_PWC(0x3F) +#define GET_DMA_PWC(x) (((x)&DMA_PWC_MASK)>>(13-DMA_CR_OFFSET)) + +#define SET_DMA_PHC(x) (((x)&0x7)<<(10-DMA_CR_OFFSET)) /* Peripheral Hold Cycles */ +#define DMA_PHC_MASK SET_DMA_PHC(0x7) +#define GET_DMA_PHC(x) (((x)&DMA_PHC_MASK)>>(10-DMA_CR_OFFSET)) + +#define DMA_ETD_OUTPUT (1<<(9-DMA_CR_OFFSET)) /* EOT pin is a TC output */ +#define SET_DMA_ETD(x) (((x)&0x1)<<(9-DMA_CR_OFFSET)) + +#define DMA_TCE_ENABLE (1<<(8-DMA_CR_OFFSET)) +#define SET_DMA_TCE(x) (((x)&0x1)<<(8-DMA_CR_OFFSET)) + +#define DMA_DEC (1<<(2)) /* Address Decrement */ +#define SET_DMA_DEC(x) (((x)&0x1)<<2) +#define GET_DMA_DEC(x) (((x)&DMA_DEC)>>2) + + +/* + * Transfer Modes + * These modes are defined in a way that makes it possible to + * simply "or" in the value in the control register. + */ + +#define DMA_MODE_MM (SET_DMA_TM(TM_S_MM)) /* memory to memory */ + + /* Device-paced memory to memory, */ + /* device is at source address */ +#define DMA_MODE_MM_DEVATSRC (DMA_TD | SET_DMA_TM(TM_D_MM)) + + /* Device-paced memory to memory, */ + /* device is at destination address */ +#define DMA_MODE_MM_DEVATDST (SET_DMA_TM(TM_D_MM)) + +#define SGL_LIST_SIZE 16384 +#define DMA_PPC4xx_SIZE SGL_LIST_SIZE + +#define SET_DMA_PRIORITY(x) (((x)&0x3)<<(6-DMA_CR_OFFSET)) /* DMA Channel Priority */ +#define DMA_PRIORITY_MASK SET_DMA_PRIORITY(3) +#define PRIORITY_LOW 0 +#define PRIORITY_MID_LOW 1 +#define PRIORITY_MID_HIGH 2 +#define PRIORITY_HIGH 3 +#define GET_DMA_PRIORITY(x) (((x)&DMA_PRIORITY_MASK)>>(6-DMA_CR_OFFSET)) + + +#define SET_DMA_PREFETCH(x) (((x)&0x3)<<(4-DMA_CR_OFFSET)) /* Memory Read Prefetch */ +#define DMA_PREFETCH_MASK SET_DMA_PREFETCH(3) +#define PREFETCH_1 0 /* Prefetch 1 Double Word */ +#define PREFETCH_2 1 +#define PREFETCH_4 2 +#define GET_DMA_PREFETCH(x) (((x)&DMA_PREFETCH_MASK)>>(4-DMA_CR_OFFSET)) + +#define DMA_PCE (1<<(3-DMA_CR_OFFSET)) /* Parity Check Enable */ +#define SET_DMA_PCE(x) (((x)&0x1)<<(3-DMA_CR_OFFSET)) +#define GET_DMA_PCE(x) (((x)&DMA_PCE)>>(3-DMA_CR_OFFSET)) + +/* + * DMA Polarity Configuration Register + */ +#define DMAReq_ActiveLow(chan) (1<<(31-(chan*3))) +#define DMAAck_ActiveLow(chan) (1<<(30-(chan*3))) +#define EOT_ActiveLow(chan) (1<<(29-(chan*3))) /* End of Transfer */ + +/* + * DMA Sleep Mode Register + */ +#define SLEEP_MODE_ENABLE (1<<21) + +/* + * DMA Status Register + */ +#define DMA_CS0 (1<<31) /* Terminal Count has been reached */ +#define DMA_CS1 (1<<30) +#define DMA_CS2 (1<<29) +#define DMA_CS3 (1<<28) + +#define DMA_TS0 (1<<27) /* End of Transfer has been requested */ +#define DMA_TS1 (1<<26) +#define DMA_TS2 (1<<25) +#define DMA_TS3 (1<<24) + +#define DMA_CH0_ERR (1<<23) /* DMA Chanel 0 Error */ +#define DMA_CH1_ERR (1<<22) +#define DMA_CH2_ERR (1<<21) +#define DMA_CH3_ERR (1<<20) + +#define DMA_IN_DMA_REQ0 (1<<19) /* Internal DMA Request is pending */ +#define DMA_IN_DMA_REQ1 (1<<18) +#define DMA_IN_DMA_REQ2 (1<<17) +#define DMA_IN_DMA_REQ3 (1<<16) + +#define DMA_EXT_DMA_REQ0 (1<<15) /* External DMA Request is pending */ +#define DMA_EXT_DMA_REQ1 (1<<14) +#define DMA_EXT_DMA_REQ2 (1<<13) +#define DMA_EXT_DMA_REQ3 (1<<12) + +#define DMA_CH0_BUSY (1<<11) /* DMA Channel 0 Busy */ +#define DMA_CH1_BUSY (1<<10) +#define DMA_CH2_BUSY (1<<9) +#define DMA_CH3_BUSY (1<<8) + +#define DMA_SG0 (1<<7) /* DMA Channel 0 Scatter/Gather in progress */ +#define DMA_SG1 (1<<6) +#define DMA_SG2 (1<<5) +#define DMA_SG3 (1<<4) + +/* DMA Channel Count Register */ +#define DMA_CTC_TCIE (1<<29) /* Terminal Count Interrupt Enable */ +#define DMA_CTC_ETIE (1<<28) /* EOT Interupt Enable */ +#define DMA_CTC_EIE (1<<27) /* Error Interrupt Enable */ +#define DMA_CTC_BTEN (1<<23) /* Burst Enable/Disable bit */ +#define DMA_CTC_BSIZ_MSK (3<<21) /* Mask of the Burst size bits */ +#define DMA_CTC_BSIZ_2 (0) +#define DMA_CTC_BSIZ_4 (1<<21) +#define DMA_CTC_BSIZ_8 (2<<21) +#define DMA_CTC_BSIZ_16 (3<<21) +#define DMA_CTC_TC_MASK 0xFFFFF + +/* + * DMA SG Command Register + */ +#define SSG_ENABLE(chan) (1<<(31-chan)) /* Start Scatter Gather */ +#define SSG_MASK_ENABLE(chan) (1<<(15-chan)) /* Enable writing to SSG0 bit */ + + +/* + * DMA Scatter/Gather Descriptor Bit fields + */ +#define SG_LINK (1<<31) /* Link */ +#define SG_TCI_ENABLE (1<<29) /* Enable Terminal Count Interrupt */ +#define SG_ETI_ENABLE (1<<28) /* Enable End of Transfer Interrupt */ +#define SG_ERI_ENABLE (1<<27) /* Enable Error Interrupt */ +#define SG_COUNT_MASK 0xFFFF /* Count Field */ + +#define SET_DMA_CONTROL \ + (SET_DMA_CIE_ENABLE(p_init->int_enable) | /* interrupt enable */ \ + SET_DMA_BEN(p_init->buffer_enable) | /* buffer enable */\ + SET_DMA_ETD(p_init->etd_output) | /* end of transfer pin */ \ + SET_DMA_TCE(p_init->tce_enable) | /* terminal count enable */ \ + SET_DMA_PL(p_init->pl) | /* peripheral location */ \ + SET_DMA_DAI(p_init->dai) | /* dest addr increment */ \ + SET_DMA_SAI(p_init->sai) | /* src addr increment */ \ + SET_DMA_PRIORITY(p_init->cp) | /* channel priority */ \ + SET_DMA_PW(p_init->pwidth) | /* peripheral/bus width */ \ + SET_DMA_PSC(p_init->psc) | /* peripheral setup cycles */ \ + SET_DMA_PWC(p_init->pwc) | /* peripheral wait cycles */ \ + SET_DMA_PHC(p_init->phc) | /* peripheral hold cycles */ \ + SET_DMA_PREFETCH(p_init->pf) /* read prefetch */) + +#define GET_DMA_POLARITY(chan) (DMAReq_ActiveLow(chan) | DMAAck_ActiveLow(chan) | EOT_ActiveLow(chan)) + + +/** + * struct ppc460ex_dma_device - internal representation of an DMA device + * @pdev: Platform device + * @id: HW DMA Device selector + * @dma_desc_pool: base of DMA descriptor region (DMA address) + * @dma_desc_pool_virt: base of DMA descriptor region (CPU address) + * @common: embedded struct dma_device + */ +typedef struct ppc460ex_plb_dma_device { + //struct platform_device *pdev; + void __iomem *reg_base; + struct device *dev; + struct resource reg; /* Resource for register */ + int id; + struct ppc460ex_plb_dma_chan *chan[MAX_PPC460EX_DMA_CHANNELS]; + wait_queue_head_t queue; +} ppc460ex_plb_dma_dev_t; + +typedef uint32_t sgl_handle_t; +/** + * struct ppc460ex_dma_chan - internal representation of an ADMA channel + * @lock: serializes enqueue/dequeue operations to the slot pool + * @device: parent device + * @chain: device chain view of the descriptors + * @common: common dmaengine channel object members + * @all_slots: complete domain of slots usable by the channel + * @reg: Resource for register + * @pending: allows batching of hardware operations + * @completed_cookie: identifier for the most recently completed operation + * @slots_allocated: records the actual size of the descriptor slot pool + * @hw_chain_inited: h/w descriptor chain initialization flag + * @irq_tasklet: bottom half where ppc460ex_adma_slot_cleanup runs + * @needs_unmap: if buffers should not be unmapped upon final processing + */ +typedef struct ppc460ex_plb_dma_chan { + void __iomem *reg_base; + struct ppc460ex_plb_dma_device *device; + struct timer_list cleanup_watchdog; + struct resource reg; /* Resource for register */ + unsigned int chan_id; + struct tasklet_struct irq_tasklet; + sgl_handle_t *phandle; + unsigned short in_use; /* set when channel is being used, clr when + * available. + */ + /* + * Valid polarity settings: + * DMAReq_ActiveLow(n) + * DMAAck_ActiveLow(n) + * EOT_ActiveLow(n) + * + * n is 0 to max dma chans + */ + unsigned int polarity; + + char buffer_enable; /* Boolean: buffer enable */ + char tce_enable; /* Boolean: terminal count enable */ + char etd_output; /* Boolean: eot pin is a tc output */ + char pce; /* Boolean: parity check enable */ + + /* + * Peripheral location: + * INTERNAL_PERIPHERAL (UART0 on the 405GP) + * EXTERNAL_PERIPHERAL + */ + char pl; /* internal/external peripheral */ + + /* + * Valid pwidth settings: + * PW_8 + * PW_16 + * PW_32 + * PW_64 + */ + unsigned int pwidth; + + char dai; /* Boolean: dst address increment */ + char sai; /* Boolean: src address increment */ + + /* + * Valid psc settings: 0-3 + */ + unsigned int psc; /* Peripheral Setup Cycles */ + + /* + * Valid pwc settings: + * 0-63 + */ + unsigned int pwc; /* Peripheral Wait Cycles */ + + /* + * Valid phc settings: + * 0-7 + */ + unsigned int phc; /* Peripheral Hold Cycles */ + + /* + * Valid cp (channel priority) settings: + * PRIORITY_LOW + * PRIORITY_MID_LOW + * PRIORITY_MID_HIGH + * PRIORITY_HIGH + */ + unsigned int cp; /* channel priority */ + + /* + * Valid pf (memory read prefetch) settings: + * + * PREFETCH_1 + * PREFETCH_2 + * PREFETCH_4 + */ + unsigned int pf; /* memory read prefetch */ + + /* + * Boolean: channel interrupt enable + * NOTE: for sgl transfers, only the last descriptor will be setup to + * interrupt. + */ + char int_enable; + + char shift; /* easy access to byte_count shift, based on */ + /* the width of the channel */ + + uint32_t control; /* channel control word */ + + /* These variabled are used ONLY in single dma transfers */ + unsigned int mode; /* transfer mode */ + phys_addr_t addr; + char ce; /* channel enable */ + char int_on_final_sg;/* for scatter/gather - only interrupt on last sg */ + +} ppc460ex_plb_dma_ch_t; + +/* + * PPC44x DMA implementations have a slightly different + * descriptor layout. Probably moved about due to the + * change to 64-bit addresses and link pointer. I don't + * know why they didn't just leave control_count after + * the dst_addr. + */ +#ifdef PPC4xx_DMA_64BIT +typedef struct { + uint32_t control; + uint32_t control_count; + phys_addr_t src_addr; + phys_addr_t dst_addr; + phys_addr_t next; +} ppc_sgl_t; +#else +typedef struct { + uint32_t control; + phys_addr_t src_addr; + phys_addr_t dst_addr; + uint32_t control_count; + uint32_t next; +} ppc_sgl_t; +#endif + + + +typedef struct { + unsigned int ch_id; + uint32_t control; /* channel ctrl word; loaded from each descrptr */ + uint32_t sgl_control; /* LK, TCI, ETI, and ERI bits in sgl descriptor */ + dma_addr_t dma_addr; /* dma (physical) address of this list */ + dma_addr_t dummy; /*Dummy variable to allow quad word alignment*/ + ppc_sgl_t *phead; + dma_addr_t phead_dma; + ppc_sgl_t *ptail; + dma_addr_t ptail_dma; +} sgl_list_info_t; + +typedef struct { + phys_addr_t *src_addr; + phys_addr_t *dst_addr; + phys_addr_t dma_src_addr; + phys_addr_t dma_dst_addr; +} pci_alloc_desc_t; + +#define PPC460EX_DMA_SGXFR_COMPLETE(id) (!((1 << (11-id)) & mfdcr(DCR_DMA2P40_SR))) +#define PPC460EX_DMA_CHAN_BUSY(id) ( (1 << (11-id)) & mfdcr(DCR_DMA2P40_SR) ) +#define DMA_STATUS(id) (mfdcr(DCR_DMA2P40_SR)) +#define CLEAR_DMA_STATUS(id) (mtdcr(DCR_DMA2P40_SR, 0xFFFFFFFF)) +#define PPC460EX_DMA_SGSTAT_FREE(id) (!((1 << (7-id)) & mfdcr(DCR_DMA2P40_SR)) ) +#define PPC460EX_DMA_TC_REACHED(id) ( (1 << (31-id)) & mfdcr(DCR_DMA2P40_SR) ) +#define PPC460EX_DMA_CHAN_XFR_COMPLETE(id) ( (!PPC460EX_DMA_CHAN_BUSY(id)) && (PPC460EX_DMA_TC_REACHED(id)) ) +#define PPC460EX_DMA_CHAN_SGXFR_COMPLETE(id) ( (!PPC460EX_DMA_CHAN_BUSY(id)) && PPC460EX_DMA_SGSTAT_FREE(id) ) +#define PPC460EX_DMA_SG_IN_PROGRESS(id) ( (1 << (7-id)) | (1 << (11-id)) ) +#define PPC460EX_DMA_SG_OP_COMPLETE(id) ( (PPC460EX_DMA_SG_IN_PROGRESS(id) & DMA_STATUS(id) ) == 0) + +extern ppc460ex_plb_dma_dev_t *adev; +int ppc460ex_init_dma_channel(ppc460ex_plb_dma_dev_t *adev, + unsigned int ch_id, + ppc460ex_plb_dma_ch_t *p_init); + +int ppc460ex_set_src_addr(int ch_id, phys_addr_t src_addr); + +int ppc460ex_set_dst_addr(int ch_id, phys_addr_t dst_addr); + +int ppc460ex_set_dma_mode(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, unsigned int mode); + +void ppc460ex_set_dma_count(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id, unsigned int count); + +int ppc460ex_enable_dma_interrupt(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id); + +int ppc460ex_enable_dma(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id); + +int ppc460ex_get_dma_channel(void); + +void ppc460ex_disable_dma(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id); + +int ppc460ex_clear_dma_status(unsigned int ch_id); + +#if 0 +extern int test_dma_memcpy(void *src, void *dst, unsigned int length, unsigned int dma_ch); + +extern int test_sgdma_memcpy(void *src, void *dst, void *src1, void *dst1, + unsigned int length, unsigned int dma_ch); +#endif diff --git a/drivers/dma/ppc460ex_4chan_sgdma.c b/drivers/dma/ppc460ex_4chan_sgdma.c new file mode 100755 index 00000000000..fb26dd76ce7 --- /dev/null +++ b/drivers/dma/ppc460ex_4chan_sgdma.c @@ -0,0 +1,1003 @@ +/* + * Copyright(c) 2008 Applied Micro Circuits Corporation(AMCC). All rights reserved. + * + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ + + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/async_tx.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/spinlock.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/of_platform.h> +#include <linux/uaccess.h> +#include <linux/proc_fs.h> +#include <asm/dcr-regs.h> +#include <asm/dcr.h> +#include <linux/delay.h> +#include <asm/cacheflush.h> +#include "ppc460ex_4chan_dma.h" +#include <asm/page.h> +#include <asm/time.h> +#include <linux/pipe_fs_i.h> +#include <linux/splice.h> + +#define SGDMA_MAX_POLL_COUNT 100000000 +#define SGDMA_POLL_DELAY 5 + +static phys_addr_t splice_src_dma_addrs[PIPE_BUFFERS]; +static dma_addr_t splice_dst_dma_addrs[PIPE_BUFFERS]; + +//#define DEBUG_SPLICE_DMA 1 +//#define SPLICE_DMA_COHERENT 1 +//#define DEBUG_SPLICE_DMA_TIMECAL 1 + +extern int ppc460ex_disable_dma_interrupt(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id); +extern int ppc460ex_disable_burst(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id); +extern int ppc460ex_enable_burst(ppc460ex_plb_dma_dev_t *adev, unsigned int ch_id); +void *dma_mem_page = NULL; + + + + + +#define dma_pr(x) printk(KERN_DEBUG,x) + + +int ppc460ex_set_sg_addr(int ch_id, phys_addr_t sg_addr) +{ + if (unlikely(ch_id >= MAX_PPC460EX_DMA_CHANNELS)) { + printk("%s: bad channel %d\n", __FUNCTION__, ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + +#ifdef PPC4xx_DMA_64BIT + mtdcr(DCR_DMA2P40_SGH0 + ch_id*8, sg_addr >> 32); +#endif + mtdcr(DCR_DMA2P40_SGL0 + ch_id*8, (u32)sg_addr); + + return 0; +} + +static int +poll_for_sgdma_done(int chan) +{ + int i; + volatile int status = 0; +#ifdef DEBUG_SPLICE_DMA_TIMECAL + u64 time1=0, time2=0, timediff=0; +#endif + +#ifdef DEBUG_SPLICE_DMA_TIMECAL + time1 = get_tbl() | (get_tbu() << 32); +#endif + for(i = 0; i < SGDMA_MAX_POLL_COUNT; i++) { +#ifdef DEBUG_SPLICE_DMA + if(i%16 == 0) + printk("%s:%s:%d - waiting %d\n", __FILE__, __FUNCTION__, __LINE__, i); +#endif + status = PPC460EX_DMA_CHAN_SGXFR_COMPLETE(chan); + if(status) { +#ifdef DEBUG_SPLICE_DMA + printk("%s:%s:%d - Breaking\n", __FILE__, __FUNCTION__, __LINE__); +#endif + break; + } + +#ifdef DEBUG_SPLICE_DMA + printk("status = %d dma_status = 0x%08x\n", status, DMA_STATUS(chan)); +#endif + //udelay(SGDMA_POLL_DELAY); + } +#ifdef DEBUG_SPLICE_DMA_TIMECAL + time2 = get_tbl() | (get_tbu() << 32); +#endif + +#ifdef DEBUG_SPLICE_DMA_TIMECAL + printk("%s:%s:%d time taken for transfer is %llu\n", + __FILE__, __FUNCTION__, __LINE__, time2-time1); +#endif + if(unlikely(i >= SGDMA_MAX_POLL_COUNT)) { + printk("%s:%s:%d - timeout\n", + __FILE__, __FUNCTION__, __LINE__); + return -ETIME; + } + + return 0; +} + +static int +get_transfer_width(u64 align) +{ + if(!(align & 0xF)) + return 128; + + if(!(align & 0x7)) + return 64; + + if(!(align & 0x3)) + return 32; + + if(!(align & 0x1)) + return 16; + + return 8; +} + + +/* + * Add a new sgl descriptor to the end of a scatter/gather list + * which was created by alloc_dma_handle(). + * + * For a memory to memory transfer, both dma addresses must be + * valid. For a peripheral to memory transfer, one of the addresses + * must be set to NULL, depending on the direction of the transfer: + * memory to peripheral: set dst_addr to NULL, + * peripheral to memory: set src_addr to NULL. + */ +int ppc460ex_add_dma_sgl(ppc460ex_plb_dma_dev_t *adev, + sgl_handle_t handle, + phys_addr_t src_addr, + phys_addr_t dst_addr, + unsigned int count) +{ + sgl_list_info_t *psgl = (sgl_list_info_t *)handle; + ppc460ex_plb_dma_ch_t *p_dma_ch; + u64 align; + int tr_width = 8; /* initial value 8 bits */ +#ifdef DEBUG_SPLICE_DMA + printk("%s:%s:%d - Filling in dma sgl list\n", __func__, __FILE__, __LINE__); +#endif + + if (unlikely(!handle)) { + printk("%s: null handle\n", __FUNCTION__); + return DMA_STATUS_BAD_HANDLE; + } + if (unlikely(psgl->ch_id >= MAX_PPC460EX_DMA_CHANNELS)) { + printk("%s: bad channel %d\n", __FUNCTION__, psgl->ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + p_dma_ch = adev->chan[psgl->ch_id]; + align = src_addr | dst_addr | count; + + tr_width = get_transfer_width(align); + switch(tr_width) { + case 128: + p_dma_ch->pwidth = PW_128; + break; + case 64: + p_dma_ch->pwidth = PW_64; + break; + case 32: + p_dma_ch->pwidth = PW_32; + break; + case 16: + p_dma_ch->pwidth = PW_16; + break; + default: + p_dma_ch->pwidth = PW_8; + break; + } + + { + int error = 0; + u64 aligned = + src_addr | dst_addr | count; + switch (p_dma_ch->pwidth) { + case PW_8: + break; + case PW_16: + if (aligned & 0x1) + error = 1; + break; + case PW_32: + if (aligned & 0x3) + error = 1; + break; + case PW_64: + if (aligned & 0x7) + error = 1; + break; + case PW_128: + if (aligned & 0xf) + error = 1; + break; + default: + printk("%s:set_dma_count: invalid bus width: 0x%x\n", __FUNCTION__, + p_dma_ch->pwidth); + return DMA_STATUS_GENERAL_ERROR; + } + if (unlikely(error)) + printk + ("Warning: set_dma_count count 0x%x bus width %d aligned= 0x%llx\n", + count, p_dma_ch->pwidth, aligned); + } + + p_dma_ch->shift = p_dma_ch->pwidth; + + + if (unlikely((unsigned) (psgl->ptail + 1) >= ((unsigned) psgl + SGL_LIST_SIZE))) { + printk("sgl handle out of memory \n"); + return DMA_STATUS_OUT_OF_MEMORY; + } + + if (!psgl->ptail) { + psgl->phead = (ppc_sgl_t *) + ((((unsigned) psgl + sizeof (sgl_list_info_t))) ); + psgl->phead_dma = (psgl->dma_addr + sizeof(sgl_list_info_t)) ; + psgl->ptail = psgl->phead; + psgl->ptail_dma = psgl->phead_dma; + } + else { + if(p_dma_ch->int_on_final_sg) { + /* mask out all dma interrupts, except error, on tail + before adding new tail. */ + psgl->ptail->control_count &= + ~(SG_TCI_ENABLE | SG_ETI_ENABLE); + } + psgl->ptail->next = psgl->ptail_dma + sizeof(ppc_sgl_t); + psgl->ptail++; + psgl->ptail_dma += sizeof(ppc_sgl_t); + } + psgl->ptail->control = psgl->control | SET_DMA_PW(p_dma_ch->pwidth); +#if !defined(CONFIG_APM82181) + /* Move to Highband segment to expect higher performance */ + psgl->ptail->src_addr = src_addr | (0x8ULL << 32); + psgl->ptail->dst_addr = dst_addr | (0x8ULL << 32); +#else /* APM821x1 */ + psgl->ptail->src_addr = src_addr; + psgl->ptail->dst_addr = dst_addr; +#endif +#ifdef DEBUG_SPLICE_DMA + psgl->ptail->control_count = (count >> p_dma_ch->shift); + // | psgl->sgl_control; +#endif + psgl->ptail->control_count = (count >> p_dma_ch->shift) | psgl->sgl_control; + + psgl->ptail->next = (uint32_t) NULL; + +#ifdef DEBUG_SPLICE_DMA + printk("count=%d control=0x%08x p_dma_ch->pwidth=%d bits=0x%08x\n", + count, psgl->ptail->control, p_dma_ch->pwidth, SET_DMA_PW(p_dma_ch->pwidth)); + printk("src_addr=0x%llx\n", psgl->ptail->src_addr); + printk("dst_addr=0x%llx\n", psgl->ptail->dst_addr); + printk("control_count=0x%08x\n", psgl->ptail->control_count); + printk("sgl_control=0x%08x\n", psgl->sgl_control); +#endif + + return DMA_STATUS_GOOD; + +} + + +/* + * Enable (start) the DMA described by the sgl handle. + */ +int ppc460ex_enable_dma_sgl(ppc460ex_plb_dma_dev_t *adev, sgl_handle_t handle) +{ + sgl_list_info_t *psgl = (sgl_list_info_t *)handle; + ppc460ex_plb_dma_ch_t *p_dma_ch; + uint32_t sg_command; + + + if (unlikely(!handle)) { + printk("%s: null handle\n", __FUNCTION__); + return DMA_STATUS_BAD_HANDLE; + } + if (unlikely(psgl->ch_id >= MAX_PPC460EX_DMA_CHANNELS)) { + printk("%s: bad channel %d\n", __FUNCTION__, psgl->ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + p_dma_ch = adev->chan[psgl->ch_id]; + psgl->ptail->control_count &= ~SG_LINK; /* make this the last dscrptr */ + sg_command = mfdcr(DCR_DMA2P40_SGC); + ppc460ex_set_sg_addr(psgl->ch_id, psgl->phead_dma); + sg_command |= SSG_ENABLE(psgl->ch_id); +//PMB - work around for PLB + sg_command &= 0xF0FFFFFF; + mtdcr(DCR_DMA2P40_SGC, sg_command); /* start transfer */ + + return 0; +} + +/* + * Halt an active scatter/gather DMA operation. + */ +int ppc460ex_disable_dma_sgl(ppc460ex_plb_dma_dev_t *adev, sgl_handle_t handle) +{ + sgl_list_info_t *psgl = (sgl_list_info_t *) handle; + uint32_t sg_command; + + if (unlikely(!handle)) { + printk("%s: null handle\n", __FUNCTION__); + return DMA_STATUS_BAD_HANDLE; + } + if (unlikely(psgl->ch_id >= MAX_PPC460EX_DMA_CHANNELS)) { + printk("%s: bad channel %d\n", __FUNCTION__, psgl->ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + sg_command = mfdcr(DCR_DMA2P40_SGC); + sg_command &= ~SSG_ENABLE(psgl->ch_id); + mtdcr(DCR_DMA2P40_SGC, sg_command); /* stop transfer */ + return 0; +} + + +/* + * Returns number of bytes left to be transferred from the entire sgl list. + * *src_addr and *dst_addr get set to the source/destination address of + * the sgl descriptor where the DMA stopped. + * + * An sgl transfer must NOT be active when this function is called. + */ +int ppc460ex_get_dma_sgl_residue(ppc460ex_plb_dma_dev_t *adev, sgl_handle_t handle, phys_addr_t * src_addr, + phys_addr_t * dst_addr) +{ + sgl_list_info_t *psgl = (sgl_list_info_t *) handle; + ppc460ex_plb_dma_ch_t *p_dma_ch; + ppc_sgl_t *pnext, *sgl_addr; + uint32_t count_left; + + if (unlikely(!handle)) { + printk("%s: null handle\n", __FUNCTION__); + return DMA_STATUS_BAD_HANDLE; + } + if (unlikely(psgl->ch_id >= MAX_PPC460EX_DMA_CHANNELS)) { + printk("%s: bad channel %d\n", __FUNCTION__, psgl->ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + sgl_addr = (ppc_sgl_t *) __va(mfdcr(DCR_DMA2P40_SGL0 + (psgl->ch_id * 0x8))); + count_left = mfdcr(DCR_DMA2P40_CTC0 + (psgl->ch_id * 0x8)) & SG_COUNT_MASK; + if (unlikely(!sgl_addr)) { + printk("%s: sgl addr register is null\n", __FUNCTION__); + goto error; + } + pnext = psgl->phead; + while (pnext && + ((unsigned) pnext < ((unsigned) psgl + SGL_LIST_SIZE) && + (pnext != sgl_addr)) + ) { + pnext++; + } + if (pnext == sgl_addr) { /* found the sgl descriptor */ + + *src_addr = pnext->src_addr; + *dst_addr = pnext->dst_addr; + + /* + * Now search the remaining descriptors and add their count. + * We already have the remaining count from this descriptor in + * count_left. + */ + pnext++; + + while ((pnext != psgl->ptail) && + ((unsigned) pnext < ((unsigned) psgl + SGL_LIST_SIZE)) + ) { + count_left += pnext->control_count & SG_COUNT_MASK; + } + if (unlikely(pnext != psgl->ptail)) { /* should never happen */ + printk + ("%s:error (1) psgl->ptail 0x%x handle 0x%x\n", __FUNCTION__, + (unsigned int) psgl->ptail, (unsigned int) handle); + goto error; + } + /* success */ + p_dma_ch = adev->chan[psgl->ch_id]; + return (count_left << p_dma_ch->shift); /* count in bytes */ + + } else { + /* this shouldn't happen */ + printk + ("get_dma_sgl_residue, unable to match current address 0x%x, handle 0x%x\n", + (unsigned int) sgl_addr, (unsigned int) handle); + + } + + error: + src_addr = NULL; + dst_addr = NULL; + return 0; + +} + +/* + * Returns the address(es) of the buffer(s) contained in the head element of + * the scatter/gather list. The element is removed from the scatter/gather + * list and the next element becomes the head. + * + * This function should only be called when the DMA is not active. + */ +int ppc460ex_delete_dma_sgl_element(sgl_handle_t handle, phys_addr_t * src_dma_addr, + phys_addr_t * dst_dma_addr) +{ + sgl_list_info_t *psgl = (sgl_list_info_t *) handle; + + if (unlikely(!handle)) { + printk("%s: null handle\n", __FUNCTION__); + return DMA_STATUS_BAD_HANDLE; + } + if (unlikely(psgl->ch_id >= MAX_PPC460EX_DMA_CHANNELS)) { + printk("%s: bad channel %d\n", __FUNCTION__, psgl->ch_id); + return DMA_STATUS_BAD_CHANNEL; + } + + if (unlikely(!psgl->phead)) { + printk("%s: sgl list empty\n", __FUNCTION__); + src_dma_addr = NULL; + dst_dma_addr = NULL; + return DMA_STATUS_SGL_LIST_EMPTY; + } + + *src_dma_addr = (phys_addr_t) psgl->phead->src_addr; + *dst_dma_addr = (phys_addr_t) psgl->phead->dst_addr; + + if (psgl->phead == psgl->ptail) { + /* last descriptor on the list */ + psgl->phead = NULL; + psgl->ptail = NULL; + } else { + psgl->phead++; + psgl->phead_dma += sizeof(ppc_sgl_t); + } + + return DMA_STATUS_GOOD; + +} + + +/* + * Create a scatter/gather list handle. This is simply a structure which + * describes a scatter/gather list. + * + * A handle is returned in "handle" which the driver should save in order to + * be able to access this list later. A chunk of memory will be allocated + * to be used by the API for internal management purposes, including managing + * the sg list and allocating memory for the sgl descriptors. One page should + * be more than enough for that purpose. Perhaps it's a bit wasteful to use + * a whole page for a single sg list, but most likely there will be only one + * sg list per channel. + * + * Interrupt notes: + * Each sgl descriptor has a copy of the DMA control word which the DMA engine + * loads in the control register. The control word has a "global" interrupt + * enable bit for that channel. Interrupts are further qualified by a few bits + * in the sgl descriptor count register. In order to setup an sgl, we have to + * know ahead of time whether or not interrupts will be enabled at the completion + * of the transfers. Thus, enable_dma_interrupt()/disable_dma_interrupt() MUST + * be called before calling alloc_dma_handle(). If the interrupt mode will never + * change after powerup, then enable_dma_interrupt()/disable_dma_interrupt() + * do not have to be called -- interrupts will be enabled or disabled based + * on how the channel was configured after powerup by the hw_init_dma_channel() + * function. Each sgl descriptor will be setup to interrupt if an error occurs; + * however, only the last descriptor will be setup to interrupt. Thus, an + * interrupt will occur (if interrupts are enabled) only after the complete + * sgl transfer is done. + */ +int ppc460ex_alloc_dma_handle(ppc460ex_plb_dma_dev_t *adev, sgl_handle_t *phandle, + unsigned int mode, unsigned int ch_id) +{ + sgl_list_info_t *psgl=NULL; + static dma_addr_t dma_addr; + ppc460ex_plb_dma_ch_t *p_dma_ch = adev->chan[ch_id]; + uint32_t sg_command; +#if 0 + void *ret; +#endif + if (unlikely(ch_id >= MAX_PPC460EX_DMA_CHANNELS)) { + printk("%s: bad channel %d\n", __FUNCTION__, p_dma_ch->chan_id); + return DMA_STATUS_BAD_CHANNEL; + } + if (unlikely(!phandle)) { + printk("%s: null handle pointer\n", __FUNCTION__); + return DMA_STATUS_NULL_POINTER; + } +#if 0 + /* Get a page of memory, which is zeroed out by consistent_alloc() */ + ret = dma_alloc_coherent(NULL, DMA_PPC4xx_SIZE, &dma_addr, GFP_KERNEL); + if (ret != NULL) { + memset(ret, 0, DMA_PPC4xx_SIZE); + psgl = (sgl_list_info_t *) ret; + } +#else + + if(!dma_mem_page) { + dma_mem_page = dma_alloc_coherent(adev->dev, DMA_PPC4xx_SIZE, &dma_addr, GFP_KERNEL); + if (unlikely(!dma_mem_page)){ + printk("dma_alloc_coherent failed\n"); + return -1; + } + } + + psgl = (sgl_list_info_t *) dma_mem_page; +#endif + + + if (unlikely(psgl == NULL)) { + *phandle = (sgl_handle_t) NULL; + return DMA_STATUS_OUT_OF_MEMORY; + } + + + psgl->dma_addr = dma_addr; + psgl->ch_id = ch_id; + /* + * Modify and save the control word. These words will be + * written to each sgl descriptor. The DMA engine then + * loads this control word into the control register + * every time it reads a new descriptor. + */ + psgl->control = p_dma_ch->control; + /* Clear all mode bits */ + psgl->control &= ~(DMA_TM_MASK | DMA_TD); + /* Save control word and mode */ + psgl->control |= (mode | DMA_CE_ENABLE); + /* In MM mode, we must set ETD/TCE */ + if (mode == DMA_MODE_MM) { /* PMB - Workaround */ + psgl->control |= DMA_ETD_OUTPUT | DMA_TCE_ENABLE; + psgl->control &= 0xFFFCFFFF; + psgl->control |= 0x00020000; + } + + if (p_dma_ch->int_enable) { + /* Enable channel interrupt */ + psgl->control |= DMA_CIE_ENABLE; + } else { + psgl->control &= ~DMA_CIE_ENABLE; + } + psgl->control &= ~DMA_CIE_ENABLE; + sg_command = mfdcr(DCR_DMA2P40_SGC); + sg_command |= SSG_MASK_ENABLE(ch_id); + + + /*Enable SGL control access */ + mtdcr(DCR_DMA2P40_SGC, sg_command); + psgl->sgl_control = SG_ERI_ENABLE | SG_LINK; + + + p_dma_ch->int_enable=0; + if (p_dma_ch->int_enable) { + if (p_dma_ch->tce_enable) + psgl->sgl_control |= SG_TCI_ENABLE | SG_ETI_ENABLE; + else + psgl->sgl_control |= SG_ETI_ENABLE | SG_TCI_ENABLE; + } + + *phandle = (sgl_handle_t) psgl; + return DMA_STATUS_GOOD; + +} + +/* + * Destroy a scatter/gather list handle that was created by alloc_dma_handle(). + * The list must be empty (contain no elements). + */ +void +ppc460ex_free_dma_handle(sgl_handle_t handle) +{ + sgl_list_info_t *psgl = (sgl_list_info_t *) handle; + + if (!handle) { + printk("%s: got NULL\n", __FUNCTION__); + return; + } else if (psgl->phead) { + printk("%s: list not empty\n", __FUNCTION__); + return; + } else if (!psgl->dma_addr) { /* should never happen */ + printk("%s: no dma address\n", __FUNCTION__); + return; + } + + //dma_free_coherent(NULL, DMA_PPC4xx_SIZE, (void *) psgl, 0); +} + +#if 0 + +int test_sgdma_memcpy(void *src, void *dst, void *src1, void *dst1, unsigned int length, unsigned int dma_ch_id) +{ + ppc460ex_plb_dma_dev_t *device; + ppc460ex_plb_dma_ch_t p_init; + phys_addr_t dma_dest, dma_src; + phys_addr_t dma_dest1, dma_src1; + int res = 0; + ppc460ex_plb_dma_ch_t *new_chan; + unsigned int control; + u32 status = 0; + u32 value = 0; + sgl_handle_t handle_p; + + /* create a device */ + if ((device = kzalloc(sizeof(*device), GFP_KERNEL)) == NULL) { + res = -ENOMEM; + } + + if ((new_chan = kzalloc(sizeof(ppc460ex_plb_dma_ch_t), GFP_KERNEL)) == NULL) { + printk("ERROR:No Free memory for allocating dma channels\n"); + res = -ENOMEM; + } + + dma_src = dma_map_single(p_init.device->dev, src, length, + DMA_TO_DEVICE); + dma_dest = dma_map_single(p_init.device->dev, dst, length, + DMA_FROM_DEVICE); + + dma_src1 = dma_map_single(p_init.device->dev, src1, length, + DMA_TO_DEVICE); + dma_dest1 = dma_map_single(p_init.device->dev, dst1, length, + DMA_FROM_DEVICE); + + memset(new_chan, 0 , sizeof(ppc460ex_plb_dma_ch_t)); + device->chan[dma_ch_id] = new_chan; + + + memset((char *)&p_init, sizeof(p_init), 0); + p_init.polarity = 0; + p_init.pwidth = PW_32; + p_init.in_use = 0; + p_init.sai = 1; + p_init.dai = 1; + p_init.tce_enable = 1; + //printk("%s:channel id = %d\n", __FUNCTION__, dma_ch_id); + + res = ppc460ex_init_dma_channel(device, dma_ch_id, &p_init); + + + ppc460ex_set_dma_count(device, dma_ch_id, length); + + res = ppc460ex_enable_dma_interrupt(device, dma_ch_id); + if (res) { + printk("%32s: en/disable_dma_interrupt\n", + __FUNCTION__); + } + + res = ppc460ex_alloc_dma_handle(device, &handle_p, DMA_MODE_MM, dma_ch_id); + + ppc460ex_add_dma_sgl(device, handle_p, dma_src, dma_dest, length); + ppc460ex_add_dma_sgl(device, handle_p, dma_src1, dma_dest1, length); + + ppc460ex_enable_dma_sgl(device, handle_p); + + + /*do { + value = mfdcr(DCR_DMA2P40_SR); + }while ((value & 0x80000000) != 0x80000000);*/ + +#if DEBUG_TEST + printk("%s:out:dump src \n", __FUNCTION__); + DMA_HEXDUMP(src, length); + printk("%s:out:dump dst\n", __FUNCTION__); + DMA_HEXDUMP(dst, length); + printk("%s:out:dump src1 \n", __FUNCTION__); + DMA_HEXDUMP(src1, length); + printk("%s:out:dump dst1\n", __FUNCTION__); + DMA_HEXDUMP(dst1, length); +#endif + + if (memcmp(src, dst, length) || memcmp(src1, dst1, length)) { + printk("Self-test copy failed compare, disabling\n"); + res = -ENODEV; + goto out; + } + + return 0; + out: + + return res; + +} +#endif + +#ifdef SPLICE_DMA_COHERENT +int +ppc460ex_sgdma_pipebufs_memcpy(struct pipe_inode_info *pipe, void *dest, dma_addr_t dma_dest, unsigned int length) +#else +int +ppc460ex_sgdma_pipebufs_memcpy(struct pipe_inode_info *pipe, void *dest, unsigned int length) +#endif +{ + sgl_list_info_t *psgl; + ppc460ex_plb_dma_dev_t *device; + ppc460ex_plb_dma_ch_t p_init; +#ifndef SPLICE_DMA_COHERENT + dma_addr_t dma_dest; +#endif + //dma_addr_t dma_addrs[32]; + phys_addr_t dma_src; + ppc460ex_plb_dma_ch_t *new_chan; + sgl_handle_t handle_p; + int dma_ch_id; + void *src; + int nrbufs = pipe->nrbufs; + int res = 0; + int len = 0; +#ifdef DEBUG_SPLICE_DMA + char *s_vaddr = NULL, *d_vaddr = NULL; + char strbuf[256]; + int firstbuf=0; +#endif + int curbuf = pipe->curbuf; /* stash away pipe->curbuf */ + +#ifdef DEBUG_SPLICE_DMA + printk("%s:%s:%d - dest = %p, length = %d len = %d\n", + __FILE__, __FUNCTION__, __LINE__, dest, length, len); +#endif + + if(unlikely(!nrbufs)) + return -EFAULT; + + /* create a device */ + if(unlikely ((device = kzalloc(sizeof(*device), GFP_KERNEL)) == NULL)) { + res = -ENOMEM; + } + + if(unlikely ((new_chan = kzalloc(sizeof(ppc460ex_plb_dma_ch_t), GFP_KERNEL)) == NULL)) { + printk("ERROR:No Free memory for allocating dma channels\n"); + res = -ENOMEM; + } + + memset(new_chan, 0 , sizeof(ppc460ex_plb_dma_ch_t)); + dma_ch_id = ppc460ex_get_dma_channel(); + if(unlikely(dma_ch_id == -ENODEV)) + return dma_ch_id; + + device->chan[dma_ch_id] = new_chan; + memset((char *)&p_init, 0, sizeof(ppc460ex_plb_dma_ch_t)); + p_init.polarity = 0; + p_init.pwidth = PW_8; + p_init.in_use = 0; + p_init.sai = 1; + p_init.dai = 1; + p_init.tce_enable = 1; + + + res = ppc460ex_init_dma_channel(device, dma_ch_id, &p_init); + if(unlikely(res != DMA_STATUS_GOOD)) + goto out; + + init_waitqueue_head(&device->queue); + + /* ppc460ex_disable_dma_interrupt(device, dma_ch_id); + ppc460ex_disable_burst(device, dma_ch_id); */ + res = ppc460ex_alloc_dma_handle(device, &handle_p, DMA_MODE_MM, dma_ch_id); + if(unlikely(res != DMA_STATUS_GOOD)) + goto out; + + +#ifdef DEBUG_SPLICE_DMA + printk("%s:%s:%d - nrbufs = %d pipe->curbuf = %d\n", + __FILE__, __FUNCTION__, __LINE__, nrbufs, pipe->curbuf); +#endif + for(;;) { + if(nrbufs) { + struct pipe_buffer *buf = pipe->bufs + curbuf; +#ifdef DEBUG_SPLICE_DMA + printk("%s:%s:%d - buf[%d] buf->len=%d length=%d len=%d\n", + __FILE__, __FUNCTION__, __LINE__, curbuf, buf->len, length, len); +#endif + if(len < length) { + + if(!buf->len) + continue; + + src = page_address(buf->page); + dma_src = dma_map_single(p_init.device->dev, src + buf->offset, buf->len, DMA_TO_DEVICE); +#ifndef SPLICE_DMA_COHERENT + dma_dest = dma_map_single(p_init.device->dev, dest + len, buf->len, DMA_FROM_DEVICE); +#endif + +#ifdef DEBUG_SPLICE_DMA + printk("maping %d src: %p, dest: %p, buf->len=%d dma_dest = 0x%08x\n", + curbuf, src + buf->offset, dest+len, buf->len, dma_dest); + printk("ADDING BUF NUMBER %d\n\n\n\n", curbuf); +#endif + ppc460ex_add_dma_sgl(device, handle_p, dma_src, dma_dest, buf->len); + len += buf->len; + curbuf = (curbuf + 1) & (PIPE_BUFFERS - 1); + --nrbufs; + } + else /* all pipe buf elements mapped to sgl */ + break; + } + else + break; + } + + + __dma_sync(dest, length, DMA_FROM_DEVICE); + ppc460ex_enable_dma_sgl(device, handle_p); + +#if 0 + res = wait_event_interruptible(device->queue, PPC460EX_DMA_CHAN_SGXFR_COMPLETE(dma_ch_id)); +#else + res = poll_for_sgdma_done(dma_ch_id); +#endif + + if(unlikely(res)) { + printk("%s:%s:%d - Timeout while waiting for SG Xfr to complete\n", + __FILE__, __FUNCTION__, __LINE__); + printk("dma_status = 0x%08x\n", DMA_STATUS(dma_ch_id)); + } + + /* Check the error status bits */ + printk("DCR_DMA2P40_SR=0x%x\n",mfdcr(DCR_DMA2P40_SR)); + if(unlikely(mfdcr(DCR_DMA2P40_SR) & (1 << (23 + dma_ch_id)))) { + printk(KERN_ERR"Error happened in the channel %d\n",dma_ch_id); + printk("DCR_DMA2P40_SR=0x%x\n",mfdcr(DCR_DMA2P40_SR)); + } + + mtdcr(DCR_DMA2P40_SR, 0xFFFFFFFF); + + +#ifdef DEBUG_SPLICE_DMA + printk("%s:%s:%d - dma status = 0x%08x\n", __FILE__, __FUNCTION__, __LINE__, DMA_STATUS(dma_ch_id)); +#endif + + + /* Hack */ + psgl = (sgl_list_info_t *) handle_p; + psgl->phead = NULL; + ppc460ex_free_dma_handle(handle_p); + handle_p = 0; + +#ifdef DEBUG_SPLICE_DMA + printk("%s:%s:%d - returning res = %d\n", __FILE__, __FUNCTION__, __LINE__, res); + + struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; + memset(strbuf, 0, 256); + s_vaddr = page_address(buf->page); + memcpy(strbuf, s_vaddr+buf->offset, 255); + *(strbuf+255) = '\0'; + printk("%s:%s:%d - source strbuf is %s\n", __FILE__, __FUNCTION__, __LINE__, strbuf); + + d_vaddr = dest; + memset(strbuf, 0, 256); + memcpy(strbuf, d_vaddr, 255); + *(strbuf+255) = '\0'; + printk("%s:%s:%d - dest strbuf is %s\n", __FILE__, __FUNCTION__, __LINE__, strbuf); +#endif + +out: + + return res; +} + +int splice_dma_memcpy(struct splice_dma_desc *sd_p, unsigned int len) +{ + sgl_list_info_t *psgl; + //static ppc460ex_plb_dma_dev_t *device; + ppc460ex_plb_dma_ch_t p_init; + dma_addr_t dma_dest; + phys_addr_t dma_src; + ppc460ex_plb_dma_ch_t *new_chan; + unsigned int size = 0; + sgl_handle_t handle_p; + int dma_ch_id; + void *src=NULL, *dst=NULL; + int res = 0; + int i = 0; + int dma_xfr_size=0; + dma_ch_id = ppc460ex_get_dma_channel(); + +#ifdef DEBUG_SPLICE_DMA + printk("%s:%s:%d - sd_p->n_elems=%d, len = %d \n", + __FILE__, __FUNCTION__, __LINE__, sd_p->n_elems, len); +#endif + + if(unlikely(!sd_p->n_elems)) + return -EFAULT; + new_chan = adev->chan[dma_ch_id]; + p_init.polarity = 0; + p_init.pwidth = PW_8; + p_init.in_use = 0; + p_init.sai = 1; + p_init.dai = 1; + p_init.tce_enable = 1; + + + res = ppc460ex_init_dma_channel(adev, dma_ch_id, &p_init); + if(unlikely(res != DMA_STATUS_GOOD)) + goto out; + + init_waitqueue_head(&adev->queue); + + ppc460ex_enable_burst(adev, dma_ch_id); + res = ppc460ex_alloc_dma_handle(adev, &handle_p, DMA_MODE_MM, dma_ch_id); + if(unlikely(res != DMA_STATUS_GOOD)) + goto out; + + for(i=0; i<sd_p->n_elems; i++) { + src = (void *)(sd_p->src_addrs[i]); + dst = (void *)(sd_p->dst_addrs[i]); + size = sd_p->xfr_size[i]; +#ifdef DEBUG_SPLICE_DMA + printk(KERN_DEBUG "index=%d src=0x%08x dst=0x%08x size=%d\n", i, src, dst, size);; +#endif + dma_src = dma_map_single(adev->dev, src, size, DMA_TO_DEVICE); + dma_dest = dma_map_single(adev->dev, dst, size, DMA_FROM_DEVICE); + ppc460ex_add_dma_sgl(adev, handle_p, dma_src, dma_dest, size); + dma_xfr_size += size; + } + +#ifdef DEBUG_SPLICE_DMA + printk(KERN_DEBUG "%s:%s:%d - dma_xfr_size=%d\n", __FILE__, __FUNCTION__, __LINE__, dma_xfr_size); +#endif + + dst = (void *)(sd_p->dst_addrs[0]); + + ppc460ex_enable_dma_sgl(adev, handle_p); + +#if 0 + res = wait_event_interruptible(device->queue, PPC460EX_DMA_CHAN_SGXFR_COMPLETE(dma_ch_id)); +#else + res = poll_for_sgdma_done(dma_ch_id); +#endif + + if(unlikely(res)) { + printk("%s:%s:%d - Timeout while waiting for SG Xfr to complete\n", + __FILE__, __FUNCTION__, __LINE__); + printk("dma_status = 0x%08x\n", DMA_STATUS(dma_ch_id)); + } + + /* Check the error status bits */ + if(unlikely(mfdcr(DCR_DMA2P40_SR) & (1 << (23 + dma_ch_id)))) { + printk(KERN_ERR"Error happened in the channel %d\n",dma_ch_id); + printk("DCR_DMA2P40_SR=0x%x\n",mfdcr(DCR_DMA2P40_SR)); + } + mtdcr(DCR_DMA2P40_SR, 0xFFFFFFFF); + + + +#ifdef DEBUG_SPLICE_DMA + printk("%s:%s:%d - dma status = 0x%08x\n", __FILE__, __FUNCTION__, __LINE__, DMA_STATUS(dma_ch_id)); +#endif + + for(i=0; i<sd_p->n_elems; i++) { + dma_unmap_single(adev->dev, splice_src_dma_addrs[i], size, DMA_TO_DEVICE); + dma_unmap_single(adev->dev, splice_dst_dma_addrs[i], size, DMA_FROM_DEVICE); + } + + /* Hack to clean up dma handle without memset */ + psgl = (sgl_list_info_t *) handle_p; + psgl->phead = NULL; + psgl->ptail = NULL; + ppc460ex_free_dma_handle(handle_p); + handle_p = 0; + +out: + return res; +} + + + +EXPORT_SYMBOL(ppc460ex_alloc_dma_handle); +EXPORT_SYMBOL(ppc460ex_free_dma_handle); +EXPORT_SYMBOL(ppc460ex_add_dma_sgl); +EXPORT_SYMBOL(ppc460ex_delete_dma_sgl_element); +EXPORT_SYMBOL(ppc460ex_enable_dma_sgl); +EXPORT_SYMBOL(ppc460ex_disable_dma_sgl); +EXPORT_SYMBOL(ppc460ex_get_dma_sgl_residue); +EXPORT_SYMBOL(ppc460ex_sgdma_pipebufs_memcpy); +EXPORT_SYMBOL(splice_dma_memcpy); |