aboutsummaryrefslogtreecommitdiff
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/dca/dca-core.c131
-rw-r--r--drivers/dca/dca-sysfs.c3
-rw-r--r--drivers/dma/Kconfig37
-rw-r--r--drivers/dma/Makefile3
-rw-r--r--drivers/dma/dmaengine.c35
-rw-r--r--drivers/dma/dmatest.c444
-rw-r--r--drivers/dma/dw_dmac.c1122
-rw-r--r--drivers/dma/dw_dmac_regs.h225
-rw-r--r--drivers/dma/fsldma.c38
-rw-r--r--drivers/dma/ioat.c15
-rw-r--r--drivers/dma/ioat_dca.c244
-rw-r--r--drivers/dma/ioat_dma.c402
-rw-r--r--drivers/dma/ioatdma.h28
-rw-r--r--drivers/dma/ioatdma_hw.h1
-rw-r--r--drivers/dma/ioatdma_registers.h20
-rw-r--r--drivers/dma/iop-adma.c53
-rw-r--r--drivers/dma/mv_xor.c1375
-rw-r--r--drivers/dma/mv_xor.h183
18 files changed, 4240 insertions, 119 deletions
diff --git a/drivers/dca/dca-core.c b/drivers/dca/dca-core.c
index bf5b92f86df..ec249d2db63 100644
--- a/drivers/dca/dca-core.c
+++ b/drivers/dca/dca-core.c
@@ -28,13 +28,29 @@
#include <linux/device.h>
#include <linux/dca.h>
-MODULE_LICENSE("GPL");
+#define DCA_VERSION "1.4"
-/* For now we're assuming a single, global, DCA provider for the system. */
+MODULE_VERSION(DCA_VERSION);
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Intel Corporation");
static DEFINE_SPINLOCK(dca_lock);
-static struct dca_provider *global_dca = NULL;
+static LIST_HEAD(dca_providers);
+
+static struct dca_provider *dca_find_provider_by_dev(struct device *dev)
+{
+ struct dca_provider *dca, *ret = NULL;
+
+ list_for_each_entry(dca, &dca_providers, node) {
+ if ((!dev) || (dca->ops->dev_managed(dca, dev))) {
+ ret = dca;
+ break;
+ }
+ }
+
+ return ret;
+}
/**
* dca_add_requester - add a dca client to the list
@@ -42,25 +58,39 @@ static struct dca_provider *global_dca = NULL;
*/
int dca_add_requester(struct device *dev)
{
- int err, slot;
+ struct dca_provider *dca;
+ int err, slot = -ENODEV;
- if (!global_dca)
- return -ENODEV;
+ if (!dev)
+ return -EFAULT;
spin_lock(&dca_lock);
- slot = global_dca->ops->add_requester(global_dca, dev);
- spin_unlock(&dca_lock);
- if (slot < 0)
+
+ /* check if the requester has not been added already */
+ dca = dca_find_provider_by_dev(dev);
+ if (dca) {
+ spin_unlock(&dca_lock);
+ return -EEXIST;
+ }
+
+ list_for_each_entry(dca, &dca_providers, node) {
+ slot = dca->ops->add_requester(dca, dev);
+ if (slot >= 0)
+ break;
+ }
+ if (slot < 0) {
+ spin_unlock(&dca_lock);
return slot;
+ }
- err = dca_sysfs_add_req(global_dca, dev, slot);
+ err = dca_sysfs_add_req(dca, dev, slot);
if (err) {
- spin_lock(&dca_lock);
- global_dca->ops->remove_requester(global_dca, dev);
+ dca->ops->remove_requester(dca, dev);
spin_unlock(&dca_lock);
return err;
}
+ spin_unlock(&dca_lock);
return 0;
}
EXPORT_SYMBOL_GPL(dca_add_requester);
@@ -71,30 +101,78 @@ EXPORT_SYMBOL_GPL(dca_add_requester);
*/
int dca_remove_requester(struct device *dev)
{
+ struct dca_provider *dca;
int slot;
- if (!global_dca)
- return -ENODEV;
+
+ if (!dev)
+ return -EFAULT;
spin_lock(&dca_lock);
- slot = global_dca->ops->remove_requester(global_dca, dev);
- spin_unlock(&dca_lock);
- if (slot < 0)
+ dca = dca_find_provider_by_dev(dev);
+ if (!dca) {
+ spin_unlock(&dca_lock);
+ return -ENODEV;
+ }
+ slot = dca->ops->remove_requester(dca, dev);
+ if (slot < 0) {
+ spin_unlock(&dca_lock);
return slot;
+ }
- dca_sysfs_remove_req(global_dca, slot);
+ dca_sysfs_remove_req(dca, slot);
+
+ spin_unlock(&dca_lock);
return 0;
}
EXPORT_SYMBOL_GPL(dca_remove_requester);
/**
- * dca_get_tag - return the dca tag for the given cpu
+ * dca_common_get_tag - return the dca tag (serves both new and old api)
+ * @dev - the device that wants dca service
* @cpu - the cpuid as returned by get_cpu()
*/
-u8 dca_get_tag(int cpu)
+u8 dca_common_get_tag(struct device *dev, int cpu)
{
- if (!global_dca)
+ struct dca_provider *dca;
+ u8 tag;
+
+ spin_lock(&dca_lock);
+
+ dca = dca_find_provider_by_dev(dev);
+ if (!dca) {
+ spin_unlock(&dca_lock);
return -ENODEV;
- return global_dca->ops->get_tag(global_dca, cpu);
+ }
+ tag = dca->ops->get_tag(dca, dev, cpu);
+
+ spin_unlock(&dca_lock);
+ return tag;
+}
+
+/**
+ * dca3_get_tag - return the dca tag to the requester device
+ * for the given cpu (new api)
+ * @dev - the device that wants dca service
+ * @cpu - the cpuid as returned by get_cpu()
+ */
+u8 dca3_get_tag(struct device *dev, int cpu)
+{
+ if (!dev)
+ return -EFAULT;
+
+ return dca_common_get_tag(dev, cpu);
+}
+EXPORT_SYMBOL_GPL(dca3_get_tag);
+
+/**
+ * dca_get_tag - return the dca tag for the given cpu (old api)
+ * @cpu - the cpuid as returned by get_cpu()
+ */
+u8 dca_get_tag(int cpu)
+{
+ struct device *dev = NULL;
+
+ return dca_common_get_tag(dev, cpu);
}
EXPORT_SYMBOL_GPL(dca_get_tag);
@@ -140,12 +218,10 @@ int register_dca_provider(struct dca_provider *dca, struct device *dev)
{
int err;
- if (global_dca)
- return -EEXIST;
err = dca_sysfs_add_provider(dca, dev);
if (err)
return err;
- global_dca = dca;
+ list_add(&dca->node, &dca_providers);
blocking_notifier_call_chain(&dca_provider_chain,
DCA_PROVIDER_ADD, NULL);
return 0;
@@ -158,11 +234,9 @@ EXPORT_SYMBOL_GPL(register_dca_provider);
*/
void unregister_dca_provider(struct dca_provider *dca)
{
- if (!global_dca)
- return;
blocking_notifier_call_chain(&dca_provider_chain,
DCA_PROVIDER_REMOVE, NULL);
- global_dca = NULL;
+ list_del(&dca->node);
dca_sysfs_remove_provider(dca);
}
EXPORT_SYMBOL_GPL(unregister_dca_provider);
@@ -187,6 +261,7 @@ EXPORT_SYMBOL_GPL(dca_unregister_notify);
static int __init dca_init(void)
{
+ printk(KERN_ERR "dca service started, version %s\n", DCA_VERSION);
return dca_sysfs_init();
}
diff --git a/drivers/dca/dca-sysfs.c b/drivers/dca/dca-sysfs.c
index 9a70377bfb3..7af4b403bd2 100644
--- a/drivers/dca/dca-sysfs.c
+++ b/drivers/dca/dca-sysfs.c
@@ -13,10 +13,11 @@ static spinlock_t dca_idr_lock;
int dca_sysfs_add_req(struct dca_provider *dca, struct device *dev, int slot)
{
struct device *cd;
+ static int req_count;
cd = device_create_drvdata(dca_class, dca->cd,
MKDEV(0, slot + 1), NULL,
- "requester%d", slot);
+ "requester%d", req_count++);
if (IS_ERR(cd))
return PTR_ERR(cd);
return 0;
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 6239c3df30a..cd303901eb5 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -4,13 +4,14 @@
menuconfig DMADEVICES
bool "DMA Engine support"
- depends on (PCI && X86) || ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX || PPC
- depends on !HIGHMEM64G
+ depends on !HIGHMEM64G && HAS_DMA
help
DMA engines can do asynchronous data transfers without
involving the host CPU. Currently, this framework can be
used to offload memory copies in the network stack and
- RAID operations in the MD driver.
+ RAID operations in the MD driver. This menu only presents
+ DMA Device drivers supported by the configured arch, it may
+ be empty in some cases.
if DMADEVICES
@@ -37,6 +38,15 @@ config INTEL_IOP_ADMA
help
Enable support for the Intel(R) IOP Series RAID engines.
+config DW_DMAC
+ tristate "Synopsys DesignWare AHB DMA support"
+ depends on AVR32
+ select DMA_ENGINE
+ default y if CPU_AT32AP7000
+ help
+ Support the Synopsys DesignWare AHB DMA controller. This
+ can be integrated in chips such as the Atmel AT32ap7000.
+
config FSL_DMA
bool "Freescale MPC85xx/MPC83xx DMA support"
depends on PPC
@@ -46,6 +56,14 @@ config FSL_DMA
MPC8560/40, MPC8555, MPC8548 and MPC8641 processors.
The MPC8349, MPC8360 is also supported.
+config MV_XOR
+ bool "Marvell XOR engine support"
+ depends on PLAT_ORION
+ select ASYNC_CORE
+ select DMA_ENGINE
+ ---help---
+ Enable support for the Marvell XOR engine.
+
config DMA_ENGINE
bool
@@ -55,10 +73,19 @@ comment "DMA Clients"
config NET_DMA
bool "Network: TCP receive copy offload"
depends on DMA_ENGINE && NET
+ default (INTEL_IOATDMA || FSL_DMA)
help
This enables the use of DMA engines in the network stack to
offload receive copy-to-user operations, freeing CPU cycles.
- Since this is the main user of the DMA engine, it should be enabled;
- say Y here.
+
+ Say Y here if you enabled INTEL_IOATDMA or FSL_DMA, otherwise
+ say N.
+
+config DMATEST
+ tristate "DMA Test client"
+ depends on DMA_ENGINE
+ help
+ Simple DMA test client. Say N unless you're debugging a
+ DMA Device driver.
endif
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index c8036d94590..14f59527d4f 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -1,6 +1,9 @@
obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
obj-$(CONFIG_NET_DMA) += iovlock.o
+obj-$(CONFIG_DMATEST) += dmatest.o
obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o
obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
obj-$(CONFIG_FSL_DMA) += fsldma.o
+obj-$(CONFIG_MV_XOR) += mv_xor.o
+obj-$(CONFIG_DW_DMAC) += dw_dmac.o
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 97b329e7679..dc003a3a787 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -169,12 +169,18 @@ static void dma_client_chan_alloc(struct dma_client *client)
enum dma_state_client ack;
/* Find a channel */
- list_for_each_entry(device, &dma_device_list, global_node)
+ list_for_each_entry(device, &dma_device_list, global_node) {
+ /* Does the client require a specific DMA controller? */
+ if (client->slave && client->slave->dma_dev
+ && client->slave->dma_dev != device->dev)
+ continue;
+
list_for_each_entry(chan, &device->channels, device_node) {
if (!dma_chan_satisfies_mask(chan, client->cap_mask))
continue;
- desc = chan->device->device_alloc_chan_resources(chan);
+ desc = chan->device->device_alloc_chan_resources(
+ chan, client);
if (desc >= 0) {
ack = client->event_callback(client,
chan,
@@ -183,12 +189,14 @@ static void dma_client_chan_alloc(struct dma_client *client)
/* we are done once this client rejects
* an available resource
*/
- if (ack == DMA_ACK)
+ if (ack == DMA_ACK) {
dma_chan_get(chan);
- else if (ack == DMA_NAK)
+ chan->client_count++;
+ } else if (ack == DMA_NAK)
return;
}
}
+ }
}
enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
@@ -272,8 +280,10 @@ static void dma_clients_notify_removed(struct dma_chan *chan)
/* client was holding resources for this channel so
* free it
*/
- if (ack == DMA_ACK)
+ if (ack == DMA_ACK) {
dma_chan_put(chan);
+ chan->client_count--;
+ }
}
mutex_unlock(&dma_list_mutex);
@@ -285,6 +295,10 @@ static void dma_clients_notify_removed(struct dma_chan *chan)
*/
void dma_async_client_register(struct dma_client *client)
{
+ /* validate client data */
+ BUG_ON(dma_has_cap(DMA_SLAVE, client->cap_mask) &&
+ !client->slave);
+
mutex_lock(&dma_list_mutex);
list_add_tail(&client->global_node, &dma_client_list);
mutex_unlock(&dma_list_mutex);
@@ -313,8 +327,10 @@ void dma_async_client_unregister(struct dma_client *client)
ack = client->event_callback(client, chan,
DMA_RESOURCE_REMOVED);
- if (ack == DMA_ACK)
+ if (ack == DMA_ACK) {
dma_chan_put(chan);
+ chan->client_count--;
+ }
}
list_del(&client->global_node);
@@ -359,6 +375,10 @@ int dma_async_device_register(struct dma_device *device)
!device->device_prep_dma_memset);
BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) &&
!device->device_prep_dma_interrupt);
+ BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) &&
+ !device->device_prep_slave_sg);
+ BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) &&
+ !device->device_terminate_all);
BUG_ON(!device->device_alloc_chan_resources);
BUG_ON(!device->device_free_chan_resources);
@@ -378,7 +398,7 @@ int dma_async_device_register(struct dma_device *device)
chan->chan_id = chancnt++;
chan->dev.class = &dma_devclass;
- chan->dev.parent = NULL;
+ chan->dev.parent = device->dev;
snprintf(chan->dev.bus_id, BUS_ID_SIZE, "dma%dchan%d",
device->dev_id, chan->chan_id);
@@ -394,6 +414,7 @@ int dma_async_device_register(struct dma_device *device)
kref_get(&device->refcount);
kref_get(&device->refcount);
kref_init(&chan->refcount);
+ chan->client_count = 0;
chan->slow_ref = 0;
INIT_RCU_HEAD(&chan->rcu);
}
diff --git a/drivers/dma/dmatest.c b/drivers/dma/dmatest.c
new file mode 100644
index 00000000000..a08d1970474
--- /dev/null
+++ b/drivers/dma/dmatest.c
@@ -0,0 +1,444 @@
+/*
+ * DMA Engine test module
+ *
+ * Copyright (C) 2007 Atmel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/init.h>
+#include <linux/kthread.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/random.h>
+#include <linux/wait.h>
+
+static unsigned int test_buf_size = 16384;
+module_param(test_buf_size, uint, S_IRUGO);
+MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer");
+
+static char test_channel[BUS_ID_SIZE];
+module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO);
+MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)");
+
+static char test_device[BUS_ID_SIZE];
+module_param_string(device, test_device, sizeof(test_device), S_IRUGO);
+MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)");
+
+static unsigned int threads_per_chan = 1;
+module_param(threads_per_chan, uint, S_IRUGO);
+MODULE_PARM_DESC(threads_per_chan,
+ "Number of threads to start per channel (default: 1)");
+
+static unsigned int max_channels;
+module_param(max_channels, uint, S_IRUGO);
+MODULE_PARM_DESC(nr_channels,
+ "Maximum number of channels to use (default: all)");
+
+/*
+ * Initialization patterns. All bytes in the source buffer has bit 7
+ * set, all bytes in the destination buffer has bit 7 cleared.
+ *
+ * Bit 6 is set for all bytes which are to be copied by the DMA
+ * engine. Bit 5 is set for all bytes which are to be overwritten by
+ * the DMA engine.
+ *
+ * The remaining bits are the inverse of a counter which increments by
+ * one for each byte address.
+ */
+#define PATTERN_SRC 0x80
+#define PATTERN_DST 0x00
+#define PATTERN_COPY 0x40
+#define PATTERN_OVERWRITE 0x20
+#define PATTERN_COUNT_MASK 0x1f
+
+struct dmatest_thread {
+ struct list_head node;
+ struct task_struct *task;
+ struct dma_chan *chan;
+ u8 *srcbuf;
+ u8 *dstbuf;
+};
+
+struct dmatest_chan {
+ struct list_head node;
+ struct dma_chan *chan;
+ struct list_head threads;
+};
+
+/*
+ * These are protected by dma_list_mutex since they're only used by
+ * the DMA client event callback
+ */
+static LIST_HEAD(dmatest_channels);
+static unsigned int nr_channels;
+
+static bool dmatest_match_channel(struct dma_chan *chan)
+{
+ if (test_channel[0] == '\0')
+ return true;
+ return strcmp(chan->dev.bus_id, test_channel) == 0;
+}
+
+static bool dmatest_match_device(struct dma_device *device)
+{
+ if (test_device[0] == '\0')
+ return true;
+ return strcmp(device->dev->bus_id, test_device) == 0;
+}
+
+static unsigned long dmatest_random(void)
+{
+ unsigned long buf;
+
+ get_random_bytes(&buf, sizeof(buf));
+ return buf;
+}
+
+static void dmatest_init_srcbuf(u8 *buf, unsigned int start, unsigned int len)
+{
+ unsigned int i;
+
+ for (i = 0; i < start; i++)
+ buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
+ for ( ; i < start + len; i++)
+ buf[i] = PATTERN_SRC | PATTERN_COPY
+ | (~i & PATTERN_COUNT_MASK);;
+ for ( ; i < test_buf_size; i++)
+ buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK);
+}
+
+static void dmatest_init_dstbuf(u8 *buf, unsigned int start, unsigned int len)
+{
+ unsigned int i;
+
+ for (i = 0; i < start; i++)
+ buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
+ for ( ; i < start + len; i++)
+ buf[i] = PATTERN_DST | PATTERN_OVERWRITE
+ | (~i & PATTERN_COUNT_MASK);
+ for ( ; i < test_buf_size; i++)
+ buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK);
+}
+
+static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index,
+ unsigned int counter, bool is_srcbuf)
+{
+ u8 diff = actual ^ pattern;
+ u8 expected = pattern | (~counter & PATTERN_COUNT_MASK);
+ const char *thread_name = current->comm;
+
+ if (is_srcbuf)
+ pr_warning("%s: srcbuf[0x%x] overwritten!"
+ " Expected %02x, got %02x\n",
+ thread_name, index, expected, actual);
+ else if ((pattern & PATTERN_COPY)
+ && (diff & (PATTERN_COPY | PATTERN_OVERWRITE)))
+ pr_warning("%s: dstbuf[0x%x] not copied!"
+ " Expected %02x, got %02x\n",
+ thread_name, index, expected, actual);
+ else if (diff & PATTERN_SRC)
+ pr_warning("%s: dstbuf[0x%x] was copied!"
+ " Expected %02x, got %02x\n",
+ thread_name, index, expected, actual);
+ else
+ pr_warning("%s: dstbuf[0x%x] mismatch!"
+ " Expected %02x, got %02x\n",
+ thread_name, index, expected, actual);
+}
+
+static unsigned int dmatest_verify(u8 *buf, unsigned int start,
+ unsigned int end, unsigned int counter, u8 pattern,
+ bool is_srcbuf)
+{
+ unsigned int i;
+ unsigned int error_count = 0;
+ u8 actual;
+
+ for (i = start; i < end; i++) {
+ actual = buf[i];
+ if (actual != (pattern | (~counter & PATTERN_COUNT_MASK))) {
+ if (error_count < 32)
+ dmatest_mismatch(actual, pattern, i, counter,
+ is_srcbuf);
+ error_count++;
+ }
+ counter++;
+ }
+
+ if (error_count > 32)
+ pr_warning("%s: %u errors suppressed\n",
+ current->comm, error_count - 32);
+
+ return error_count;
+}
+
+/*
+ * This function repeatedly tests DMA transfers of various lengths and
+ * offsets until it is told to exit by kthread_stop(). There may be
+ * multiple threads running this function in parallel for a single
+ * channel, and there may be multiple channels being tested in
+ * parallel.
+ *
+ * Before each test, the source and destination buffer is initialized
+ * with a known pattern. This pattern is different depending on
+ * whether it's in an area which is supposed to be copied or
+ * overwritten, and different in the source and destination buffers.
+ * So if the DMA engine doesn't copy exactly what we tell it to copy,
+ * we'll notice.
+ */
+static int dmatest_func(void *data)
+{
+ struct dmatest_thread *thread = data;
+ struct dma_chan *chan;
+ const char *thread_name;
+ unsigned int src_off, dst_off, len;
+ unsigned int error_count;
+ unsigned int failed_tests = 0;
+ unsigned int total_tests = 0;
+ dma_cookie_t cookie;
+ enum dma_status status;
+ int ret;
+
+ thread_name = current->comm;
+
+ ret = -ENOMEM;
+ thread->srcbuf = kmalloc(test_buf_size, GFP_KERNEL);
+ if (!thread->srcbuf)
+ goto err_srcbuf;
+ thread->dstbuf = kmalloc(test_buf_size, GFP_KERNEL);
+ if (!thread->dstbuf)
+ goto err_dstbuf;
+
+ smp_rmb();
+ chan = thread->chan;
+ dma_chan_get(chan);
+
+ while (!kthread_should_stop()) {
+ total_tests++;
+
+ len = dmatest_random() % test_buf_size + 1;
+ src_off = dmatest_random() % (test_buf_size - len + 1);
+ dst_off = dmatest_random() % (test_buf_size - len + 1);
+
+ dmatest_init_srcbuf(thread->srcbuf, src_off, len);
+ dmatest_init_dstbuf(thread->dstbuf, dst_off, len);
+
+ cookie = dma_async_memcpy_buf_to_buf(chan,
+ thread->dstbuf + dst_off,
+ thread->srcbuf + src_off,
+ len);
+ if (dma_submit_error(cookie)) {
+ pr_warning("%s: #%u: submit error %d with src_off=0x%x "
+ "dst_off=0x%x len=0x%x\n",
+ thread_name, total_tests - 1, cookie,
+ src_off, dst_off, len);
+ msleep(100);
+ failed_tests++;
+ continue;
+ }
+ dma_async_memcpy_issue_pending(chan);
+
+ do {
+ msleep(1);
+ status = dma_async_memcpy_complete(
+ chan, cookie, NULL, NULL);
+ } while (status == DMA_IN_PROGRESS);
+
+ if (status == DMA_ERROR) {
+ pr_warning("%s: #%u: error during copy\n",
+ thread_name, total_tests - 1);
+ failed_tests++;
+ continue;
+ }
+
+ error_count = 0;
+
+ pr_debug("%s: verifying source buffer...\n", thread_name);
+ error_count += dmatest_verify(thread->srcbuf, 0, src_off,
+ 0, PATTERN_SRC, true);
+ error_count += dmatest_verify(thread->srcbuf, src_off,
+ src_off + len, src_off,
+ PATTERN_SRC | PATTERN_COPY, true);
+ error_count += dmatest_verify(thread->srcbuf, src_off + len,
+ test_buf_size, src_off + len,
+ PATTERN_SRC, true);
+
+ pr_debug("%s: verifying dest buffer...\n",
+ thread->task->comm);
+ error_count += dmatest_verify(thread->dstbuf, 0, dst_off,
+ 0, PATTERN_DST, false);
+ error_count += dmatest_verify(thread->dstbuf, dst_off,
+ dst_off + len, src_off,
+ PATTERN_SRC | PATTERN_COPY, false);
+ error_count += dmatest_verify(thread->dstbuf, dst_off + len,
+ test_buf_size, dst_off + len,
+ PATTERN_DST, false);
+
+ if (error_count) {
+ pr_warning("%s: #%u: %u errors with "
+ "src_off=0x%x dst_off=0x%x len=0x%x\n",
+ thread_name, total_tests - 1, error_count,
+ src_off, dst_off, len);
+ failed_tests++;
+ } else {
+ pr_debug("%s: #%u: No errors with "
+ "src_off=0x%x dst_off=0x%x len=0x%x\n",
+ thread_name, total_tests - 1,
+ src_off, dst_off, len);
+ }
+ }
+
+ ret = 0;
+ dma_chan_put(chan);
+ kfree(thread->dstbuf);
+err_dstbuf:
+ kfree(thread->srcbuf);
+err_srcbuf:
+ pr_notice("%s: terminating after %u tests, %u failures (status %d)\n",
+ thread_name, total_tests, failed_tests, ret);
+ return ret;
+}
+
+static void dmatest_cleanup_channel(struct dmatest_chan *dtc)
+{
+ struct dmatest_thread *thread;
+ struct dmatest_thread *_thread;
+ int ret;
+
+ list_for_each_entry_safe(thread, _thread, &dtc->threads, node) {
+ ret = kthread_stop(thread->task);
+ pr_debug("dmatest: thread %s exited with status %d\n",
+ thread->task->comm, ret);
+ list_del(&thread->node);
+ kfree(thread);
+ }
+ kfree(dtc);
+}
+
+static enum dma_state_client dmatest_add_channel(struct dma_chan *chan)
+{
+ struct dmatest_chan *dtc;
+ struct dmatest_thread *thread;
+ unsigned int i;
+
+ dtc = kmalloc(sizeof(struct dmatest_chan), GFP_ATOMIC);
+ if (!dtc) {
+ pr_warning("dmatest: No memory for %s\n", chan->dev.bus_id);
+ return DMA_NAK;
+ }
+
+ dtc->chan = chan;
+ INIT_LIST_HEAD(&dtc->threads);
+
+ for (i = 0; i < threads_per_chan; i++) {
+ thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL);
+ if (!thread) {
+ pr_warning("dmatest: No memory for %s-test%u\n",
+ chan->dev.bus_id, i);
+ break;
+ }
+ thread->chan = dtc->chan;
+ smp_wmb();
+ thread->task = kthread_run(dmatest_func, thread, "%s-test%u",
+ chan->dev.bus_id, i);
+ if (IS_ERR(thread->task)) {
+ pr_warning("dmatest: Failed to run thread %s-test%u\n",
+ chan->dev.bus_id, i);
+ kfree(thread);
+ break;
+ }
+
+ /* srcbuf and dstbuf are allocated by the thread itself */
+
+ list_add_tail(&thread->node, &dtc->threads);
+ }
+
+ pr_info("dmatest: Started %u threads using %s\n", i, chan->dev.bus_id);
+
+ list_add_tail(&dtc->node, &dmatest_channels);
+ nr_channels++;
+
+ return DMA_ACK;
+}
+
+static enum dma_state_client dmatest_remove_channel(struct dma_chan *chan)
+{
+ struct dmatest_chan *dtc, *_dtc;
+
+ list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) {
+ if (dtc->chan == chan) {
+ list_del(&dtc->node);
+ dmatest_cleanup_channel(dtc);
+ pr_debug("dmatest: lost channel %s\n",
+ chan->dev.bus_id);
+ return DMA_ACK;
+ }
+ }
+
+ return DMA_DUP;
+}
+
+/*
+ * Start testing threads as new channels are assigned to us, and kill
+ * them when the channels go away.
+ *
+ * When we unregister the client, all channels are removed so this
+ * will also take care of cleaning things up when the module is
+ * unloaded.
+ */
+static enum dma_state_client
+dmatest_event(struct dma_client *client, struct dma_chan *chan,
+ enum dma_state state)
+{
+ enum dma_state_client ack = DMA_NAK;
+
+ switch (state) {
+ case DMA_RESOURCE_AVAILABLE:
+ if (!dmatest_match_channel(chan)
+ || !dmatest_match_device(chan->device))
+ ack = DMA_DUP;
+ else if (max_channels && nr_channels >= max_channels)
+ ack = DMA_NAK;
+ else
+ ack = dmatest_add_channel(chan);
+ break;
+
+ case DMA_RESOURCE_REMOVED:
+ ack = dmatest_remove_channel(chan);
+ break;
+
+ default:
+ pr_info("dmatest: Unhandled event %u (%s)\n",
+ state, chan->dev.bus_id);
+ break;
+ }
+
+ return ack;
+}
+
+static struct dma_client dmatest_client = {
+ .event_callback = dmatest_event,
+};
+
+static int __init dmatest_init(void)
+{
+ dma_cap_set(DMA_MEMCPY, dmatest_client.cap_mask);
+ dma_async_client_register(&dmatest_client);
+ dma_async_client_chan_request(&dmatest_client);
+
+ return 0;
+}
+module_init(dmatest_init);
+
+static void __exit dmatest_exit(void)
+{
+ dma_async_client_unregister(&dmatest_client);
+}
+module_exit(dmatest_exit);
+
+MODULE_AUTHOR("Haavard Skinnemoen <hskinnemoen@atmel.com>");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/dma/dw_dmac.c b/drivers/dma/dw_dmac.c
new file mode 100644
index 00000000000..94df9177124
--- /dev/null
+++ b/drivers/dma/dw_dmac.c
@@ -0,0 +1,1122 @@
+/*
+ * Driver for the Synopsys DesignWare DMA Controller (aka DMACA on
+ * AVR32 systems.)
+ *
+ * Copyright (C) 2007-2008 Atmel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+
+#include "dw_dmac_regs.h"
+
+/*
+ * This supports the Synopsys "DesignWare AHB Central DMA Controller",
+ * (DW_ahb_dmac) which is used with various AMBA 2.0 systems (not all
+ * of which use ARM any more). See the "Databook" from Synopsys for
+ * information beyond what licensees probably provide.
+ *
+ * The driver has currently been tested only with the Atmel AT32AP7000,
+ * which does not support descriptor writeback.
+ */
+
+/* NOTE: DMS+SMS is system-specific. We should get this information
+ * from the platform code somehow.
+ */
+#define DWC_DEFAULT_CTLLO (DWC_CTLL_DST_MSIZE(0) \
+ | DWC_CTLL_SRC_MSIZE(0) \
+ | DWC_CTLL_DMS(0) \
+ | DWC_CTLL_SMS(1) \
+ | DWC_CTLL_LLP_D_EN \
+ | DWC_CTLL_LLP_S_EN)
+
+/*
+ * This is configuration-dependent and usually a funny size like 4095.
+ * Let's round it down to the nearest power of two.
+ *
+ * Note that this is a transfer count, i.e. if we transfer 32-bit
+ * words, we can do 8192 bytes per descriptor.
+ *
+ * This parameter is also system-specific.
+ */
+#define DWC_MAX_COUNT 2048U
+
+/*
+ * Number of descriptors to allocate for each channel. This should be
+ * made configurable somehow; preferably, the clients (at least the
+ * ones using slave transfers) should be able to give us a hint.
+ */
+#define NR_DESCS_PER_CHANNEL 64
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * Because we're not relying on writeback from the controller (it may not
+ * even be configured into the core!) we don't need to use dma_pool. These
+ * descriptors -- and associated data -- are cacheable. We do need to make
+ * sure their dcache entries are written back before handing them off to
+ * the controller, though.
+ */
+
+static struct dw_desc *dwc_first_active(struct dw_dma_chan *dwc)
+{
+ return list_entry(dwc->active_list.next, struct dw_desc, desc_node);
+}
+
+static struct dw_desc *dwc_first_queued(struct dw_dma_chan *dwc)
+{
+ return list_entry(dwc->queue.next, struct dw_desc, desc_node);
+}
+
+static struct dw_desc *dwc_desc_get(struct dw_dma_chan *dwc)
+{
+ struct dw_desc *desc, *_desc;
+ struct dw_desc *ret = NULL;
+ unsigned int i = 0;
+
+ spin_lock_bh(&dwc->lock);
+ list_for_each_entry_safe(desc, _desc, &dwc->free_list, desc_node) {
+ if (async_tx_test_ack(&desc->txd)) {
+ list_del(&desc->desc_node);
+ ret = desc;
+ break;
+ }
+ dev_dbg(&dwc->chan.dev, "desc %p not ACKed\n", desc);
+ i++;
+ }
+ spin_unlock_bh(&dwc->lock);
+
+ dev_vdbg(&dwc->chan.dev, "scanned %u descriptors on freelist\n", i);
+
+ return ret;
+}
+
+static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
+{
+ struct dw_desc *child;
+
+ list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+ dma_sync_single_for_cpu(dwc->chan.dev.parent,
+ child->txd.phys, sizeof(child->lli),
+ DMA_TO_DEVICE);
+ dma_sync_single_for_cpu(dwc->chan.dev.parent,
+ desc->txd.phys, sizeof(desc->lli),
+ DMA_TO_DEVICE);
+}
+
+/*
+ * Move a descriptor, including any children, to the free list.
+ * `desc' must not be on any lists.
+ */
+static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
+{
+ if (desc) {
+ struct dw_desc *child;
+
+ dwc_sync_desc_for_cpu(dwc, desc);
+
+ spin_lock_bh(&dwc->lock);
+ list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+ dev_vdbg(&dwc->chan.dev,
+ "moving child desc %p to freelist\n",
+ child);
+ list_splice_init(&desc->txd.tx_list, &dwc->free_list);
+ dev_vdbg(&dwc->chan.dev, "moving desc %p to freelist\n", desc);
+ list_add(&desc->desc_node, &dwc->free_list);
+ spin_unlock_bh(&dwc->lock);
+ }
+}
+
+/* Called with dwc->lock held and bh disabled */
+static dma_cookie_t
+dwc_assign_cookie(struct dw_dma_chan *dwc, struct dw_desc *desc)
+{
+ dma_cookie_t cookie = dwc->chan.cookie;
+
+ if (++cookie < 0)
+ cookie = 1;
+
+ dwc->chan.cookie = cookie;
+ desc->txd.cookie = cookie;
+
+ return cookie;
+}
+
+/*----------------------------------------------------------------------*/