diff options
Diffstat (limited to 'arch/arm/include/asm/dma-mapping.h')
| -rw-r--r-- | arch/arm/include/asm/dma-mapping.h | 562 |
1 files changed, 199 insertions, 363 deletions
diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 7b95d205839..c45b61a4b4a 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -5,34 +5,102 @@ #include <linux/mm_types.h> #include <linux/scatterlist.h> +#include <linux/dma-attrs.h> +#include <linux/dma-debug.h> #include <asm-generic/dma-coherent.h> #include <asm/memory.h> +#include <xen/xen.h> +#include <asm/xen/hypervisor.h> + +#define DMA_ERROR_CODE (~0) +extern struct dma_map_ops arm_dma_ops; +extern struct dma_map_ops arm_coherent_dma_ops; + +static inline struct dma_map_ops *__generic_dma_ops(struct device *dev) +{ + if (dev && dev->archdata.dma_ops) + return dev->archdata.dma_ops; + return &arm_dma_ops; +} + +static inline struct dma_map_ops *get_dma_ops(struct device *dev) +{ + if (xen_initial_domain()) + return xen_dma_ops; + else + return __generic_dma_ops(dev); +} + +static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops) +{ + BUG_ON(!dev); + dev->archdata.dma_ops = ops; +} + +#include <asm-generic/dma-mapping-common.h> + +static inline int dma_set_mask(struct device *dev, u64 mask) +{ + return get_dma_ops(dev)->set_dma_mask(dev, mask); +} + +#ifdef __arch_page_to_dma +#error Please update to __arch_pfn_to_dma +#endif + /* - * page_to_dma/dma_to_virt/virt_to_dma are architecture private functions - * used internally by the DMA-mapping API to provide DMA addresses. They - * must not be used by drivers. + * dma_to_pfn/pfn_to_dma/dma_to_virt/virt_to_dma are architecture private + * functions used internally by the DMA-mapping API to provide DMA + * addresses. They must not be used by drivers. */ -#ifndef __arch_page_to_dma -static inline dma_addr_t page_to_dma(struct device *dev, struct page *page) +#ifndef __arch_pfn_to_dma +static inline dma_addr_t pfn_to_dma(struct device *dev, unsigned long pfn) { - return (dma_addr_t)__virt_to_bus((unsigned long)page_address(page)); + if (dev) + pfn -= dev->dma_pfn_offset; + return (dma_addr_t)__pfn_to_bus(pfn); +} + +static inline unsigned long dma_to_pfn(struct device *dev, dma_addr_t addr) +{ + unsigned long pfn = __bus_to_pfn(addr); + + if (dev) + pfn += dev->dma_pfn_offset; + + return pfn; } static inline void *dma_to_virt(struct device *dev, dma_addr_t addr) { - return (void *)__bus_to_virt(addr); + if (dev) { + unsigned long pfn = dma_to_pfn(dev, addr); + + return phys_to_virt(__pfn_to_phys(pfn)); + } + + return (void *)__bus_to_virt((unsigned long)addr); } static inline dma_addr_t virt_to_dma(struct device *dev, void *addr) { + if (dev) + return pfn_to_dma(dev, virt_to_pfn(addr)); + return (dma_addr_t)__virt_to_bus((unsigned long)(addr)); } + #else -static inline dma_addr_t page_to_dma(struct device *dev, struct page *page) +static inline dma_addr_t pfn_to_dma(struct device *dev, unsigned long pfn) { - return __arch_page_to_dma(dev, page); + return __arch_pfn_to_dma(dev, pfn); +} + +static inline unsigned long dma_to_pfn(struct device *dev, dma_addr_t addr) +{ + return __arch_dma_to_pfn(dev, addr); } static inline void *dma_to_virt(struct device *dev, dma_addr_t addr) @@ -46,411 +114,180 @@ static inline dma_addr_t virt_to_dma(struct device *dev, void *addr) } #endif -/* - * DMA-consistent mapping functions. These allocate/free a region of - * uncached, unwrite-buffered mapped memory space for use with DMA - * devices. This is the "generic" version. The PCI specific version - * is in pci.h - * - * Note: Drivers should NOT use this function directly, as it will break - * platforms with CONFIG_DMABOUNCE. - * Use the driver DMA support - see dma-mapping.h (dma_sync_*) - */ -extern void dma_cache_maint(const void *kaddr, size_t size, int rw); - -/* - * Return whether the given device DMA address mask can be supported - * properly. For example, if your device can only drive the low 24-bits - * during bus mastering, then you would pass 0x00ffffff as the mask - * to this function. - * - * FIXME: This should really be a platform specific issue - we should - * return false if GFP_DMA allocations may not satisfy the supplied 'mask'. - */ -static inline int dma_supported(struct device *dev, u64 mask) +/* The ARM override for dma_max_pfn() */ +static inline unsigned long dma_max_pfn(struct device *dev) { - return dev->dma_mask && *dev->dma_mask != 0; + return PHYS_PFN_OFFSET + dma_to_pfn(dev, *dev->dma_mask); } +#define dma_max_pfn(dev) dma_max_pfn(dev) -static inline int dma_set_mask(struct device *dev, u64 dma_mask) +static inline int set_arch_dma_coherent_ops(struct device *dev) { - if (!dev->dma_mask || !dma_supported(dev, dma_mask)) - return -EIO; - - *dev->dma_mask = dma_mask; - + set_dma_ops(dev, &arm_coherent_dma_ops); return 0; } +#define set_arch_dma_coherent_ops(dev) set_arch_dma_coherent_ops(dev) -static inline int dma_get_cache_alignment(void) +static inline dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr) { - return 32; + unsigned int offset = paddr & ~PAGE_MASK; + return pfn_to_dma(dev, __phys_to_pfn(paddr)) + offset; } -static inline int dma_is_consistent(struct device *dev, dma_addr_t handle) +static inline phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr) { - return !!arch_is_coherent(); + unsigned int offset = dev_addr & ~PAGE_MASK; + return __pfn_to_phys(dma_to_pfn(dev, dev_addr)) + offset; +} + +static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) +{ + u64 limit, mask; + + if (!dev->dma_mask) + return 0; + + mask = *dev->dma_mask; + + limit = (mask + 1) & ~mask; + if (limit && size > limit) + return 0; + + if ((addr | (addr + size - 1)) & ~mask) + return 0; + + return 1; } +static inline void dma_mark_clean(void *addr, size_t size) { } + /* * DMA errors are defined by all-bits-set in the DMA address. */ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) { - return dma_addr == ~0; + debug_dma_mapping_error(dev, dma_addr); + return dma_addr == DMA_ERROR_CODE; } /* * Dummy noncoherent implementation. We don't provide a dma_cache_sync * function so drivers using this API are highlighted with build warnings. */ -static inline void * -dma_alloc_noncoherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp) +static inline void *dma_alloc_noncoherent(struct device *dev, size_t size, + dma_addr_t *handle, gfp_t gfp) { return NULL; } -static inline void -dma_free_noncoherent(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle) +static inline void dma_free_noncoherent(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t handle) { } +extern int dma_supported(struct device *dev, u64 mask); + +extern int arm_dma_set_mask(struct device *dev, u64 dma_mask); + /** - * dma_alloc_coherent - allocate consistent memory for DMA + * arm_dma_alloc - allocate consistent memory for DMA * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices * @size: required memory size * @handle: bus-specific DMA address + * @attrs: optinal attributes that specific mapping properties * - * Allocate some uncached, unbuffered memory for a device for - * performing DMA. This function allocates pages, and will - * return the CPU-viewed address, and sets @handle to be the - * device-viewed address. + * Allocate some memory for a device for performing DMA. This function + * allocates pages, and will return the CPU-viewed address, and sets @handle + * to be the device-viewed address. */ -extern void * -dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp); +extern void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, + gfp_t gfp, struct dma_attrs *attrs); + +#define dma_alloc_coherent(d, s, h, f) dma_alloc_attrs(d, s, h, f, NULL) + +static inline void *dma_alloc_attrs(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag, + struct dma_attrs *attrs) +{ + struct dma_map_ops *ops = get_dma_ops(dev); + void *cpu_addr; + BUG_ON(!ops); + + cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); + debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); + return cpu_addr; +} /** - * dma_free_coherent - free memory allocated by dma_alloc_coherent + * arm_dma_free - free memory allocated by arm_dma_alloc * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices * @size: size of memory originally requested in dma_alloc_coherent * @cpu_addr: CPU-view address returned from dma_alloc_coherent * @handle: device-view address returned from dma_alloc_coherent + * @attrs: optinal attributes that specific mapping properties * * Free (and unmap) a DMA buffer previously allocated by - * dma_alloc_coherent(). + * arm_dma_alloc(). * * References to memory and mappings associated with cpu_addr/handle * during and after this call executing are illegal. */ -extern void -dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t handle); +extern void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t handle, struct dma_attrs *attrs); + +#define dma_free_coherent(d, s, c, h) dma_free_attrs(d, s, c, h, NULL) + +static inline void dma_free_attrs(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_handle, + struct dma_attrs *attrs) +{ + struct dma_map_ops *ops = get_dma_ops(dev); + BUG_ON(!ops); + + debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); + ops->free(dev, size, cpu_addr, dma_handle, attrs); +} /** - * dma_mmap_coherent - map a coherent DMA allocation into user space + * arm_dma_mmap - map a coherent DMA allocation into user space * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices * @vma: vm_area_struct describing requested user mapping * @cpu_addr: kernel CPU-view address returned from dma_alloc_coherent * @handle: device-view address returned from dma_alloc_coherent * @size: size of memory originally requested in dma_alloc_coherent + * @attrs: optinal attributes that specific mapping properties * * Map a coherent DMA buffer previously allocated by dma_alloc_coherent * into user space. The coherent DMA buffer must not be freed by the * driver until the user space mapping has been released. */ -int dma_mmap_coherent(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t handle, size_t size); - - -/** - * dma_alloc_writecombine - allocate writecombining memory for DMA - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @size: required memory size - * @handle: bus-specific DMA address - * - * Allocate some uncached, buffered memory for a device for - * performing DMA. This function allocates pages, and will - * return the CPU-viewed address, and sets @handle to be the - * device-viewed address. - */ -extern void * -dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp); - -#define dma_free_writecombine(dev,size,cpu_addr,handle) \ - dma_free_coherent(dev,size,cpu_addr,handle) +extern int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + struct dma_attrs *attrs); -int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma, - void *cpu_addr, dma_addr_t handle, size_t size); - - -/** - * dma_map_single - map a single buffer for streaming DMA - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @cpu_addr: CPU direct mapped address of buffer - * @size: size of buffer to map - * @dir: DMA transfer direction - * - * Ensure that any data held in the cache is appropriately discarded - * or written back. - * - * The device owns this memory once this call has completed. The CPU - * can regain ownership by calling dma_unmap_single() or - * dma_sync_single_for_cpu(). - */ -#ifndef CONFIG_DMABOUNCE -static inline dma_addr_t -dma_map_single(struct device *dev, void *cpu_addr, size_t size, - enum dma_data_direction dir) +static inline void *dma_alloc_writecombine(struct device *dev, size_t size, + dma_addr_t *dma_handle, gfp_t flag) { - if (!arch_is_coherent()) - dma_cache_maint(cpu_addr, size, dir); - - return virt_to_dma(dev, cpu_addr); + DEFINE_DMA_ATTRS(attrs); + dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + return dma_alloc_attrs(dev, size, dma_handle, flag, &attrs); } -#else -extern dma_addr_t dma_map_single(struct device *,void *, size_t, enum dma_data_direction); -#endif -/** - * dma_map_page - map a portion of a page for streaming DMA - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @page: page that buffer resides in - * @offset: offset into page for start of buffer - * @size: size of buffer to map - * @dir: DMA transfer direction - * - * Ensure that any data held in the cache is appropriately discarded - * or written back. - * - * The device owns this memory once this call has completed. The CPU - * can regain ownership by calling dma_unmap_page() or - * dma_sync_single_for_cpu(). - */ -static inline dma_addr_t -dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir) +static inline void dma_free_writecombine(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_handle) { - return dma_map_single(dev, page_address(page) + offset, size, dir); + DEFINE_DMA_ATTRS(attrs); + dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs); + return dma_free_attrs(dev, size, cpu_addr, dma_handle, &attrs); } -/** - * dma_unmap_single - unmap a single buffer previously mapped - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @handle: DMA address of buffer - * @size: size of buffer to map - * @dir: DMA transfer direction - * - * Unmap a single streaming mode DMA translation. The handle and size - * must match what was provided in the previous dma_map_single() call. - * All other usages are undefined. - * - * After this call, reads by the CPU to the buffer are guaranteed to see - * whatever the device wrote there. - */ -#ifndef CONFIG_DMABOUNCE -static inline void -dma_unmap_single(struct device *dev, dma_addr_t handle, size_t size, - enum dma_data_direction dir) -{ - /* nothing to do */ -} -#else -extern void dma_unmap_single(struct device *, dma_addr_t, size_t, enum dma_data_direction); -#endif - -/** - * dma_unmap_page - unmap a buffer previously mapped through dma_map_page() - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @handle: DMA address of buffer - * @size: size of buffer to map - * @dir: DMA transfer direction - * - * Unmap a single streaming mode DMA translation. The handle and size - * must match what was provided in the previous dma_map_single() call. - * All other usages are undefined. - * - * After this call, reads by the CPU to the buffer are guaranteed to see - * whatever the device wrote there. - */ -static inline void -dma_unmap_page(struct device *dev, dma_addr_t handle, size_t size, - enum dma_data_direction dir) -{ - dma_unmap_single(dev, handle, size, dir); -} - -/** - * dma_map_sg - map a set of SG buffers for streaming mode DMA - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to map - * @dir: DMA transfer direction - * - * Map a set of buffers described by scatterlist in streaming - * mode for DMA. This is the scatter-gather version of the - * above dma_map_single interface. Here the scatter gather list - * elements are each tagged with the appropriate dma address - * and length. They are obtained via sg_dma_{address,length}(SG). - * - * NOTE: An implementation may be able to use a smaller number of - * DMA address/length pairs than there are SG table elements. - * (for example via virtual mapping capabilities) - * The routine returns the number of addr/length pairs actually - * used, at most nents. - * - * Device ownership issues as mentioned above for dma_map_single are - * the same here. - */ -#ifndef CONFIG_DMABOUNCE -static inline int -dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir) -{ - int i; - - for (i = 0; i < nents; i++, sg++) { - char *virt; - - sg->dma_address = page_to_dma(dev, sg_page(sg)) + sg->offset; - virt = sg_virt(sg); - - if (!arch_is_coherent()) - dma_cache_maint(virt, sg->length, dir); - } - - return nents; -} -#else -extern int dma_map_sg(struct device *, struct scatterlist *, int, enum dma_data_direction); -#endif - -/** - * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to map - * @dir: DMA transfer direction - * - * Unmap a set of streaming mode DMA translations. - * Again, CPU read rules concerning calls here are the same as for - * dma_unmap_single() above. - */ -#ifndef CONFIG_DMABOUNCE -static inline void -dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir) -{ - - /* nothing to do */ -} -#else -extern void dma_unmap_sg(struct device *, struct scatterlist *, int, enum dma_data_direction); -#endif - - -/** - * dma_sync_single_range_for_cpu - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @handle: DMA address of buffer - * @offset: offset of region to start sync - * @size: size of region to sync - * @dir: DMA transfer direction (same as passed to dma_map_single) - * - * Make physical memory consistent for a single streaming mode DMA - * translation after a transfer. - * - * If you perform a dma_map_single() but wish to interrogate the - * buffer using the cpu, yet do not wish to teardown the PCI dma - * mapping, you must call this function before doing so. At the - * next point you give the PCI dma address back to the card, you - * must first the perform a dma_sync_for_device, and then the - * device again owns the buffer. +/* + * This can be called during early boot to increase the size of the atomic + * coherent DMA pool above the default value of 256KiB. It must be called + * before postcore_initcall. */ -#ifndef CONFIG_DMABOUNCE -static inline void -dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t handle, - unsigned long offset, size_t size, - enum dma_data_direction dir) -{ - if (!arch_is_coherent()) - dma_cache_maint(dma_to_virt(dev, handle) + offset, size, dir); -} - -static inline void -dma_sync_single_range_for_device(struct device *dev, dma_addr_t handle, - unsigned long offset, size_t size, - enum dma_data_direction dir) -{ - if (!arch_is_coherent()) - dma_cache_maint(dma_to_virt(dev, handle) + offset, size, dir); -} -#else -extern void dma_sync_single_range_for_cpu(struct device *, dma_addr_t, unsigned long, size_t, enum dma_data_direction); -extern void dma_sync_single_range_for_device(struct device *, dma_addr_t, unsigned long, size_t, enum dma_data_direction); -#endif - -static inline void -dma_sync_single_for_cpu(struct device *dev, dma_addr_t handle, size_t size, - enum dma_data_direction dir) -{ - dma_sync_single_range_for_cpu(dev, handle, 0, size, dir); -} - -static inline void -dma_sync_single_for_device(struct device *dev, dma_addr_t handle, size_t size, - enum dma_data_direction dir) -{ - dma_sync_single_range_for_device(dev, handle, 0, size, dir); -} - +extern void __init init_dma_coherent_pool_size(unsigned long size); -/** - * dma_sync_sg_for_cpu - * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices - * @sg: list of buffers - * @nents: number of buffers to map - * @dir: DMA transfer direction - * - * Make physical memory consistent for a set of streaming - * mode DMA translations after a transfer. - * - * The same as dma_sync_single_for_* but for a scatter-gather list, - * same rules and usage. - */ -#ifndef CONFIG_DMABOUNCE -static inline void -dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir) -{ - int i; - - for (i = 0; i < nents; i++, sg++) { - char *virt = sg_virt(sg); - if (!arch_is_coherent()) - dma_cache_maint(virt, sg->length, dir); - } -} - -static inline void -dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nents, - enum dma_data_direction dir) -{ - int i; - - for (i = 0; i < nents; i++, sg++) { - char *virt = sg_virt(sg); - if (!arch_is_coherent()) - dma_cache_maint(virt, sg->length, dir); - } -} -#else -extern void dma_sync_sg_for_cpu(struct device*, struct scatterlist*, int, enum dma_data_direction); -extern void dma_sync_sg_for_device(struct device*, struct scatterlist*, int, enum dma_data_direction); -#endif - -#ifdef CONFIG_DMABOUNCE /* * For SA-1111, IXP425, and ADI systems the dma-mapping functions are "magic" * and utilize bounce buffers as needed to work around limited DMA windows. @@ -469,13 +306,14 @@ extern void dma_sync_sg_for_device(struct device*, struct scatterlist*, int, enu * @dev: valid struct device pointer * @small_buf_size: size of buffers to use with small buffer pool * @large_buf_size: size of buffers to use with large buffer pool (can be 0) + * @needs_bounce_fn: called to determine whether buffer needs bouncing * * This function should be called by low-level platform code to register * a device as requireing DMA buffer bouncing. The function will allocate * appropriate DMA pools for the device. - * */ -extern int dmabounce_register_dev(struct device *, unsigned long, unsigned long); +extern int dmabounce_register_dev(struct device *, unsigned long, + unsigned long, int (*)(struct device *, dma_addr_t, size_t)); /** * dmabounce_unregister_dev @@ -489,24 +327,22 @@ extern int dmabounce_register_dev(struct device *, unsigned long, unsigned long) */ extern void dmabounce_unregister_dev(struct device *); -/** - * dma_needs_bounce - * - * @dev: valid struct device pointer - * @dma_handle: dma_handle of unbounced buffer - * @size: size of region being mapped - * - * Platforms that utilize the dmabounce mechanism must implement - * this function. - * - * The dmabounce routines call this function whenever a dma-mapping - * is requested to determine whether a given buffer needs to be bounced - * or not. The function must return 0 if the buffer is OK for - * DMA access and 1 if the buffer needs to be bounced. - * + + +/* + * The scatter list versions of the above methods. */ -extern int dma_needs_bounce(struct device*, dma_addr_t, size_t); -#endif /* CONFIG_DMABOUNCE */ +extern int arm_dma_map_sg(struct device *, struct scatterlist *, int, + enum dma_data_direction, struct dma_attrs *attrs); +extern void arm_dma_unmap_sg(struct device *, struct scatterlist *, int, + enum dma_data_direction, struct dma_attrs *attrs); +extern void arm_dma_sync_sg_for_cpu(struct device *, struct scatterlist *, int, + enum dma_data_direction); +extern void arm_dma_sync_sg_for_device(struct device *, struct scatterlist *, int, + enum dma_data_direction); +extern int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, + void *cpu_addr, dma_addr_t dma_addr, size_t size, + struct dma_attrs *attrs); #endif /* __KERNEL__ */ #endif |