diff options
Diffstat (limited to 'arch/metag')
| -rw-r--r-- | arch/metag/include/asm/io.h | 165 | ||||
| -rw-r--r-- | arch/metag/include/asm/uaccess.h | 241 | ||||
| -rw-r--r-- | arch/metag/lib/usercopy.c | 1341 | ||||
| -rw-r--r-- | arch/metag/mm/ioremap.c | 89 | ||||
| -rw-r--r-- | arch/metag/mm/maccess.c | 68 |
5 files changed, 1904 insertions, 0 deletions
diff --git a/arch/metag/include/asm/io.h b/arch/metag/include/asm/io.h new file mode 100644 index 00000000000..9359e504844 --- /dev/null +++ b/arch/metag/include/asm/io.h @@ -0,0 +1,165 @@ +#ifndef _ASM_METAG_IO_H +#define _ASM_METAG_IO_H + +#include <linux/types.h> + +#define IO_SPACE_LIMIT 0 + +#define page_to_bus page_to_phys +#define bus_to_page phys_to_page + +/* + * Generic I/O + */ + +#define __raw_readb __raw_readb +static inline u8 __raw_readb(const volatile void __iomem *addr) +{ + u8 ret; + asm volatile("GETB %0,[%1]" + : "=da" (ret) + : "da" (addr) + : "memory"); + return ret; +} + +#define __raw_readw __raw_readw +static inline u16 __raw_readw(const volatile void __iomem *addr) +{ + u16 ret; + asm volatile("GETW %0,[%1]" + : "=da" (ret) + : "da" (addr) + : "memory"); + return ret; +} + +#define __raw_readl __raw_readl +static inline u32 __raw_readl(const volatile void __iomem *addr) +{ + u32 ret; + asm volatile("GETD %0,[%1]" + : "=da" (ret) + : "da" (addr) + : "memory"); + return ret; +} + +#define __raw_readq __raw_readq +static inline u64 __raw_readq(const volatile void __iomem *addr) +{ + u64 ret; + asm volatile("GETL %0,%t0,[%1]" + : "=da" (ret) + : "da" (addr) + : "memory"); + return ret; +} + +#define __raw_writeb __raw_writeb +static inline void __raw_writeb(u8 b, volatile void __iomem *addr) +{ + asm volatile("SETB [%0],%1" + : + : "da" (addr), + "da" (b) + : "memory"); +} + +#define __raw_writew __raw_writew +static inline void __raw_writew(u16 b, volatile void __iomem *addr) +{ + asm volatile("SETW [%0],%1" + : + : "da" (addr), + "da" (b) + : "memory"); +} + +#define __raw_writel __raw_writel +static inline void __raw_writel(u32 b, volatile void __iomem *addr) +{ + asm volatile("SETD [%0],%1" + : + : "da" (addr), + "da" (b) + : "memory"); +} + +#define __raw_writeq __raw_writeq +static inline void __raw_writeq(u64 b, volatile void __iomem *addr) +{ + asm volatile("SETL [%0],%1,%t1" + : + : "da" (addr), + "da" (b) + : "memory"); +} + +/* + * The generic io.h can define all the other generic accessors + */ + +#include <asm-generic/io.h> + +/* + * Despite being a 32bit architecture, Meta can do 64bit memory accesses + * (assuming the bus supports it). + */ + +#define readq __raw_readq +#define writeq __raw_writeq + +/* + * Meta specific I/O for accessing non-MMU areas. + * + * These can be provided with a physical address rather than an __iomem pointer + * and should only be used by core architecture code for accessing fixed core + * registers. Generic drivers should use ioremap and the generic I/O accessors. + */ + +#define metag_in8(addr) __raw_readb((volatile void __iomem *)(addr)) +#define metag_in16(addr) __raw_readw((volatile void __iomem *)(addr)) +#define metag_in32(addr) __raw_readl((volatile void __iomem *)(addr)) +#define metag_in64(addr) __raw_readq((volatile void __iomem *)(addr)) + +#define metag_out8(b, addr) __raw_writeb(b, (volatile void __iomem *)(addr)) +#define metag_out16(b, addr) __raw_writew(b, (volatile void __iomem *)(addr)) +#define metag_out32(b, addr) __raw_writel(b, (volatile void __iomem *)(addr)) +#define metag_out64(b, addr) __raw_writeq(b, (volatile void __iomem *)(addr)) + +/* + * io remapping functions + */ + +extern void __iomem *__ioremap(unsigned long offset, + size_t size, unsigned long flags); +extern void __iounmap(void __iomem *addr); + +/** + * ioremap - map bus memory into CPU space + * @offset: bus address of the memory + * @size: size of the resource to map + * + * ioremap performs a platform specific sequence of operations to + * make bus memory CPU accessible via the readb/readw/readl/writeb/ + * writew/writel functions and the other mmio helpers. The returned + * address is not guaranteed to be usable directly as a virtual + * address. + */ +#define ioremap(offset, size) \ + __ioremap((offset), (size), 0) + +#define ioremap_nocache(offset, size) \ + __ioremap((offset), (size), 0) + +#define ioremap_cached(offset, size) \ + __ioremap((offset), (size), _PAGE_CACHEABLE) + +#define ioremap_wc(offset, size) \ + __ioremap((offset), (size), _PAGE_WR_COMBINE) + +#define iounmap(addr) \ + __iounmap(addr) + +#endif /* _ASM_METAG_IO_H */ diff --git a/arch/metag/include/asm/uaccess.h b/arch/metag/include/asm/uaccess.h new file mode 100644 index 00000000000..0748b0a9798 --- /dev/null +++ b/arch/metag/include/asm/uaccess.h @@ -0,0 +1,241 @@ +#ifndef __METAG_UACCESS_H +#define __METAG_UACCESS_H + +/* + * User space memory access functions + */ +#include <linux/sched.h> + +#define VERIFY_READ 0 +#define VERIFY_WRITE 1 + +/* + * The fs value determines whether argument validity checking should be + * performed or not. If get_fs() == USER_DS, checking is performed, with + * get_fs() == KERNEL_DS, checking is bypassed. + * + * For historical reasons, these macros are grossly misnamed. + */ + +#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) + +#define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFF) +#define USER_DS MAKE_MM_SEG(PAGE_OFFSET) + +#define get_ds() (KERNEL_DS) +#define get_fs() (current_thread_info()->addr_limit) +#define set_fs(x) (current_thread_info()->addr_limit = (x)) + +#define segment_eq(a, b) ((a).seg == (b).seg) + +#define __kernel_ok (segment_eq(get_fs(), KERNEL_DS)) +/* + * Explicitly allow NULL pointers here. Parts of the kernel such + * as readv/writev use access_ok to validate pointers, but want + * to allow NULL pointers for various reasons. NULL pointers are + * safe to allow through because the first page is not mappable on + * Meta. + * + * We also wish to avoid letting user code access the system area + * and the kernel half of the address space. + */ +#define __user_bad(addr, size) (((addr) > 0 && (addr) < META_MEMORY_BASE) || \ + ((addr) > PAGE_OFFSET && \ + (addr) < LINCORE_BASE)) + +static inline int __access_ok(unsigned long addr, unsigned long size) +{ + return __kernel_ok || !__user_bad(addr, size); +} + +#define access_ok(type, addr, size) __access_ok((unsigned long)(addr), \ + (unsigned long)(size)) + +static inline int verify_area(int type, const void *addr, unsigned long size) +{ + return access_ok(type, addr, size) ? 0 : -EFAULT; +} + +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ +struct exception_table_entry { + unsigned long insn, fixup; +}; + +extern int fixup_exception(struct pt_regs *regs); + +/* + * These are the main single-value transfer routines. They automatically + * use the right size if we just have the right pointer type. + */ + +#define put_user(x, ptr) \ + __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) +#define __put_user(x, ptr) \ + __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) + +extern void __put_user_bad(void); + +#define __put_user_nocheck(x, ptr, size) \ +({ \ + long __pu_err; \ + __put_user_size((x), (ptr), (size), __pu_err); \ + __pu_err; \ +}) + +#define __put_user_check(x, ptr, size) \ +({ \ + long __pu_err = -EFAULT; \ + __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ + if (access_ok(VERIFY_WRITE, __pu_addr, size)) \ + __put_user_size((x), __pu_addr, (size), __pu_err); \ + __pu_err; \ +}) + +extern long __put_user_asm_b(unsigned int x, void __user *addr); +extern long __put_user_asm_w(unsigned int x, void __user *addr); +extern long __put_user_asm_d(unsigned int x, void __user *addr); +extern long __put_user_asm_l(unsigned long long x, void __user *addr); + +#define __put_user_size(x, ptr, size, retval) \ +do { \ + retval = 0; \ + switch (size) { \ + case 1: \ + retval = __put_user_asm_b((unsigned int)x, ptr); break; \ + case 2: \ + retval = __put_user_asm_w((unsigned int)x, ptr); break; \ + case 4: \ + retval = __put_user_asm_d((unsigned int)x, ptr); break; \ + case 8: \ + retval = __put_user_asm_l((unsigned long long)x, ptr); break; \ + default: \ + __put_user_bad(); \ + } \ +} while (0) + +#define get_user(x, ptr) \ + __get_user_check((x), (ptr), sizeof(*(ptr))) +#define __get_user(x, ptr) \ + __get_user_nocheck((x), (ptr), sizeof(*(ptr))) + +extern long __get_user_bad(void); + +#define __get_user_nocheck(x, ptr, size) \ +({ \ + long __gu_err, __gu_val; \ + __get_user_size(__gu_val, (ptr), (size), __gu_err); \ + (x) = (__typeof__(*(ptr)))__gu_val; \ + __gu_err; \ +}) + +#define __get_user_check(x, ptr, size) \ +({ \ + long __gu_err = -EFAULT, __gu_val = 0; \ + const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ + if (access_ok(VERIFY_READ, __gu_addr, size)) \ + __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ + (x) = (__typeof__(*(ptr)))__gu_val; \ + __gu_err; \ +}) + +extern unsigned char __get_user_asm_b(const void __user *addr, long *err); +extern unsigned short __get_user_asm_w(const void __user *addr, long *err); +extern unsigned int __get_user_asm_d(const void __user *addr, long *err); + +#define __get_user_size(x, ptr, size, retval) \ +do { \ + retval = 0; \ + switch (size) { \ + case 1: \ + x = __get_user_asm_b(ptr, &retval); break; \ + case 2: \ + x = __get_user_asm_w(ptr, &retval); break; \ + case 4: \ + x = __get_user_asm_d(ptr, &retval); break; \ + default: \ + (x) = __get_user_bad(); \ + } \ +} while (0) + +/* + * Copy a null terminated string from userspace. + * + * Must return: + * -EFAULT for an exception + * count if we hit the buffer limit + * bytes copied if we hit a null byte + * (without the null byte) + */ + +extern long __must_check __strncpy_from_user(char *dst, const char __user *src, + long count); + +#define strncpy_from_user(dst, src, count) __strncpy_from_user(dst, src, count) + +/* + * Return the size of a string (including the ending 0) + * + * Return 0 on exception, a value greater than N if too long + */ +extern long __must_check strnlen_user(const char __user *src, long count); + +#define strlen_user(str) strnlen_user(str, 32767) + +extern unsigned long __must_check __copy_user_zeroing(void *to, + const void __user *from, + unsigned long n); + +static inline unsigned long +copy_from_user(void *to, const void __user *from, unsigned long n) +{ + if (access_ok(VERIFY_READ, from, n)) + return __copy_user_zeroing(to, from, n); + return n; +} + +#define __copy_from_user(to, from, n) __copy_user_zeroing(to, from, n) +#define __copy_from_user_inatomic __copy_from_user + +extern unsigned long __must_check __copy_user(void __user *to, + const void *from, + unsigned long n); + +static inline unsigned long copy_to_user(void __user *to, const void *from, + unsigned long n) +{ + if (access_ok(VERIFY_WRITE, to, n)) + return __copy_user(to, from, n); + return n; +} + +#define __copy_to_user(to, from, n) __copy_user(to, from, n) +#define __copy_to_user_inatomic __copy_to_user + +/* + * Zero Userspace + */ + +extern unsigned long __must_check __do_clear_user(void __user *to, + unsigned long n); + +static inline unsigned long clear_user(void __user *to, unsigned long n) +{ + if (access_ok(VERIFY_WRITE, to, n)) + return __do_clear_user(to, n); + return n; +} + +#define __clear_user(to, n) __do_clear_user(to, n) + +#endif /* _METAG_UACCESS_H */ diff --git a/arch/metag/lib/usercopy.c b/arch/metag/lib/usercopy.c new file mode 100644 index 00000000000..92f6dbb34e8 --- /dev/null +++ b/arch/metag/lib/usercopy.c @@ -0,0 +1,1341 @@ +/* + * User address space access functions. + * The non-inlined parts of asm-metag/uaccess.h are here. + * + * Copyright (C) 2006, Imagination Technologies. + * Copyright (C) 2000, Axis Communications AB. + * + * Written by Hans-Peter Nilsson. + * Pieces used from memcpy, originally by Kenny Ranerup long time ago. + * Modified for Meta by Will Newton. + */ + +#include <linux/uaccess.h> +#include <asm/cache.h> /* def of L1_CACHE_BYTES */ + +#define USE_RAPF +#define RAPF_MIN_BUF_SIZE (3*L1_CACHE_BYTES) + + +/* The "double write" in this code is because the Meta will not fault + * immediately unless the memory pipe is forced to by e.g. a data stall or + * another memory op. The second write should be discarded by the write + * combiner so should have virtually no cost. + */ + +#define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + asm volatile ( \ + COPY \ + "1:\n" \ + " .section .fixup,\"ax\"\n" \ + " MOV D1Ar1,#0\n" \ + FIXUP \ + " MOVT D1Ar1,#HI(1b)\n" \ + " JUMP D1Ar1,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + TENTRY \ + " .previous\n" \ + : "=r" (to), "=r" (from), "=r" (ret) \ + : "0" (to), "1" (from), "2" (ret) \ + : "D1Ar1", "memory") + + +#define __asm_copy_to_user_1(to, from, ret) \ + __asm_copy_user_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "2: SETB [%0++],D1Ar1\n", \ + "3: ADD %2,%2,#1\n", \ + " .long 2b,3b\n") + +#define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_user_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + " SETW [%0],D1Ar1\n" \ + "2: SETW [%0++],D1Ar1\n" COPY, \ + "3: ADD %2,%2,#2\n" FIXUP, \ + " .long 2b,3b\n" TENTRY) + +#define __asm_copy_to_user_2(to, from, ret) \ + __asm_copy_to_user_2x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_3(to, from, ret) \ + __asm_copy_to_user_2x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "4: SETB [%0++],D1Ar1\n", \ + "5: ADD %2,%2,#1\n", \ + " .long 4b,5b\n") + +#define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_user_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + " SETD [%0],D1Ar1\n" \ + "2: SETD [%0++],D1Ar1\n" COPY, \ + "3: ADD %2,%2,#4\n" FIXUP, \ + " .long 2b,3b\n" TENTRY) + +#define __asm_copy_to_user_4(to, from, ret) \ + __asm_copy_to_user_4x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_5(to, from, ret) \ + __asm_copy_to_user_4x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "4: SETB [%0++],D1Ar1\n", \ + "5: ADD %2,%2,#1\n", \ + " .long 4b,5b\n") + +#define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_4x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + " SETW [%0],D1Ar1\n" \ + "4: SETW [%0++],D1Ar1\n" COPY, \ + "5: ADD %2,%2,#2\n" FIXUP, \ + " .long 4b,5b\n" TENTRY) + +#define __asm_copy_to_user_6(to, from, ret) \ + __asm_copy_to_user_6x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_7(to, from, ret) \ + __asm_copy_to_user_6x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "6: SETB [%0++],D1Ar1\n", \ + "7: ADD %2,%2,#1\n", \ + " .long 6b,7b\n") + +#define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_4x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + " SETD [%0],D1Ar1\n" \ + "4: SETD [%0++],D1Ar1\n" COPY, \ + "5: ADD %2,%2,#4\n" FIXUP, \ + " .long 4b,5b\n" TENTRY) + +#define __asm_copy_to_user_8(to, from, ret) \ + __asm_copy_to_user_8x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_9(to, from, ret) \ + __asm_copy_to_user_8x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "6: SETB [%0++],D1Ar1\n", \ + "7: ADD %2,%2,#1\n", \ + " .long 6b,7b\n") + +#define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_8x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + " SETW [%0],D1Ar1\n" \ + "6: SETW [%0++],D1Ar1\n" COPY, \ + "7: ADD %2,%2,#2\n" FIXUP, \ + " .long 6b,7b\n" TENTRY) + +#define __asm_copy_to_user_10(to, from, ret) \ + __asm_copy_to_user_10x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_11(to, from, ret) \ + __asm_copy_to_user_10x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "8: SETB [%0++],D1Ar1\n", \ + "9: ADD %2,%2,#1\n", \ + " .long 8b,9b\n") + +#define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_8x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + " SETD [%0],D1Ar1\n" \ + "6: SETD [%0++],D1Ar1\n" COPY, \ + "7: ADD %2,%2,#4\n" FIXUP, \ + " .long 6b,7b\n" TENTRY) +#define __asm_copy_to_user_12(to, from, ret) \ + __asm_copy_to_user_12x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_13(to, from, ret) \ + __asm_copy_to_user_12x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "8: SETB [%0++],D1Ar1\n", \ + "9: ADD %2,%2,#1\n", \ + " .long 8b,9b\n") + +#define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_12x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + " SETW [%0],D1Ar1\n" \ + "8: SETW [%0++],D1Ar1\n" COPY, \ + "9: ADD %2,%2,#2\n" FIXUP, \ + " .long 8b,9b\n" TENTRY) + +#define __asm_copy_to_user_14(to, from, ret) \ + __asm_copy_to_user_14x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_15(to, from, ret) \ + __asm_copy_to_user_14x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + " SETB [%0],D1Ar1\n" \ + "10: SETB [%0++],D1Ar1\n", \ + "11: ADD %2,%2,#1\n", \ + " .long 10b,11b\n") + +#define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_to_user_12x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + " SETD [%0],D1Ar1\n" \ + "8: SETD [%0++],D1Ar1\n" COPY, \ + "9: ADD %2,%2,#4\n" FIXUP, \ + " .long 8b,9b\n" TENTRY) + +#define __asm_copy_to_user_16(to, from, ret) \ + __asm_copy_to_user_16x_cont(to, from, ret, "", "", "") + +#define __asm_copy_to_user_8x64(to, from, ret) \ + asm volatile ( \ + " GETL D0Ar2,D1Ar1,[%1++]\n" \ + " SETL [%0],D0Ar2,D1Ar1\n" \ + "2: SETL [%0++],D0Ar2,D1Ar1\n" \ + "1:\n" \ + " .section .fixup,\"ax\"\n" \ + "3: ADD %2,%2,#8\n" \ + " MOVT D0Ar2,#HI(1b)\n" \ + " JUMP D0Ar2,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .long 2b,3b\n" \ + " .previous\n" \ + : "=r" (to), "=r" (from), "=r" (ret) \ + : "0" (to), "1" (from), "2" (ret) \ + : "D1Ar1", "D0Ar2", "memory") + +/* + * optimized copying loop using RAPF when 64 bit aligned + * + * n will be automatically decremented inside the loop + * ret will be left intact. if error occurs we will rewind + * so that the original non optimized code will fill up + * this value correctly. + * + * on fault: + * > n will hold total number of uncopied bytes + * + * > {'to','from'} will be rewind back so that + * the non-optimized code will do the proper fix up + * + * DCACHE drops the cacheline which helps in reducing cache + * pollution. + * + * We introduce an extra SETL at the end of the loop to + * ensure we don't fall off the loop before we catch all + * erros. + * + * NOTICE: + * LSM_STEP in TXSTATUS must be cleared in fix up code. + * since we're using M{S,G}ETL, a fault might happen at + * any address in the middle of M{S,G}ETL causing + * the value of LSM_STEP to be incorrect which can + * cause subsequent use of M{S,G}ET{L,D} to go wrong. + * ie: if LSM_STEP was 1 when a fault occurs, the + * next call to M{S,G}ET{L,D} will skip the first + * copy/getting as it think that the first 1 has already + * been done. + * + */ +#define __asm_copy_user_64bit_rapf_loop( \ + to, from, ret, n, id, FIXUP) \ + asm volatile ( \ + ".balign 8\n" \ + "MOV RAPF, %1\n" \ + "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \ + "MOV D0Ar6, #0\n" \ + "LSR D1Ar5, %3, #6\n" \ + "SUB TXRPT, D1Ar5, #2\n" \ + "MOV RAPF, %1\n" \ + "$Lloop"id":\n" \ + "ADD RAPF, %1, #64\n" \ + "21:\n" \ + "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "22:\n" \ + "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #32\n" \ + "23:\n" \ + "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "24:\n" \ + "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #32\n" \ + "DCACHE [%1+#-64], D0Ar6\n" \ + "BR $Lloop"id"\n" \ + \ + "MOV RAPF, %1\n" \ + "25:\n" \ + "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "26:\n" \ + "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #32\n" \ + "27:\n" \ + "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "28:\n" \ + "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %0, %0, #8\n" \ + "29:\n" \ + "SETL [%0++], D0.7, D1.7\n" \ + "SUB %3, %3, #32\n" \ + "1:" \ + "DCACHE [%1+#-64], D0Ar6\n" \ + "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \ + "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \ + "GETL D0.5, D1.5, [A0StP+#-24]\n" \ + "GETL D0.6, D1.6, [A0StP+#-16]\n" \ + "GETL D0.7, D1.7, [A0StP+#-8]\n" \ + "SUB A0StP, A0StP, #40\n" \ + " .section .fixup,\"ax\"\n" \ + "4:\n" \ + " ADD %0, %0, #8\n" \ + "3:\n" \ + " MOV D0Ar2, TXSTATUS\n" \ + " MOV D1Ar1, TXSTATUS\n" \ + " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \ + " MOV TXSTATUS, D1Ar1\n" \ + FIXUP \ + " MOVT D0Ar2,#HI(1b)\n" \ + " JUMP D0Ar2,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .long 21b,3b\n" \ + " .long 22b,3b\n" \ + " .long 23b,3b\n" \ + " .long 24b,3b\n" \ + " .long 25b,3b\n" \ + " .long 26b,3b\n" \ + " .long 27b,3b\n" \ + " .long 28b,3b\n" \ + " .long 29b,4b\n" \ + " .previous\n" \ + : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \ + : "0" (to), "1" (from), "2" (ret), "3" (n) \ + : "D1Ar1", "D0Ar2", "memory") + +/* rewind 'to' and 'from' pointers when a fault occurs + * + * Rationale: + * A fault always occurs on writing to user buffer. A fault + * is at a single address, so we need to rewind by only 4 + * bytes. + * Since we do a complete read from kernel buffer before + * writing, we need to rewind it also. The amount to be + * rewind equals the number of faulty writes in MSETD + * which is: [4 - (LSM_STEP-1)]*8 + * LSM_STEP is bits 10:8 in TXSTATUS which is already read + * and stored in D0Ar2 + * + * NOTE: If a fault occurs at the last operation in M{G,S}ETL + * LSM_STEP will be 0. ie: we do 4 writes in our case, if + * a fault happens at the 4th write, LSM_STEP will be 0 + * instead of 4. The code copes with that. + * + * n is updated by the number of successful writes, which is: + * n = n - (LSM_STEP-1)*8 + */ +#define __asm_copy_to_user_64bit_rapf_loop(to, from, ret, n, id)\ + __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \ + "LSR D0Ar2, D0Ar2, #8\n" \ + "AND D0Ar2, D0Ar2, #0x7\n" \ + "ADDZ D0Ar2, D0Ar2, #4\n" \ + "SUB D0Ar2, D0Ar2, #1\n" \ + "MOV D1Ar1, #4\n" \ + "SUB D0Ar2, D1Ar1, D0Ar2\n" \ + "LSL D0Ar2, D0Ar2, #3\n" \ + "LSL D1Ar1, D1Ar1, #3\n" \ + "SUB D1Ar1, D1Ar1, D0Ar2\n" \ + "SUB %0, %0, #8\n" \ + "SUB %1, %1,D0Ar2\n" \ + "SUB %3, %3, D1Ar1\n") + +/* + * optimized copying loop using RAPF when 32 bit aligned + * + * n will be automatically decremented inside the loop + * ret will be left intact. if error occurs we will rewind + * so that the original non optimized code will fill up + * this value correctly. + * + * on fault: + * > n will hold total number of uncopied bytes + * + * > {'to','from'} will be rewind back so that + * the non-optimized code will do the proper fix up + * + * DCACHE drops the cacheline which helps in reducing cache + * pollution. + * + * We introduce an extra SETD at the end of the loop to + * ensure we don't fall off the loop before we catch all + * erros. + * + * NOTICE: + * LSM_STEP in TXSTATUS must be cleared in fix up code. + * since we're using M{S,G}ETL, a fault might happen at + * any address in the middle of M{S,G}ETL causing + * the value of LSM_STEP to be incorrect which can + * cause subsequent use of M{S,G}ET{L,D} to go wrong. + * ie: if LSM_STEP was 1 when a fault occurs, the + * next call to M{S,G}ET{L,D} will skip the first + * copy/getting as it think that the first 1 has already + * been done. + * + */ +#define __asm_copy_user_32bit_rapf_loop( \ + to, from, ret, n, id, FIXUP) \ + asm volatile ( \ + ".balign 8\n" \ + "MOV RAPF, %1\n" \ + "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \ + "MOV D0Ar6, #0\n" \ + "LSR D1Ar5, %3, #6\n" \ + "SUB TXRPT, D1Ar5, #2\n" \ + "MOV RAPF, %1\n" \ + "$Lloop"id":\n" \ + "ADD RAPF, %1, #64\n" \ + "21:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "22:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "23:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "24:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "25:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "26:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "27:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "28:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "DCACHE [%1+#-64], D0Ar6\n" \ + "BR $Lloop"id"\n" \ + \ + "MOV RAPF, %1\n" \ + "29:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "30:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "31:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "32:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "33:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "34:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %3, %3, #16\n" \ + "35:\n" \ + "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ + "36:\n" \ + "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ + "SUB %0, %0, #4\n" \ + "37:\n" \ + "SETD [%0++], D0.7\n" \ + "SUB %3, %3, #16\n" \ + "1:" \ + "DCACHE [%1+#-64], D0Ar6\n" \ + "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \ + "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \ + "GETL D0.5, D1.5, [A0StP+#-24]\n" \ + "GETL D0.6, D1.6, [A0StP+#-16]\n" \ + "GETL D0.7, D1.7, [A0StP+#-8]\n" \ + "SUB A0StP, A0StP, #40\n" \ + " .section .fixup,\"ax\"\n" \ + "4:\n" \ + " ADD %0, %0, #4\n" \ + "3:\n" \ + " MOV D0Ar2, TXSTATUS\n" \ + " MOV D1Ar1, TXSTATUS\n" \ + " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \ + " MOV TXSTATUS, D1Ar1\n" \ + FIXUP \ + " MOVT D0Ar2,#HI(1b)\n" \ + " JUMP D0Ar2,#LO(1b)\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .long 21b,3b\n" \ + " .long 22b,3b\n" \ + " .long 23b,3b\n" \ + " .long 24b,3b\n" \ + " .long 25b,3b\n" \ + " .long 26b,3b\n" \ + " .long 27b,3b\n" \ + " .long 28b,3b\n" \ + " .long 29b,3b\n" \ + " .long 30b,3b\n" \ + " .long 31b,3b\n" \ + " .long 32b,3b\n" \ + " .long 33b,3b\n" \ + " .long 34b,3b\n" \ + " .long 35b,3b\n" \ + " .long 36b,3b\n" \ + " .long 37b,4b\n" \ + " .previous\n" \ + : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \ + : "0" (to), "1" (from), "2" (ret), "3" (n) \ + : "D1Ar1", "D0Ar2", "memory") + +/* rewind 'to' and 'from' pointers when a fault occurs + * + * Rationale: + * A fault always occurs on writing to user buffer. A fault + * is at a single address, so we need to rewind by only 4 + * bytes. + * Since we do a complete read from kernel buffer before + * writing, we need to rewind it also. The amount to be + * rewind equals the number of faulty writes in MSETD + * which is: [4 - (LSM_STEP-1)]*4 + * LSM_STEP is bits 10:8 in TXSTATUS which is already read + * and stored in D0Ar2 + * + * NOTE: If a fault occurs at the last operation in M{G,S}ETL + * LSM_STEP will be 0. ie: we do 4 writes in our case, if + * a fault happens at the 4th write, LSM_STEP will be 0 + * instead of 4. The code copes with that. + * + * n is updated by the number of successful writes, which is: + * n = n - (LSM_STEP-1)*4 + */ +#define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\ + __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \ + "LSR D0Ar2, D0Ar2, #8\n" \ + "AND D0Ar2, D0Ar2, #0x7\n" \ + "ADDZ D0Ar2, D0Ar2, #4\n" \ + "SUB D0Ar2, D0Ar2, #1\n" \ + "MOV D1Ar1, #4\n" \ + "SUB D0Ar2, D1Ar1, D0Ar2\n" \ + "LSL D0Ar2, D0Ar2, #2\n" \ + "LSL D1Ar1, D1Ar1, #2\n" \ + "SUB D1Ar1, D1Ar1, D0Ar2\n" \ + "SUB %0, %0, #4\n" \ + "SUB %1, %1, D0Ar2\n" \ + "SUB %3, %3, D1Ar1\n") + +unsigned long __copy_user(void __user *pdst, const void *psrc, + unsigned long n) +{ + register char __user *dst asm ("A0.2") = pdst; + register const char *src asm ("A1.2") = psrc; + unsigned long retn = 0; + + if (n == 0) + return 0; + + if ((unsigned long) src & 1) { + __asm_copy_to_user_1(dst, src, retn); + n--; + } + if ((unsigned long) dst & 1) { + /* Worst case - byte copy */ + while (n > 0) { + __asm_copy_to_user_1(dst, src, retn); + n--; + } + } + if (((unsigned long) src & 2) && n >= 2) { + __asm_copy_to_user_2(dst, src, retn); + n -= 2; + } + if ((unsigned long) dst & 2) { + /* Second worst case - word copy */ + while (n >= 2) { + __asm_copy_to_user_2(dst, src, retn); + n -= 2; + } + } + +#ifdef USE_RAPF + /* 64 bit copy loop */ + if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) { + if (n >= RAPF_MIN_BUF_SIZE) { + /* copy user using 64 bit rapf copy */ + __asm_copy_to_user_64bit_rapf_loop(dst, src, retn, + n, "64cu"); + } + while (n >= 8) { + __asm_copy_to_user_8x64(dst, src, retn); + n -= 8; + } + } + if (n >= RAPF_MIN_BUF_SIZE) { + /* copy user using 32 bit rapf copy */ + __asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu"); + } +#else + /* 64 bit copy loop */ + if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) { + while (n >= 8) { + __asm_copy_to_user_8x64(dst, src, retn); + n -= 8; + } + } +#endif + + while (n >= 16) { + __asm_copy_to_user_16(dst, src, retn); + n -= 16; + } + + while (n >= 4) { + __asm_copy_to_user_4(dst, src, retn); + n -= 4; + } + + switch (n) { + case 0: + break; + case 1: + __asm_copy_to_user_1(dst, src, retn); + break; + case 2: + __asm_copy_to_user_2(dst, src, retn); + break; + case 3: + __asm_copy_to_user_3(dst, src, retn); + break; + } + + return retn; +} + +#define __asm_copy_from_user_1(to, from, ret) \ + __asm_copy_user_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "2: SETB [%0++],D1Ar1\n", \ + "3: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 2b,3b\n") + +#define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_user_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + "2: SETW [%0++],D1Ar1\n" COPY, \ + "3: ADD %2,%2,#2\n" \ + " SETW [%0++],D1Ar1\n" FIXUP, \ + " .long 2b,3b\n" TENTRY) + +#define __asm_copy_from_user_2(to, from, ret) \ + __asm_copy_from_user_2x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_3(to, from, ret) \ + __asm_copy_from_user_2x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "4: SETB [%0++],D1Ar1\n", \ + "5: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 4b,5b\n") + +#define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_user_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ + "2: SETD [%0++],D1Ar1\n" COPY, \ + "3: ADD %2,%2,#4\n" \ + " SETD [%0++],D1Ar1\n" FIXUP, \ + " .long 2b,3b\n" TENTRY) + +#define __asm_copy_from_user_4(to, from, ret) \ + __asm_copy_from_user_4x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_5(to, from, ret) \ + __asm_copy_from_user_4x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "4: SETB [%0++],D1Ar1\n", \ + "5: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 4b,5b\n") + +#define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_from_user_4x_cont(to, from, ret, \ + " GETW D1Ar1,[%1++]\n" \ + "4: SETW [%0++],D1Ar1\n" COPY, \ + "5: ADD %2,%2,#2\n" \ + " SETW [%0++],D1Ar1\n" FIXUP, \ + " .long 4b,5b\n" TENTRY) + +#define __asm_copy_from_user_6(to, from, ret) \ + __asm_copy_from_user_6x_cont(to, from, ret, "", "", "") + +#define __asm_copy_from_user_7(to, from, ret) \ + __asm_copy_from_user_6x_cont(to, from, ret, \ + " GETB D1Ar1,[%1++]\n" \ + "6: SETB [%0++],D1Ar1\n", \ + "7: ADD %2,%2,#1\n" \ + " SETB [%0++],D1Ar1\n", \ + " .long 6b,7b\n") + +#define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ + __asm_copy_from_user_4x_cont(to, from, ret, \ + " GETD D1Ar1,[%1++]\n" \ |