diff options
Diffstat (limited to 'arch/mips/include/asm/cmpxchg.h')
| -rw-r--r-- | arch/mips/include/asm/cmpxchg.h | 139 |
1 files changed, 132 insertions, 7 deletions
diff --git a/arch/mips/include/asm/cmpxchg.h b/arch/mips/include/asm/cmpxchg.h index d8d1c2805ac..eefcaa363a8 100644 --- a/arch/mips/include/asm/cmpxchg.h +++ b/arch/mips/include/asm/cmpxchg.h @@ -8,7 +8,132 @@ #ifndef __ASM_CMPXCHG_H #define __ASM_CMPXCHG_H +#include <linux/bug.h> #include <linux/irqflags.h> +#include <asm/war.h> + +static inline unsigned long __xchg_u32(volatile int * m, unsigned int val) +{ + __u32 retval; + + smp_mb__before_llsc(); + + if (kernel_uses_llsc && R10000_LLSC_WAR) { + unsigned long dummy; + + __asm__ __volatile__( + " .set arch=r4000 \n" + "1: ll %0, %3 # xchg_u32 \n" + " .set mips0 \n" + " move %2, %z4 \n" + " .set arch=r4000 \n" + " sc %2, %1 \n" + " beqzl %2, 1b \n" + " .set mips0 \n" + : "=&r" (retval), "=m" (*m), "=&r" (dummy) + : "R" (*m), "Jr" (val) + : "memory"); + } else if (kernel_uses_llsc) { + unsigned long dummy; + + do { + __asm__ __volatile__( + " .set arch=r4000 \n" + " ll %0, %3 # xchg_u32 \n" + " .set mips0 \n" + " move %2, %z4 \n" + " .set arch=r4000 \n" + " sc %2, %1 \n" + " .set mips0 \n" + : "=&r" (retval), "=m" (*m), "=&r" (dummy) + : "R" (*m), "Jr" (val) + : "memory"); + } while (unlikely(!dummy)); + } else { + unsigned long flags; + + raw_local_irq_save(flags); + retval = *m; + *m = val; + raw_local_irq_restore(flags); /* implies memory barrier */ + } + + smp_llsc_mb(); + + return retval; +} + +#ifdef CONFIG_64BIT +static inline __u64 __xchg_u64(volatile __u64 * m, __u64 val) +{ + __u64 retval; + + smp_mb__before_llsc(); + + if (kernel_uses_llsc && R10000_LLSC_WAR) { + unsigned long dummy; + + __asm__ __volatile__( + " .set arch=r4000 \n" + "1: lld %0, %3 # xchg_u64 \n" + " move %2, %z4 \n" + " scd %2, %1 \n" + " beqzl %2, 1b \n" + " .set mips0 \n" + : "=&r" (retval), "=m" (*m), "=&r" (dummy) + : "R" (*m), "Jr" (val) + : "memory"); + } else if (kernel_uses_llsc) { + unsigned long dummy; + + do { + __asm__ __volatile__( + " .set arch=r4000 \n" + " lld %0, %3 # xchg_u64 \n" + " move %2, %z4 \n" + " scd %2, %1 \n" + " .set mips0 \n" + : "=&r" (retval), "=m" (*m), "=&r" (dummy) + : "R" (*m), "Jr" (val) + : "memory"); + } while (unlikely(!dummy)); + } else { + unsigned long flags; + + raw_local_irq_save(flags); + retval = *m; + *m = val; + raw_local_irq_restore(flags); /* implies memory barrier */ + } + + smp_llsc_mb(); + + return retval; +} +#else +extern __u64 __xchg_u64_unsupported_on_32bit_kernels(volatile __u64 * m, __u64 val); +#define __xchg_u64 __xchg_u64_unsupported_on_32bit_kernels +#endif + +static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) +{ + switch (size) { + case 4: + return __xchg_u32(ptr, x); + case 8: + return __xchg_u64(ptr, x); + } + + return x; +} + +#define xchg(ptr, x) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) & ~0xc); \ + \ + ((__typeof__(*(ptr))) \ + __xchg((unsigned long)(x), (ptr), sizeof(*(ptr)))); \ +}) #define __HAVE_ARCH_CMPXCHG 1 @@ -20,12 +145,12 @@ __asm__ __volatile__( \ " .set push \n" \ " .set noat \n" \ - " .set mips3 \n" \ - "1: " ld " %0, %2 # __cmpxchg_asm \n" \ + " .set arch=r4000 \n" \ + "1: " ld " %0, %2 # __cmpxchg_asm \n" \ " bne %0, %z3, 2f \n" \ " .set mips0 \n" \ " move $1, %z4 \n" \ - " .set mips3 \n" \ + " .set arch=r4000 \n" \ " " st " $1, %1 \n" \ " beqzl $1, 1b \n" \ "2: \n" \ @@ -37,12 +162,12 @@ __asm__ __volatile__( \ " .set push \n" \ " .set noat \n" \ - " .set mips3 \n" \ - "1: " ld " %0, %2 # __cmpxchg_asm \n" \ + " .set arch=r4000 \n" \ + "1: " ld " %0, %2 # __cmpxchg_asm \n" \ " bne %0, %z3, 2f \n" \ " .set mips0 \n" \ " move $1, %z4 \n" \ - " .set mips3 \n" \ + " .set arch=r4000 \n" \ " " st " $1, %1 \n" \ " beqz $1, 1b \n" \ " .set pop \n" \ @@ -80,7 +205,7 @@ extern void __cmpxchg_called_with_bad_pointer(void); \ switch (sizeof(*(__ptr))) { \ case 4: \ - __res = __cmpxchg_asm("ll", "sc", __ptr, __old, __new); \ + __res = __cmpxchg_asm("ll", "sc", __ptr, __old, __new); \ break; \ case 8: \ if (sizeof(long) == 8) { \ |