diff options
| -rw-r--r-- | arch/alpha/include/asm/atomic.h | 68 | ||||
| -rw-r--r-- | arch/alpha/include/asm/cmpxchg.h | 71 | ||||
| -rw-r--r-- | arch/alpha/include/asm/xchg.h | 4 | ||||
| -rw-r--r-- | arch/ia64/include/asm/cmpxchg.h | 148 | ||||
| -rw-r--r-- | arch/ia64/include/asm/intrinsics.h | 114 | ||||
| -rw-r--r-- | kernel/irq_work.c | 1 |
6 files changed, 223 insertions, 183 deletions
diff --git a/arch/alpha/include/asm/atomic.h b/arch/alpha/include/asm/atomic.h index f62251e82ff..3bb7ffeae3b 100644 --- a/arch/alpha/include/asm/atomic.h +++ b/arch/alpha/include/asm/atomic.h @@ -3,6 +3,7 @@ #include <linux/types.h> #include <asm/barrier.h> +#include <asm/cmpxchg.h> /* * Atomic operations that C can't guarantee us. Useful for @@ -168,73 +169,6 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t * v) return result; } -/* - * Atomic exchange routines. - */ - -#define __ASM__MB -#define ____xchg(type, args...) __xchg ## type ## _local(args) -#define ____cmpxchg(type, args...) __cmpxchg ## type ## _local(args) -#include <asm/xchg.h> - -#define xchg_local(ptr,x) \ - ({ \ - __typeof__(*(ptr)) _x_ = (x); \ - (__typeof__(*(ptr))) __xchg_local((ptr), (unsigned long)_x_, \ - sizeof(*(ptr))); \ - }) - -#define cmpxchg_local(ptr, o, n) \ - ({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \ - (unsigned long)_n_, \ - sizeof(*(ptr))); \ - }) - -#define cmpxchg64_local(ptr, o, n) \ - ({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg_local((ptr), (o), (n)); \ - }) - -#ifdef CONFIG_SMP -#undef __ASM__MB -#define __ASM__MB "\tmb\n" -#endif -#undef ____xchg -#undef ____cmpxchg -#define ____xchg(type, args...) __xchg ##type(args) -#define ____cmpxchg(type, args...) __cmpxchg ##type(args) -#include <asm/xchg.h> - -#define xchg(ptr,x) \ - ({ \ - __typeof__(*(ptr)) _x_ = (x); \ - (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, \ - sizeof(*(ptr))); \ - }) - -#define cmpxchg(ptr, o, n) \ - ({ \ - __typeof__(*(ptr)) _o_ = (o); \ - __typeof__(*(ptr)) _n_ = (n); \ - (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ - (unsigned long)_n_, sizeof(*(ptr)));\ - }) - -#define cmpxchg64(ptr, o, n) \ - ({ \ - BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ - cmpxchg((ptr), (o), (n)); \ - }) - -#undef __ASM__MB -#undef ____cmpxchg - -#define __HAVE_ARCH_CMPXCHG 1 - #define atomic64_cmpxchg(v, old, new) (cmpxchg(&((v)->counter), old, new)) #define atomic64_xchg(v, new) (xchg(&((v)->counter), new)) diff --git a/arch/alpha/include/asm/cmpxchg.h b/arch/alpha/include/asm/cmpxchg.h new file mode 100644 index 00000000000..429e8cd0d78 --- /dev/null +++ b/arch/alpha/include/asm/cmpxchg.h @@ -0,0 +1,71 @@ +#ifndef _ALPHA_CMPXCHG_H +#define _ALPHA_CMPXCHG_H + +/* + * Atomic exchange routines. + */ + +#define __ASM__MB +#define ____xchg(type, args...) __xchg ## type ## _local(args) +#define ____cmpxchg(type, args...) __cmpxchg ## type ## _local(args) +#include <asm/xchg.h> + +#define xchg_local(ptr, x) \ +({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __xchg_local((ptr), (unsigned long)_x_, \ + sizeof(*(ptr))); \ +}) + +#define cmpxchg_local(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \ + (unsigned long)_n_, \ + sizeof(*(ptr))); \ +}) + +#define cmpxchg64_local(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + cmpxchg_local((ptr), (o), (n)); \ +}) + +#ifdef CONFIG_SMP +#undef __ASM__MB +#define __ASM__MB "\tmb\n" +#endif +#undef ____xchg +#undef ____cmpxchg +#define ____xchg(type, args...) __xchg ##type(args) +#define ____cmpxchg(type, args...) __cmpxchg ##type(args) +#include <asm/xchg.h> + +#define xchg(ptr, x) \ +({ \ + __typeof__(*(ptr)) _x_ = (x); \ + (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, \ + sizeof(*(ptr))); \ +}) + +#define cmpxchg(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ + (unsigned long)_n_, sizeof(*(ptr)));\ +}) + +#define cmpxchg64(ptr, o, n) \ +({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + cmpxchg((ptr), (o), (n)); \ +}) + +#undef __ASM__MB +#undef ____cmpxchg + +#define __HAVE_ARCH_CMPXCHG 1 + +#endif /* _ALPHA_CMPXCHG_H */ diff --git a/arch/alpha/include/asm/xchg.h b/arch/alpha/include/asm/xchg.h index 1d1b436fbff..0ca9724597c 100644 --- a/arch/alpha/include/asm/xchg.h +++ b/arch/alpha/include/asm/xchg.h @@ -1,10 +1,10 @@ -#ifndef _ALPHA_ATOMIC_H +#ifndef _ALPHA_CMPXCHG_H #error Do not include xchg.h directly! #else /* * xchg/xchg_local and cmpxchg/cmpxchg_local share the same code * except that local version do not have the expensive memory barrier. - * So this file is included twice from asm/system.h. + * So this file is included twice from asm/cmpxchg.h. */ /* diff --git a/arch/ia64/include/asm/cmpxchg.h b/arch/ia64/include/asm/cmpxchg.h index 4c96187e204..4f37dbbb864 100644 --- a/arch/ia64/include/asm/cmpxchg.h +++ b/arch/ia64/include/asm/cmpxchg.h @@ -1 +1,147 @@ -#include <asm/intrinsics.h> +#ifndef _ASM_IA64_CMPXCHG_H +#define _ASM_IA64_CMPXCHG_H + +/* + * Compare/Exchange, forked from asm/intrinsics.h + * which was: + * + * Copyright (C) 2002-2003 Hewlett-Packard Co + * David Mosberger-Tang <davidm@hpl.hp.com> + */ + +#ifndef __ASSEMBLY__ + +#include <linux/types.h> +/* include compiler specific intrinsics */ +#include <asm/ia64regs.h> +#ifdef __INTEL_COMPILER +# include <asm/intel_intrin.h> +#else +# include <asm/gcc_intrin.h> +#endif + +/* + * This function doesn't exist, so you'll get a linker error if + * something tries to do an invalid xchg(). + */ +extern void ia64_xchg_called_with_bad_pointer(void); + +#define __xchg(x, ptr, size) \ +({ \ + unsigned long __xchg_result; \ + \ + switch (size) { \ + case 1: \ + __xchg_result = ia64_xchg1((__u8 *)ptr, x); \ + break; \ + \ + case 2: \ + __xchg_result = ia64_xchg2((__u16 *)ptr, x); \ + break; \ + \ + case 4: \ + __xchg_result = ia64_xchg4((__u32 *)ptr, x); \ + break; \ + \ + case 8: \ + __xchg_result = ia64_xchg8((__u64 *)ptr, x); \ + break; \ + default: \ + ia64_xchg_called_with_bad_pointer(); \ + } \ + __xchg_result; \ +}) + +#define xchg(ptr, x) \ +((__typeof__(*(ptr))) __xchg((unsigned long) (x), (ptr), sizeof(*(ptr)))) + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + */ + +#define __HAVE_ARCH_CMPXCHG 1 + +/* + * This function doesn't exist, so you'll get a linker error + * if something tries to do an invalid cmpxchg(). + */ +extern long ia64_cmpxchg_called_with_bad_pointer(void); + +#define ia64_cmpxchg(sem, ptr, old, new, size) \ +({ \ + __u64 _o_, _r_; \ + \ + switch (size) { \ + case 1: \ + _o_ = (__u8) (long) (old); \ + break; \ + case 2: \ + _o_ = (__u16) (long) (old); \ + break; \ + case 4: \ + _o_ = (__u32) (long) (old); \ + break; \ + case 8: \ + _o_ = (__u64) (long) (old); \ + break; \ + default: \ + break; \ + } \ + switch (size) { \ + case 1: \ + _r_ = ia64_cmpxchg1_##sem((__u8 *) ptr, new, _o_); \ + break; \ + \ + case 2: \ + _r_ = ia64_cmpxchg2_##sem((__u16 *) ptr, new, _o_); \ + break; \ + \ + case 4: \ + _r_ = ia64_cmpxchg4_##sem((__u32 *) ptr, new, _o_); \ + break; \ + \ + case 8: \ + _r_ = ia64_cmpxchg8_##sem((__u64 *) ptr, new, _o_); \ + break; \ + \ + default: \ + _r_ = ia64_cmpxchg_called_with_bad_pointer(); \ + break; \ + } \ + (__typeof__(old)) _r_; \ +}) + +#define cmpxchg_acq(ptr, o, n) \ + ia64_cmpxchg(acq, (ptr), (o), (n), sizeof(*(ptr))) +#define cmpxchg_rel(ptr, o, n) \ + ia64_cmpxchg(rel, (ptr), (o), (n), sizeof(*(ptr))) + +/* for compatibility with other platforms: */ +#define cmpxchg(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) +#define cmpxchg64(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) + +#define cmpxchg_local cmpxchg +#define cmpxchg64_local cmpxchg64 + +#ifdef CONFIG_IA64_DEBUG_CMPXCHG +# define CMPXCHG_BUGCHECK_DECL int _cmpxchg_bugcheck_count = 128; +# define CMPXCHG_BUGCHECK(v) \ +do { \ + if (_cmpxchg_bugcheck_count-- <= 0) { \ + void *ip; \ + extern int printk(const char *fmt, ...); \ + ip = (void *) ia64_getreg(_IA64_REG_IP); \ + printk("CMPXCHG_BUGCHECK: stuck at %p on word %p\n", ip, (v));\ + break; \ + } \ +} while (0) +#else /* !CONFIG_IA64_DEBUG_CMPXCHG */ +# define CMPXCHG_BUGCHECK_DECL +# define CMPXCHG_BUGCHECK(v) +#endif /* !CONFIG_IA64_DEBUG_CMPXCHG */ + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_IA64_CMPXCHG_H */ diff --git a/arch/ia64/include/asm/intrinsics.h b/arch/ia64/include/asm/intrinsics.h index e4076b51182..d129e367e76 100644 --- a/arch/ia64/include/asm/intrinsics.h +++ b/arch/ia64/include/asm/intrinsics.h @@ -18,6 +18,7 @@ #else # include <asm/gcc_intrin.h> #endif +#include <asm/cmpxchg.h> #define ia64_native_get_psr_i() (ia64_native_getreg(_IA64_REG_PSR) & IA64_PSR_I) @@ -81,119 +82,6 @@ extern unsigned long __bad_increment_for_ia64_fetch_and_add (void); #define ia64_fetch_and_add(i,v) (ia64_fetchadd(i, v, rel) + (i)) /* return new value */ -/* - * This function doesn't exist, so you'll get a linker error if - * something tries to do an invalid xchg(). - */ -extern void ia64_xchg_called_with_bad_pointer (void); - -#define __xchg(x,ptr,size) \ -({ \ - unsigned long __xchg_result; \ - \ - switch (size) { \ - case 1: \ - __xchg_result = ia64_xchg1((__u8 *)ptr, x); \ - break; \ - \ - case 2: \ - __xchg_result = ia64_xchg2((__u16 *)ptr, x); \ - break; \ - \ - case 4: \ - __xchg_result = ia64_xchg4((__u32 *)ptr, x); \ - break; \ - \ - case 8: \ - __xchg_result = ia64_xchg8((__u64 *)ptr, x); \ - break; \ - default: \ - ia64_xchg_called_with_bad_pointer(); \ - } \ - __xchg_result; \ -}) - -#define xchg(ptr,x) \ - ((__typeof__(*(ptr))) __xchg ((unsigned long) (x), (ptr), sizeof(*(ptr)))) - -/* - * Atomic compare and exchange. Compare OLD with MEM, if identical, - * store NEW in MEM. Return the initial value in MEM. Success is - * indicated by comparing RETURN with OLD. - */ - -#define __HAVE_ARCH_CMPXCHG 1 - -/* - * This function doesn't exist, so you'll get a linker error - * if something tries to do an invalid cmpxchg(). - */ -extern long ia64_cmpxchg_called_with_bad_pointer (void); - -#define ia64_cmpxchg(sem,ptr,old,new,size) \ -({ \ - __u64 _o_, _r_; \ - \ - switch (size) { \ - case 1: _o_ = (__u8 ) (long) (old); break; \ - case 2: _o_ = (__u16) (long) (old); break; \ - case 4: _o_ = (__u32) (long) (old); break; \ - case 8: _o_ = (__u64) (long) (old); break; \ - default: break; \ - } \ - switch (size) { \ - case 1: \ - _r_ = ia64_cmpxchg1_##sem((__u8 *) ptr, new, _o_); \ - break; \ - \ - case 2: \ - _r_ = ia64_cmpxchg2_##sem((__u16 *) ptr, new, _o_); \ - break; \ - \ - case 4: \ - _r_ = ia64_cmpxchg4_##sem((__u32 *) ptr, new, _o_); \ - break; \ - \ - case 8: \ - _r_ = ia64_cmpxchg8_##sem((__u64 *) ptr, new, _o_); \ - break; \ - \ - default: \ - _r_ = ia64_cmpxchg_called_with_bad_pointer(); \ - break; \ - } \ - (__typeof__(old)) _r_; \ -}) - -#define cmpxchg_acq(ptr, o, n) \ - ia64_cmpxchg(acq, (ptr), (o), (n), sizeof(*(ptr))) -#define cmpxchg_rel(ptr, o, n) \ - ia64_cmpxchg(rel, (ptr), (o), (n), sizeof(*(ptr))) - -/* for compatibility with other platforms: */ -#define cmpxchg(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) -#define cmpxchg64(ptr, o, n) cmpxchg_acq((ptr), (o), (n)) - -#define cmpxchg_local cmpxchg -#define cmpxchg64_local cmpxchg64 - -#ifdef CONFIG_IA64_DEBUG_CMPXCHG -# define CMPXCHG_BUGCHECK_DECL int _cmpxchg_bugcheck_count = 128; -# define CMPXCHG_BUGCHECK(v) \ - do { \ - if (_cmpxchg_bugcheck_count-- <= 0) { \ - void *ip; \ - extern int printk(const char *fmt, ...); \ - ip = (void *) ia64_getreg(_IA64_REG_IP); \ - printk("CMPXCHG_BUGCHECK: stuck at %p on word %p\n", ip, (v)); \ - break; \ - } \ - } while (0) -#else /* !CONFIG_IA64_DEBUG_CMPXCHG */ -# define CMPXCHG_BUGCHECK_DECL -# define CMPXCHG_BUGCHECK(v) -#endif /* !CONFIG_IA64_DEBUG_CMPXCHG */ - #endif #ifdef __KERNEL__ diff --git a/kernel/irq_work.c b/kernel/irq_work.c index 0c56d44b9fd..1588e3b2871 100644 --- a/kernel/irq_work.c +++ b/kernel/irq_work.c @@ -11,6 +11,7 @@ #include <linux/irq_work.h> #include <linux/percpu.h> #include <linux/hardirq.h> +#include <linux/irqflags.h> #include <asm/processor.h> /* |