aboutsummaryrefslogtreecommitdiff
path: root/arch/mips/include/asm/spinlock.h
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/include/asm/spinlock.h')
-rw-r--r--arch/mips/include/asm/spinlock.h300
1 files changed, 118 insertions, 182 deletions
diff --git a/arch/mips/include/asm/spinlock.h b/arch/mips/include/asm/spinlock.h
index 5b60a09a0f0..78d201fb6c8 100644
--- a/arch/mips/include/asm/spinlock.h
+++ b/arch/mips/include/asm/spinlock.h
@@ -17,7 +17,7 @@
/*
* Your basic SMP spinlocks, allowing only a single CPU anywhere
*
- * Simple spin lock operations. There are two variants, one clears IRQ's
+ * Simple spin lock operations. There are two variants, one clears IRQ's
* on the local processor, one does not.
*
* These are fair FIFO ticket locks
@@ -34,55 +34,54 @@
* becomes equal to the the initial value of the tail.
*/
-static inline int __raw_spin_is_locked(raw_spinlock_t *lock)
+static inline int arch_spin_is_locked(arch_spinlock_t *lock)
{
- unsigned int counters = ACCESS_ONCE(lock->lock);
+ u32 counters = ACCESS_ONCE(lock->lock);
- return ((counters >> 14) ^ counters) & 0x1fff;
+ return ((counters >> 16) ^ counters) & 0xffff;
}
-#define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
-#define __raw_spin_unlock_wait(x) \
- while (__raw_spin_is_locked(x)) { cpu_relax(); }
+#define arch_spin_lock_flags(lock, flags) arch_spin_lock(lock)
+#define arch_spin_unlock_wait(x) \
+ while (arch_spin_is_locked(x)) { cpu_relax(); }
-static inline int __raw_spin_is_contended(raw_spinlock_t *lock)
+static inline int arch_spin_is_contended(arch_spinlock_t *lock)
{
- unsigned int counters = ACCESS_ONCE(lock->lock);
+ u32 counters = ACCESS_ONCE(lock->lock);
- return (((counters >> 14) - counters) & 0x1fff) > 1;
+ return (((counters >> 16) - counters) & 0xffff) > 1;
}
-#define __raw_spin_is_contended __raw_spin_is_contended
+#define arch_spin_is_contended arch_spin_is_contended
-static inline void __raw_spin_lock(raw_spinlock_t *lock)
+static inline void arch_spin_lock(arch_spinlock_t *lock)
{
int my_ticket;
int tmp;
+ int inc = 0x10000;
if (R10000_LLSC_WAR) {
__asm__ __volatile__ (
- " .set push # __raw_spin_lock \n"
+ " .set push # arch_spin_lock \n"
" .set noreorder \n"
" \n"
"1: ll %[ticket], %[ticket_ptr] \n"
- " addiu %[my_ticket], %[ticket], 0x4000 \n"
+ " addu %[my_ticket], %[ticket], %[inc] \n"
" sc %[my_ticket], %[ticket_ptr] \n"
" beqzl %[my_ticket], 1b \n"
" nop \n"
- " srl %[my_ticket], %[ticket], 14 \n"
- " andi %[my_ticket], %[my_ticket], 0x1fff \n"
- " andi %[ticket], %[ticket], 0x1fff \n"
+ " srl %[my_ticket], %[ticket], 16 \n"
+ " andi %[ticket], %[ticket], 0xffff \n"
" bne %[ticket], %[my_ticket], 4f \n"
" subu %[ticket], %[my_ticket], %[ticket] \n"
"2: \n"
" .subsection 2 \n"
- "4: andi %[ticket], %[ticket], 0x1fff \n"
+ "4: andi %[ticket], %[ticket], 0xffff \n"
" sll %[ticket], 5 \n"
" \n"
"6: bnez %[ticket], 6b \n"
" subu %[ticket], 1 \n"
" \n"
- " lw %[ticket], %[ticket_ptr] \n"
- " andi %[ticket], %[ticket], 0x1fff \n"
+ " lhu %[ticket], %[serving_now_ptr] \n"
" beq %[ticket], %[my_ticket], 2b \n"
" subu %[ticket], %[my_ticket], %[ticket] \n"
" b 4b \n"
@@ -90,36 +89,32 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock)
" .previous \n"
" .set pop \n"
: [ticket_ptr] "+m" (lock->lock),
+ [serving_now_ptr] "+m" (lock->h.serving_now),
[ticket] "=&r" (tmp),
- [my_ticket] "=&r" (my_ticket));
+ [my_ticket] "=&r" (my_ticket)
+ : [inc] "r" (inc));
} else {
__asm__ __volatile__ (
- " .set push # __raw_spin_lock \n"
+ " .set push # arch_spin_lock \n"
" .set noreorder \n"
" \n"
- " ll %[ticket], %[ticket_ptr] \n"
- "1: addiu %[my_ticket], %[ticket], 0x4000 \n"
+ "1: ll %[ticket], %[ticket_ptr] \n"
+ " addu %[my_ticket], %[ticket], %[inc] \n"
" sc %[my_ticket], %[ticket_ptr] \n"
- " beqz %[my_ticket], 3f \n"
- " nop \n"
- " srl %[my_ticket], %[ticket], 14 \n"
- " andi %[my_ticket], %[my_ticket], 0x1fff \n"
- " andi %[ticket], %[ticket], 0x1fff \n"
+ " beqz %[my_ticket], 1b \n"
+ " srl %[my_ticket], %[ticket], 16 \n"
+ " andi %[ticket], %[ticket], 0xffff \n"
" bne %[ticket], %[my_ticket], 4f \n"
" subu %[ticket], %[my_ticket], %[ticket] \n"
"2: \n"
" .subsection 2 \n"
- "3: b 1b \n"
- " ll %[ticket], %[ticket_ptr] \n"
- " \n"
"4: andi %[ticket], %[ticket], 0x1fff \n"
" sll %[ticket], 5 \n"
" \n"
"6: bnez %[ticket], 6b \n"
" subu %[ticket], 1 \n"
" \n"
- " lw %[ticket], %[ticket_ptr] \n"
- " andi %[ticket], %[ticket], 0x1fff \n"
+ " lhu %[ticket], %[serving_now_ptr] \n"
" beq %[ticket], %[my_ticket], 2b \n"
" subu %[ticket], %[my_ticket], %[ticket] \n"
" b 4b \n"
@@ -127,68 +122,38 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock)
" .previous \n"
" .set pop \n"
: [ticket_ptr] "+m" (lock->lock),
+ [serving_now_ptr] "+m" (lock->h.serving_now),
[ticket] "=&r" (tmp),
- [my_ticket] "=&r" (my_ticket));
+ [my_ticket] "=&r" (my_ticket)
+ : [inc] "r" (inc));
}
smp_llsc_mb();
}
-static inline void __raw_spin_unlock(raw_spinlock_t *lock)
+static inline void arch_spin_unlock(arch_spinlock_t *lock)
{
- int tmp;
-
- smp_llsc_mb();
-
- if (R10000_LLSC_WAR) {
- __asm__ __volatile__ (
- " # __raw_spin_unlock \n"
- "1: ll %[ticket], %[ticket_ptr] \n"
- " addiu %[ticket], %[ticket], 1 \n"
- " ori %[ticket], %[ticket], 0x2000 \n"
- " xori %[ticket], %[ticket], 0x2000 \n"
- " sc %[ticket], %[ticket_ptr] \n"
- " beqzl %[ticket], 1b \n"
- : [ticket_ptr] "+m" (lock->lock),
- [ticket] "=&r" (tmp));
- } else {
- __asm__ __volatile__ (
- " .set push # __raw_spin_unlock \n"
- " .set noreorder \n"
- " \n"
- " ll %[ticket], %[ticket_ptr] \n"
- "1: addiu %[ticket], %[ticket], 1 \n"
- " ori %[ticket], %[ticket], 0x2000 \n"
- " xori %[ticket], %[ticket], 0x2000 \n"
- " sc %[ticket], %[ticket_ptr] \n"
- " beqz %[ticket], 2f \n"
- " nop \n"
- " \n"
- " .subsection 2 \n"
- "2: b 1b \n"
- " ll %[ticket], %[ticket_ptr] \n"
- " .previous \n"
- " .set pop \n"
- : [ticket_ptr] "+m" (lock->lock),
- [ticket] "=&r" (tmp));
- }
+ unsigned int serving_now = lock->h.serving_now + 1;
+ wmb();
+ lock->h.serving_now = (u16)serving_now;
+ nudge_writes();
}
-static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock)
+static inline unsigned int arch_spin_trylock(arch_spinlock_t *lock)
{
int tmp, tmp2, tmp3;
+ int inc = 0x10000;
if (R10000_LLSC_WAR) {
__asm__ __volatile__ (
- " .set push # __raw_spin_trylock \n"
+ " .set push # arch_spin_trylock \n"
" .set noreorder \n"
" \n"
"1: ll %[ticket], %[ticket_ptr] \n"
- " srl %[my_ticket], %[ticket], 14 \n"
- " andi %[my_ticket], %[my_ticket], 0x1fff \n"
- " andi %[now_serving], %[ticket], 0x1fff \n"
+ " srl %[my_ticket], %[ticket], 16 \n"
+ " andi %[now_serving], %[ticket], 0xffff \n"
" bne %[my_ticket], %[now_serving], 3f \n"
- " addiu %[ticket], %[ticket], 0x4000 \n"
+ " addu %[ticket], %[ticket], %[inc] \n"
" sc %[ticket], %[ticket_ptr] \n"
" beqzl %[ticket], 1b \n"
" li %[ticket], 1 \n"
@@ -201,33 +166,32 @@ static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock)
: [ticket_ptr] "+m" (lock->lock),
[ticket] "=&r" (tmp),
[my_ticket] "=&r" (tmp2),
- [now_serving] "=&r" (tmp3));
+ [now_serving] "=&r" (tmp3)
+ : [inc] "r" (inc));
} else {
__asm__ __volatile__ (
- " .set push # __raw_spin_trylock \n"
+ " .set push # arch_spin_trylock \n"
" .set noreorder \n"
" \n"
- " ll %[ticket], %[ticket_ptr] \n"
- "1: srl %[my_ticket], %[ticket], 14 \n"
- " andi %[my_ticket], %[my_ticket], 0x1fff \n"
- " andi %[now_serving], %[ticket], 0x1fff \n"
+ "1: ll %[ticket], %[ticket_ptr] \n"
+ " srl %[my_ticket], %[ticket], 16 \n"
+ " andi %[now_serving], %[ticket], 0xffff \n"
" bne %[my_ticket], %[now_serving], 3f \n"
- " addiu %[ticket], %[ticket], 0x4000 \n"
+ " addu %[ticket], %[ticket], %[inc] \n"
" sc %[ticket], %[ticket_ptr] \n"
- " beqz %[ticket], 4f \n"
+ " beqz %[ticket], 1b \n"
" li %[ticket], 1 \n"
"2: \n"
" .subsection 2 \n"
"3: b 2b \n"
" li %[ticket], 0 \n"
- "4: b 1b \n"
- " ll %[ticket], %[ticket_ptr] \n"
" .previous \n"
" .set pop \n"
: [ticket_ptr] "+m" (lock->lock),
[ticket] "=&r" (tmp),
[my_ticket] "=&r" (tmp2),
- [now_serving] "=&r" (tmp3));
+ [now_serving] "=&r" (tmp3)
+ : [inc] "r" (inc));
}
smp_llsc_mb();
@@ -248,21 +212,21 @@ static inline unsigned int __raw_spin_trylock(raw_spinlock_t *lock)
* read_can_lock - would read_trylock() succeed?
* @lock: the rwlock in question.
*/
-#define __raw_read_can_lock(rw) ((rw)->lock >= 0)
+#define arch_read_can_lock(rw) ((rw)->lock >= 0)
/*
* write_can_lock - would write_trylock() succeed?
* @lock: the rwlock in question.
*/
-#define __raw_write_can_lock(rw) (!(rw)->lock)
+#define arch_write_can_lock(rw) (!(rw)->lock)
-static inline void __raw_read_lock(raw_rwlock_t *rw)
+static inline void arch_read_lock(arch_rwlock_t *rw)
{
unsigned int tmp;
if (R10000_LLSC_WAR) {
__asm__ __volatile__(
- " .set noreorder # __raw_read_lock \n"
+ " .set noreorder # arch_read_lock \n"
"1: ll %1, %2 \n"
" bltz %1, 1b \n"
" addu %1, 1 \n"
@@ -274,25 +238,16 @@ static inline void __raw_read_lock(raw_rwlock_t *rw)
: "m" (rw->lock)
: "memory");
} else {
- __asm__ __volatile__(
- " .set noreorder # __raw_read_lock \n"
- "1: ll %1, %2 \n"
- " bltz %1, 2f \n"
- " addu %1, 1 \n"
- " sc %1, %0 \n"
- " beqz %1, 1b \n"
- " nop \n"
- " .subsection 2 \n"
- "2: ll %1, %2 \n"
- " bltz %1, 2b \n"
- " addu %1, 1 \n"
- " b 1b \n"
- " nop \n"
- " .previous \n"
- " .set reorder \n"
- : "=m" (rw->lock), "=&r" (tmp)
- : "m" (rw->lock)
- : "memory");
+ do {
+ __asm__ __volatile__(
+ "1: ll %1, %2 # arch_read_lock \n"
+ " bltz %1, 1b \n"
+ " addu %1, 1 \n"
+ "2: sc %1, %0 \n"
+ : "=m" (rw->lock), "=&r" (tmp)
+ : "m" (rw->lock)
+ : "memory");
+ } while (unlikely(!tmp));
}
smp_llsc_mb();
@@ -301,15 +256,15 @@ static inline void __raw_read_lock(raw_rwlock_t *rw)
/* Note the use of sub, not subu which will make the kernel die with an
overflow exception if we ever try to unlock an rwlock that is already
unlocked or is being held by a writer. */
-static inline void __raw_read_unlock(raw_rwlock_t *rw)
+static inline void arch_read_unlock(arch_rwlock_t *rw)
{
unsigned int tmp;
- smp_llsc_mb();
+ smp_mb__before_llsc();
if (R10000_LLSC_WAR) {
__asm__ __volatile__(
- "1: ll %1, %2 # __raw_read_unlock \n"
+ "1: ll %1, %2 # arch_read_unlock \n"
" sub %1, 1 \n"
" sc %1, %0 \n"
" beqzl %1, 1b \n"
@@ -317,31 +272,25 @@ static inline void __raw_read_unlock(raw_rwlock_t *rw)
: "m" (rw->lock)
: "memory");
} else {
- __asm__ __volatile__(
- " .set noreorder # __raw_read_unlock \n"
- "1: ll %1, %2 \n"
- " sub %1, 1 \n"
- " sc %1, %0 \n"
- " beqz %1, 2f \n"
- " nop \n"
- " .subsection 2 \n"
- "2: b 1b \n"
- " nop \n"
- " .previous \n"
- " .set reorder \n"
- : "=m" (rw->lock), "=&r" (tmp)
- : "m" (rw->lock)
- : "memory");
+ do {
+ __asm__ __volatile__(
+ "1: ll %1, %2 # arch_read_unlock \n"
+ " sub %1, 1 \n"
+ " sc %1, %0 \n"
+ : "=m" (rw->lock), "=&r" (tmp)
+ : "m" (rw->lock)
+ : "memory");
+ } while (unlikely(!tmp));
}
}
-static inline void __raw_write_lock(raw_rwlock_t *rw)
+static inline void arch_write_lock(arch_rwlock_t *rw)
{
unsigned int tmp;
if (R10000_LLSC_WAR) {
__asm__ __volatile__(
- " .set noreorder # __raw_write_lock \n"
+ " .set noreorder # arch_write_lock \n"
"1: ll %1, %2 \n"
" bnez %1, 1b \n"
" lui %1, 0x8000 \n"
@@ -353,50 +302,41 @@ static inline void __raw_write_lock(raw_rwlock_t *rw)
: "m" (rw->lock)
: "memory");
} else {
- __asm__ __volatile__(
- " .set noreorder # __raw_write_lock \n"
- "1: ll %1, %2 \n"
- " bnez %1, 2f \n"
- " lui %1, 0x8000 \n"
- " sc %1, %0 \n"
- " beqz %1, 2f \n"
- " nop \n"
- " .subsection 2 \n"
- "2: ll %1, %2 \n"
- " bnez %1, 2b \n"
- " lui %1, 0x8000 \n"
- " b 1b \n"
- " nop \n"
- " .previous \n"
- " .set reorder \n"
- : "=m" (rw->lock), "=&r" (tmp)
- : "m" (rw->lock)
- : "memory");
+ do {
+ __asm__ __volatile__(
+ "1: ll %1, %2 # arch_write_lock \n"
+ " bnez %1, 1b \n"
+ " lui %1, 0x8000 \n"
+ "2: sc %1, %0 \n"
+ : "=m" (rw->lock), "=&r" (tmp)
+ : "m" (rw->lock)
+ : "memory");
+ } while (unlikely(!tmp));
}
smp_llsc_mb();
}
-static inline void __raw_write_unlock(raw_rwlock_t *rw)
+static inline void arch_write_unlock(arch_rwlock_t *rw)
{
smp_mb();
__asm__ __volatile__(
- " # __raw_write_unlock \n"
+ " # arch_write_unlock \n"
" sw $0, %0 \n"
: "=m" (rw->lock)
: "m" (rw->lock)
: "memory");
}
-static inline int __raw_read_trylock(raw_rwlock_t *rw)
+static inline int arch_read_trylock(arch_rwlock_t *rw)
{
unsigned int tmp;
int ret;
if (R10000_LLSC_WAR) {
__asm__ __volatile__(
- " .set noreorder # __raw_read_trylock \n"
+ " .set noreorder # arch_read_trylock \n"
" li %2, 0 \n"
"1: ll %1, %3 \n"
" bltz %1, 2f \n"
@@ -413,7 +353,7 @@ static inline int __raw_read_trylock(raw_rwlock_t *rw)
: "memory");
} else {
__asm__ __volatile__(
- " .set noreorder # __raw_read_trylock \n"
+ " .set noreorder # arch_read_trylock \n"
" li %2, 0 \n"
"1: ll %1, %3 \n"
" bltz %1, 2f \n"
@@ -433,14 +373,14 @@ static inline int __raw_read_trylock(raw_rwlock_t *rw)
return ret;
}
-static inline int __raw_write_trylock(raw_rwlock_t *rw)
+static inline int arch_write_trylock(arch_rwlock_t *rw)
{
unsigned int tmp;
int ret;
if (R10000_LLSC_WAR) {
__asm__ __volatile__(
- " .set noreorder # __raw_write_trylock \n"
+ " .set noreorder # arch_write_trylock \n"
" li %2, 0 \n"
"1: ll %1, %3 \n"
" bnez %1, 2f \n"
@@ -456,35 +396,31 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)
: "m" (rw->lock)
: "memory");
} else {
- __asm__ __volatile__(
- " .set noreorder # __raw_write_trylock \n"
- " li %2, 0 \n"
- "1: ll %1, %3 \n"
- " bnez %1, 2f \n"
- " lui %1, 0x8000 \n"
- " sc %1, %0 \n"
- " beqz %1, 3f \n"
- " li %2, 1 \n"
- "2: \n"
- __WEAK_LLSC_MB
- " .subsection 2 \n"
- "3: b 1b \n"
- " li %2, 0 \n"
- " .previous \n"
- " .set reorder \n"
- : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
- : "m" (rw->lock)
- : "memory");
+ do {
+ __asm__ __volatile__(
+ " ll %1, %3 # arch_write_trylock \n"
+ " li %2, 0 \n"
+ " bnez %1, 2f \n"
+ " lui %1, 0x8000 \n"
+ " sc %1, %0 \n"
+ " li %2, 1 \n"
+ "2: \n"
+ : "=m" (rw->lock), "=&r" (tmp), "=&r" (ret)
+ : "m" (rw->lock)
+ : "memory");
+ } while (unlikely(!tmp));
+
+ smp_llsc_mb();
}
return ret;
}
-#define __raw_read_lock_flags(lock, flags) __raw_read_lock(lock)
-#define __raw_write_lock_flags(lock, flags) __raw_write_lock(lock)
+#define arch_read_lock_flags(lock, flags) arch_read_lock(lock)
+#define arch_write_lock_flags(lock, flags) arch_write_lock(lock)
-#define _raw_spin_relax(lock) cpu_relax()
-#define _raw_read_relax(lock) cpu_relax()
-#define _raw_write_relax(lock) cpu_relax()
+#define arch_spin_relax(lock) cpu_relax()
+#define arch_read_relax(lock) cpu_relax()
+#define arch_write_relax(lock) cpu_relax()
#endif /* _ASM_SPINLOCK_H */
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2025-12-05 19:48:38 +0000