Commit 8453eab9 authored by Harry Fearnhamm's avatar Harry Fearnhamm Committed by Catalin Marinas

Spinlocks using LDREX and STREX instructions can livelock

According to the ARM11MPCore Erratum 351422 (r0p0), under extremely
rare conditions, in an MPCore node consisting of at least 3 CPUs, two
CPUs trying to perform a STREX to data on the same shared cache line
can enter a livelock situation. This patch adds variable spinning time
to the locking routines.
Signed-off-by: default avatarHarry Fearnhamm <Harry.Fearnhamm@arm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>


parent 7b182a2d
...@@ -537,6 +537,18 @@ config ARM_ERRATA_411920 ...@@ -537,6 +537,18 @@ config ARM_ERRATA_411920
does not affect the MPCore. This option enables the ARM Ltd. does not affect the MPCore. This option enables the ARM Ltd.
recommended workaround. recommended workaround.
config ARM_ERRATA_351422
bool "Spinlocks using LDREX and STREX instructions can livelock"
depends on CPU_V6 && SMP
default n
help
According to the ARM11MPCore Erratum 351422 (r0p0), under
extremely rare conditions, in an MPCore node consisting of
at least 3 CPUs, two CPUs trying to perform a STREX to data
on the same shared cache line can enter a livelock
situation. This option adds variable spinning time to the
locking routines.
endmenu endmenu
source "arch/arm/common/Kconfig" source "arch/arm/common/Kconfig"
......
...@@ -5,6 +5,13 @@ ...@@ -5,6 +5,13 @@
and r3, r0, #7 @ Get bit offset and r3, r0, #7 @ Get bit offset
add r1, r1, r0, lsr #3 @ Get byte offset add r1, r1, r0, lsr #3 @ Get byte offset
mov r3, r2, lsl r3 mov r3, r2, lsl r3
#ifdef CONFIG_ARM_ERRATA_351422
mrc p15, 0, r0, c0, c0, 5
and r0, r0, #0xf
mov r0, r0, lsl #8
3: subs r0, r0, #1
bpl 3b
#endif
1: ldrexb r2, [r1] 1: ldrexb r2, [r1]
\instr r2, r2, r3 \instr r2, r2, r3
strexb r0, r2, [r1] strexb r0, r2, [r1]
...@@ -18,6 +25,13 @@ ...@@ -18,6 +25,13 @@
mov r2, #1 mov r2, #1
add r1, r1, r0, lsr #3 @ Get byte offset add r1, r1, r0, lsr #3 @ Get byte offset
mov r3, r2, lsl r3 @ create mask mov r3, r2, lsl r3 @ create mask
#ifdef CONFIG_ARM_ERRATA_351422
mrc p15, 0, r0, c0, c0, 5
and r0, r0, #0xf
mov r0, r0, lsl #8
3: subs r0, r0, #1
bpl 3b
#endif
1: ldrexb r2, [r1] 1: ldrexb r2, [r1]
ands r0, r2, r3 @ save old value of bit ands r0, r2, r3 @ save old value of bit
.ifnc \cond,al .ifnc \cond,al
......
...@@ -25,6 +25,26 @@ typedef struct { volatile int counter; } atomic_t; ...@@ -25,6 +25,26 @@ typedef struct { volatile int counter; } atomic_t;
#if __LINUX_ARM_ARCH__ >= 6 #if __LINUX_ARM_ARCH__ >= 6
#ifdef CONFIG_ARM_ERRATA_351422
static inline int atomic_backoff_delay(void)
{
unsigned int delay;
__asm__ __volatile__(
" mrc p15, 0, %0, c0, c0, 5\n"
" and %0, %0, #0xf\n"
" mov %0, %0, lsl #8\n"
"1: subs %0, %0, #1\n"
" bpl 1b\n"
: "=&r" (delay)
:
: "cc" );
return 1;
}
#else
#define atomic_backoff_delay() 1
#endif
/* /*
* ARMv6 UP and SMP safe atomic ops. We use load exclusive and * ARMv6 UP and SMP safe atomic ops. We use load exclusive and
* store exclusive to ensure that these are atomic. We may loop * store exclusive to ensure that these are atomic. We may loop
...@@ -32,20 +52,36 @@ typedef struct { volatile int counter; } atomic_t; ...@@ -32,20 +52,36 @@ typedef struct { volatile int counter; } atomic_t;
* without using the following operations WILL break the atomic * without using the following operations WILL break the atomic
* nature of these ops. * nature of these ops.
*/ */
#if 0 /* atomic set implemented as a simple STR by a previous patch */
static inline void atomic_set(atomic_t *v, int i)
{
unsigned long tmp;
do {
__asm__ __volatile__("@ atomic_set\n"
"1: ldrex %0, [%1]\n"
" strex %0, %2, [%1]\n"
: "=&r" (tmp)
: "r" (&v->counter), "r" (i)
: "cc");
} while (tmp && atomic_backoff_delay());
}
#endif
static inline int atomic_add_return(int i, atomic_t *v) static inline int atomic_add_return(int i, atomic_t *v)
{ {
unsigned long tmp; unsigned long tmp;
int result; int result;
do {
__asm__ __volatile__("@ atomic_add_return\n" __asm__ __volatile__("@ atomic_add_return\n"
"1: ldrex %0, [%2]\n" "1: ldrex %0, [%2]\n"
" add %0, %0, %3\n" " add %0, %0, %3\n"
" strex %1, %0, [%2]\n" " strex %1, %0, [%2]\n"
" teq %1, #0\n"
" bne 1b"
: "=&r" (result), "=&r" (tmp) : "=&r" (result), "=&r" (tmp)
: "r" (&v->counter), "Ir" (i) : "r" (&v->counter), "Ir" (i)
: "cc"); : "cc");
} while (tmp && atomic_backoff_delay());
return result; return result;
} }
...@@ -55,15 +91,15 @@ static inline int atomic_sub_return(int i, atomic_t *v) ...@@ -55,15 +91,15 @@ static inline int atomic_sub_return(int i, atomic_t *v)
unsigned long tmp; unsigned long tmp;
int result; int result;
do {
__asm__ __volatile__("@ atomic_sub_return\n" __asm__ __volatile__("@ atomic_sub_return\n"
"1: ldrex %0, [%2]\n" "1: ldrex %0, [%2]\n"
" sub %0, %0, %3\n" " sub %0, %0, %3\n"
" strex %1, %0, [%2]\n" " strex %1, %0, [%2]\n"
" teq %1, #0\n"
" bne 1b"
: "=&r" (result), "=&r" (tmp) : "=&r" (result), "=&r" (tmp)
: "r" (&v->counter), "Ir" (i) : "r" (&v->counter), "Ir" (i)
: "cc"); : "cc");
} while (tmp && atomic_backoff_delay());
return result; return result;
} }
...@@ -82,7 +118,7 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new) ...@@ -82,7 +118,7 @@ static inline int atomic_cmpxchg(atomic_t *ptr, int old, int new)
: "=&r" (res), "=&r" (oldval) : "=&r" (res), "=&r" (oldval)
: "r" (&ptr->counter), "Ir" (old), "r" (new) : "r" (&ptr->counter), "Ir" (old), "r" (new)
: "cc"); : "cc");
} while (res); } while (res && atomic_backoff_delay());
return oldval; return oldval;
} }
...@@ -91,15 +127,15 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr) ...@@ -91,15 +127,15 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
{ {
unsigned long tmp, tmp2; unsigned long tmp, tmp2;
do {
__asm__ __volatile__("@ atomic_clear_mask\n" __asm__ __volatile__("@ atomic_clear_mask\n"
"1: ldrex %0, [%2]\n" "1: ldrex %0, [%2]\n"
" bic %0, %0, %3\n" " bic %0, %0, %3\n"
" strex %1, %0, [%2]\n" " strex %1, %0, [%2]\n"
" teq %1, #0\n"
" bne 1b"
: "=&r" (tmp), "=&r" (tmp2) : "=&r" (tmp), "=&r" (tmp2)
: "r" (addr), "Ir" (mask) : "r" (addr), "Ir" (mask)
: "cc"); : "cc");
} while (tmp && atomic_backoff_delay());
} }
#else /* ARM_ARCH_6 */ #else /* ARM_ARCH_6 */
......
...@@ -23,12 +23,32 @@ ...@@ -23,12 +23,32 @@
#define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock) #define __raw_spin_lock_flags(lock, flags) __raw_spin_lock(lock)
#ifdef CONFIG_ARM_ERRATA_351422
#define spinlock_backoff_delay() \
{ \
unsigned int delay; \
__asm__ __volatile__( \
"1: mrc p15, 0, %0, c0, c0, 5\n" \
" and %0, %0, #0xf\n" \
" mov %0, %0, lsl #8\n" \
"2: subs %0, %0, #1\n" \
" bpl 2b\n" \
: "=&r" (delay) \
: \
: "cc" ); \
}
#else
#define spinlock_backoff_delay() \
__asm__ __volatile__("1: \n");
#endif
static inline void __raw_spin_lock(raw_spinlock_t *lock) static inline void __raw_spin_lock(raw_spinlock_t *lock)
{ {
unsigned long tmp; unsigned long tmp;
spinlock_backoff_delay();
__asm__ __volatile__( __asm__ __volatile__(
"1: ldrex %0, [%1]\n" " ldrex %0, [%1]\n"
" teq %0, #0\n" " teq %0, #0\n"
#ifdef CONFIG_CPU_32v6K #ifdef CONFIG_CPU_32v6K
" itee ne\n" " itee ne\n"
...@@ -50,6 +70,7 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) ...@@ -50,6 +70,7 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock)
{ {
unsigned long tmp; unsigned long tmp;
spinlock_backoff_delay();
__asm__ __volatile__( __asm__ __volatile__(
" ldrex %0, [%1]\n" " ldrex %0, [%1]\n"
" teq %0, #0\n" " teq %0, #0\n"
...@@ -94,8 +115,9 @@ static inline void __raw_write_lock(raw_rwlock_t *rw) ...@@ -94,8 +115,9 @@ static inline void __raw_write_lock(raw_rwlock_t *rw)
{ {
unsigned long tmp; unsigned long tmp;
spinlock_backoff_delay();
__asm__ __volatile__( __asm__ __volatile__(
"1: ldrex %0, [%1]\n" " ldrex %0, [%1]\n"
" teq %0, #0\n" " teq %0, #0\n"
#ifdef CONFIG_CPU_32v6K #ifdef CONFIG_CPU_32v6K
" itee ne\n" " itee ne\n"
...@@ -117,6 +139,7 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw) ...@@ -117,6 +139,7 @@ static inline int __raw_write_trylock(raw_rwlock_t *rw)
{ {
unsigned long tmp; unsigned long tmp;
spinlock_backoff_delay();
__asm__ __volatile__( __asm__ __volatile__(
"1: ldrex %0, [%1]\n" "1: ldrex %0, [%1]\n"
" teq %0, #0\n" " teq %0, #0\n"
...@@ -168,8 +191,9 @@ static inline void __raw_read_lock(raw_rwlock_t *rw) ...@@ -168,8 +191,9 @@ static inline void __raw_read_lock(raw_rwlock_t *rw)
{ {
unsigned long tmp, tmp2; unsigned long tmp, tmp2;
spinlock_backoff_delay();
__asm__ __volatile__( __asm__ __volatile__(
"1: ldrex %0, [%2]\n" " ldrex %0, [%2]\n"
" adds %0, %0, #1\n" " adds %0, %0, #1\n"
#ifdef CONFIG_CPU_32v6K #ifdef CONFIG_CPU_32v6K
" itet pl\n" " itet pl\n"
...@@ -195,8 +219,9 @@ static inline void __raw_read_unlock(raw_rwlock_t *rw) ...@@ -195,8 +219,9 @@ static inline void __raw_read_unlock(raw_rwlock_t *rw)
smp_mb(); smp_mb();
spinlock_backoff_delay();
__asm__ __volatile__( __asm__ __volatile__(
"1: ldrex %0, [%2]\n" " ldrex %0, [%2]\n"
" sub %0, %0, #1\n" " sub %0, %0, #1\n"
" strex %1, %0, [%2]\n" " strex %1, %0, [%2]\n"
" teq %1, #0\n" " teq %1, #0\n"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment