Skip to content

L
linux
Commit 5ef3fe4c authored by Will Deacon's avatar Will Deacon
Browse files
Options

arm64: Avoid redundant type conversions in xchg() and cmpxchg() 

Our atomic instructions (either LSE atomics of LDXR/STXR sequences)
natively support byte, half-word, word and double-word memory accesses
so there is no need to mask the data register prior to being stored.
Signed-off-by: default avatarWill Deacon <will.deacon@arm.com>
parent 39624469
Hide whitespace changes
Inline Side-by-side
Showing with 108 additions and 107 deletions
arch/arm64/include/asm/atomic_ll_sc.h
View file @ 5ef3fe4c
...@@ -248,48 +248,49 @@ __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v)) ...@@ -248,48 +248,49 @@ __LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
} }
__LL_SC_EXPORT(atomic64_dec_if_positive); __LL_SC_EXPORT(atomic64_dec_if_positive);
#define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl) \ #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl) \
__LL_SC_INLINE unsigned long \ __LL_SC_INLINE u##sz \
__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \ __LL_SC_PREFIX(__cmpxchg_case_##name##sz(volatile void *ptr, \
unsigned long old, \ unsigned long old, \
unsigned long new)) \ u##sz new)) \
{ \ { \
unsigned long tmp, oldval; \ unsigned long tmp; \
u##sz oldval; \
\ \
asm volatile( \ asm volatile( \
" prfm pstl1strm, %[v]\n" \ " prfm pstl1strm, %[v]\n" \
"1: ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n" \ "1: ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n" \
" eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \ " eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
" cbnz %" #w "[tmp], 2f\n" \ " cbnz %" #w "[tmp], 2f\n" \
" st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \ " st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n" \
" cbnz %w[tmp], 1b\n" \ " cbnz %w[tmp], 1b\n" \
" " #mb "\n" \ " " #mb "\n" \
"2:" \ "2:" \
: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \ : [tmp] "=&r" (tmp), [oldval] "=&r" (oldval), \
[v] "+Q" (*(unsigned long *)ptr) \ [v] "+Q" (*(u##sz *)ptr) \
: [old] "Lr" (old), [new] "r" (new) \ : [old] "Lr" (old), [new] "r" (new) \
: cl); \ : cl); \
\ \
return oldval; \ return oldval; \
} \ } \
__LL_SC_EXPORT(__cmpxchg_case_##name); __LL_SC_EXPORT(__cmpxchg_case_##name##sz);
__CMPXCHG_CASE(w, b, 1, , , , ) __CMPXCHG_CASE(w, b, , 8, , , , )
__CMPXCHG_CASE(w, h, 2, , , , ) __CMPXCHG_CASE(w, h, , 16, , , , )
__CMPXCHG_CASE(w, , 4, , , , ) __CMPXCHG_CASE(w, , , 32, , , , )
__CMPXCHG_CASE( , , 8, , , , ) __CMPXCHG_CASE( , , , 64, , , , )
__CMPXCHG_CASE(w, b, acq_1, , a, , "memory") __CMPXCHG_CASE(w, b, acq_, 8, , a, , "memory")
__CMPXCHG_CASE(w, h, acq_2, , a, , "memory") __CMPXCHG_CASE(w, h, acq_, 16, , a, , "memory")
__CMPXCHG_CASE(w, , acq_4, , a, , "memory") __CMPXCHG_CASE(w, , acq_, 32, , a, , "memory")
__CMPXCHG_CASE( , , acq_8, , a, , "memory") __CMPXCHG_CASE( , , acq_, 64, , a, , "memory")
__CMPXCHG_CASE(w, b, rel_1, , , l, "memory") __CMPXCHG_CASE(w, b, rel_, 8, , , l, "memory")
__CMPXCHG_CASE(w, h, rel_2, , , l, "memory") __CMPXCHG_CASE(w, h, rel_, 16, , , l, "memory")
__CMPXCHG_CASE(w, , rel_4, , , l, "memory") __CMPXCHG_CASE(w, , rel_, 32, , , l, "memory")
__CMPXCHG_CASE( , , rel_8, , , l, "memory") __CMPXCHG_CASE( , , rel_, 64, , , l, "memory")
__CMPXCHG_CASE(w, b, mb_1, dmb ish, , l, "memory") __CMPXCHG_CASE(w, b, mb_, 8, dmb ish, , l, "memory")
__CMPXCHG_CASE(w, h, mb_2, dmb ish, , l, "memory") __CMPXCHG_CASE(w, h, mb_, 16, dmb ish, , l, "memory")
__CMPXCHG_CASE(w, , mb_4, dmb ish, , l, "memory") __CMPXCHG_CASE(w, , mb_, 32, dmb ish, , l, "memory")
__CMPXCHG_CASE( , , mb_8, dmb ish, , l, "memory") __CMPXCHG_CASE( , , mb_, 64, dmb ish, , l, "memory")
#undef __CMPXCHG_CASE #undef __CMPXCHG_CASE
......
arch/arm64/include/asm/atomic_lse.h
View file @ 5ef3fe4c
...@@ -446,22 +446,22 @@ static inline long atomic64_dec_if_positive(atomic64_t *v) ...@@ -446,22 +446,22 @@ static inline long atomic64_dec_if_positive(atomic64_t *v)
#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op) #define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \ #define __CMPXCHG_CASE(w, sfx, name, sz, mb, cl...) \
static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \ static inline u##sz __cmpxchg_case_##name##sz(volatile void *ptr, \
unsigned long old, \ unsigned long old, \
unsigned long new) \ u##sz new) \
{ \ { \
register unsigned long x0 asm ("x0") = (unsigned long)ptr; \ register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
register unsigned long x1 asm ("x1") = old; \ register unsigned long x1 asm ("x1") = old; \
register unsigned long x2 asm ("x2") = new; \ register u##sz x2 asm ("x2") = new; \
\ \
asm volatile(ARM64_LSE_ATOMIC_INSN( \ asm volatile(ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \ /* LL/SC */ \
__LL_SC_CMPXCHG(name) \ __LL_SC_CMPXCHG(name##sz) \
__nops(2), \ __nops(2), \
/* LSE atomics */ \ /* LSE atomics */ \
" mov " #w "30, %" #w "[old]\n" \ " mov " #w "30, %" #w "[old]\n" \
" cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \ " cas" #mb #sfx "\t" #w "30, %" #w "[new], %[v]\n" \
" mov %" #w "[ret], " #w "30") \ " mov %" #w "[ret], " #w "30") \
: [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \ : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
: [old] "r" (x1), [new] "r" (x2) \ : [old] "r" (x1), [new] "r" (x2) \
...@@ -470,22 +470,22 @@ static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \ ...@@ -470,22 +470,22 @@ static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
return x0; \ return x0; \
} }
__CMPXCHG_CASE(w, b, 1, ) __CMPXCHG_CASE(w, b, , 8, )
__CMPXCHG_CASE(w, h, 2, ) __CMPXCHG_CASE(w, h, , 16, )
__CMPXCHG_CASE(w, , 4, ) __CMPXCHG_CASE(w, , , 32, )
__CMPXCHG_CASE(x, , 8, ) __CMPXCHG_CASE(x, , , 64, )
__CMPXCHG_CASE(w, b, acq_1, a, "memory") __CMPXCHG_CASE(w, b, acq_, 8, a, "memory")
__CMPXCHG_CASE(w, h, acq_2, a, "memory") __CMPXCHG_CASE(w, h, acq_, 16, a, "memory")
__CMPXCHG_CASE(w, , acq_4, a, "memory") __CMPXCHG_CASE(w, , acq_, 32, a, "memory")
__CMPXCHG_CASE(x, , acq_8, a, "memory") __CMPXCHG_CASE(x, , acq_, 64, a, "memory")
__CMPXCHG_CASE(w, b, rel_1, l, "memory") __CMPXCHG_CASE(w, b, rel_, 8, l, "memory")
__CMPXCHG_CASE(w, h, rel_2, l, "memory") __CMPXCHG_CASE(w, h, rel_, 16, l, "memory")
__CMPXCHG_CASE(w, , rel_4, l, "memory") __CMPXCHG_CASE(w, , rel_, 32, l, "memory")
__CMPXCHG_CASE(x, , rel_8, l, "memory") __CMPXCHG_CASE(x, , rel_, 64, l, "memory")
__CMPXCHG_CASE(w, b, mb_1, al, "memory") __CMPXCHG_CASE(w, b, mb_, 8, al, "memory")
__CMPXCHG_CASE(w, h, mb_2, al, "memory") __CMPXCHG_CASE(w, h, mb_, 16, al, "memory")
__CMPXCHG_CASE(w, , mb_4, al, "memory") __CMPXCHG_CASE(w, , mb_, 32, al, "memory")
__CMPXCHG_CASE(x, , mb_8, al, "memory") __CMPXCHG_CASE(x, , mb_, 64, al, "memory")
#undef __LL_SC_CMPXCHG #undef __LL_SC_CMPXCHG
#undef __CMPXCHG_CASE #undef __CMPXCHG_CASE
......
arch/arm64/include/asm/cmpxchg.h
View file @ 5ef3fe4c
...@@ -30,46 +30,46 @@ ...@@ -30,46 +30,46 @@
* barrier case is generated as release+dmb for the former and * barrier case is generated as release+dmb for the former and
* acquire+release for the latter. * acquire+release for the latter.
*/ */
#define __XCHG_CASE(w, sz, name, mb, nop_lse, acq, acq_lse, rel, cl) \ #define __XCHG_CASE(w, sfx, name, sz, mb, nop_lse, acq, acq_lse, rel, cl) \
static inline unsigned long __xchg_case_##name(unsigned long x, \ static inline u##sz __xchg_case_##name##sz(u##sz x, volatile void *ptr) \
volatile void *ptr) \ { \
{ \ u##sz ret; \
unsigned long ret, tmp; \ unsigned long tmp; \
\ \
asm volatile(ARM64_LSE_ATOMIC_INSN( \ asm volatile(ARM64_LSE_ATOMIC_INSN( \
/* LL/SC */ \ /* LL/SC */ \
" prfm pstl1strm, %2\n" \ " prfm pstl1strm, %2\n" \
"1: ld" #acq "xr" #sz "\t%" #w "0, %2\n" \ "1: ld" #acq "xr" #sfx "\t%" #w "0, %2\n" \
" st" #rel "xr" #sz "\t%w1, %" #w "3, %2\n" \ " st" #rel "xr" #sfx "\t%w1, %" #w "3, %2\n" \
" cbnz %w1, 1b\n" \ " cbnz %w1, 1b\n" \
" " #mb, \ " " #mb, \
/* LSE atomics */ \ /* LSE atomics */ \
" swp" #acq_lse #rel #sz "\t%" #w "3, %" #w "0, %2\n" \ " swp" #acq_lse #rel #sfx "\t%" #w "3, %" #w "0, %2\n" \
__nops(3) \ __nops(3) \
" " #nop_lse) \ " " #nop_lse) \
: "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned long *)ptr) \ : "=&r" (ret), "=&r" (tmp), "+Q" (*(u##sz *)ptr) \
: "r" (x) \ : "r" (x) \
: cl); \ : cl); \
\ \
return ret; \ return ret; \
} }
__XCHG_CASE(w, b, 1, , , , , , ) __XCHG_CASE(w, b, , 8, , , , , , )
__XCHG_CASE(w, h, 2, , , , , , ) __XCHG_CASE(w, h, , 16, , , , , , )
__XCHG_CASE(w, , 4, , , , , , ) __XCHG_CASE(w, , , 32, , , , , , )
__XCHG_CASE( , , 8, , , , , , ) __XCHG_CASE( , , , 64, , , , , , )
__XCHG_CASE(w, b, acq_1, , , a, a, , "memory") __XCHG_CASE(w, b, acq_, 8, , , a, a, , "memory")
__XCHG_CASE(w, h, acq_2, , , a, a, , "memory") __XCHG_CASE(w, h, acq_, 16, , , a, a, , "memory")
__XCHG_CASE(w, , acq_4, , , a, a, , "memory") __XCHG_CASE(w, , acq_, 32, , , a, a, , "memory")
__XCHG_CASE( , , acq_8, , , a, a, , "memory") __XCHG_CASE( , , acq_, 64, , , a, a, , "memory")
__XCHG_CASE(w, b, rel_1, , , , , l, "memory") __XCHG_CASE(w, b, rel_, 8, , , , , l, "memory")
__XCHG_CASE(w, h, rel_2, , , , , l, "memory") __XCHG_CASE(w, h, rel_, 16, , , , , l, "memory")
__XCHG_CASE(w, , rel_4, , , , , l, "memory") __XCHG_CASE(w, , rel_, 32, , , , , l, "memory")
__XCHG_CASE( , , rel_8, , , , , l, "memory") __XCHG_CASE( , , rel_, 64, , , , , l, "memory")
__XCHG_CASE(w, b, mb_1, dmb ish, nop, , a, l, "memory") __XCHG_CASE(w, b, mb_, 8, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, h, mb_2, dmb ish, nop, , a, l, "memory") __XCHG_CASE(w, h, mb_, 16, dmb ish, nop, , a, l, "memory")
__XCHG_CASE(w, , mb_4, dmb ish, nop, , a, l, "memory") __XCHG_CASE(w, , mb_, 32, dmb ish, nop, , a, l, "memory")
__XCHG_CASE( , , mb_8, dmb ish, nop, , a, l, "memory") __XCHG_CASE( , , mb_, 64, dmb ish, nop, , a, l, "memory")
#undef __XCHG_CASE #undef __XCHG_CASE
...@@ -80,13 +80,13 @@ static inline unsigned long __xchg##sfx(unsigned long x, \ ...@@ -80,13 +80,13 @@ static inline unsigned long __xchg##sfx(unsigned long x, \
{ \ { \
switch (size) { \ switch (size) { \
case 1: \ case 1: \
return __xchg_case##sfx##_1(x, ptr); \ return __xchg_case##sfx##_8(x, ptr); \
case 2: \ case 2: \
return __xchg_case##sfx##_2(x, ptr); \ return __xchg_case##sfx##_16(x, ptr); \
case 4: \ case 4: \
return __xchg_case##sfx##_4(x, ptr); \ return __xchg_case##sfx##_32(x, ptr); \
case 8: \ case 8: \
return __xchg_case##sfx##_8(x, ptr); \ return __xchg_case##sfx##_64(x, ptr); \
default: \ default: \
BUILD_BUG(); \ BUILD_BUG(); \
} \ } \
...@@ -123,13 +123,13 @@ static inline unsigned long __cmpxchg##sfx(volatile void *ptr, \ ...@@ -123,13 +123,13 @@ static inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
{ \ { \
switch (size) { \ switch (size) { \
case 1: \ case 1: \
return __cmpxchg_case##sfx##_1(ptr, (u8)old, new); \ return __cmpxchg_case##sfx##_8(ptr, (u8)old, new); \
case 2: \ case 2: \
return __cmpxchg_case##sfx##_2(ptr, (u16)old, new); \ return __cmpxchg_case##sfx##_16(ptr, (u16)old, new); \
case 4: \ case 4: \
return __cmpxchg_case##sfx##_4(ptr, old, new); \ return __cmpxchg_case##sfx##_32(ptr, old, new); \
case 8: \ case 8: \
return __cmpxchg_case##sfx##_8(ptr, old, new); \ return __cmpxchg_case##sfx##_64(ptr, old, new); \
default: \ default: \
BUILD_BUG(); \ BUILD_BUG(); \
} \ } \
...@@ -197,16 +197,16 @@ __CMPXCHG_GEN(_mb) ...@@ -197,16 +197,16 @@ __CMPXCHG_GEN(_mb)
__ret; \ __ret; \
}) })
#define __CMPWAIT_CASE(w, sz, name) \ #define __CMPWAIT_CASE(w, sfx, sz) \
static inline void __cmpwait_case_##name(volatile void *ptr, \ static inline void __cmpwait_case_##sz(volatile void *ptr, \
unsigned long val) \ unsigned long val) \
{ \ { \
unsigned long tmp; \ unsigned long tmp; \
\ \
asm volatile( \ asm volatile( \
" sevl\n" \ " sevl\n" \
" wfe\n" \ " wfe\n" \
" ldxr" #sz "\t%" #w "[tmp], %[v]\n" \ " ldxr" #sfx "\t%" #w "[tmp], %[v]\n" \
" eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \ " eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
" cbnz %" #w "[tmp], 1f\n" \ " cbnz %" #w "[tmp], 1f\n" \
" wfe\n" \ " wfe\n" \
...@@ -215,10 +215,10 @@ static inline void __cmpwait_case_##name(volatile void *ptr, \ ...@@ -215,10 +215,10 @@ static inline void __cmpwait_case_##name(volatile void *ptr, \
: [val] "r" (val)); \ : [val] "r" (val)); \
} }
__CMPWAIT_CASE(w, b, 1); __CMPWAIT_CASE(w, b, 8);
__CMPWAIT_CASE(w, h, 2); __CMPWAIT_CASE(w, h, 16);
__CMPWAIT_CASE(w, , 4); __CMPWAIT_CASE(w, , 32);
__CMPWAIT_CASE( , , 8); __CMPWAIT_CASE( , , 64);
#undef __CMPWAIT_CASE #undef __CMPWAIT_CASE
...@@ -229,13 +229,13 @@ static inline void __cmpwait##sfx(volatile void *ptr, \ ...@@ -229,13 +229,13 @@ static inline void __cmpwait##sfx(volatile void *ptr, \
{ \ { \
switch (size) { \ switch (size) { \
case 1: \ case 1: \
return __cmpwait_case##sfx##_1(ptr, (u8)val); \ return __cmpwait_case##sfx##_8(ptr, (u8)val); \
case 2: \ case 2: \
return __cmpwait_case##sfx##_2(ptr, (u16)val); \ return __cmpwait_case##sfx##_16(ptr, (u16)val); \
case 4: \ case 4: \
return __cmpwait_case##sfx##_4(ptr, val); \ return __cmpwait_case##sfx##_32(ptr, val); \
case 8: \ case 8: \
return __cmpwait_case##sfx##_8(ptr, val); \ return __cmpwait_case##sfx##_64(ptr, val); \
default: \ default: \
BUILD_BUG(); \ BUILD_BUG(); \
} \ } \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7