Commit 0b2fcfdb authored by Nick Piggin's avatar Nick Piggin Committed by Linus Torvalds

[PATCH] atomic: add_unless cmpxchg optimise

Without branch hints, the very unlikely chance of the loop repeating due to
cmpxchg failure is unrolled with gcc-4 that I have tested.

Improve this for architectures with a native cas/cmpxchg.  llsc archs
should try to implement this natively.
Signed-off-by: default avatarNick Piggin <npiggin@suse.de>
Cc: Andi Kleen <ak@muc.de>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: Roman Zippel <zippel@linux-m68k.org>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 713729e8
...@@ -225,8 +225,14 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v) ...@@ -225,8 +225,14 @@ static __inline__ int atomic_sub_return(int i, atomic_t *v)
({ \ ({ \
int c, old; \ int c, old; \
c = atomic_read(v); \ c = atomic_read(v); \
while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ for (;;) { \
if (unlikely(c == (u))) \
break; \
old = atomic_cmpxchg((v), c, c + (a)); \
if (likely(old == c)) \
break; \
c = old; \ c = old; \
} \
c != (u); \ c != (u); \
}) })
#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
......
...@@ -95,8 +95,14 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v) ...@@ -95,8 +95,14 @@ ia64_atomic64_sub (__s64 i, atomic64_t *v)
({ \ ({ \
int c, old; \ int c, old; \
c = atomic_read(v); \ c = atomic_read(v); \
while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ for (;;) { \
if (unlikely(c == (u))) \
break; \
old = atomic_cmpxchg((v), c, c + (a)); \
if (likely(old == c)) \
break; \
c = old; \ c = old; \
} \
c != (u); \ c != (u); \
}) })
#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
......
...@@ -175,8 +175,14 @@ static inline void atomic_set_mask(unsigned long mask, unsigned long *v) ...@@ -175,8 +175,14 @@ static inline void atomic_set_mask(unsigned long mask, unsigned long *v)
({ \ ({ \
int c, old; \ int c, old; \
c = atomic_read(v); \ c = atomic_read(v); \
while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ for (;;) { \
if (unlikely(c == (u))) \
break; \
old = atomic_cmpxchg((v), c, c + (a)); \
if (likely(old == c)) \
break; \
c = old; \ c = old; \
} \
c != (u); \ c != (u); \
}) })
#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
......
...@@ -89,10 +89,15 @@ static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new) ...@@ -89,10 +89,15 @@ static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new)
static __inline__ int atomic_add_unless(atomic_t *v, int a, int u) static __inline__ int atomic_add_unless(atomic_t *v, int a, int u)
{ {
int c, old; int c, old;
c = atomic_read(v); c = atomic_read(v);
while (c != u && (old = atomic_cmpxchg(v, c, c + a)) != c) for (;;) {
if (unlikely(c == u))
break;
old = atomic_cmpxchg(v, c, c + a);
if (likely(old == c))
break;
c = old; c = old;
}
return c != u; return c != u;
} }
...@@ -167,10 +172,15 @@ static __inline__ int atomic64_add_unless(atomic64_t *v, ...@@ -167,10 +172,15 @@ static __inline__ int atomic64_add_unless(atomic64_t *v,
long long a, long long u) long long a, long long u)
{ {
long long c, old; long long c, old;
c = atomic64_read(v); c = atomic64_read(v);
while (c != u && (old = atomic64_cmpxchg(v, c, c + a)) != c) for (;;) {
if (unlikely(c == u))
break;
old = atomic64_cmpxchg(v, c, c + a);
if (likely(old == c))
break;
c = old; c = old;
}
return c != u; return c != u;
} }
......
...@@ -78,9 +78,15 @@ extern int atomic64_sub_ret(int, atomic64_t *); ...@@ -78,9 +78,15 @@ extern int atomic64_sub_ret(int, atomic64_t *);
({ \ ({ \
int c, old; \ int c, old; \
c = atomic_read(v); \ c = atomic_read(v); \
while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ for (;;) { \
if (unlikely(c == (u))) \
break; \
old = atomic_cmpxchg((v), c, c + (a)); \
if (likely(old == c)) \
break; \
c = old; \ c = old; \
c != (u); \ } \
likely(c != (u)); \
}) })
#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
......
...@@ -405,8 +405,14 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v) ...@@ -405,8 +405,14 @@ static __inline__ long atomic64_sub_return(long i, atomic64_t *v)
({ \ ({ \
int c, old; \ int c, old; \
c = atomic_read(v); \ c = atomic_read(v); \
while (c != (u) && (old = atomic_cmpxchg((v), c, c + (a))) != c) \ for (;;) { \
if (unlikely(c == (u))) \
break; \
old = atomic_cmpxchg((v), c, c + (a)); \
if (likely(old == c)) \
break; \
c = old; \ c = old; \
} \
c != (u); \ c != (u); \
}) })
#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment