Skip to content

L
linux
Commit 4f3316c2 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar
Browse files
Options

locking,arch,sparc: Fold atomic_ops 

Many of the atomic op implementations are the same except for one
instruction; fold the lot into a few CPP macros and reduce LoC.

This also prepares for easy addition of new ops.
Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Acked-by: default avatarDavid S. Miller <davem@davemloft.net>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Kirill Tkhai <tkhai@yandex.ru>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Sam Ravnborg <sam@ravnborg.org>
Cc: sparclinux@vger.kernel.org
Link: http://lkml.kernel.org/r/20140508135852.825281379@infradead.orgSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent c6470150
Hide whitespace changes
Inline Side-by-side
Showing with 133 additions and 148 deletions
arch/sparc/include/asm/atomic_32.h
View file @ 4f3316c2
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
#define ATOMIC_INIT(i) { (i) } #define ATOMIC_INIT(i) { (i) }
int __atomic_add_return(int, atomic_t *); int atomic_add_return(int, atomic_t *);
int atomic_cmpxchg(atomic_t *, int, int); int atomic_cmpxchg(atomic_t *, int, int);
#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
int __atomic_add_unless(atomic_t *, int, int); int __atomic_add_unless(atomic_t *, int, int);
...@@ -28,15 +28,14 @@ void atomic_set(atomic_t *, int); ...@@ -28,15 +28,14 @@ void atomic_set(atomic_t *, int);
#define atomic_read(v) (*(volatile int *)&(v)->counter) #define atomic_read(v) (*(volatile int *)&(v)->counter)
#define atomic_add(i, v) ((void)__atomic_add_return( (int)(i), (v))) #define atomic_add(i, v) ((void)atomic_add_return( (int)(i), (v)))
#define atomic_sub(i, v) ((void)__atomic_add_return(-(int)(i), (v))) #define atomic_sub(i, v) ((void)atomic_add_return(-(int)(i), (v)))
#define atomic_inc(v) ((void)__atomic_add_return( 1, (v))) #define atomic_inc(v) ((void)atomic_add_return( 1, (v)))
#define atomic_dec(v) ((void)__atomic_add_return( -1, (v))) #define atomic_dec(v) ((void)atomic_add_return( -1, (v)))
#define atomic_add_return(i, v) (__atomic_add_return( (int)(i), (v))) #define atomic_sub_return(i, v) (atomic_add_return(-(int)(i), (v)))
#define atomic_sub_return(i, v) (__atomic_add_return(-(int)(i), (v))) #define atomic_inc_return(v) (atomic_add_return( 1, (v)))
#define atomic_inc_return(v) (__atomic_add_return( 1, (v))) #define atomic_dec_return(v) (atomic_add_return( -1, (v)))
#define atomic_dec_return(v) (__atomic_add_return( -1, (v)))
#define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0) #define atomic_add_negative(a, v) (atomic_add_return((a), (v)) < 0)
......
arch/sparc/include/asm/atomic_64.h
View file @ 4f3316c2
...@@ -20,27 +20,28 @@ ...@@ -20,27 +20,28 @@
#define atomic_set(v, i) (((v)->counter) = i) #define atomic_set(v, i) (((v)->counter) = i)
#define atomic64_set(v, i) (((v)->counter) = i) #define atomic64_set(v, i) (((v)->counter) = i)
void atomic_add(int, atomic_t *); #define ATOMIC_OP(op) \
void atomic64_add(long, atomic64_t *); void atomic_##op(int, atomic_t *); \
void atomic_sub(int, atomic_t *); void atomic64_##op(long, atomic64_t *);
void atomic64_sub(long, atomic64_t *);
int atomic_add_ret(int, atomic_t *); #define ATOMIC_OP_RETURN(op) \
long atomic64_add_ret(long, atomic64_t *); int atomic_##op##_return(int, atomic_t *); \
int atomic_sub_ret(int, atomic_t *); long atomic64_##op##_return(long, atomic64_t *);
long atomic64_sub_ret(long, atomic64_t *);
#define atomic_dec_return(v) atomic_sub_ret(1, v) #define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
#define atomic64_dec_return(v) atomic64_sub_ret(1, v)
#define atomic_inc_return(v) atomic_add_ret(1, v) ATOMIC_OPS(add)
#define atomic64_inc_return(v) atomic64_add_ret(1, v) ATOMIC_OPS(sub)
#define atomic_sub_return(i, v) atomic_sub_ret(i, v) #undef ATOMIC_OPS
#define atomic64_sub_return(i, v) atomic64_sub_ret(i, v) #undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
#define atomic_add_return(i, v) atomic_add_ret(i, v) #define atomic_dec_return(v) atomic_sub_return(1, v)
#define atomic64_add_return(i, v) atomic64_add_ret(i, v) #define atomic64_dec_return(v) atomic64_sub_return(1, v)
#define atomic_inc_return(v) atomic_add_return(1, v)
#define atomic64_inc_return(v) atomic64_add_return(1, v)
/* /*
* atomic_inc_and_test - increment and test * atomic_inc_and_test - increment and test
...@@ -53,11 +54,11 @@ long atomic64_sub_ret(long, atomic64_t *); ...@@ -53,11 +54,11 @@ long atomic64_sub_ret(long, atomic64_t *);
#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0) #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0) #define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
#define atomic_sub_and_test(i, v) (atomic_sub_ret(i, v) == 0) #define atomic_sub_and_test(i, v) (atomic_sub_return(i, v) == 0)
#define atomic64_sub_and_test(i, v) (atomic64_sub_ret(i, v) == 0) #define atomic64_sub_and_test(i, v) (atomic64_sub_return(i, v) == 0)
#define atomic_dec_and_test(v) (atomic_sub_ret(1, v) == 0) #define atomic_dec_and_test(v) (atomic_sub_return(1, v) == 0)
#define atomic64_dec_and_test(v) (atomic64_sub_ret(1, v) == 0) #define atomic64_dec_and_test(v) (atomic64_sub_return(1, v) == 0)
#define atomic_inc(v) atomic_add(1, v) #define atomic_inc(v) atomic_add(1, v)
#define atomic64_inc(v) atomic64_add(1, v) #define atomic64_inc(v) atomic64_add(1, v)
...@@ -65,8 +66,8 @@ long atomic64_sub_ret(long, atomic64_t *); ...@@ -65,8 +66,8 @@ long atomic64_sub_ret(long, atomic64_t *);
#define atomic_dec(v) atomic_sub(1, v) #define atomic_dec(v) atomic_sub(1, v)
#define atomic64_dec(v) atomic64_sub(1, v) #define atomic64_dec(v) atomic64_sub(1, v)
#define atomic_add_negative(i, v) (atomic_add_ret(i, v) < 0) #define atomic_add_negative(i, v) (atomic_add_return(i, v) < 0)
#define atomic64_add_negative(i, v) (atomic64_add_ret(i, v) < 0) #define atomic64_add_negative(i, v) (atomic64_add_return(i, v) < 0)
#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n))) #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
#define atomic_xchg(v, new) (xchg(&((v)->counter), new)) #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
......
arch/sparc/kernel/smp_64.c
View file @ 4f3316c2
...@@ -1138,7 +1138,7 @@ static unsigned long penguins_are_doing_time; ...@@ -1138,7 +1138,7 @@ static unsigned long penguins_are_doing_time;
void smp_capture(void) void smp_capture(void)
{ {
int result = atomic_add_ret(1, &smp_capture_depth); int result = atomic_add_return(1, &smp_capture_depth);
if (result == 1) { if (result == 1) {
int ncpus = num_online_cpus(); int ncpus = num_online_cpus();
......
arch/sparc/lib/atomic32.c
View file @ 4f3316c2
...@@ -27,18 +27,23 @@ static DEFINE_SPINLOCK(dummy); ...@@ -27,18 +27,23 @@ static DEFINE_SPINLOCK(dummy);
#endif /* SMP */ #endif /* SMP */
int __atomic_add_return(int i, atomic_t *v) #define ATOMIC_OP(op, cop) \
{ int atomic_##op##_return(int i, atomic_t *v) \
int ret; { \
unsigned long flags; int ret; \
spin_lock_irqsave(ATOMIC_HASH(v), flags); unsigned long flags; \
spin_lock_irqsave(ATOMIC_HASH(v), flags); \
ret = (v->counter += i); \
ret = (v->counter cop i); \
spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \
return ret; spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \
} return ret; \
EXPORT_SYMBOL(__atomic_add_return); } \
EXPORT_SYMBOL(atomic_##op##_return);
ATOMIC_OP(add, +=)
#undef ATOMIC_OP
int atomic_cmpxchg(atomic_t *v, int old, int new) int atomic_cmpxchg(atomic_t *v, int old, int new)
{ {
......
arch/sparc/lib/atomic_64.S
View file @ 4f3316c2
...@@ -14,109 +14,80 @@ ...@@ -14,109 +14,80 @@
* memory barriers, and a second which returns * memory barriers, and a second which returns
* a value and does the barriers. * a value and does the barriers.
*/ */
ENTRY(atomic_add) /* %o0 = increment, %o1 = atomic_ptr */
BACKOFF_SETUP(%o2)
1: lduw [%o1], %g1
add %g1, %o0, %g7
cas [%o1], %g1, %g7
cmp %g1, %g7
bne,pn %icc, BACKOFF_LABEL(2f, 1b)
nop
retl
nop
2: BACKOFF_SPIN(%o2, %o3, 1b)
ENDPROC(atomic_add)
ENTRY(atomic_sub) /* %o0 = decrement, %o1 = atomic_ptr */ #define ATOMIC_OP(op) \
BACKOFF_SETUP(%o2) ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
1: lduw [%o1], %g1 BACKOFF_SETUP(%o2); \
sub %g1, %o0, %g7 1: lduw [%o1], %g1; \
cas [%o1], %g1, %g7 op %g1, %o0, %g7; \
cmp %g1, %g7 cas [%o1], %g1, %g7; \
bne,pn %icc, BACKOFF_LABEL(2f, 1b) cmp %g1, %g7; \
nop bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
retl nop; \
nop retl; \
2: BACKOFF_SPIN(%o2, %o3, 1b) nop; \
ENDPROC(atomic_sub) 2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic_##op); \
ENTRY(atomic_add_ret) /* %o0 = increment, %o1 = atomic_ptr */ #define ATOMIC_OP_RETURN(op) \
BACKOFF_SETUP(%o2) ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
1: lduw [%o1], %g1 BACKOFF_SETUP(%o2); \
add %g1, %o0, %g7 1: lduw [%o1], %g1; \
cas [%o1], %g1, %g7 op %g1, %o0, %g7; \
cmp %g1, %g7 cas [%o1], %g1, %g7; \
bne,pn %icc, BACKOFF_LABEL(2f, 1b) cmp %g1, %g7; \
add %g1, %o0, %g1 bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
retl add %g1, %o0, %g1; \
sra %g1, 0, %o0 retl; \
2: BACKOFF_SPIN(%o2, %o3, 1b) sra %g1, 0, %o0; \
ENDPROC(atomic_add_ret) 2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic_##op##_return);
ENTRY(atomic_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */ #define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
BACKOFF_SETUP(%o2)
1: lduw [%o1], %g1
sub %g1, %o0, %g7
cas [%o1], %g1, %g7
cmp %g1, %g7
bne,pn %icc, BACKOFF_LABEL(2f, 1b)
sub %g1, %o0, %g1
retl
sra %g1, 0, %o0
2: BACKOFF_SPIN(%o2, %o3, 1b)
ENDPROC(atomic_sub_ret)
ENTRY(atomic64_add) /* %o0 = increment, %o1 = atomic_ptr */ ATOMIC_OPS(add)
BACKOFF_SETUP(%o2) ATOMIC_OPS(sub)
1: ldx [%o1], %g1
add %g1, %o0, %g7
casx [%o1], %g1, %g7
cmp %g1, %g7
bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
nop
retl
nop
2: BACKOFF_SPIN(%o2, %o3, 1b)
ENDPROC(atomic64_add)
ENTRY(atomic64_sub) /* %o0 = decrement, %o1 = atomic_ptr */ #undef ATOMIC_OPS
BACKOFF_SETUP(%o2) #undef ATOMIC_OP_RETURN
1: ldx [%o1], %g1 #undef ATOMIC_OP
sub %g1, %o0, %g7
casx [%o1], %g1, %g7
cmp %g1, %g7
bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
nop
retl
nop
2: BACKOFF_SPIN(%o2, %o3, 1b)
ENDPROC(atomic64_sub)
ENTRY(atomic64_add_ret) /* %o0 = increment, %o1 = atomic_ptr */ #define ATOMIC64_OP(op) \
BACKOFF_SETUP(%o2) ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
1: ldx [%o1], %g1 BACKOFF_SETUP(%o2); \
add %g1, %o0, %g7 1: ldx [%o1], %g1; \
casx [%o1], %g1, %g7 op %g1, %o0, %g7; \
cmp %g1, %g7 casx [%o1], %g1, %g7; \
bne,pn %xcc, BACKOFF_LABEL(2f, 1b) cmp %g1, %g7; \
nop bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
retl nop; \
add %g1, %o0, %o0 retl; \
2: BACKOFF_SPIN(%o2, %o3, 1b) nop; \
ENDPROC(atomic64_add_ret) 2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic64_##op); \
ENTRY(atomic64_sub_ret) /* %o0 = decrement, %o1 = atomic_ptr */ #define ATOMIC64_OP_RETURN(op) \
BACKOFF_SETUP(%o2) ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
1: ldx [%o1], %g1 BACKOFF_SETUP(%o2); \
sub %g1, %o0, %g7 1: ldx [%o1], %g1; \
casx [%o1], %g1, %g7 op %g1, %o0, %g7; \
cmp %g1, %g7 casx [%o1], %g1, %g7; \
bne,pn %xcc, BACKOFF_LABEL(2f, 1b) cmp %g1, %g7; \
nop bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
retl nop; \
sub %g1, %o0, %o0 retl; \
2: BACKOFF_SPIN(%o2, %o3, 1b) add %g1, %o0, %o0; \
ENDPROC(atomic64_sub_ret) 2: BACKOFF_SPIN(%o2, %o3, 1b); \
ENDPROC(atomic64_##op##_return);
#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op)
ATOMIC64_OPS(add)
ATOMIC64_OPS(sub)
#undef ATOMIC64_OPS
#undef ATOMIC64_OP_RETURN
#undef ATOMIC64_OP
ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */ ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
BACKOFF_SETUP(%o2) BACKOFF_SETUP(%o2)
......
arch/sparc/lib/ksyms.c
View file @ 4f3316c2
...@@ -99,14 +99,23 @@ EXPORT_SYMBOL(___copy_in_user); ...@@ -99,14 +99,23 @@ EXPORT_SYMBOL(___copy_in_user);
EXPORT_SYMBOL(__clear_user); EXPORT_SYMBOL(__clear_user);
/* Atomic counter implementation. */ /* Atomic counter implementation. */
EXPORT_SYMBOL(atomic_add); #define ATOMIC_OP(op) \
EXPORT_SYMBOL(atomic_add_ret); EXPORT_SYMBOL(atomic_##op); \
EXPORT_SYMBOL(atomic_sub); EXPORT_SYMBOL(atomic64_##op);
EXPORT_SYMBOL(atomic_sub_ret);
EXPORT_SYMBOL(atomic64_add); #define ATOMIC_OP_RETURN(op) \
EXPORT_SYMBOL(atomic64_add_ret); EXPORT_SYMBOL(atomic_##op##_return); \
EXPORT_SYMBOL(atomic64_sub); EXPORT_SYMBOL(atomic64_##op##_return);
EXPORT_SYMBOL(atomic64_sub_ret);
#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
ATOMIC_OPS(add)
ATOMIC_OPS(sub)
#undef ATOMIC_OPS
#undef ATOMIC_OP_RETURN
#undef ATOMIC_OP
EXPORT_SYMBOL(atomic64_dec_if_positive); EXPORT_SYMBOL(atomic64_dec_if_positive);
/* Atomic bit operations. */ /* Atomic bit operations. */
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7