Skip to content

L
linux
Commit 2d6b1cee authored by David S. Miller's avatar David S. Miller
Browse files
Options

[SPARC64]: atomic and bitop fixes. 

1) Correct memory barriers.  Routines not returning a value need
   no memory barriers, however routines returning values do need
   them.
2) Actually implement non-atomic ext2 bitops.

Thanks to Anton Blanchard for pointing out the memory barrier
requirements.
Signed-off-by: default avatarDavid S. Miller <davem@davemloft.net>
parent d4ba27b9
Hide whitespace changes
Inline Side-by-side
Showing with 288 additions and 129 deletions
arch/sparc64/kernel/smp.c
View file @ 2d6b1cee
......@@ -894,9 +894,8 @@ static unsigned long penguins_are_doing_time;
void smp_capture(void)
{
int result = __atomic_add(1, &smp_capture_depth);
int result = atomic_add_ret(1, &smp_capture_depth);
membar("#StoreStore | #LoadStore");
if (result == 1) {
int ncpus = num_online_cpus();
......
arch/sparc64/kernel/sparc64_ksyms.c
View file @ 2d6b1cee
......@@ -172,18 +172,25 @@ EXPORT_SYMBOL(down_interruptible);
EXPORT_SYMBOL(up);
/* Atomic counter implementation. */
EXPORT_SYMBOL(__atomic_add);
EXPORT_SYMBOL(__atomic_sub);
EXPORT_SYMBOL(__atomic64_add);
EXPORT_SYMBOL(__atomic64_sub);
EXPORT_SYMBOL(atomic_add);
EXPORT_SYMBOL(atomic_add_ret);
EXPORT_SYMBOL(atomic_sub);
EXPORT_SYMBOL(atomic_sub_ret);
EXPORT_SYMBOL(atomic64_add);
EXPORT_SYMBOL(atomic64_add_ret);
EXPORT_SYMBOL(atomic64_sub);
EXPORT_SYMBOL(atomic64_sub_ret);
#ifdef CONFIG_SMP
EXPORT_SYMBOL(_atomic_dec_and_lock);
#endif
/* Atomic bit operations. */
EXPORT_SYMBOL(___test_and_set_bit);
EXPORT_SYMBOL(___test_and_clear_bit);
EXPORT_SYMBOL(___test_and_change_bit);
EXPORT_SYMBOL(test_and_set_bit);
EXPORT_SYMBOL(test_and_clear_bit);
EXPORT_SYMBOL(test_and_change_bit);
EXPORT_SYMBOL(set_bit);
EXPORT_SYMBOL(clear_bit);
EXPORT_SYMBOL(change_bit);
/* Bit searching */
EXPORT_SYMBOL(find_next_bit);
......
arch/sparc64/lib/atomic.S
View file @ 2d6b1cee
......@@ -4,73 +4,136 @@
* Copyright (C) 1999 David S. Miller (davem@redhat.com)
*/
#include <linux/config.h>
#include <asm/asi.h>
/* On SMP we need to use memory barriers to ensure
* correct memory operation ordering, nop these out
* for uniprocessor.
*/
#ifdef CONFIG_SMP
#define ATOMIC_PRE_BARRIER membar #StoreLoad | #LoadLoad
#define ATOMIC_POST_BARRIER membar #StoreLoad | #StoreStore
#else
#define ATOMIC_PRE_BARRIER nop
#define ATOMIC_POST_BARRIER nop
#endif
.text
/* We use these stubs for the uncommon case
* of contention on the atomic value. This is
* so that we can keep the main fast path 8
* instructions long and thus fit into a single
* L2 cache line.
/* Two versions of the atomic routines, one that
* does not return a value and does not perform
* memory barriers, and a second which returns
* a value and does the barriers.
*/
__atomic_add_membar:
ba,pt %xcc, __atomic_add
membar #StoreLoad | #StoreStore
.globl atomic_add
.type atomic_add,#function
atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
1: lduw [%o1], %g5
add %g5, %o0, %g7
cas [%o1], %g5, %g7
cmp %g5, %g7
bne,pn %icc, 1b
nop
retl
nop
.size atomic_add, .-atomic_add
__atomic_sub_membar:
ba,pt %xcc, __atomic_sub
membar #StoreLoad | #StoreStore
.globl atomic_sub
.type atomic_sub,#function
atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
1: lduw [%o1], %g5
sub %g5, %o0, %g7
cas [%o1], %g5, %g7
cmp %g5, %g7
bne,pn %icc, 1b
nop
retl
nop
.size atomic_sub, .-atomic_sub
.align 64
.globl __atomic_add
.type __atomic_add,#function
__atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
lduw [%o1], %g5
.globl atomic_add_ret
.type atomic_add_ret,#function
atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
ATOMIC_PRE_BARRIER
1: lduw [%o1], %g5
add %g5, %o0, %g7
cas [%o1], %g5, %g7
cmp %g5, %g7
bne,pn %icc, __atomic_add_membar
bne,pn %icc, 1b
add %g7, %o0, %g7
ATOMIC_POST_BARRIER
retl
sra %g7, 0, %o0
.size __atomic_add, .-__atomic_add
.size atomic_add_ret, .-atomic_add_ret
.globl __atomic_sub
.type __atomic_sub,#function
__atomic_sub: /* %o0 = increment, %o1 = atomic_ptr */
lduw [%o1], %g5
.globl atomic_sub_ret
.type atomic_sub_ret,#function
atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
ATOMIC_PRE_BARRIER
1: lduw [%o1], %g5
sub %g5, %o0, %g7
cas [%o1], %g5, %g7
cmp %g5, %g7
bne,pn %icc, __atomic_sub_membar
bne,pn %icc, 1b
sub %g7, %o0, %g7
ATOMIC_POST_BARRIER
retl
sra %g7, 0, %o0
.size __atomic_sub, .-__atomic_sub
.size atomic_sub_ret, .-atomic_sub_ret
.globl atomic64_add
.type atomic64_add,#function
atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
1: ldx [%o1], %g5
add %g5, %o0, %g7
casx [%o1], %g5, %g7
cmp %g5, %g7
bne,pn %xcc, 1b
nop
retl
nop
.size atomic64_add, .-atomic64_add
.globl __atomic64_add
.type __atomic64_add,#function
__atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
ldx [%o1], %g5
.globl atomic64_sub
.type atomic64_sub,#function
atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
1: ldx [%o1], %g5
sub %g5, %o0, %g7
casx [%o1], %g5, %g7
cmp %g5, %g7
bne,pn %xcc, 1b
nop
retl
nop
.size atomic64_sub, .-atomic64_sub
.globl atomic64_add_ret
.type atomic64_add_ret,#function
atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
ATOMIC_PRE_BARRIER
1: ldx [%o1], %g5
add %g5, %o0, %g7
casx [%o1], %g5, %g7
cmp %g5, %g7
bne,pn %xcc, __atomic64_add
membar #StoreLoad | #StoreStore
bne,pn %xcc, 1b
add %g7, %o0, %g7
ATOMIC_POST_BARRIER
retl
add %g7, %o0, %o0
.size __atomic64_add, .-__atomic64_add
mov %g7, %o0
.size atomic64_add_ret, .-atomic64_add_ret
.globl __atomic64_sub
.type __atomic64_sub,#function
__atomic64_sub: /* %o0 = increment, %o1 = atomic_ptr */
ldx [%o1], %g5
.globl atomic64_sub_ret
.type atomic64_sub_ret,#function
atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
ATOMIC_PRE_BARRIER
1: ldx [%o1], %g5
sub %g5, %o0, %g7
casx [%o1], %g5, %g7
cmp %g5, %g7
bne,pn %xcc, __atomic64_sub
membar #StoreLoad | #StoreStore
bne,pn %xcc, 1b
sub %g7, %o0, %g7
ATOMIC_POST_BARRIER
retl
sub %g7, %o0, %o0
.size __atomic64_sub, .-__atomic64_sub
mov %g7, %o0
.size atomic64_sub_ret, .-atomic64_sub_ret
arch/sparc64/lib/bitops.S
View file @ 2d6b1cee
......@@ -4,69 +4,142 @@
* Copyright (C) 2000 David S. Miller (davem@redhat.com)
*/
#include <linux/config.h>
#include <asm/asi.h>
/* On SMP we need to use memory barriers to ensure
* correct memory operation ordering, nop these out
* for uniprocessor.
*/
#ifdef CONFIG_SMP
#define BITOP_PRE_BARRIER membar #StoreLoad | #LoadLoad
#define BITOP_POST_BARRIER membar #StoreLoad | #StoreStore
#else
#define BITOP_PRE_BARRIER nop
#define BITOP_POST_BARRIER nop
#endif
.text
.align 64
.globl ___test_and_set_bit
.type ___test_and_set_bit,#function
___test_and_set_bit: /* %o0=nr, %o1=addr */
.globl test_and_set_bit
.type test_and_set_bit,#function
test_and_set_bit: /* %o0=nr, %o1=addr */
BITOP_PRE_BARRIER
srlx %o0, 6, %g1
mov 1, %g5
sllx %g1, 3, %g3
and %o0, 63, %g2
sllx %g5, %g2, %g5
add %o1, %g3, %o1
1: ldx [%o1], %g7
or %g7, %g5, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
bne,pn %xcc, 1b
and %g7, %g5, %g2
BITOP_POST_BARRIER
clr %o0
retl
movrne %g2, 1, %o0
.size test_and_set_bit, .-test_and_set_bit
.globl test_and_clear_bit
.type test_and_clear_bit,#function
test_and_clear_bit: /* %o0=nr, %o1=addr */
BITOP_PRE_BARRIER
srlx %o0, 6, %g1
mov 1, %g5
sllx %g1, 3, %g3
and %o0, 63, %g2
sllx %g5, %g2, %g5
add %o1, %g3, %o1
1: ldx [%o1], %g7
andn %g7, %g5, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
bne,pn %xcc, 1b
and %g7, %g5, %g2
BITOP_POST_BARRIER
clr %o0
retl
movrne %g2, 1, %o0
.size test_and_clear_bit, .-test_and_clear_bit
.globl test_and_change_bit
.type test_and_change_bit,#function
test_and_change_bit: /* %o0=nr, %o1=addr */
BITOP_PRE_BARRIER
srlx %o0, 6, %g1
mov 1, %g5
sllx %g1, 3, %g3
and %o0, 63, %g2
sllx %g5, %g2, %g5
add %o1, %g3, %o1
1: ldx [%o1], %g7
xor %g7, %g5, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
bne,pn %xcc, 1b
and %g7, %g5, %g2
BITOP_POST_BARRIER
clr %o0
retl
movrne %g2, 1, %o0
.size test_and_change_bit, .-test_and_change_bit
.globl set_bit
.type set_bit,#function
set_bit: /* %o0=nr, %o1=addr */
srlx %o0, 6, %g1
mov 1, %g5
sllx %g1, 3, %g3
and %o0, 63, %g2
sllx %g5, %g2, %g5
add %o1, %g3, %o1
ldx [%o1], %g7
1: andcc %g7, %g5, %o0
bne,pn %xcc, 2f
xor %g7, %g5, %g1
1: ldx [%o1], %g7
or %g7, %g5, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
bne,a,pn %xcc, 1b
ldx [%o1], %g7
2: retl
membar #StoreLoad | #StoreStore
.size ___test_and_set_bit, .-___test_and_set_bit
bne,pn %xcc, 1b
nop
retl
nop
.size set_bit, .-set_bit
.globl ___test_and_clear_bit
.type ___test_and_clear_bit,#function
___test_and_clear_bit: /* %o0=nr, %o1=addr */
.globl clear_bit
.type clear_bit,#function
clear_bit: /* %o0=nr, %o1=addr */
srlx %o0, 6, %g1
mov 1, %g5
sllx %g1, 3, %g3
and %o0, 63, %g2
sllx %g5, %g2, %g5
add %o1, %g3, %o1
ldx [%o1], %g7
1: andcc %g7, %g5, %o0
be,pn %xcc, 2f
xor %g7, %g5, %g1
1: ldx [%o1], %g7
andn %g7, %g5, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
bne,a,pn %xcc, 1b
ldx [%o1], %g7
2: retl
membar #StoreLoad | #StoreStore
.size ___test_and_clear_bit, .-___test_and_clear_bit
bne,pn %xcc, 1b
nop
retl
nop
.size clear_bit, .-clear_bit
.globl ___test_and_change_bit
.type ___test_and_change_bit,#function
___test_and_change_bit: /* %o0=nr, %o1=addr */
.globl change_bit
.type change_bit,#function
change_bit: /* %o0=nr, %o1=addr */
srlx %o0, 6, %g1
mov 1, %g5
sllx %g1, 3, %g3
and %o0, 63, %g2
sllx %g5, %g2, %g5
add %o1, %g3, %o1
ldx [%o1], %g7
1: and %g7, %g5, %o0
1: ldx [%o1], %g7
xor %g7, %g5, %g1
casx [%o1], %g7, %g1
cmp %g7, %g1
bne,a,pn %xcc, 1b
ldx [%o1], %g7
2: retl
membar #StoreLoad | #StoreStore
nop
.size ___test_and_change_bit, .-___test_and_change_bit
bne,pn %xcc, 1b
nop
retl
nop
.size change_bit, .-change_bit
include/asm-sparc64/atomic.h
View file @ 2d6b1cee
......@@ -8,6 +8,7 @@
#ifndef __ARCH_SPARC64_ATOMIC__
#define __ARCH_SPARC64_ATOMIC__
#include <linux/config.h>
#include <linux/types.h>
typedef struct { volatile int counter; } atomic_t;
......@@ -22,29 +23,27 @@ typedef struct { volatile __s64 counter; } atomic64_t;
#define atomic_set(v, i) (((v)->counter) = i)
#define atomic64_set(v, i) (((v)->counter) = i)
extern int __atomic_add(int, atomic_t *);
extern int __atomic64_add(__s64, atomic64_t *);
extern void atomic_add(int, atomic_t *);
extern void atomic64_add(int, atomic64_t *);
extern void atomic_sub(int, atomic_t *);
extern void atomic64_sub(int, atomic64_t *);
extern int __atomic_sub(int, atomic_t *);
extern int __atomic64_sub(__s64, atomic64_t *);
extern int atomic_add_ret(int, atomic_t *);
extern int atomic64_add_ret(int, atomic64_t *);
extern int atomic_sub_ret(int, atomic_t *);
extern int atomic64_sub_ret(int, atomic64_t *);
#define atomic_add(i, v) ((void)__atomic_add(i, v))
#define atomic64_add(i, v) ((void)__atomic64_add(i, v))
#define atomic_dec_return(v) atomic_sub_ret(1, v)
#define atomic64_dec_return(v) atomic64_sub_ret(1, v)
#define atomic_sub(i, v) ((void)__atomic_sub(i, v))
#define atomic64_sub(i, v) ((void)__atomic64_sub(i, v))
#define atomic_inc_return(v) atomic_add_ret(1, v)
#define atomic64_inc_return(v) atomic64_add_ret(1, v)
#define atomic_dec_return(v) __atomic_sub(1, v)
#define atomic64_dec_return(v) __atomic64_sub(1, v)
#define atomic_sub_return(i, v) atomic_sub_ret(i, v)
#define atomic64_sub_return(i, v) atomic64_sub_ret(i, v)
#define atomic_inc_return(v) __atomic_add(1, v)
#define atomic64_inc_return(v) __atomic64_add(1, v)
#define atomic_sub_return(i, v) __atomic_sub(i, v)
#define atomic64_sub_return(i, v) __atomic64_sub(i, v)
#define atomic_add_return(i, v) __atomic_add(i, v)
#define atomic64_add_return(i, v) __atomic64_add(i, v)
#define atomic_add_return(i, v) atomic_add_ret(i, v)
#define atomic64_add_return(i, v) atomic64_add_ret(i, v)
/*
* atomic_inc_and_test - increment and test
......@@ -56,25 +55,32 @@ extern int __atomic64_sub(__s64, atomic64_t *);
*/
#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
#define atomic_sub_and_test(i, v) (__atomic_sub(i, v) == 0)
#define atomic64_sub_and_test(i, v) (__atomic64_sub(i, v) == 0)
#define atomic_sub_and_test(i, v) (atomic_sub_ret(i, v) == 0)
#define atomic64_sub_and_test(i, v) (atomic64_sub_ret(i, v) == 0)
#define atomic_dec_and_test(v) (__atomic_sub(1, v) == 0)
#define atomic64_dec_and_test(v) (__atomic64_sub(1, v) == 0)
#define atomic_dec_and_test(v) (atomic_sub_ret(1, v) == 0)
#define atomic64_dec_and_test(v) (atomic64_sub_ret(1, v) == 0)
#define atomic_inc(v) ((void)__atomic_add(1, v))
#define atomic64_inc(v) ((void)__atomic64_add(1, v))
#define atomic_inc(v) atomic_add(1, v)
#define atomic64_inc(v) atomic64_add(1, v)
#define atomic_dec(v) ((void)__atomic_sub(1, v))
#define atomic64_dec(v) ((void)__atomic64_sub(1, v))
#define atomic_dec(v) atomic_sub(1, v)
#define atomic64_dec(v) atomic64_sub(1, v)
#define atomic_add_negative(i, v) (__atomic_add(i, v) < 0)
#define atomic64_add_negative(i, v) (__atomic64_add(i, v) < 0)
#define atomic_add_negative(i, v) (atomic_add_ret(i, v) < 0)
#define atomic64_add_negative(i, v) (atomic64_add_ret(i, v) < 0)
/* Atomic operations are already serializing */
#ifdef CONFIG_SMP
#define smp_mb__before_atomic_dec() membar("#StoreLoad | #LoadLoad")
#define smp_mb__after_atomic_dec() membar("#StoreLoad | #StoreStore")
#define smp_mb__before_atomic_inc() membar("#StoreLoad | #LoadLoad")
#define smp_mb__after_atomic_inc() membar("#StoreLoad | #StoreStore")
#else
#define smp_mb__before_atomic_dec() barrier()
#define smp_mb__after_atomic_dec() barrier()
#define smp_mb__before_atomic_inc() barrier()
#define smp_mb__after_atomic_inc() barrier()
#endif
#endif /* !(__ARCH_SPARC64_ATOMIC__) */
include/asm-sparc64/bitops.h
View file @ 2d6b1cee
......@@ -7,19 +7,16 @@
#ifndef _SPARC64_BITOPS_H
#define _SPARC64_BITOPS_H
#include <linux/config.h>
#include <linux/compiler.h>
#include <asm/byteorder.h>
extern long ___test_and_set_bit(unsigned long nr, volatile unsigned long *addr);
extern long ___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr);
extern long ___test_and_change_bit(unsigned long nr, volatile unsigned long *addr);
#define test_and_set_bit(nr,addr) ({___test_and_set_bit(nr,addr)!=0;})
#define test_and_clear_bit(nr,addr) ({___test_and_clear_bit(nr,addr)!=0;})
#define test_and_change_bit(nr,addr) ({___test_and_change_bit(nr,addr)!=0;})
#define set_bit(nr,addr) ((void)___test_and_set_bit(nr,addr))
#define clear_bit(nr,addr) ((void)___test_and_clear_bit(nr,addr))
#define change_bit(nr,addr) ((void)___test_and_change_bit(nr,addr))
extern int test_and_set_bit(unsigned long nr, volatile unsigned long *addr);
extern int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr);
extern int test_and_change_bit(unsigned long nr, volatile unsigned long *addr);
extern void set_bit(unsigned long nr, volatile unsigned long *addr);
extern void clear_bit(unsigned long nr, volatile unsigned long *addr);
extern void change_bit(unsigned long nr, volatile unsigned long *addr);
/* "non-atomic" versions... */
......@@ -74,8 +71,13 @@ static __inline__ int __test_and_change_bit(int nr, volatile unsigned long *addr
return ((old & mask) != 0);
}
#define smp_mb__before_clear_bit() do { } while(0)
#define smp_mb__after_clear_bit() do { } while(0)
#ifdef CONFIG_SMP
#define smp_mb__before_clear_bit() membar("#StoreLoad | #LoadLoad")
#define smp_mb__after_clear_bit() membar("#StoreLoad | #StoreStore")
#else
#define smp_mb__before_clear_bit() barrier()
#define smp_mb__after_clear_bit() barrier()
#endif
static __inline__ int test_bit(int nr, __const__ volatile unsigned long *addr)
{
......@@ -230,9 +232,9 @@ extern unsigned long find_next_zero_bit(const unsigned long *,
find_next_zero_bit((addr), (size), 0)
#define test_and_set_le_bit(nr,addr) \
({ ___test_and_set_bit((nr) ^ 0x38, (addr)) != 0; })
test_and_set_bit((nr) ^ 0x38, (addr))
#define test_and_clear_le_bit(nr,addr) \
({ ___test_and_clear_bit((nr) ^ 0x38, (addr)) != 0; })
test_and_clear_bit((nr) ^ 0x38, (addr))
static __inline__ int test_le_bit(int nr, __const__ unsigned long * addr)
{
......@@ -251,12 +253,21 @@ extern unsigned long find_next_zero_le_bit(unsigned long *, unsigned long, unsig
#ifdef __KERNEL__
#define __set_le_bit(nr, addr) \
__set_bit((nr) ^ 0x38, (addr))
#define __clear_le_bit(nr, addr) \
__clear_bit((nr) ^ 0x38, (addr))
#define __test_and_clear_le_bit(nr, addr) \
__test_and_clear_bit((nr) ^ 0x38, (addr))
#define __test_and_set_le_bit(nr, addr) \
__test_and_set_bit((nr) ^ 0x38, (addr))
#define ext2_set_bit(nr,addr) \
test_and_set_le_bit((nr),(unsigned long *)(addr))
__test_and_set_le_bit((nr),(unsigned long *)(addr))
#define ext2_set_bit_atomic(lock,nr,addr) \
test_and_set_le_bit((nr),(unsigned long *)(addr))
#define ext2_clear_bit(nr,addr) \
test_and_clear_le_bit((nr),(unsigned long *)(addr))
__test_and_clear_le_bit((nr),(unsigned long *)(addr))
#define ext2_clear_bit_atomic(lock,nr,addr) \
test_and_clear_le_bit((nr),(unsigned long *)(addr))
#define ext2_test_bit(nr,addr) \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7