[SPARC64]: atomic and bitop fixes.

1) Correct memory barriers.  Routines not returning a value need
   no memory barriers, however routines returning values do need
   them.
2) Actually implement non-atomic ext2 bitops.

Thanks to Anton Blanchard for pointing out the memory barrier
requirements.
Signed-off-by: David S. Miller <davem@davemloft.net>

arch/sparc64/kernel/smp.c

@@ -894,9 +894,8 @@ static unsigned long penguins_are_doing_time;
 
 void smp_capture(void)
 {
-	int result = __atomic_add(1, &smp_capture_depth);
+	int result = atomic_add_ret(1, &smp_capture_depth);
 
-	membar("#StoreStore | #LoadStore");
 	if (result == 1) {
 		int ncpus = num_online_cpus();
 

arch/sparc64/kernel/sparc64_ksyms.c

@@ -172,18 +172,25 @@ EXPORT_SYMBOL(down_interruptible);
 EXPORT_SYMBOL(up);
 
 /* Atomic counter implementation. */
-EXPORT_SYMBOL(__atomic_add);
-EXPORT_SYMBOL(__atomic_sub);
-EXPORT_SYMBOL(__atomic64_add);
-EXPORT_SYMBOL(__atomic64_sub);
+EXPORT_SYMBOL(atomic_add);
+EXPORT_SYMBOL(atomic_add_ret);
+EXPORT_SYMBOL(atomic_sub);
+EXPORT_SYMBOL(atomic_sub_ret);
+EXPORT_SYMBOL(atomic64_add);
+EXPORT_SYMBOL(atomic64_add_ret);
+EXPORT_SYMBOL(atomic64_sub);
+EXPORT_SYMBOL(atomic64_sub_ret);
 #ifdef CONFIG_SMP
 EXPORT_SYMBOL(_atomic_dec_and_lock);
 #endif
 
 /* Atomic bit operations. */
-EXPORT_SYMBOL(___test_and_set_bit);
-EXPORT_SYMBOL(___test_and_clear_bit);
-EXPORT_SYMBOL(___test_and_change_bit);
+EXPORT_SYMBOL(test_and_set_bit);
+EXPORT_SYMBOL(test_and_clear_bit);
+EXPORT_SYMBOL(test_and_change_bit);
+EXPORT_SYMBOL(set_bit);
+EXPORT_SYMBOL(clear_bit);
+EXPORT_SYMBOL(change_bit);
 
 /* Bit searching */
 EXPORT_SYMBOL(find_next_bit);

arch/sparc64/lib/atomic.S

@@ -4,73 +4,136 @@
  * Copyright (C) 1999 David S. Miller (davem@redhat.com)
  */
 
+#include <linux/config.h>
 #include <asm/asi.h>
 
+	/* On SMP we need to use memory barriers to ensure
+	 * correct memory operation ordering, nop these out
+	 * for uniprocessor.
+	 */
+#ifdef CONFIG_SMP
+#define ATOMIC_PRE_BARRIER	membar #StoreLoad | #LoadLoad
+#define ATOMIC_POST_BARRIER	membar #StoreLoad | #StoreStore
+#else
+#define ATOMIC_PRE_BARRIER	nop
+#define ATOMIC_POST_BARRIER	nop
+#endif
+
 	.text
 
-	/* We use these stubs for the uncommon case
-	 * of contention on the atomic value.  This is
-	 * so that we can keep the main fast path 8
-	 * instructions long and thus fit into a single
-	 * L2 cache line.
+	/* Two versions of the atomic routines, one that
+	 * does not return a value and does not perform
+	 * memory barriers, and a second which returns
+	 * a value and does the barriers.
 	 */
-__atomic_add_membar:
-	ba,pt	%xcc, __atomic_add
-	 membar	#StoreLoad | #StoreStore
+	.globl	atomic_add
+	.type	atomic_add,#function
+atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
+1:	lduw	[%o1], %g5
+	add	%g5, %o0, %g7
+	cas	[%o1], %g5, %g7
+	cmp	%g5, %g7
+	bne,pn	%icc, 1b
+	 nop
+	retl
+	 nop
+	.size	atomic_add, .-atomic_add
 
-__atomic_sub_membar:
-	ba,pt	%xcc, __atomic_sub
-	 membar	#StoreLoad | #StoreStore
+	.globl	atomic_sub
+	.type	atomic_sub,#function
+atomic_sub: /* %o0 = decrement, %o1 = atomic_ptr */
+1:	lduw	[%o1], %g5
+	sub	%g5, %o0, %g7
+	cas	[%o1], %g5, %g7
+	cmp	%g5, %g7
+	bne,pn	%icc, 1b
+	 nop
+	retl
+	 nop
+	.size	atomic_sub, .-atomic_sub
 
-	.align	64
-	.globl	__atomic_add
-	.type	__atomic_add,#function
-__atomic_add: /* %o0 = increment, %o1 = atomic_ptr */
-	lduw	[%o1], %g5
+	.globl	atomic_add_ret
+	.type	atomic_add_ret,#function
+atomic_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
+	ATOMIC_PRE_BARRIER
+1:	lduw	[%o1], %g5
 	add	%g5, %o0, %g7
 	cas	[%o1], %g5, %g7
 	cmp	%g5, %g7
-	bne,pn	%icc, __atomic_add_membar
+	bne,pn	%icc, 1b
 	 add	%g7, %o0, %g7
+	ATOMIC_POST_BARRIER
 	retl
 	 sra	%g7, 0, %o0
-	.size	__atomic_add, .-__atomic_add
+	.size	atomic_add_ret, .-atomic_add_ret
 
-	.globl	__atomic_sub
-	.type	__atomic_sub,#function
-__atomic_sub: /* %o0 = increment, %o1 = atomic_ptr */
-	lduw	[%o1], %g5
+	.globl	atomic_sub_ret
+	.type	atomic_sub_ret,#function
+atomic_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
+	ATOMIC_PRE_BARRIER
+1:	lduw	[%o1], %g5
 	sub	%g5, %o0, %g7
 	cas	[%o1], %g5, %g7
 	cmp	%g5, %g7
-	bne,pn	%icc, __atomic_sub_membar
+	bne,pn	%icc, 1b
 	 sub	%g7, %o0, %g7
+	ATOMIC_POST_BARRIER
 	retl
 	 sra	%g7, 0, %o0
-	.size	__atomic_sub, .-__atomic_sub
+	.size	atomic_sub_ret, .-atomic_sub_ret
+
+	.globl	atomic64_add
+	.type	atomic64_add,#function
+atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
+1:	ldx	[%o1], %g5
+	add	%g5, %o0, %g7
+	casx	[%o1], %g5, %g7
+	cmp	%g5, %g7
+	bne,pn	%xcc, 1b
+	 nop
+	retl
+	 nop
+	.size	atomic64_add, .-atomic64_add
 
-	.globl	__atomic64_add
-	.type	__atomic64_add,#function
-__atomic64_add: /* %o0 = increment, %o1 = atomic_ptr */
-	ldx	[%o1], %g5
+	.globl	atomic64_sub
+	.type	atomic64_sub,#function
+atomic64_sub: /* %o0 = decrement, %o1 = atomic_ptr */
+1:	ldx	[%o1], %g5
+	sub	%g5, %o0, %g7
+	casx	[%o1], %g5, %g7
+	cmp	%g5, %g7
+	bne,pn	%xcc, 1b
+	 nop
+	retl
+	 nop
+	.size	atomic64_sub, .-atomic64_sub
+
+	.globl	atomic64_add_ret
+	.type	atomic64_add_ret,#function
+atomic64_add_ret: /* %o0 = increment, %o1 = atomic_ptr */
+	ATOMIC_PRE_BARRIER
+1:	ldx	[%o1], %g5
 	add	%g5, %o0, %g7
 	casx	[%o1], %g5, %g7
 	cmp	%g5, %g7
-	bne,pn	%xcc, __atomic64_add
-	 membar	#StoreLoad | #StoreStore
+	bne,pn	%xcc, 1b
+	 add	%g7, %o0, %g7
+	ATOMIC_POST_BARRIER
 	retl
-	 add	%g7, %o0, %o0
-	.size	__atomic64_add, .-__atomic64_add
+	 mov	%g7, %o0
+	.size	atomic64_add_ret, .-atomic64_add_ret
 
-	.globl	__atomic64_sub
-	.type	__atomic64_sub,#function
-__atomic64_sub: /* %o0 = increment, %o1 = atomic_ptr */
-	ldx	[%o1], %g5
+	.globl	atomic64_sub_ret
+	.type	atomic64_sub_ret,#function
+atomic64_sub_ret: /* %o0 = decrement, %o1 = atomic_ptr */
+	ATOMIC_PRE_BARRIER
+1:	ldx	[%o1], %g5
 	sub	%g5, %o0, %g7
 	casx	[%o1], %g5, %g7
 	cmp	%g5, %g7
-	bne,pn	%xcc, __atomic64_sub
-	 membar	#StoreLoad | #StoreStore
+	bne,pn	%xcc, 1b
+	 sub	%g7, %o0, %g7
+	ATOMIC_POST_BARRIER
 	retl
-	 sub	%g7, %o0, %o0
-	.size	__atomic64_sub, .-__atomic64_sub
+	 mov	%g7, %o0
+	.size	atomic64_sub_ret, .-atomic64_sub_ret

arch/sparc64/lib/bitops.S

@@ -4,69 +4,142 @@
  * Copyright (C) 2000 David S. Miller (davem@redhat.com)
  */
 
+#include <linux/config.h>
 #include <asm/asi.h>
 
+	/* On SMP we need to use memory barriers to ensure
+	 * correct memory operation ordering, nop these out
+	 * for uniprocessor.
+	 */
+#ifdef CONFIG_SMP
+#define BITOP_PRE_BARRIER	membar #StoreLoad | #LoadLoad
+#define BITOP_POST_BARRIER	membar #StoreLoad | #StoreStore
+#else
+#define BITOP_PRE_BARRIER	nop
+#define BITOP_POST_BARRIER	nop
+#endif
+
 	.text
-	.align	64
-	.globl	___test_and_set_bit
-	.type	___test_and_set_bit,#function
-___test_and_set_bit:	/* %o0=nr, %o1=addr */
+
+	.globl	test_and_set_bit
+	.type	test_and_set_bit,#function
+test_and_set_bit:	/* %o0=nr, %o1=addr */
+	BITOP_PRE_BARRIER
 	srlx	%o0, 6, %g1
 	mov	1, %g5
 	sllx	%g1, 3, %g3
 	and	%o0, 63, %g2
 	sllx	%g5, %g2, %g5
 	add	%o1, %g3, %o1
-	ldx	[%o1], %g7
-1:	andcc	%g7, %g5, %o0
-	bne,pn	%xcc, 2f
-	 xor	%g7, %g5, %g1
+1:	ldx	[%o1], %g7
+	or	%g7, %g5, %g1
+	casx	[%o1], %g7, %g1
+	cmp	%g7, %g1
+	bne,pn	%xcc, 1b
+	 and	%g7, %g5, %g2
+	BITOP_POST_BARRIER
+	clr	%o0
+	retl
+	 movrne	%g2, 1, %o0
+	.size	test_and_set_bit, .-test_and_set_bit
+
+	.globl	test_and_clear_bit
+	.type	test_and_clear_bit,#function
+test_and_clear_bit:	/* %o0=nr, %o1=addr */
+	BITOP_PRE_BARRIER
+	srlx	%o0, 6, %g1
+	mov	1, %g5
+	sllx	%g1, 3, %g3
+	and	%o0, 63, %g2
+	sllx	%g5, %g2, %g5
+	add	%o1, %g3, %o1
+1:	ldx	[%o1], %g7
+	andn	%g7, %g5, %g1
 	casx	[%o1], %g7, %g1
 	cmp	%g7, %g1
-	bne,a,pn %xcc, 1b
-	 ldx	[%o1], %g7
-2:	retl
-	 membar	#StoreLoad | #StoreStore
-	.size	___test_and_set_bit, .-___test_and_set_bit
+	bne,pn	%xcc, 1b
+	 and	%g7, %g5, %g2
+	BITOP_POST_BARRIER
+	clr	%o0
+	retl
+	 movrne	%g2, 1, %o0
+	.size	test_and_clear_bit, .-test_and_clear_bit
 
-	.globl	___test_and_clear_bit
-	.type	___test_and_clear_bit,#function
-___test_and_clear_bit:	/* %o0=nr, %o1=addr */
+	.globl	test_and_change_bit
+	.type	test_and_change_bit,#function
+test_and_change_bit:	/* %o0=nr, %o1=addr */
+	BITOP_PRE_BARRIER
 	srlx	%o0, 6, %g1
 	mov	1, %g5
 	sllx	%g1, 3, %g3
 	and	%o0, 63, %g2
 	sllx	%g5, %g2, %g5
 	add	%o1, %g3, %o1
-	ldx	[%o1], %g7
-1:	andcc	%g7, %g5, %o0
-	be,pn	%xcc, 2f
+1:	ldx	[%o1], %g7
 	xor	%g7, %g5, %g1
 	casx	[%o1], %g7, %g1
 	cmp	%g7, %g1
-	bne,a,pn %xcc, 1b
-	 ldx	[%o1], %g7
-2:	retl
-	 membar	#StoreLoad | #StoreStore
-	.size	___test_and_clear_bit, .-___test_and_clear_bit
+	bne,pn	%xcc, 1b
+	 and	%g7, %g5, %g2
+	BITOP_POST_BARRIER
+	clr	%o0
+	retl
+	 movrne	%g2, 1, %o0
+	.size	test_and_change_bit, .-test_and_change_bit
+
+	.globl	set_bit
+	.type	set_bit,#function
+set_bit:		/* %o0=nr, %o1=addr */
+	srlx	%o0, 6, %g1
+	mov	1, %g5
+	sllx	%g1, 3, %g3
+	and	%o0, 63, %g2
+	sllx	%g5, %g2, %g5
+	add	%o1, %g3, %o1
+1:	ldx	[%o1], %g7
+	or	%g7, %g5, %g1
+	casx	[%o1], %g7, %g1
+	cmp	%g7, %g1
+	bne,pn	%xcc, 1b
+	 nop
+	retl
+	 nop
+	.size	set_bit, .-set_bit
+
+	.globl	clear_bit
+	.type	clear_bit,#function
+clear_bit:		/* %o0=nr, %o1=addr */
+	srlx	%o0, 6, %g1
+	mov	1, %g5
+	sllx	%g1, 3, %g3
+	and	%o0, 63, %g2
+	sllx	%g5, %g2, %g5
+	add	%o1, %g3, %o1
+1:	ldx	[%o1], %g7
+	andn	%g7, %g5, %g1
+	casx	[%o1], %g7, %g1
+	cmp	%g7, %g1
+	bne,pn	%xcc, 1b
+	 nop
+	retl
+	 nop
+	.size	clear_bit, .-clear_bit
 
-	.globl	___test_and_change_bit
-	.type	___test_and_change_bit,#function
-___test_and_change_bit:	/* %o0=nr, %o1=addr */
+	.globl	change_bit
+	.type	change_bit,#function
+change_bit:		/* %o0=nr, %o1=addr */
 	srlx	%o0, 6, %g1
 	mov	1, %g5
 	sllx	%g1, 3, %g3
 	and	%o0, 63, %g2
 	sllx	%g5, %g2, %g5
 	add	%o1, %g3, %o1
-	ldx	[%o1], %g7
-1:	and	%g7, %g5, %o0
+1:	ldx	[%o1], %g7
 	xor	%g7, %g5, %g1
 	casx	[%o1], %g7, %g1
 	cmp	%g7, %g1
-	bne,a,pn %xcc, 1b
-	 ldx	[%o1], %g7
-2:	retl
-	 membar	#StoreLoad | #StoreStore
+	bne,pn	%xcc, 1b
+	 nop
+	retl
 	 nop
-	.size	___test_and_change_bit, .-___test_and_change_bit
+	.size	change_bit, .-change_bit

include/asm-sparc64/atomic.h

@@ -8,6 +8,7 @@
 #ifndef __ARCH_SPARC64_ATOMIC__
 #define __ARCH_SPARC64_ATOMIC__
 
+#include <linux/config.h>
 #include <linux/types.h>
 
 typedef struct { volatile int counter; } atomic_t;
@@ -22,29 +23,27 @@ typedef struct { volatile __s64 counter; } atomic64_t;
 #define atomic_set(v, i)	(((v)->counter) = i)
 #define atomic64_set(v, i)	(((v)->counter) = i)
 
-extern int __atomic_add(int, atomic_t *);
-extern int __atomic64_add(__s64, atomic64_t *);
+extern void atomic_add(int, atomic_t *);
+extern void atomic64_add(int, atomic64_t *);
+extern void atomic_sub(int, atomic_t *);
+extern void atomic64_sub(int, atomic64_t *);
 
-extern int __atomic_sub(int, atomic_t *);
-extern int __atomic64_sub(__s64, atomic64_t *);
+extern int atomic_add_ret(int, atomic_t *);
+extern int atomic64_add_ret(int, atomic64_t *);
+extern int atomic_sub_ret(int, atomic_t *);
+extern int atomic64_sub_ret(int, atomic64_t *);
 
-#define atomic_add(i, v) ((void)__atomic_add(i, v))
-#define atomic64_add(i, v) ((void)__atomic64_add(i, v))
+#define atomic_dec_return(v) atomic_sub_ret(1, v)
+#define atomic64_dec_return(v) atomic64_sub_ret(1, v)
 
-#define atomic_sub(i, v) ((void)__atomic_sub(i, v))
-#define atomic64_sub(i, v) ((void)__atomic64_sub(i, v))
+#define atomic_inc_return(v) atomic_add_ret(1, v)
+#define atomic64_inc_return(v) atomic64_add_ret(1, v)
 
-#define atomic_dec_return(v) __atomic_sub(1, v)
-#define atomic64_dec_return(v) __atomic64_sub(1, v)
+#define atomic_sub_return(i, v) atomic_sub_ret(i, v)
+#define atomic64_sub_return(i, v) atomic64_sub_ret(i, v)
 
-#define atomic_inc_return(v) __atomic_add(1, v)
-#define atomic64_inc_return(v) __atomic64_add(1, v)
-
-#define atomic_sub_return(i, v) __atomic_sub(i, v)
-#define atomic64_sub_return(i, v) __atomic64_sub(i, v)
-
-#define atomic_add_return(i, v) __atomic_add(i, v)
-#define atomic64_add_return(i, v) __atomic64_add(i, v)
+#define atomic_add_return(i, v) atomic_add_ret(i, v)
+#define atomic64_add_return(i, v) atomic64_add_ret(i, v)
 
 /*
  * atomic_inc_and_test - increment and test
@@ -56,25 +55,32 @@ extern int __atomic64_sub(__s64, atomic64_t *);
  */
 #define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
 
-#define atomic_sub_and_test(i, v) (__atomic_sub(i, v) == 0)
-#define atomic64_sub_and_test(i, v) (__atomic64_sub(i, v) == 0)
+#define atomic_sub_and_test(i, v) (atomic_sub_ret(i, v) == 0)
+#define atomic64_sub_and_test(i, v) (atomic64_sub_ret(i, v) == 0)
 
-#define atomic_dec_and_test(v) (__atomic_sub(1, v) == 0)
-#define atomic64_dec_and_test(v) (__atomic64_sub(1, v) == 0)
+#define atomic_dec_and_test(v) (atomic_sub_ret(1, v) == 0)
+#define atomic64_dec_and_test(v) (atomic64_sub_ret(1, v) == 0)
 
-#define atomic_inc(v) ((void)__atomic_add(1, v))
-#define atomic64_inc(v) ((void)__atomic64_add(1, v))
+#define atomic_inc(v) atomic_add(1, v)
+#define atomic64_inc(v) atomic64_add(1, v)
 
-#define atomic_dec(v) ((void)__atomic_sub(1, v))
-#define atomic64_dec(v) ((void)__atomic64_sub(1, v))
+#define atomic_dec(v) atomic_sub(1, v)
+#define atomic64_dec(v) atomic64_sub(1, v)
 
-#define atomic_add_negative(i, v) (__atomic_add(i, v) < 0)
-#define atomic64_add_negative(i, v) (__atomic64_add(i, v) < 0)
+#define atomic_add_negative(i, v) (atomic_add_ret(i, v) < 0)
+#define atomic64_add_negative(i, v) (atomic64_add_ret(i, v) < 0)
 
 /* Atomic operations are already serializing */
+#ifdef CONFIG_SMP
+#define smp_mb__before_atomic_dec()	membar("#StoreLoad | #LoadLoad")
+#define smp_mb__after_atomic_dec()	membar("#StoreLoad | #StoreStore")
+#define smp_mb__before_atomic_inc()	membar("#StoreLoad | #LoadLoad")
+#define smp_mb__after_atomic_inc()	membar("#StoreLoad | #StoreStore")
+#else
 #define smp_mb__before_atomic_dec()	barrier()
 #define smp_mb__after_atomic_dec()	barrier()
 #define smp_mb__before_atomic_inc()	barrier()
 #define smp_mb__after_atomic_inc()	barrier()
+#endif
 
 #endif /* !(__ARCH_SPARC64_ATOMIC__) */

include/asm-sparc64/bitops.h

@@ -7,19 +7,16 @@
 #ifndef _SPARC64_BITOPS_H
 #define _SPARC64_BITOPS_H
 
+#include <linux/config.h>
 #include <linux/compiler.h>
 #include <asm/byteorder.h>
 
-extern long ___test_and_set_bit(unsigned long nr, volatile unsigned long *addr);
-extern long ___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr);
-extern long ___test_and_change_bit(unsigned long nr, volatile unsigned long *addr);
-
-#define test_and_set_bit(nr,addr)	({___test_and_set_bit(nr,addr)!=0;})
-#define test_and_clear_bit(nr,addr)	({___test_and_clear_bit(nr,addr)!=0;})
-#define test_and_change_bit(nr,addr)	({___test_and_change_bit(nr,addr)!=0;})
-#define set_bit(nr,addr)		((void)___test_and_set_bit(nr,addr))
-#define clear_bit(nr,addr)		((void)___test_and_clear_bit(nr,addr))
-#define change_bit(nr,addr)		((void)___test_and_change_bit(nr,addr))
+extern int test_and_set_bit(unsigned long nr, volatile unsigned long *addr);
+extern int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr);
+extern int test_and_change_bit(unsigned long nr, volatile unsigned long *addr);
+extern void set_bit(unsigned long nr, volatile unsigned long *addr);
+extern void clear_bit(unsigned long nr, volatile unsigned long *addr);
+extern void change_bit(unsigned long nr, volatile unsigned long *addr);
 
 /* "non-atomic" versions... */
 
@@ -74,8 +71,13 @@ static __inline__ int __test_and_change_bit(int nr, volatile unsigned long *addr
 	return ((old & mask) != 0);
 }
 
-#define smp_mb__before_clear_bit()	do { } while(0)
-#define smp_mb__after_clear_bit()	do { } while(0)
+#ifdef CONFIG_SMP
+#define smp_mb__before_clear_bit()	membar("#StoreLoad | #LoadLoad")
+#define smp_mb__after_clear_bit()	membar("#StoreLoad | #StoreStore")
+#else
+#define smp_mb__before_clear_bit()	barrier()
+#define smp_mb__after_clear_bit()	barrier()
+#endif
 
 static __inline__ int test_bit(int nr, __const__ volatile unsigned long *addr)
 {
@@ -230,9 +232,9 @@ extern unsigned long find_next_zero_bit(const unsigned long *,
         find_next_zero_bit((addr), (size), 0)
 
 #define test_and_set_le_bit(nr,addr)	\
-	({ ___test_and_set_bit((nr) ^ 0x38, (addr)) != 0; })
+	test_and_set_bit((nr) ^ 0x38, (addr))
 #define test_and_clear_le_bit(nr,addr)	\
-	({ ___test_and_clear_bit((nr) ^ 0x38, (addr)) != 0; })
+	test_and_clear_bit((nr) ^ 0x38, (addr))
 
 static __inline__ int test_le_bit(int nr, __const__ unsigned long * addr)
 {
@@ -251,12 +253,21 @@ extern unsigned long find_next_zero_le_bit(unsigned long *, unsigned long, unsig
 
 #ifdef __KERNEL__
 
+#define __set_le_bit(nr, addr) \
+	__set_bit((nr) ^ 0x38, (addr))
+#define __clear_le_bit(nr, addr) \
+	__clear_bit((nr) ^ 0x38, (addr))
+#define __test_and_clear_le_bit(nr, addr) \
+	__test_and_clear_bit((nr) ^ 0x38, (addr))
+#define __test_and_set_le_bit(nr, addr) \
+	__test_and_set_bit((nr) ^ 0x38, (addr))
+
 #define ext2_set_bit(nr,addr)	\
-	test_and_set_le_bit((nr),(unsigned long *)(addr))
+	__test_and_set_le_bit((nr),(unsigned long *)(addr))
 #define ext2_set_bit_atomic(lock,nr,addr) \
 	test_and_set_le_bit((nr),(unsigned long *)(addr))
 #define ext2_clear_bit(nr,addr)	\
-	test_and_clear_le_bit((nr),(unsigned long *)(addr))
+	__test_and_clear_le_bit((nr),(unsigned long *)(addr))
 #define ext2_clear_bit_atomic(lock,nr,addr) \
 	test_and_clear_le_bit((nr),(unsigned long *)(addr))
 #define ext2_test_bit(nr,addr)	\