[PATCH] RCU helper patchset 1/2

This first RCU helper patch adds a read_barrier_depends() primitive
to all archs which is NOP for archs that doesn't require an rmb()
for data dependent reads when writes are ordered using a wmb().
In reality, only alpha requires an rmb(), the rest are NOPs.
It is likely to be necessary in most situations that would use RCU.
Please apply.

 Description :

Sometime ago, during a discussion on lock-free lookups, it was
agreed that an additional memory barrier interface,
read_barrier_depends() that is lighter than an rmb(),
is necessary to make sure that data-dependent reads are not
re-ordered over this barrier. For many processors, data-dependency
enforces order, so this interface is a NOP, but for those that don't
(like alpha), it can be a rmb().

For example, the following code would force ordering (the initial
value of "a" is zero, "b" is one, and "p" is "&a"):

    CPU 0                           CPU 1

    b = 2;
    memory_barrier();
    p = &b;                         q = p;
                                    read_barrier_depends();
                                    d = *q;

because the read of "*q" depends on the read of "p" and these
two reads should be separated by a read_barrier_depends().  However,
the following code, with the same initial values for "a" and "b":

    CPU 0                           CPU 1

    a = 2;
    memory_barrier();
    b = 3;                          y = b;
                                    read_barrier_depends();
                                    x = a;

does not enforce ordering, since there is no data dependency between
the read of "a" and the read of "b".  Therefore, on some CPUs, such
as Alpha, "y" could be set to 3 and "x" to 0.  rmb()
needs to be used here, not read_barrier_depends().

The original discussion can be found at -
http://marc.theaimsgroup.com/?t=100259422200002&r=1&w=2

Explanation of the need for read_barrier_depends()
can be found at http://lse.sf.net/locking/wmbdd.html

include/asm-alpha/system.h

@@ -151,14 +151,19 @@ __asm__ __volatile__("mb": : :"memory")
 #define wmb() \
 __asm__ __volatile__("wmb": : :"memory")
 
+#define read_barrier_depends() \
+__asm__ __volatile__("mb": : :"memory")
+
 #ifdef CONFIG_SMP
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_read_barrier_depends()	read_barrier_depends()
 #else
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
+#define smp_read_barrier_depends()	barrier()
 #endif
 
 #define set_mb(var, value) \

include/asm-arm/system.h

@@ -52,6 +52,7 @@ extern asmlinkage void __backtrace(void);
 #define mb() __asm__ __volatile__ ("" : : : "memory")
 #define rmb() mb()
 #define wmb() mb()
+#define read_barrier_depends() do { } while(0)
 #define nop() __asm__ __volatile__("mov\tr0,r0\t@ nop\n\t");
 
 #define prepare_to_switch()    do { } while(0)
@@ -76,12 +77,14 @@ extern struct task_struct *__switch_to(struct thread_info *, struct thread_info
 #define smp_mb()		mb()
 #define smp_rmb()		rmb()
 #define smp_wmb()		wmb()
+#define smp_read_barrier_depends()		read_barrier_depends()
 
 #else
 
 #define smp_mb()		barrier()
 #define smp_rmb()		barrier()
 #define smp_wmb()		barrier()
+#define smp_read_barrier_depends()		do { } while(0)
 
 #define clf()			__clf()
 #define stf()			__stf()

include/asm-cris/system.h

@@ -150,15 +150,18 @@ static inline unsigned long __xchg(unsigned long x, void * ptr, int size)
 #define mb() __asm__ __volatile__ ("" : : : "memory")
 #define rmb() mb()
 #define wmb() mb()
+#define read_barrier_depends() do { } while(0)
 
 #ifdef CONFIG_SMP
 #define smp_mb()        mb()
 #define smp_rmb()       rmb()
 #define smp_wmb()       wmb()
+#define smp_read_barrier_depends()     read_barrier_depends()
 #else
 #define smp_mb()        barrier()
 #define smp_rmb()       barrier()
 #define smp_wmb()       barrier()
+#define smp_read_barrier_depends()     do { } while(0)
 #endif
 
 #define iret()

include/asm-i386/system.h

@@ -286,6 +286,60 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 #define mb() 	__asm__ __volatile__ ("lock; addl $0,0(%%esp)": : :"memory")
 #define rmb()	mb()
 
+/**
+ * read_barrier_depends - Flush all pending reads that subsequents reads
+ * depend on.
+ *
+ * No data-dependent reads from memory-like regions are ever reordered
+ * over this barrier.  All reads preceding this primitive are guaranteed
+ * to access memory (but not necessarily other CPUs' caches) before any
+ * reads following this primitive that depend on the data return by
+ * any of the preceding reads.  This primitive is much lighter weight than
+ * rmb() on most CPUs, and is never heavier weight than is
+ * rmb().
+ *
+ * These ordering constraints are respected by both the local CPU
+ * and the compiler.
+ *
+ * Ordering is not guaranteed by anything other than these primitives,
+ * not even by data dependencies.  See the documentation for
+ * memory_barrier() for examples and URLs to more information.
+ *
+ * For example, the following code would force ordering (the initial
+ * value of "a" is zero, "b" is one, and "p" is "&a"):
+ *
+ * <programlisting>
+ *	CPU 0				CPU 1
+ *
+ *	b = 2;
+ *	memory_barrier();
+ *	p = &b;				q = p;
+ *					read_barrier_depends();
+ *					d = *q;
+ * </programlisting>
+ *
+ * because the read of "*q" depends on the read of "p" and these
+ * two reads are separated by a read_barrier_depends().  However,
+ * the following code, with the same initial values for "a" and "b":
+ *
+ * <programlisting>
+ *	CPU 0				CPU 1
+ *
+ *	a = 2;
+ *	memory_barrier();
+ *	b = 3;				y = b;
+ *					read_barrier_depends();
+ *					x = a;
+ * </programlisting>
+ *
+ * does not enforce ordering, since there is no data dependency between
+ * the read of "a" and the read of "b".  Therefore, on some CPUs, such
+ * as Alpha, "y" could be set to 3 and "x" to 0.  Use rmb()
+ * in cases like thiswhere there are no data dependencies.
+ **/
+
+#define read_barrier_depends()	do { } while(0)
+
 #ifdef CONFIG_X86_OOSTORE
 #define wmb() 	__asm__ __volatile__ ("lock; addl $0,0(%%esp)": : :"memory")
 #else
@@ -296,10 +350,12 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_read_barrier_depends()	read_barrier_depends()
 #else
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
+#define smp_read_barrier_depends()	do { } while(0)
 #endif
 
 #define set_mb(var, value) do { xchg(&var, value); } while (0)

include/asm-ia64/system.h

@@ -85,15 +85,18 @@ ia64_insn_group_barrier (void)
 #define mb()	__asm__ __volatile__ ("mf" ::: "memory")
 #define rmb()	mb()
 #define wmb()	mb()
+#define read_barrier_depends()	do { } while(0)
 
 #ifdef CONFIG_SMP
 # define smp_mb()	mb()
 # define smp_rmb()	rmb()
 # define smp_wmb()	wmb()
+# define smp_read_barrier_depends()	read_barrier_depends()
 #else
 # define smp_mb()	barrier()
 # define smp_rmb()	barrier()
 # define smp_wmb()	barrier()
+# define smp_read_barrier_depends()	do { } while(0)
 #endif
 
 /*

include/asm-mips/system.h

@@ -146,6 +146,7 @@ extern void __global_restore_flags(unsigned long);
 #define rmb()	do { } while(0)
 #define wmb()	wbflush()
 #define mb()	wbflush()
+#define read_barrier_depends()	do { } while(0)
 
 #else /* CONFIG_CPU_HAS_WB  */
 
@@ -161,6 +162,7 @@ __asm__ __volatile__(					\
 	: "memory")
 #define rmb() mb()
 #define wmb() mb()
+#define read_barrier_depends()	do { } while(0)
 
 #endif /* CONFIG_CPU_HAS_WB  */
 
@@ -168,10 +170,12 @@ __asm__ __volatile__(					\
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_read_barrier_depends()	read_barrier_depends()
 #else
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
+#define smp_read_barrier_depends()	do { } while(0)
 #endif
 
 #define set_mb(var, value) \

include/asm-mips64/system.h

@@ -142,15 +142,18 @@ __asm__ __volatile__(					\
 	: "memory")
 #define rmb() mb()
 #define wmb() mb()
+#define read_barrier_depends()	do { } while(0)
 
 #ifdef CONFIG_SMP
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_read_barrier_depends()	read_barrier_depends()
 #else
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
+#define smp_read_barrier_depends()	do { } while(0)
 #endif
 
 #define set_mb(var, value) \

include/asm-parisc/system.h

@@ -51,6 +51,7 @@ extern struct task_struct *_switch_to(struct task_struct *, struct task_struct *
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_read_barrier_depends()	do { } while(0)
 #else
 /* This is simply the barrier() macro from linux/kernel.h but when serial.c
  * uses tqueue.h uses smp_mb() defined using barrier(), linux/kernel.h
@@ -59,6 +60,7 @@ extern struct task_struct *_switch_to(struct task_struct *, struct task_struct *
 #define smp_mb()	__asm__ __volatile__("":::"memory");
 #define smp_rmb()	__asm__ __volatile__("":::"memory");
 #define smp_wmb()	__asm__ __volatile__("":::"memory");
+#define smp_read_barrier_depends()	do { } while(0)
 #endif
 
 /* interrupt control */
@@ -120,6 +122,7 @@ static inline void set_eiem(unsigned long val)
 
 #define mb()  __asm__ __volatile__ ("sync" : : :"memory")
 #define wmb() mb()
+#define read_barrier_depends()	do { } while(0)
 
 extern unsigned long __xchg(unsigned long, unsigned long *, int);
 

include/asm-ppc/system.h

@@ -22,6 +22,8 @@
  * mb() prevents loads and stores being reordered across this point.
  * rmb() prevents loads being reordered across this point.
  * wmb() prevents stores being reordered across this point.
+ * read_barrier_depends() prevents data-dependant loads being reordered
+ *	across this point (nop on PPC).
  *
  * We can use the eieio instruction for wmb, but since it doesn't
  * give any ordering guarantees about loads, we have to use the
@@ -30,6 +32,7 @@
 #define mb()  __asm__ __volatile__ ("sync" : : : "memory")
 #define rmb()  __asm__ __volatile__ ("sync" : : : "memory")
 #define wmb()  __asm__ __volatile__ ("eieio" : : : "memory")
+#define read_barrier_depends()  do { } while(0)
 
 #define set_mb(var, value)	do { var = value; mb(); } while (0)
 #define set_wmb(var, value)	do { var = value; wmb(); } while (0)
@@ -38,10 +41,12 @@
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_read_barrier_depends()	read_barrier_depends()
 #else
 #define smp_mb()	__asm__ __volatile__("": : :"memory")
 #define smp_rmb()	__asm__ __volatile__("": : :"memory")
 #define smp_wmb()	__asm__ __volatile__("": : :"memory")
+#define smp_read_barrier_depends()	do { } while(0)
 #endif /* CONFIG_SMP */
 
 #ifdef __KERNEL__

include/asm-ppc64/system.h

@@ -25,6 +25,8 @@
  * mb() prevents loads and stores being reordered across this point.
  * rmb() prevents loads being reordered across this point.
  * wmb() prevents stores being reordered across this point.
+ * read_barrier_depends() prevents data-dependant loads being reordered
+ *	across this point (nop on PPC).
  *
  * We can use the eieio instruction for wmb, but since it doesn't
  * give any ordering guarantees about loads, we have to use the
@@ -33,6 +35,7 @@
 #define mb()   __asm__ __volatile__ ("sync" : : : "memory")
 #define rmb()  __asm__ __volatile__ ("lwsync" : : : "memory")
 #define wmb()  __asm__ __volatile__ ("eieio" : : : "memory")
+#define read_barrier_depends()  do { } while(0)
 
 #define set_mb(var, value)	do { var = value; mb(); } while (0)
 #define set_wmb(var, value)	do { var = value; wmb(); } while (0)
@@ -41,10 +44,12 @@
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_read_barrier_depends()  read_barrier_depends()
 #else
 #define smp_mb()	__asm__ __volatile__("": : :"memory")
 #define smp_rmb()	__asm__ __volatile__("": : :"memory")
 #define smp_wmb()	__asm__ __volatile__("": : :"memory")
+#define smp_read_barrier_depends()  do { } while(0)
 #endif /* CONFIG_SMP */
 
 #ifdef CONFIG_DEBUG_KERNEL

include/asm-s390/system.h

@@ -227,9 +227,11 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
 #define mb()    eieio()
 #define rmb()   eieio()
 #define wmb()   eieio()
+#define read_barrier_depends() do { } while(0)
 #define smp_mb()       mb()
 #define smp_rmb()      rmb()
 #define smp_wmb()      wmb()
+#define smp_read_barrier_depends()    read_barrier_depends()
 #define smp_mb__before_clear_bit()     smp_mb()
 #define smp_mb__after_clear_bit()      smp_mb()
 

include/asm-s390x/system.h

@@ -238,9 +238,11 @@ __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size)
 #define mb()    eieio()
 #define rmb()   eieio()
 #define wmb()   eieio()
+#define read_barrier_depends()	do { } while(0)
 #define smp_mb()       mb()
 #define smp_rmb()      rmb()
 #define smp_wmb()      wmb()
+#define smp_read_barrier_depends()    read_barrier_depends()
 #define smp_mb__before_clear_bit()     smp_mb()
 #define smp_mb__after_clear_bit()      smp_mb()
 

include/asm-sh/system.h

@@ -89,15 +89,18 @@ extern void __xchg_called_with_bad_pointer(void);
 #define mb()	__asm__ __volatile__ ("": : :"memory")
 #define rmb()	mb()
 #define wmb()	__asm__ __volatile__ ("": : :"memory")
+#define read_barrier_depends()	do { } while(0)
 
 #ifdef CONFIG_SMP
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_read_barrier_depends()	read_barrier_depends()
 #else
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
+#define smp_read_barrier_depends()	do { } while(0)
 #endif
 
 #define set_mb(var, value) do { xchg(&var, value); } while (0)

include/asm-sparc/system.h

@@ -310,11 +310,13 @@ extern void __global_restore_flags(unsigned long flags);
 #define mb()	__asm__ __volatile__ ("" : : : "memory")
 #define rmb()	mb()
 #define wmb()	mb()
+#define read_barrier_depends()	do { } while(0)
 #define set_mb(__var, __value)  do { __var = __value; mb(); } while(0)
 #define set_wmb(__var, __value) set_mb(__var, __value)
 #define smp_mb()	__asm__ __volatile__("":::"memory");
 #define smp_rmb()	__asm__ __volatile__("":::"memory");
 #define smp_wmb()	__asm__ __volatile__("":::"memory");
+#define smp_read_barrier_depends()	do { } while(0)
 
 #define nop() __asm__ __volatile__ ("nop");
 

include/asm-x86_64/system.h

@@ -215,10 +215,12 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 #define smp_mb()	mb()
 #define smp_rmb()	rmb()
 #define smp_wmb()	wmb()
+#define smp_read_barrier_depends()	do {} while(0)
 #else
 #define smp_mb()	barrier()
 #define smp_rmb()	barrier()
 #define smp_wmb()	barrier()
+#define smp_read_barrier_depends()	do {} while(0)
 #endif
 
     
@@ -230,6 +232,7 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
 #define mb() 	asm volatile("mfence":::"memory")
 #define rmb()	asm volatile("lfence":::"memory")
 #define wmb()	asm volatile("sfence":::"memory")
+#define read_barrier_depends()	do {} while(0)
 #define set_mb(var, value) do { xchg(&var, value); } while (0)
 #define set_wmb(var, value) do { var = value; wmb(); } while (0)