Commit 5c1eabe6 authored by Mikulas Patocka's avatar Mikulas Patocka Committed by Linus Torvalds

percpu-rw-semaphores: use light/heavy barriers

This patch introduces new barrier pair light_mb() and heavy_mb() for
percpu rw semaphores.

This patch fixes a bug in percpu-rw-semaphores where a barrier was
missing in percpu_up_write.

This patch improves performance on the read path of
percpu-rw-semaphores: on non-x86 cpus, there was a smp_mb() in
percpu_up_read. This patch changes it to a compiler barrier and removes
the "#if defined(X86) ..." condition.

From: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: default avatarMikulas Patocka <mpatocka@redhat.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent e657e078
...@@ -12,6 +12,9 @@ struct percpu_rw_semaphore { ...@@ -12,6 +12,9 @@ struct percpu_rw_semaphore {
struct mutex mtx; struct mutex mtx;
}; };
#define light_mb() barrier()
#define heavy_mb() synchronize_sched()
static inline void percpu_down_read(struct percpu_rw_semaphore *p) static inline void percpu_down_read(struct percpu_rw_semaphore *p)
{ {
rcu_read_lock(); rcu_read_lock();
...@@ -24,22 +27,12 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *p) ...@@ -24,22 +27,12 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *p)
} }
this_cpu_inc(*p->counters); this_cpu_inc(*p->counters);
rcu_read_unlock(); rcu_read_unlock();
light_mb(); /* A, between read of p->locked and read of data, paired with D */
} }
static inline void percpu_up_read(struct percpu_rw_semaphore *p) static inline void percpu_up_read(struct percpu_rw_semaphore *p)
{ {
/* light_mb(); /* B, between read of the data and write to p->counter, paired with C */
* On X86, write operation in this_cpu_dec serves as a memory unlock
* barrier (i.e. memory accesses may be moved before the write, but
* no memory accesses are moved past the write).
* On other architectures this may not be the case, so we need smp_mb()
* there.
*/
#if defined(CONFIG_X86) && (!defined(CONFIG_X86_PPRO_FENCE) && !defined(CONFIG_X86_OOSTORE))
barrier();
#else
smp_mb();
#endif
this_cpu_dec(*p->counters); this_cpu_dec(*p->counters);
} }
...@@ -61,11 +54,12 @@ static inline void percpu_down_write(struct percpu_rw_semaphore *p) ...@@ -61,11 +54,12 @@ static inline void percpu_down_write(struct percpu_rw_semaphore *p)
synchronize_rcu(); synchronize_rcu();
while (__percpu_count(p->counters)) while (__percpu_count(p->counters))
msleep(1); msleep(1);
smp_rmb(); /* paired with smp_mb() in percpu_sem_up_read() */ heavy_mb(); /* C, between read of p->counter and write to data, paired with B */
} }
static inline void percpu_up_write(struct percpu_rw_semaphore *p) static inline void percpu_up_write(struct percpu_rw_semaphore *p)
{ {
heavy_mb(); /* D, between write to data and write to p->locked, paired with A */
p->locked = false; p->locked = false;
mutex_unlock(&p->mtx); mutex_unlock(&p->mtx);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment