Commit a31a369b authored by Michel Lespinasse's avatar Michel Lespinasse Committed by Linus Torvalds

x86 rwsem: avoid taking slow path when stealing write lock

modify __down_write[_nested] and __down_write_trylock to grab the write
lock whenever the active count is 0, even if there are queued waiters
(they must be writers pending wakeup, since the active count is 0).

Note that this is an optimization only; architectures without this
optimization will still work fine:

- __down_write() would take the slow path which would take the wait_lock
  and then try stealing the lock (as in the spinlocked rwsem implementation)

- __down_write_trylock() would fail, but callers must be ready to deal
  with that - since there are some writers pending wakeup, they could
  have raced with us and obtained the lock before we steal it.
Signed-off-by: default avatarMichel Lespinasse <walken@google.com>
Reviewed-by: default avatarPeter Hurley <peter@hurleysoftware.com>
Acked-by: default avatarDavidlohr Bueso <davidlohr.bueso@hp.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 25c39325
...@@ -105,8 +105,8 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) ...@@ -105,8 +105,8 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
asm volatile("# beginning down_write\n\t" asm volatile("# beginning down_write\n\t"
LOCK_PREFIX " xadd %1,(%2)\n\t" LOCK_PREFIX " xadd %1,(%2)\n\t"
/* adds 0xffff0001, returns the old value */ /* adds 0xffff0001, returns the old value */
" test %1,%1\n\t" " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t"
/* was the count 0 before? */ /* was the active mask 0 before? */
" jz 1f\n" " jz 1f\n"
" call call_rwsem_down_write_failed\n" " call call_rwsem_down_write_failed\n"
"1:\n" "1:\n"
...@@ -126,11 +126,25 @@ static inline void __down_write(struct rw_semaphore *sem) ...@@ -126,11 +126,25 @@ static inline void __down_write(struct rw_semaphore *sem)
*/ */
static inline int __down_write_trylock(struct rw_semaphore *sem) static inline int __down_write_trylock(struct rw_semaphore *sem)
{ {
long ret = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE, long result, tmp;
RWSEM_ACTIVE_WRITE_BIAS); asm volatile("# beginning __down_write_trylock\n\t"
if (ret == RWSEM_UNLOCKED_VALUE) " mov %0,%1\n\t"
return 1; "1:\n\t"
return 0; " test " __ASM_SEL(%w1,%k1) "," __ASM_SEL(%w1,%k1) "\n\t"
/* was the active mask 0 before? */
" jnz 2f\n\t"
" mov %1,%2\n\t"
" add %3,%2\n\t"
LOCK_PREFIX " cmpxchg %2,%0\n\t"
" jnz 1b\n\t"
"2:\n\t"
" sete %b1\n\t"
" movzbl %b1, %k1\n\t"
"# ending __down_write_trylock\n\t"
: "+m" (sem->count), "=&a" (result), "=&r" (tmp)
: "er" (RWSEM_ACTIVE_WRITE_BIAS)
: "memory", "cc");
return result;
} }
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment