Commit 5293c2ef authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Thomas Gleixner

futex,rt_mutex: Provide futex specific rt_mutex API

Part of what makes futex_unlock_pi() intricate is that
rt_mutex_futex_unlock() -> rt_mutex_slowunlock() can drop
rt_mutex::wait_lock.

This means it cannot rely on the atomicy of wait_lock, which would be
preferred in order to not rely on hb->lock so much.

The reason rt_mutex_slowunlock() needs to drop wait_lock is because it can
race with the rt_mutex fastpath, however futexes have their own fast path.

Since futexes already have a bunch of separate rt_mutex accessors, complete
that set and implement a rt_mutex variant without fastpath for them.
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Cc: juri.lelli@arm.com
Cc: bigeasy@linutronix.de
Cc: xlpang@redhat.com
Cc: rostedt@goodmis.org
Cc: mathieu.desnoyers@efficios.com
Cc: jdesfossez@efficios.com
Cc: dvhart@infradead.org
Cc: bristot@redhat.com
Link: http://lkml.kernel.org/r/20170322104151.702962446@infradead.orgSigned-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
parent fffa954f
...@@ -916,7 +916,7 @@ void exit_pi_state_list(struct task_struct *curr) ...@@ -916,7 +916,7 @@ void exit_pi_state_list(struct task_struct *curr)
pi_state->owner = NULL; pi_state->owner = NULL;
raw_spin_unlock_irq(&curr->pi_lock); raw_spin_unlock_irq(&curr->pi_lock);
rt_mutex_unlock(&pi_state->pi_mutex); rt_mutex_futex_unlock(&pi_state->pi_mutex);
spin_unlock(&hb->lock); spin_unlock(&hb->lock);
...@@ -1364,20 +1364,18 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *top_waiter ...@@ -1364,20 +1364,18 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *top_waiter
pi_state->owner = new_owner; pi_state->owner = new_owner;
raw_spin_unlock(&new_owner->pi_lock); raw_spin_unlock(&new_owner->pi_lock);
raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
/* /*
* First unlock HB so the waiter does not spin on it once he got woken * We've updated the uservalue, this unlock cannot fail.
* up. Second wake up the waiter before the priority is adjusted. If we
* deboost first (and lose our higher priority), then the task might get
* scheduled away before the wake up can take place.
*/ */
deboost = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
spin_unlock(&hb->lock); spin_unlock(&hb->lock);
wake_up_q(&wake_q);
if (deboost) if (deboost) {
wake_up_q(&wake_q);
rt_mutex_adjust_prio(current); rt_mutex_adjust_prio(current);
}
return 0; return 0;
} }
...@@ -2253,7 +2251,7 @@ static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked) ...@@ -2253,7 +2251,7 @@ static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked)
* task acquired the rt_mutex after we removed ourself from the * task acquired the rt_mutex after we removed ourself from the
* rt_mutex waiters list. * rt_mutex waiters list.
*/ */
if (rt_mutex_trylock(&q->pi_state->pi_mutex)) { if (rt_mutex_futex_trylock(&q->pi_state->pi_mutex)) {
locked = 1; locked = 1;
goto out; goto out;
} }
...@@ -2568,7 +2566,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ...@@ -2568,7 +2566,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,
if (!trylock) { if (!trylock) {
ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to); ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);
} else { } else {
ret = rt_mutex_trylock(&q.pi_state->pi_mutex); ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
/* Fixup the trylock return value: */ /* Fixup the trylock return value: */
ret = ret ? 0 : -EWOULDBLOCK; ret = ret ? 0 : -EWOULDBLOCK;
} }
...@@ -2591,7 +2589,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, ...@@ -2591,7 +2589,7 @@ static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,
* it and return the fault to userspace. * it and return the fault to userspace.
*/ */
if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current)) if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current))
rt_mutex_unlock(&q.pi_state->pi_mutex); rt_mutex_futex_unlock(&q.pi_state->pi_mutex);
/* Unqueue and drop the lock */ /* Unqueue and drop the lock */
unqueue_me_pi(&q); unqueue_me_pi(&q);
...@@ -2898,7 +2896,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, ...@@ -2898,7 +2896,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
spin_lock(q.lock_ptr); spin_lock(q.lock_ptr);
ret = fixup_pi_state_owner(uaddr2, &q, current); ret = fixup_pi_state_owner(uaddr2, &q, current);
if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current)
rt_mutex_unlock(&q.pi_state->pi_mutex); rt_mutex_futex_unlock(&q.pi_state->pi_mutex);
/* /*
* Drop the reference to the pi state which * Drop the reference to the pi state which
* the requeue_pi() code acquired for us. * the requeue_pi() code acquired for us.
...@@ -2938,7 +2936,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags, ...@@ -2938,7 +2936,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
* userspace. * userspace.
*/ */
if (ret && rt_mutex_owner(pi_mutex) == current) if (ret && rt_mutex_owner(pi_mutex) == current)
rt_mutex_unlock(pi_mutex); rt_mutex_futex_unlock(pi_mutex);
/* Unqueue and drop the lock. */ /* Unqueue and drop the lock. */
unqueue_me_pi(&q); unqueue_me_pi(&q);
......
...@@ -1488,15 +1488,23 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); ...@@ -1488,15 +1488,23 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
/* /*
* Futex variant with full deadlock detection. * Futex variant with full deadlock detection.
* Futex variants must not use the fast-path, see __rt_mutex_futex_unlock().
*/ */
int rt_mutex_timed_futex_lock(struct rt_mutex *lock, int __sched rt_mutex_timed_futex_lock(struct rt_mutex *lock,
struct hrtimer_sleeper *timeout) struct hrtimer_sleeper *timeout)
{ {
might_sleep(); might_sleep();
return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, return rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE,
RT_MUTEX_FULL_CHAINWALK, timeout, RT_MUTEX_FULL_CHAINWALK);
rt_mutex_slowlock); }
/*
* Futex variant, must not use fastpath.
*/
int __sched rt_mutex_futex_trylock(struct rt_mutex *lock)
{
return rt_mutex_slowtrylock(lock);
} }
/** /**
...@@ -1555,19 +1563,38 @@ void __sched rt_mutex_unlock(struct rt_mutex *lock) ...@@ -1555,19 +1563,38 @@ void __sched rt_mutex_unlock(struct rt_mutex *lock)
EXPORT_SYMBOL_GPL(rt_mutex_unlock); EXPORT_SYMBOL_GPL(rt_mutex_unlock);
/** /**
* rt_mutex_futex_unlock - Futex variant of rt_mutex_unlock * Futex variant, that since futex variants do not use the fast-path, can be
* @lock: the rt_mutex to be unlocked * simple and will not need to retry.
*
* Returns: true/false indicating whether priority adjustment is
* required or not.
*/ */
bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock, bool __sched __rt_mutex_futex_unlock(struct rt_mutex *lock,
struct wake_q_head *wqh) struct wake_q_head *wake_q)
{ {
if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) lockdep_assert_held(&lock->wait_lock);
return false;
debug_rt_mutex_unlock(lock);
if (!rt_mutex_has_waiters(lock)) {
lock->owner = NULL;
return false; /* done */
}
mark_wakeup_next_waiter(wake_q, lock);
return true; /* deboost and wakeups */
}
return rt_mutex_slowunlock(lock, wqh); void __sched rt_mutex_futex_unlock(struct rt_mutex *lock)
{
DEFINE_WAKE_Q(wake_q);
bool deboost;
raw_spin_lock_irq(&lock->wait_lock);
deboost = __rt_mutex_futex_unlock(lock, &wake_q);
raw_spin_unlock_irq(&lock->wait_lock);
if (deboost) {
wake_up_q(&wake_q);
rt_mutex_adjust_prio(current);
}
} }
/** /**
......
...@@ -109,9 +109,14 @@ extern int rt_mutex_start_proxy_lock(struct rt_mutex *lock, ...@@ -109,9 +109,14 @@ extern int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
extern int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, extern int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
struct hrtimer_sleeper *to, struct hrtimer_sleeper *to,
struct rt_mutex_waiter *waiter); struct rt_mutex_waiter *waiter);
extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to); extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to);
extern bool rt_mutex_futex_unlock(struct rt_mutex *lock, extern int rt_mutex_futex_trylock(struct rt_mutex *l);
struct wake_q_head *wqh);
extern void rt_mutex_futex_unlock(struct rt_mutex *lock);
extern bool __rt_mutex_futex_unlock(struct rt_mutex *lock,
struct wake_q_head *wqh);
extern void rt_mutex_adjust_prio(struct task_struct *task); extern void rt_mutex_adjust_prio(struct task_struct *task);
#ifdef CONFIG_DEBUG_RT_MUTEXES #ifdef CONFIG_DEBUG_RT_MUTEXES
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment