Commit 74602315 authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar

sched: Fix race in migrate_swap_stop()

There is a subtle race in migrate_swap, when task P, on CPU A, decides to swap
places with task T, on CPU B.

Task P:
  - call migrate_swap
Task T:
  - go to sleep, removing itself from the runqueue
Task P:
  - double lock the runqueues on CPU A & B
Task T:
  - get woken up, place itself on the runqueue of CPU C
Task P:
  - see that task T is on a runqueue, and pretend to remove it
    from the runqueue on CPU B

Now CPUs B & C both have corrupted scheduler data structures.

This patch fixes it, by holding the pi_lock for both of the tasks
involved in the migrate swap. This prevents task T from waking up,
and placing itself onto another runqueue, until after migrate_swap
has released all locks.

This means that, when migrate_swap checks, task T will be either
on the runqueue where it was originally seen, or not on any
runqueue at all. Migrate_swap deals correctly with of those cases.
Tested-by: default avatarJoe Mario <jmario@redhat.com>
Acked-by: default avatarMel Gorman <mgorman@suse.de>
Reviewed-by: default avatarRik van Riel <riel@redhat.com>
Signed-off-by: default avatarPeter Zijlstra <peterz@infradead.org>
Cc: hannes@cmpxchg.org
Cc: aarcange@redhat.com
Cc: srikar@linux.vnet.ibm.com
Cc: tglx@linutronix.de
Cc: hpa@zytor.com
Link: http://lkml.kernel.org/r/20131010181722.GO13848@laptop.programming.kicks-ass.netSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 7c3f2ab7
...@@ -1049,6 +1049,8 @@ static int migrate_swap_stop(void *data) ...@@ -1049,6 +1049,8 @@ static int migrate_swap_stop(void *data)
src_rq = cpu_rq(arg->src_cpu); src_rq = cpu_rq(arg->src_cpu);
dst_rq = cpu_rq(arg->dst_cpu); dst_rq = cpu_rq(arg->dst_cpu);
double_raw_lock(&arg->src_task->pi_lock,
&arg->dst_task->pi_lock);
double_rq_lock(src_rq, dst_rq); double_rq_lock(src_rq, dst_rq);
if (task_cpu(arg->dst_task) != arg->dst_cpu) if (task_cpu(arg->dst_task) != arg->dst_cpu)
goto unlock; goto unlock;
...@@ -1069,6 +1071,8 @@ static int migrate_swap_stop(void *data) ...@@ -1069,6 +1071,8 @@ static int migrate_swap_stop(void *data)
unlock: unlock:
double_rq_unlock(src_rq, dst_rq); double_rq_unlock(src_rq, dst_rq);
raw_spin_unlock(&arg->dst_task->pi_lock);
raw_spin_unlock(&arg->src_task->pi_lock);
return ret; return ret;
} }
......
...@@ -1448,15 +1448,6 @@ static inline void put_numa_group(struct numa_group *grp) ...@@ -1448,15 +1448,6 @@ static inline void put_numa_group(struct numa_group *grp)
kfree_rcu(grp, rcu); kfree_rcu(grp, rcu);
} }
static void double_lock(spinlock_t *l1, spinlock_t *l2)
{
if (l1 > l2)
swap(l1, l2);
spin_lock(l1);
spin_lock_nested(l2, SINGLE_DEPTH_NESTING);
}
static void task_numa_group(struct task_struct *p, int cpupid, int flags, static void task_numa_group(struct task_struct *p, int cpupid, int flags,
int *priv) int *priv)
{ {
......
...@@ -1249,6 +1249,24 @@ static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) ...@@ -1249,6 +1249,24 @@ static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_); lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
} }
static inline void double_lock(spinlock_t *l1, spinlock_t *l2)
{
if (l1 > l2)
swap(l1, l2);
spin_lock(l1);
spin_lock_nested(l2, SINGLE_DEPTH_NESTING);
}
static inline void double_raw_lock(raw_spinlock_t *l1, raw_spinlock_t *l2)
{
if (l1 > l2)
swap(l1, l2);
raw_spin_lock(l1);
raw_spin_lock_nested(l2, SINGLE_DEPTH_NESTING);
}
/* /*
* double_rq_lock - safely lock two runqueues * double_rq_lock - safely lock two runqueues
* *
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment