Commit 81368f8b authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull RCU fix from Ingo Molnar:
 "A boot hang fix for the offloaded callback RCU model (RCU_NOCB_CPU=y
  && (TREE_CPU=y || TREE_PREEMPT_RC)) in certain bootup scenarios"

* 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  rcu: Make nocb leader kthreads process pending callbacks after spawning
parents ebc54f27 651bc1a4
...@@ -358,7 +358,7 @@ struct rcu_data { ...@@ -358,7 +358,7 @@ struct rcu_data {
struct rcu_head **nocb_gp_tail; struct rcu_head **nocb_gp_tail;
long nocb_gp_count; long nocb_gp_count;
long nocb_gp_count_lazy; long nocb_gp_count_lazy;
bool nocb_leader_wake; /* Is the nocb leader thread awake? */ bool nocb_leader_sleep; /* Is the nocb leader thread asleep? */
struct rcu_data *nocb_next_follower; struct rcu_data *nocb_next_follower;
/* Next follower in wakeup chain. */ /* Next follower in wakeup chain. */
......
...@@ -2074,9 +2074,9 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force) ...@@ -2074,9 +2074,9 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force)
if (!ACCESS_ONCE(rdp_leader->nocb_kthread)) if (!ACCESS_ONCE(rdp_leader->nocb_kthread))
return; return;
if (!ACCESS_ONCE(rdp_leader->nocb_leader_wake) || force) { if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) {
/* Prior xchg orders against prior callback enqueue. */ /* Prior xchg orders against prior callback enqueue. */
ACCESS_ONCE(rdp_leader->nocb_leader_wake) = true; ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false;
wake_up(&rdp_leader->nocb_wq); wake_up(&rdp_leader->nocb_wq);
} }
} }
...@@ -2253,7 +2253,7 @@ static void nocb_leader_wait(struct rcu_data *my_rdp) ...@@ -2253,7 +2253,7 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
if (!rcu_nocb_poll) { if (!rcu_nocb_poll) {
trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep"); trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
wait_event_interruptible(my_rdp->nocb_wq, wait_event_interruptible(my_rdp->nocb_wq,
ACCESS_ONCE(my_rdp->nocb_leader_wake)); !ACCESS_ONCE(my_rdp->nocb_leader_sleep));
/* Memory barrier handled by smp_mb() calls below and repoll. */ /* Memory barrier handled by smp_mb() calls below and repoll. */
} else if (firsttime) { } else if (firsttime) {
firsttime = false; /* Don't drown trace log with "Poll"! */ firsttime = false; /* Don't drown trace log with "Poll"! */
...@@ -2292,12 +2292,12 @@ static void nocb_leader_wait(struct rcu_data *my_rdp) ...@@ -2292,12 +2292,12 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
schedule_timeout_interruptible(1); schedule_timeout_interruptible(1);
/* Rescan in case we were a victim of memory ordering. */ /* Rescan in case we were a victim of memory ordering. */
my_rdp->nocb_leader_wake = false; my_rdp->nocb_leader_sleep = true;
smp_mb(); /* Ensure _wake false before scan. */ smp_mb(); /* Ensure _sleep true before scan. */
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
if (ACCESS_ONCE(rdp->nocb_head)) { if (ACCESS_ONCE(rdp->nocb_head)) {
/* Found CB, so short-circuit next wait. */ /* Found CB, so short-circuit next wait. */
my_rdp->nocb_leader_wake = true; my_rdp->nocb_leader_sleep = false;
break; break;
} }
goto wait_again; goto wait_again;
...@@ -2307,17 +2307,17 @@ static void nocb_leader_wait(struct rcu_data *my_rdp) ...@@ -2307,17 +2307,17 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
rcu_nocb_wait_gp(my_rdp); rcu_nocb_wait_gp(my_rdp);
/* /*
* We left ->nocb_leader_wake set to reduce cache thrashing. * We left ->nocb_leader_sleep unset to reduce cache thrashing.
* We clear it now, but recheck for new callbacks while * We set it now, but recheck for new callbacks while
* traversing our follower list. * traversing our follower list.
*/ */
my_rdp->nocb_leader_wake = false; my_rdp->nocb_leader_sleep = true;
smp_mb(); /* Ensure _wake false before scan of ->nocb_head. */ smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */
/* Each pass through the following loop wakes a follower, if needed. */ /* Each pass through the following loop wakes a follower, if needed. */
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
if (ACCESS_ONCE(rdp->nocb_head)) if (ACCESS_ONCE(rdp->nocb_head))
my_rdp->nocb_leader_wake = true; /* No need to wait. */ my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
if (!rdp->nocb_gp_head) if (!rdp->nocb_gp_head)
continue; /* No CBs, so no need to wake follower. */ continue; /* No CBs, so no need to wake follower. */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment