Commit 11ed7f93 authored by Pranith Kumar's avatar Pranith Kumar Committed by Paul E. McKenney

rcu: Make nocb leader kthreads process pending callbacks after spawning

The nocb callbacks generated before the nocb kthreads are spawned are
enqueued in the nocb queue for later processing. Commit fbce7497 ("rcu:
Parallelize and economize NOCB kthread wakeups") introduced nocb leader kthreads
which checked the nocb_leader_wake flag to see if there were any such pending
callbacks. A case was reported in which newly spawned leader kthreads were not
processing the pending callbacks as this flag was not set, which led to a boot
hang.

The following commit ensures that the newly spawned nocb kthreads process the
pending callbacks by allowing the kthreads to run immediately after spawning
instead of waiting. This is done by inverting the logic of nocb_leader_wake
tests to nocb_leader_sleep which allows us to use the default initialization of
this flag to 0 to let the kthreads run.
Reported-by: default avatarAmit Shah <amit.shah@redhat.com>
Signed-off-by: default avatarPranith Kumar <bobby.prani@gmail.com>
Link: http://www.spinics.net/lists/kernel/msg1802899.html
[ paulmck: Backported to v3.17-rc2. ]
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: default avatarAmit Shah <amit.shah@redhat.com>
parent 52addcf9
...@@ -358,7 +358,7 @@ struct rcu_data { ...@@ -358,7 +358,7 @@ struct rcu_data {
struct rcu_head **nocb_gp_tail; struct rcu_head **nocb_gp_tail;
long nocb_gp_count; long nocb_gp_count;
long nocb_gp_count_lazy; long nocb_gp_count_lazy;
bool nocb_leader_wake; /* Is the nocb leader thread awake? */ bool nocb_leader_sleep; /* Is the nocb leader thread asleep? */
struct rcu_data *nocb_next_follower; struct rcu_data *nocb_next_follower;
/* Next follower in wakeup chain. */ /* Next follower in wakeup chain. */
......
...@@ -2074,9 +2074,9 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force) ...@@ -2074,9 +2074,9 @@ static void wake_nocb_leader(struct rcu_data *rdp, bool force)
if (!ACCESS_ONCE(rdp_leader->nocb_kthread)) if (!ACCESS_ONCE(rdp_leader->nocb_kthread))
return; return;
if (!ACCESS_ONCE(rdp_leader->nocb_leader_wake) || force) { if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) {
/* Prior xchg orders against prior callback enqueue. */ /* Prior xchg orders against prior callback enqueue. */
ACCESS_ONCE(rdp_leader->nocb_leader_wake) = true; ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false;
wake_up(&rdp_leader->nocb_wq); wake_up(&rdp_leader->nocb_wq);
} }
} }
...@@ -2253,7 +2253,7 @@ static void nocb_leader_wait(struct rcu_data *my_rdp) ...@@ -2253,7 +2253,7 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
if (!rcu_nocb_poll) { if (!rcu_nocb_poll) {
trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep"); trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep");
wait_event_interruptible(my_rdp->nocb_wq, wait_event_interruptible(my_rdp->nocb_wq,
ACCESS_ONCE(my_rdp->nocb_leader_wake)); !ACCESS_ONCE(my_rdp->nocb_leader_sleep));
/* Memory barrier handled by smp_mb() calls below and repoll. */ /* Memory barrier handled by smp_mb() calls below and repoll. */
} else if (firsttime) { } else if (firsttime) {
firsttime = false; /* Don't drown trace log with "Poll"! */ firsttime = false; /* Don't drown trace log with "Poll"! */
...@@ -2292,12 +2292,12 @@ static void nocb_leader_wait(struct rcu_data *my_rdp) ...@@ -2292,12 +2292,12 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
schedule_timeout_interruptible(1); schedule_timeout_interruptible(1);
/* Rescan in case we were a victim of memory ordering. */ /* Rescan in case we were a victim of memory ordering. */
my_rdp->nocb_leader_wake = false; my_rdp->nocb_leader_sleep = true;
smp_mb(); /* Ensure _wake false before scan. */ smp_mb(); /* Ensure _sleep true before scan. */
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower)
if (ACCESS_ONCE(rdp->nocb_head)) { if (ACCESS_ONCE(rdp->nocb_head)) {
/* Found CB, so short-circuit next wait. */ /* Found CB, so short-circuit next wait. */
my_rdp->nocb_leader_wake = true; my_rdp->nocb_leader_sleep = false;
break; break;
} }
goto wait_again; goto wait_again;
...@@ -2307,17 +2307,17 @@ static void nocb_leader_wait(struct rcu_data *my_rdp) ...@@ -2307,17 +2307,17 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
rcu_nocb_wait_gp(my_rdp); rcu_nocb_wait_gp(my_rdp);
/* /*
* We left ->nocb_leader_wake set to reduce cache thrashing. * We left ->nocb_leader_sleep unset to reduce cache thrashing.
* We clear it now, but recheck for new callbacks while * We set it now, but recheck for new callbacks while
* traversing our follower list. * traversing our follower list.
*/ */
my_rdp->nocb_leader_wake = false; my_rdp->nocb_leader_sleep = true;
smp_mb(); /* Ensure _wake false before scan of ->nocb_head. */ smp_mb(); /* Ensure _sleep true before scan of ->nocb_head. */
/* Each pass through the following loop wakes a follower, if needed. */ /* Each pass through the following loop wakes a follower, if needed. */
for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) { for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
if (ACCESS_ONCE(rdp->nocb_head)) if (ACCESS_ONCE(rdp->nocb_head))
my_rdp->nocb_leader_wake = true; /* No need to wait. */ my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
if (!rdp->nocb_gp_head) if (!rdp->nocb_gp_head)
continue; /* No CBs, so no need to wake follower. */ continue; /* No CBs, so no need to wake follower. */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment