Commit 281d150c authored by Paul E. McKenney's avatar Paul E. McKenney Committed by Ingo Molnar

rcu: Prepare for synchronization fixes: clean up for non-NO_HZ handling of ->completed counter

Impose a clear locking design on non-NO_HZ handling of the
->completed counter.  This increases the distance between the
RCU and the CPU-hotplug mechanisms.
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: laijs@cn.fujitsu.com
Cc: dipankar@in.ibm.com
Cc: mathieu.desnoyers@polymtl.ca
Cc: josh@joshtriplett.org
Cc: dvhltc@us.ibm.com
Cc: niv@us.ibm.com
Cc: peterz@infradead.org
Cc: rostedt@goodmis.org
Cc: Valdis.Kletnieks@vt.edu
Cc: dhowells@redhat.com
Cc: <stable@kernel.org> # .32.x
LKML-Reference: <12571987491353-git-send-email->
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 7e1a2766
...@@ -178,8 +178,28 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp) ...@@ -178,8 +178,28 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp)
return &rsp->node[0]; return &rsp->node[0];
} }
/*
* Record the specified "completed" value, which is later used to validate
* dynticks counter manipulations and CPU-offline checks. Specify
* "rsp->completed - 1" to unconditionally invalidate any future dynticks
* manipulations and CPU-offline checks. Such invalidation is useful at
* the beginning of a grace period.
*/
static void dyntick_record_completed(struct rcu_state *rsp, long comp)
{
rsp->dynticks_completed = comp;
}
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
/*
* Recall the previously recorded value of the completion for dynticks.
*/
static long dyntick_recall_completed(struct rcu_state *rsp)
{
return rsp->dynticks_completed;
}
/* /*
* If the specified CPU is offline, tell the caller that it is in * If the specified CPU is offline, tell the caller that it is in
* a quiescent state. Otherwise, whack it with a reschedule IPI. * a quiescent state. Otherwise, whack it with a reschedule IPI.
...@@ -337,27 +357,8 @@ void rcu_irq_exit(void) ...@@ -337,27 +357,8 @@ void rcu_irq_exit(void)
set_need_resched(); set_need_resched();
} }
/*
* Record the specified "completed" value, which is later used to validate
* dynticks counter manipulations. Specify "rsp->completed - 1" to
* unconditionally invalidate any future dynticks manipulations (which is
* useful at the beginning of a grace period).
*/
static void dyntick_record_completed(struct rcu_state *rsp, long comp)
{
rsp->dynticks_completed = comp;
}
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
/*
* Recall the previously recorded value of the completion for dynticks.
*/
static long dyntick_recall_completed(struct rcu_state *rsp)
{
return rsp->dynticks_completed;
}
/* /*
* Snapshot the specified CPU's dynticks counter so that we can later * Snapshot the specified CPU's dynticks counter so that we can later
* credit them with an implicit quiescent state. Return 1 if this CPU * credit them with an implicit quiescent state. Return 1 if this CPU
...@@ -421,24 +422,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) ...@@ -421,24 +422,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
#else /* #ifdef CONFIG_NO_HZ */ #else /* #ifdef CONFIG_NO_HZ */
static void dyntick_record_completed(struct rcu_state *rsp, long comp)
{
}
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
/*
* If there are no dynticks, then the only way that a CPU can passively
* be in a quiescent state is to be offline. Unlike dynticks idle, which
* is a point in time during the prior (already finished) grace period,
* an offline CPU is always in a quiescent state, and thus can be
* unconditionally applied. So just return the current value of completed.
*/
static long dyntick_recall_completed(struct rcu_state *rsp)
{
return rsp->completed;
}
static int dyntick_save_progress_counter(struct rcu_data *rdp) static int dyntick_save_progress_counter(struct rcu_data *rdp)
{ {
return 0; return 0;
...@@ -1146,6 +1131,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) ...@@ -1146,6 +1131,7 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
long lastcomp; long lastcomp;
struct rcu_node *rnp = rcu_get_root(rsp); struct rcu_node *rnp = rcu_get_root(rsp);
u8 signaled; u8 signaled;
u8 forcenow;
if (!rcu_gp_in_progress(rsp)) if (!rcu_gp_in_progress(rsp))
return; /* No grace period in progress, nothing to force. */ return; /* No grace period in progress, nothing to force. */
...@@ -1182,16 +1168,23 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) ...@@ -1182,16 +1168,23 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
if (rcu_process_dyntick(rsp, lastcomp, if (rcu_process_dyntick(rsp, lastcomp,
dyntick_save_progress_counter)) dyntick_save_progress_counter))
goto unlock_ret; goto unlock_ret;
/* fall into next case. */
case RCU_SAVE_COMPLETED:
/* Update state, record completion counter. */ /* Update state, record completion counter. */
forcenow = 0;
spin_lock(&rnp->lock); spin_lock(&rnp->lock);
if (lastcomp == rsp->completed && if (lastcomp == rsp->completed &&
rsp->signaled == RCU_SAVE_DYNTICK) { rsp->signaled == signaled) {
rsp->signaled = RCU_FORCE_QS; rsp->signaled = RCU_FORCE_QS;
dyntick_record_completed(rsp, lastcomp); dyntick_record_completed(rsp, lastcomp);
forcenow = signaled == RCU_SAVE_COMPLETED;
} }
spin_unlock(&rnp->lock); spin_unlock(&rnp->lock);
if (!forcenow)
break; break;
/* fall into next case. */
case RCU_FORCE_QS: case RCU_FORCE_QS:
......
...@@ -204,11 +204,12 @@ struct rcu_data { ...@@ -204,11 +204,12 @@ struct rcu_data {
#define RCU_GP_IDLE 0 /* No grace period in progress. */ #define RCU_GP_IDLE 0 /* No grace period in progress. */
#define RCU_GP_INIT 1 /* Grace period being initialized. */ #define RCU_GP_INIT 1 /* Grace period being initialized. */
#define RCU_SAVE_DYNTICK 2 /* Need to scan dyntick state. */ #define RCU_SAVE_DYNTICK 2 /* Need to scan dyntick state. */
#define RCU_FORCE_QS 3 /* Need to force quiescent state. */ #define RCU_SAVE_COMPLETED 3 /* Need to save rsp->completed. */
#define RCU_FORCE_QS 4 /* Need to force quiescent state. */
#ifdef CONFIG_NO_HZ #ifdef CONFIG_NO_HZ
#define RCU_SIGNAL_INIT RCU_SAVE_DYNTICK #define RCU_SIGNAL_INIT RCU_SAVE_DYNTICK
#else /* #ifdef CONFIG_NO_HZ */ #else /* #ifdef CONFIG_NO_HZ */
#define RCU_SIGNAL_INIT RCU_FORCE_QS #define RCU_SIGNAL_INIT RCU_SAVE_COMPLETED
#endif /* #else #ifdef CONFIG_NO_HZ */ #endif /* #else #ifdef CONFIG_NO_HZ */
#define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */ #define RCU_JIFFIES_TILL_FORCE_QS 3 /* for rsp->jiffies_force_qs */
...@@ -274,9 +275,8 @@ struct rcu_state { ...@@ -274,9 +275,8 @@ struct rcu_state {
unsigned long jiffies_stall; /* Time at which to check */ unsigned long jiffies_stall; /* Time at which to check */
/* for CPU stalls. */ /* for CPU stalls. */
#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */ #endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
#ifdef CONFIG_NO_HZ
long dynticks_completed; /* Value of completed @ snap. */ long dynticks_completed; /* Value of completed @ snap. */
#endif /* #ifdef CONFIG_NO_HZ */ /* Protected by fqslock. */
}; };
#ifdef RCU_TREE_NONCORE #ifdef RCU_TREE_NONCORE
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment