Commit 08bca60a authored by Peter Zijlstra's avatar Peter Zijlstra Committed by Ingo Molnar

rcu: Remove waitqueue usage for cpu, node, and boost kthreads

It is not necessary to use waitqueues for the RCU kthreads because
we always know exactly which thread is to be awakened.  In addition,
wake_up() only issues an actual wakeup when there is a thread waiting on
the queue, which was why there was an extra explicit wake_up_process()
to get the RCU kthreads started.

Eliminating the waitqueues (and wake_up()) in favor of wake_up_process()
eliminates the need for the initial wake_up_process() and also shrinks
the data structure size a bit.  The wakeup logic is placed in a new
rcu_wait() macro.
Signed-off-by: default avatarPeter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
parent 8826f3b0
...@@ -95,7 +95,6 @@ static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); ...@@ -95,7 +95,6 @@ static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
DEFINE_PER_CPU(int, rcu_cpu_kthread_cpu); DEFINE_PER_CPU(int, rcu_cpu_kthread_cpu);
DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
static DEFINE_PER_CPU(wait_queue_head_t, rcu_cpu_wq);
DEFINE_PER_CPU(char, rcu_cpu_has_work); DEFINE_PER_CPU(char, rcu_cpu_has_work);
static char rcu_kthreads_spawnable; static char rcu_kthreads_spawnable;
...@@ -1476,7 +1475,7 @@ static void invoke_rcu_cpu_kthread(void) ...@@ -1476,7 +1475,7 @@ static void invoke_rcu_cpu_kthread(void)
local_irq_restore(flags); local_irq_restore(flags);
return; return;
} }
wake_up(&__get_cpu_var(rcu_cpu_wq)); wake_up_process(__this_cpu_read(rcu_cpu_kthread_task));
local_irq_restore(flags); local_irq_restore(flags);
} }
...@@ -1596,14 +1595,12 @@ static int rcu_cpu_kthread(void *arg) ...@@ -1596,14 +1595,12 @@ static int rcu_cpu_kthread(void *arg)
unsigned long flags; unsigned long flags;
int spincnt = 0; int spincnt = 0;
unsigned int *statusp = &per_cpu(rcu_cpu_kthread_status, cpu); unsigned int *statusp = &per_cpu(rcu_cpu_kthread_status, cpu);
wait_queue_head_t *wqp = &per_cpu(rcu_cpu_wq, cpu);
char work; char work;
char *workp = &per_cpu(rcu_cpu_has_work, cpu); char *workp = &per_cpu(rcu_cpu_has_work, cpu);
for (;;) { for (;;) {
*statusp = RCU_KTHREAD_WAITING; *statusp = RCU_KTHREAD_WAITING;
wait_event_interruptible(*wqp, rcu_wait(*workp != 0 || kthread_should_stop());
*workp != 0 || kthread_should_stop());
local_bh_disable(); local_bh_disable();
if (rcu_cpu_kthread_should_stop(cpu)) { if (rcu_cpu_kthread_should_stop(cpu)) {
local_bh_enable(); local_bh_enable();
...@@ -1654,7 +1651,6 @@ static int __cpuinit rcu_spawn_one_cpu_kthread(int cpu) ...@@ -1654,7 +1651,6 @@ static int __cpuinit rcu_spawn_one_cpu_kthread(int cpu)
per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu; per_cpu(rcu_cpu_kthread_cpu, cpu) = cpu;
WARN_ON_ONCE(per_cpu(rcu_cpu_kthread_task, cpu) != NULL); WARN_ON_ONCE(per_cpu(rcu_cpu_kthread_task, cpu) != NULL);
per_cpu(rcu_cpu_kthread_task, cpu) = t; per_cpu(rcu_cpu_kthread_task, cpu) = t;
wake_up_process(t);
sp.sched_priority = RCU_KTHREAD_PRIO; sp.sched_priority = RCU_KTHREAD_PRIO;
sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
return 0; return 0;
...@@ -1677,8 +1673,7 @@ static int rcu_node_kthread(void *arg) ...@@ -1677,8 +1673,7 @@ static int rcu_node_kthread(void *arg)
for (;;) { for (;;) {
rnp->node_kthread_status = RCU_KTHREAD_WAITING; rnp->node_kthread_status = RCU_KTHREAD_WAITING;
wait_event_interruptible(rnp->node_wq, rcu_wait(atomic_read(&rnp->wakemask) != 0);
atomic_read(&rnp->wakemask) != 0);
rnp->node_kthread_status = RCU_KTHREAD_RUNNING; rnp->node_kthread_status = RCU_KTHREAD_RUNNING;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
mask = atomic_xchg(&rnp->wakemask, 0); mask = atomic_xchg(&rnp->wakemask, 0);
...@@ -1762,7 +1757,6 @@ static int __cpuinit rcu_spawn_one_node_kthread(struct rcu_state *rsp, ...@@ -1762,7 +1757,6 @@ static int __cpuinit rcu_spawn_one_node_kthread(struct rcu_state *rsp,
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
rnp->node_kthread_task = t; rnp->node_kthread_task = t;
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
wake_up_process(t);
sp.sched_priority = 99; sp.sched_priority = 99;
sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
} }
...@@ -1779,19 +1773,14 @@ static int __init rcu_spawn_kthreads(void) ...@@ -1779,19 +1773,14 @@ static int __init rcu_spawn_kthreads(void)
rcu_kthreads_spawnable = 1; rcu_kthreads_spawnable = 1;
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
init_waitqueue_head(&per_cpu(rcu_cpu_wq, cpu));
per_cpu(rcu_cpu_has_work, cpu) = 0; per_cpu(rcu_cpu_has_work, cpu) = 0;
if (cpu_online(cpu)) if (cpu_online(cpu))
(void)rcu_spawn_one_cpu_kthread(cpu); (void)rcu_spawn_one_cpu_kthread(cpu);
} }
rnp = rcu_get_root(rcu_state); rnp = rcu_get_root(rcu_state);
init_waitqueue_head(&rnp->node_wq);
rcu_init_boost_waitqueue(rnp);
(void)rcu_spawn_one_node_kthread(rcu_state, rnp); (void)rcu_spawn_one_node_kthread(rcu_state, rnp);
if (NUM_RCU_NODES > 1) if (NUM_RCU_NODES > 1) {
rcu_for_each_leaf_node(rcu_state, rnp) { rcu_for_each_leaf_node(rcu_state, rnp)
init_waitqueue_head(&rnp->node_wq);
rcu_init_boost_waitqueue(rnp);
(void)rcu_spawn_one_node_kthread(rcu_state, rnp); (void)rcu_spawn_one_node_kthread(rcu_state, rnp);
} }
return 0; return 0;
......
...@@ -159,9 +159,6 @@ struct rcu_node { ...@@ -159,9 +159,6 @@ struct rcu_node {
struct task_struct *boost_kthread_task; struct task_struct *boost_kthread_task;
/* kthread that takes care of priority */ /* kthread that takes care of priority */
/* boosting for this rcu_node structure. */ /* boosting for this rcu_node structure. */
wait_queue_head_t boost_wq;
/* Wait queue on which to park the boost */
/* kthread. */
unsigned int boost_kthread_status; unsigned int boost_kthread_status;
/* State of boost_kthread_task for tracing. */ /* State of boost_kthread_task for tracing. */
unsigned long n_tasks_boosted; unsigned long n_tasks_boosted;
...@@ -188,9 +185,6 @@ struct rcu_node { ...@@ -188,9 +185,6 @@ struct rcu_node {
/* kthread that takes care of this rcu_node */ /* kthread that takes care of this rcu_node */
/* structure, for example, awakening the */ /* structure, for example, awakening the */
/* per-CPU kthreads as needed. */ /* per-CPU kthreads as needed. */
wait_queue_head_t node_wq;
/* Wait queue on which to park the per-node */
/* kthread. */
unsigned int node_kthread_status; unsigned int node_kthread_status;
/* State of node_kthread_task for tracing. */ /* State of node_kthread_task for tracing. */
} ____cacheline_internodealigned_in_smp; } ____cacheline_internodealigned_in_smp;
...@@ -336,6 +330,16 @@ struct rcu_data { ...@@ -336,6 +330,16 @@ struct rcu_data {
/* scheduling clock irq */ /* scheduling clock irq */
/* before ratting on them. */ /* before ratting on them. */
#define rcu_wait(cond) \
do { \
for (;;) { \
set_current_state(TASK_INTERRUPTIBLE); \
if (cond) \
break; \
schedule(); \
} \
__set_current_state(TASK_RUNNING); \
} while (0)
/* /*
* RCU global state, including node hierarchy. This hierarchy is * RCU global state, including node hierarchy. This hierarchy is
...@@ -445,7 +449,6 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu); ...@@ -445,7 +449,6 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu);
static void rcu_preempt_send_cbs_to_online(void); static void rcu_preempt_send_cbs_to_online(void);
static void __init __rcu_init_preempt(void); static void __init __rcu_init_preempt(void);
static void rcu_needs_cpu_flush(void); static void rcu_needs_cpu_flush(void);
static void __init rcu_init_boost_waitqueue(struct rcu_node *rnp);
static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp,
cpumask_var_t cm); cpumask_var_t cm);
......
...@@ -1196,8 +1196,7 @@ static int rcu_boost_kthread(void *arg) ...@@ -1196,8 +1196,7 @@ static int rcu_boost_kthread(void *arg)
for (;;) { for (;;) {
rnp->boost_kthread_status = RCU_KTHREAD_WAITING; rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
wait_event_interruptible(rnp->boost_wq, rnp->boost_tasks || rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
rnp->exp_tasks);
rnp->boost_kthread_status = RCU_KTHREAD_RUNNING; rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
more2boost = rcu_boost(rnp); more2boost = rcu_boost(rnp);
if (more2boost) if (more2boost)
...@@ -1274,14 +1273,6 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) ...@@ -1274,14 +1273,6 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES; rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
} }
/*
* Initialize the RCU-boost waitqueue.
*/
static void __init rcu_init_boost_waitqueue(struct rcu_node *rnp)
{
init_waitqueue_head(&rnp->boost_wq);
}
/* /*
* Create an RCU-boost kthread for the specified node if one does not * Create an RCU-boost kthread for the specified node if one does not
* already exist. We only create this kthread for preemptible RCU. * already exist. We only create this kthread for preemptible RCU.
...@@ -1306,7 +1297,6 @@ static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, ...@@ -1306,7 +1297,6 @@ static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
rnp->boost_kthread_task = t; rnp->boost_kthread_task = t;
raw_spin_unlock_irqrestore(&rnp->lock, flags); raw_spin_unlock_irqrestore(&rnp->lock, flags);
wake_up_process(t);
sp.sched_priority = RCU_KTHREAD_PRIO; sp.sched_priority = RCU_KTHREAD_PRIO;
sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
return 0; return 0;
...@@ -1328,10 +1318,6 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp) ...@@ -1328,10 +1318,6 @@ static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
{ {
} }
static void __init rcu_init_boost_waitqueue(struct rcu_node *rnp)
{
}
static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
struct rcu_node *rnp, struct rcu_node *rnp,
int rnp_index) int rnp_index)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment