Commit cba6d0d6 authored by Paul E. McKenney's avatar Paul E. McKenney

Revert "rcu: Move PREEMPT_RCU preemption to switch_to() invocation"

This reverts commit 616c310e.
(Move PREEMPT_RCU preemption to switch_to() invocation).
Testing by Sasha Levin <levinsasha928@gmail.com> showed that this
can result in deadlock due to invoking the scheduler when one of
the runqueue locks is held.  Because this commit was simply a
performance optimization, revert it.
Reported-by: default avatarSasha Levin <levinsasha928@gmail.com>
Signed-off-by: default avatarPaul E. McKenney <paulmck@linux.vnet.ibm.com>
Tested-by: default avatarSasha Levin <levinsasha928@gmail.com>
parent 6887a413
...@@ -705,7 +705,6 @@ static void stack_proc(void *arg) ...@@ -705,7 +705,6 @@ static void stack_proc(void *arg)
struct task_struct *from = current, *to = arg; struct task_struct *from = current, *to = arg;
to->thread.saved_task = from; to->thread.saved_task = from;
rcu_switch_from(from);
switch_to(from, to, from); switch_to(from, to, from);
} }
......
...@@ -184,7 +184,6 @@ static inline int rcu_preempt_depth(void) ...@@ -184,7 +184,6 @@ static inline int rcu_preempt_depth(void)
/* Internal to kernel */ /* Internal to kernel */
extern void rcu_sched_qs(int cpu); extern void rcu_sched_qs(int cpu);
extern void rcu_bh_qs(int cpu); extern void rcu_bh_qs(int cpu);
extern void rcu_preempt_note_context_switch(void);
extern void rcu_check_callbacks(int cpu, int user); extern void rcu_check_callbacks(int cpu, int user);
struct notifier_block; struct notifier_block;
extern void rcu_idle_enter(void); extern void rcu_idle_enter(void);
......
...@@ -87,6 +87,10 @@ static inline void kfree_call_rcu(struct rcu_head *head, ...@@ -87,6 +87,10 @@ static inline void kfree_call_rcu(struct rcu_head *head,
#ifdef CONFIG_TINY_RCU #ifdef CONFIG_TINY_RCU
static inline void rcu_preempt_note_context_switch(void)
{
}
static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies) static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
{ {
*delta_jiffies = ULONG_MAX; *delta_jiffies = ULONG_MAX;
...@@ -95,6 +99,7 @@ static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies) ...@@ -95,6 +99,7 @@ static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
#else /* #ifdef CONFIG_TINY_RCU */ #else /* #ifdef CONFIG_TINY_RCU */
void rcu_preempt_note_context_switch(void);
int rcu_preempt_needs_cpu(void); int rcu_preempt_needs_cpu(void);
static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies) static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
...@@ -108,6 +113,7 @@ static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies) ...@@ -108,6 +113,7 @@ static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
static inline void rcu_note_context_switch(int cpu) static inline void rcu_note_context_switch(int cpu)
{ {
rcu_sched_qs(cpu); rcu_sched_qs(cpu);
rcu_preempt_note_context_switch();
} }
/* /*
......
...@@ -1871,22 +1871,12 @@ static inline void rcu_copy_process(struct task_struct *p) ...@@ -1871,22 +1871,12 @@ static inline void rcu_copy_process(struct task_struct *p)
INIT_LIST_HEAD(&p->rcu_node_entry); INIT_LIST_HEAD(&p->rcu_node_entry);
} }
static inline void rcu_switch_from(struct task_struct *prev)
{
if (prev->rcu_read_lock_nesting != 0)
rcu_preempt_note_context_switch();
}
#else #else
static inline void rcu_copy_process(struct task_struct *p) static inline void rcu_copy_process(struct task_struct *p)
{ {
} }
static inline void rcu_switch_from(struct task_struct *prev)
{
}
#endif #endif
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
......
...@@ -201,6 +201,7 @@ void rcu_note_context_switch(int cpu) ...@@ -201,6 +201,7 @@ void rcu_note_context_switch(int cpu)
{ {
trace_rcu_utilization("Start context switch"); trace_rcu_utilization("Start context switch");
rcu_sched_qs(cpu); rcu_sched_qs(cpu);
rcu_preempt_note_context_switch(cpu);
trace_rcu_utilization("End context switch"); trace_rcu_utilization("End context switch");
} }
EXPORT_SYMBOL_GPL(rcu_note_context_switch); EXPORT_SYMBOL_GPL(rcu_note_context_switch);
......
...@@ -444,6 +444,7 @@ DECLARE_PER_CPU(char, rcu_cpu_has_work); ...@@ -444,6 +444,7 @@ DECLARE_PER_CPU(char, rcu_cpu_has_work);
/* Forward declarations for rcutree_plugin.h */ /* Forward declarations for rcutree_plugin.h */
static void rcu_bootup_announce(void); static void rcu_bootup_announce(void);
long rcu_batches_completed(void); long rcu_batches_completed(void);
static void rcu_preempt_note_context_switch(int cpu);
static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp); static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp);
#ifdef CONFIG_HOTPLUG_CPU #ifdef CONFIG_HOTPLUG_CPU
static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp,
......
...@@ -153,7 +153,7 @@ static void rcu_preempt_qs(int cpu) ...@@ -153,7 +153,7 @@ static void rcu_preempt_qs(int cpu)
* *
* Caller must disable preemption. * Caller must disable preemption.
*/ */
void rcu_preempt_note_context_switch(void) static void rcu_preempt_note_context_switch(int cpu)
{ {
struct task_struct *t = current; struct task_struct *t = current;
unsigned long flags; unsigned long flags;
...@@ -164,7 +164,7 @@ void rcu_preempt_note_context_switch(void) ...@@ -164,7 +164,7 @@ void rcu_preempt_note_context_switch(void)
(t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) { (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
/* Possibly blocking in an RCU read-side critical section. */ /* Possibly blocking in an RCU read-side critical section. */
rdp = __this_cpu_ptr(rcu_preempt_state.rda); rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu);
rnp = rdp->mynode; rnp = rdp->mynode;
raw_spin_lock_irqsave(&rnp->lock, flags); raw_spin_lock_irqsave(&rnp->lock, flags);
t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED; t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BLOCKED;
...@@ -228,7 +228,7 @@ void rcu_preempt_note_context_switch(void) ...@@ -228,7 +228,7 @@ void rcu_preempt_note_context_switch(void)
* means that we continue to block the current grace period. * means that we continue to block the current grace period.
*/ */
local_irq_save(flags); local_irq_save(flags);
rcu_preempt_qs(smp_processor_id()); rcu_preempt_qs(cpu);
local_irq_restore(flags); local_irq_restore(flags);
} }
...@@ -1001,6 +1001,14 @@ void rcu_force_quiescent_state(void) ...@@ -1001,6 +1001,14 @@ void rcu_force_quiescent_state(void)
} }
EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
/*
* Because preemptible RCU does not exist, we never have to check for
* CPUs being in quiescent states.
*/
static void rcu_preempt_note_context_switch(int cpu)
{
}
/* /*
* Because preemptible RCU does not exist, there are never any preempted * Because preemptible RCU does not exist, there are never any preempted
* RCU readers. * RCU readers.
......
...@@ -2081,7 +2081,6 @@ context_switch(struct rq *rq, struct task_struct *prev, ...@@ -2081,7 +2081,6 @@ context_switch(struct rq *rq, struct task_struct *prev,
#endif #endif
/* Here we just switch the register state and the stack. */ /* Here we just switch the register state and the stack. */
rcu_switch_from(prev);
switch_to(prev, next, prev); switch_to(prev, next, prev);
barrier(); barrier();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment