powerpc/irq: Run softirqs off the top of the irq stack

Nowadays, irq_exit() calls __do_softirq() pretty much directly
instead of calling do_softirq() which switches to the decicated
softirq stack.

This has lead to observed stack overflows on powerpc since we call
irq_enter() and irq_exit() outside of the scope that switches to
the irq stack.

This fixes it by moving the stack switching up a level, making
irq_enter() and irq_exit() run off the irq stack.
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
parent 4a10c2ac
...@@ -69,9 +69,9 @@ extern struct thread_info *softirq_ctx[NR_CPUS]; ...@@ -69,9 +69,9 @@ extern struct thread_info *softirq_ctx[NR_CPUS];
extern void irq_ctx_init(void); extern void irq_ctx_init(void);
extern void call_do_softirq(struct thread_info *tp); extern void call_do_softirq(struct thread_info *tp);
extern int call_handle_irq(int irq, void *p1, extern void call_do_irq(struct pt_regs *regs, struct thread_info *tp);
struct thread_info *tp, void *func);
extern void do_IRQ(struct pt_regs *regs); extern void do_IRQ(struct pt_regs *regs);
extern void __do_irq(struct pt_regs *regs);
int irq_choose_cpu(const struct cpumask *mask); int irq_choose_cpu(const struct cpumask *mask);
......
...@@ -441,50 +441,6 @@ void migrate_irqs(void) ...@@ -441,50 +441,6 @@ void migrate_irqs(void)
} }
#endif #endif
static inline void handle_one_irq(unsigned int irq)
{
struct thread_info *curtp, *irqtp;
unsigned long saved_sp_limit;
struct irq_desc *desc;
desc = irq_to_desc(irq);
if (!desc)
return;
/* Switch to the irq stack to handle this */
curtp = current_thread_info();
irqtp = hardirq_ctx[smp_processor_id()];
if (curtp == irqtp) {
/* We're already on the irq stack, just handle it */
desc->handle_irq(irq, desc);
return;
}
saved_sp_limit = current->thread.ksp_limit;
irqtp->task = curtp->task;
irqtp->flags = 0;
/* Copy the softirq bits in preempt_count so that the
* softirq checks work in the hardirq context. */
irqtp->preempt_count = (irqtp->preempt_count & ~SOFTIRQ_MASK) |
(curtp->preempt_count & SOFTIRQ_MASK);
current->thread.ksp_limit = (unsigned long)irqtp +
_ALIGN_UP(sizeof(struct thread_info), 16);
call_handle_irq(irq, desc, irqtp, desc->handle_irq);
current->thread.ksp_limit = saved_sp_limit;
irqtp->task = NULL;
/* Set any flag that may have been set on the
* alternate stack
*/
if (irqtp->flags)
set_bits(irqtp->flags, &curtp->flags);
}
static inline void check_stack_overflow(void) static inline void check_stack_overflow(void)
{ {
#ifdef CONFIG_DEBUG_STACKOVERFLOW #ifdef CONFIG_DEBUG_STACKOVERFLOW
...@@ -501,9 +457,9 @@ static inline void check_stack_overflow(void) ...@@ -501,9 +457,9 @@ static inline void check_stack_overflow(void)
#endif #endif
} }
void do_IRQ(struct pt_regs *regs) void __do_irq(struct pt_regs *regs)
{ {
struct pt_regs *old_regs = set_irq_regs(regs); struct irq_desc *desc;
unsigned int irq; unsigned int irq;
irq_enter(); irq_enter();
...@@ -519,18 +475,64 @@ void do_IRQ(struct pt_regs *regs) ...@@ -519,18 +475,64 @@ void do_IRQ(struct pt_regs *regs)
*/ */
irq = ppc_md.get_irq(); irq = ppc_md.get_irq();
/* We can hard enable interrupts now */ /* We can hard enable interrupts now to allow perf interrupts */
may_hard_irq_enable(); may_hard_irq_enable();
/* And finally process it */ /* And finally process it */
if (irq != NO_IRQ) if (unlikely(irq == NO_IRQ))
handle_one_irq(irq);
else
__get_cpu_var(irq_stat).spurious_irqs++; __get_cpu_var(irq_stat).spurious_irqs++;
else {
desc = irq_to_desc(irq);
if (likely(desc))
desc->handle_irq(irq, desc);
}
trace_irq_exit(regs); trace_irq_exit(regs);
irq_exit(); irq_exit();
}
void do_IRQ(struct pt_regs *regs)
{
struct pt_regs *old_regs = set_irq_regs(regs);
struct thread_info *curtp, *irqtp;
unsigned long saved_sp_limit;
/* Switch to the irq stack to handle this */
curtp = current_thread_info();
irqtp = hardirq_ctx[raw_smp_processor_id()];
/* Already there ? */
if (unlikely(curtp == irqtp)) {
__do_irq(regs);
set_irq_regs(old_regs);
return;
}
/* Adjust the stack limit */
saved_sp_limit = current->thread.ksp_limit;
current->thread.ksp_limit = (unsigned long)irqtp +
_ALIGN_UP(sizeof(struct thread_info), 16);
/* Prepare the thread_info in the irq stack */
irqtp->task = curtp->task;
irqtp->flags = 0;
/* Copy the preempt_count so that the [soft]irq checks work. */
irqtp->preempt_count = curtp->preempt_count;
/* Switch stack and call */
call_do_irq(regs, irqtp);
/* Restore stack limit */
current->thread.ksp_limit = saved_sp_limit;
irqtp->task = NULL;
/* Copy back updates to the thread_info */
if (irqtp->flags)
set_bits(irqtp->flags, &curtp->flags);
set_irq_regs(old_regs); set_irq_regs(old_regs);
} }
...@@ -592,12 +594,10 @@ void irq_ctx_init(void) ...@@ -592,12 +594,10 @@ void irq_ctx_init(void)
memset((void *)softirq_ctx[i], 0, THREAD_SIZE); memset((void *)softirq_ctx[i], 0, THREAD_SIZE);
tp = softirq_ctx[i]; tp = softirq_ctx[i];
tp->cpu = i; tp->cpu = i;
tp->preempt_count = 0;
memset((void *)hardirq_ctx[i], 0, THREAD_SIZE); memset((void *)hardirq_ctx[i], 0, THREAD_SIZE);
tp = hardirq_ctx[i]; tp = hardirq_ctx[i];
tp->cpu = i; tp->cpu = i;
tp->preempt_count = HARDIRQ_OFFSET;
} }
} }
......
...@@ -47,13 +47,12 @@ _GLOBAL(call_do_softirq) ...@@ -47,13 +47,12 @@ _GLOBAL(call_do_softirq)
mtlr r0 mtlr r0
blr blr
_GLOBAL(call_handle_irq) _GLOBAL(call_do_irq)
mflr r0 mflr r0
stw r0,4(r1) stw r0,4(r1)
mtctr r6 stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
stwu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r5) mr r1,r4
mr r1,r5 bl __do_irq
bctrl
lwz r1,0(r1) lwz r1,0(r1)
lwz r0,4(r1) lwz r0,4(r1)
mtlr r0 mtlr r0
......
...@@ -40,14 +40,12 @@ _GLOBAL(call_do_softirq) ...@@ -40,14 +40,12 @@ _GLOBAL(call_do_softirq)
mtlr r0 mtlr r0
blr blr
_GLOBAL(call_handle_irq) _GLOBAL(call_do_irq)
ld r8,0(r6)
mflr r0 mflr r0
std r0,16(r1) std r0,16(r1)
mtctr r8 stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
stdu r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r5) mr r1,r4
mr r1,r5 bl .__do_irq
bctrl
ld r1,0(r1) ld r1,0(r1)
ld r0,16(r1) ld r0,16(r1)
mtlr r0 mtlr r0
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment