Commit cb2c9b27 authored by Anton Blanchard's avatar Anton Blanchard Committed by Paul Mackerras

[PATCH] powerpc: Fix runlatch performance issues

The runlatch SPR can take a lot of time to write. My original runlatch
code would set it on every exception entry even though most of the time
this was not required. It would also continually set it in the idle
loop, which is an issue on an SMT capable processor.

Now we cache the runlatch value in a threadinfo bit, and only check for
it in decrementer and hardware interrupt exceptions as well as the idle
loop. Boot on POWER3, POWER5 and iseries, and compile tested on pmac32.
Signed-off-by: default avatarAnton Blanchard <anton@samba.org>
Signed-off-by: default avatarPaul Mackerras <paulus@samba.org>
parent 47f78a49
...@@ -319,7 +319,6 @@ exception_marker: ...@@ -319,7 +319,6 @@ exception_marker:
label##_pSeries: \ label##_pSeries: \
HMT_MEDIUM; \ HMT_MEDIUM; \
mtspr SPRN_SPRG1,r13; /* save r13 */ \ mtspr SPRN_SPRG1,r13; /* save r13 */ \
RUNLATCH_ON(r13); \
EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common) EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
#define STD_EXCEPTION_ISERIES(n, label, area) \ #define STD_EXCEPTION_ISERIES(n, label, area) \
...@@ -327,7 +326,6 @@ label##_pSeries: \ ...@@ -327,7 +326,6 @@ label##_pSeries: \
label##_iSeries: \ label##_iSeries: \
HMT_MEDIUM; \ HMT_MEDIUM; \
mtspr SPRN_SPRG1,r13; /* save r13 */ \ mtspr SPRN_SPRG1,r13; /* save r13 */ \
RUNLATCH_ON(r13); \
EXCEPTION_PROLOG_ISERIES_1(area); \ EXCEPTION_PROLOG_ISERIES_1(area); \
EXCEPTION_PROLOG_ISERIES_2; \ EXCEPTION_PROLOG_ISERIES_2; \
b label##_common b label##_common
...@@ -337,7 +335,6 @@ label##_iSeries: \ ...@@ -337,7 +335,6 @@ label##_iSeries: \
label##_iSeries: \ label##_iSeries: \
HMT_MEDIUM; \ HMT_MEDIUM; \
mtspr SPRN_SPRG1,r13; /* save r13 */ \ mtspr SPRN_SPRG1,r13; /* save r13 */ \
RUNLATCH_ON(r13); \
EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN); \ EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN); \
lbz r10,PACAPROCENABLED(r13); \ lbz r10,PACAPROCENABLED(r13); \
cmpwi 0,r10,0; \ cmpwi 0,r10,0; \
...@@ -390,6 +387,7 @@ label##_common: \ ...@@ -390,6 +387,7 @@ label##_common: \
label##_common: \ label##_common: \
EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN); \ EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN); \
DISABLE_INTS; \ DISABLE_INTS; \
bl .ppc64_runlatch_on; \
addi r3,r1,STACK_FRAME_OVERHEAD; \ addi r3,r1,STACK_FRAME_OVERHEAD; \
bl hdlr; \ bl hdlr; \
b .ret_from_except_lite b .ret_from_except_lite
...@@ -407,7 +405,6 @@ __start_interrupts: ...@@ -407,7 +405,6 @@ __start_interrupts:
_machine_check_pSeries: _machine_check_pSeries:
HMT_MEDIUM HMT_MEDIUM
mtspr SPRN_SPRG1,r13 /* save r13 */ mtspr SPRN_SPRG1,r13 /* save r13 */
RUNLATCH_ON(r13)
EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common) EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
. = 0x300 . = 0x300
...@@ -434,7 +431,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_SLB) ...@@ -434,7 +431,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
data_access_slb_pSeries: data_access_slb_pSeries:
HMT_MEDIUM HMT_MEDIUM
mtspr SPRN_SPRG1,r13 mtspr SPRN_SPRG1,r13
RUNLATCH_ON(r13)
mfspr r13,SPRN_SPRG3 /* get paca address into r13 */ mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
std r3,PACA_EXSLB+EX_R3(r13) std r3,PACA_EXSLB+EX_R3(r13)
mfspr r3,SPRN_DAR mfspr r3,SPRN_DAR
...@@ -460,7 +456,6 @@ data_access_slb_pSeries: ...@@ -460,7 +456,6 @@ data_access_slb_pSeries:
instruction_access_slb_pSeries: instruction_access_slb_pSeries:
HMT_MEDIUM HMT_MEDIUM
mtspr SPRN_SPRG1,r13 mtspr SPRN_SPRG1,r13
RUNLATCH_ON(r13)
mfspr r13,SPRN_SPRG3 /* get paca address into r13 */ mfspr r13,SPRN_SPRG3 /* get paca address into r13 */
std r3,PACA_EXSLB+EX_R3(r13) std r3,PACA_EXSLB+EX_R3(r13)
mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */ mfspr r3,SPRN_SRR0 /* SRR0 is faulting address */
...@@ -491,7 +486,6 @@ instruction_access_slb_pSeries: ...@@ -491,7 +486,6 @@ instruction_access_slb_pSeries:
.globl system_call_pSeries .globl system_call_pSeries
system_call_pSeries: system_call_pSeries:
HMT_MEDIUM HMT_MEDIUM
RUNLATCH_ON(r9)
mr r9,r13 mr r9,r13
mfmsr r10 mfmsr r10
mfspr r13,SPRN_SPRG3 mfspr r13,SPRN_SPRG3
...@@ -575,7 +569,6 @@ slb_miss_user_pseries: ...@@ -575,7 +569,6 @@ slb_miss_user_pseries:
system_reset_fwnmi: system_reset_fwnmi:
HMT_MEDIUM HMT_MEDIUM
mtspr SPRN_SPRG1,r13 /* save r13 */ mtspr SPRN_SPRG1,r13 /* save r13 */
RUNLATCH_ON(r13)
EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common) EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common)
.globl machine_check_fwnmi .globl machine_check_fwnmi
...@@ -583,7 +576,6 @@ system_reset_fwnmi: ...@@ -583,7 +576,6 @@ system_reset_fwnmi:
machine_check_fwnmi: machine_check_fwnmi:
HMT_MEDIUM HMT_MEDIUM
mtspr SPRN_SPRG1,r13 /* save r13 */ mtspr SPRN_SPRG1,r13 /* save r13 */
RUNLATCH_ON(r13)
EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common) EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
#ifdef CONFIG_PPC_ISERIES #ifdef CONFIG_PPC_ISERIES
...@@ -894,7 +886,6 @@ unrecov_fer: ...@@ -894,7 +886,6 @@ unrecov_fer:
.align 7 .align 7
.globl data_access_common .globl data_access_common
data_access_common: data_access_common:
RUNLATCH_ON(r10) /* It wont fit in the 0x300 handler */
mfspr r10,SPRN_DAR mfspr r10,SPRN_DAR
std r10,PACA_EXGEN+EX_DAR(r13) std r10,PACA_EXGEN+EX_DAR(r13)
mfspr r10,SPRN_DSISR mfspr r10,SPRN_DSISR
...@@ -1042,6 +1033,7 @@ hardware_interrupt_common: ...@@ -1042,6 +1033,7 @@ hardware_interrupt_common:
EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN) EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
hardware_interrupt_entry: hardware_interrupt_entry:
DISABLE_INTS DISABLE_INTS
bl .ppc64_runlatch_on
addi r3,r1,STACK_FRAME_OVERHEAD addi r3,r1,STACK_FRAME_OVERHEAD
bl .do_IRQ bl .do_IRQ
b .ret_from_except_lite b .ret_from_except_lite
......
...@@ -888,3 +888,35 @@ void dump_stack(void) ...@@ -888,3 +888,35 @@ void dump_stack(void)
show_stack(current, NULL); show_stack(current, NULL);
} }
EXPORT_SYMBOL(dump_stack); EXPORT_SYMBOL(dump_stack);
#ifdef CONFIG_PPC64
void ppc64_runlatch_on(void)
{
unsigned long ctrl;
if (cpu_has_feature(CPU_FTR_CTRL) && !test_thread_flag(TIF_RUNLATCH)) {
HMT_medium();
ctrl = mfspr(SPRN_CTRLF);
ctrl |= CTRL_RUNLATCH;
mtspr(SPRN_CTRLT, ctrl);
set_thread_flag(TIF_RUNLATCH);
}
}
void ppc64_runlatch_off(void)
{
unsigned long ctrl;
if (cpu_has_feature(CPU_FTR_CTRL) && test_thread_flag(TIF_RUNLATCH)) {
HMT_medium();
clear_thread_flag(TIF_RUNLATCH);
ctrl = mfspr(SPRN_CTRLF);
ctrl &= ~CTRL_RUNLATCH;
mtspr(SPRN_CTRLT, ctrl);
}
}
#endif
...@@ -648,6 +648,7 @@ static void yield_shared_processor(void) ...@@ -648,6 +648,7 @@ static void yield_shared_processor(void)
* here and let the timer_interrupt code sort out the actual time. * here and let the timer_interrupt code sort out the actual time.
*/ */
get_lppaca()->int_dword.fields.decr_int = 1; get_lppaca()->int_dword.fields.decr_int = 1;
ppc64_runlatch_on();
process_iSeries_events(); process_iSeries_events();
} }
......
...@@ -615,27 +615,9 @@ ...@@ -615,27 +615,9 @@
#define proc_trap() asm volatile("trap") #define proc_trap() asm volatile("trap")
#ifdef CONFIG_PPC64 #ifdef CONFIG_PPC64
static inline void ppc64_runlatch_on(void)
{ extern void ppc64_runlatch_on(void);
unsigned long ctrl; extern void ppc64_runlatch_off(void);
if (cpu_has_feature(CPU_FTR_CTRL)) {
ctrl = mfspr(SPRN_CTRLF);
ctrl |= CTRL_RUNLATCH;
mtspr(SPRN_CTRLT, ctrl);
}
}
static inline void ppc64_runlatch_off(void)
{
unsigned long ctrl;
if (cpu_has_feature(CPU_FTR_CTRL)) {
ctrl = mfspr(SPRN_CTRLF);
ctrl &= ~CTRL_RUNLATCH;
mtspr(SPRN_CTRLT, ctrl);
}
}
extern unsigned long scom970_read(unsigned int address); extern unsigned long scom970_read(unsigned int address);
extern void scom970_write(unsigned int address, unsigned long value); extern void scom970_write(unsigned int address, unsigned long value);
...@@ -645,15 +627,6 @@ extern void scom970_write(unsigned int address, unsigned long value); ...@@ -645,15 +627,6 @@ extern void scom970_write(unsigned int address, unsigned long value);
#define __get_SP() ({unsigned long sp; \ #define __get_SP() ({unsigned long sp; \
asm volatile("mr %0,1": "=r" (sp)); sp;}) asm volatile("mr %0,1": "=r" (sp)); sp;})
#else /* __ASSEMBLY__ */
#define RUNLATCH_ON(REG) \
BEGIN_FTR_SECTION \
mfspr (REG),SPRN_CTRLF; \
ori (REG),(REG),CTRL_RUNLATCH; \
mtspr SPRN_CTRLT,(REG); \
END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
#endif /* __KERNEL__ */ #endif /* __KERNEL__ */
#endif /* _ASM_POWERPC_REG_H */ #endif /* _ASM_POWERPC_REG_H */
...@@ -113,7 +113,7 @@ static inline struct thread_info *current_thread_info(void) ...@@ -113,7 +113,7 @@ static inline struct thread_info *current_thread_info(void)
#define TIF_POLLING_NRFLAG 4 /* true if poll_idle() is polling #define TIF_POLLING_NRFLAG 4 /* true if poll_idle() is polling
TIF_NEED_RESCHED */ TIF_NEED_RESCHED */
#define TIF_32BIT 5 /* 32 bit binary */ #define TIF_32BIT 5 /* 32 bit binary */
/* #define SPARE 6 */ #define TIF_RUNLATCH 6 /* Is the runlatch enabled? */
#define TIF_ABI_PENDING 7 /* 32/64 bit switch needed */ #define TIF_ABI_PENDING 7 /* 32/64 bit switch needed */
#define TIF_SYSCALL_AUDIT 8 /* syscall auditing active */ #define TIF_SYSCALL_AUDIT 8 /* syscall auditing active */
#define TIF_SINGLESTEP 9 /* singlestepping active */ #define TIF_SINGLESTEP 9 /* singlestepping active */
...@@ -131,7 +131,7 @@ static inline struct thread_info *current_thread_info(void) ...@@ -131,7 +131,7 @@ static inline struct thread_info *current_thread_info(void)
#define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED) #define _TIF_NEED_RESCHED (1<<TIF_NEED_RESCHED)
#define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG) #define _TIF_POLLING_NRFLAG (1<<TIF_POLLING_NRFLAG)
#define _TIF_32BIT (1<<TIF_32BIT) #define _TIF_32BIT (1<<TIF_32BIT)
/* #define _SPARE (1<<SPARE) */ #define _TIF_RUNLATCH (1<<TIF_RUNLATCH)
#define _TIF_ABI_PENDING (1<<TIF_ABI_PENDING) #define _TIF_ABI_PENDING (1<<TIF_ABI_PENDING)
#define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT) #define _TIF_SYSCALL_AUDIT (1<<TIF_SYSCALL_AUDIT)
#define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP) #define _TIF_SINGLESTEP (1<<TIF_SINGLESTEP)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment