Commit 9d292501 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman

powerpc/64s/idle: Avoid SRR usage in idle sleep/wake paths

Idle code now always runs at the 0xc... effective address whether
in real or virtual mode. This means rfid can be ditched, along
with a lot of SRR manipulations.

In the wakeup path, carry SRR1 around in r12. Use mtmsrd to change
MSR states as required.

This also balances the return prediction for the idle call, by
doing blr rather than rfid to return to the idle caller.

On POWER9, 2-process context switch on different cores, with snooze
disabled, increases performance by 2%.
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
[mpe: Incorporate v2 fixes from Nick]
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent b51351e2
...@@ -130,6 +130,7 @@ EXC_VIRT_NONE(0x4100, 0x100) ...@@ -130,6 +130,7 @@ EXC_VIRT_NONE(0x4100, 0x100)
#ifdef CONFIG_PPC_P7_NAP #ifdef CONFIG_PPC_P7_NAP
EXC_COMMON_BEGIN(system_reset_idle_common) EXC_COMMON_BEGIN(system_reset_idle_common)
mfspr r12,SPRN_SRR1
b pnv_powersave_wakeup b pnv_powersave_wakeup
#endif #endif
......
...@@ -111,7 +111,7 @@ core_idle_lock_held: ...@@ -111,7 +111,7 @@ core_idle_lock_held:
* r3 - PNV_THREAD_NAP/SLEEP/WINKLE in POWER8 * r3 - PNV_THREAD_NAP/SLEEP/WINKLE in POWER8
* - Requested PSSCR value in POWER9 * - Requested PSSCR value in POWER9
* *
* Address of idle handler to 'rfid' to in r4 * Address of idle handler to branch to in realmode in r4
*/ */
pnv_powersave_common: pnv_powersave_common:
/* Use r3 to pass state nap/sleep/winkle */ /* Use r3 to pass state nap/sleep/winkle */
...@@ -121,14 +121,14 @@ pnv_powersave_common: ...@@ -121,14 +121,14 @@ pnv_powersave_common:
* need to save PC, some CR bits and the NV GPRs, * need to save PC, some CR bits and the NV GPRs,
* but for now an interrupt frame will do. * but for now an interrupt frame will do.
*/ */
mtctr r4
mflr r0 mflr r0
std r0,16(r1) std r0,16(r1)
stdu r1,-INT_FRAME_SIZE(r1) stdu r1,-INT_FRAME_SIZE(r1)
std r0,_LINK(r1) std r0,_LINK(r1)
std r0,_NIP(r1) std r0,_NIP(r1)
mfmsr r9
/* We haven't lost state ... yet */ /* We haven't lost state ... yet */
li r0,0 li r0,0
stb r0,PACA_NAPSTATELOST(r13) stb r0,PACA_NAPSTATELOST(r13)
...@@ -138,7 +138,6 @@ pnv_powersave_common: ...@@ -138,7 +138,6 @@ pnv_powersave_common:
SAVE_NVGPRS(r1) SAVE_NVGPRS(r1)
mfcr r5 mfcr r5
std r5,_CCR(r1) std r5,_CCR(r1)
std r9,_MSR(r1)
std r1,PACAR1(r13) std r1,PACAR1(r13)
/* /*
...@@ -148,12 +147,8 @@ pnv_powersave_common: ...@@ -148,12 +147,8 @@ pnv_powersave_common:
* the MMU context to the guest. * the MMU context to the guest.
*/ */
LOAD_REG_IMMEDIATE(r7, MSR_IDLE) LOAD_REG_IMMEDIATE(r7, MSR_IDLE)
li r6, MSR_RI mtmsrd r7,0
andc r6, r9, r6 bctr
mtmsrd r6, 1 /* clear RI before setting SRR0/1 */
mtspr SPRN_SRR0, r4
mtspr SPRN_SRR1, r7
rfid
.globl pnv_enter_arch207_idle_mode .globl pnv_enter_arch207_idle_mode
pnv_enter_arch207_idle_mode: pnv_enter_arch207_idle_mode:
...@@ -305,11 +300,10 @@ _GLOBAL(power7_idle_insn) ...@@ -305,11 +300,10 @@ _GLOBAL(power7_idle_insn)
b pnv_powersave_common b pnv_powersave_common
#define CHECK_HMI_INTERRUPT \ #define CHECK_HMI_INTERRUPT \
mfspr r0,SPRN_SRR1; \
BEGIN_FTR_SECTION_NESTED(66); \ BEGIN_FTR_SECTION_NESTED(66); \
rlwinm r0,r0,45-31,0xf; /* extract wake reason field (P8) */ \ rlwinm r0,r12,45-31,0xf; /* extract wake reason field (P8) */ \
FTR_SECTION_ELSE_NESTED(66); \ FTR_SECTION_ELSE_NESTED(66); \
rlwinm r0,r0,45-31,0xe; /* P7 wake reason field is 3 bits */ \ rlwinm r0,r12,45-31,0xe; /* P7 wake reason field is 3 bits */ \
ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66); \ ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_ARCH_207S, 66); \
cmpwi r0,0xa; /* Hypervisor maintenance ? */ \ cmpwi r0,0xa; /* Hypervisor maintenance ? */ \
bne 20f; \ bne 20f; \
...@@ -388,17 +382,17 @@ pnv_powersave_wakeup_mce: ...@@ -388,17 +382,17 @@ pnv_powersave_wakeup_mce:
/* /*
* Now put the original SRR1 with SRR1_WAKEMCE_RESVD as the wake * Now put the original SRR1 with SRR1_WAKEMCE_RESVD as the wake
* reason into SRR1, which allows reuse of the system reset wakeup * reason into r12, which allows reuse of the system reset wakeup
* code without being mistaken for another type of wakeup. * code without being mistaken for another type of wakeup.
*/ */
oris r3,r3,SRR1_WAKEMCE_RESVD@h oris r12,r3,SRR1_WAKEMCE_RESVD@h
mtspr SPRN_SRR1,r3
b pnv_powersave_wakeup b pnv_powersave_wakeup
/* /*
* Called from reset vector for powersave wakeups. * Called from reset vector for powersave wakeups.
* cr3 - set to gt if waking up with partial/complete hypervisor state loss * cr3 - set to gt if waking up with partial/complete hypervisor state loss
* r12 - SRR1
*/ */
.global pnv_powersave_wakeup .global pnv_powersave_wakeup
pnv_powersave_wakeup: pnv_powersave_wakeup:
...@@ -416,6 +410,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) ...@@ -416,6 +410,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
li r0,PNV_THREAD_RUNNING li r0,PNV_THREAD_RUNNING
stb r0,PACA_THREAD_IDLE_STATE(r13) /* Clear thread state */ stb r0,PACA_THREAD_IDLE_STATE(r13) /* Clear thread state */
mr r3,r12
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
li r0,KVM_HWTHREAD_IN_KERNEL li r0,KVM_HWTHREAD_IN_KERNEL
stb r0,HSTATE_HWTHREAD_STATE(r13) stb r0,HSTATE_HWTHREAD_STATE(r13)
...@@ -429,7 +425,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) ...@@ -429,7 +425,6 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300)
#endif #endif
/* Return SRR1 from power7_nap() */ /* Return SRR1 from power7_nap() */
mfspr r3,SPRN_SRR1
blt cr3,pnv_wakeup_noloss blt cr3,pnv_wakeup_noloss
b pnv_wakeup_loss b pnv_wakeup_loss
...@@ -529,9 +524,9 @@ pnv_wakeup_tb_loss: ...@@ -529,9 +524,9 @@ pnv_wakeup_tb_loss:
* is required to return back to reset vector after hypervisor state * is required to return back to reset vector after hypervisor state
* restore is complete. * restore is complete.
*/ */
mr r19,r12
mr r18,r4 mr r18,r4
mflr r17 mflr r17
mfspr r16,SPRN_SRR1
BEGIN_FTR_SECTION BEGIN_FTR_SECTION
CHECK_HMI_INTERRUPT CHECK_HMI_INTERRUPT
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
...@@ -781,7 +776,7 @@ BEGIN_FTR_SECTION ...@@ -781,7 +776,7 @@ BEGIN_FTR_SECTION
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
hypervisor_state_restored: hypervisor_state_restored:
mtspr SPRN_SRR1,r16 mr r12,r19
mtlr r17 mtlr r17
blr /* return to pnv_powersave_wakeup */ blr /* return to pnv_powersave_wakeup */
...@@ -794,6 +789,7 @@ fastsleep_workaround_at_exit: ...@@ -794,6 +789,7 @@ fastsleep_workaround_at_exit:
/* /*
* R3 here contains the value that will be returned to the caller * R3 here contains the value that will be returned to the caller
* of power7_nap. * of power7_nap.
* R12 contains SRR1 for CHECK_HMI_INTERRUPT.
*/ */
.global pnv_wakeup_loss .global pnv_wakeup_loss
pnv_wakeup_loss: pnv_wakeup_loss:
...@@ -803,32 +799,33 @@ BEGIN_FTR_SECTION ...@@ -803,32 +799,33 @@ BEGIN_FTR_SECTION
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
REST_NVGPRS(r1) REST_NVGPRS(r1)
REST_GPR(2, r1) REST_GPR(2, r1)
ld r4,PACAKMSR(r13)
ld r5,_LINK(r1)
ld r6,_CCR(r1) ld r6,_CCR(r1)
ld r4,_MSR(r1)
ld r5,_NIP(r1)
addi r1,r1,INT_FRAME_SIZE addi r1,r1,INT_FRAME_SIZE
mtlr r5
mtcr r6 mtcr r6
mtspr SPRN_SRR1,r4 mtmsrd r4
mtspr SPRN_SRR0,r5 blr
rfid
/* /*
* R3 here contains the value that will be returned to the caller * R3 here contains the value that will be returned to the caller
* of power7_nap. * of power7_nap.
* R12 contains SRR1 for CHECK_HMI_INTERRUPT.
*/ */
pnv_wakeup_noloss: pnv_wakeup_noloss:
lbz r0,PACA_NAPSTATELOST(r13) lbz r0,PACA_NAPSTATELOST(r13)
cmpwi r0,0 cmpwi r0,0
bne pnv_wakeup_loss bne pnv_wakeup_loss
ld r1,PACAR1(r13)
BEGIN_FTR_SECTION BEGIN_FTR_SECTION
CHECK_HMI_INTERRUPT CHECK_HMI_INTERRUPT
END_FTR_SECTION_IFSET(CPU_FTR_HVMODE) END_FTR_SECTION_IFSET(CPU_FTR_HVMODE)
ld r1,PACAR1(r13) ld r4,PACAKMSR(r13)
ld r6,_CCR(r1)
ld r4,_MSR(r1)
ld r5,_NIP(r1) ld r5,_NIP(r1)
ld r6,_CCR(r1)
addi r1,r1,INT_FRAME_SIZE addi r1,r1,INT_FRAME_SIZE
mtlr r5
mtcr r6 mtcr r6
mtspr SPRN_SRR1,r4 mtmsrd r4
mtspr SPRN_SRR0,r5 blr
rfid
...@@ -329,15 +329,21 @@ kvm_novcpu_exit: ...@@ -329,15 +329,21 @@ kvm_novcpu_exit:
* We come in here when wakened from nap mode. * We come in here when wakened from nap mode.
* Relocation is off and most register values are lost. * Relocation is off and most register values are lost.
* r13 points to the PACA. * r13 points to the PACA.
* r3 contains the SRR1 wakeup value, SRR1 is trashed.
*/ */
.globl kvm_start_guest .globl kvm_start_guest
kvm_start_guest: kvm_start_guest:
/* Set runlatch bit the minute you wake up from nap */ /* Set runlatch bit the minute you wake up from nap */
mfspr r0, SPRN_CTRLF mfspr r0, SPRN_CTRLF
ori r0, r0, 1 ori r0, r0, 1
mtspr SPRN_CTRLT, r0 mtspr SPRN_CTRLT, r0
/*
* Could avoid this and pass it through in r3. For now,
* code expects it to be in SRR1.
*/
mtspr SPRN_SRR1,r3
ld r2,PACATOC(r13) ld r2,PACATOC(r13)
li r0,KVM_HWTHREAD_IN_KVM li r0,KVM_HWTHREAD_IN_KVM
...@@ -456,13 +462,15 @@ kvm_no_guest: ...@@ -456,13 +462,15 @@ kvm_no_guest:
/* /*
* We jump to pnv_wakeup_loss, which will return to the caller * We jump to pnv_wakeup_loss, which will return to the caller
* of power7_nap in the powernv cpu offline loop. The value we * of power7_nap in the powernv cpu offline loop. The value we
* put in r3 becomes the return value for power7_nap. * put in r3 becomes the return value for power7_nap. pnv_wakeup_loss
* requires SRR1 in r12.
*/ */
li r3, LPCR_PECE0 li r3, LPCR_PECE0
mfspr r4, SPRN_LPCR mfspr r4, SPRN_LPCR
rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
mtspr SPRN_LPCR, r4 mtspr SPRN_LPCR, r4
li r3, 0 li r3, 0
mfspr r12,SPRN_SRR1
b pnv_wakeup_loss b pnv_wakeup_loss
53: HMT_LOW 53: HMT_LOW
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment