Skip to content

L
linux
Commit c7c5cbb4 authored by Nicholas Piggin's avatar Nicholas Piggin Committed by Michael Ellerman
Browse files
Options

powerpc/64s/exception: move interrupt entry code above the common handler 

This better reflects the order in which the code is executed.

No generated code change except BUG line number constants.
Signed-off-by: default avatarNicholas Piggin <npiggin@gmail.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20190802105709.27696-34-npiggin@gmail.com
parent d1a84718
Show whitespace changes
Inline Side-by-side
Showing with 225 additions and 225 deletions
arch/powerpc/kernel/exceptions-64s.S
View file @ c7c5cbb4
......@@ -180,6 +180,130 @@ BEGIN_FTR_SECTION_NESTED(943) \
std ra,offset(r13); \
END_FTR_SECTION_NESTED(ftr,ftr,943)
/*
* Branch to label using its 0xC000 address. This results in instruction
* address suitable for MSR[IR]=0 or 1, which allows relocation to be turned
* on using mtmsr rather than rfid.
*
* This could set the 0xc bits for !RELOCATABLE as an immediate, rather than
* load KBASE for a slight optimisation.
*/
#define BRANCH_TO_C000(reg, label) \
__LOAD_FAR_HANDLER(reg, label); \
mtctr reg; \
bctr
.macro INT_KVM_HANDLER vec, hsrr, area, skip
.if \hsrr
TRAMP_KVM_BEGIN(do_kvm_H\vec\())
.else
TRAMP_KVM_BEGIN(do_kvm_\vec\())
.endif
KVM_HANDLER \vec, \hsrr, \area, \skip
.endm
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
/*
* If hv is possible, interrupts come into to the hv version
* of the kvmppc_interrupt code, which then jumps to the PR handler,
* kvmppc_interrupt_pr, if the guest is a PR guest.
*/
#define kvmppc_interrupt kvmppc_interrupt_hv
#else
#define kvmppc_interrupt kvmppc_interrupt_pr
#endif
.macro KVMTEST hsrr, n
lbz r10,HSTATE_IN_GUEST(r13)
cmpwi r10,0
.if \hsrr == EXC_HV_OR_STD
BEGIN_FTR_SECTION
bne do_kvm_H\n
FTR_SECTION_ELSE
bne do_kvm_\n
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
.elseif \hsrr
bne do_kvm_H\n
.else
bne do_kvm_\n
.endif
.endm
.macro KVM_HANDLER vec, hsrr, area, skip
.if \skip
cmpwi r10,KVM_GUEST_MODE_SKIP
beq 89f
.else
BEGIN_FTR_SECTION_NESTED(947)
ld r10,\area+EX_CFAR(r13)
std r10,HSTATE_CFAR(r13)
END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947)
.endif
BEGIN_FTR_SECTION_NESTED(948)
ld r10,\area+EX_PPR(r13)
std r10,HSTATE_PPR(r13)
END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)
ld r10,\area+EX_R10(r13)
std r12,HSTATE_SCRATCH0(r13)
sldi r12,r9,32
/* HSRR variants have the 0x2 bit added to their trap number */
.if \hsrr == EXC_HV_OR_STD
BEGIN_FTR_SECTION
ori r12,r12,(\vec + 0x2)
FTR_SECTION_ELSE
ori r12,r12,(\vec)
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
.elseif \hsrr
ori r12,r12,(\vec + 0x2)
.else
ori r12,r12,(\vec)
.endif
#ifdef CONFIG_RELOCATABLE
/*
* KVM requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives
* outside the head section. CONFIG_RELOCATABLE KVM expects CTR
* to be saved in HSTATE_SCRATCH1.
*/
mfctr r9
std r9,HSTATE_SCRATCH1(r13)
__LOAD_FAR_HANDLER(r9, kvmppc_interrupt)
mtctr r9
ld r9,\area+EX_R9(r13)
bctr
#else
ld r9,\area+EX_R9(r13)
b kvmppc_interrupt
#endif
.if \skip
89: mtocrf 0x80,r9
ld r9,\area+EX_R9(r13)
ld r10,\area+EX_R10(r13)
.if \hsrr == EXC_HV_OR_STD
BEGIN_FTR_SECTION
b kvmppc_skip_Hinterrupt
FTR_SECTION_ELSE
b kvmppc_skip_interrupt
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
.elseif \hsrr
b kvmppc_skip_Hinterrupt
.else
b kvmppc_skip_interrupt
.endif
.endif
.endm
#else
.macro KVMTEST hsrr, n
.endm
.macro KVM_HANDLER vec, hsrr, area, skip
.endm
#endif
.macro INT_SAVE_SRR_AND_JUMP label, hsrr, set_ri
ld r10,PACAKMSR(r13) /* get MSR value for kernel */
.if ! \set_ri
......@@ -276,119 +400,125 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
.endm
/*
* Branch to label using its 0xC000 address. This results in instruction
* address suitable for MSR[IR]=0 or 1, which allows relocation to be turned
* on using mtmsr rather than rfid.
* This is the BOOK3S interrupt entry code macro.
*
* This could set the 0xc bits for !RELOCATABLE as an immediate, rather than
* load KBASE for a slight optimisation.
*/
#define BRANCH_TO_C000(reg, label) \
__LOAD_FAR_HANDLER(reg, label); \
mtctr reg; \
bctr
#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
/*
* If hv is possible, interrupts come into to the hv version
* of the kvmppc_interrupt code, which then jumps to the PR handler,
* kvmppc_interrupt_pr, if the guest is a PR guest.
* This can result in one of several things happening:
* - Branch to the _common handler, relocated, in virtual mode.
* These are normal interrupts (synchronous and asynchronous) handled by
* the kernel.
* - Branch to KVM, relocated but real mode interrupts remain in real mode.
* These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by
* / intended for host or guest kernel, but KVM must always be involved
* because the machine state is set for guest execution.
* - Branch to the masked handler, unrelocated.
* These occur when maskable asynchronous interrupts are taken with the
* irq_soft_mask set.
* - Branch to an "early" handler in real mode but relocated.
* This is done if early=1. MCE and HMI use these to handle errors in real
* mode.
* - Fall through and continue executing in real, unrelocated mode.
* This is done if early=2.
*/
#define kvmppc_interrupt kvmppc_interrupt_hv
#else
#define kvmppc_interrupt kvmppc_interrupt_pr
#endif
.macro KVMTEST hsrr, n
lbz r10,HSTATE_IN_GUEST(r13)
cmpwi r10,0
.if \hsrr == EXC_HV_OR_STD
BEGIN_FTR_SECTION
bne do_kvm_H\n
FTR_SECTION_ELSE
bne do_kvm_\n
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
.elseif \hsrr
bne do_kvm_H\n
.macro INT_HANDLER name, vec, ool=0, early=0, virt=0, hsrr=0, area=PACA_EXGEN, ri=1, dar=0, dsisr=0, bitmask=0, kvm=0
SET_SCRATCH0(r13) /* save r13 */
GET_PACA(r13)
std r9,\area\()+EX_R9(r13) /* save r9 */
OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR)
HMT_MEDIUM
std r10,\area\()+EX_R10(r13) /* save r10 - r12 */
OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR)
.if \ool
.if !\virt
b tramp_real_\name
.pushsection .text
TRAMP_REAL_BEGIN(tramp_real_\name)
.else
bne do_kvm_\n
b tramp_virt_\name
.pushsection .text
TRAMP_VIRT_BEGIN(tramp_virt_\name)
.endif
.endif
.endm
.macro KVM_HANDLER vec, hsrr, area, skip
.if \skip
cmpwi r10,KVM_GUEST_MODE_SKIP
beq 89f
OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR)
OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR)
INTERRUPT_TO_KERNEL
SAVE_CTR(r10, \area\())
mfcr r9
.if \kvm
KVMTEST \hsrr \vec
.endif
.if \bitmask
lbz r10,PACAIRQSOFTMASK(r13)
andi. r10,r10,\bitmask
/* Associate vector numbers with bits in paca->irq_happened */
.if \vec == 0x500 || \vec == 0xea0
li r10,PACA_IRQ_EE
.elseif \vec == 0x900
li r10,PACA_IRQ_DEC
.elseif \vec == 0xa00 || \vec == 0xe80
li r10,PACA_IRQ_DBELL
.elseif \vec == 0xe60
li r10,PACA_IRQ_HMI
.elseif \vec == 0xf00
li r10,PACA_IRQ_PMI
.else
BEGIN_FTR_SECTION_NESTED(947)
ld r10,\area+EX_CFAR(r13)
std r10,HSTATE_CFAR(r13)
END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947)
.abort "Bad maskable vector"
.endif
BEGIN_FTR_SECTION_NESTED(948)
ld r10,\area+EX_PPR(r13)
std r10,HSTATE_PPR(r13)
END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)
ld r10,\area+EX_R10(r13)
std r12,HSTATE_SCRATCH0(r13)
sldi r12,r9,32
/* HSRR variants have the 0x2 bit added to their trap number */
.if \hsrr == EXC_HV_OR_STD
BEGIN_FTR_SECTION
ori r12,r12,(\vec + 0x2)
bne masked_Hinterrupt
FTR_SECTION_ELSE
ori r12,r12,(\vec)
bne masked_interrupt
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
.elseif \hsrr
ori r12,r12,(\vec + 0x2)
bne masked_Hinterrupt
.else
ori r12,r12,(\vec)
bne masked_interrupt
.endif
.endif
#ifdef CONFIG_RELOCATABLE
std r11,\area\()+EX_R11(r13)
std r12,\area\()+EX_R12(r13)
/*
* KVM requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives
* outside the head section. CONFIG_RELOCATABLE KVM expects CTR
* to be saved in HSTATE_SCRATCH1.
* DAR/DSISR, SCRATCH0 must be read before setting MSR[RI],
* because a d-side MCE will clobber those registers so is
* not recoverable if they are live.
*/
mfctr r9
std r9,HSTATE_SCRATCH1(r13)
__LOAD_FAR_HANDLER(r9, kvmppc_interrupt)
mtctr r9
ld r9,\area+EX_R9(r13)
bctr
#else
ld r9,\area+EX_R9(r13)
b kvmppc_interrupt
#endif
.if \skip
89: mtocrf 0x80,r9
ld r9,\area+EX_R9(r13)
ld r10,\area+EX_R10(r13)
.if \hsrr == EXC_HV_OR_STD
BEGIN_FTR_SECTION
b kvmppc_skip_Hinterrupt
FTR_SECTION_ELSE
b kvmppc_skip_interrupt
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
.elseif \hsrr
b kvmppc_skip_Hinterrupt
GET_SCRATCH0(r10)
std r10,\area\()+EX_R13(r13)
.if \dar
.if \hsrr
mfspr r10,SPRN_HDAR
.else
b kvmppc_skip_interrupt
mfspr r10,SPRN_DAR
.endif
std r10,\area\()+EX_DAR(r13)
.endif
.if \dsisr
.if \hsrr
mfspr r10,SPRN_HDSISR
.else
mfspr r10,SPRN_DSISR
.endif
stw r10,\area\()+EX_DSISR(r13)
.endif
.endm
#else
.macro KVMTEST hsrr, n
.endm
.macro KVM_HANDLER vec, hsrr, area, skip
.if \early == 2
/* nothing more */
.elseif \early
mfctr r10 /* save ctr, even for !RELOCATABLE */
BRANCH_TO_C000(r11, \name\()_early_common)
.elseif !\virt
INT_SAVE_SRR_AND_JUMP \name\()_common, \hsrr, \ri
.else
INT_VIRT_SAVE_SRR_AND_JUMP \name\()_common, \hsrr
.endif
.if \ool
.popsection
.endif
.endm
#endif
/*
* On entry r13 points to the paca, r9-r13 are saved in the paca,
......@@ -555,136 +685,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
#define FINISH_NAP
#endif
/*
* This is the BOOK3S interrupt entry code macro.
*
* This can result in one of several things happening:
* - Branch to the _common handler, relocated, in virtual mode.
* These are normal interrupts (synchronous and asynchronous) handled by
* the kernel.
* - Branch to KVM, relocated but real mode interrupts remain in real mode.
* These occur when HSTATE_IN_GUEST is set. The interrupt may be caused by
* / intended for host or guest kernel, but KVM must always be involved
* because the machine state is set for guest execution.
* - Branch to the masked handler, unrelocated.
* These occur when maskable asynchronous interrupts are taken with the
* irq_soft_mask set.
* - Branch to an "early" handler in real mode but relocated.
* This is done if early=1. MCE and HMI use these to handle errors in real
* mode.
* - Fall through and continue executing in real, unrelocated mode.
* This is done if early=2.
*/
.macro INT_HANDLER name, vec, ool=0, early=0, virt=0, hsrr=0, area=PACA_EXGEN, ri=1, dar=0, dsisr=0, bitmask=0, kvm=0
SET_SCRATCH0(r13) /* save r13 */
GET_PACA(r13)
std r9,\area\()+EX_R9(r13) /* save r9 */
OPT_GET_SPR(r9, SPRN_PPR, CPU_FTR_HAS_PPR)
HMT_MEDIUM
std r10,\area\()+EX_R10(r13) /* save r10 - r12 */
OPT_GET_SPR(r10, SPRN_CFAR, CPU_FTR_CFAR)
.if \ool
.if !\virt
b tramp_real_\name
.pushsection .text
TRAMP_REAL_BEGIN(tramp_real_\name)
.else
b tramp_virt_\name
.pushsection .text
TRAMP_VIRT_BEGIN(tramp_virt_\name)
.endif
.endif
OPT_SAVE_REG_TO_PACA(\area\()+EX_PPR, r9, CPU_FTR_HAS_PPR)
OPT_SAVE_REG_TO_PACA(\area\()+EX_CFAR, r10, CPU_FTR_CFAR)
INTERRUPT_TO_KERNEL
SAVE_CTR(r10, \area\())
mfcr r9
.if \kvm
KVMTEST \hsrr \vec
.endif
.if \bitmask
lbz r10,PACAIRQSOFTMASK(r13)
andi. r10,r10,\bitmask
/* Associate vector numbers with bits in paca->irq_happened */
.if \vec == 0x500 || \vec == 0xea0
li r10,PACA_IRQ_EE
.elseif \vec == 0x900
li r10,PACA_IRQ_DEC
.elseif \vec == 0xa00 || \vec == 0xe80
li r10,PACA_IRQ_DBELL
.elseif \vec == 0xe60
li r10,PACA_IRQ_HMI
.elseif \vec == 0xf00
li r10,PACA_IRQ_PMI
.else
.abort "Bad maskable vector"
.endif
.if \hsrr == EXC_HV_OR_STD
BEGIN_FTR_SECTION
bne masked_Hinterrupt
FTR_SECTION_ELSE
bne masked_interrupt
ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
.elseif \hsrr
bne masked_Hinterrupt
.else
bne masked_interrupt
.endif
.endif
std r11,\area\()+EX_R11(r13)
std r12,\area\()+EX_R12(r13)
/*
* DAR/DSISR, SCRATCH0 must be read before setting MSR[RI],
* because a d-side MCE will clobber those registers so is
* not recoverable if they are live.
*/
GET_SCRATCH0(r10)
std r10,\area\()+EX_R13(r13)
.if \dar
.if \hsrr
mfspr r10,SPRN_HDAR
.else
mfspr r10,SPRN_DAR
.endif
std r10,\area\()+EX_DAR(r13)
.endif
.if \dsisr
.if \hsrr
mfspr r10,SPRN_HDSISR
.else
mfspr r10,SPRN_DSISR
.endif
stw r10,\area\()+EX_DSISR(r13)
.endif
.if \early == 2
/* nothing more */
.elseif \early
mfctr r10 /* save ctr, even for !RELOCATABLE */
BRANCH_TO_C000(r11, \name\()_early_common)
.elseif !\virt
INT_SAVE_SRR_AND_JUMP \name\()_common, \hsrr, \ri
.else
INT_VIRT_SAVE_SRR_AND_JUMP \name\()_common, \hsrr
.endif
.if \ool
.popsection
.endif
.endm
.macro INT_KVM_HANDLER vec, hsrr, area, skip
.if \hsrr
TRAMP_KVM_BEGIN(do_kvm_H\vec\())
.else
TRAMP_KVM_BEGIN(do_kvm_\vec\())
.endif
KVM_HANDLER \vec, \hsrr, \area, \skip
.endm
#define EXC_COMMON(name, realvec, hdlr) \
EXC_COMMON_BEGIN(name); \
INT_COMMON realvec, PACA_EXGEN, 1, 1, 1, 0, 0 ; \
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7