Skip to content

L
linux
Commit cd08f109 authored by Christophe Leroy's avatar Christophe Leroy Committed by Michael Ellerman
Browse files
Options

powerpc/32s: Enable CONFIG_VMAP_STACK 

A few changes to retrieve DAR and DSISR from struct regs
instead of retrieving them directly, as they may have
changed due to a TLB miss.

Also modifies hash_page() and friends to work with virtual
data addresses instead of physical ones. Same on load_up_fpu()
and load_up_altivec().
Signed-off-by: default avatarChristophe Leroy <christophe.leroy@c-s.fr>
[mpe: Fix tovirt_vmstack call in head_32.S to fix CHRP build]
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/2e2509a242fd5f3e23df4a06530c18060c4d321e.1576916812.git.christophe.leroy@c-s.fr
parent 94dd54c5
Hide whitespace changes
Inline Side-by-side
Showing with 61 additions and 24 deletions
arch/powerpc/kernel/entry_32.S
View file @ cd08f109
...@@ -1339,7 +1339,7 @@ _GLOBAL(enter_rtas) ...@@ -1339,7 +1339,7 @@ _GLOBAL(enter_rtas)
lis r6,1f@ha /* physical return address for rtas */ lis r6,1f@ha /* physical return address for rtas */
addi r6,r6,1f@l addi r6,r6,1f@l
tophys(r6,r6) tophys(r6,r6)
tophys(r7,r1) tophys_novmstack r7, r1
lwz r8,RTASENTRY(r4) lwz r8,RTASENTRY(r4)
lwz r4,RTASBASE(r4) lwz r4,RTASBASE(r4)
mfmsr r9 mfmsr r9
......
arch/powerpc/kernel/fpu.S
View file @ cd08f109
...@@ -94,6 +94,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX) ...@@ -94,6 +94,9 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX)
/* enable use of FP after return */ /* enable use of FP after return */
#ifdef CONFIG_PPC32 #ifdef CONFIG_PPC32
mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
#ifdef CONFIG_VMAP_STACK
tovirt(r5, r5)
#endif
lwz r4,THREAD_FPEXC_MODE(r5) lwz r4,THREAD_FPEXC_MODE(r5)
ori r9,r9,MSR_FP /* enable FP for current */ ori r9,r9,MSR_FP /* enable FP for current */
or r9,r9,r4 or r9,r9,r4
......
arch/powerpc/kernel/head_32.S
View file @ cd08f109
...@@ -272,14 +272,21 @@ __secondary_hold_acknowledge: ...@@ -272,14 +272,21 @@ __secondary_hold_acknowledge:
*/ */
. = 0x200 . = 0x200
DO_KVM 0x200 DO_KVM 0x200
MachineCheck:
EXCEPTION_PROLOG_0 EXCEPTION_PROLOG_0
#ifdef CONFIG_VMAP_STACK
li r11, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */
mtmsr r11
isync
#endif
#ifdef CONFIG_PPC_CHRP #ifdef CONFIG_PPC_CHRP
mfspr r11, SPRN_SPRG_THREAD mfspr r11, SPRN_SPRG_THREAD
tovirt_vmstack r11, r11
lwz r11, RTAS_SP(r11) lwz r11, RTAS_SP(r11)
cmpwi cr1, r11, 0 cmpwi cr1, r11, 0
bne cr1, 7f bne cr1, 7f
#endif /* CONFIG_PPC_CHRP */ #endif /* CONFIG_PPC_CHRP */
EXCEPTION_PROLOG_1 EXCEPTION_PROLOG_1 for_rtas=1
7: EXCEPTION_PROLOG_2 7: EXCEPTION_PROLOG_2
addi r3,r1,STACK_FRAME_OVERHEAD addi r3,r1,STACK_FRAME_OVERHEAD
#ifdef CONFIG_PPC_CHRP #ifdef CONFIG_PPC_CHRP
...@@ -294,7 +301,7 @@ __secondary_hold_acknowledge: ...@@ -294,7 +301,7 @@ __secondary_hold_acknowledge:
. = 0x300 . = 0x300
DO_KVM 0x300 DO_KVM 0x300
DataAccess: DataAccess:
EXCEPTION_PROLOG EXCEPTION_PROLOG handle_dar_dsisr=1
get_and_save_dar_dsisr_on_stack r4, r5, r11 get_and_save_dar_dsisr_on_stack r4, r5, r11
BEGIN_MMU_FTR_SECTION BEGIN_MMU_FTR_SECTION
#ifdef CONFIG_PPC_KUAP #ifdef CONFIG_PPC_KUAP
...@@ -334,7 +341,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE) ...@@ -334,7 +341,7 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_HPTE_TABLE)
. = 0x600 . = 0x600
DO_KVM 0x600 DO_KVM 0x600
Alignment: Alignment:
EXCEPTION_PROLOG EXCEPTION_PROLOG handle_dar_dsisr=1
save_dar_dsisr_on_stack r4, r5, r11 save_dar_dsisr_on_stack r4, r5, r11
addi r3,r1,STACK_FRAME_OVERHEAD addi r3,r1,STACK_FRAME_OVERHEAD
EXC_XFER_STD(0x600, alignment_exception) EXC_XFER_STD(0x600, alignment_exception)
...@@ -645,6 +652,9 @@ handle_page_fault_tramp_1: ...@@ -645,6 +652,9 @@ handle_page_fault_tramp_1:
handle_page_fault_tramp_2: handle_page_fault_tramp_2:
EXC_XFER_LITE(0x300, handle_page_fault) EXC_XFER_LITE(0x300, handle_page_fault)
stack_overflow:
vmap_stack_overflow_exception
AltiVecUnavailable: AltiVecUnavailable:
EXCEPTION_PROLOG EXCEPTION_PROLOG
#ifdef CONFIG_ALTIVEC #ifdef CONFIG_ALTIVEC
......
arch/powerpc/kernel/head_32.h
View file @ cd08f109
...@@ -38,11 +38,13 @@ ...@@ -38,11 +38,13 @@
andi. r11, r11, MSR_PR andi. r11, r11, MSR_PR
.endm .endm
.macro EXCEPTION_PROLOG_1 .macro EXCEPTION_PROLOG_1 for_rtas=0
#ifdef CONFIG_VMAP_STACK #ifdef CONFIG_VMAP_STACK
.ifeq \for_rtas
li r11, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */ li r11, MSR_KERNEL & ~(MSR_IR | MSR_RI) /* can take DTLB miss */
mtmsr r11 mtmsr r11
isync isync
.endif
subi r11, r1, INT_FRAME_SIZE /* use r1 if kernel */ subi r11, r1, INT_FRAME_SIZE /* use r1 if kernel */
#else #else
tophys(r11,r1) /* use tophys(r1) if kernel */ tophys(r11,r1) /* use tophys(r1) if kernel */
......
arch/powerpc/kernel/vector.S
View file @ cd08f109
...@@ -67,6 +67,9 @@ _GLOBAL(load_up_altivec) ...@@ -67,6 +67,9 @@ _GLOBAL(load_up_altivec)
#ifdef CONFIG_PPC32 #ifdef CONFIG_PPC32
mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */ mfspr r5,SPRN_SPRG_THREAD /* current task's THREAD (phys) */
oris r9,r9,MSR_VEC@h oris r9,r9,MSR_VEC@h
#ifdef CONFIG_VMAP_STACK
tovirt(r5, r5)
#endif
#else #else
ld r4,PACACURRENT(r13) ld r4,PACACURRENT(r13)
addi r5,r4,THREAD /* Get THREAD */ addi r5,r4,THREAD /* Get THREAD */
......
arch/powerpc/mm/book3s32/hash_low.S
View file @ cd08f109
...@@ -25,6 +25,12 @@ ...@@ -25,6 +25,12 @@
#include <asm/feature-fixups.h> #include <asm/feature-fixups.h>
#include <asm/code-patching-asm.h> #include <asm/code-patching-asm.h>
#ifdef CONFIG_VMAP_STACK
#define ADDR_OFFSET 0
#else
#define ADDR_OFFSET PAGE_OFFSET
#endif
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
.section .bss .section .bss
.align 2 .align 2
...@@ -47,8 +53,8 @@ mmu_hash_lock: ...@@ -47,8 +53,8 @@ mmu_hash_lock:
.text .text
_GLOBAL(hash_page) _GLOBAL(hash_page)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
lis r8, (mmu_hash_lock - PAGE_OFFSET)@h lis r8, (mmu_hash_lock - ADDR_OFFSET)@h
ori r8, r8, (mmu_hash_lock - PAGE_OFFSET)@l ori r8, r8, (mmu_hash_lock - ADDR_OFFSET)@l
lis r0,0x0fff lis r0,0x0fff
b 10f b 10f
11: lwz r6,0(r8) 11: lwz r6,0(r8)
...@@ -66,9 +72,12 @@ _GLOBAL(hash_page) ...@@ -66,9 +72,12 @@ _GLOBAL(hash_page)
cmplw 0,r4,r0 cmplw 0,r4,r0
ori r3,r3,_PAGE_USER|_PAGE_PRESENT /* test low addresses as user */ ori r3,r3,_PAGE_USER|_PAGE_PRESENT /* test low addresses as user */
mfspr r5, SPRN_SPRG_PGDIR /* phys page-table root */ mfspr r5, SPRN_SPRG_PGDIR /* phys page-table root */
#ifdef CONFIG_VMAP_STACK
tovirt(r5, r5)
#endif
blt+ 112f /* assume user more likely */ blt+ 112f /* assume user more likely */
lis r5, (swapper_pg_dir - PAGE_OFFSET)@ha /* if kernel address, use */ lis r5, (swapper_pg_dir - ADDR_OFFSET)@ha /* if kernel address, use */
addi r5 ,r5 ,(swapper_pg_dir - PAGE_OFFSET)@l /* kernel page table */ addi r5 ,r5 ,(swapper_pg_dir - ADDR_OFFSET)@l /* kernel page table */
rlwimi r3,r9,32-12,29,29 /* MSR_PR -> _PAGE_USER */ rlwimi r3,r9,32-12,29,29 /* MSR_PR -> _PAGE_USER */
112: 112:
#ifndef CONFIG_PTE_64BIT #ifndef CONFIG_PTE_64BIT
...@@ -80,6 +89,9 @@ _GLOBAL(hash_page) ...@@ -80,6 +89,9 @@ _GLOBAL(hash_page)
lwzx r8,r8,r5 /* Get L1 entry */ lwzx r8,r8,r5 /* Get L1 entry */
rlwinm. r8,r8,0,0,20 /* extract pt base address */ rlwinm. r8,r8,0,0,20 /* extract pt base address */
#endif #endif
#ifdef CONFIG_VMAP_STACK
tovirt(r8, r8)
#endif
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
beq- hash_page_out /* return if no mapping */ beq- hash_page_out /* return if no mapping */
#else #else
...@@ -137,9 +149,9 @@ retry: ...@@ -137,9 +149,9 @@ retry:
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
eieio eieio
lis r8, (mmu_hash_lock - PAGE_OFFSET)@ha lis r8, (mmu_hash_lock - ADDR_OFFSET)@ha
li r0,0 li r0,0
stw r0, (mmu_hash_lock - PAGE_OFFSET)@l(r8) stw r0, (mmu_hash_lock - ADDR_OFFSET)@l(r8)
#endif #endif
/* Return from the exception */ /* Return from the exception */
...@@ -152,9 +164,9 @@ retry: ...@@ -152,9 +164,9 @@ retry:
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
hash_page_out: hash_page_out:
eieio eieio
lis r8, (mmu_hash_lock - PAGE_OFFSET)@ha lis r8, (mmu_hash_lock - ADDR_OFFSET)@ha
li r0,0 li r0,0
stw r0, (mmu_hash_lock - PAGE_OFFSET)@l(r8) stw r0, (mmu_hash_lock - ADDR_OFFSET)@l(r8)
blr blr
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
...@@ -329,7 +341,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT) ...@@ -329,7 +341,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
patch_site 1f, patch__hash_page_A1 patch_site 1f, patch__hash_page_A1
patch_site 2f, patch__hash_page_A2 patch_site 2f, patch__hash_page_A2
/* Get the address of the primary PTE group in the hash table (r3) */ /* Get the address of the primary PTE group in the hash table (r3) */
0: lis r0, (Hash_base - PAGE_OFFSET)@h /* base address of hash table */ 0: lis r0, (Hash_base - ADDR_OFFSET)@h /* base address of hash table */
1: rlwimi r0,r3,LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* VSID -> hash */ 1: rlwimi r0,r3,LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* VSID -> hash */
2: rlwinm r3,r4,20+LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* PI -> hash */ 2: rlwinm r3,r4,20+LG_PTEG_SIZE,HASH_LEFT,HASH_RIGHT /* PI -> hash */
xor r3,r3,r0 /* make primary hash */ xor r3,r3,r0 /* make primary hash */
...@@ -343,10 +355,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT) ...@@ -343,10 +355,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
beq+ 10f /* no PTE: go look for an empty slot */ beq+ 10f /* no PTE: go look for an empty slot */
tlbie r4 tlbie r4
lis r4, (htab_hash_searches - PAGE_OFFSET)@ha lis r4, (htab_hash_searches - ADDR_OFFSET)@ha
lwz r6, (htab_hash_searches - PAGE_OFFSET)@l(r4) lwz r6, (htab_hash_searches - ADDR_OFFSET)@l(r4)
addi r6,r6,1 /* count how many searches we do */ addi r6,r6,1 /* count how many searches we do */
stw r6, (htab_hash_searches - PAGE_OFFSET)@l(r4) stw r6, (htab_hash_searches - ADDR_OFFSET)@l(r4)
/* Search the primary PTEG for a PTE whose 1st (d)word matches r5 */ /* Search the primary PTEG for a PTE whose 1st (d)word matches r5 */
mtctr r0 mtctr r0
...@@ -378,10 +390,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT) ...@@ -378,10 +390,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
beq+ found_empty beq+ found_empty
/* update counter of times that the primary PTEG is full */ /* update counter of times that the primary PTEG is full */
lis r4, (primary_pteg_full - PAGE_OFFSET)@ha lis r4, (primary_pteg_full - ADDR_OFFSET)@ha
lwz r6, (primary_pteg_full - PAGE_OFFSET)@l(r4) lwz r6, (primary_pteg_full - ADDR_OFFSET)@l(r4)
addi r6,r6,1 addi r6,r6,1
stw r6, (primary_pteg_full - PAGE_OFFSET)@l(r4) stw r6, (primary_pteg_full - ADDR_OFFSET)@l(r4)
patch_site 0f, patch__hash_page_C patch_site 0f, patch__hash_page_C
/* Search the secondary PTEG for an empty slot */ /* Search the secondary PTEG for an empty slot */
...@@ -415,8 +427,8 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT) ...@@ -415,8 +427,8 @@ END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
* lockup here but that shouldn't happen * lockup here but that shouldn't happen
*/ */
1: lis r4, (next_slot - PAGE_OFFSET)@ha /* get next evict slot */ 1: lis r4, (next_slot - ADDR_OFFSET)@ha /* get next evict slot */
lwz r6, (next_slot - PAGE_OFFSET)@l(r4) lwz r6, (next_slot - ADDR_OFFSET)@l(r4)
addi r6,r6,HPTE_SIZE /* search for candidate */ addi r6,r6,HPTE_SIZE /* search for candidate */
andi. r6,r6,7*HPTE_SIZE andi. r6,r6,7*HPTE_SIZE
stw r6,next_slot@l(r4) stw r6,next_slot@l(r4)
......
arch/powerpc/mm/book3s32/mmu.c
View file @ cd08f109
...@@ -413,6 +413,7 @@ void __init MMU_init_hw(void) ...@@ -413,6 +413,7 @@ void __init MMU_init_hw(void)
void __init MMU_init_hw_patch(void) void __init MMU_init_hw_patch(void)
{ {
unsigned int hmask = Hash_mask >> (16 - LG_HPTEG_SIZE); unsigned int hmask = Hash_mask >> (16 - LG_HPTEG_SIZE);
unsigned int hash;
if (ppc_md.progress) if (ppc_md.progress)
ppc_md.progress("hash:patch", 0x345); ppc_md.progress("hash:patch", 0x345);
...@@ -424,8 +425,12 @@ void __init MMU_init_hw_patch(void) ...@@ -424,8 +425,12 @@ void __init MMU_init_hw_patch(void)
/* /*
* Patch up the instructions in hashtable.S:create_hpte * Patch up the instructions in hashtable.S:create_hpte
*/ */
modify_instruction_site(&patch__hash_page_A0, 0xffff, if (IS_ENABLED(CONFIG_VMAP_STACK))
((unsigned int)Hash - PAGE_OFFSET) >> 16); hash = (unsigned int)Hash;
else
hash = (unsigned int)Hash - PAGE_OFFSET;
modify_instruction_site(&patch__hash_page_A0, 0xffff, hash >> 16);
modify_instruction_site(&patch__hash_page_A1, 0x7c0, hash_mb << 6); modify_instruction_site(&patch__hash_page_A1, 0x7c0, hash_mb << 6);
modify_instruction_site(&patch__hash_page_A2, 0x7c0, hash_mb2 << 6); modify_instruction_site(&patch__hash_page_A2, 0x7c0, hash_mb2 << 6);
modify_instruction_site(&patch__hash_page_B, 0xffff, hmask); modify_instruction_site(&patch__hash_page_B, 0xffff, hmask);
......
arch/powerpc/platforms/Kconfig.cputype
View file @ cd08f109
...@@ -31,12 +31,14 @@ config PPC_BOOK3S_6xx ...@@ -31,12 +31,14 @@ config PPC_BOOK3S_6xx
select PPC_HAVE_PMU_SUPPORT select PPC_HAVE_PMU_SUPPORT
select PPC_HAVE_KUEP select PPC_HAVE_KUEP
select PPC_HAVE_KUAP select PPC_HAVE_KUAP
select HAVE_ARCH_VMAP_STACK
config PPC_BOOK3S_601 config PPC_BOOK3S_601
bool "PowerPC 601" bool "PowerPC 601"
select PPC_BOOK3S_32 select PPC_BOOK3S_32
select PPC_FPU select PPC_FPU
select PPC_HAVE_KUAP select PPC_HAVE_KUAP
select HAVE_ARCH_VMAP_STACK
config PPC_85xx config PPC_85xx
bool "Freescale 85xx" bool "Freescale 85xx"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7