Commit a3059b0c authored by Christophe Leroy's avatar Christophe Leroy Committed by Michael Ellerman

powerpc/8xx: Make pinning of ITLBs optional

As stated in a comment in head_8xx.S, today we "Always pin the first
8 MB ITLB to prevent ITLB misses while mucking around with SRR0/SRR1
in asm".

This issue has just been cleared by the preceding patch, therefore
we can make this pinning optional (on by default) and independent
of DATA pinning.

This patch also makes pinning of IMMR independent of pinning of DATA.
Signed-off-by: default avatarChristophe Leroy <christophe.leroy@c-s.fr>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent 0eb0d2e7
...@@ -1178,10 +1178,20 @@ config PIN_TLB ...@@ -1178,10 +1178,20 @@ config PIN_TLB
bool "Pinned Kernel TLBs (860 ONLY)" bool "Pinned Kernel TLBs (860 ONLY)"
depends on ADVANCED_OPTIONS && PPC_8xx depends on ADVANCED_OPTIONS && PPC_8xx
config PIN_TLB_DATA
bool "Pinned TLB for DATA"
depends on PIN_TLB
default y
config PIN_TLB_IMMR config PIN_TLB_IMMR
bool "Pinned TLB for IMMR" bool "Pinned TLB for IMMR"
depends on PIN_TLB depends on PIN_TLB
default y default y
config PIN_TLB_TEXT
bool "Pinned TLB for TEXT"
depends on PIN_TLB
default y
endmenu endmenu
if PPC64 if PPC64
......
...@@ -55,6 +55,15 @@ ...@@ -55,6 +55,15 @@
#define SIMPLE_KERNEL_ADDRESS 1 #define SIMPLE_KERNEL_ADDRESS 1
#endif #endif
/*
* We need an ITLB miss handler for kernel addresses if:
* - Either we have modules
* - Or we have not pinned the first 8M
*/
#if defined(CONFIG_MODULES) || !defined(CONFIG_PIN_TLB_TEXT) || \
defined(CONFIG_DEBUG_PAGEALLOC)
#define ITLB_MISS_KERNEL 1
#endif
/* /*
* Value for the bits that have fixed value in RPN entries. * Value for the bits that have fixed value in RPN entries.
...@@ -317,7 +326,7 @@ SystemCall: ...@@ -317,7 +326,7 @@ SystemCall:
#endif #endif
InstructionTLBMiss: InstructionTLBMiss:
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE) #if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
mtspr SPRN_SPRG_SCRATCH2, r3 mtspr SPRN_SPRG_SCRATCH2, r3
#endif #endif
EXCEPTION_PROLOG_0 EXCEPTION_PROLOG_0
...@@ -335,23 +344,31 @@ InstructionTLBMiss: ...@@ -335,23 +344,31 @@ InstructionTLBMiss:
INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10) INVALIDATE_ADJACENT_PAGES_CPU15(r11, r10)
/* Only modules will cause ITLB Misses as we always /* Only modules will cause ITLB Misses as we always
* pin the first 8MB of kernel memory */ * pin the first 8MB of kernel memory */
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE) #if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
mfcr r3 mfcr r3
#endif #endif
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) #ifdef ITLB_MISS_KERNEL
#ifdef SIMPLE_KERNEL_ADDRESS #if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
andis. r11, r10, 0x8000 /* Address >= 0x80000000 */ andis. r11, r10, 0x8000 /* Address >= 0x80000000 */
#else #else
rlwinm r11, r10, 16, 0xfff8 rlwinm r11, r10, 16, 0xfff8
cmpli cr0, r11, PAGE_OFFSET@h cmpli cr0, r11, PAGE_OFFSET@h
#ifndef CONFIG_PIN_TLB_TEXT
/* It is assumed that kernel code fits into the first 8M page */
_ENTRY(ITLBMiss_cmp)
cmpli cr7, r11, (PAGE_OFFSET + 0x0800000)@h
#endif
#endif #endif
#endif #endif
mfspr r11, SPRN_M_TW /* Get level 1 table */ mfspr r11, SPRN_M_TW /* Get level 1 table */
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) #ifdef ITLB_MISS_KERNEL
#ifdef SIMPLE_KERNEL_ADDRESS #if defined(SIMPLE_KERNEL_ADDRESS) && defined(CONFIG_PIN_TLB_TEXT)
beq+ 3f beq+ 3f
#else #else
blt+ 3f blt+ 3f
#endif
#ifndef CONFIG_PIN_TLB_TEXT
blt cr7, ITLBMissLinear
#endif #endif
lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha lis r11, (swapper_pg_dir-PAGE_OFFSET)@ha
3: 3:
...@@ -370,7 +387,7 @@ InstructionTLBMiss: ...@@ -370,7 +387,7 @@ InstructionTLBMiss:
rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */ rlwimi r10, r11, 0, 0, 32 - PAGE_SHIFT - 1 /* Add level 2 base */
lwz r10, 0(r10) /* Get the pte */ lwz r10, 0(r10) /* Get the pte */
4: 4:
#if defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE) #if defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
mtcr r3 mtcr r3
#endif #endif
/* Insert the APG into the TWC from the Linux PTE. */ /* Insert the APG into the TWC from the Linux PTE. */
...@@ -401,7 +418,7 @@ InstructionTLBMiss: ...@@ -401,7 +418,7 @@ InstructionTLBMiss:
MTSPR_CPU6(SPRN_MI_RPN, r10, r3) /* Update TLB entry */ MTSPR_CPU6(SPRN_MI_RPN, r10, r3) /* Update TLB entry */
/* Restore registers */ /* Restore registers */
#if defined(CONFIG_8xx_CPU6) || defined(CONFIG_MODULES) || defined (CONFIG_DEBUG_PAGEALLOC) || defined (CONFIG_HUGETLB_PAGE) #if defined(CONFIG_8xx_CPU6) || defined(ITLB_MISS_KERNEL) || defined(CONFIG_HUGETLB_PAGE)
mfspr r3, SPRN_SPRG_SCRATCH2 mfspr r3, SPRN_SPRG_SCRATCH2
#endif #endif
EXCEPTION_EPILOG_0 EXCEPTION_EPILOG_0
...@@ -696,6 +713,22 @@ DTLBMissLinear: ...@@ -696,6 +713,22 @@ DTLBMissLinear:
EXCEPTION_EPILOG_0 EXCEPTION_EPILOG_0
rfi rfi
#ifndef CONFIG_PIN_TLB_TEXT
ITLBMissLinear:
mtcr r3
/* Set 8M byte page and mark it valid */
li r11, MI_PS8MEG | MI_SVALID | _PAGE_EXEC
MTSPR_CPU6(SPRN_MI_TWC, r11, r3)
rlwinm r10, r10, 0, 0x0f800000 /* 8xx supports max 256Mb RAM */
ori r10, r10, 0xf0 | MI_SPS16K | _PAGE_SHARED | _PAGE_DIRTY | \
_PAGE_PRESENT
MTSPR_CPU6(SPRN_MI_RPN, r10, r11) /* Update TLB entry */
mfspr r3, SPRN_SPRG_SCRATCH2
EXCEPTION_EPILOG_0
rfi
#endif
/* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions /* This is the procedure to calculate the data EA for buggy dcbx,dcbi instructions
* by decoding the registers used by the dcbx instruction and adding them. * by decoding the registers used by the dcbx instruction and adding them.
* DAR is set to the calculated address. * DAR is set to the calculated address.
...@@ -955,15 +988,14 @@ initial_mmu: ...@@ -955,15 +988,14 @@ initial_mmu:
mtspr SPRN_MD_CTR, r10 /* remove PINNED DTLB entries */ mtspr SPRN_MD_CTR, r10 /* remove PINNED DTLB entries */
tlbia /* Invalidate all TLB entries */ tlbia /* Invalidate all TLB entries */
/* Always pin the first 8 MB ITLB to prevent ITLB #ifdef CONFIG_PIN_TLB_TEXT
misses while mucking around with SRR0/SRR1 in asm
*/
lis r8, MI_RSV4I@h lis r8, MI_RSV4I@h
ori r8, r8, 0x1c00 ori r8, r8, 0x1c00
mtspr SPRN_MI_CTR, r8 /* Set instruction MMU control */ mtspr SPRN_MI_CTR, r8 /* Set instruction MMU control */
#endif
#ifdef CONFIG_PIN_TLB #ifdef CONFIG_PIN_TLB_DATA
oris r10, r10, MD_RSV4I@h oris r10, r10, MD_RSV4I@h
mtspr SPRN_MD_CTR, r10 /* Set data TLB control */ mtspr SPRN_MD_CTR, r10 /* Set data TLB control */
#endif #endif
...@@ -989,6 +1021,7 @@ initial_mmu: ...@@ -989,6 +1021,7 @@ initial_mmu:
* internal registers (among other things). * internal registers (among other things).
*/ */
#ifdef CONFIG_PIN_TLB_IMMR #ifdef CONFIG_PIN_TLB_IMMR
oris r10, r10, MD_RSV4I@h
ori r10, r10, 0x1c00 ori r10, r10, 0x1c00
mtspr SPRN_MD_CTR, r10 mtspr SPRN_MD_CTR, r10
......
...@@ -65,7 +65,7 @@ unsigned long p_block_mapped(phys_addr_t pa) ...@@ -65,7 +65,7 @@ unsigned long p_block_mapped(phys_addr_t pa)
void __init MMU_init_hw(void) void __init MMU_init_hw(void)
{ {
/* PIN up to the 3 first 8Mb after IMMR in DTLB table */ /* PIN up to the 3 first 8Mb after IMMR in DTLB table */
#ifdef CONFIG_PIN_TLB #ifdef CONFIG_PIN_TLB_DATA
unsigned long ctr = mfspr(SPRN_MD_CTR) & 0xfe000000; unsigned long ctr = mfspr(SPRN_MD_CTR) & 0xfe000000;
unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY; unsigned long flags = 0xf0 | MD_SPS16K | _PAGE_SHARED | _PAGE_DIRTY;
#ifdef CONFIG_PIN_TLB_IMMR #ifdef CONFIG_PIN_TLB_IMMR
...@@ -103,6 +103,9 @@ static void mmu_mapin_immr(void) ...@@ -103,6 +103,9 @@ static void mmu_mapin_immr(void)
extern unsigned int DTLBMiss_jmp; extern unsigned int DTLBMiss_jmp;
#endif #endif
extern unsigned int DTLBMiss_cmp, FixupDAR_cmp; extern unsigned int DTLBMiss_cmp, FixupDAR_cmp;
#ifndef CONFIG_PIN_TLB_TEXT
extern unsigned int ITLBMiss_cmp;
#endif
void mmu_patch_cmp_limit(unsigned int *addr, unsigned long mapped) void mmu_patch_cmp_limit(unsigned int *addr, unsigned long mapped)
{ {
...@@ -122,6 +125,9 @@ unsigned long __init mmu_mapin_ram(unsigned long top) ...@@ -122,6 +125,9 @@ unsigned long __init mmu_mapin_ram(unsigned long top)
mmu_mapin_immr(); mmu_mapin_immr();
#ifndef CONFIG_PIN_TLB_IMMR #ifndef CONFIG_PIN_TLB_IMMR
patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP); patch_instruction(&DTLBMiss_jmp, PPC_INST_NOP);
#endif
#ifndef CONFIG_PIN_TLB_TEXT
mmu_patch_cmp_limit(&ITLBMiss_cmp, 0);
#endif #endif
} else { } else {
mapped = top & ~(LARGE_PAGE_SIZE_8M - 1); mapped = top & ~(LARGE_PAGE_SIZE_8M - 1);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment