Skip to content

L
linux
Commit 9c7fd8c7 authored by Russell King's avatar Russell King
Browse files
Options

[ARM] Optimise ARM TLB handling 

Sanitise includes of asm/tlbflush.h, asm/cacheflush.h, asm/proc-fns.h
Implement ARM-specific TLB "shootdown" code.  It turns out that it
is overall more efficient to unconditionally invalidate the whole
TLB rather than entry by entry when removing areas.
parent d6494ca3
Hide whitespace changes
Inline Side-by-side
Showing with 348 additions and 382 deletions
arch/arm/kernel/arch.c
View file @ 9c7fd8c7
......@@ -8,6 +8,7 @@
#include <linux/types.h>
#include <asm/elf.h>
#include <asm/page.h>
#include <asm/setup.h>
#include <asm/mach-types.h>
#include <asm/mach/arch.h>
......
arch/arm/mm/fault-armv.c
View file @ 9c7fd8c7
......@@ -16,8 +16,10 @@
#include <linux/bitops.h>
#include <linux/init.h>
#include <asm/cacheflush.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
#include "fault.h"
......
arch/arm/mm/init.c
View file @ 9c7fd8c7
......@@ -519,6 +519,10 @@ void __init paging_init(struct meminfo *mi, struct machine_desc *mdesc)
bdata->node_boot_start >> PAGE_SHIFT, zhole_size);
}
#ifndef CONFIG_DISCONTIGMEM
mem_map = contig_page_data.node_mem_map;
#endif
/*
* finish off the bad pages once
* the mem_map is initialised
......@@ -559,7 +563,9 @@ void __init mem_init(void)
initpages = &__init_end - &__init_begin;
high_memory = (void *)__va(meminfo.end);
#ifndef CONFIG_DISCONTIGMEM
max_mapnr = virt_to_page(high_memory) - mem_map;
#endif
/*
* We may have non-contiguous memory.
......
arch/arm/mm/minicache.c
View file @ 9c7fd8c7
......@@ -15,9 +15,11 @@
*/
#include <linux/init.h>
#include <linux/mm.h>
#include <asm/page.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
/*
* 0xffff8000 to 0xffffffff is reserved for any ARM architecture
......
arch/arm/mm/mm-armv.c
View file @ 9c7fd8c7
......@@ -13,6 +13,7 @@
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/bootmem.h>
#include <linux/highmem.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
......@@ -20,6 +21,7 @@
#include <asm/rmap.h>
#include <asm/io.h>
#include <asm/setup.h>
#include <asm/tlbflush.h>
#include <asm/mach/map.h>
......
arch/arm/mm/proc-syms.c
View file @ 9c7fd8c7
......@@ -10,8 +10,10 @@
#include <linux/module.h>
#include <linux/mm.h>
#include <asm/cacheflush.h>
#include <asm/pgalloc.h>
#include <asm/proc-fns.h>
#include <asm/tlbflush.h>
#ifndef MULTI_CPU
EXPORT_SYMBOL(cpu_cache_clean_invalidate_all);
......@@ -29,11 +31,11 @@ EXPORT_SYMBOL(cpu_set_pte);
EXPORT_SYMBOL(processor);
#endif
#ifndef MULTI_TLB
EXPORT_SYMBOL_NOVERS(__cpu_flush_kern_tlb_all);
EXPORT_SYMBOL_NOVERS(__cpu_flush_user_tlb_mm);
EXPORT_SYMBOL_NOVERS(__cpu_flush_user_tlb_range);
EXPORT_SYMBOL_NOVERS(__cpu_flush_user_tlb_page);
#else
/*
* No module should need to touch the TLB (and currently
* no modules do. We export this for "loadkernel" support
* (booting a new kernel from within a running kernel.)
*/
#ifdef MULTI_TLB
EXPORT_SYMBOL(cpu_tlb);
#endif
arch/arm/mm/tlb-v3.S
View file @ 9c7fd8c7
......@@ -13,31 +13,10 @@
*/
#include <linux/linkage.h>
#include <asm/constants.h>
#include <asm/tlbflush.h>
#include "proc-macros.S"
.align 5
/*
* v3_flush_user_tlb_mm(mm)
*
* Invalidate all TLB entries in a particular address space
*
* - mm - mm_struct describing address space
*/
ENTRY(v3_flush_user_tlb_mm)
act_mm r1 @ get current->active_mm
teq r0, r1 @ == mm ?
movne pc, lr @ no, we dont do anything
/*
* v3_flush_kern_tlb_all()
*
* Invalidate the entire TLB
*/
ENTRY(v3_flush_kern_tlb_all)
mov r0, #0
mcr p15, 0, r0, c5, c0, 0 @ invalidate TLB
mov pc, lr
/*
* v3_flush_user_tlb_range(start, end, mm)
*
......@@ -62,32 +41,11 @@ ENTRY(v3_flush_kern_tlb_range)
blo 1b
mov pc, lr
/*
* v3_flush_user_tlb_page(vaddr,vma)
*
* Invalidate the specified page in the specified address space.
*
* - vaddr - virtual address (may not be aligned)
* - vma - vma_struct describing address range
*/
.align 5
ENTRY(v3_flush_user_tlb_page)
vma_vm_mm r2, r1 @ get vma->vm_mm
act_mm r3 @ get current->active_mm
teq r2, r3 @ equal
movne pc, lr @ no
ENTRY(v3_flush_kern_tlb_page)
mcr p15, 0, r0, c6, c0, 0 @ invalidate TLB entry
mov pc, lr
.section ".text.init", #alloc, #execinstr
.type v3_tlb_fns, #object
ENTRY(v3_tlb_fns)
.long v3_flush_kern_tlb_all
.long v3_flush_user_tlb_mm
.long v3_flush_user_tlb_range
.long v3_flush_user_tlb_page
.long v3_flush_kern_tlb_range
.long v3_flush_kern_tlb_page
.long v3_tlb_flags
.size v3_tlb_fns, . - v3_tlb_fns
arch/arm/mm/tlb-v4.S
View file @ 9c7fd8c7
......@@ -14,31 +14,10 @@
*/
#include <linux/linkage.h>
#include <asm/constants.h>
#include <asm/tlbflush.h>
#include "proc-macros.S"
.align 5
/*
* v4_flush_user_tlb_mm(mm)
*
* Invalidate all TLB entries in a particular address space
*
* - mm - mm_struct describing address space
*/
ENTRY(v4_flush_user_tlb_mm)
act_mm r1 @ get current->active_mm
teq r0, r1 @ == mm ?
movne pc, lr @ no, we dont do anything
/*
* v4_flush_kern_tlb_all()
*
* Invalidate the entire TLB
*/
ENTRY(v4_flush_kern_tlb_all)
mov r0, #0
mcr p15, 0, r0, c8, c7, 0 @ invalidate I + D TLBs
mov pc, lr
/*
* v4_flush_user_tlb_range(start, end, mm)
*
......@@ -64,25 +43,6 @@ ENTRY(v4_flush_user_tlb_range)
blo 1b
mov pc, lr
/*
* v4_flush_user_tlb_page(vaddr,vma)
*
* Invalidate the specified page in the specified address space.
*
* - vaddr - virtual address (may not be aligned)
* - vma - vma_struct describing address range
*/
.align 5
ENTRY(v4_flush_user_tlb_page)
vma_vm_mm r2, r1 @ get vma->vm_mm
act_mm r3 @ get current->active_mm
teq r2, r3 @ equal
movne pc, lr @ no
vma_vm_flags r2, r1
.v4_flush_kern_tlb_page:
mcr p15, 0, r0, c8, c7, 1 @ invalidate TLB entry
mov pc, lr
/*
* v4_flush_kerm_tlb_range(start, end)
*
......@@ -95,27 +55,11 @@ ENTRY(v4_flush_user_tlb_page)
.globl v4_flush_kern_tlb_range
.equ v4_flush_kern_tlb_range, .v4_flush_kern_tlb_range
/*
* v4_flush_kern_tlb_page(kaddr)
*
* Invalidate the TLB entry for the specified page. The address
* will be in the kernels virtual memory space. Current uses
* only require the D-TLB to be invalidated.
*
* - kaddr - Kernel virtual memory address
*/
.globl v4_flush_kern_tlb_page
.equ v4_flush_kern_tlb_page, .v4_flush_kern_tlb_page
.section ".text.init", #alloc, #execinstr
.type v4_tlb_fns, #object
ENTRY(v4_tlb_fns)
.long v4_flush_kern_tlb_all
.long v4_flush_user_tlb_mm
.long v4_flush_user_tlb_range
.long v4_flush_user_tlb_page
.long v4_flush_kern_tlb_range
.long v4_flush_kern_tlb_page
.long v4_tlb_flags
.size v4_tlb_fns, . - v4_tlb_fns
arch/arm/mm/tlb-v4wb.S
View file @ 9c7fd8c7
......@@ -14,34 +14,10 @@
*/
#include <linux/linkage.h>
#include <asm/constants.h>
#include <asm/tlbflush.h>
#include "proc-macros.S"
.align 5
/*
* v4wb_flush_user_tlb_mm(mm)
*
* Invalidate all TLB entries in a particular address space
*
* - mm - mm_struct describing address space
*/
ENTRY(v4wb_flush_user_tlb_mm)
ENTRY(v4wbi_flush_user_tlb_mm)
act_mm r1 @ get current->active_mm
teq r0, r1 @ == mm ?
movne pc, lr @ no, we dont do anything
/*
* v4wb_flush_tlb_all()
*
* Invalidate the entire TLB
*/
ENTRY(v4wb_flush_kern_tlb_all)
ENTRY(v4wbi_flush_kern_tlb_all)
mov r0, #0
mcr p15, 0, r0, c7, c10, 4 @ drain WB
mcr p15, 0, r0, c8, c7, 0 @ invalidate I + D TLBs
mov pc, lr
/*
* v4wb_flush_user_tlb_range(start, end, mm)
*
......@@ -69,28 +45,6 @@ ENTRY(v4wb_flush_user_tlb_range)
blo 1b
mov pc, lr
/*
* v4wb_flush_user_tlb_page(vaddr,vma)
*
* Invalidate the specified page in the specified address space.
*
* - vaddr - virtual address (may not be aligned)
* - vma - vma_struct describing address range
*/
.align 5
ENTRY(v4wb_flush_user_tlb_page)
vma_vm_mm r2, r1 @ get vma->vm_mm
act_mm r3 @ get current->active_mm
teq r2, r3 @ equal
movne pc, lr @ no
vma_vm_flags r2, r1
mov r3, #0
mcr p15, 0, r3, c7, c10, 4 @ drain WB
tst r2, #VM_EXEC
mcrne p15, 0, r3, c8, c5, 0 @ invalidate I TLB
mcr p15, 0, r0, c8, c6, 1 @ invalidate D TLB entry
mov pc, lr
/*
* v4_flush_kerm_tlb_range(start, end)
*
......@@ -112,20 +66,6 @@ ENTRY(v4wb_flush_kern_tlb_range)
blo 1b
mov pc, lr
/*
* v4_flush_kern_tlb_page(kaddr)
*
* Invalidate the TLB entry for the specified page. The address
* will be in the kernels virtual memory space. Current uses
* only require the D-TLB to be invalidated.
*
* - kaddr - Kernel virtual memory address
*/
ENTRY(v4wb_flush_kern_tlb_page)
mcr p15, 0, r3, c8, c5, 0 @ invalidate I TLB
mcr p15, 0, r0, c8, c6, 1 @ invalidate D TLB entry
mov pc, lr
/*
* These two are optimised for ARM920, ARM922, ARM926, Xscale
*/
......@@ -158,28 +98,6 @@ ENTRY(v4wbi_flush_user_tlb_range)
blo 1b
mov pc, lr
/*
* v4wb_flush_tlb_page(vaddr,vma)
*
* Invalidate the specified page in the specified address space.
*
* - vaddr - virtual address (may not be aligned)
* - vma - vma_struct describing address range
*/
.align 5
ENTRY(v4wbi_flush_user_tlb_page)
vma_vm_mm r2, r1 @ get vma->vm_mm
act_mm r3 @ get current->active_mm
teq r2, r3 @ equal
movne pc, lr @ no
vma_vm_flags r2, r1
mov r3, #0
mcr p15, 0, r3, c7, c10, 4 @ drain WB
tst r2, #VM_EXEC
mcrne p15, 0, r0, c8, c5, 1 @ invalidate I TLB entry
mcr p15, 0, r0, c8, c6, 1 @ invalidate D TLB entry
mov pc, lr
ENTRY(v4wbi_flush_kern_tlb_range)
mov r3, #0
mcr p15, 0, r3, c7, c10, 4 @ drain WB
......@@ -192,29 +110,18 @@ ENTRY(v4wbi_flush_kern_tlb_range)
blo 1b
mov pc, lr
ENTRY(v4wbi_flush_kern_tlb_page)
mcr p15, 0, r0, c8, c5, 1 @ invalidate I TLB entry
mcr p15, 0, r0, c8, c6, 1 @ invalidate D TLB entry
mov pc, lr
.section ".text.init", #alloc, #execinstr
.type v4wb_tlb_fns, #object
ENTRY(v4wb_tlb_fns)
.long v4wb_flush_kern_tlb_all
.long v4wb_flush_user_tlb_mm
.long v4wb_flush_user_tlb_range
.long v4wb_flush_user_tlb_page
.long v4wb_flush_kern_tlb_range
.long v4wb_flush_kern_tlb_page
.long v4wb_tlb_flags
.size v4wb_tlb_fns, . - v4wb_tlb_fns
.type v4wbi_tlb_fns, #object
ENTRY(v4wbi_tlb_fns)
.long v4wbi_flush_kern_tlb_all
.long v4wbi_flush_user_tlb_mm
.long v4wbi_flush_user_tlb_range
.long v4wbi_flush_user_tlb_page
.long v4wbi_flush_kern_tlb_range
.long v4wbi_flush_kern_tlb_page
.long v4wbi_tlb_flags
.size v4wbi_tlb_fns, . - v4wbi_tlb_fns
include/asm-arm/glue.h
View file @ 9c7fd8c7
......@@ -26,59 +26,6 @@
/*
* MMU TLB Model
* =============
*
* We have the following to choose from:
* v3 - ARMv3
* v4 - ARMv4 without write buffer
* v4wb - ARMv4 with write buffer without I TLB flush entry instruction
* v4wbi - ARMv4 with write buffer with I TLB flush entry instruction
*/
#undef _TLB
#undef MULTI_TLB
#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710)
# ifdef _TLB
# define MULTI_TLB 1
# else
# define _TLB v3
# endif
#endif
#if defined(CONFIG_CPU_ARM720T)
# ifdef _TLB
# define MULTI_TLB 1
# else
# define _TLB v4
# endif
#endif
#if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \
defined(CONFIG_CPU_ARM926T) || defined(CONFIG_CPU_ARM1020) || \
defined(CONFIG_CPU_XSCALE)
# ifdef _TLB
# define MULTI_TLB 1
# else
# define _TLB v4wbi
# endif
#endif
#if defined(CONFIG_CPU_SA110) || defined(CONFIG_CPU_SA1100)
# ifdef _TLB
# define MULTI_TLB 1
# else
# define _TLB v4wb
# endif
#endif
#ifndef _TLB
#error Unknown TLB model
#endif
/*
* Data Abort Model
* ================
......@@ -156,69 +103,4 @@
#error Unknown data abort handler type
#endif
/*
* User Space Model
* ================
*
* This section selects the correct set of functions for dealing with
* page-based copying and clearing for user space for the particular
* processor(s) we're building for.
*
* We have the following to choose from:
* v3 - ARMv3
* v4wt - ARMv4 with writethrough cache, without minicache
* v4wb - ARMv4 with writeback cache, without minicache
* v4_mc - ARMv4 with minicache
* xscale - Xscale
*/
#undef _USER
#undef MULTI_USER
#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER v3
# endif
#endif
#if defined(CONFIG_CPU_ARM720T)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER v4wt
# endif
#endif
#if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \
defined(CONFIG_CPU_ARM926T) || defined(CONFIG_CPU_SA110) || \
defined(CONFIG_CPU_ARM1020)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER v4wb
# endif
#endif
#if defined(CONFIG_CPU_SA1100)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER v4_mc
# endif
#endif
#if defined(CONFIG_CPU_XSCALE)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER xscale_mc
# endif
#endif
#ifndef _USER
#error Unknown user operations model
#endif
#endif
include/asm-arm/page.h
View file @ 9c7fd8c7
......@@ -8,6 +8,70 @@
#include <asm/glue.h>
/*
* User Space Model
* ================
*
* This section selects the correct set of functions for dealing with
* page-based copying and clearing for user space for the particular
* processor(s) we're building for.
*
* We have the following to choose from:
* v3 - ARMv3
* v4wt - ARMv4 with writethrough cache, without minicache
* v4wb - ARMv4 with writeback cache, without minicache
* v4_mc - ARMv4 with minicache
* xscale - Xscale
*/
#undef _USER
#undef MULTI_USER
#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER v3
# endif
#endif
#if defined(CONFIG_CPU_ARM720T)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER v4wt
# endif
#endif
#if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \
defined(CONFIG_CPU_ARM926T) || defined(CONFIG_CPU_SA110) || \
defined(CONFIG_CPU_ARM1020)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER v4wb
# endif
#endif
#if defined(CONFIG_CPU_SA1100)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER v4_mc
# endif
#endif
#if defined(CONFIG_CPU_XSCALE)
# ifdef _USER
# define MULTI_USER 1
# else
# define _USER xscale_mc
# endif
#endif
#ifndef _USER
#error Unknown user operations model
#endif
struct cpu_user_fns {
void (*cpu_clear_user_page)(void *p, unsigned long user);
void (*cpu_copy_user_page)(void *to, const void *from,
......
include/asm-arm/pgalloc.h
View file @ 9c7fd8c7
......@@ -11,9 +11,6 @@
#define _ASMARM_PGALLOC_H
#include <asm/processor.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/proc/pgalloc.h>
/*
......
include/asm-arm/pgtable.h
View file @ 9c7fd8c7
......@@ -12,6 +12,7 @@
#include <linux/config.h>
#include <asm/memory.h>
#include <asm/proc-fns.h>
#include <asm/arch/vmalloc.h>
/*
......
include/asm-arm/proc-armo/system.h
View file @ 9c7fd8c7
......@@ -10,8 +10,6 @@
#ifndef __ASM_PROC_SYSTEM_H
#define __ASM_PROC_SYSTEM_H
#include <asm/proc-fns.h>
#define vectors_base() (0)
static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
......
include/asm-arm/proc-armv/pgalloc.h
View file @ 9c7fd8c7
......@@ -5,6 +5,7 @@
*
* Page table allocation/freeing primitives for 32-bit ARM processors.
*/
#include <asm/cacheflush.h>
#include "pgtable.h"
/*
......
include/asm-arm/proc-armv/tlbflush.h
View file @ 9c7fd8c7
......@@ -7,6 +7,113 @@
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*/
#include <linux/config.h>
#include <asm/glue.h>
#define TLB_V3_PAGE (1 << 0)
#define TLB_V4_U_PAGE (1 << 1)
#define TLB_V4_D_PAGE (1 << 2)
#define TLB_V4_I_PAGE (1 << 3)
#define TLB_V3_FULL (1 << 8)
#define TLB_V4_U_FULL (1 << 9)
#define TLB_V4_D_FULL (1 << 10)
#define TLB_V4_I_FULL (1 << 11)
#define TLB_WB (1 << 31)
/*
* MMU TLB Model
* =============
*
* We have the following to choose from:
* v3 - ARMv3
* v4 - ARMv4 without write buffer
* v4wb - ARMv4 with write buffer without I TLB flush entry instruction
* v4wbi - ARMv4 with write buffer with I TLB flush entry instruction
*/
#undef _TLB
#undef MULTI_TLB
#define v3_tlb_flags (TLB_V3_FULL | TLB_V3_PAGE)
#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710)
# ifdef _TLB
# define MULTI_TLB 1
# else
# define _TLB v3
# endif
#endif
#define v4_tlb_flags (TLB_V4_U_FULL | TLB_V4_U_PAGE)
#if defined(CONFIG_CPU_ARM720T)
# ifdef _TLB
# define MULTI_TLB 1
# else
# define _TLB v4
# endif
#endif
#define v4wbi_tlb_flags (TLB_WB | \
TLB_V4_I_FULL | TLB_V4_D_FULL | \
TLB_V4_I_PAGE | TLB_V4_D_PAGE)
#if defined(CONFIG_CPU_ARM920T) || defined(CONFIG_CPU_ARM922T) || \
defined(CONFIG_CPU_ARM926T) || defined(CONFIG_CPU_ARM1020) || \
defined(CONFIG_CPU_XSCALE)
# ifdef _TLB
# define MULTI_TLB 1
# else
# define _TLB v4wbi
# endif
#endif
#define v4wb_tlb_flags (TLB_WB | \
TLB_V4_I_FULL | TLB_V4_D_FULL | \
TLB_V4_D_PAGE)
#if defined(CONFIG_CPU_SA110) || defined(CONFIG_CPU_SA1100)
# ifdef _TLB
# define MULTI_TLB 1
# else
# define _TLB v4wb
# endif
#endif
#ifndef _TLB
#error Unknown TLB model
#endif
#ifndef __ASSEMBLY__
struct cpu_tlb_fns {
void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *);
void (*flush_kern_range)(unsigned long, unsigned long);
unsigned long tlb_flags;
};
/*
* Select the calling method
*/
#ifdef MULTI_TLB
extern struct cpu_tlb_fns cpu_tlb;
#define __cpu_flush_user_tlb_range cpu_tlb.flush_user_range
#define __cpu_flush_kern_tlb_range cpu_tlb.flush_kern_range
#define __cpu_tlb_flags cpu_tlb.tlb_flags
#else
#define __cpu_flush_user_tlb_range __glue(_TLB,_flush_user_tlb_range)
#define __cpu_flush_kern_tlb_range __glue(_TLB,_flush_kern_tlb_range)
#define __cpu_tlb_flags __glue(_TLB,_tlb_flags)
extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
#endif
/*
* TLB Management
......@@ -51,56 +158,94 @@
* - kaddr - Kernel virtual memory address
*/
struct cpu_tlb_fns {
void (*flush_kern_all)(void);
void (*flush_user_mm)(struct mm_struct *);
void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *);
void (*flush_user_page)(unsigned long, struct vm_area_struct *);
void (*flush_kern_range)(unsigned long, unsigned long);
void (*flush_kern_page)(unsigned long);
};
#define tlb_flag(f) (__cpu_tlb_flags & (f))
/*
* Convert calls to our calling convention.
*/
#define flush_tlb_all() __cpu_flush_kern_tlb_all()
#define flush_tlb_mm(mm) __cpu_flush_user_tlb_mm(mm)
#define flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma)
#define flush_tlb_page(vma,vaddr) __cpu_flush_user_tlb_page(vaddr,vma)
#define flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e)
#define flush_tlb_kernel_page(kaddr) __cpu_flush_kern_tlb_page(kaddr)
static inline void flush_tlb_all(void)
{
const int zero = 0;
/*
* Now select the calling method
*/
#ifdef MULTI_TLB
if (tlb_flag(TLB_WB))
asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero));
extern struct cpu_tlb_fns cpu_tlb;
if (tlb_flag(TLB_V3_FULL))
asm("mcr%? p15, 0, %0, c6, c0, 0" : : "r" (zero));
if (tlb_flag(TLB_V4_U_FULL))
asm("mcr%? p15, 0, %0, c8, c7, 0" : : "r" (zero));
if (tlb_flag(TLB_V4_D_FULL))
asm("mcr%? p15, 0, %0, c8, c6, 0" : : "r" (zero));
if (tlb_flag(TLB_V4_I_FULL))
asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero));
}
#define __cpu_flush_kern_tlb_all cpu_tlb.flush_kern_all
#define __cpu_flush_user_tlb_mm cpu_tlb.flush_user_mm
#define __cpu_flush_user_tlb_range cpu_tlb.flush_user_range
#define __cpu_flush_user_tlb_page cpu_tlb.flush_user_page
#define __cpu_flush_kern_tlb_range cpu_tlb.flush_kern_range
#define __cpu_flush_kern_tlb_page cpu_tlb.flush_kern_page
static inline void flush_tlb_mm(struct mm_struct *mm)
{
const int zero = 0;
#else
if (tlb_flag(TLB_WB))
asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero));
#define __cpu_flush_kern_tlb_all __glue(_TLB,_flush_kern_tlb_all)
#define __cpu_flush_user_tlb_mm __glue(_TLB,_flush_user_tlb_mm)
#define __cpu_flush_user_tlb_range __glue(_TLB,_flush_user_tlb_range)
#define __cpu_flush_user_tlb_page __glue(_TLB,_flush_user_tlb_page)
#define __cpu_flush_kern_tlb_range __glue(_TLB,_flush_kern_tlb_range)
#define __cpu_flush_kern_tlb_page __glue(_TLB,_flush_kern_tlb_page)
if (mm == current->active_mm) {
if (tlb_flag(TLB_V3_FULL))
asm("mcr%? p15, 0, %0, c6, c0, 0" : : "r" (zero));
if (tlb_flag(TLB_V4_U_FULL))
asm("mcr%? p15, 0, %0, c8, c7, 0" : : "r" (zero));
if (tlb_flag(TLB_V4_D_FULL))
asm("mcr%? p15, 0, %0, c8, c6, 0" : : "r" (zero));
if (tlb_flag(TLB_V4_I_FULL))
asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero));
}
}
extern void __cpu_flush_kern_tlb_all(void);
extern void __cpu_flush_user_tlb_mm(struct mm_struct *);
extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
extern void __cpu_flush_user_tlb_page(unsigned long, struct vm_area_struct *);
extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
extern void __cpu_flush_kern_tlb_page(unsigned long);
static inline void
flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
{
const int zero = 0;
#endif
uaddr &= PAGE_MASK;
if (tlb_flag(TLB_WB))
asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero));
if (vma->vm_mm == current->active_mm) {
if (tlb_flag(TLB_V3_PAGE))
asm("mcr%? p15, 0, %0, c6, c0, 0" : : "r" (uaddr));
if (tlb_flag(TLB_V4_U_PAGE))
asm("mcr%? p15, 0, %0, c8, c7, 1" : : "r" (uaddr));
if (tlb_flag(TLB_V4_D_PAGE))
asm("mcr%? p15, 0, %0, c8, c6, 1" : : "r" (uaddr));
if (tlb_flag(TLB_V4_I_PAGE))
asm("mcr%? p15, 0, %0, c8, c5, 1" : : "r" (uaddr));
if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero));
}
}
static inline void flush_tlb_kernel_page(unsigned long kaddr)
{
const int zero = 0;
kaddr &= PAGE_MASK;
if (tlb_flag(TLB_WB))
asm("mcr%? p15, 0, %0, c7, c10, 4" : : "r" (zero));
if (tlb_flag(TLB_V3_PAGE))
asm("mcr%? p15, 0, %0, c6, c0, 0" : : "r" (kaddr));
if (tlb_flag(TLB_V4_U_PAGE))
asm("mcr%? p15, 0, %0, c8, c7, 1" : : "r" (kaddr));
if (tlb_flag(TLB_V4_D_PAGE))
asm("mcr%? p15, 0, %0, c8, c6, 1" : : "r" (kaddr));
if (tlb_flag(TLB_V4_I_PAGE))
asm("mcr%? p15, 0, %0, c8, c5, 1" : : "r" (kaddr));
if (!tlb_flag(TLB_V4_I_PAGE) && tlb_flag(TLB_V4_I_FULL))
asm("mcr%? p15, 0, %0, c8, c5, 0" : : "r" (zero));
}
/*
* Convert calls to our calling convention.
*/
#define flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma)
#define flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e)
/*
* if PG_dcache_dirty is set for the page, we need to ensure that any
......@@ -123,3 +268,4 @@ extern void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte
#define memc_update_addr(mm,pte,log) do { } while (0)
#define memc_clear(mm,physaddr) do { } while (0)
#endif
include/asm-arm/procinfo.h
View file @ 9c7fd8c7
......@@ -12,8 +12,6 @@
#ifndef __ASSEMBLY__
#include <asm/proc-fns.h>
struct cpu_tlb_fns;
struct cpu_user_fns;
struct processor;
......
include/asm-arm/tlb.h
View file @ 9c7fd8c7
/*
* linux/include/asm-arm/tlb.h
*
* Copyright (C) 2002 Russell King
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* Experimentation shows that on a StrongARM, it appears to be faster
* to use the "invalidate whole tlb" rather than "invalidate single
* tlb" for this.
*
* This appears true for both the process fork+exit case, as well as
* the munmap-large-area case.
*/
#ifndef __ASMARM_TLB_H
#define __ASMARM_TLB_H
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#define tlb_flush(tlb) \
flush_tlb_mm((tlb)->mm)
#define tlb_start_vma(tlb,vma) \
flush_cache_range(vma, vma->vm_start, vma->vm_end)
#define tlb_end_vma(tlb,vma) \
flush_tlb_range(vma, vma->vm_start, vma->vm_end)
/*
* TLB handling. This allows us to remove pages from the page
* tables, and efficiently handle the TLB issues.
*/
typedef struct free_pte_ctx {
struct mm_struct *mm;
unsigned int freed;
#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
unsigned int flushes;
unsigned int avoided_flushes;
} mmu_gather_t;
#include <asm-generic/tlb.h>
extern mmu_gather_t mmu_gathers[NR_CPUS];
#define __pmd_free_tlb(tlb, pmd) pmd_free(pmd)
#define __pte_free_tlb(tlb, pte) pte_free(pte)
static inline mmu_gather_t *tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
{
int cpu = smp_processor_id();
mmu_gather_t *tlb = &mmu_gathers[cpu];
tlb->mm = mm;
tlb->freed = 0;
return tlb;
}
static inline void tlb_finish_mmu(mmu_gather_t *tlb, unsigned long start, unsigned long end)
{
struct mm_struct *mm = tlb->mm;
unsigned long freed = tlb->freed;
int rss = mm->rss;
if (rss < freed)
freed = rss;
mm->rss = rss - freed;
if (freed) {
flush_tlb_mm(mm);
tlb->flushes++;
} else {
tlb->avoided_flushes++;
}
/* keep the page table cache within bounds */
check_pgt_cache();
}
#define tlb_remove_tlb_entry(tlb,ptep,address) do { } while (0)
#define tlb_start_vma(tlb,vma) do { } while (0)
#define tlb_end_vma(tlb,vma) do { } while (0)
#define tlb_remove_page(tlb,page) free_page_and_swap_cache(page)
#define pte_free_tlb(tlb,ptep) pte_free(ptep)
#define pmd_free_tlb(tlb,pmdp) pmd_free(pmdp)
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7