Commit a741e679 authored by Benjamin Herrenschmidt's avatar Benjamin Herrenschmidt Committed by Paul Mackerras

[POWERPC] Make tlb flush batch use lazy MMU mode

The current tlb flush code on powerpc 64 bits has a subtle race since we
lost the page table lock due to the possible faulting in of new PTEs
after a previous one has been removed but before the corresponding hash
entry has been evicted, which can leads to all sort of fatal problems.

This patch reworks the batch code completely. It doesn't use the mmu_gather
stuff anymore. Instead, we use the lazy mmu hooks that were added by the
paravirt code. They have the nice property that the enter/leave lazy mmu
mode pair is always fully contained by the PTE lock for a given range
of PTEs. Thus we can guarantee that all batches are flushed on a given
CPU before it drops that lock.

We also generalize batching for any PTE update that require a flush.

Batching is now enabled on a CPU by arch_enter_lazy_mmu_mode() and
disabled by arch_leave_lazy_mmu_mode(). The code epects that this is
always contained within a PTE lock section so no preemption can happen
and no PTE insertion in that range from another CPU. When batching
is enabled on a CPU, every PTE updates that need a hash flush will
use the batch for that flush.
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: default avatarPaul Mackerras <paulus@samba.org>
parent e4ee3891
...@@ -305,9 +305,7 @@ struct task_struct *__switch_to(struct task_struct *prev, ...@@ -305,9 +305,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
set_dabr(new->thread.dabr); set_dabr(new->thread.dabr);
__get_cpu_var(current_dabr) = new->thread.dabr; __get_cpu_var(current_dabr) = new->thread.dabr;
} }
#endif /* CONFIG_PPC64 */
flush_tlb_pending();
#endif
new_thread = &new->thread; new_thread = &new->thread;
old_thread = &current->thread; old_thread = &current->thread;
......
...@@ -428,10 +428,6 @@ void generic_mach_cpu_die(void) ...@@ -428,10 +428,6 @@ void generic_mach_cpu_die(void)
smp_wmb(); smp_wmb();
while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE) while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE)
cpu_relax(); cpu_relax();
#ifdef CONFIG_PPC64
flush_tlb_pending();
#endif
cpu_set(cpu, cpu_online_map); cpu_set(cpu, cpu_online_map);
local_irq_enable(); local_irq_enable();
} }
......
...@@ -316,12 +316,11 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, ...@@ -316,12 +316,11 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
{ {
if (pte_present(*ptep)) { if (pte_present(*ptep)) {
/* We open-code pte_clear because we need to pass the right /* We open-code pte_clear because we need to pass the right
* argument to hpte_update (huge / !huge) * argument to hpte_need_flush (huge / !huge). Might not be
* necessary anymore if we make hpte_need_flush() get the
* page size from the slices
*/ */
unsigned long old = pte_update(ptep, ~0UL); pte_update(mm, addr & HPAGE_MASK, ptep, ~0UL, 1);
if (old & _PAGE_HASHPTE)
hpte_update(mm, addr & HPAGE_MASK, ptep, old, 1);
flush_tlb_pending();
} }
*ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
} }
...@@ -329,12 +328,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, ...@@ -329,12 +328,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
pte_t *ptep) pte_t *ptep)
{ {
unsigned long old = pte_update(ptep, ~0UL); unsigned long old = pte_update(mm, addr, ptep, ~0UL, 1);
if (old & _PAGE_HASHPTE)
hpte_update(mm, addr & HPAGE_MASK, ptep, old, 1);
*ptep = __pte(0);
return __pte(old); return __pte(old);
} }
......
...@@ -120,17 +120,20 @@ void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf) ...@@ -120,17 +120,20 @@ void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf)
} }
/* /*
* Update the MMU hash table to correspond with a change to * A linux PTE was changed and the corresponding hash table entry
* a Linux PTE. If wrprot is true, it is permissible to * neesd to be flushed. This function will either perform the flush
* change the existing HPTE to read-only rather than removing it * immediately or will batch it up if the current CPU has an active
* (if we remove it we should clear the _PTE_HPTEFLAGS bits). * batch on it.
*
* Must be called from within some kind of spinlock/non-preempt region...
*/ */
void hpte_update(struct mm_struct *mm, unsigned long addr, void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned long pte, int huge) pte_t *ptep, unsigned long pte, int huge)
{ {
struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
unsigned long vsid; unsigned long vsid, vaddr;
unsigned int psize; unsigned int psize;
real_pte_t rpte;
int i; int i;
i = batch->index; i = batch->index;
...@@ -151,6 +154,26 @@ void hpte_update(struct mm_struct *mm, unsigned long addr, ...@@ -151,6 +154,26 @@ void hpte_update(struct mm_struct *mm, unsigned long addr,
} else } else
psize = pte_pagesize_index(pte); psize = pte_pagesize_index(pte);
/* Build full vaddr */
if (!is_kernel_addr(addr)) {
vsid = get_vsid(mm->context.id, addr);
WARN_ON(vsid == 0);
} else
vsid = get_kernel_vsid(addr);
vaddr = (vsid << 28 ) | (addr & 0x0fffffff);
rpte = __real_pte(__pte(pte), ptep);
/*
* Check if we have an active batch on this CPU. If not, just
* flush now and return. For now, we don global invalidates
* in that case, might be worth testing the mm cpu mask though
* and decide to use local invalidates instead...
*/
if (!batch->active) {
flush_hash_page(vaddr, rpte, psize, 0);
return;
}
/* /*
* This can happen when we are in the middle of a TLB batch and * This can happen when we are in the middle of a TLB batch and
* we encounter memory pressure (eg copy_page_range when it tries * we encounter memory pressure (eg copy_page_range when it tries
...@@ -162,47 +185,42 @@ void hpte_update(struct mm_struct *mm, unsigned long addr, ...@@ -162,47 +185,42 @@ void hpte_update(struct mm_struct *mm, unsigned long addr,
* batch * batch
*/ */
if (i != 0 && (mm != batch->mm || batch->psize != psize)) { if (i != 0 && (mm != batch->mm || batch->psize != psize)) {
flush_tlb_pending(); __flush_tlb_pending(batch);
i = 0; i = 0;
} }
if (i == 0) { if (i == 0) {
batch->mm = mm; batch->mm = mm;
batch->psize = psize; batch->psize = psize;
} }
if (!is_kernel_addr(addr)) { batch->pte[i] = rpte;
vsid = get_vsid(mm->context.id, addr); batch->vaddr[i] = vaddr;
WARN_ON(vsid == 0);
} else
vsid = get_kernel_vsid(addr);
batch->vaddr[i] = (vsid << 28 ) | (addr & 0x0fffffff);
batch->pte[i] = __real_pte(__pte(pte), ptep);
batch->index = ++i; batch->index = ++i;
if (i >= PPC64_TLB_BATCH_NR) if (i >= PPC64_TLB_BATCH_NR)
flush_tlb_pending(); __flush_tlb_pending(batch);
} }
/*
* This function is called when terminating an mmu batch or when a batch
* is full. It will perform the flush of all the entries currently stored
* in a batch.
*
* Must be called from within some kind of spinlock/non-preempt region...
*/
void __flush_tlb_pending(struct ppc64_tlb_batch *batch) void __flush_tlb_pending(struct ppc64_tlb_batch *batch)
{ {
int i;
int cpu;
cpumask_t tmp; cpumask_t tmp;
int local = 0; int i, local = 0;
BUG_ON(in_interrupt());
cpu = get_cpu();
i = batch->index; i = batch->index;
tmp = cpumask_of_cpu(cpu); tmp = cpumask_of_cpu(smp_processor_id());
if (cpus_equal(batch->mm->cpu_vm_mask, tmp)) if (cpus_equal(batch->mm->cpu_vm_mask, tmp))
local = 1; local = 1;
if (i == 1) if (i == 1)
flush_hash_page(batch->vaddr[0], batch->pte[0], flush_hash_page(batch->vaddr[0], batch->pte[0],
batch->psize, local); batch->psize, local);
else else
flush_hash_range(i, local); flush_hash_range(i, local);
batch->index = 0; batch->index = 0;
put_cpu();
} }
void pte_free_finish(void) void pte_free_finish(void)
......
...@@ -272,7 +272,10 @@ static inline pte_t pte_mkhuge(pte_t pte) { ...@@ -272,7 +272,10 @@ static inline pte_t pte_mkhuge(pte_t pte) {
return pte; } return pte; }
/* Atomic PTE updates */ /* Atomic PTE updates */
static inline unsigned long pte_update(pte_t *p, unsigned long clr) static inline unsigned long pte_update(struct mm_struct *mm,
unsigned long addr,
pte_t *ptep, unsigned long clr,
int huge)
{ {
unsigned long old, tmp; unsigned long old, tmp;
...@@ -283,20 +286,15 @@ static inline unsigned long pte_update(pte_t *p, unsigned long clr) ...@@ -283,20 +286,15 @@ static inline unsigned long pte_update(pte_t *p, unsigned long clr)
andc %1,%0,%4 \n\ andc %1,%0,%4 \n\
stdcx. %1,0,%3 \n\ stdcx. %1,0,%3 \n\
bne- 1b" bne- 1b"
: "=&r" (old), "=&r" (tmp), "=m" (*p) : "=&r" (old), "=&r" (tmp), "=m" (*ptep)
: "r" (p), "r" (clr), "m" (*p), "i" (_PAGE_BUSY) : "r" (ptep), "r" (clr), "m" (*ptep), "i" (_PAGE_BUSY)
: "cc" ); : "cc" );
if (old & _PAGE_HASHPTE)
hpte_need_flush(mm, addr, ptep, old, huge);
return old; return old;
} }
/* PTE updating functions, this function puts the PTE in the
* batch, doesn't actually triggers the hash flush immediately,
* you need to call flush_tlb_pending() to do that.
* Pass -1 for "normal" size (4K or 64K)
*/
extern void hpte_update(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned long pte, int huge);
static inline int __ptep_test_and_clear_young(struct mm_struct *mm, static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
unsigned long addr, pte_t *ptep) unsigned long addr, pte_t *ptep)
{ {
...@@ -304,11 +302,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm, ...@@ -304,11 +302,7 @@ static inline int __ptep_test_and_clear_young(struct mm_struct *mm,
if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0) if ((pte_val(*ptep) & (_PAGE_ACCESSED | _PAGE_HASHPTE)) == 0)
return 0; return 0;
old = pte_update(ptep, _PAGE_ACCESSED); old = pte_update(mm, addr, ptep, _PAGE_ACCESSED, 0);
if (old & _PAGE_HASHPTE) {
hpte_update(mm, addr, ptep, old, 0);
flush_tlb_pending();
}
return (old & _PAGE_ACCESSED) != 0; return (old & _PAGE_ACCESSED) != 0;
} }
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
...@@ -331,9 +325,7 @@ static inline int __ptep_test_and_clear_dirty(struct mm_struct *mm, ...@@ -331,9 +325,7 @@ static inline int __ptep_test_and_clear_dirty(struct mm_struct *mm,
if ((pte_val(*ptep) & _PAGE_DIRTY) == 0) if ((pte_val(*ptep) & _PAGE_DIRTY) == 0)
return 0; return 0;
old = pte_update(ptep, _PAGE_DIRTY); old = pte_update(mm, addr, ptep, _PAGE_DIRTY, 0);
if (old & _PAGE_HASHPTE)
hpte_update(mm, addr, ptep, old, 0);
return (old & _PAGE_DIRTY) != 0; return (old & _PAGE_DIRTY) != 0;
} }
#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
...@@ -352,9 +344,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, ...@@ -352,9 +344,7 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
if ((pte_val(*ptep) & _PAGE_RW) == 0) if ((pte_val(*ptep) & _PAGE_RW) == 0)
return; return;
old = pte_update(ptep, _PAGE_RW); old = pte_update(mm, addr, ptep, _PAGE_RW, 0);
if (old & _PAGE_HASHPTE)
hpte_update(mm, addr, ptep, old, 0);
} }
/* /*
...@@ -378,7 +368,6 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, ...@@ -378,7 +368,6 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
({ \ ({ \
int __dirty = __ptep_test_and_clear_dirty((__vma)->vm_mm, __address, \ int __dirty = __ptep_test_and_clear_dirty((__vma)->vm_mm, __address, \
__ptep); \ __ptep); \
flush_tlb_page(__vma, __address); \
__dirty; \ __dirty; \
}) })
...@@ -386,20 +375,14 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, ...@@ -386,20 +375,14 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr,
static inline pte_t ptep_get_and_clear(struct mm_struct *mm, static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
unsigned long addr, pte_t *ptep) unsigned long addr, pte_t *ptep)
{ {
unsigned long old = pte_update(ptep, ~0UL); unsigned long old = pte_update(mm, addr, ptep, ~0UL, 0);
if (old & _PAGE_HASHPTE)
hpte_update(mm, addr, ptep, old, 0);
return __pte(old); return __pte(old);
} }
static inline void pte_clear(struct mm_struct *mm, unsigned long addr, static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
pte_t * ptep) pte_t * ptep)
{ {
unsigned long old = pte_update(ptep, ~0UL); pte_update(mm, addr, ptep, ~0UL, 0);
if (old & _PAGE_HASHPTE)
hpte_update(mm, addr, ptep, old, 0);
} }
/* /*
...@@ -408,10 +391,8 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, ...@@ -408,10 +391,8 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr,
static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, pte_t pte) pte_t *ptep, pte_t pte)
{ {
if (pte_present(*ptep)) { if (pte_present(*ptep))
pte_clear(mm, addr, ptep); pte_clear(mm, addr, ptep);
flush_tlb_pending();
}
pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS); pte = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
*ptep = pte; *ptep = pte;
} }
...@@ -522,6 +503,7 @@ void pgtable_cache_init(void); ...@@ -522,6 +503,7 @@ void pgtable_cache_init(void);
return pt; return pt;
} }
#include <asm-generic/pgtable.h> #include <asm-generic/pgtable.h>
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
...@@ -38,7 +38,6 @@ extern void pte_free_finish(void); ...@@ -38,7 +38,6 @@ extern void pte_free_finish(void);
static inline void tlb_flush(struct mmu_gather *tlb) static inline void tlb_flush(struct mmu_gather *tlb)
{ {
flush_tlb_pending();
pte_free_finish(); pte_free_finish();
} }
......
...@@ -28,6 +28,7 @@ struct mm_struct; ...@@ -28,6 +28,7 @@ struct mm_struct;
#define PPC64_TLB_BATCH_NR 192 #define PPC64_TLB_BATCH_NR 192
struct ppc64_tlb_batch { struct ppc64_tlb_batch {
int active;
unsigned long index; unsigned long index;
struct mm_struct *mm; struct mm_struct *mm;
real_pte_t pte[PPC64_TLB_BATCH_NR]; real_pte_t pte[PPC64_TLB_BATCH_NR];
...@@ -38,15 +39,30 @@ DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); ...@@ -38,15 +39,30 @@ DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch);
extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch); extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch);
static inline void flush_tlb_pending(void) extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
pte_t *ptep, unsigned long pte, int huge);
#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE
static inline void arch_enter_lazy_mmu_mode(void)
{
struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
batch->active = 1;
}
static inline void arch_leave_lazy_mmu_mode(void)
{ {
struct ppc64_tlb_batch *batch = &get_cpu_var(ppc64_tlb_batch); struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch);
if (batch->index) if (batch->index)
__flush_tlb_pending(batch); __flush_tlb_pending(batch);
put_cpu_var(ppc64_tlb_batch); batch->active = 0;
} }
#define arch_flush_lazy_mmu_mode() do {} while (0)
extern void flush_hash_page(unsigned long va, real_pte_t pte, int psize, extern void flush_hash_page(unsigned long va, real_pte_t pte, int psize,
int local); int local);
extern void flush_hash_range(unsigned long number, int local); extern void flush_hash_range(unsigned long number, int local);
...@@ -88,15 +104,12 @@ extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t); ...@@ -88,15 +104,12 @@ extern void update_mmu_cache(struct vm_area_struct *, unsigned long, pte_t);
static inline void flush_tlb_mm(struct mm_struct *mm) static inline void flush_tlb_mm(struct mm_struct *mm)
{ {
flush_tlb_pending();
} }
static inline void flush_tlb_page(struct vm_area_struct *vma, static inline void flush_tlb_page(struct vm_area_struct *vma,
unsigned long vmaddr) unsigned long vmaddr)
{ {
#ifdef CONFIG_PPC64 #ifndef CONFIG_PPC64
flush_tlb_pending();
#else
_tlbie(vmaddr); _tlbie(vmaddr);
#endif #endif
} }
...@@ -112,13 +125,11 @@ static inline void flush_tlb_page_nohash(struct vm_area_struct *vma, ...@@ -112,13 +125,11 @@ static inline void flush_tlb_page_nohash(struct vm_area_struct *vma,
static inline void flush_tlb_range(struct vm_area_struct *vma, static inline void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end) unsigned long start, unsigned long end)
{ {
flush_tlb_pending();
} }
static inline void flush_tlb_kernel_range(unsigned long start, static inline void flush_tlb_kernel_range(unsigned long start,
unsigned long end) unsigned long end)
{ {
flush_tlb_pending();
} }
#else /* 6xx, 7xx, 7xxx cpus */ #else /* 6xx, 7xx, 7xxx cpus */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment