Commit 858eaaa7 authored by Nadav Amit's avatar Nadav Amit Committed by Linus Torvalds

mm/rmap: batched invalidations should use existing api

The recently introduced batched invalidations mechanism uses its own
mechanism for shootdown.  However, it does wrong accounting of
interrupts (e.g., inc_irq_stat is called for local invalidations),
trace-points (e.g., TLB_REMOTE_SHOOTDOWN for local invalidations) and
may break some platforms as it bypasses the invalidation mechanisms of
Xen and SGI UV.

This patch reuses the existing TLB flushing mechnaisms instead.  We use
NULL as mm to indicate a global invalidation is required.

Fixes 72b252ae ("mm: send one IPI per CPU to TLB flush all entries after unmapping pages")
Signed-off-by: default avatarNadav Amit <namit@vmware.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Rik van Riel <riel@redhat.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Ingo Molnar <mingo@kernel.org>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 18c98243
...@@ -319,12 +319,6 @@ static inline void reset_lazy_tlbstate(void) ...@@ -319,12 +319,6 @@ static inline void reset_lazy_tlbstate(void)
#endif /* SMP */ #endif /* SMP */
/* Not inlined due to inc_irq_stat not being defined yet */
#define flush_tlb_local() { \
inc_irq_stat(irq_tlb_count); \
local_flush_tlb(); \
}
#ifndef CONFIG_PARAVIRT #ifndef CONFIG_PARAVIRT
#define flush_tlb_others(mask, mm, start, end) \ #define flush_tlb_others(mask, mm, start, end) \
native_flush_tlb_others(mask, mm, start, end) native_flush_tlb_others(mask, mm, start, end)
......
...@@ -104,7 +104,7 @@ static void flush_tlb_func(void *info) ...@@ -104,7 +104,7 @@ static void flush_tlb_func(void *info)
inc_irq_stat(irq_tlb_count); inc_irq_stat(irq_tlb_count);
if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) if (f->flush_mm && f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
return; return;
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED); count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
......
...@@ -569,19 +569,6 @@ void page_unlock_anon_vma_read(struct anon_vma *anon_vma) ...@@ -569,19 +569,6 @@ void page_unlock_anon_vma_read(struct anon_vma *anon_vma)
} }
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
static void percpu_flush_tlb_batch_pages(void *data)
{
/*
* All TLB entries are flushed on the assumption that it is
* cheaper to flush all TLBs and let them be refilled than
* flushing individual PFNs. Note that we do not track mm's
* to flush as that might simply be multiple full TLB flushes
* for no gain.
*/
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
flush_tlb_local();
}
/* /*
* Flush TLB entries for recently unmapped pages from remote CPUs. It is * Flush TLB entries for recently unmapped pages from remote CPUs. It is
* important if a PTE was dirty when it was unmapped that it's flushed * important if a PTE was dirty when it was unmapped that it's flushed
...@@ -598,15 +585,14 @@ void try_to_unmap_flush(void) ...@@ -598,15 +585,14 @@ void try_to_unmap_flush(void)
cpu = get_cpu(); cpu = get_cpu();
trace_tlb_flush(TLB_REMOTE_SHOOTDOWN, -1UL); if (cpumask_test_cpu(cpu, &tlb_ubc->cpumask)) {
count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
if (cpumask_test_cpu(cpu, &tlb_ubc->cpumask)) local_flush_tlb();
percpu_flush_tlb_batch_pages(&tlb_ubc->cpumask); trace_tlb_flush(TLB_LOCAL_SHOOTDOWN, TLB_FLUSH_ALL);
if (cpumask_any_but(&tlb_ubc->cpumask, cpu) < nr_cpu_ids) {
smp_call_function_many(&tlb_ubc->cpumask,
percpu_flush_tlb_batch_pages, (void *)tlb_ubc, true);
} }
if (cpumask_any_but(&tlb_ubc->cpumask, cpu) < nr_cpu_ids)
flush_tlb_others(&tlb_ubc->cpumask, NULL, 0, TLB_FLUSH_ALL);
cpumask_clear(&tlb_ubc->cpumask); cpumask_clear(&tlb_ubc->cpumask);
tlb_ubc->flush_required = false; tlb_ubc->flush_required = false;
tlb_ubc->writable = false; tlb_ubc->writable = false;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment