Commit 08f051ed authored by Andrew Waterman's avatar Andrew Waterman Committed by Palmer Dabbelt

RISC-V: Flush I$ when making a dirty page executable

The RISC-V ISA allows for instruction caches that are not coherent WRT
stores, even on a single hart.  As a result, we need to explicitly flush
the instruction cache whenever marking a dirty page as executable in
order to preserve the correct system behavior.

Local instruction caches aren't that scary (our implementations actually
flush the cache, but RISC-V is defined to allow higher-performance
implementations to exist), but RISC-V defines no way to perform an
instruction cache shootdown.  When explicitly asked to do so we can
shoot down remote instruction caches via an IPI, but this is a bit on
the slow side.

Instead of requiring an IPI to all harts whenever marking a page as
executable, we simply flush the currently running harts.  In order to
maintain correct behavior, we additionally mark every other hart as
needing a deferred instruction cache which will be taken before anything
runs on it.
Signed-off-by: default avatarAndrew Waterman <andrew@sifive.com>
Signed-off-by: default avatarPalmer Dabbelt <palmer@sifive.com>
parent 28dfbe6e
...@@ -18,21 +18,37 @@ ...@@ -18,21 +18,37 @@
#undef flush_icache_range #undef flush_icache_range
#undef flush_icache_user_range #undef flush_icache_user_range
#undef flush_dcache_page
static inline void local_flush_icache_all(void) static inline void local_flush_icache_all(void)
{ {
asm volatile ("fence.i" ::: "memory"); asm volatile ("fence.i" ::: "memory");
} }
#define PG_dcache_clean PG_arch_1
static inline void flush_dcache_page(struct page *page)
{
if (test_bit(PG_dcache_clean, &page->flags))
clear_bit(PG_dcache_clean, &page->flags);
}
/*
* RISC-V doesn't have an instruction to flush parts of the instruction cache,
* so instead we just flush the whole thing.
*/
#define flush_icache_range(start, end) flush_icache_all()
#define flush_icache_user_range(vma, pg, addr, len) flush_icache_all()
#ifndef CONFIG_SMP #ifndef CONFIG_SMP
#define flush_icache_range(start, end) local_flush_icache_all() #define flush_icache_all() local_flush_icache_all()
#define flush_icache_user_range(vma, pg, addr, len) local_flush_icache_all() #define flush_icache_mm(mm, local) flush_icache_all()
#else /* CONFIG_SMP */ #else /* CONFIG_SMP */
#define flush_icache_range(start, end) sbi_remote_fence_i(0) #define flush_icache_all() sbi_remote_fence_i(0)
#define flush_icache_user_range(vma, pg, addr, len) sbi_remote_fence_i(0) void flush_icache_mm(struct mm_struct *mm, bool local);
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
......
...@@ -19,6 +19,10 @@ ...@@ -19,6 +19,10 @@
typedef struct { typedef struct {
void *vdso; void *vdso;
#ifdef CONFIG_SMP
/* A local icache flush is needed before user execution can resume. */
cpumask_t icache_stale_mask;
#endif
} mm_context_t; } mm_context_t;
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
/* /*
* Copyright (C) 2012 Regents of the University of California * Copyright (C) 2012 Regents of the University of California
* Copyright (C) 2017 SiFive
* *
* This program is free software; you can redistribute it and/or * This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License * modify it under the terms of the GNU General Public License
...@@ -19,6 +20,7 @@ ...@@ -19,6 +20,7 @@
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include <asm/cacheflush.h>
static inline void enter_lazy_tlb(struct mm_struct *mm, static inline void enter_lazy_tlb(struct mm_struct *mm,
struct task_struct *task) struct task_struct *task)
...@@ -46,12 +48,54 @@ static inline void set_pgdir(pgd_t *pgd) ...@@ -46,12 +48,54 @@ static inline void set_pgdir(pgd_t *pgd)
csr_write(sptbr, virt_to_pfn(pgd) | SPTBR_MODE); csr_write(sptbr, virt_to_pfn(pgd) | SPTBR_MODE);
} }
/*
* When necessary, performs a deferred icache flush for the given MM context,
* on the local CPU. RISC-V has no direct mechanism for instruction cache
* shoot downs, so instead we send an IPI that informs the remote harts they
* need to flush their local instruction caches. To avoid pathologically slow
* behavior in a common case (a bunch of single-hart processes on a many-hart
* machine, ie 'make -j') we avoid the IPIs for harts that are not currently
* executing a MM context and instead schedule a deferred local instruction
* cache flush to be performed before execution resumes on each hart. This
* actually performs that local instruction cache flush, which implicitly only
* refers to the current hart.
*/
static inline void flush_icache_deferred(struct mm_struct *mm)
{
#ifdef CONFIG_SMP
unsigned int cpu = smp_processor_id();
cpumask_t *mask = &mm->context.icache_stale_mask;
if (cpumask_test_cpu(cpu, mask)) {
cpumask_clear_cpu(cpu, mask);
/*
* Ensure the remote hart's writes are visible to this hart.
* This pairs with a barrier in flush_icache_mm.
*/
smp_mb();
local_flush_icache_all();
}
#endif
}
static inline void switch_mm(struct mm_struct *prev, static inline void switch_mm(struct mm_struct *prev,
struct mm_struct *next, struct task_struct *task) struct mm_struct *next, struct task_struct *task)
{ {
if (likely(prev != next)) { if (likely(prev != next)) {
/*
* Mark the current MM context as inactive, and the next as
* active. This is at least used by the icache flushing
* routines in order to determine who should
*/
unsigned int cpu = smp_processor_id();
cpumask_clear_cpu(cpu, mm_cpumask(prev));
cpumask_set_cpu(cpu, mm_cpumask(next));
set_pgdir(next->pgd); set_pgdir(next->pgd);
local_flush_tlb_all(); local_flush_tlb_all();
flush_icache_deferred(next);
} }
} }
......
...@@ -178,28 +178,6 @@ static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long addr) ...@@ -178,28 +178,6 @@ static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long addr)
#define pte_offset_map(dir, addr) pte_offset_kernel((dir), (addr)) #define pte_offset_map(dir, addr) pte_offset_kernel((dir), (addr))
#define pte_unmap(pte) ((void)(pte)) #define pte_unmap(pte) ((void)(pte))
/*
* Certain architectures need to do special things when PTEs within
* a page table are directly modified. Thus, the following hook is
* made available.
*/
static inline void set_pte(pte_t *ptep, pte_t pteval)
{
*ptep = pteval;
}
static inline void set_pte_at(struct mm_struct *mm,
unsigned long addr, pte_t *ptep, pte_t pteval)
{
set_pte(ptep, pteval);
}
static inline void pte_clear(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
set_pte_at(mm, addr, ptep, __pte(0));
}
static inline int pte_present(pte_t pte) static inline int pte_present(pte_t pte)
{ {
return (pte_val(pte) & _PAGE_PRESENT); return (pte_val(pte) & _PAGE_PRESENT);
...@@ -210,21 +188,22 @@ static inline int pte_none(pte_t pte) ...@@ -210,21 +188,22 @@ static inline int pte_none(pte_t pte)
return (pte_val(pte) == 0); return (pte_val(pte) == 0);
} }
/* static inline int pte_read(pte_t pte) */
static inline int pte_write(pte_t pte) static inline int pte_write(pte_t pte)
{ {
return pte_val(pte) & _PAGE_WRITE; return pte_val(pte) & _PAGE_WRITE;
} }
static inline int pte_exec(pte_t pte)
{
return pte_val(pte) & _PAGE_EXEC;
}
static inline int pte_huge(pte_t pte) static inline int pte_huge(pte_t pte)
{ {
return pte_present(pte) return pte_present(pte)
&& (pte_val(pte) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC)); && (pte_val(pte) & (_PAGE_READ | _PAGE_WRITE | _PAGE_EXEC));
} }
/* static inline int pte_exec(pte_t pte) */
static inline int pte_dirty(pte_t pte) static inline int pte_dirty(pte_t pte)
{ {
return pte_val(pte) & _PAGE_DIRTY; return pte_val(pte) & _PAGE_DIRTY;
...@@ -311,6 +290,33 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b) ...@@ -311,6 +290,33 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b)
return pte_val(pte_a) == pte_val(pte_b); return pte_val(pte_a) == pte_val(pte_b);
} }
/*
* Certain architectures need to do special things when PTEs within
* a page table are directly modified. Thus, the following hook is
* made available.
*/
static inline void set_pte(pte_t *ptep, pte_t pteval)
{
*ptep = pteval;
}
void flush_icache_pte(pte_t pte);
static inline void set_pte_at(struct mm_struct *mm,
unsigned long addr, pte_t *ptep, pte_t pteval)
{
if (pte_present(pteval) && pte_exec(pteval))
flush_icache_pte(pteval);
set_pte(ptep, pteval);
}
static inline void pte_clear(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
set_pte_at(mm, addr, ptep, __pte(0));
}
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
static inline int ptep_set_access_flags(struct vm_area_struct *vma, static inline int ptep_set_access_flags(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep, unsigned long address, pte_t *ptep,
......
...@@ -17,6 +17,8 @@ ...@@ -17,6 +17,8 @@
#ifdef CONFIG_MMU #ifdef CONFIG_MMU
#include <linux/mm_types.h>
/* Flush entire local TLB */ /* Flush entire local TLB */
static inline void local_flush_tlb_all(void) static inline void local_flush_tlb_all(void)
{ {
......
...@@ -108,3 +108,51 @@ void smp_send_reschedule(int cpu) ...@@ -108,3 +108,51 @@ void smp_send_reschedule(int cpu)
{ {
send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE); send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
} }
/*
* Performs an icache flush for the given MM context. RISC-V has no direct
* mechanism for instruction cache shoot downs, so instead we send an IPI that
* informs the remote harts they need to flush their local instruction caches.
* To avoid pathologically slow behavior in a common case (a bunch of
* single-hart processes on a many-hart machine, ie 'make -j') we avoid the
* IPIs for harts that are not currently executing a MM context and instead
* schedule a deferred local instruction cache flush to be performed before
* execution resumes on each hart.
*/
void flush_icache_mm(struct mm_struct *mm, bool local)
{
unsigned int cpu;
cpumask_t others, *mask;
preempt_disable();
/* Mark every hart's icache as needing a flush for this MM. */
mask = &mm->context.icache_stale_mask;
cpumask_setall(mask);
/* Flush this hart's I$ now, and mark it as flushed. */
cpu = smp_processor_id();
cpumask_clear_cpu(cpu, mask);
local_flush_icache_all();
/*
* Flush the I$ of other harts concurrently executing, and mark them as
* flushed.
*/
cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
local |= cpumask_empty(&others);
if (mm != current->active_mm || !local)
sbi_remote_fence_i(others.bits);
else {
/*
* It's assumed that at least one strongly ordered operation is
* performed on this hart between setting a hart's cpumask bit
* and scheduling this MM context on that hart. Sending an SBI
* remote message will do this, but in the case where no
* messages are sent we still need to order this hart's writes
* with flush_icache_deferred().
*/
smp_mb();
}
preempt_enable();
}
...@@ -2,3 +2,4 @@ obj-y += init.o ...@@ -2,3 +2,4 @@ obj-y += init.o
obj-y += fault.o obj-y += fault.o
obj-y += extable.o obj-y += extable.o
obj-y += ioremap.o obj-y += ioremap.o
obj-y += cacheflush.o
/*
* Copyright (C) 2017 SiFive
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation, version 2.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*/
#include <asm/pgtable.h>
#include <asm/cacheflush.h>
void flush_icache_pte(pte_t pte)
{
struct page *page = pte_page(pte);
if (!test_and_set_bit(PG_dcache_clean, &page->flags))
flush_icache_all();
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment