Commit 95dff480 authored by Michael Ellerman's avatar Michael Ellerman

Merge branch 'fixes' into next

Merge our fixes branch from the 4.16 cycle.

There were a number of important fixes merged, in particular some Power9
workarounds that we want in next for testing purposes. There's also been
some conflicting changes in the CPU features code which are best merged
and tested before going upstream.
parents c0b34672 52396500
...@@ -152,6 +152,11 @@ OCXL_IOCTL_IRQ_SET_FD: ...@@ -152,6 +152,11 @@ OCXL_IOCTL_IRQ_SET_FD:
Associate an event fd to an AFU interrupt so that the user process Associate an event fd to an AFU interrupt so that the user process
can be notified when the AFU sends an interrupt. can be notified when the AFU sends an interrupt.
OCXL_IOCTL_GET_METADATA:
Obtains configuration information from the card, such at the size of
MMIO areas, the AFU version, and the PASID for the current context.
mmap mmap
---- ----
......
...@@ -101,7 +101,8 @@ $(addprefix $(obj)/,$(zlib-y)): \ ...@@ -101,7 +101,8 @@ $(addprefix $(obj)/,$(zlib-y)): \
libfdt := fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c libfdt := fdt.c fdt_ro.c fdt_wip.c fdt_sw.c fdt_rw.c fdt_strerror.c
libfdtheader := fdt.h libfdt.h libfdt_internal.h libfdtheader := fdt.h libfdt.h libfdt_internal.h
$(addprefix $(obj)/,$(libfdt) libfdt-wrapper.o simpleboot.o epapr.o opal.o): \ $(addprefix $(obj)/,$(libfdt) libfdt-wrapper.o simpleboot.o epapr.o opal.o \
treeboot-akebono.o treeboot-currituck.o treeboot-iss4xx.o): \
$(addprefix $(obj)/,$(libfdtheader)) $(addprefix $(obj)/,$(libfdtheader))
src-wlib-y := string.S crt0.S stdio.c decompress.c main.c \ src-wlib-y := string.S crt0.S stdio.c decompress.c main.c \
......
...@@ -97,6 +97,9 @@ typedef struct { ...@@ -97,6 +97,9 @@ typedef struct {
/* Number of bits in the mm_cpumask */ /* Number of bits in the mm_cpumask */
atomic_t active_cpus; atomic_t active_cpus;
/* Number of users of the external (Nest) MMU */
atomic_t copros;
/* NPU NMMU context */ /* NPU NMMU context */
struct npu_context *npu_context; struct npu_context *npu_context;
......
...@@ -47,9 +47,6 @@ extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad ...@@ -47,9 +47,6 @@ extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad
#endif #endif
extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr); extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
unsigned long page_size);
extern void radix__flush_tlb_lpid(unsigned long lpid);
extern void radix__flush_tlb_all(void); extern void radix__flush_tlb_all(void);
extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm, extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
unsigned long address); unsigned long address);
......
...@@ -215,6 +215,7 @@ static inline void cpu_feature_keys_init(void) { } ...@@ -215,6 +215,7 @@ static inline void cpu_feature_keys_init(void) { }
#define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x0000080000000000) #define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x0000080000000000)
#define CPU_FTR_P9_TM_HV_ASSIST LONG_ASM_CONST(0x0000100000000000) #define CPU_FTR_P9_TM_HV_ASSIST LONG_ASM_CONST(0x0000100000000000)
#define CPU_FTR_P9_TM_XER_SO_BUG LONG_ASM_CONST(0x0000200000000000) #define CPU_FTR_P9_TM_XER_SO_BUG LONG_ASM_CONST(0x0000200000000000)
#define CPU_FTR_P9_TLBIE_BUG LONG_ASM_CONST(0x0000400000000000)
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
...@@ -465,7 +466,8 @@ static inline void cpu_feature_keys_init(void) { } ...@@ -465,7 +466,8 @@ static inline void cpu_feature_keys_init(void) { }
CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \ CPU_FTR_STCX_CHECKS_ADDRESS | CPU_FTR_POPCNTB | CPU_FTR_POPCNTD | \
CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \ CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \ CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \
CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY) CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY | \
CPU_FTR_P9_TLBIE_BUG)
#define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \ #define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \
(~CPU_FTR_SAO)) (~CPU_FTR_SAO))
#define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9 #define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9
......
...@@ -92,15 +92,23 @@ static inline void dec_mm_active_cpus(struct mm_struct *mm) ...@@ -92,15 +92,23 @@ static inline void dec_mm_active_cpus(struct mm_struct *mm)
static inline void mm_context_add_copro(struct mm_struct *mm) static inline void mm_context_add_copro(struct mm_struct *mm)
{ {
/* /*
* On hash, should only be called once over the lifetime of * If any copro is in use, increment the active CPU count
* the context, as we can't decrement the active cpus count * in order to force TLB invalidations to be global as to
* and flush properly for the time being. * propagate to the Nest MMU.
*/ */
inc_mm_active_cpus(mm); if (atomic_inc_return(&mm->context.copros) == 1)
inc_mm_active_cpus(mm);
} }
static inline void mm_context_remove_copro(struct mm_struct *mm) static inline void mm_context_remove_copro(struct mm_struct *mm)
{ {
int c;
c = atomic_dec_if_positive(&mm->context.copros);
/* Detect imbalance between add and remove */
WARN_ON(c < 0);
/* /*
* Need to broadcast a global flush of the full mm before * Need to broadcast a global flush of the full mm before
* decrementing active_cpus count, as the next TLBI may be * decrementing active_cpus count, as the next TLBI may be
...@@ -111,7 +119,7 @@ static inline void mm_context_remove_copro(struct mm_struct *mm) ...@@ -111,7 +119,7 @@ static inline void mm_context_remove_copro(struct mm_struct *mm)
* for the time being. Invalidations will remain global if * for the time being. Invalidations will remain global if
* used on hash. * used on hash.
*/ */
if (radix_enabled()) { if (c == 0 && radix_enabled()) {
flush_all_mm(mm); flush_all_mm(mm);
dec_mm_active_cpus(mm); dec_mm_active_cpus(mm);
} }
......
...@@ -714,8 +714,10 @@ static __init void cpufeatures_cpu_quirks(void) ...@@ -714,8 +714,10 @@ static __init void cpufeatures_cpu_quirks(void)
cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST | cur_cpu_spec->cpu_features |= CPU_FTR_P9_TM_HV_ASSIST |
CPU_FTR_P9_TM_XER_SO_BUG; CPU_FTR_P9_TM_XER_SO_BUG;
if ((version & 0xffff0000) == 0x004e0000) if ((version & 0xffff0000) == 0x004e0000) {
cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR); cur_cpu_spec->cpu_features &= ~(CPU_FTR_DAWR);
cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_BUG;
}
} }
static void __init cpufeatures_setup_finished(void) static void __init cpufeatures_setup_finished(void)
...@@ -727,6 +729,9 @@ static void __init cpufeatures_setup_finished(void) ...@@ -727,6 +729,9 @@ static void __init cpufeatures_setup_finished(void)
cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE; cur_cpu_spec->cpu_features |= CPU_FTR_HVMODE;
} }
/* Make sure powerpc_base_platform is non-NULL */
powerpc_base_platform = cur_cpu_spec->platform;
system_registers.lpcr = mfspr(SPRN_LPCR); system_registers.lpcr = mfspr(SPRN_LPCR);
system_registers.hfscr = mfspr(SPRN_HFSCR); system_registers.hfscr = mfspr(SPRN_HFSCR);
system_registers.fscr = mfspr(SPRN_FSCR); system_registers.fscr = mfspr(SPRN_FSCR);
......
...@@ -706,7 +706,7 @@ EXC_COMMON_BEGIN(bad_addr_slb) ...@@ -706,7 +706,7 @@ EXC_COMMON_BEGIN(bad_addr_slb)
ld r3, PACA_EXSLB+EX_DAR(r13) ld r3, PACA_EXSLB+EX_DAR(r13)
std r3, _DAR(r1) std r3, _DAR(r1)
beq cr6, 2f beq cr6, 2f
li r10, 0x480 /* fix trap number for I-SLB miss */ li r10, 0x481 /* fix trap number for I-SLB miss */
std r10, _TRAP(r1) std r10, _TRAP(r1)
2: bl save_nvgprs 2: bl save_nvgprs
addi r3, r1, STACK_FRAME_OVERHEAD addi r3, r1, STACK_FRAME_OVERHEAD
......
...@@ -476,6 +476,14 @@ void force_external_irq_replay(void) ...@@ -476,6 +476,14 @@ void force_external_irq_replay(void)
*/ */
WARN_ON(!arch_irqs_disabled()); WARN_ON(!arch_irqs_disabled());
/*
* Interrupts must always be hard disabled before irq_happened is
* modified (to prevent lost update in case of interrupt between
* load and store).
*/
__hard_irq_disable();
local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
/* Indicate in the PACA that we have an interrupt to replay */ /* Indicate in the PACA that we have an interrupt to replay */
local_paca->irq_happened |= PACA_IRQ_EE; local_paca->irq_happened |= PACA_IRQ_EE;
} }
......
...@@ -874,7 +874,6 @@ struct ibm_arch_vec __cacheline_aligned ibm_architecture_vec = { ...@@ -874,7 +874,6 @@ struct ibm_arch_vec __cacheline_aligned ibm_architecture_vec = {
.mmu = 0, .mmu = 0,
.hash_ext = 0, .hash_ext = 0,
.radix_ext = 0, .radix_ext = 0,
.byte22 = 0,
}, },
/* option vector 6: IBM PAPR hints */ /* option vector 6: IBM PAPR hints */
......
...@@ -157,6 +157,9 @@ static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr, ...@@ -157,6 +157,9 @@ static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1) asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
: : "r" (addr), "r" (kvm->arch.lpid) : "memory"); : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG))
asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
: : "r" (addr), "r" (kvm->arch.lpid) : "memory");
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
} }
......
...@@ -473,6 +473,17 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues, ...@@ -473,6 +473,17 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
trace_tlbie(kvm->arch.lpid, 0, rbvalues[i], trace_tlbie(kvm->arch.lpid, 0, rbvalues[i],
kvm->arch.lpid, 0, 0, 0); kvm->arch.lpid, 0, 0, 0);
} }
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
/*
* Need the extra ptesync to make sure we don't
* re-order the tlbie
*/
asm volatile("ptesync": : :"memory");
asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
"r" (rbvalues[0]), "r" (kvm->arch.lpid));
}
asm volatile("eieio; tlbsync; ptesync" : : : "memory"); asm volatile("eieio; tlbsync; ptesync" : : : "memory");
kvm->arch.tlbie_lock = 0; kvm->arch.tlbie_lock = 0;
} else { } else {
......
...@@ -201,6 +201,15 @@ static inline unsigned long ___tlbie(unsigned long vpn, int psize, ...@@ -201,6 +201,15 @@ static inline unsigned long ___tlbie(unsigned long vpn, int psize,
return va; return va;
} }
static inline void fixup_tlbie(unsigned long vpn, int psize, int apsize, int ssize)
{
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
/* Need the extra ptesync to ensure we don't reorder tlbie*/
asm volatile("ptesync": : :"memory");
___tlbie(vpn, psize, apsize, ssize);
}
}
static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize) static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
{ {
unsigned long rb; unsigned long rb;
...@@ -278,6 +287,7 @@ static inline void tlbie(unsigned long vpn, int psize, int apsize, ...@@ -278,6 +287,7 @@ static inline void tlbie(unsigned long vpn, int psize, int apsize,
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
} else { } else {
__tlbie(vpn, psize, apsize, ssize); __tlbie(vpn, psize, apsize, ssize);
fixup_tlbie(vpn, psize, apsize, ssize);
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
if (lock_tlbie && !use_local) if (lock_tlbie && !use_local)
...@@ -771,7 +781,7 @@ static void native_hpte_clear(void) ...@@ -771,7 +781,7 @@ static void native_hpte_clear(void)
*/ */
static void native_flush_hash_range(unsigned long number, int local) static void native_flush_hash_range(unsigned long number, int local)
{ {
unsigned long vpn; unsigned long vpn = 0;
unsigned long hash, index, hidx, shift, slot; unsigned long hash, index, hidx, shift, slot;
struct hash_pte *hptep; struct hash_pte *hptep;
unsigned long hpte_v; unsigned long hpte_v;
...@@ -843,6 +853,10 @@ static void native_flush_hash_range(unsigned long number, int local) ...@@ -843,6 +853,10 @@ static void native_flush_hash_range(unsigned long number, int local)
__tlbie(vpn, psize, psize, ssize); __tlbie(vpn, psize, psize, ssize);
} pte_iterate_hashed_end(); } pte_iterate_hashed_end();
} }
/*
* Just do one more with the last used values.
*/
fixup_tlbie(vpn, psize, psize, ssize);
asm volatile("eieio; tlbsync; ptesync":::"memory"); asm volatile("eieio; tlbsync; ptesync":::"memory");
if (lock_tlbie) if (lock_tlbie)
......
...@@ -166,6 +166,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) ...@@ -166,6 +166,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
mm_iommu_init(mm); mm_iommu_init(mm);
#endif #endif
atomic_set(&mm->context.active_cpus, 0); atomic_set(&mm->context.active_cpus, 0);
atomic_set(&mm->context.copros, 0);
return 0; return 0;
} }
......
...@@ -481,6 +481,7 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0, ...@@ -481,6 +481,7 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
"r" (TLBIEL_INVAL_SET_LPID), "r" (lpid)); "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0); trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0);
} }
/* do we need fixup here ?*/
asm volatile("eieio; tlbsync; ptesync" : : : "memory"); asm volatile("eieio; tlbsync; ptesync" : : : "memory");
} }
EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry); EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
......
...@@ -119,6 +119,49 @@ static inline void __tlbie_pid(unsigned long pid, unsigned long ric) ...@@ -119,6 +119,49 @@ static inline void __tlbie_pid(unsigned long pid, unsigned long ric)
trace_tlbie(0, 0, rb, rs, ric, prs, r); trace_tlbie(0, 0, rb, rs, ric, prs, r);
} }
static inline void __tlbiel_va(unsigned long va, unsigned long pid,
unsigned long ap, unsigned long ric)
{
unsigned long rb,rs,prs,r;
rb = va & ~(PPC_BITMASK(52, 63));
rb |= ap << PPC_BITLSHIFT(58);
rs = pid << PPC_BITLSHIFT(31);
prs = 1; /* process scoped */
r = 1; /* radix format */
asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
trace_tlbie(0, 1, rb, rs, ric, prs, r);
}
static inline void __tlbie_va(unsigned long va, unsigned long pid,
unsigned long ap, unsigned long ric)
{
unsigned long rb,rs,prs,r;
rb = va & ~(PPC_BITMASK(52, 63));
rb |= ap << PPC_BITLSHIFT(58);
rs = pid << PPC_BITLSHIFT(31);
prs = 1; /* process scoped */
r = 1; /* radix format */
asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
trace_tlbie(0, 0, rb, rs, ric, prs, r);
}
static inline void fixup_tlbie(void)
{
unsigned long pid = 0;
unsigned long va = ((1UL << 52) - 1);
if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
asm volatile("ptesync": : :"memory");
__tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
}
}
/* /*
* We use 128 set in radix mode and 256 set in hpt mode. * We use 128 set in radix mode and 256 set in hpt mode.
*/ */
...@@ -151,24 +194,25 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric) ...@@ -151,24 +194,25 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
static inline void _tlbie_pid(unsigned long pid, unsigned long ric) static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
{ {
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
__tlbie_pid(pid, ric);
asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
static inline void __tlbiel_va(unsigned long va, unsigned long pid, /*
unsigned long ap, unsigned long ric) * Workaround the fact that the "ric" argument to __tlbie_pid
{ * must be a compile-time contraint to match the "i" constraint
unsigned long rb,rs,prs,r; * in the asm statement.
*/
rb = va & ~(PPC_BITMASK(52, 63)); switch (ric) {
rb |= ap << PPC_BITLSHIFT(58); case RIC_FLUSH_TLB:
rs = pid << PPC_BITLSHIFT(31); __tlbie_pid(pid, RIC_FLUSH_TLB);
prs = 1; /* process scoped */ break;
r = 1; /* radix format */ case RIC_FLUSH_PWC:
__tlbie_pid(pid, RIC_FLUSH_PWC);
asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1) break;
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); case RIC_FLUSH_ALL:
trace_tlbie(0, 1, rb, rs, ric, prs, r); default:
__tlbie_pid(pid, RIC_FLUSH_ALL);
}
fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
static inline void __tlbiel_va_range(unsigned long start, unsigned long end, static inline void __tlbiel_va_range(unsigned long start, unsigned long end,
...@@ -203,22 +247,6 @@ static inline void _tlbiel_va_range(unsigned long start, unsigned long end, ...@@ -203,22 +247,6 @@ static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
} }
static inline void __tlbie_va(unsigned long va, unsigned long pid,
unsigned long ap, unsigned long ric)
{
unsigned long rb,rs,prs,r;
rb = va & ~(PPC_BITMASK(52, 63));
rb |= ap << PPC_BITLSHIFT(58);
rs = pid << PPC_BITLSHIFT(31);
prs = 1; /* process scoped */
r = 1; /* radix format */
asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
trace_tlbie(0, 0, rb, rs, ric, prs, r);
}
static inline void __tlbie_va_range(unsigned long start, unsigned long end, static inline void __tlbie_va_range(unsigned long start, unsigned long end,
unsigned long pid, unsigned long page_size, unsigned long pid, unsigned long page_size,
unsigned long psize) unsigned long psize)
...@@ -237,6 +265,7 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid, ...@@ -237,6 +265,7 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid,
asm volatile("ptesync": : :"memory"); asm volatile("ptesync": : :"memory");
__tlbie_va(va, pid, ap, ric); __tlbie_va(va, pid, ap, ric);
fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
...@@ -248,6 +277,7 @@ static inline void _tlbie_va_range(unsigned long start, unsigned long end, ...@@ -248,6 +277,7 @@ static inline void _tlbie_va_range(unsigned long start, unsigned long end,
if (also_pwc) if (also_pwc)
__tlbie_pid(pid, RIC_FLUSH_PWC); __tlbie_pid(pid, RIC_FLUSH_PWC);
__tlbie_va_range(start, end, pid, page_size, psize); __tlbie_va_range(start, end, pid, page_size, psize);
fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
...@@ -311,6 +341,16 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd ...@@ -311,6 +341,16 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd
} }
EXPORT_SYMBOL(radix__local_flush_tlb_page); EXPORT_SYMBOL(radix__local_flush_tlb_page);
static bool mm_needs_flush_escalation(struct mm_struct *mm)
{
/*
* P9 nest MMU has issues with the page walk cache
* caching PTEs and not flushing them properly when
* RIC = 0 for a PID/LPID invalidate
*/
return atomic_read(&mm->context.copros) != 0;
}
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
void radix__flush_tlb_mm(struct mm_struct *mm) void radix__flush_tlb_mm(struct mm_struct *mm)
{ {
...@@ -321,9 +361,12 @@ void radix__flush_tlb_mm(struct mm_struct *mm) ...@@ -321,9 +361,12 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
return; return;
preempt_disable(); preempt_disable();
if (!mm_is_thread_local(mm)) if (!mm_is_thread_local(mm)) {
_tlbie_pid(pid, RIC_FLUSH_TLB); if (mm_needs_flush_escalation(mm))
else _tlbie_pid(pid, RIC_FLUSH_ALL);
else
_tlbie_pid(pid, RIC_FLUSH_TLB);
} else
_tlbiel_pid(pid, RIC_FLUSH_TLB); _tlbiel_pid(pid, RIC_FLUSH_TLB);
preempt_enable(); preempt_enable();
} }
...@@ -435,10 +478,14 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, ...@@ -435,10 +478,14 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
} }
if (full) { if (full) {
if (local) if (local) {
_tlbiel_pid(pid, RIC_FLUSH_TLB); _tlbiel_pid(pid, RIC_FLUSH_TLB);
else } else {
_tlbie_pid(pid, RIC_FLUSH_TLB); if (mm_needs_flush_escalation(mm))
_tlbie_pid(pid, RIC_FLUSH_ALL);
else
_tlbie_pid(pid, RIC_FLUSH_TLB);
}
} else { } else {
bool hflush = false; bool hflush = false;
unsigned long hstart, hend; unsigned long hstart, hend;
...@@ -465,6 +512,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start, ...@@ -465,6 +512,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
if (hflush) if (hflush)
__tlbie_va_range(hstart, hend, pid, __tlbie_va_range(hstart, hend, pid,
HPAGE_PMD_SIZE, MMU_PAGE_2M); HPAGE_PMD_SIZE, MMU_PAGE_2M);
fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory"); asm volatile("eieio; tlbsync; ptesync": : :"memory");
} }
} }
...@@ -548,6 +596,9 @@ static inline void __radix__flush_tlb_range_psize(struct mm_struct *mm, ...@@ -548,6 +596,9 @@ static inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
} }
if (full) { if (full) {
if (!local && mm_needs_flush_escalation(mm))
also_pwc = true;
if (local) if (local)
_tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB); _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
else else
...@@ -603,46 +654,6 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr) ...@@ -603,46 +654,6 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
} }
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
unsigned long page_size)
{
unsigned long rb,rs,prs,r;
unsigned long ap;
unsigned long ric = RIC_FLUSH_TLB;
ap = mmu_get_ap(radix_get_mmu_psize(page_size));
rb = gpa & ~(PPC_BITMASK(52, 63));
rb |= ap << PPC_BITLSHIFT(58);
rs = lpid & ((1UL << 32) - 1);
prs = 0; /* process scoped */
r = 1; /* radix format */
asm volatile("ptesync": : :"memory");
asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
asm volatile("eieio; tlbsync; ptesync": : :"memory");
trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
}
EXPORT_SYMBOL(radix__flush_tlb_lpid_va);
void radix__flush_tlb_lpid(unsigned long lpid)
{
unsigned long rb,rs,prs,r;
unsigned long ric = RIC_FLUSH_ALL;
rb = 0x2 << PPC_BITLSHIFT(53); /* IS = 2 */
rs = lpid & ((1UL << 32) - 1);
prs = 0; /* partition scoped */
r = 1; /* radix format */
asm volatile("ptesync": : :"memory");
asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
asm volatile("eieio; tlbsync; ptesync": : :"memory");
trace_tlbie(lpid, 0, rb, rs, ric, prs, r);
}
EXPORT_SYMBOL(radix__flush_tlb_lpid);
void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end) unsigned long start, unsigned long end)
{ {
......
...@@ -102,10 +102,32 @@ static long afu_ioctl_attach(struct ocxl_context *ctx, ...@@ -102,10 +102,32 @@ static long afu_ioctl_attach(struct ocxl_context *ctx,
return rc; return rc;
} }
static long afu_ioctl_get_metadata(struct ocxl_context *ctx,
struct ocxl_ioctl_metadata __user *uarg)
{
struct ocxl_ioctl_metadata arg;
memset(&arg, 0, sizeof(arg));
arg.version = 0;
arg.afu_version_major = ctx->afu->config.version_major;
arg.afu_version_minor = ctx->afu->config.version_minor;
arg.pasid = ctx->pasid;
arg.pp_mmio_size = ctx->afu->config.pp_mmio_stride;
arg.global_mmio_size = ctx->afu->config.global_mmio_size;
if (copy_to_user(uarg, &arg, sizeof(arg)))
return -EFAULT;
return 0;
}
#define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" : \ #define CMD_STR(x) (x == OCXL_IOCTL_ATTACH ? "ATTACH" : \
x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" : \ x == OCXL_IOCTL_IRQ_ALLOC ? "IRQ_ALLOC" : \
x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" : \ x == OCXL_IOCTL_IRQ_FREE ? "IRQ_FREE" : \
x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" : \ x == OCXL_IOCTL_IRQ_SET_FD ? "IRQ_SET_FD" : \
x == OCXL_IOCTL_GET_METADATA ? "GET_METADATA" : \
"UNKNOWN") "UNKNOWN")
static long afu_ioctl(struct file *file, unsigned int cmd, static long afu_ioctl(struct file *file, unsigned int cmd,
...@@ -159,6 +181,11 @@ static long afu_ioctl(struct file *file, unsigned int cmd, ...@@ -159,6 +181,11 @@ static long afu_ioctl(struct file *file, unsigned int cmd,
irq_fd.eventfd); irq_fd.eventfd);
break; break;
case OCXL_IOCTL_GET_METADATA:
rc = afu_ioctl_get_metadata(ctx,
(struct ocxl_ioctl_metadata __user *) args);
break;
default: default:
rc = -EINVAL; rc = -EINVAL;
} }
......
...@@ -32,6 +32,22 @@ struct ocxl_ioctl_attach { ...@@ -32,6 +32,22 @@ struct ocxl_ioctl_attach {
__u64 reserved3; __u64 reserved3;
}; };
struct ocxl_ioctl_metadata {
__u16 version; // struct version, always backwards compatible
// Version 0 fields
__u8 afu_version_major;
__u8 afu_version_minor;
__u32 pasid; // PASID assigned to the current context
__u64 pp_mmio_size; // Per PASID MMIO size
__u64 global_mmio_size;
// End version 0 fields
__u64 reserved[13]; // Total of 16*u64
};
struct ocxl_ioctl_irq_fd { struct ocxl_ioctl_irq_fd {
__u64 irq_offset; __u64 irq_offset;
__s32 eventfd; __s32 eventfd;
...@@ -45,5 +61,6 @@ struct ocxl_ioctl_irq_fd { ...@@ -45,5 +61,6 @@ struct ocxl_ioctl_irq_fd {
#define OCXL_IOCTL_IRQ_ALLOC _IOR(OCXL_MAGIC, 0x11, __u64) #define OCXL_IOCTL_IRQ_ALLOC _IOR(OCXL_MAGIC, 0x11, __u64)
#define OCXL_IOCTL_IRQ_FREE _IOW(OCXL_MAGIC, 0x12, __u64) #define OCXL_IOCTL_IRQ_FREE _IOW(OCXL_MAGIC, 0x12, __u64)
#define OCXL_IOCTL_IRQ_SET_FD _IOW(OCXL_MAGIC, 0x13, struct ocxl_ioctl_irq_fd) #define OCXL_IOCTL_IRQ_SET_FD _IOW(OCXL_MAGIC, 0x13, struct ocxl_ioctl_irq_fd)
#define OCXL_IOCTL_GET_METADATA _IOR(OCXL_MAGIC, 0x14, struct ocxl_ioctl_metadata)
#endif /* _UAPI_MISC_OCXL_H */ #endif /* _UAPI_MISC_OCXL_H */
...@@ -135,6 +135,16 @@ static int run_test(void *addr, unsigned long size) ...@@ -135,6 +135,16 @@ static int run_test(void *addr, unsigned long size)
return 0; return 0;
} }
static int syscall_available(void)
{
int rc;
errno = 0;
rc = syscall(__NR_subpage_prot, 0, 0, 0);
return rc == 0 || (errno != ENOENT && errno != ENOSYS);
}
int test_anon(void) int test_anon(void)
{ {
unsigned long align; unsigned long align;
...@@ -145,6 +155,8 @@ int test_anon(void) ...@@ -145,6 +155,8 @@ int test_anon(void)
void *mallocblock; void *mallocblock;
unsigned long mallocsize; unsigned long mallocsize;
SKIP_IF(!syscall_available());
if (getpagesize() != 0x10000) { if (getpagesize() != 0x10000) {
fprintf(stderr, "Kernel page size must be 64K!\n"); fprintf(stderr, "Kernel page size must be 64K!\n");
return 1; return 1;
...@@ -180,6 +192,8 @@ int test_file(void) ...@@ -180,6 +192,8 @@ int test_file(void)
off_t filesize; off_t filesize;
int fd; int fd;
SKIP_IF(!syscall_available());
fd = open(file_name, O_RDWR); fd = open(file_name, O_RDWR);
if (fd == -1) { if (fd == -1) {
perror("failed to open file"); perror("failed to open file");
......
...@@ -16,7 +16,7 @@ $(OUTPUT)/tm-syscall: tm-syscall-asm.S ...@@ -16,7 +16,7 @@ $(OUTPUT)/tm-syscall: tm-syscall-asm.S
$(OUTPUT)/tm-syscall: CFLAGS += -I../../../../../usr/include $(OUTPUT)/tm-syscall: CFLAGS += -I../../../../../usr/include
$(OUTPUT)/tm-tmspr: CFLAGS += -pthread $(OUTPUT)/tm-tmspr: CFLAGS += -pthread
$(OUTPUT)/tm-vmx-unavail: CFLAGS += -pthread -m64 $(OUTPUT)/tm-vmx-unavail: CFLAGS += -pthread -m64
$(OUTPUT)/tm-resched-dscr: ../pmu/lib.o $(OUTPUT)/tm-resched-dscr: ../pmu/lib.c
$(OUTPUT)/tm-unavailable: CFLAGS += -O0 -pthread -m64 -Wno-error=uninitialized -mvsx $(OUTPUT)/tm-unavailable: CFLAGS += -O0 -pthread -m64 -Wno-error=uninitialized -mvsx
$(OUTPUT)/tm-trap: CFLAGS += -O0 -pthread -m64 $(OUTPUT)/tm-trap: CFLAGS += -O0 -pthread -m64
......
...@@ -255,6 +255,8 @@ int tm_trap_test(void) ...@@ -255,6 +255,8 @@ int tm_trap_test(void)
struct sigaction trap_sa; struct sigaction trap_sa;
SKIP_IF(!have_htm());
trap_sa.sa_flags = SA_SIGINFO; trap_sa.sa_flags = SA_SIGINFO;
trap_sa.sa_sigaction = trap_signal_handler; trap_sa.sa_sigaction = trap_signal_handler;
sigaction(SIGTRAP, &trap_sa, NULL); sigaction(SIGTRAP, &trap_sa, NULL);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment