Commit 50d7280d authored by Martin Schwidefsky's avatar Martin Schwidefsky Committed by Martin Schwidefsky

[S390] fault handler performance optimization.

Slim down the do_exception function to handle only the fast path of a
fault and move the exceptional cases into a new function. That slightly
increases the performance of the fault handling.

Build fix for !CONFIG_COMPAT by
Kamalesh Babulal <kamalesh@linux.vnet.ibm.com>
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent 7ecb344a
...@@ -34,16 +34,15 @@ ...@@ -34,16 +34,15 @@
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/s390_ext.h> #include <asm/s390_ext.h>
#include <asm/mmu_context.h> #include <asm/mmu_context.h>
#include <asm/compat.h>
#include "../kernel/entry.h" #include "../kernel/entry.h"
#ifndef CONFIG_64BIT #ifndef CONFIG_64BIT
#define __FAIL_ADDR_MASK 0x7ffff000 #define __FAIL_ADDR_MASK 0x7ffff000
#define __FIXUP_MASK 0x7fffffff
#define __SUBCODE_MASK 0x0200 #define __SUBCODE_MASK 0x0200
#define __PF_RES_FIELD 0ULL #define __PF_RES_FIELD 0ULL
#else /* CONFIG_64BIT */ #else /* CONFIG_64BIT */
#define __FAIL_ADDR_MASK -4096L #define __FAIL_ADDR_MASK -4096L
#define __FIXUP_MASK ~0L
#define __SUBCODE_MASK 0x0600 #define __SUBCODE_MASK 0x0600
#define __PF_RES_FIELD 0x8000000000000000ULL #define __PF_RES_FIELD 0x8000000000000000ULL
#endif /* CONFIG_64BIT */ #endif /* CONFIG_64BIT */
...@@ -52,6 +51,10 @@ ...@@ -52,6 +51,10 @@
extern int sysctl_userprocess_debug; extern int sysctl_userprocess_debug;
#endif #endif
#define VM_FAULT_BADCONTEXT 0x010000
#define VM_FAULT_BADMAP 0x020000
#define VM_FAULT_BADACCESS 0x040000
static inline int notify_page_fault(struct pt_regs *regs) static inline int notify_page_fault(struct pt_regs *regs)
{ {
int ret = 0; int ret = 0;
...@@ -122,18 +125,22 @@ static inline int user_space_fault(unsigned long trans_exc_code) ...@@ -122,18 +125,22 @@ static inline int user_space_fault(unsigned long trans_exc_code)
* Send SIGSEGV to task. This is an external routine * Send SIGSEGV to task. This is an external routine
* to keep the stack usage of do_page_fault small. * to keep the stack usage of do_page_fault small.
*/ */
static void do_sigsegv(struct pt_regs *regs, unsigned long error_code, static noinline void do_sigsegv(struct pt_regs *regs, long int_code,
int si_code, unsigned long address) int si_code, unsigned long trans_exc_code)
{ {
struct siginfo si; struct siginfo si;
unsigned long address;
address = trans_exc_code & __FAIL_ADDR_MASK;
current->thread.prot_addr = address;
current->thread.trap_no = int_code;
#if defined(CONFIG_SYSCTL) || defined(CONFIG_PROCESS_DEBUG) #if defined(CONFIG_SYSCTL) || defined(CONFIG_PROCESS_DEBUG)
#if defined(CONFIG_SYSCTL) #if defined(CONFIG_SYSCTL)
if (sysctl_userprocess_debug) if (sysctl_userprocess_debug)
#endif #endif
{ {
printk("User process fault: interruption code 0x%lX\n", printk("User process fault: interruption code 0x%lX\n",
error_code); int_code);
printk("failing address: %lX\n", address); printk("failing address: %lX\n", address);
show_regs(regs); show_regs(regs);
} }
...@@ -144,14 +151,14 @@ static void do_sigsegv(struct pt_regs *regs, unsigned long error_code, ...@@ -144,14 +151,14 @@ static void do_sigsegv(struct pt_regs *regs, unsigned long error_code,
force_sig_info(SIGSEGV, &si, current); force_sig_info(SIGSEGV, &si, current);
} }
static void do_no_context(struct pt_regs *regs, unsigned long error_code, static noinline void do_no_context(struct pt_regs *regs, long int_code,
unsigned long trans_exc_code) unsigned long trans_exc_code)
{ {
const struct exception_table_entry *fixup; const struct exception_table_entry *fixup;
unsigned long address; unsigned long address;
/* Are we prepared to handle this kernel fault? */ /* Are we prepared to handle this kernel fault? */
fixup = search_exception_tables(regs->psw.addr & __FIXUP_MASK); fixup = search_exception_tables(regs->psw.addr & PSW_ADDR_INSN);
if (fixup) { if (fixup) {
regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE; regs->psw.addr = fixup->fixup | PSW_ADDR_AMODE;
return; return;
...@@ -169,107 +176,127 @@ static void do_no_context(struct pt_regs *regs, unsigned long error_code, ...@@ -169,107 +176,127 @@ static void do_no_context(struct pt_regs *regs, unsigned long error_code,
printk(KERN_ALERT "Unable to handle kernel paging request" printk(KERN_ALERT "Unable to handle kernel paging request"
" at virtual user address %p\n", (void *)address); " at virtual user address %p\n", (void *)address);
die("Oops", regs, error_code); die("Oops", regs, int_code);
do_exit(SIGKILL); do_exit(SIGKILL);
} }
static void do_low_address(struct pt_regs *regs, unsigned long error_code, static noinline void do_low_address(struct pt_regs *regs, long int_code,
unsigned long trans_exc_code) unsigned long trans_exc_code)
{ {
/* Low-address protection hit in kernel mode means /* Low-address protection hit in kernel mode means
NULL pointer write access in kernel mode. */ NULL pointer write access in kernel mode. */
if (regs->psw.mask & PSW_MASK_PSTATE) { if (regs->psw.mask & PSW_MASK_PSTATE) {
/* Low-address protection hit in user mode 'cannot happen'. */ /* Low-address protection hit in user mode 'cannot happen'. */
die ("Low-address protection", regs, error_code); die ("Low-address protection", regs, int_code);
do_exit(SIGKILL); do_exit(SIGKILL);
} }
do_no_context(regs, error_code, trans_exc_code); do_no_context(regs, int_code, trans_exc_code);
} }
static void do_sigbus(struct pt_regs *regs, unsigned long error_code, static noinline void do_sigbus(struct pt_regs *regs, long int_code,
unsigned long trans_exc_code) unsigned long trans_exc_code)
{ {
struct task_struct *tsk = current; struct task_struct *tsk = current;
struct mm_struct *mm = tsk->mm;
up_read(&mm->mmap_sem);
/* /*
* Send a sigbus, regardless of whether we were in kernel * Send a sigbus, regardless of whether we were in kernel
* or user mode. * or user mode.
*/ */
tsk->thread.prot_addr = trans_exc_code & __FAIL_ADDR_MASK; tsk->thread.prot_addr = trans_exc_code & __FAIL_ADDR_MASK;
tsk->thread.trap_no = error_code; tsk->thread.trap_no = int_code;
force_sig(SIGBUS, tsk); force_sig(SIGBUS, tsk);
/* Kernel mode? Handle exceptions or die */
if (!(regs->psw.mask & PSW_MASK_PSTATE))
do_no_context(regs, error_code, trans_exc_code);
} }
#ifdef CONFIG_S390_EXEC_PROTECT #ifdef CONFIG_S390_EXEC_PROTECT
static int signal_return(struct mm_struct *mm, struct pt_regs *regs, static noinline int signal_return(struct pt_regs *regs, long int_code,
unsigned long address, unsigned long error_code) unsigned long trans_exc_code)
{ {
u16 instruction; u16 instruction;
int rc; int rc;
#ifdef CONFIG_COMPAT
int compat;
#endif
pagefault_disable();
rc = __get_user(instruction, (u16 __user *) regs->psw.addr); rc = __get_user(instruction, (u16 __user *) regs->psw.addr);
pagefault_enable();
if (rc)
return -EFAULT;
up_read(&mm->mmap_sem); if (!rc && instruction == 0x0a77) {
clear_tsk_thread_flag(current, TIF_SINGLE_STEP); clear_tsk_thread_flag(current, TIF_SINGLE_STEP);
#ifdef CONFIG_COMPAT if (is_compat_task())
compat = is_compat_task(); sys32_sigreturn();
if (compat && instruction == 0x0a77) else
sys32_sigreturn(); sys_sigreturn();
else if (compat && instruction == 0x0aad) } else if (!rc && instruction == 0x0aad) {
sys32_rt_sigreturn(); clear_tsk_thread_flag(current, TIF_SINGLE_STEP);
else if (is_compat_task())
#endif sys32_rt_sigreturn();
if (instruction == 0x0a77) else
sys_sigreturn(); sys_rt_sigreturn();
else if (instruction == 0x0aad) } else
sys_rt_sigreturn(); do_sigsegv(regs, int_code, SEGV_MAPERR, trans_exc_code);
else {
current->thread.prot_addr = address;
current->thread.trap_no = error_code;
do_sigsegv(regs, error_code, SEGV_MAPERR, address);
}
return 0; return 0;
} }
#endif /* CONFIG_S390_EXEC_PROTECT */ #endif /* CONFIG_S390_EXEC_PROTECT */
static noinline void do_fault_error(struct pt_regs *regs, long int_code,
unsigned long trans_exc_code, int fault)
{
int si_code;
switch (fault) {
case VM_FAULT_BADACCESS:
#ifdef CONFIG_S390_EXEC_PROTECT
if ((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY &&
(trans_exc_code & 3) == 0) {
signal_return(regs, int_code, trans_exc_code);
break;
}
#endif /* CONFIG_S390_EXEC_PROTECT */
case VM_FAULT_BADMAP:
/* Bad memory access. Check if it is kernel or user space. */
if (regs->psw.mask & PSW_MASK_PSTATE) {
/* User mode accesses just cause a SIGSEGV */
si_code = (fault == VM_FAULT_BADMAP) ?
SEGV_MAPERR : SEGV_ACCERR;
do_sigsegv(regs, int_code, si_code, trans_exc_code);
return;
}
case VM_FAULT_BADCONTEXT:
do_no_context(regs, int_code, trans_exc_code);
break;
default: /* fault & VM_FAULT_ERROR */
if (fault & VM_FAULT_OOM)
pagefault_out_of_memory();
else if (fault & VM_FAULT_SIGBUS) {
do_sigbus(regs, int_code, trans_exc_code);
/* Kernel mode? Handle exceptions or die */
if (!(regs->psw.mask & PSW_MASK_PSTATE))
do_no_context(regs, int_code, trans_exc_code);
} else
BUG();
break;
}
}
/* /*
* This routine handles page faults. It determines the address, * This routine handles page faults. It determines the address,
* and the problem, and then passes it off to one of the appropriate * and the problem, and then passes it off to one of the appropriate
* routines. * routines.
* *
* error_code: * interruption code (int_code):
* 04 Protection -> Write-Protection (suprression) * 04 Protection -> Write-Protection (suprression)
* 10 Segment translation -> Not present (nullification) * 10 Segment translation -> Not present (nullification)
* 11 Page translation -> Not present (nullification) * 11 Page translation -> Not present (nullification)
* 3b Region third trans. -> Not present (nullification) * 3b Region third trans. -> Not present (nullification)
*/ */
static inline void static inline int do_exception(struct pt_regs *regs, int write,
do_exception(struct pt_regs *regs, unsigned long error_code, int write, unsigned long trans_exc_code)
unsigned long trans_exc_code)
{ {
struct task_struct *tsk; struct task_struct *tsk;
struct mm_struct *mm; struct mm_struct *mm;
struct vm_area_struct *vma; struct vm_area_struct *vma;
unsigned long address; unsigned long address;
int si_code;
int fault; int fault;
if (notify_page_fault(regs)) if (notify_page_fault(regs))
return; return 0;
tsk = current; tsk = current;
mm = tsk->mm; mm = tsk->mm;
...@@ -279,8 +306,9 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write, ...@@ -279,8 +306,9 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write,
* we are not in an interrupt and that there is a * we are not in an interrupt and that there is a
* user context. * user context.
*/ */
fault = VM_FAULT_BADCONTEXT;
if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
goto no_context; goto out;
address = trans_exc_code & __FAIL_ADDR_MASK; address = trans_exc_code & __FAIL_ADDR_MASK;
/* /*
...@@ -292,41 +320,35 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write, ...@@ -292,41 +320,35 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write,
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address);
down_read(&mm->mmap_sem); down_read(&mm->mmap_sem);
si_code = SEGV_MAPERR; fault = VM_FAULT_BADMAP;
vma = find_vma(mm, address); vma = find_vma(mm, address);
if (!vma) if (!vma)
goto bad_area; goto out_up;
if (unlikely(vma->vm_start > address)) {
if (!(vma->vm_flags & VM_GROWSDOWN))
goto out_up;
if (expand_stack(vma, address))
goto out_up;
}
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
fault = VM_FAULT_BADACCESS;
#ifdef CONFIG_S390_EXEC_PROTECT #ifdef CONFIG_S390_EXEC_PROTECT
if (unlikely((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY && if (unlikely((regs->psw.mask & PSW_MASK_ASC) == PSW_ASC_SECONDARY &&
(trans_exc_code & 3) == 0 && !(vma->vm_flags & VM_EXEC))) (trans_exc_code & 3) == 0 && !(vma->vm_flags & VM_EXEC)))
if (!signal_return(mm, regs, address, error_code)) goto out_up;
/*
* signal_return() has done an up_read(&mm->mmap_sem)
* if it returns 0.
*/
return;
#endif #endif
if (vma->vm_start <= address)
goto good_area;
if (!(vma->vm_flags & VM_GROWSDOWN))
goto bad_area;
if (expand_stack(vma, address))
goto bad_area;
/*
* Ok, we have a good vm_area for this memory access, so
* we can handle it..
*/
good_area:
si_code = SEGV_ACCERR;
if (!write) { if (!write) {
/* page not present, check vm flags */ /* page not present, check vm flags */
if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))) if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
goto bad_area; goto out_up;
} else { } else {
if (!(vma->vm_flags & VM_WRITE)) if (!(vma->vm_flags & VM_WRITE))
goto bad_area; goto out_up;
} }
if (is_vm_hugetlb_page(vma)) if (is_vm_hugetlb_page(vma))
...@@ -337,17 +359,9 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write, ...@@ -337,17 +359,9 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write,
* the fault. * the fault.
*/ */
fault = handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0); fault = handle_mm_fault(mm, vma, address, write ? FAULT_FLAG_WRITE : 0);
if (unlikely(fault & VM_FAULT_ERROR)) { if (unlikely(fault & VM_FAULT_ERROR))
if (fault & VM_FAULT_OOM) { goto out_up;
up_read(&mm->mmap_sem);
pagefault_out_of_memory();
return;
} else if (fault & VM_FAULT_SIGBUS) {
do_sigbus(regs, error_code, address);
return;
}
BUG();
}
if (fault & VM_FAULT_MAJOR) { if (fault & VM_FAULT_MAJOR) {
tsk->maj_flt++; tsk->maj_flt++;
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0,
...@@ -357,67 +371,55 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write, ...@@ -357,67 +371,55 @@ do_exception(struct pt_regs *regs, unsigned long error_code, int write,
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0,
regs, address); regs, address);
} }
up_read(&mm->mmap_sem);
/* /*
* The instruction that caused the program check will * The instruction that caused the program check will
* be repeated. Don't signal single step via SIGTRAP. * be repeated. Don't signal single step via SIGTRAP.
*/ */
clear_tsk_thread_flag(tsk, TIF_SINGLE_STEP); clear_tsk_thread_flag(tsk, TIF_SINGLE_STEP);
return; fault = 0;
out_up:
/*
* Something tried to access memory that isn't in our memory map..
* Fix it, but check if it's kernel or user first..
*/
bad_area:
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
out:
/* User mode accesses just cause a SIGSEGV */ return fault;
if (regs->psw.mask & PSW_MASK_PSTATE) {
tsk->thread.prot_addr = address;
tsk->thread.trap_no = error_code;
do_sigsegv(regs, error_code, si_code, address);
return;
}
no_context:
do_no_context(regs, error_code, trans_exc_code);
} }
void __kprobes do_protection_exception(struct pt_regs *regs, void __kprobes do_protection_exception(struct pt_regs *regs, long int_code)
long error_code)
{ {
unsigned long trans_exc_code = S390_lowcore.trans_exc_code; unsigned long trans_exc_code = S390_lowcore.trans_exc_code;
int fault;
/* Protection exception is supressing, decrement psw address. */ /* Protection exception is supressing, decrement psw address. */
regs->psw.addr -= (error_code >> 16); regs->psw.addr -= (int_code >> 16);
/* /*
* Check for low-address protection. This needs to be treated * Check for low-address protection. This needs to be treated
* as a special case because the translation exception code * as a special case because the translation exception code
* field is not guaranteed to contain valid data in this case. * field is not guaranteed to contain valid data in this case.
*/ */
if (unlikely(!(trans_exc_code & 4))) { if (unlikely(!(trans_exc_code & 4))) {
do_low_address(regs, error_code, trans_exc_code); do_low_address(regs, int_code, trans_exc_code);
return; return;
} }
do_exception(regs, 4, 1, trans_exc_code); fault = do_exception(regs, 1, trans_exc_code);
if (unlikely(fault))
do_fault_error(regs, 4, trans_exc_code, fault);
} }
void __kprobes do_dat_exception(struct pt_regs *regs, long error_code) void __kprobes do_dat_exception(struct pt_regs *regs, long int_code)
{ {
do_exception(regs, error_code & 0xff, 0, S390_lowcore.trans_exc_code); unsigned long trans_exc_code = S390_lowcore.trans_exc_code;
int fault;
fault = do_exception(regs, 0, trans_exc_code);
if (unlikely(fault))
do_fault_error(regs, int_code & 255, trans_exc_code, fault);
} }
#ifdef CONFIG_64BIT #ifdef CONFIG_64BIT
void __kprobes do_asce_exception(struct pt_regs *regs, unsigned long error_code) void __kprobes do_asce_exception(struct pt_regs *regs, long int_code)
{ {
unsigned long trans_exc_code = S390_lowcore.trans_exc_code; unsigned long trans_exc_code = S390_lowcore.trans_exc_code;
struct mm_struct *mm; struct mm_struct *mm = current->mm;
struct vm_area_struct *vma; struct vm_area_struct *vma;
unsigned long address;
mm = current->mm;
address = trans_exc_code & __FAIL_ADDR_MASK;
if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm)) if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm))
goto no_context; goto no_context;
...@@ -425,7 +427,7 @@ void __kprobes do_asce_exception(struct pt_regs *regs, unsigned long error_code) ...@@ -425,7 +427,7 @@ void __kprobes do_asce_exception(struct pt_regs *regs, unsigned long error_code)
local_irq_enable(); local_irq_enable();
down_read(&mm->mmap_sem); down_read(&mm->mmap_sem);
vma = find_vma(mm, address); vma = find_vma(mm, trans_exc_code & __FAIL_ADDR_MASK);
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
if (vma) { if (vma) {
...@@ -435,14 +437,12 @@ void __kprobes do_asce_exception(struct pt_regs *regs, unsigned long error_code) ...@@ -435,14 +437,12 @@ void __kprobes do_asce_exception(struct pt_regs *regs, unsigned long error_code)
/* User mode accesses just cause a SIGSEGV */ /* User mode accesses just cause a SIGSEGV */
if (regs->psw.mask & PSW_MASK_PSTATE) { if (regs->psw.mask & PSW_MASK_PSTATE) {
current->thread.prot_addr = address; do_sigsegv(regs, int_code, SEGV_MAPERR, trans_exc_code);
current->thread.trap_no = error_code;
do_sigsegv(regs, error_code, SEGV_MAPERR, address);
return; return;
} }
no_context: no_context:
do_no_context(regs, error_code, trans_exc_code); do_no_context(regs, int_code, trans_exc_code);
} }
#endif #endif
...@@ -507,7 +507,7 @@ void pfault_fini(void) ...@@ -507,7 +507,7 @@ void pfault_fini(void)
: : "a" (&refbk), "m" (refbk) : "cc"); : : "a" (&refbk), "m" (refbk) : "cc");
} }
static void pfault_interrupt(__u16 error_code) static void pfault_interrupt(__u16 int_code)
{ {
struct task_struct *tsk; struct task_struct *tsk;
__u16 subcode; __u16 subcode;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment