Commit 8546c008 authored by Linus Torvalds's avatar Linus Torvalds Committed by H. Peter Anvin

i387: Uninline the generic FP helpers that we expose to kernel modules

Instead of exporting the very low-level internals of the FPU state
save/restore code (ie things like 'fpu_owner_task'), we should export
the higher-level interfaces.

Inlining these things is pointless anyway: sure, sometimes the end
result is small, but while 'stts()' can result in just three x86
instructions, those are not cheap instructions (writing %cr0 is a
serializing instruction and a very slow one at that).

So the overhead of a function call is not noticeable, and we really
don't want random modules mucking about with our internal state save
logic anyway.

So this unexports 'fpu_owner_task', and instead uninlines and exports
the actual functions that modules can use: fpu_kernel_begin/end() and
unlazy_fpu().
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
Link: http://lkml.kernel.org/r/alpine.LFD.2.02.1202211339590.5354@i5.linux-foundation.orgSigned-off-by: default avatarH. Peter Anvin <hpa@linux.intel.com>
parent 27e74da9
...@@ -419,70 +419,9 @@ static inline void __clear_fpu(struct task_struct *tsk) ...@@ -419,70 +419,9 @@ static inline void __clear_fpu(struct task_struct *tsk)
} }
} }
/* extern bool irq_fpu_usable(void);
* Were we in an interrupt that interrupted kernel mode? extern void kernel_fpu_begin(void);
* extern void kernel_fpu_end(void);
* We can do a kernel_fpu_begin/end() pair *ONLY* if that
* pair does nothing at all: the thread must not have fpu (so
* that we don't try to save the FPU state), and TS must
* be set (so that the clts/stts pair does nothing that is
* visible in the interrupted kernel thread).
*/
static inline bool interrupted_kernel_fpu_idle(void)
{
return !__thread_has_fpu(current) &&
(read_cr0() & X86_CR0_TS);
}
/*
* Were we in user mode (or vm86 mode) when we were
* interrupted?
*
* Doing kernel_fpu_begin/end() is ok if we are running
* in an interrupt context from user mode - we'll just
* save the FPU state as required.
*/
static inline bool interrupted_user_mode(void)
{
struct pt_regs *regs = get_irq_regs();
return regs && user_mode_vm(regs);
}
/*
* Can we use the FPU in kernel mode with the
* whole "kernel_fpu_begin/end()" sequence?
*
* It's always ok in process context (ie "not interrupt")
* but it is sometimes ok even from an irq.
*/
static inline bool irq_fpu_usable(void)
{
return !in_interrupt() ||
interrupted_user_mode() ||
interrupted_kernel_fpu_idle();
}
static inline void kernel_fpu_begin(void)
{
struct task_struct *me = current;
WARN_ON_ONCE(!irq_fpu_usable());
preempt_disable();
if (__thread_has_fpu(me)) {
__save_init_fpu(me);
__thread_clear_has_fpu(me);
/* We do 'stts()' in kernel_fpu_end() */
} else {
percpu_write(fpu_owner_task, NULL);
clts();
}
}
static inline void kernel_fpu_end(void)
{
stts();
preempt_enable();
}
/* /*
* Some instructions like VIA's padlock instructions generate a spurious * Some instructions like VIA's padlock instructions generate a spurious
...@@ -566,16 +505,7 @@ static inline void save_init_fpu(struct task_struct *tsk) ...@@ -566,16 +505,7 @@ static inline void save_init_fpu(struct task_struct *tsk)
preempt_enable(); preempt_enable();
} }
static inline void unlazy_fpu(struct task_struct *tsk) extern void unlazy_fpu(struct task_struct *tsk);
{
preempt_disable();
if (__thread_has_fpu(tsk)) {
__save_init_fpu(tsk);
__thread_fpu_end(tsk);
} else
tsk->fpu_counter = 0;
preempt_enable();
}
static inline void clear_fpu(struct task_struct *tsk) static inline void clear_fpu(struct task_struct *tsk)
{ {
......
...@@ -1045,7 +1045,6 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) = ...@@ -1045,7 +1045,6 @@ DEFINE_PER_CPU(char *, irq_stack_ptr) =
DEFINE_PER_CPU(unsigned int, irq_count) = -1; DEFINE_PER_CPU(unsigned int, irq_count) = -1;
DEFINE_PER_CPU(struct task_struct *, fpu_owner_task); DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
EXPORT_PER_CPU_SYMBOL(fpu_owner_task);
/* /*
* Special IST stacks which the CPU switches to when it calls * Special IST stacks which the CPU switches to when it calls
...@@ -1115,7 +1114,6 @@ void debug_stack_reset(void) ...@@ -1115,7 +1114,6 @@ void debug_stack_reset(void)
DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task; DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task); EXPORT_PER_CPU_SYMBOL(current_task);
DEFINE_PER_CPU(struct task_struct *, fpu_owner_task); DEFINE_PER_CPU(struct task_struct *, fpu_owner_task);
EXPORT_PER_CPU_SYMBOL(fpu_owner_task);
#ifdef CONFIG_CC_STACKPROTECTOR #ifdef CONFIG_CC_STACKPROTECTOR
DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary); DEFINE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
......
...@@ -32,6 +32,86 @@ ...@@ -32,6 +32,86 @@
# define user32_fxsr_struct user_fxsr_struct # define user32_fxsr_struct user_fxsr_struct
#endif #endif
/*
* Were we in an interrupt that interrupted kernel mode?
*
* We can do a kernel_fpu_begin/end() pair *ONLY* if that
* pair does nothing at all: the thread must not have fpu (so
* that we don't try to save the FPU state), and TS must
* be set (so that the clts/stts pair does nothing that is
* visible in the interrupted kernel thread).
*/
static inline bool interrupted_kernel_fpu_idle(void)
{
return !__thread_has_fpu(current) &&
(read_cr0() & X86_CR0_TS);
}
/*
* Were we in user mode (or vm86 mode) when we were
* interrupted?
*
* Doing kernel_fpu_begin/end() is ok if we are running
* in an interrupt context from user mode - we'll just
* save the FPU state as required.
*/
static inline bool interrupted_user_mode(void)
{
struct pt_regs *regs = get_irq_regs();
return regs && user_mode_vm(regs);
}
/*
* Can we use the FPU in kernel mode with the
* whole "kernel_fpu_begin/end()" sequence?
*
* It's always ok in process context (ie "not interrupt")
* but it is sometimes ok even from an irq.
*/
bool irq_fpu_usable(void)
{
return !in_interrupt() ||
interrupted_user_mode() ||
interrupted_kernel_fpu_idle();
}
EXPORT_SYMBOL(irq_fpu_usable);
void kernel_fpu_begin(void)
{
struct task_struct *me = current;
WARN_ON_ONCE(!irq_fpu_usable());
preempt_disable();
if (__thread_has_fpu(me)) {
__save_init_fpu(me);
__thread_clear_has_fpu(me);
/* We do 'stts()' in kernel_fpu_end() */
} else {
percpu_write(fpu_owner_task, NULL);
clts();
}
}
EXPORT_SYMBOL(kernel_fpu_begin);
void kernel_fpu_end(void)
{
stts();
preempt_enable();
}
EXPORT_SYMBOL(kernel_fpu_end);
void unlazy_fpu(struct task_struct *tsk)
{
preempt_disable();
if (__thread_has_fpu(tsk)) {
__save_init_fpu(tsk);
__thread_fpu_end(tsk);
} else
tsk->fpu_counter = 0;
preempt_enable();
}
EXPORT_SYMBOL(unlazy_fpu);
#ifdef CONFIG_MATH_EMULATION #ifdef CONFIG_MATH_EMULATION
# define HAVE_HWFP (boot_cpu_data.hard_math) # define HAVE_HWFP (boot_cpu_data.hard_math)
#else #else
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment