Commit 9aaefe7b authored by Andy Lutomirski's avatar Andy Lutomirski Committed by Ingo Molnar

x86/entry/64: Separate cpu_current_top_of_stack from TSS.sp0

On 64-bit kernels, we used to assume that TSS.sp0 was the current
top of stack.  With the addition of an entry trampoline, this will
no longer be the case.  Store the current top of stack in TSS.sp1,
which is otherwise unused but shares the same cacheline.
Signed-off-by: default avatarAndy Lutomirski <luto@kernel.org>
Signed-off-by: default avatarThomas Gleixner <tglx@linutronix.de>
Reviewed-by: default avatarThomas Gleixner <tglx@linutronix.de>
Reviewed-by: default avatarBorislav Petkov <bp@suse.de>
Cc: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Borislav Petkov <bpetkov@suse.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Dave Hansen <dave.hansen@intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: David Laight <David.Laight@aculab.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Eduardo Valentin <eduval@amazon.com>
Cc: Greg KH <gregkh@linuxfoundation.org>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Rik van Riel <riel@redhat.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: aliguori@amazon.com
Cc: daniel.gruss@iaik.tugraz.at
Cc: hughd@google.com
Cc: keescook@google.com
Link: https://lkml.kernel.org/r/20171204150606.050864668@linutronix.deSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 72f5e08d
...@@ -309,7 +309,13 @@ struct x86_hw_tss { ...@@ -309,7 +309,13 @@ struct x86_hw_tss {
struct x86_hw_tss { struct x86_hw_tss {
u32 reserved1; u32 reserved1;
u64 sp0; u64 sp0;
/*
* We store cpu_current_top_of_stack in sp1 so it's always accessible.
* Linux does not use ring 1, so sp1 is not otherwise needed.
*/
u64 sp1; u64 sp1;
u64 sp2; u64 sp2;
u64 reserved2; u64 reserved2;
u64 ist[7]; u64 ist[7];
...@@ -368,6 +374,8 @@ DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss); ...@@ -368,6 +374,8 @@ DECLARE_PER_CPU_PAGE_ALIGNED(struct tss_struct, cpu_tss);
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack); DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
#else
#define cpu_current_top_of_stack cpu_tss.x86_tss.sp1
#endif #endif
/* /*
...@@ -539,12 +547,12 @@ static inline void native_swapgs(void) ...@@ -539,12 +547,12 @@ static inline void native_swapgs(void)
static inline unsigned long current_top_of_stack(void) static inline unsigned long current_top_of_stack(void)
{ {
#ifdef CONFIG_X86_64 /*
return this_cpu_read_stable(cpu_tss.x86_tss.sp0); * We can't read directly from tss.sp0: sp0 on x86_32 is special in
#else * and around vm86 mode and sp0 on x86_64 is special because of the
/* sp0 on x86_32 is special in and around vm86 mode. */ * entry trampoline.
*/
return this_cpu_read_stable(cpu_current_top_of_stack); return this_cpu_read_stable(cpu_current_top_of_stack);
#endif
} }
static inline bool on_thread_stack(void) static inline bool on_thread_stack(void)
......
...@@ -207,7 +207,7 @@ static inline int arch_within_stack_frames(const void * const stack, ...@@ -207,7 +207,7 @@ static inline int arch_within_stack_frames(const void * const stack,
#else /* !__ASSEMBLY__ */ #else /* !__ASSEMBLY__ */
#ifdef CONFIG_X86_64 #ifdef CONFIG_X86_64
# define cpu_current_top_of_stack (cpu_tss + TSS_sp0) # define cpu_current_top_of_stack (cpu_tss + TSS_sp1)
#endif #endif
#endif #endif
......
...@@ -66,6 +66,7 @@ int main(void) ...@@ -66,6 +66,7 @@ int main(void)
OFFSET(TSS_ist, tss_struct, x86_tss.ist); OFFSET(TSS_ist, tss_struct, x86_tss.ist);
OFFSET(TSS_sp0, tss_struct, x86_tss.sp0); OFFSET(TSS_sp0, tss_struct, x86_tss.sp0);
OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
BLANK(); BLANK();
#ifdef CONFIG_CC_STACKPROTECTOR #ifdef CONFIG_CC_STACKPROTECTOR
......
...@@ -56,6 +56,16 @@ __visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = { ...@@ -56,6 +56,16 @@ __visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = {
* Poison it. * Poison it.
*/ */
.sp0 = (1UL << (BITS_PER_LONG-1)) + 1, .sp0 = (1UL << (BITS_PER_LONG-1)) + 1,
#ifdef CONFIG_X86_64
/*
* .sp1 is cpu_current_top_of_stack. The init task never
* runs user code, but cpu_current_top_of_stack should still
* be well defined before the first context switch.
*/
.sp1 = TOP_OF_INIT_STACK,
#endif
#ifdef CONFIG_X86_32 #ifdef CONFIG_X86_32
.ss0 = __KERNEL_DS, .ss0 = __KERNEL_DS,
.ss1 = __KERNEL_CS, .ss1 = __KERNEL_CS,
......
...@@ -461,6 +461,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) ...@@ -461,6 +461,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
* Switch the PDA and FPU contexts. * Switch the PDA and FPU contexts.
*/ */
this_cpu_write(current_task, next_p); this_cpu_write(current_task, next_p);
this_cpu_write(cpu_current_top_of_stack, task_top_of_stack(next_p));
/* Reload sp0. */ /* Reload sp0. */
update_sp0(next_p); update_sp0(next_p);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment