Move "used FPU status" into new non-atomic thread_info->status field.

This allows us to avoid having to use atomic updates for the lazy FP
status setting, since we don't have to worry about other CPU's racing
on the fields.

Also, fix x86 FP state after fork() by making sure the FP is unlazied
_before_ we copy the state information. Otherwise, if a process did a
fork() while holding the FP state lazily in the registers, the child
would incorrectly unlazy bogus state.

arch/i386/kernel/cpu/common.c

@@ -480,7 +480,7 @@ void __init cpu_init (void)
 	 */
 	atomic_inc(&init_mm.mm_count);
 	current->active_mm = &init_mm;
-	if(current->mm)
+	if (current->mm)
 		BUG();
 	enter_lazy_tlb(&init_mm, current, cpu);
 
@@ -508,7 +508,7 @@ void __init cpu_init (void)
 	/*
 	 * Force FPU initialization:
 	 */
-	clear_thread_flag(TIF_USEDFPU);
+	current_thread_info()->status = 0;
 	current->used_math = 0;
 	stts();
 }

arch/i386/kernel/i387.c

@@ -54,9 +54,11 @@ void init_fpu(struct task_struct *tsk)
 
 void kernel_fpu_begin(void)
 {
+	struct thread_info *thread = current_thread_info();
+
 	preempt_disable();
-	if (test_thread_flag(TIF_USEDFPU)) {
-		__save_init_fpu(current);
+	if (thread->status & TS_USEDFPU) {
+		__save_init_fpu(thread->task);
 		return;
 	}
 	clts();

arch/i386/kernel/process.c

@@ -275,6 +275,15 @@ void release_thread(struct task_struct *dead_task)
 	release_x86_irqs(dead_task);
 }
 
+/*
+ * This gets called before we allocate a new thread and copy
+ * the current task into it.
+ */
+void prepare_to_copy(struct task_struct *tsk)
+{
+	unlazy_fpu(tsk);
+}
+
 int copy_thread(int nr, unsigned long clone_flags, unsigned long esp,
 	unsigned long unused,
 	struct task_struct * p, struct pt_regs * regs)
@@ -297,9 +306,6 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long esp,
 	savesegment(gs,p->thread.gs);
 
 	tsk = current;
-	unlazy_fpu(tsk);
-	struct_cpy(&p->thread.i387, &tsk->thread.i387);
-
 	if (unlikely(NULL != tsk->thread.ts_io_bitmap)) {
 		p->thread.ts_io_bitmap = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
 		if (!p->thread.ts_io_bitmap)

arch/i386/kernel/traps.c

@@ -737,13 +737,14 @@ asmlinkage void do_spurious_interrupt_bug(struct pt_regs * regs,
  */
 asmlinkage void math_state_restore(struct pt_regs regs)
 {
-	struct task_struct *tsk = current;
-	clts();		/* Allow maths ops (or we recurse) */
+	struct thread_info *thread = current_thread_info();
+	struct task_struct *tsk = thread->task;
 
+	clts();		/* Allow maths ops (or we recurse) */
 	if (!tsk->used_math)
 		init_fpu(tsk);
 	restore_fpu(tsk);
-	set_thread_flag(TIF_USEDFPU);	/* So we fnsave on switch_to() */
+	thread->status |= TS_USEDFPU;	/* So we fnsave on switch_to() */
 }
 
 #ifndef CONFIG_MATH_EMULATION

include/asm-alpha/processor.h

@@ -51,6 +51,9 @@ extern void start_thread(struct pt_regs *, unsigned long, unsigned long);
 /* Free all resources held by a thread. */
 extern void release_thread(struct task_struct *);
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 /* Create a kernel thread without removing it from tasklists.  */
 extern long kernel_thread(int (*fn)(void *), void *arg, unsigned long flags);
 

include/asm-arm/processor.h

@@ -62,6 +62,9 @@ struct task_struct;
 /* Free all resources held by a thread. */
 extern void release_thread(struct task_struct *);
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 unsigned long get_wchan(struct task_struct *p);
 
 #define cpu_relax()			barrier()

include/asm-cris/processor.h

@@ -123,6 +123,9 @@ static inline void release_thread(struct task_struct *dead_task)
         /* Nothing needs to be done.  */
 }
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 /*
  * Return saved PC of a blocked thread.
  */

include/asm-i386/i387.h

@@ -36,7 +36,7 @@ static inline void __save_init_fpu( struct task_struct *tsk )
 		asm volatile( "fnsave %0 ; fwait"
 			      : "=m" (tsk->thread.i387.fsave) );
 	}
-	tsk->thread_info->flags &= ~_TIF_USEDFPU;
+	tsk->thread_info->status &= ~TS_USEDFPU;
 }
 
 static inline void save_init_fpu( struct task_struct *tsk )
@@ -47,15 +47,15 @@ static inline void save_init_fpu( struct task_struct *tsk )
 
 
 #define unlazy_fpu( tsk ) do { \
-	if ((tsk)->thread_info->flags & _TIF_USEDFPU) \
+	if ((tsk)->thread_info->status & TS_USEDFPU) \
 		save_init_fpu( tsk ); \
 } while (0)
 
 #define clear_fpu( tsk )					\
 do {								\
-	if ((tsk)->thread_info->flags & _TIF_USEDFPU) {		\
+	if ((tsk)->thread_info->status & TS_USEDFPU) {		\
 		asm volatile("fwait");				\
-		(tsk)->thread_info->flags &= ~_TIF_USEDFPU;	\
+		(tsk)->thread_info->status &= ~TS_USEDFPU;	\
 		stts();						\
 	}							\
 } while (0)

include/asm-i386/processor.h

@@ -446,6 +446,10 @@ struct mm_struct;
 
 /* Free all resources held by a thread. */
 extern void release_thread(struct task_struct *);
+
+/* Prepare to copy thread state - unlazy all lazy status */
+extern void prepare_to_copy(struct task_struct *tsk);
+
 /*
  * create a kernel thread without removing it from tasklists
  */

include/asm-i386/thread_info.h

@@ -25,6 +25,7 @@ struct thread_info {
 	struct task_struct	*task;		/* main task structure */
 	struct exec_domain	*exec_domain;	/* execution domain */
 	unsigned long		flags;		/* low level flags */
+	unsigned long		status;		/* thread-synchronous flags */
 	__u32			cpu;		/* current CPU */
 	__s32			preempt_count; /* 0 => preemptable, <0 => BUG */
 
@@ -43,10 +44,11 @@ struct thread_info {
 #define TI_TASK		0x00000000
 #define TI_EXEC_DOMAIN	0x00000004
 #define TI_FLAGS	0x00000008
-#define TI_CPU		0x0000000C
-#define TI_PRE_COUNT	0x00000010
-#define TI_ADDR_LIMIT	0x00000014
-#define TI_RESTART_BLOCK 0x0000018
+#define TI_STATUS	0x0000000C
+#define TI_CPU		0x00000010
+#define TI_PRE_COUNT	0x00000014
+#define TI_ADDR_LIMIT	0x00000018
+#define TI_RESTART_BLOCK 0x000001C
 
 #endif
 
@@ -111,8 +113,7 @@ static inline struct thread_info *current_thread_info(void)
 #define TIF_NEED_RESCHED	3	/* rescheduling necessary */
 #define TIF_SINGLESTEP		4	/* restore singlestep on return to user mode */
 #define TIF_IRET		5	/* return with iret */
-#define TIF_USEDFPU		16	/* FPU was used by this task this quantum (SMP) */
-#define TIF_POLLING_NRFLAG	17	/* true if poll_idle() is polling TIF_NEED_RESCHED */
+#define TIF_POLLING_NRFLAG	16	/* true if poll_idle() is polling TIF_NEED_RESCHED */
 
 #define _TIF_SYSCALL_TRACE	(1<<TIF_SYSCALL_TRACE)
 #define _TIF_NOTIFY_RESUME	(1<<TIF_NOTIFY_RESUME)
@@ -120,12 +121,20 @@ static inline struct thread_info *current_thread_info(void)
 #define _TIF_NEED_RESCHED	(1<<TIF_NEED_RESCHED)
 #define _TIF_SINGLESTEP		(1<<TIF_SINGLESTEP)
 #define _TIF_IRET		(1<<TIF_IRET)
-#define _TIF_USEDFPU		(1<<TIF_USEDFPU)
 #define _TIF_POLLING_NRFLAG	(1<<TIF_POLLING_NRFLAG)
 
 #define _TIF_WORK_MASK		0x0000FFFE	/* work to do on interrupt/exception return */
 #define _TIF_ALLWORK_MASK	0x0000FFFF	/* work to do on any return to u-space */
 
+/*
+ * Thread-synchronous status.
+ *
+ * This is different from the flags in that nobody else
+ * ever touches our thread-synchronous status, so we don't
+ * have to worry about atomic accesses.
+ */
+#define TS_USEDFPU		0x0001	/* FPU was used by this task this quantum (SMP) */
+
 #endif /* __KERNEL__ */
 
 #endif /* _ASM_THREAD_INFO_H */

include/asm-ia64/processor.h

@@ -344,6 +344,9 @@ struct task_struct;
 # define release_thread(dead_task)
 #endif
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 /*
  * This is the mechanism for creating a new kernel thread.
  *

include/asm-m68k/processor.h

@@ -112,6 +112,9 @@ static inline void release_thread(struct task_struct *dead_task)
 {
 }
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
 
 /*

include/asm-m68knommu/processor.h

@@ -108,6 +108,9 @@ static inline void release_thread(struct task_struct *dead_task)
 {
 }
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
 
 /*

include/asm-mips/processor.h

@@ -208,6 +208,9 @@ struct thread_struct {
 /* Free all resources held by a thread. */
 #define release_thread(thread) do { } while(0)
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
 
 /*

include/asm-mips64/processor.h

@@ -231,6 +231,9 @@ struct thread_struct {
 /* Free all resources held by a thread. */
 #define release_thread(thread) do { } while(0)
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
 
 /*

include/asm-parisc/processor.h

@@ -290,6 +290,9 @@ struct mm_struct;
 extern void release_thread(struct task_struct *);
 extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 extern void map_hpux_gateway_page(struct task_struct *tsk, struct mm_struct *mm);
 
 static inline unsigned long get_wchan(struct task_struct *p)

include/asm-ppc/processor.h

@@ -608,6 +608,9 @@ struct task_struct;
 void start_thread(struct pt_regs *regs, unsigned long nip, unsigned long sp);
 void release_thread(struct task_struct *);
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 /*
  * Create a new kernel thread.
  */

include/asm-ppc64/processor.h

@@ -601,6 +601,9 @@ struct task_struct;
 void start_thread(struct pt_regs *regs, unsigned long fdptr, unsigned long sp);
 void release_thread(struct task_struct *);
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 /*
  * Create a new kernel thread.
  */

include/asm-s390/processor.h

@@ -114,6 +114,9 @@ struct mm_struct;
 extern void release_thread(struct task_struct *);
 extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 /*
  * Return saved PC of a blocked thread.
  */

include/asm-s390x/processor.h

@@ -129,6 +129,9 @@ struct mm_struct;
 extern void release_thread(struct task_struct *);
 extern int kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 /*
  * Return saved PC of a blocked thread.
  */

include/asm-sh/processor.h

@@ -134,6 +134,10 @@ struct mm_struct;
 
 /* Free all resources held by a thread. */
 extern void release_thread(struct task_struct *);
+
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 /*
  * create a kernel thread without removing it from tasklists
  */

include/asm-sparc/processor.h

@@ -139,6 +139,9 @@ extern __inline__ void start_thread(struct pt_regs * regs, unsigned long pc,
 #define release_thread(tsk)		do { } while(0)
 extern pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 extern unsigned long get_wchan(struct task_struct *);
 
 #define KSTK_EIP(tsk)  ((tsk)->thread.kregs->pc)

include/asm-sparc64/processor.h

@@ -186,6 +186,9 @@ do { \
 /* Free all resources held by a thread. */
 #define release_thread(tsk)		do { } while (0)
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 extern pid_t kernel_thread(int (*fn)(void *), void * arg, unsigned long flags);
 
 extern unsigned long get_wchan(struct task_struct *task);

include/asm-v850/processor.h

@@ -81,6 +81,9 @@ extern inline void release_thread (struct task_struct *dead_task)
 {
 }
 
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 extern int kernel_thread (int (*fn)(void *), void * arg, unsigned long flags);
 
 /* Free current thread data structures etc.  */

include/asm-x86_64/processor.h

@@ -267,6 +267,10 @@ struct mm_struct;
 
 /* Free all resources held by a thread. */
 extern void release_thread(struct task_struct *);
+
+/* Prepare to copy thread state - unlazy all lazy status */
+#define prepare_to_copy(tsk)	do { } while (0)
+
 /*
  * create a kernel thread without removing it from tasklists
  */

kernel/fork.c

@@ -214,6 +214,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig)
 	struct thread_info *ti;
 	int cpu = get_cpu();
 
+	prepare_to_copy(orig);
+
 	tsk = task_cache[cpu];
 	task_cache[cpu] = NULL;
 	put_cpu();