Split out copy_user_page/clear_user_page to allow more efficient

selection depending on processor features.

arch/arm/kernel/setup.c

@@ -76,6 +76,9 @@ struct processor processor;
 #ifdef MULTI_TLB
 struct cpu_tlb_fns cpu_tlb;
 #endif
+#ifdef MULTI_USER
+struct cpu_user_fns cpu_user;
+#endif
 
 unsigned char aux_device_present;
 char elf_platform[ELF_PLATFORM_SIZE];
@@ -248,6 +251,9 @@ static void __init setup_processor(void)
 #ifdef MULTI_TLB
 	cpu_tlb = *list->tlb;
 #endif
+#ifdef MULTI_USER
+	cpu_user = *list->user;
+#endif
 
 	printk("Processor: %s %s revision %d\n",
 	       proc_info.manufacturer, proc_info.cpu_name,

arch/arm/mm/copypage-v3.S

@@ -20,7 +20,7 @@
  *
  * FIXME: do we need to handle cache stuff...
  */
-ENTRY(armv3_copy_user_page)
+ENTRY(v3_copy_user_page)
 	stmfd	sp!, {r4, lr}			@	2
 	mov	r2, #PAGE_SZ/64			@	1
 	ldmia	r1!, {r3, r4, ip, lr}		@	4+1
@@ -42,7 +42,7 @@ ENTRY(armv3_copy_user_page)
  *
  * FIXME: do we need to handle cache stuff...
  */
-ENTRY(armv3_clear_user_page)
+ENTRY(v3_clear_user_page)
 	str	lr, [sp, #-4]!
 	mov	r1, #PAGE_SZ/64			@ 1
 	mov	r2, #0				@ 1
@@ -57,3 +57,8 @@ ENTRY(armv3_clear_user_page)
 	bne	1b				@ 1
 	ldr	pc, [sp], #4
 
+	.section ".text.init", #alloc, #execinstr
+
+ENTRY(v3_user_fns)
+	.long	v3_clear_user_page
+	.long	v3_copy_user_page

arch/arm/mm/copypage-v4.S

@@ -26,7 +26,7 @@
  * instruction.  If your processor does not supply this, you have to write your
  * own copy_user_page that does the right thing.
  */
-ENTRY(armv4_copy_user_page)
+ENTRY(v4_copy_user_page)
 	stmfd	sp!, {r4, lr}			@ 2
 	mov	r2, #PAGE_SZ/64			@ 1
 	ldmia	r1!, {r3, r4, ip, lr}		@ 4
@@ -51,7 +51,7 @@ ENTRY(armv4_copy_user_page)
  *
  * Same story as above.
  */
-ENTRY(armv4_clear_user_page)
+ENTRY(v4_clear_user_page)
 	str	lr, [sp, #-4]!
 	mov	r1, #PAGE_SZ/64			@ 1
 	mov	r2, #0				@ 1
@@ -68,3 +68,10 @@ ENTRY(armv4_clear_user_page)
 	bne	1b				@ 1
 	mcr	p15, 0, r1, c7, c10, 4		@ 1   drain WB
 	ldr	pc, [sp], #4
+
+	.section ".text.init", #alloc, #execinstr
+
+ENTRY(v4_user_fns)
+	.long	v4_clear_user_page
+	.long	v4_copy_user_page
+

arch/arm/mm/copypage-v4mc.S

@@ -26,7 +26,7 @@
  * instruction.  If your processor does not supply this, you have to write your
  * own copy_user_page that does the right thing.
  */
-ENTRY(armv4_mc_copy_user_page)
+ENTRY(v4_mc_copy_user_page)
 	stmfd	sp!, {r4, lr}			@ 2
 	mov	r4, r0
 	mov	r0, r1
@@ -53,7 +53,7 @@ ENTRY(armv4_mc_copy_user_page)
  *
  * Same story as above.
  */
-ENTRY(armv4_mc_clear_user_page)
+ENTRY(v4_mc_clear_user_page)
 	str	lr, [sp, #-4]!
 	mov	r1, #PAGE_SZ/64			@ 1
 	mov	r2, #0				@ 1
@@ -69,3 +69,10 @@ ENTRY(armv4_mc_clear_user_page)
 	subs	r1, r1, #1			@ 1
 	bne	1b				@ 1
 	ldr	pc, [sp], #4
+
+	.section ".text.init", #alloc, #execinstr
+
+ENTRY(v4_mc_user_fns)
+	.long	v4_mc_clear_user_page
+	.long	v4_mc_copy_user_page
+

arch/arm/mm/copypage-v5te.S

@@ -32,7 +32,7 @@
  * page.  We rely on the mini-cache being smaller than one page, so we'll
  * cycle through the complete cache anyway.
  */
-ENTRY(armv5te_copy_user_page)
+ENTRY(v5te_mc_copy_user_page)
 	stmfd	sp!, {r4, r5, lr}
 	mov	r5, r0
 	mov	r0, r1
@@ -62,7 +62,7 @@ ENTRY(armv5te_copy_user_page)
  *  r0 = destination
  *  r1 = virtual user address of ultimate destination page
  */
-ENTRY(armv5te_clear_user_page)
+ENTRY(v5te_mc_clear_user_page)
 	str	lr, [sp, #-4]!
 	mov	r1, #PAGE_SZ/32
 	mov	r2, #0
@@ -77,3 +77,9 @@ ENTRY(armv5te_clear_user_page)
 	subs	r1, r1, #1
 	bne	1b
 	ldr	pc, [sp], #4
+
+	.section ".text.init", #alloc, #execinstr
+
+ENTRY(v5te_mc_user_fns)
+	.long	v5te_mc_clear_user_page
+	.long	v5te_mc_copy_user_page

arch/arm/mm/proc-arm1020.S

@@ -635,10 +635,6 @@ arm1020_processor_functions:
 	.word	cpu_arm1020_set_pmd
 	.word	cpu_arm1020_set_pte
 
-	/* misc */
-	.word	armv4_clear_user_page
-	.word	armv4_copy_user_page
-
 	.size	arm1020_processor_functions, . - arm1020_processor_functions
 
 	.type	cpu_arm1020_info, #object
@@ -672,4 +668,5 @@ __arm1020_proc_info:
 	.long	cpu_arm1020_info
 	.long	arm1020_processor_functions
 	.long	v4wbi_tlb_fns
+	.long	v4_user_fns
 	.size	__arm1020_proc_info, . - __arm1020_proc_info

arch/arm/mm/proc-arm2,3.S

@@ -342,6 +342,7 @@ arm3_elf_name:	.asciz	"v2"
 		.long	cpu_arm2_info
 		.long	SYMBOL_NAME(arm2_processor_functions)
 		.long	0
+		.long	0
 
 		.long	0x41560250
 		.long	0xfffffff0
@@ -353,6 +354,7 @@ arm3_elf_name:	.asciz	"v2"
 		.long	cpu_arm250_info
 		.long	SYMBOL_NAME(arm250_processor_functions)
 		.long	0
+		.long	0
 
 		.long	0x41560300
 		.long	0xfffffff0
@@ -364,3 +366,5 @@ arm3_elf_name:	.asciz	"v2"
 		.long	cpu_arm3_info
 		.long	SYMBOL_NAME(arm3_processor_functions)
 		.long	0
+		.long	0
+

arch/arm/mm/proc-arm6,7.S

@@ -373,10 +373,6 @@ ENTRY(arm6_processor_functions)
 		.word	cpu_arm6_set_pmd
 		.word	cpu_arm6_set_pte
 
-		/* other */
-		.word	armv3_clear_user_page
-		.word	armv3_copy_user_page
-
 		.size	arm6_processor_functions, . - arm6_processor_functions
 
 /*
@@ -412,10 +408,6 @@ ENTRY(arm7_processor_functions)
 		.word	cpu_arm7_set_pmd
 		.word	cpu_arm7_set_pte
 
-		/* other */
-		.word	armv3_clear_user_page
-		.word	armv3_copy_user_page
-
 		.size	arm7_processor_functions, . - arm7_processor_functions
 
 		.type	cpu_arm6_info, #object
@@ -465,6 +457,7 @@ __arm6_proc_info:
 		.long	cpu_arm6_info
 		.long	arm6_processor_functions
 		.long	v3_tlb_fns
+		.long	v3_user_fns
 		.size	__arm6_proc_info, . - __arm6_proc_info
 
 		.type	__arm610_proc_info, #object
@@ -479,6 +472,7 @@ __arm610_proc_info:
 		.long	cpu_arm610_info
 		.long	arm6_processor_functions
 		.long	v3_tlb_fns
+		.long	v3_user_fns
 		.size	__arm610_proc_info, . - __arm610_proc_info
 
 		.type	__arm7_proc_info, #object
@@ -493,6 +487,7 @@ __arm7_proc_info:
 		.long	cpu_arm7_info
 		.long	arm7_processor_functions
 		.long	v3_tlb_fns
+		.long	v3_user_fns
 		.size	__arm7_proc_info, . - __arm7_proc_info
 
 		.type	__arm710_proc_info, #object
@@ -507,4 +502,5 @@ __arm710_proc_info:
 		.long	cpu_arm710_info
 		.long	arm7_processor_functions
 		.long	v3_tlb_fns
+		.long	v3_user_fns
 		.size	__arm710_proc_info, . - __arm710_proc_info

arch/arm/mm/proc-arm720.S

@@ -226,10 +226,6 @@ ENTRY(arm720_processor_functions)
 		.word	cpu_arm720_set_pmd
 		.word	cpu_arm720_set_pte
 
-		/* misc */
-		.word	armv4_clear_user_page
-		.word	armv4_copy_user_page
-
 		.size	arm720_processor_functions, . - arm720_processor_functions
 
 		.type	cpu_arm720_info, #object
@@ -265,4 +261,5 @@ __arm720_proc_info:
 		.long	cpu_arm720_info				@ info
 		.long	arm720_processor_functions
 		.long	v4_tlb_fns
+		.long	v4_user_fns
 		.size	__arm720_proc_info, . - __arm720_proc_info

arch/arm/mm/proc-arm920.S

@@ -538,10 +538,6 @@ arm920_processor_functions:
 	.word	cpu_arm920_set_pmd
 	.word	cpu_arm920_set_pte
 
-	/* misc */
-	.word	armv4_clear_user_page
-	.word	armv4_copy_user_page
-
 	.size	arm920_processor_functions, . - arm920_processor_functions
 
 	.type	cpu_arm920_info, #object
@@ -575,4 +571,5 @@ __arm920_proc_info:
 	.long	cpu_arm920_info
 	.long	arm920_processor_functions
 	.long	v4wbi_tlb_fns
+	.long	v4_user_fns
 	.size	__arm920_proc_info, . - __arm920_proc_info

arch/arm/mm/proc-arm922.S

@@ -539,10 +539,6 @@ arm922_processor_functions:
 	.word	cpu_arm922_set_pmd
 	.word	cpu_arm922_set_pte
 
-	/* misc */
-	.word	armv4_clear_user_page
-	.word	armv4_copy_user_page
-
 	.size	arm922_processor_functions, . - arm922_processor_functions
 
 	.type	cpu_arm922_info, #object
@@ -576,4 +572,5 @@ __arm922_proc_info:
 	.long	cpu_arm922_info
 	.long	arm922_processor_functions
 	.long	v4wbi_tlb_fns
+	.long	v4_user_fns
 	.size	__arm922_proc_info, . - __arm922_proc_info

arch/arm/mm/proc-arm926.S

@@ -576,10 +576,6 @@ arm926_processor_functions:
 	.word	cpu_arm926_set_pmd
 	.word	cpu_arm926_set_pte
 
-	/* misc */
-	.word	armv4_clear_user_page
-	.word	armv4_copy_user_page
-
 	.size	arm926_processor_functions, . - arm926_processor_functions
 
 	.type	cpu_arm926_info, #object
@@ -613,4 +609,5 @@ __arm926_proc_info:
 	.long	cpu_arm926_info
 	.long	arm926_processor_functions
 	.long	v4wbi_tlb_fns
+	.long	v4_user_fns
 	.size	__arm926_proc_info, . - __arm926_proc_info

arch/arm/mm/proc-sa110.S

@@ -565,10 +565,6 @@ ENTRY(sa110_processor_functions)
 	.word	cpu_sa110_set_pmd
 	.word	cpu_sa110_set_pte
 
-	/* misc */
-	.word	armv4_clear_user_page
-	.word	armv4_copy_user_page
-
 	.size	sa110_processor_functions, . - sa110_processor_functions
 
 	.type	cpu_sa110_info, #object
@@ -610,10 +606,6 @@ ENTRY(sa1100_processor_functions)
 	.word	cpu_sa1100_set_pmd
 	.word	cpu_sa1100_set_pte
 
-	/* misc */
-	.word	armv4_mc_clear_user_page
-	.word	armv4_mc_copy_user_page
-
 	.size	sa1100_processor_functions, . - sa1100_processor_functions
 
 cpu_sa1100_info:
@@ -651,6 +643,7 @@ __sa110_proc_info:
 	.long	cpu_sa110_info
 	.long	sa110_processor_functions
 	.long	v4wb_tlb_fns
+	.long	v4_user_fns
 	.size	__sa110_proc_info, . - __sa110_proc_info
 
 	.type	__sa1100_proc_info,#object
@@ -665,6 +658,7 @@ __sa1100_proc_info:
 	.long	cpu_sa1100_info
 	.long	sa1100_processor_functions
 	.long	v4wb_tlb_fns
+	.long	v4_mc_user_fns
 	.size	__sa1100_proc_info, . - __sa1100_proc_info
 
 	.type	__sa1110_proc_info,#object
@@ -679,4 +673,5 @@ __sa1110_proc_info:
 	.long	cpu_sa1110_info
 	.long	sa1100_processor_functions
 	.long	v4wb_tlb_fns
+	.long	v4_mc_user_fns
 	.size	__sa1110_proc_info, . - __sa1110_proc_info

arch/arm/mm/proc-xscale.S

@@ -722,10 +722,6 @@ ENTRY(xscale_processor_functions)
 	.word	cpu_xscale_set_pmd
 	.word	cpu_xscale_set_pte
 
-	/* misc */
-	.word	armv5te_clear_user_page
-	.word	armv5te_copy_user_page
-
 	.size	xscale_processor_functions, . - xscale_processor_functions
 
 	.type	cpu_80200_info, #object
@@ -765,6 +761,7 @@ __80200_proc_info:
 	.long	cpu_80200_info
 	.long	xscale_processor_functions
 	.long	v4wbi_tlb_fns
+	.long	v5te_mc_user_fns
 	.size	__80200_proc_info, . - __80200_proc_info
 
 	.type	__pxa250_proc_info,#object
@@ -779,6 +776,7 @@ __pxa250_proc_info:
 	.long	cpu_pxa250_info
 	.long	xscale_processor_functions
 	.long	v4wbi_tlb_fns
+	.long	v5te_mc_user_fns
 	.size	__cotulla_proc_info, . - __cotulla_proc_info
 	.size	__pxa250_proc_info, . - __pxa250_proc_info
 

include/asm-arm/cpu-multi32.h

@@ -107,11 +107,6 @@ extern struct processor {
 		 */
 		void (*set_pte)(pte_t *ptep, pte_t pte);
 	} pgtable;
-
-	struct {	/* other */
-		void (*clear_user_page)(void *page, unsigned long u_addr);
-		void (*copy_user_page)(void *to, void *from, unsigned long u_addr);
-	} misc;
 } processor;
 
 extern const struct processor arm6_processor_functions;
@@ -140,9 +135,6 @@ extern const struct processor sa110_processor_functions;
 #define cpu_set_pmd(pmdp, pmd)			processor.pgtable.set_pmd(pmdp, pmd)
 #define cpu_set_pte(ptep, pte)			processor.pgtable.set_pte(ptep, pte)
 
-#define cpu_copy_user_page(to,from,uaddr)	processor.misc.copy_user_page(to,from,uaddr)
-#define cpu_clear_user_page(page,uaddr)		processor.misc.clear_user_page(page,uaddr)
-
 #define cpu_switch_mm(pgd,tsk)			cpu_set_pgd(__virt_to_phys((unsigned long)(pgd)))
 
 #define cpu_get_pgd()	\

include/asm-arm/cpu-single.h

@@ -39,8 +39,6 @@
 #define cpu_set_pgd			__cpu_fn(CPU_NAME,_set_pgd)
 #define cpu_set_pmd			__cpu_fn(CPU_NAME,_set_pmd)
 #define cpu_set_pte			__cpu_fn(CPU_NAME,_set_pte)
-#define cpu_copy_user_page		__cpu_fn(MMU_ARCH,_copy_user_page)
-#define cpu_clear_user_page		__cpu_fn(MMU_ARCH,_clear_user_page)
 
 #ifndef __ASSEMBLY__
 
@@ -73,9 +71,6 @@ extern void cpu_set_pgd(unsigned long pgd_phys);
 extern void cpu_set_pmd(pmd_t *pmdp, pmd_t pmd);
 extern void cpu_set_pte(pte_t *ptep, pte_t pte);
 
-extern void cpu_copy_user_page(void *to, void *from, unsigned long u_addr);
-extern void cpu_clear_user_page(void *page, unsigned long u_addr);
-
 extern volatile void cpu_reset(unsigned long addr);
 
 #define cpu_switch_mm(pgd,tsk) cpu_set_pgd(__virt_to_phys((unsigned long)(pgd)))

include/asm-arm/glue.h

@@ -147,4 +147,60 @@
 #error Unknown data abort handler type
 #endif
 
+
+/*
+ *	User Space Model
+ *	================
+ *
+ *	This section selects the correct set of functions for dealing with
+ *	page-based copying and clearing for user space for the particular
+ *	processor(s) we're building for.
+ *
+ *	We have the following to choose from:
+ *	  v3		- ARMv3
+ *	  v4		- ARMv4 without minicache
+ *	  v4_mc		- ARMv4 with minicache
+ *	  v5te_mc	- ARMv5TE with minicache
+ */
+#undef _USER
+#undef MULTI_USER
+
+#if defined(CONFIG_CPU_ARM610) || defined(CONFIG_CPU_ARM710)
+# ifdef _USER
+#  define MULTI_USER 1
+# else
+#  define _USER v3
+# endif
+#endif
+
+#if defined(CONFIG_CPU_ARM720T) || defined(CONFIG_CPU_ARM920T) || \
+    defined(CONFIG_CPU_ARM922T) || defined(CONFIG_CPU_ARM926T) || \
+    defined(CONFIG_CPU_SA110)   || defined(CONFIG_CPU_ARM1020)
+# ifdef _USER
+#  define MULTI_USER 1
+# else
+#  define _USER v4
+# endif
+#endif
+
+#if defined(CONFIG_CPU_SA1100)
+# ifdef _USER
+#  define MULTI_USER 1
+# else
+#  define _USER v4_mc
+# endif
+#endif
+
+#if defined(CONFIG_CPU_XSCALE)
+# ifdef _USER
+#  define MULTI_USER 1
+# else
+#  define _USER v5te_mc
+# endif
+#endif
+
+#ifndef _USER
+#error Unknown user operations model
+#endif
+
 #endif

include/asm-arm/page.h

@@ -9,25 +9,49 @@
 #ifdef __KERNEL__
 #ifndef __ASSEMBLY__
 
-#define STRICT_MM_TYPECHECKS
+#include <asm/glue.h>
 
-#define clear_page(page)	memzero((void *)(page), PAGE_SIZE)
-extern void copy_page(void *to, void *from);
+struct cpu_user_fns {
+	void (*cpu_clear_user_page)(void *p, unsigned long user);
+	void (*cpu_copy_user_page)(void *to, const void *from,
+				   unsigned long user);
+};
+
+#ifdef MULTI_USER
+extern struct cpu_user_fns cpu_user;
+
+#define __cpu_clear_user_page	cpu_user.cpu_clear_user_page
+#define __cpu_copy_user_page	cpu_user.cpu_copy_user_page
+
+#else
+
+#define __cpu_clear_user_page	__glue(_USER,_clear_user_page)
+#define __cpu_copy_user_page	__glue(_USER,_copy_user_page)
+
+extern void __cpu_clear_user_page(void *p, unsigned long user);
+extern void __cpu_copy_user_page(void *to, const void *from,
+				 unsigned long user);
+#endif
 
 #define clear_user_page(addr,vaddr)			\
 	do {						\
 		preempt_disable();			\
-		cpu_clear_user_page(addr, vaddr);	\
+		__cpu_clear_user_page(addr, vaddr);	\
 		preempt_enable();			\
 	} while (0)
 
 #define copy_user_page(to,from,vaddr)			\
 	do {						\
 		preempt_disable();			\
-		cpu_copy_user_page(to, from, vaddr);	\
+		__cpu_copy_user_page(to, from, vaddr);	\
 		preempt_enable();			\
 	} while (0)
 
+#define clear_page(page)	memzero((void *)(page), PAGE_SIZE)
+extern void copy_page(void *to, void *from);
+
+#undef STRICT_MM_TYPECHECKS
+
 #ifdef STRICT_MM_TYPECHECKS
 /*
  * These are used to make use of C type-checking..

include/asm-arm/proc-fns.h

@@ -28,7 +28,6 @@
 
 /*
  * CPU_NAME - the prefix for CPU related functions
- * MMU_ARCH - the prefix for copy_user_page/clear_user_page
  */
 
 #ifdef CONFIG_CPU_32
@@ -39,7 +38,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_arm6
-#   define MMU_ARCH armv3
 #  endif
 # endif
 # ifdef CONFIG_CPU_ARM710
@@ -48,7 +46,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_arm7
-#   define MMU_ARCH armv3
 #  endif
 # endif
 # ifdef CONFIG_CPU_ARM720T
@@ -57,7 +54,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_arm720
-#   define MMU_ARCH armv4
 #  endif
 # endif
 # ifdef CONFIG_CPU_ARM920T
@@ -66,7 +62,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_arm920
-#   define MMU_ARCH armv4
 #  endif
 # endif
 # ifdef CONFIG_CPU_ARM922T
@@ -75,7 +70,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_arm922
-#   define MMU_ARCH armv4
 #  endif
 # endif
 # ifdef CONFIG_CPU_ARM926T
@@ -84,7 +78,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_arm926
-#   define MMU_ARCH armv4
 #  endif
 # endif
 # ifdef CONFIG_CPU_SA110
@@ -93,7 +86,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_sa110
-#   define MMU_ARCH armv4
 #  endif
 # endif
 # ifdef CONFIG_CPU_SA1100
@@ -102,7 +94,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_sa1100
-#   define MMU_ARCH armv4_mc
 #  endif
 # endif
 # ifdef CONFIG_CPU_ARM1020
@@ -111,7 +102,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_arm1020
-#   define MMU_ARCH armv4
 #  endif
 # endif
 # ifdef CONFIG_CPU_XSCALE
@@ -120,7 +110,6 @@
 #   define MULTI_CPU
 #  else
 #   define CPU_NAME cpu_xscale
-#   define MMU_ARCH armv5te
 #  endif
 # endif
 #endif

include/asm-arm/procinfo.h

@@ -15,6 +15,7 @@
 #include <asm/proc-fns.h>
 
 struct cpu_tlb_fns;
+struct cpu_user_fns;
 struct processor;
 
 struct proc_info_item {
@@ -42,11 +43,12 @@ struct proc_info_list {
 	struct proc_info_item	*info;
 	struct processor	*proc;
 	struct cpu_tlb_fns	*tlb;
+	struct cpu_user_fns	*user;
 };
 
 #endif	/* __ASSEMBLY__ */
 
-#define PROC_INFO_SZ	40
+#define PROC_INFO_SZ	44
 
 #define HWCAP_SWP	 1
 #define HWCAP_HALF	 2