Commit 40426882 authored by Feng Kan's avatar Feng Kan Committed by Catalin Marinas

arm64: copy_to-from-in_user optimization using copy template

This patch optimize copy_to-from-in_user for arm 64bit architecture. The
copy template is used as template file for all the copy*.S files. Minor
change was made to it to accommodate the copy to/from/in user files.
Signed-off-by: default avatarFeng Kan <fkan@apm.com>
Signed-off-by: default avatarBalamurugan Shanmugam <bshanmugam@apm.com>
Signed-off-by: default avatarCatalin Marinas <catalin.marinas@arm.com>
parent e5c88e3f
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/assembler.h> #include <asm/assembler.h>
#include <asm/cache.h>
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#include <asm/sysreg.h> #include <asm/sysreg.h>
...@@ -31,49 +32,58 @@ ...@@ -31,49 +32,58 @@
* Returns: * Returns:
* x0 - bytes not copied * x0 - bytes not copied
*/ */
.macro ldrb1 ptr, regB, val
USER(9998f, ldrb \ptr, [\regB], \val)
.endm
.macro strb1 ptr, regB, val
strb \ptr, [\regB], \val
.endm
.macro ldrh1 ptr, regB, val
USER(9998f, ldrh \ptr, [\regB], \val)
.endm
.macro strh1 ptr, regB, val
strh \ptr, [\regB], \val
.endm
.macro ldr1 ptr, regB, val
USER(9998f, ldr \ptr, [\regB], \val)
.endm
.macro str1 ptr, regB, val
str \ptr, [\regB], \val
.endm
.macro ldp1 ptr, regB, regC, val
USER(9998f, ldp \ptr, \regB, [\regC], \val)
.endm
.macro stp1 ptr, regB, regC, val
stp \ptr, \regB, [\regC], \val
.endm
end .req x5
ENTRY(__copy_from_user) ENTRY(__copy_from_user)
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
add x5, x1, x2 // upper user buffer boundary add end, x0, x2
subs x2, x2, #16 #include "copy_template.S"
b.mi 1f
0:
USER(9f, ldp x3, x4, [x1], #16)
subs x2, x2, #16
stp x3, x4, [x0], #16
b.pl 0b
1: adds x2, x2, #8
b.mi 2f
USER(9f, ldr x3, [x1], #8 )
sub x2, x2, #8
str x3, [x0], #8
2: adds x2, x2, #4
b.mi 3f
USER(9f, ldr w3, [x1], #4 )
sub x2, x2, #4
str w3, [x0], #4
3: adds x2, x2, #2
b.mi 4f
USER(9f, ldrh w3, [x1], #2 )
sub x2, x2, #2
strh w3, [x0], #2
4: adds x2, x2, #1
b.mi 5f
USER(9f, ldrb w3, [x1] )
strb w3, [x0]
5: mov x0, #0
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
mov x0, #0 // Nothing to copy
ret ret
ENDPROC(__copy_from_user) ENDPROC(__copy_from_user)
.section .fixup,"ax" .section .fixup,"ax"
.align 2 .align 2
9: sub x2, x5, x1 9998:
mov x3, x2 sub x0, end, dst
10: strb wzr, [x0], #1 // zero remaining buffer space 9999:
subs x3, x3, #1 strb wzr, [dst], #1 // zero remaining buffer space
b.ne 10b cmp dst, end
mov x0, x2 // bytes not copied b.lo 9999b
ret ret
.previous .previous
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/assembler.h> #include <asm/assembler.h>
#include <asm/cache.h>
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#include <asm/sysreg.h> #include <asm/sysreg.h>
...@@ -33,44 +34,52 @@ ...@@ -33,44 +34,52 @@
* Returns: * Returns:
* x0 - bytes not copied * x0 - bytes not copied
*/ */
.macro ldrb1 ptr, regB, val
USER(9998f, ldrb \ptr, [\regB], \val)
.endm
.macro strb1 ptr, regB, val
USER(9998f, strb \ptr, [\regB], \val)
.endm
.macro ldrh1 ptr, regB, val
USER(9998f, ldrh \ptr, [\regB], \val)
.endm
.macro strh1 ptr, regB, val
USER(9998f, strh \ptr, [\regB], \val)
.endm
.macro ldr1 ptr, regB, val
USER(9998f, ldr \ptr, [\regB], \val)
.endm
.macro str1 ptr, regB, val
USER(9998f, str \ptr, [\regB], \val)
.endm
.macro ldp1 ptr, regB, regC, val
USER(9998f, ldp \ptr, \regB, [\regC], \val)
.endm
.macro stp1 ptr, regB, regC, val
USER(9998f, stp \ptr, \regB, [\regC], \val)
.endm
end .req x5
ENTRY(__copy_in_user) ENTRY(__copy_in_user)
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
add x5, x0, x2 // upper user buffer boundary add end, x0, x2
subs x2, x2, #16 #include "copy_template.S"
b.mi 1f
0:
USER(9f, ldp x3, x4, [x1], #16)
subs x2, x2, #16
USER(9f, stp x3, x4, [x0], #16)
b.pl 0b
1: adds x2, x2, #8
b.mi 2f
USER(9f, ldr x3, [x1], #8 )
sub x2, x2, #8
USER(9f, str x3, [x0], #8 )
2: adds x2, x2, #4
b.mi 3f
USER(9f, ldr w3, [x1], #4 )
sub x2, x2, #4
USER(9f, str w3, [x0], #4 )
3: adds x2, x2, #2
b.mi 4f
USER(9f, ldrh w3, [x1], #2 )
sub x2, x2, #2
USER(9f, strh w3, [x0], #2 )
4: adds x2, x2, #1
b.mi 5f
USER(9f, ldrb w3, [x1] )
USER(9f, strb w3, [x0] )
5: mov x0, #0
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
mov x0, #0
ret ret
ENDPROC(__copy_in_user) ENDPROC(__copy_in_user)
.section .fixup,"ax" .section .fixup,"ax"
.align 2 .align 2
9: sub x0, x5, x0 // bytes not copied 9998: sub x0, end, dst // bytes not copied
ret ret
.previous .previous
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include <asm/alternative.h> #include <asm/alternative.h>
#include <asm/assembler.h> #include <asm/assembler.h>
#include <asm/cache.h>
#include <asm/cpufeature.h> #include <asm/cpufeature.h>
#include <asm/sysreg.h> #include <asm/sysreg.h>
...@@ -31,44 +32,52 @@ ...@@ -31,44 +32,52 @@
* Returns: * Returns:
* x0 - bytes not copied * x0 - bytes not copied
*/ */
.macro ldrb1 ptr, regB, val
ldrb \ptr, [\regB], \val
.endm
.macro strb1 ptr, regB, val
USER(9998f, strb \ptr, [\regB], \val)
.endm
.macro ldrh1 ptr, regB, val
ldrh \ptr, [\regB], \val
.endm
.macro strh1 ptr, regB, val
USER(9998f, strh \ptr, [\regB], \val)
.endm
.macro ldr1 ptr, regB, val
ldr \ptr, [\regB], \val
.endm
.macro str1 ptr, regB, val
USER(9998f, str \ptr, [\regB], \val)
.endm
.macro ldp1 ptr, regB, regC, val
ldp \ptr, \regB, [\regC], \val
.endm
.macro stp1 ptr, regB, regC, val
USER(9998f, stp \ptr, \regB, [\regC], \val)
.endm
end .req x5
ENTRY(__copy_to_user) ENTRY(__copy_to_user)
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(0)), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
add x5, x0, x2 // upper user buffer boundary add end, x0, x2
subs x2, x2, #16 #include "copy_template.S"
b.mi 1f
0:
ldp x3, x4, [x1], #16
subs x2, x2, #16
USER(9f, stp x3, x4, [x0], #16)
b.pl 0b
1: adds x2, x2, #8
b.mi 2f
ldr x3, [x1], #8
sub x2, x2, #8
USER(9f, str x3, [x0], #8 )
2: adds x2, x2, #4
b.mi 3f
ldr w3, [x1], #4
sub x2, x2, #4
USER(9f, str w3, [x0], #4 )
3: adds x2, x2, #2
b.mi 4f
ldrh w3, [x1], #2
sub x2, x2, #2
USER(9f, strh w3, [x0], #2 )
4: adds x2, x2, #1
b.mi 5f
ldrb w3, [x1]
USER(9f, strb w3, [x0] )
5: mov x0, #0
ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \ ALTERNATIVE("nop", __stringify(SET_PSTATE_PAN(1)), ARM64_HAS_PAN, \
CONFIG_ARM64_PAN) CONFIG_ARM64_PAN)
mov x0, #0
ret ret
ENDPROC(__copy_to_user) ENDPROC(__copy_to_user)
.section .fixup,"ax" .section .fixup,"ax"
.align 2 .align 2
9: sub x0, x5, x0 // bytes not copied 9998: sub x0, end, dst // bytes not copied
ret ret
.previous .previous
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment