Skip to content

L
linux
Commit 13d38e96 authored by Matthew Wilcox's avatar Matthew Wilcox Committed by Linus Torvalds
Browse files
Options

[PATCH] PA-RISC copy/clear_user_page improvements 

 - Performance improvement to __clear_user_page_asm
 - White space cleanup
 - 64-bit version of copy_user_page_asm
 - Add prefetching to copy_user_page_asm
 - remove NOP and "bundle" comments

Contributions from Joel Soete
Signed-off-by: default avatarGrant Grundler <grundler@parisc-linux.org>
Signed-off-by: default avatarMatthew Wilcox <willy@parisc-linux.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent ade02163
Hide whitespace changes
Inline Side-by-side
Showing with 609 additions and 523 deletions
arch/parisc/kernel/pacache.S
View file @ 13d38e96
...@@ -27,17 +27,17 @@ ...@@ -27,17 +27,17 @@
*/ */
#ifdef __LP64__ #ifdef __LP64__
#define ADDIB addib,* #define ADDIB addib,*
#define CMPB cmpb,* #define CMPB cmpb,*
#define ANDCM andcm,* #define ANDCM andcm,*
.level 2.0w .level 2.0w
#else #else
#define ADDIB addib, #define ADDIB addib,
#define CMPB cmpb, #define CMPB cmpb,
#define ANDCM andcm #define ANDCM andcm
.level 2.0 .level 2.0
#endif #endif
#include <asm/assembly.h> #include <asm/assembly.h>
...@@ -46,7 +46,7 @@ ...@@ -46,7 +46,7 @@
#include <asm/cache.h> #include <asm/cache.h>
.text .text
.align 128 .align 128
.export flush_tlb_all_local,code .export flush_tlb_all_local,code
...@@ -68,7 +68,7 @@ flush_tlb_all_local: ...@@ -68,7 +68,7 @@ flush_tlb_all_local:
* consolidated. * consolidated.
*/ */
rsm PSW_SM_I,%r19 /* relied upon translation! PA 2.0 Arch. F-5 */ rsm PSW_SM_I, %r19 /* relied upon translation! PA 2.0 Arch. F-5 */
nop nop
nop nop
nop nop
...@@ -77,127 +77,127 @@ flush_tlb_all_local: ...@@ -77,127 +77,127 @@ flush_tlb_all_local:
nop nop
nop nop
rsm PSW_SM_Q,%r0 /* Turn off Q bit to load iia queue */ rsm PSW_SM_Q, %r0 /* Turn off Q bit to load iia queue */
ldil L%REAL_MODE_PSW, %r1 ldil L%REAL_MODE_PSW, %r1
ldo R%REAL_MODE_PSW(%r1), %r1 ldo R%REAL_MODE_PSW(%r1), %r1
mtctl %r1, %cr22 mtctl %r1, %cr22
mtctl %r0, %cr17 /* Clear IIASQ tail */ mtctl %r0, %cr17 /* Clear IIASQ tail */
mtctl %r0, %cr17 /* Clear IIASQ head */ mtctl %r0, %cr17 /* Clear IIASQ head */
ldil L%PA(1f),%r1 ldil L%PA(1f), %r1
ldo R%PA(1f)(%r1),%r1 ldo R%PA(1f)(%r1), %r1
mtctl %r1, %cr18 /* IIAOQ head */ mtctl %r1, %cr18 /* IIAOQ head */
ldo 4(%r1), %r1 ldo 4(%r1), %r1
mtctl %r1, %cr18 /* IIAOQ tail */ mtctl %r1, %cr18 /* IIAOQ tail */
rfi rfi
nop nop
1: ldil L%PA(cache_info),%r1 1: ldil L%PA(cache_info), %r1
ldo R%PA(cache_info)(%r1),%r1 ldo R%PA(cache_info)(%r1), %r1
/* Flush Instruction Tlb */ /* Flush Instruction Tlb */
LDREG ITLB_SID_BASE(%r1),%r20 LDREG ITLB_SID_BASE(%r1), %r20
LDREG ITLB_SID_STRIDE(%r1),%r21 LDREG ITLB_SID_STRIDE(%r1), %r21
LDREG ITLB_SID_COUNT(%r1),%r22 LDREG ITLB_SID_COUNT(%r1), %r22
LDREG ITLB_OFF_BASE(%r1),%arg0 LDREG ITLB_OFF_BASE(%r1), %arg0
LDREG ITLB_OFF_STRIDE(%r1),%arg1 LDREG ITLB_OFF_STRIDE(%r1), %arg1
LDREG ITLB_OFF_COUNT(%r1),%arg2 LDREG ITLB_OFF_COUNT(%r1), %arg2
LDREG ITLB_LOOP(%r1),%arg3 LDREG ITLB_LOOP(%r1), %arg3
ADDIB= -1,%arg3,fitoneloop /* Preadjust and test */ ADDIB= -1, %arg3, fitoneloop /* Preadjust and test */
movb,<,n %arg3,%r31,fitdone /* If loop < 0, skip */ movb,<,n %arg3, %r31, fitdone /* If loop < 0, skip */
copy %arg0,%r28 /* Init base addr */ copy %arg0, %r28 /* Init base addr */
fitmanyloop: /* Loop if LOOP >= 2 */ fitmanyloop: /* Loop if LOOP >= 2 */
mtsp %r20,%sr1 mtsp %r20, %sr1
add %r21,%r20,%r20 /* increment space */ add %r21, %r20, %r20 /* increment space */
copy %arg2,%r29 /* Init middle loop count */ copy %arg2, %r29 /* Init middle loop count */
fitmanymiddle: /* Loop if LOOP >= 2 */ fitmanymiddle: /* Loop if LOOP >= 2 */
ADDIB> -1,%r31,fitmanymiddle /* Adjusted inner loop decr */ ADDIB> -1, %r31, fitmanymiddle /* Adjusted inner loop decr */
pitlbe 0(%sr1,%r28) pitlbe 0(%sr1, %r28)
pitlbe,m %arg1(%sr1,%r28) /* Last pitlbe and addr adjust */ pitlbe,m %arg1(%sr1, %r28) /* Last pitlbe and addr adjust */
ADDIB> -1,%r29,fitmanymiddle /* Middle loop decr */ ADDIB> -1, %r29, fitmanymiddle /* Middle loop decr */
copy %arg3,%r31 /* Re-init inner loop count */ copy %arg3, %r31 /* Re-init inner loop count */
movb,tr %arg0,%r28,fitmanyloop /* Re-init base addr */ movb,tr %arg0, %r28, fitmanyloop /* Re-init base addr */
ADDIB<=,n -1,%r22,fitdone /* Outer loop count decr */ ADDIB<=,n -1, %r22, fitdone /* Outer loop count decr */
fitoneloop: /* Loop if LOOP = 1 */ fitoneloop: /* Loop if LOOP = 1 */
mtsp %r20,%sr1 mtsp %r20, %sr1
copy %arg0,%r28 /* init base addr */ copy %arg0, %r28 /* init base addr */
copy %arg2,%r29 /* init middle loop count */ copy %arg2, %r29 /* init middle loop count */
fitonemiddle: /* Loop if LOOP = 1 */ fitonemiddle: /* Loop if LOOP = 1 */
ADDIB> -1,%r29,fitonemiddle /* Middle loop count decr */ ADDIB> -1, %r29, fitonemiddle /* Middle loop count decr */
pitlbe,m %arg1(%sr1,%r28) /* pitlbe for one loop */ pitlbe,m %arg1(%sr1, %r28) /* pitlbe for one loop */
ADDIB> -1,%r22,fitoneloop /* Outer loop count decr */ ADDIB> -1, %r22, fitoneloop /* Outer loop count decr */
add %r21,%r20,%r20 /* increment space */ add %r21, %r20, %r20 /* increment space */
fitdone: fitdone:
/* Flush Data Tlb */ /* Flush Data Tlb */
LDREG DTLB_SID_BASE(%r1),%r20 LDREG DTLB_SID_BASE(%r1), %r20
LDREG DTLB_SID_STRIDE(%r1),%r21 LDREG DTLB_SID_STRIDE(%r1), %r21
LDREG DTLB_SID_COUNT(%r1),%r22 LDREG DTLB_SID_COUNT(%r1), %r22
LDREG DTLB_OFF_BASE(%r1),%arg0 LDREG DTLB_OFF_BASE(%r1), %arg0
LDREG DTLB_OFF_STRIDE(%r1),%arg1 LDREG DTLB_OFF_STRIDE(%r1), %arg1
LDREG DTLB_OFF_COUNT(%r1),%arg2 LDREG DTLB_OFF_COUNT(%r1), %arg2
LDREG DTLB_LOOP(%r1),%arg3 LDREG DTLB_LOOP(%r1), %arg3
ADDIB= -1,%arg3,fdtoneloop /* Preadjust and test */ ADDIB= -1, %arg3, fdtoneloop /* Preadjust and test */
movb,<,n %arg3,%r31,fdtdone /* If loop < 0, skip */ movb,<,n %arg3, %r31, fdtdone /* If loop < 0, skip */
copy %arg0,%r28 /* Init base addr */ copy %arg0, %r28 /* Init base addr */
fdtmanyloop: /* Loop if LOOP >= 2 */ fdtmanyloop: /* Loop if LOOP >= 2 */
mtsp %r20,%sr1 mtsp %r20, %sr1
add %r21,%r20,%r20 /* increment space */ add %r21, %r20, %r20 /* increment space */
copy %arg2,%r29 /* Init middle loop count */ copy %arg2, %r29 /* Init middle loop count */
fdtmanymiddle: /* Loop if LOOP >= 2 */ fdtmanymiddle: /* Loop if LOOP >= 2 */
ADDIB> -1,%r31,fdtmanymiddle /* Adjusted inner loop decr */ ADDIB> -1, %r31, fdtmanymiddle /* Adjusted inner loop decr */
pdtlbe 0(%sr1,%r28) pdtlbe 0(%sr1, %r28)
pdtlbe,m %arg1(%sr1,%r28) /* Last pdtlbe and addr adjust */ pdtlbe,m %arg1(%sr1, %r28) /* Last pdtlbe and addr adjust */
ADDIB> -1,%r29,fdtmanymiddle /* Middle loop decr */ ADDIB> -1, %r29, fdtmanymiddle /* Middle loop decr */
copy %arg3,%r31 /* Re-init inner loop count */ copy %arg3, %r31 /* Re-init inner loop count */
movb,tr %arg0,%r28,fdtmanyloop /* Re-init base addr */ movb,tr %arg0, %r28, fdtmanyloop /* Re-init base addr */
ADDIB<=,n -1,%r22,fdtdone /* Outer loop count decr */ ADDIB<=,n -1, %r22,fdtdone /* Outer loop count decr */
fdtoneloop: /* Loop if LOOP = 1 */ fdtoneloop: /* Loop if LOOP = 1 */
mtsp %r20,%sr1 mtsp %r20, %sr1
copy %arg0,%r28 /* init base addr */ copy %arg0, %r28 /* init base addr */
copy %arg2,%r29 /* init middle loop count */ copy %arg2, %r29 /* init middle loop count */
fdtonemiddle: /* Loop if LOOP = 1 */ fdtonemiddle: /* Loop if LOOP = 1 */
ADDIB> -1,%r29,fdtonemiddle /* Middle loop count decr */ ADDIB> -1, %r29, fdtonemiddle /* Middle loop count decr */
pdtlbe,m %arg1(%sr1,%r28) /* pdtlbe for one loop */ pdtlbe,m %arg1(%sr1, %r28) /* pdtlbe for one loop */
ADDIB> -1,%r22,fdtoneloop /* Outer loop count decr */ ADDIB> -1, %r22, fdtoneloop /* Outer loop count decr */
add %r21,%r20,%r20 /* increment space */ add %r21, %r20, %r20 /* increment space */
fdtdone: fdtdone:
/* Switch back to virtual mode */ /* Switch back to virtual mode */
rsm PSW_SM_Q,%r0 /* clear Q bit to load iia queue */ rsm PSW_SM_Q, %r0 /* clear Q bit to load iia queue */
ldil L%KERNEL_PSW, %r1 ldil L%KERNEL_PSW, %r1
ldo R%KERNEL_PSW(%r1), %r1 ldo R%KERNEL_PSW(%r1), %r1
or %r1,%r19,%r1 /* Set I bit if set on entry */ or %r1, %r19, %r1 /* Set I bit if set on entry */
mtctl %r1, %cr22 mtctl %r1, %cr22
mtctl %r0, %cr17 /* Clear IIASQ tail */ mtctl %r0, %cr17 /* Clear IIASQ tail */
mtctl %r0, %cr17 /* Clear IIASQ head */ mtctl %r0, %cr17 /* Clear IIASQ head */
ldil L%(2f), %r1 ldil L%(2f), %r1
ldo R%(2f)(%r1), %r1 ldo R%(2f)(%r1), %r1
mtctl %r1, %cr18 /* IIAOQ head */ mtctl %r1, %cr18 /* IIAOQ head */
ldo 4(%r1), %r1 ldo 4(%r1), %r1
mtctl %r1, %cr18 /* IIAOQ tail */ mtctl %r1, %cr18 /* IIAOQ tail */
rfi rfi
nop nop
2: bv %r0(%r2) 2: bv %r0(%r2)
nop nop
.exit .exit
...@@ -211,140 +211,198 @@ flush_instruction_cache_local: ...@@ -211,140 +211,198 @@ flush_instruction_cache_local:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
mtsp %r0,%sr1 mtsp %r0, %sr1
ldil L%cache_info,%r1 ldil L%cache_info, %r1
ldo R%cache_info(%r1),%r1 ldo R%cache_info(%r1), %r1
/* Flush Instruction Cache */ /* Flush Instruction Cache */
LDREG ICACHE_BASE(%r1),%arg0 LDREG ICACHE_BASE(%r1), %arg0
LDREG ICACHE_STRIDE(%r1),%arg1 LDREG ICACHE_STRIDE(%r1), %arg1
LDREG ICACHE_COUNT(%r1),%arg2 LDREG ICACHE_COUNT(%r1), %arg2
LDREG ICACHE_LOOP(%r1),%arg3 LDREG ICACHE_LOOP(%r1), %arg3
rsm PSW_SM_I,%r22 /* No mmgt ops during loop*/ rsm PSW_SM_I, %r22 /* No mmgt ops during loop*/
ADDIB= -1,%arg3,fioneloop /* Preadjust and test */ ADDIB= -1, %arg3, fioneloop /* Preadjust and test */
movb,<,n %arg3,%r31,fisync /* If loop < 0, do sync */ movb,<,n %arg3, %r31, fisync /* If loop < 0, do sync */
fimanyloop: /* Loop if LOOP >= 2 */ fimanyloop: /* Loop if LOOP >= 2 */
ADDIB> -1,%r31,fimanyloop /* Adjusted inner loop decr */ ADDIB> -1, %r31, fimanyloop /* Adjusted inner loop decr */
fice 0(%sr1,%arg0) fice 0(%sr1, %arg0)
fice,m %arg1(%sr1,%arg0) /* Last fice and addr adjust */ fice,m %arg1(%sr1, %arg0) /* Last fice and addr adjust */
movb,tr %arg3,%r31,fimanyloop /* Re-init inner loop count */ movb,tr %arg3, %r31, fimanyloop /* Re-init inner loop count */
ADDIB<=,n -1,%arg2,fisync /* Outer loop decr */ ADDIB<=,n -1, %arg2, fisync /* Outer loop decr */
fioneloop: /* Loop if LOOP = 1 */ fioneloop: /* Loop if LOOP = 1 */
ADDIB> -1,%arg2,fioneloop /* Outer loop count decr */ ADDIB> -1, %arg2, fioneloop /* Outer loop count decr */
fice,m %arg1(%sr1,%arg0) /* Fice for one loop */ fice,m %arg1(%sr1, %arg0) /* Fice for one loop */
fisync: fisync:
sync sync
mtsm %r22 mtsm %r22
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
.procend .procend
.export flush_data_cache_local,code .export flush_data_cache_local, code
.import cache_info,data .import cache_info, data
flush_data_cache_local: flush_data_cache_local:
.proc .proc
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
mtsp %r0,%sr1 mtsp %r0, %sr1
ldil L%cache_info,%r1 ldil L%cache_info, %r1
ldo R%cache_info(%r1),%r1 ldo R%cache_info(%r1), %r1
/* Flush Data Cache */ /* Flush Data Cache */
LDREG DCACHE_BASE(%r1),%arg0 LDREG DCACHE_BASE(%r1), %arg0
LDREG DCACHE_STRIDE(%r1),%arg1 LDREG DCACHE_STRIDE(%r1), %arg1
LDREG DCACHE_COUNT(%r1),%arg2 LDREG DCACHE_COUNT(%r1), %arg2
LDREG DCACHE_LOOP(%r1),%arg3 LDREG DCACHE_LOOP(%r1), %arg3
rsm PSW_SM_I,%r22 rsm PSW_SM_I, %r22
ADDIB= -1,%arg3,fdoneloop /* Preadjust and test */ ADDIB= -1, %arg3, fdoneloop /* Preadjust and test */
movb,<,n %arg3,%r31,fdsync /* If loop < 0, do sync */ movb,<,n %arg3, %r31, fdsync /* If loop < 0, do sync */
fdmanyloop: /* Loop if LOOP >= 2 */ fdmanyloop: /* Loop if LOOP >= 2 */
ADDIB> -1,%r31,fdmanyloop /* Adjusted inner loop decr */ ADDIB> -1, %r31, fdmanyloop /* Adjusted inner loop decr */
fdce 0(%sr1,%arg0) fdce 0(%sr1, %arg0)
fdce,m %arg1(%sr1,%arg0) /* Last fdce and addr adjust */ fdce,m %arg1(%sr1, %arg0) /* Last fdce and addr adjust */
movb,tr %arg3,%r31,fdmanyloop /* Re-init inner loop count */ movb,tr %arg3, %r31, fdmanyloop /* Re-init inner loop count */
ADDIB<=,n -1,%arg2,fdsync /* Outer loop decr */ ADDIB<=,n -1, %arg2, fdsync /* Outer loop decr */
fdoneloop: /* Loop if LOOP = 1 */ fdoneloop: /* Loop if LOOP = 1 */
ADDIB> -1,%arg2,fdoneloop /* Outer loop count decr */ ADDIB> -1, %arg2, fdoneloop /* Outer loop count decr */
fdce,m %arg1(%sr1,%arg0) /* Fdce for one loop */ fdce,m %arg1(%sr1, %arg0) /* Fdce for one loop */
fdsync: fdsync:
syncdma syncdma
sync sync
mtsm %r22 mtsm %r22
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
.procend .procend
.export copy_user_page_asm,code .export copy_user_page_asm,code
.align 16
copy_user_page_asm: copy_user_page_asm:
.proc .proc
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldi 64,%r1 #ifdef __LP64__
/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
* Unroll the loop by hand and arrange insn appropriately.
* GCC probably can do this just as well.
*/
ldd 0(%r25), %r19
ldi 32, %r1 /* PAGE_SIZE/128 == 32 */
ldw 64(%r25), %r0 /* prefetch 1 cacheline ahead */
ldw 128(%r25), %r0 /* prefetch 2 */
1: ldd 8(%r25), %r20
ldw 192(%r25), %r0 /* prefetch 3 */
ldw 256(%r25), %r0 /* prefetch 4 */
ldd 16(%r25), %r21
ldd 24(%r25), %r22
std %r19, 0(%r26)
std %r20, 8(%r26)
ldd 32(%r25), %r19
ldd 40(%r25), %r20
std %r21, 16(%r26)
std %r22, 24(%r26)
ldd 48(%r25), %r21
ldd 56(%r25), %r22
std %r19, 32(%r26)
std %r20, 40(%r26)
ldd 64(%r25), %r19
ldd 72(%r25), %r20
std %r21, 48(%r26)
std %r22, 56(%r26)
ldd 80(%r25), %r21
ldd 88(%r25), %r22
std %r19, 64(%r26)
std %r20, 72(%r26)
ldd 96(%r25), %r19
ldd 104(%r25), %r20
std %r21, 80(%r26)
std %r22, 88(%r26)
ldd 112(%r25), %r21
ldd 120(%r25), %r22
std %r19, 96(%r26)
std %r20, 104(%r26)
ldo 128(%r25), %r25
std %r21, 112(%r26)
std %r22, 120(%r26)
ldo 128(%r26), %r26
ADDIB> -1, %r1, 1b /* bundle 10 */
ldd 0(%r25), %r19 /* start next loads */
#else
/* /*
* This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
* bundles (very restricted rules for bundling). It probably * bundles (very restricted rules for bundling).
* does OK on PCXU and better, but we could do better with * Note that until (if) we start saving
* ldd/std instructions. Note that until (if) we start saving
* the full 64 bit register values on interrupt, we can't * the full 64 bit register values on interrupt, we can't
* use ldd/std on a 32 bit kernel. * use ldd/std on a 32 bit kernel.
*/ */
ldi 64, %r1 /* PAGE_SIZE/64 == 64 */
1: 1:
ldw 0(%r25),%r19 ldw 0(%r25), %r19
ldw 4(%r25),%r20 ldw 4(%r25), %r20
ldw 8(%r25),%r21 ldw 8(%r25), %r21
ldw 12(%r25),%r22 ldw 12(%r25), %r22
stw %r19,0(%r26) stw %r19, 0(%r26)
stw %r20,4(%r26) stw %r20, 4(%r26)
stw %r21,8(%r26) stw %r21, 8(%r26)
stw %r22,12(%r26) stw %r22, 12(%r26)
ldw 16(%r25),%r19 ldw 16(%r25), %r19
ldw 20(%r25),%r20 ldw 20(%r25), %r20
ldw 24(%r25),%r21 ldw 24(%r25), %r21
ldw 28(%r25),%r22 ldw 28(%r25), %r22
stw %r19,16(%r26) stw %r19, 16(%r26)
stw %r20,20(%r26) stw %r20, 20(%r26)
stw %r21,24(%r26) stw %r21, 24(%r26)
stw %r22,28(%r26) stw %r22, 28(%r26)
ldw 32(%r25),%r19 ldw 32(%r25), %r19
ldw 36(%r25),%r20 ldw 36(%r25), %r20
ldw 40(%r25),%r21 ldw 40(%r25), %r21
ldw 44(%r25),%r22 ldw 44(%r25), %r22
stw %r19,32(%r26) stw %r19, 32(%r26)
stw %r20,36(%r26) stw %r20, 36(%r26)
stw %r21,40(%r26) stw %r21, 40(%r26)
stw %r22,44(%r26) stw %r22, 44(%r26)
ldw 48(%r25),%r19 ldw 48(%r25), %r19
ldw 52(%r25),%r20 ldw 52(%r25), %r20
ldw 56(%r25),%r21 ldw 56(%r25), %r21
ldw 60(%r25),%r22 ldw 60(%r25), %r22
stw %r19,48(%r26) stw %r19, 48(%r26)
stw %r20,52(%r26) stw %r20, 52(%r26)
stw %r21,56(%r26) stw %r21, 56(%r26)
stw %r22,60(%r26) stw %r22, 60(%r26)
ldo 64(%r26),%r26 ldo 64(%r26), %r26
ADDIB> -1,%r1,1b ADDIB> -1, %r1, 1b
ldo 64(%r25),%r25 ldo 64(%r25), %r25
#endif
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -393,33 +451,33 @@ copy_user_page_asm: ...@@ -393,33 +451,33 @@ copy_user_page_asm:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%(__PAGE_OFFSET),%r1 ldil L%(__PAGE_OFFSET), %r1
sub %r26,%r1,%r26 sub %r26, %r1, %r26
sub %r25,%r1,%r23 /* move physical addr into non shadowed reg */ sub %r25, %r1, %r23 /* move physical addr into non shadowed reg */
ldil L%(TMPALIAS_MAP_START),%r28 ldil L%(TMPALIAS_MAP_START), %r28
#ifdef __LP64__ #ifdef __LP64__
extrd,u %r26,56,32,%r26 /* convert phys addr to tlb insert format */ extrd,u %r26,56,32, %r26 /* convert phys addr to tlb insert format */
extrd,u %r23,56,32,%r23 /* convert phys addr to tlb insert format */ extrd,u %r23,56,32, %r23 /* convert phys addr to tlb insert format */
depd %r24,63,22,%r28 /* Form aliased virtual address 'to' */ depd %r24,63,22, %r28 /* Form aliased virtual address 'to' */
depdi 0,63,12,%r28 /* Clear any offset bits */ depdi 0, 63,12, %r28 /* Clear any offset bits */
copy %r28,%r29 copy %r28, %r29
depdi 1,41,1,%r29 /* Form aliased virtual address 'from' */ depdi 1, 41,1, %r29 /* Form aliased virtual address 'from' */
#else #else
extrw,u %r26,24,25,%r26 /* convert phys addr to tlb insert format */ extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
extrw,u %r23,24,25,%r23 /* convert phys addr to tlb insert format */ extrw,u %r23, 24,25, %r23 /* convert phys addr to tlb insert format */
depw %r24,31,22,%r28 /* Form aliased virtual address 'to' */ depw %r24, 31,22, %r28 /* Form aliased virtual address 'to' */
depwi 0,31,12,%r28 /* Clear any offset bits */ depwi 0, 31,12, %r28 /* Clear any offset bits */
copy %r28,%r29 copy %r28, %r29
depwi 1,9,1,%r29 /* Form aliased virtual address 'from' */ depwi 1, 9,1, %r29 /* Form aliased virtual address 'from' */
#endif #endif
/* Purge any old translations */ /* Purge any old translations */
pdtlb 0(%r28) pdtlb 0(%r28)
pdtlb 0(%r29) pdtlb 0(%r29)
ldi 64,%r1 ldi 64, %r1
/* /*
* This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
...@@ -432,43 +490,43 @@ copy_user_page_asm: ...@@ -432,43 +490,43 @@ copy_user_page_asm:
1: 1:
ldw 0(%r29),%r19 ldw 0(%r29), %r19
ldw 4(%r29),%r20 ldw 4(%r29), %r20
ldw 8(%r29),%r21 ldw 8(%r29), %r21
ldw 12(%r29),%r22 ldw 12(%r29), %r22
stw %r19,0(%r28) stw %r19, 0(%r28)
stw %r20,4(%r28) stw %r20, 4(%r28)
stw %r21,8(%r28) stw %r21, 8(%r28)
stw %r22,12(%r28) stw %r22, 12(%r28)
ldw 16(%r29),%r19 ldw 16(%r29), %r19
ldw 20(%r29),%r20 ldw 20(%r29), %r20
ldw 24(%r29),%r21 ldw 24(%r29), %r21
ldw 28(%r29),%r22 ldw 28(%r29), %r22
stw %r19,16(%r28) stw %r19, 16(%r28)
stw %r20,20(%r28) stw %r20, 20(%r28)
stw %r21,24(%r28) stw %r21, 24(%r28)
stw %r22,28(%r28) stw %r22, 28(%r28)
ldw 32(%r29),%r19 ldw 32(%r29), %r19
ldw 36(%r29),%r20 ldw 36(%r29), %r20
ldw 40(%r29),%r21 ldw 40(%r29), %r21
ldw 44(%r29),%r22 ldw 44(%r29), %r22
stw %r19,32(%r28) stw %r19, 32(%r28)
stw %r20,36(%r28) stw %r20, 36(%r28)
stw %r21,40(%r28) stw %r21, 40(%r28)
stw %r22,44(%r28) stw %r22, 44(%r28)
ldw 48(%r29),%r19 ldw 48(%r29), %r19
ldw 52(%r29),%r20 ldw 52(%r29), %r20
ldw 56(%r29),%r21 ldw 56(%r29), %r21
ldw 60(%r29),%r22 ldw 60(%r29), %r22
stw %r19,48(%r28) stw %r19, 48(%r28)
stw %r20,52(%r28) stw %r20, 52(%r28)
stw %r21,56(%r28) stw %r21, 56(%r28)
stw %r22,60(%r28) stw %r22, 60(%r28)
ldo 64(%r28),%r28 ldo 64(%r28), %r28
ADDIB> -1,%r1,1b ADDIB> -1, %r1,1b
ldo 64(%r29),%r29 ldo 64(%r29), %r29
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -482,49 +540,77 @@ __clear_user_page_asm: ...@@ -482,49 +540,77 @@ __clear_user_page_asm:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
tophys_r1 %r26 tophys_r1 %r26
ldil L%(TMPALIAS_MAP_START),%r28 ldil L%(TMPALIAS_MAP_START), %r28
#ifdef __LP64__ #ifdef __LP64__
#if (TMPALIAS_MAP_START >= 0x80000000) #if (TMPALIAS_MAP_START >= 0x80000000)
depdi 0,31,32,%r28 /* clear any sign extension */ depdi 0, 31,32, %r28 /* clear any sign extension */
#endif #endif
extrd,u %r26,56,32,%r26 /* convert phys addr to tlb insert format */ extrd,u %r26, 56,32, %r26 /* convert phys addr to tlb insert format */
depd %r25,63,22,%r28 /* Form aliased virtual address 'to' */ depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
depdi 0,63,12,%r28 /* Clear any offset bits */ depdi 0, 63,12, %r28 /* Clear any offset bits */
#else #else
extrw,u %r26,24,25,%r26 /* convert phys addr to tlb insert format */ extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
depw %r25,31,22,%r28 /* Form aliased virtual address 'to' */ depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
depwi 0,31,12,%r28 /* Clear any offset bits */ depwi 0, 31,12, %r28 /* Clear any offset bits */
#endif #endif
/* Purge any old translation */ /* Purge any old translation */
pdtlb 0(%r28) pdtlb 0(%r28)
ldi 64,%r1 #ifdef __LP64__
ldi 32, %r1 /* PAGE_SIZE/128 == 32 */
/* PREFETCH (Write) has not (yet) been proven to help here */
/* #define PREFETCHW_OP ldd 256(%0), %r0 */
1: std %r0, 0(%r28)
std %r0, 8(%r28)
std %r0, 16(%r28)
std %r0, 24(%r28)
std %r0, 32(%r28)
std %r0, 40(%r28)
std %r0, 48(%r28)
std %r0, 56(%r28)
std %r0, 64(%r28)
std %r0, 72(%r28)
std %r0, 80(%r28)
std %r0, 88(%r28)
std %r0, 96(%r28)
std %r0, 104(%r28)
std %r0, 112(%r28)
std %r0, 120(%r28)
ADDIB> -1, %r1, 1b
ldo 128(%r28), %r28
#else /* ! __LP64 */
ldi 64, %r1 /* PAGE_SIZE/64 == 64 */
1: 1:
stw %r0,0(%r28) stw %r0, 0(%r28)
stw %r0,4(%r28) stw %r0, 4(%r28)
stw %r0,8(%r28) stw %r0, 8(%r28)
stw %r0,12(%r28) stw %r0, 12(%r28)
stw %r0,16(%r28) stw %r0, 16(%r28)
stw %r0,20(%r28) stw %r0, 20(%r28)
stw %r0,24(%r28) stw %r0, 24(%r28)
stw %r0,28(%r28) stw %r0, 28(%r28)
stw %r0,32(%r28) stw %r0, 32(%r28)
stw %r0,36(%r28) stw %r0, 36(%r28)
stw %r0,40(%r28) stw %r0, 40(%r28)
stw %r0,44(%r28) stw %r0, 44(%r28)
stw %r0,48(%r28) stw %r0, 48(%r28)
stw %r0,52(%r28) stw %r0, 52(%r28)
stw %r0,56(%r28) stw %r0, 56(%r28)
stw %r0,60(%r28) stw %r0, 60(%r28)
ADDIB> -1,%r1,1b ADDIB> -1, %r1, 1b
ldo 64(%r28),%r28 ldo 64(%r28), %r28
#endif /* __LP64 */
bv %r0(%r2)
bv %r0(%r2)
nop nop
.exit .exit
...@@ -537,38 +623,38 @@ flush_kernel_dcache_page: ...@@ -537,38 +623,38 @@ flush_kernel_dcache_page:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%dcache_stride,%r1 ldil L%dcache_stride, %r1
ldw R%dcache_stride(%r1),%r23 ldw R%dcache_stride(%r1), %r23
#ifdef __LP64__ #ifdef __LP64__
depdi,z 1,63-PAGE_SHIFT,1,%r25 depdi,z 1, 63-PAGE_SHIFT,1, %r25
#else #else
depwi,z 1,31-PAGE_SHIFT,1,%r25 depwi,z 1, 31-PAGE_SHIFT,1, %r25
#endif #endif
add %r26,%r25,%r25 add %r26, %r25, %r25
sub %r25,%r23,%r25 sub %r25, %r23, %r25
1: fdc,m %r23(%r26) 1: fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
fdc,m %r23(%r26) fdc,m %r23(%r26)
CMPB<< %r26,%r25,1b CMPB<< %r26, %r25,1b
fdc,m %r23(%r26) fdc,m %r23(%r26)
sync sync
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -581,38 +667,38 @@ flush_user_dcache_page: ...@@ -581,38 +667,38 @@ flush_user_dcache_page:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%dcache_stride,%r1 ldil L%dcache_stride, %r1
ldw R%dcache_stride(%r1),%r23 ldw R%dcache_stride(%r1), %r23
#ifdef __LP64__ #ifdef __LP64__
depdi,z 1,63-PAGE_SHIFT,1,%r25 depdi,z 1,63-PAGE_SHIFT,1, %r25
#else #else
depwi,z 1,31-PAGE_SHIFT,1,%r25 depwi,z 1,31-PAGE_SHIFT,1, %r25
#endif #endif
add %r26,%r25,%r25 add %r26, %r25, %r25
sub %r25,%r23,%r25 sub %r25, %r23, %r25
1: fdc,m %r23(%sr3,%r26) 1: fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
CMPB<< %r26,%r25,1b CMPB<< %r26, %r25,1b
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
sync sync
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -625,38 +711,38 @@ flush_user_icache_page: ...@@ -625,38 +711,38 @@ flush_user_icache_page:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%dcache_stride,%r1 ldil L%dcache_stride, %r1
ldw R%dcache_stride(%r1),%r23 ldw R%dcache_stride(%r1), %r23
#ifdef __LP64__ #ifdef __LP64__
depdi,z 1,63-PAGE_SHIFT,1,%r25 depdi,z 1, 63-PAGE_SHIFT,1, %r25
#else #else
depwi,z 1,31-PAGE_SHIFT,1,%r25 depwi,z 1, 31-PAGE_SHIFT,1, %r25
#endif #endif
add %r26,%r25,%r25 add %r26, %r25, %r25
sub %r25,%r23,%r25 sub %r25, %r23, %r25
1: fic,m %r23(%sr3,%r26) 1: fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
CMPB<< %r26,%r25,1b CMPB<< %r26, %r25,1b
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
sync sync
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -670,37 +756,37 @@ purge_kernel_dcache_page: ...@@ -670,37 +756,37 @@ purge_kernel_dcache_page:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%dcache_stride,%r1 ldil L%dcache_stride, %r1
ldw R%dcache_stride(%r1),%r23 ldw R%dcache_stride(%r1), %r23
#ifdef __LP64__ #ifdef __LP64__
depdi,z 1,63-PAGE_SHIFT,1,%r25 depdi,z 1, 63-PAGE_SHIFT,1, %r25
#else #else
depwi,z 1,31-PAGE_SHIFT,1,%r25 depwi,z 1, 31-PAGE_SHIFT,1, %r25
#endif #endif
add %r26,%r25,%r25 add %r26, %r25, %r25
sub %r25,%r23,%r25 sub %r25, %r23, %r25
1: pdc,m %r23(%r26) 1: pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
pdc,m %r23(%r26) pdc,m %r23(%r26)
CMPB<< %r26,%r25,1b CMPB<< %r26, %r25, 1b
pdc,m %r23(%r26) pdc,m %r23(%r26)
sync sync
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -718,54 +804,54 @@ flush_alias_page: ...@@ -718,54 +804,54 @@ flush_alias_page:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
tophys_r1 %r26 tophys_r1 %r26
ldil L%(TMPALIAS_MAP_START),%r28 ldil L%(TMPALIAS_MAP_START), %r28
#ifdef __LP64__ #ifdef __LP64__
extrd,u %r26,56,32,%r26 /* convert phys addr to tlb insert format */ extrd,u %r26, 56,32, %r26 /* convert phys addr to tlb insert format */
depd %r25,63,22,%r28 /* Form aliased virtual address 'to' */ depd %r25, 63,22, %r28 /* Form aliased virtual address 'to' */
depdi 0,63,12,%r28 /* Clear any offset bits */ depdi 0, 63,12, %r28 /* Clear any offset bits */
#else #else
extrw,u %r26,24,25,%r26 /* convert phys addr to tlb insert format */ extrw,u %r26, 24,25, %r26 /* convert phys addr to tlb insert format */
depw %r25,31,22,%r28 /* Form aliased virtual address 'to' */ depw %r25, 31,22, %r28 /* Form aliased virtual address 'to' */
depwi 0,31,12,%r28 /* Clear any offset bits */ depwi 0, 31,12, %r28 /* Clear any offset bits */
#endif #endif
/* Purge any old translation */ /* Purge any old translation */
pdtlb 0(%r28) pdtlb 0(%r28)
ldil L%dcache_stride,%r1 ldil L%dcache_stride, %r1
ldw R%dcache_stride(%r1),%r23 ldw R%dcache_stride(%r1), %r23
#ifdef __LP64__ #ifdef __LP64__
depdi,z 1,63-PAGE_SHIFT,1,%r29 depdi,z 1, 63-PAGE_SHIFT,1, %r29
#else #else
depwi,z 1,31-PAGE_SHIFT,1,%r29 depwi,z 1, 31-PAGE_SHIFT,1, %r29
#endif #endif
add %r28,%r29,%r29 add %r28, %r29, %r29
sub %r29,%r23,%r29 sub %r29, %r23, %r29
1: fdc,m %r23(%r28) 1: fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
fdc,m %r23(%r28) fdc,m %r23(%r28)
CMPB<< %r28,%r29,1b CMPB<< %r28, %r29, 1b
fdc,m %r23(%r28) fdc,m %r23(%r28)
sync sync
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -779,16 +865,16 @@ flush_user_dcache_range_asm: ...@@ -779,16 +865,16 @@ flush_user_dcache_range_asm:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%dcache_stride,%r1 ldil L%dcache_stride, %r1
ldw R%dcache_stride(%r1),%r23 ldw R%dcache_stride(%r1), %r23
ldo -1(%r23),%r21 ldo -1(%r23), %r21
ANDCM %r26,%r21,%r26 ANDCM %r26, %r21, %r26
1: CMPB<<,n %r26,%r25,1b 1: CMPB<<,n %r26, %r25, 1b
fdc,m %r23(%sr3,%r26) fdc,m %r23(%sr3, %r26)
sync sync
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -801,17 +887,17 @@ flush_kernel_dcache_range_asm: ...@@ -801,17 +887,17 @@ flush_kernel_dcache_range_asm:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%dcache_stride,%r1 ldil L%dcache_stride, %r1
ldw R%dcache_stride(%r1),%r23 ldw R%dcache_stride(%r1), %r23
ldo -1(%r23),%r21 ldo -1(%r23), %r21
ANDCM %r26,%r21,%r26 ANDCM %r26, %r21, %r26
1: CMPB<<,n %r26,%r25,1b 1: CMPB<<,n %r26, %r25,1b
fdc,m %r23(%r26) fdc,m %r23(%r26)
sync sync
syncdma syncdma
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -824,16 +910,16 @@ flush_user_icache_range_asm: ...@@ -824,16 +910,16 @@ flush_user_icache_range_asm:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%icache_stride,%r1 ldil L%icache_stride, %r1
ldw R%icache_stride(%r1),%r23 ldw R%icache_stride(%r1), %r23
ldo -1(%r23),%r21 ldo -1(%r23), %r21
ANDCM %r26,%r21,%r26 ANDCM %r26, %r21, %r26
1: CMPB<<,n %r26,%r25,1b 1: CMPB<<,n %r26, %r25,1b
fic,m %r23(%sr3,%r26) fic,m %r23(%sr3, %r26)
sync sync
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -846,38 +932,38 @@ flush_kernel_icache_page: ...@@ -846,38 +932,38 @@ flush_kernel_icache_page:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%icache_stride,%r1 ldil L%icache_stride, %r1
ldw R%icache_stride(%r1),%r23 ldw R%icache_stride(%r1), %r23
#ifdef __LP64__ #ifdef __LP64__
depdi,z 1,63-PAGE_SHIFT,1,%r25 depdi,z 1, 63-PAGE_SHIFT,1, %r25
#else #else
depwi,z 1,31-PAGE_SHIFT,1,%r25 depwi,z 1, 31-PAGE_SHIFT,1, %r25
#endif #endif
add %r26,%r25,%r25 add %r26, %r25, %r25
sub %r25,%r23,%r25 sub %r25, %r23, %r25
1: fic,m %r23(%r26) 1: fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
fic,m %r23(%r26) fic,m %r23(%r26)
CMPB<< %r26,%r25,1b CMPB<< %r26, %r25, 1b
fic,m %r23(%r26) fic,m %r23(%r26)
sync sync
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
...@@ -890,22 +976,22 @@ flush_kernel_icache_range_asm: ...@@ -890,22 +976,22 @@ flush_kernel_icache_range_asm:
.callinfo NO_CALLS .callinfo NO_CALLS
.entry .entry
ldil L%icache_stride,%r1 ldil L%icache_stride, %r1
ldw R%icache_stride(%r1),%r23 ldw R%icache_stride(%r1), %r23
ldo -1(%r23),%r21 ldo -1(%r23), %r21
ANDCM %r26,%r21,%r26 ANDCM %r26, %r21, %r26
1: CMPB<<,n %r26,%r25,1b 1: CMPB<<,n %r26, %r25, 1b
fic,m %r23(%r26) fic,m %r23(%r26)
sync sync
bv %r0(%r2) bv %r0(%r2)
nop nop
.exit .exit
.procend .procend
.align 128 .align 128
.export disable_sr_hashing_asm,code .export disable_sr_hashing_asm,code
...@@ -916,7 +1002,7 @@ disable_sr_hashing_asm: ...@@ -916,7 +1002,7 @@ disable_sr_hashing_asm:
/* Switch to real mode */ /* Switch to real mode */
ssm 0,%r0 /* relied upon translation! */ ssm 0, %r0 /* relied upon translation! */
nop nop
nop nop
nop nop
...@@ -925,73 +1011,73 @@ disable_sr_hashing_asm: ...@@ -925,73 +1011,73 @@ disable_sr_hashing_asm:
nop nop
nop nop
rsm (PSW_SM_Q|PSW_SM_I),%r0 /* disable Q&I to load the iia queue */ rsm (PSW_SM_Q|PSW_SM_I), %r0 /* disable Q&I to load the iia queue */
ldil L%REAL_MODE_PSW, %r1 ldil L%REAL_MODE_PSW, %r1
ldo R%REAL_MODE_PSW(%r1), %r1 ldo R%REAL_MODE_PSW(%r1), %r1
mtctl %r1, %cr22 mtctl %r1, %cr22
mtctl %r0, %cr17 /* Clear IIASQ tail */ mtctl %r0, %cr17 /* Clear IIASQ tail */
mtctl %r0, %cr17 /* Clear IIASQ head */ mtctl %r0, %cr17 /* Clear IIASQ head */
ldil L%PA(1f),%r1 ldil L%PA(1f), %r1
ldo R%PA(1f)(%r1),%r1 ldo R%PA(1f)(%r1), %r1
mtctl %r1, %cr18 /* IIAOQ head */ mtctl %r1, %cr18 /* IIAOQ head */
ldo 4(%r1), %r1 ldo 4(%r1), %r1
mtctl %r1, %cr18 /* IIAOQ tail */ mtctl %r1, %cr18 /* IIAOQ tail */
rfi rfi
nop nop
1: cmpib,=,n SRHASH_PCXST,%r26,srdis_pcxs 1: cmpib,=,n SRHASH_PCXST, %r26,srdis_pcxs
cmpib,=,n SRHASH_PCXL,%r26,srdis_pcxl cmpib,=,n SRHASH_PCXL, %r26,srdis_pcxl
cmpib,=,n SRHASH_PA20,%r26,srdis_pa20 cmpib,=,n SRHASH_PA20, %r26,srdis_pa20
b,n srdis_done b,n srdis_done
srdis_pcxs: srdis_pcxs:
/* Disable Space Register Hashing for PCXS,PCXT,PCXT' */ /* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
.word 0x141c1a00 /* mfdiag %dr0,%r28 */ .word 0x141c1a00 /* mfdiag %dr0, %r28 */
.word 0x141c1a00 /* must issue twice */ .word 0x141c1a00 /* must issue twice */
depwi 0,18,1,%r28 /* Clear DHE (dcache hash enable) */ depwi 0,18,1, %r28 /* Clear DHE (dcache hash enable) */
depwi 0,20,1,%r28 /* Clear IHE (icache hash enable) */ depwi 0,20,1, %r28 /* Clear IHE (icache hash enable) */
.word 0x141c1600 /* mtdiag %r28,%dr0 */ .word 0x141c1600 /* mtdiag %r28, %dr0 */
.word 0x141c1600 /* must issue twice */ .word 0x141c1600 /* must issue twice */
b,n srdis_done b,n srdis_done
srdis_pcxl: srdis_pcxl:
/* Disable Space Register Hashing for PCXL */ /* Disable Space Register Hashing for PCXL */
.word 0x141c0600 /* mfdiag %dr0,%r28 */ .word 0x141c0600 /* mfdiag %dr0, %r28 */
depwi 0,28,2,%r28 /* Clear DHASH_EN & IHASH_EN */ depwi 0,28,2, %r28 /* Clear DHASH_EN & IHASH_EN */
.word 0x141c0240 /* mtdiag %r28,%dr0 */ .word 0x141c0240 /* mtdiag %r28, %dr0 */
b,n srdis_done b,n srdis_done
srdis_pa20: srdis_pa20:
/* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+ */ /* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+ */
.word 0x144008bc /* mfdiag %dr2,%r28 */ .word 0x144008bc /* mfdiag %dr2, %r28 */
depdi 0,54,1,%r28 /* clear DIAG_SPHASH_ENAB (bit 54) */ depdi 0, 54,1, %r28 /* clear DIAG_SPHASH_ENAB (bit 54) */
.word 0x145c1840 /* mtdiag %r28,%dr2 */ .word 0x145c1840 /* mtdiag %r28, %dr2 */
srdis_done: srdis_done:
/* Switch back to virtual mode */ /* Switch back to virtual mode */
rsm PSW_SM_Q,%r0 /* clear Q bit to load iia queue */ rsm PSW_SM_Q, %r0 /* clear Q bit to load iia queue */
ldil L%KERNEL_PSW, %r1 ldil L%KERNEL_PSW, %r1
ldo R%KERNEL_PSW(%r1), %r1 ldo R%KERNEL_PSW(%r1), %r1
mtctl %r1, %cr22 mtctl %r1, %cr22
mtctl %r0, %cr17 /* Clear IIASQ tail */ mtctl %r0, %cr17 /* Clear IIASQ tail */
mtctl %r0, %cr17 /* Clear IIASQ head */ mtctl %r0, %cr17 /* Clear IIASQ head */
ldil L%(2f), %r1 ldil L%(2f), %r1
ldo R%(2f)(%r1), %r1 ldo R%(2f)(%r1), %r1
mtctl %r1, %cr18 /* IIAOQ head */ mtctl %r1, %cr18 /* IIAOQ head */
ldo 4(%r1), %r1 ldo 4(%r1), %r1
mtctl %r1, %cr18 /* IIAOQ tail */ mtctl %r1, %cr18 /* IIAOQ tail */
rfi rfi
nop nop
2: bv %r0(%r2) 2: bv %r0(%r2)
nop nop
.exit .exit
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7