Skip to content

L
linux
Commit 162c65f2 authored by Nicolas Pitre's avatar Nicolas Pitre Committed by Russell King
Browse files
Options

[ARM PATCH] 1674/1: better ARM division routines 

Patch from Nicolas Pitre

Here's my rewrite of the ARM division routines:

 - take advantage of the clz instruction on ARMv5+
 - simpler and faster code for modulo routines
 - fast path for power of 2 divisors
 - other miscellaneous optimizations

This code has already been merged into gcc-3.4.
parent 157e3d70
Hide whitespace changes
Inline Side-by-side
Showing with 262 additions and 260 deletions
arch/arm/lib/lib1funcs.S
View file @ 162c65f2
@ libgcc1 routines for ARM cpu.
@ Division routines, written by Richard Earnshaw, (rearnsha@armltd.co.uk)
/*
* linux/arch/arm/lib/lib1funcs.S: Optimized ARM division routines
*
* Author: Nicolas Pitre <nico@cam.org>
* - contributed to gcc-3.4 on Sep 30, 2003
* - adapted for the Linux kernel on Oct 2, 2003
*/
/* Copyright (C) 1995, 1996, 1998 Free Software Foundation, Inc.
/* Copyright 1995, 1996, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
This file is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
......@@ -10,11 +15,12 @@ later version.
In addition to the permissions in the GNU General Public License, the
Free Software Foundation gives you unlimited permission to link the
compiled version of this file with other programs, and to distribute
those programs without any restriction coming from the use of this
file. (The General Public License restrictions do apply in other
respects; for example, they cover modification of the file, and
distribution when not linked into another program.)
compiled version of this file into combinations with other programs,
and to distribute those combinations without any restriction coming
from the use of this file. (The General Public License restrictions
do apply in other respects; for example, they cover modification of
the file, and distribution when not linked into a combine
executable.)
This file is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
......@@ -26,287 +32,283 @@ along with this program; see the file COPYING. If not, write to
the Free Software Foundation, 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA. */
/* As a special exception, if you link this library with other files,
some of which are compiled with GCC, to produce an executable,
this library does not by itself cause the resulting executable
to be covered by the GNU General Public License.
This exception does not however invalidate any other reasons why
the executable file might be covered by the GNU General Public License.
*/
/* This code is derived from gcc 2.95.3
* 29/07/01 Adapted for linux
* 27/03/03 Ian Molton Clean up CONFIG_CPU
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/hardware.h>
#include <linux/config.h>
#define RET mov
#define RETc(x) mov##x
#define RETCOND
dividend .req r0
divisor .req r1
result .req r2
overdone .req r2
curbit .req r3
.macro ARM_DIV_BODY dividend, divisor, result, curbit
#if __LINUX_ARM_ARCH__ >= 5
clz \curbit, \divisor
clz \result, \dividend
sub \result, \curbit, \result
mov \curbit, #1
mov \divisor, \divisor, lsl \result
mov \curbit, \curbit, lsl \result
mov \result, #0
ENTRY(__udivsi3)
cmp divisor, #0
beq Ldiv0
mov curbit, #1
mov result, #0
cmp dividend, divisor
bcc Lgot_result_udivsi3
1:
#else
@ Initially shift the divisor left 3 bits if possible,
@ set curbit accordingly. This allows for curbit to be located
@ at the left end of each 4 bit nibbles in the division loop
@ to save one loop in most cases.
tst \divisor, #0xe0000000
moveq \divisor, \divisor, lsl #3
moveq \curbit, #8
movne \curbit, #1
@ Unless the divisor is very big, shift it up in multiples of
@ four bits, since this is the amount of unwinding in the main
@ division loop. Continue shifting until the divisor is
@ larger than the dividend.
cmp divisor, #0x10000000
cmpcc divisor, dividend
movcc divisor, divisor, lsl #4
movcc curbit, curbit, lsl #4
bcc 1b
1: cmp \divisor, #0x10000000
cmplo \divisor, \dividend
movlo \divisor, \divisor, lsl #4
movlo \curbit, \curbit, lsl #4
blo 1b
2:
@ For very big divisors, we must shift it a bit at a time, or
@ we will be in danger of overflowing.
cmp divisor, #0x80000000
cmpcc divisor, dividend
movcc divisor, divisor, lsl #1
movcc curbit, curbit, lsl #1
bcc 2b
3:
@ Test for possible subtractions, and note which bits
@ are done in the result. On the final pass, this may subtract
@ too much from the dividend, but the result will be ok, since the
@ "bit" will have been shifted out at the bottom.
cmp dividend, divisor
subcs dividend, dividend, divisor
orrcs result, result, curbit
cmp dividend, divisor, lsr #1
subcs dividend, dividend, divisor, lsr #1
orrcs result, result, curbit, lsr #1
cmp dividend, divisor, lsr #2
subcs dividend, dividend, divisor, lsr #2
orrcs result, result, curbit, lsr #2
cmp dividend, divisor, lsr #3
subcs dividend, dividend, divisor, lsr #3
orrcs result, result, curbit, lsr #3
cmp dividend, #0 @ Early termination?
movnes curbit, curbit, lsr #4 @ No, any more bits to do?
movne divisor, divisor, lsr #4
bne 3b
Lgot_result_udivsi3:
mov r0, result
RET pc, lr
1: cmp \divisor, #0x80000000
cmplo \divisor, \dividend
movlo \divisor, \divisor, lsl #1
movlo \curbit, \curbit, lsl #1
blo 1b
Ldiv0:
str lr, [sp, #-4]!
bl __div0
mov r0, #0 @ about as wrong as it could be
ldmia sp!, {pc}RETCOND
mov \result, #0
/* __umodsi3 ----------------------- */
#endif
@ Division loop
1: cmp \dividend, \divisor
subhs \dividend, \dividend, \divisor
orrhs \result, \result, \curbit
cmp \dividend, \divisor, lsr #1
subhs \dividend, \dividend, \divisor, lsr #1
orrhs \result, \result, \curbit, lsr #1
cmp \dividend, \divisor, lsr #2
subhs \dividend, \dividend, \divisor, lsr #2
orrhs \result, \result, \curbit, lsr #2
cmp \dividend, \divisor, lsr #3
subhs \dividend, \dividend, \divisor, lsr #3
orrhs \result, \result, \curbit, lsr #3
cmp \dividend, #0 @ Early termination?
movnes \curbit, \curbit, lsr #4 @ No, any more bits to do?
movne \divisor, \divisor, lsr #4
bne 1b
.endm
.macro ARM_DIV2_ORDER divisor, order
#if __LINUX_ARM_ARCH__ >= 5
clz \order, \divisor
rsb \order, \order, #31
#else
cmp \divisor, #(1 << 16)
movhs \divisor, \divisor, lsr #16
movhs \order, #16
movlo \order, #0
cmp \divisor, #(1 << 8)
movhs \divisor, \divisor, lsr #8
addhs \order, \order, #8
cmp \divisor, #(1 << 4)
movhs \divisor, \divisor, lsr #4
addhs \order, \order, #4
cmp \divisor, #(1 << 2)
addhi \order, \order, #3
addls \order, \order, \divisor, lsr #1
#endif
.endm
.macro ARM_MOD_BODY dividend, divisor, order, spare
#if __LINUX_ARM_ARCH__ >= 5
clz \order, \divisor
clz \spare, \dividend
sub \order, \order, \spare
mov \divisor, \divisor, lsl \order
#else
mov \order, #0
ENTRY(__umodsi3)
cmp divisor, #0
beq Ldiv0
mov curbit, #1
cmp dividend, divisor
RETc(cc) pc, lr
1:
@ Unless the divisor is very big, shift it up in multiples of
@ four bits, since this is the amount of unwinding in the main
@ division loop. Continue shifting until the divisor is
@ larger than the dividend.
cmp divisor, #0x10000000
cmpcc divisor, dividend
movcc divisor, divisor, lsl #4
movcc curbit, curbit, lsl #4
bcc 1b
1: cmp \divisor, #0x10000000
cmplo \divisor, \dividend
movlo \divisor, \divisor, lsl #4
addlo \order, \order, #4
blo 1b
2:
@ For very big divisors, we must shift it a bit at a time, or
@ we will be in danger of overflowing.
cmp divisor, #0x80000000
cmpcc divisor, dividend
movcc divisor, divisor, lsl #1
movcc curbit, curbit, lsl #1
bcc 2b
3:
@ Test for possible subtractions. On the final pass, this may
@ subtract too much from the dividend, so keep track of which
@ subtractions are done, we can fix them up afterwards...
mov overdone, #0
cmp dividend, divisor
subcs dividend, dividend, divisor
cmp dividend, divisor, lsr #1
subcs dividend, dividend, divisor, lsr #1
orrcs overdone, overdone, curbit, ror #1
cmp dividend, divisor, lsr #2
subcs dividend, dividend, divisor, lsr #2
orrcs overdone, overdone, curbit, ror #2
cmp dividend, divisor, lsr #3
subcs dividend, dividend, divisor, lsr #3
orrcs overdone, overdone, curbit, ror #3
mov ip, curbit
cmp dividend, #0 @ Early termination?
movnes curbit, curbit, lsr #4 @ No, any more bits to do?
movne divisor, divisor, lsr #4
bne 3b
@ Any subtractions that we should not have done will be recorded in
@ the top three bits of "overdone". Exactly which were not needed
@ are governed by the position of the bit, stored in ip.
@ If we terminated early, because dividend became zero,
@ then none of the below will match, since the bit in ip will not be
@ in the bottom nibble.
ands overdone, overdone, #0xe0000000
RETc(eq) pc, lr @ No fixups needed
tst overdone, ip, ror #3
addne dividend, dividend, divisor, lsr #3
tst overdone, ip, ror #2
addne dividend, dividend, divisor, lsr #2
tst overdone, ip, ror #1
addne dividend, dividend, divisor, lsr #1
RET pc, lr
1: cmp \divisor, #0x80000000
cmplo \divisor, \dividend
movlo \divisor, \divisor, lsl #1
addlo \order, \order, #1
blo 1b
#endif
@ Perform all needed substractions to keep only the reminder.
@ Do comparisons in batch of 4 first.
subs \order, \order, #3 @ yes, 3 is intended here
blt 2f
1: cmp \dividend, \divisor
subhs \dividend, \dividend, \divisor
cmp \dividend, \divisor, lsr #1
subhs \dividend, \dividend, \divisor, lsr #1
cmp \dividend, \divisor, lsr #2
subhs \dividend, \dividend, \divisor, lsr #2
cmp \dividend, \divisor, lsr #3
subhs \dividend, \dividend, \divisor, lsr #3
cmp \dividend, #1
mov \divisor, \divisor, lsr #4
subges \order, \order, #4
bge 1b
tst \order, #3
teqne \dividend, #0
beq 5f
@ Either 1, 2 or 3 comparison/substractions are left.
2: cmn \order, #2
blt 4f
beq 3f
cmp \dividend, \divisor
subhs \dividend, \dividend, \divisor
mov \divisor, \divisor, lsr #1
3: cmp \dividend, \divisor
subhs \dividend, \dividend, \divisor
mov \divisor, \divisor, lsr #1
4: cmp \dividend, \divisor
subhs \dividend, \dividend, \divisor
5:
.endm
ENTRY(__udivsi3)
subs r2, r1, #1
moveq pc, lr
bcc Ldiv0
cmp r0, r1
bls 11f
tst r1, r2
beq 12f
ARM_DIV_BODY r0, r1, r2, r3
mov r0, r2
mov pc, lr
11: moveq r0, #1
movne r0, #0
mov pc, lr
12: ARM_DIV2_ORDER r1, r2
mov r0, r0, lsr r2
mov pc, lr
ENTRY(__umodsi3)
subs r2, r1, #1 @ compare divisor with 1
bcc Ldiv0
cmpne r0, r1 @ compare dividend with divisor
moveq r0, #0
tsthi r1, r2 @ see if divisor is power of 2
andeq r0, r0, r2
movls pc, lr
ARM_MOD_BODY r0, r1, r2, r3
mov pc, lr
ENTRY(__divsi3)
eor ip, dividend, divisor @ Save the sign of the result.
mov curbit, #1
mov result, #0
cmp divisor, #0
rsbmi divisor, divisor, #0 @ Loops below use unsigned.
cmp r1, #0
eor ip, r0, r1 @ save the sign of the result.
beq Ldiv0
cmp dividend, #0
rsbmi dividend, dividend, #0
cmp dividend, divisor
bcc Lgot_result_divsi3
rsbmi r1, r1, #0 @ loops below use unsigned.
subs r2, r1, #1 @ division by 1 or -1 ?
beq 10f
movs r3, r0
rsbmi r3, r0, #0 @ positive dividend value
cmp r3, r1
bls 11f
tst r1, r2 @ divisor is power of 2 ?
beq 12f
1:
@ Unless the divisor is very big, shift it up in multiples of
@ four bits, since this is the amount of unwinding in the main
@ division loop. Continue shifting until the divisor is
@ larger than the dividend.
cmp divisor, #0x10000000
cmpcc divisor, dividend
movcc divisor, divisor, lsl #4
movcc curbit, curbit, lsl #4
bcc 1b
ARM_DIV_BODY r3, r1, r0, r2
cmp ip, #0
rsbmi r0, r0, #0
mov pc, lr
10: teq ip, r0 @ same sign ?
rsbmi r0, r0, #0
mov pc, lr
11: movlo r0, #0
moveq r0, ip, asr #31
orreq r0, r0, #1
mov pc, lr
12: ARM_DIV2_ORDER r1, r2
2:
@ For very big divisors, we must shift it a bit at a time, or
@ we will be in danger of overflowing.
cmp divisor, #0x80000000
cmpcc divisor, dividend
movcc divisor, divisor, lsl #1
movcc curbit, curbit, lsl #1
bcc 2b
3:
@ Test for possible subtractions, and note which bits
@ are done in the result. On the final pass, this may subtract
@ too much from the dividend, but the result will be ok, since the
@ "bit" will have been shifted out at the bottom.
cmp dividend, divisor
subcs dividend, dividend, divisor
orrcs result, result, curbit
cmp dividend, divisor, lsr #1
subcs dividend, dividend, divisor, lsr #1
orrcs result, result, curbit, lsr #1
cmp dividend, divisor, lsr #2
subcs dividend, dividend, divisor, lsr #2
orrcs result, result, curbit, lsr #2
cmp dividend, divisor, lsr #3
subcs dividend, dividend, divisor, lsr #3
orrcs result, result, curbit, lsr #3
cmp dividend, #0 @ Early termination?
movnes curbit, curbit, lsr #4 @ No, any more bits to do?
movne divisor, divisor, lsr #4
bne 3b
Lgot_result_divsi3:
mov r0, result
cmp ip, #0
mov r0, r3, lsr r2
rsbmi r0, r0, #0
RET pc, lr
mov pc, lr
ENTRY(__modsi3)
mov curbit, #1
cmp divisor, #0
rsbmi divisor, divisor, #0 @ Loops below use unsigned.
cmp r1, #0
beq Ldiv0
@ Need to save the sign of the dividend, unfortunately, we need
@ ip later on; this is faster than pushing lr and using that.
str dividend, [sp, #-4]!
cmp dividend, #0
rsbmi dividend, dividend, #0
cmp dividend, divisor
bcc Lgot_result_modsi3
1:
@ Unless the divisor is very big, shift it up in multiples of
@ four bits, since this is the amount of unwinding in the main
@ division loop. Continue shifting until the divisor is
@ larger than the dividend.
cmp divisor, #0x10000000
cmpcc divisor, dividend
movcc divisor, divisor, lsl #4
movcc curbit, curbit, lsl #4
bcc 1b
rsbmi r1, r1, #0 @ loops below use unsigned.
movs ip, r0 @ preserve sign of dividend
rsbmi r0, r0, #0 @ if negative make positive
subs r2, r1, #1 @ compare divisor with 1
cmpne r0, r1 @ compare dividend with divisor
moveq r0, #0
tsthi r1, r2 @ see if divisor is power of 2
andeq r0, r0, r2
bls 10f
ARM_MOD_BODY r0, r1, r2, r3
10: cmp ip, #0
rsbmi r0, r0, #0
mov pc, lr
Ldiv0:
str lr, [sp, #-4]!
bl __div0
mov r0, #0 @ About as wrong as it could be.
ldr pc, [sp], #4
2:
@ For very big divisors, we must shift it a bit at a time, or
@ we will be in danger of overflowing.
cmp divisor, #0x80000000
cmpcc divisor, dividend
movcc divisor, divisor, lsl #1
movcc curbit, curbit, lsl #1
bcc 2b
3:
@ Test for possible subtractions. On the final pass, this may
@ subtract too much from the dividend, so keep track of which
@ subtractions are done, we can fix them up afterwards...
mov overdone, #0
cmp dividend, divisor
subcs dividend, dividend, divisor
cmp dividend, divisor, lsr #1
subcs dividend, dividend, divisor, lsr #1
orrcs overdone, overdone, curbit, ror #1
cmp dividend, divisor, lsr #2
subcs dividend, dividend, divisor, lsr #2
orrcs overdone, overdone, curbit, ror #2
cmp dividend, divisor, lsr #3
subcs dividend, dividend, divisor, lsr #3
orrcs overdone, overdone, curbit, ror #3
mov ip, curbit
cmp dividend, #0 @ Early termination?
movnes curbit, curbit, lsr #4 @ No, any more bits to do?
movne divisor, divisor, lsr #4
bne 3b
@ Any subtractions that we should not have done will be recorded in
@ the top three bits of "overdone". Exactly which were not needed
@ are governed by the position of the bit, stored in ip.
@ If we terminated early, because dividend became zero,
@ then none of the below will match, since the bit in ip will not be
@ in the bottom nibble.
ands overdone, overdone, #0xe0000000
beq Lgot_result_modsi3
tst overdone, ip, ror #3
addne dividend, dividend, divisor, lsr #3
tst overdone, ip, ror #2
addne dividend, dividend, divisor, lsr #2
tst overdone, ip, ror #1
addne dividend, dividend, divisor, lsr #1
Lgot_result_modsi3:
ldr ip, [sp], #4
cmp ip, #0
rsbmi dividend, dividend, #0
RET pc, lr
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7