crypto: blowfish-x86_64 - improve x86_64 blowfish 4-way performance

This patch adds improved F-macro for 4-way parallel functions. With new
F-macro for 4-way parallel functions, blowfish sees ~15% improvement in
speed tests on AMD Phenom II (~5% on Intel Xeon E7330).

However when used in 1-way blowfish function new macro would be ~10%
slower than original, so old F-macro is kept for 1-way functions.
Patch cleans up old F-macro as it is no longer needed in 4-way part.

Patch also does register macro renaming to reduce stack usage.
Signed-off-by: Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

arch/x86/crypto/blowfish-x86_64-asm_64.S

@@ -56,38 +56,32 @@
 
 #define RT0 %rbp
 #define RT1 %rsi
+#define RT2 %r8
+#define RT3 %r9
 
 #define RT0d %ebp
 #define RT1d %esi
+#define RT2d %r8d
+#define RT3d %r9d
 
-#define RK0 %r8
-#define RK1 %r9
-#define RK2 %r10
-#define RK3 %r11
-
-#define RK0d %r8d
-#define RK1d %r9d
-#define RK2d %r10d
-#define RK3d %r11d
-
-#define RKEY %r12
+#define RKEY %r10
 
 /***********************************************************************
  * 1-way blowfish
  ***********************************************************************/
-#define F(x, k) \
-	rorq $16,		x; \
-	movzbl x ## bh,		RT0d; \
-	movzbl x ## bl,		RT1d; \
-	rolq $16,		x; \
-	movl s0(CTX,RT0,4),	k ## d; \
-	addl s1(CTX,RT1,4),	k ## d; \
-	movzbl x ## bh,		RT0d; \
-	movzbl x ## bl,		RT1d; \
-	rolq $32,		x; \
-	xorl s2(CTX,RT0,4),	k ## d; \
-	addl s3(CTX,RT1,4),	k ## d; \
-	xorq k,			x;
+#define F() \
+	rorq $16,		RX0; \
+	movzbl RX0bh,		RT0d; \
+	movzbl RX0bl,		RT1d; \
+	rolq $16,		RX0; \
+	movl s0(CTX,RT0,4),	RT0d; \
+	addl s1(CTX,RT1,4),	RT0d; \
+	movzbl RX0bh,		RT1d; \
+	movzbl RX0bl,		RT2d; \
+	rolq $32,		RX0; \
+	xorl s2(CTX,RT1,4),	RT0d; \
+	addl s3(CTX,RT2,4),	RT0d; \
+	xorq RT0,		RX0;
 
 #define add_roundkey_enc(n) \
 	xorq p+4*(n)(CTX), 	RX0;
@@ -95,11 +89,8 @@
 #define round_enc(n) \
 	add_roundkey_enc(n); \
 	\
-	F(RX0, RK0); \
-	F(RX0, RK0);
-
-#define round_final_enc(n) \
-	xorq p+4*(n)(CTX), 	RX0;
+	F(); \
+	F();
 
 #define add_roundkey_dec(n) \
 	movq p+4*(n-1)(CTX),	RT0; \
@@ -109,8 +100,8 @@
 #define round_dec(n) \
 	add_roundkey_dec(n); \
 	\
-	F(RX0, RK0); \
-	F(RX0, RK0); \
+	F(); \
+	F(); \
 
 #define read_block() \
 	movq (RIO), 		RX0; \
@@ -130,16 +121,15 @@
 .type   __blowfish_enc_blk,@function;
 
 __blowfish_enc_blk:
-	// input:
-	//	%rdi: ctx, CTX
-	//	%rsi: dst
-	//	%rdx: src
-	//	%rcx: bool xor
-	pushq %rbp;
-	pushq %rbx;
-
-	pushq %rsi;
-	pushq %rcx;
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst
+	 *	%rdx: src
+	 *	%rcx: bool, if true: xor output
+	 */
+	movq %rbp, %r11;
+
+	movq %rsi, %r10;
 	movq %rdx, RIO;
 
 	read_block();
@@ -154,38 +144,31 @@ __blowfish_enc_blk:
 	round_enc(14);
 	add_roundkey_enc(16);
 
-	popq %rbp;
-	popq RIO;
+	movq %r11, %rbp;
 
-	test %bpl, %bpl;
+	movq %r10, RIO;
+	test %cl, %cl;
 	jnz __enc_xor;
 
 	write_block();
-
-__enc_ret:
-	popq %rbx;
-	popq %rbp;
-
 	ret;
-
 __enc_xor:
 	xor_block();
-
-	jmp __enc_ret;
+	ret;
 
 .align 8
 .global blowfish_dec_blk
 .type   blowfish_dec_blk,@function;
 
 blowfish_dec_blk:
-	// input:
-	//	%rdi: ctx, CTX
-	//	%rsi: dst
-	//	%rdx: src
-	pushq %rbp;
-	pushq %rbx;
-
-	pushq %rsi;
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst
+	 *	%rdx: src
+	 */
+	movq %rbp, %r11;
+
+	movq %rsi, %r10;
 	movq %rdx, RIO;
 
 	read_block();
@@ -200,17 +183,33 @@ blowfish_dec_blk:
 	round_dec(3);
 	add_roundkey_dec(1);
 
-	popq RIO;
+	movq %r10, RIO;
 	write_block();
 
-	popq %rbx;
-	popq %rbp;
+	movq %r11, %rbp;
 
 	ret;
 
 /**********************************************************************
   4-way blowfish, four blocks parallel
  **********************************************************************/
+
+/* F() for 4-way. Slower when used alone/1-way, but faster when used
+ * parallel/4-way (tested on AMD Phenom II & Intel Xeon E7330).
+ */
+#define F4(x) \
+	movzbl x ## bh,		RT1d; \
+	movzbl x ## bl,		RT3d; \
+	rorq $16,		x; \
+	movzbl x ## bh,		RT0d; \
+	movzbl x ## bl,		RT2d; \
+	rorq $16,		x; \
+	movl s0(CTX,RT0,4),	RT0d; \
+	addl s1(CTX,RT2,4),	RT0d; \
+	xorl s2(CTX,RT1,4),	RT0d; \
+	addl s3(CTX,RT3,4),	RT0d; \
+	xorq RT0,		x;
+
 #define add_preloaded_roundkey4() \
 	xorq RKEY,		RX0; \
 	xorq RKEY,		RX1; \
@@ -227,15 +226,15 @@ blowfish_dec_blk:
 #define round_enc4(n) \
 	add_roundkey_enc4(n); \
 	\
-	F(RX0, RK0); \
-	F(RX1, RK1); \
-	F(RX2, RK2); \
-	F(RX3, RK3); \
+	F4(RX0); \
+	F4(RX1); \
+	F4(RX2); \
+	F4(RX3); \
 	\
-	F(RX0, RK0); \
-	F(RX1, RK1); \
-	F(RX2, RK2); \
-	F(RX3, RK3);
+	F4(RX0); \
+	F4(RX1); \
+	F4(RX2); \
+	F4(RX3);
 
 #define preload_roundkey_dec(n) \
 	movq p+4*((n)-1)(CTX),	RKEY; \
@@ -248,15 +247,15 @@ blowfish_dec_blk:
 #define round_dec4(n) \
 	add_roundkey_dec4(n); \
 	\
-	F(RX0, RK0); \
-	F(RX1, RK1); \
-	F(RX2, RK2); \
-	F(RX3, RK3); \
+	F4(RX0); \
+	F4(RX1); \
+	F4(RX2); \
+	F4(RX3); \
 	\
-	F(RX0, RK0); \
-	F(RX1, RK1); \
-	F(RX2, RK2); \
-	F(RX3, RK3);
+	F4(RX0); \
+	F4(RX1); \
+	F4(RX2); \
+	F4(RX3);
 
 #define read_block4() \
 	movq (RIO),		RX0; \
@@ -306,18 +305,19 @@ blowfish_dec_blk:
 .type   __blowfish_enc_blk_4way,@function;
 
 __blowfish_enc_blk_4way:
-	// input:
-	//	%rdi: ctx, CTX
-	//	%rsi: dst
-	//	%rdx: src
-	//	%rcx: bool xor
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst
+	 *	%rdx: src
+	 *	%rcx: bool, if true: xor output
+	 */
 	pushq %rbp;
 	pushq %rbx;
-	pushq RKEY;
+	pushq %rcx;
+
 	preload_roundkey_enc(0);
 
-	pushq %rsi;
-	pushq %rcx;
+	movq %rsi, %r11;
 	movq %rdx, RIO;
 
 	read_block4();
@@ -333,40 +333,39 @@ __blowfish_enc_blk_4way:
 	add_preloaded_roundkey4();
 
 	popq %rbp;
-	popq RIO;
+	movq %r11, RIO;
 
 	test %bpl, %bpl;
 	jnz __enc_xor4;
 
 	write_block4();
 
-__enc_ret4:
-	popq RKEY;
 	popq %rbx;
 	popq %rbp;
-
 	ret;
 
 __enc_xor4:
 	xor_block4();
 
-	jmp __enc_ret4;
+	popq %rbx;
+	popq %rbp;
+	ret;
 
 .align 8
 .global blowfish_dec_blk_4way
 .type   blowfish_dec_blk_4way,@function;
 
 blowfish_dec_blk_4way:
-	// input:
-	//	%rdi: ctx, CTX
-	//	%rsi: dst
-	//	%rdx: src
+	/* input:
+	 *	%rdi: ctx, CTX
+	 *	%rsi: dst
+	 *	%rdx: src
+	 */
 	pushq %rbp;
 	pushq %rbx;
-	pushq RKEY;
 	preload_roundkey_dec(17);
 
-	pushq %rsi;
+	movq %rsi, %r11;
 	movq %rdx, RIO;
 
 	read_block4();
@@ -381,10 +380,9 @@ blowfish_dec_blk_4way:
 	round_dec4(3);
 	add_preloaded_roundkey4();
 
-	popq RIO;
+	movq %r11, RIO;
 	write_block4();
 
-	popq RKEY;
 	popq %rbx;
 	popq %rbp;