UBUNTU: SAUCE: rfi-flush: Implement congruence-first fallback flush

CVE-2017-5754

BugLink: http://bugs.launchpad.net/bugs/1742772

This patch chnages the fallback flush to load all ways of a set,
then move to the next set. This is the best way to flush the cache,
accoring to HW people.
Signed-off-by: Marcelo Henrique Cerri <marcelo.cerri@canonical.com>

arch/powerpc/include/asm/paca.h

@@ -209,8 +209,8 @@ struct paca_struct {
 	 * other paca data leaking into the L1d
 	 */
 	u64 exrfi[13] __aligned(0x80);
-	/* Number of consecutive 128 byte lines that must be loaded */
-	u64 l1d_flush_lines;
+	u64 l1d_flush_congruence;
+	u64 l1d_flush_sets;
 #endif
 };
 

arch/powerpc/kernel/asm-offsets.c

@@ -245,7 +245,8 @@ int main(void)
 	DEFINE(PACA_IN_MCE, offsetof(struct paca_struct, in_mce));
 	OFFSET(PACA_RFI_FLUSH_FALLBACK_AREA, paca_struct, rfi_flush_fallback_area);
 	OFFSET(PACA_EXRFI, paca_struct, exrfi);
-	OFFSET(PACA_L1D_FLUSH_LINES, paca_struct, l1d_flush_lines);
+	OFFSET(PACA_L1D_FLUSH_CONGRUENCE, paca_struct, l1d_flush_congruence);
+	OFFSET(PACA_L1D_FLUSH_SETS, paca_struct, l1d_flush_sets);
 #endif
 	DEFINE(PACAHWCPUID, offsetof(struct paca_struct, hw_cpu_id));
 	DEFINE(PACAKEXECSTATE, offsetof(struct paca_struct, kexec_state));

arch/powerpc/kernel/exceptions-64s.S

@@ -727,29 +727,36 @@ rfi_flush_fallback:
 	std	r9,PACA_EXRFI+EX_R9(r13)
 	std	r10,PACA_EXRFI+EX_R10(r13)
 	std	r11,PACA_EXRFI+EX_R11(r13)
+	std	r12,PACA_EXRFI+EX_R12(r13)
+	std	r8,PACA_EXRFI+EX_R13(r13)
 	mfctr	r9
 	ld	r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
-	ld	r11,PACA_L1D_FLUSH_LINES(r13)
-	srdi	r11,r11,2	/* Unrolled x4 */
-	mtctr	r11
-	DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
-	/* XXX: Should an instruction synchronizing operation be done here? */
-1:
+	ld	r11,PACA_L1D_FLUSH_SETS(r13)
+	ld	r12,PACA_L1D_FLUSH_CONGRUENCE(r13)
 	/*
 	 * The load adresses are at staggered offsets within cachelines,
 	 * which suits some pipelines better (on others it should not
 	 * hurt.
 	 */
-	ld	r11,128*0+0(r10)
-	ld	r11,128*1+8(r10)
-	ld	r11,128*2+16(r10)
-	ld	r11,128*3+24(r10)
-	addi	r10,r10,(128 * 4)
+	addi	r12,r12,8
+	mtctr	r11
+	DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
+	/* XXX: Should an instruction synchronizing operation be done here? */
+
+1:	li	r8,0
+	.rept	8 /* 8-way set associative */
+	ldx	r11,r10,r8
+	add	r8,r8,r12
+	.endr
+	addi	r10,r10,128 /* 128 byte cache line */
 	bdnz	1b
+
 	mtctr	r9
 	ld	r9,PACA_EXRFI+EX_R9(r13)
 	ld	r10,PACA_EXRFI+EX_R10(r13)
 	ld	r11,PACA_EXRFI+EX_R11(r13)
+	ld	r12,PACA_EXRFI+EX_R12(r13)
+	ld	r8,PACA_EXRFI+EX_R13(r13)
 	GET_SCRATCH0(r13);
 	rfid
 
@@ -760,29 +767,36 @@ hrfi_flush_fallback:
 	std	r9,PACA_EXRFI+EX_R9(r13)
 	std	r10,PACA_EXRFI+EX_R10(r13)
 	std	r11,PACA_EXRFI+EX_R11(r13)
+	std	r12,PACA_EXRFI+EX_R12(r13)
+	std	r8,PACA_EXRFI+EX_R13(r13)
 	mfctr	r9
 	ld	r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
-	ld	r11,PACA_L1D_FLUSH_LINES(r13)
-	srdi	r11,r11,2	/* Unrolled x4 */
-	mtctr	r11
-	DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
-	/* XXX: Should an instruction synchronizing operation be done here? */
-1:
+	ld	r11,PACA_L1D_FLUSH_SETS(r13)
+	ld	r12,PACA_L1D_FLUSH_CONGRUENCE(r13)
 	/*
 	 * The load adresses are at staggered offsets within cachelines,
 	 * which suits some pipelines better (on others it should not
 	 * hurt.
 	 */
-	ld	r11,128*0+0(r10)
-	ld	r11,128*1+8(r10)
-	ld	r11,128*2+16(r10)
-	ld	r11,128*3+24(r10)
-	addi	r10,r10,(128 * 4)
+	addi	r12,r12,8
+	mtctr	r11
+	DCBT_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
+	/* XXX: Should an instruction synchronizing operation be done here? */
+
+1:	li	r8,0
+	.rept	8 /* 8-way set associative */
+	ldx	r11,r10,r8
+	add	r8,r8,r12
+	.endr
+	addi	r10,r10,128 /* 128 byte cache line */
 	bdnz	1b
+
 	mtctr	r9
 	ld	r9,PACA_EXRFI+EX_R9(r13)
 	ld	r10,PACA_EXRFI+EX_R10(r13)
 	ld	r11,PACA_EXRFI+EX_R11(r13)
+	ld	r12,PACA_EXRFI+EX_R12(r13)
+	ld	r8,PACA_EXRFI+EX_R13(r13)
 	GET_SCRATCH0(r13);
 	hrfid
 

arch/powerpc/kernel/setup_64.c

@@ -902,8 +902,19 @@ void __init setup_rfi_flush(enum l1d_flush_type type, bool enable)
 		memset(l1d_flush_fallback_area, 0, l1d_size * 2);
 
 		for_each_possible_cpu(cpu) {
+			/*
+			 * The fallback flush is currently coded for 8-way
+			 * associativity. Different associativity is possible,
+			 * but it will be treated as 8-way and may not evict
+			 * the lines as effectively.
+			 *
+			 * 128 byte lines are mandatory.
+			 */
+			u64 c = l1d_size / 8;
+
 			paca[cpu].rfi_flush_fallback_area = l1d_flush_fallback_area;
-			paca[cpu].l1d_flush_lines = l1d_size / 128;
+			paca[cpu].l1d_flush_congruence = c;
+			paca[cpu].l1d_flush_sets = c / 128;
 		}
 	}