Commit 859deea9 authored by Benjamin Herrenschmidt's avatar Benjamin Herrenschmidt Committed by Paul Mackerras

[POWERPC] Cell timebase bug workaround

The Cell CPU timebase has an erratum. When reading the entire 64 bits
of the timebase with one mftb instruction, there is a handful of cycles
window during which one might read a value with the low order 32 bits
already reset to 0x00000000 but the high order bits not yet incremeted
by one. This fixes it by reading the timebase again until the low order
32 bits is no longer 0. That might introduce occasional latencies if
hitting mftb just at the wrong time, but no more than 70ns on a cell
blade, and that was considered acceptable.
Signed-off-by: default avatarBenjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: default avatarOlof Johansson <olof@lixom.net>
Signed-off-by: default avatarPaul Mackerras <paulus@samba.org>
parent 21c4ff80
...@@ -229,8 +229,10 @@ V_FUNCTION_BEGIN(__do_get_xsec) ...@@ -229,8 +229,10 @@ V_FUNCTION_BEGIN(__do_get_xsec)
xor r0,r8,r8 /* create dependency */ xor r0,r8,r8 /* create dependency */
add r3,r3,r0 add r3,r3,r0
/* Get TB & offset it */ /* Get TB & offset it. We use the MFTB macro which will generate
mftb r7 * workaround code for Cell.
*/
MFTB(r7)
ld r9,CFG_TB_ORIG_STAMP(r3) ld r9,CFG_TB_ORIG_STAMP(r3)
subf r7,r9,r7 subf r7,r9,r7
......
...@@ -147,6 +147,7 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start, ...@@ -147,6 +147,7 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start,
#define CPU_FTR_CI_LARGE_PAGE LONG_ASM_CONST(0x0000100000000000) #define CPU_FTR_CI_LARGE_PAGE LONG_ASM_CONST(0x0000100000000000)
#define CPU_FTR_PAUSE_ZERO LONG_ASM_CONST(0x0000200000000000) #define CPU_FTR_PAUSE_ZERO LONG_ASM_CONST(0x0000200000000000)
#define CPU_FTR_PURR LONG_ASM_CONST(0x0000400000000000) #define CPU_FTR_PURR LONG_ASM_CONST(0x0000400000000000)
#define CPU_FTR_CELL_TB_BUG LONG_ASM_CONST(0x0000800000000000)
#ifndef __ASSEMBLY__ #ifndef __ASSEMBLY__
...@@ -335,7 +336,7 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start, ...@@ -335,7 +336,7 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start,
#define CPU_FTRS_CELL (CPU_FTR_SPLIT_ID_CACHE | CPU_FTR_USE_TB | \ #define CPU_FTRS_CELL (CPU_FTR_SPLIT_ID_CACHE | CPU_FTR_USE_TB | \
CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \ CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | \
CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_MMCRA | CPU_FTR_SMT | \
CPU_FTR_PAUSE_ZERO | CPU_FTR_CI_LARGE_PAGE) CPU_FTR_PAUSE_ZERO | CPU_FTR_CI_LARGE_PAGE | CPU_FTR_CELL_TB_BUG)
#define CPU_FTRS_PA6T (CPU_FTR_SPLIT_ID_CACHE | CPU_FTR_USE_TB | \ #define CPU_FTRS_PA6T (CPU_FTR_SPLIT_ID_CACHE | CPU_FTR_USE_TB | \
CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | \ CPU_FTR_HPTE_TABLE | CPU_FTR_PPCAS_ARCH_V2 | \
CPU_FTR_ALTIVEC_COMP | CPU_FTR_CI_LARGE_PAGE | \ CPU_FTR_ALTIVEC_COMP | CPU_FTR_CI_LARGE_PAGE | \
......
...@@ -30,7 +30,7 @@ BEGIN_FTR_SECTION; \ ...@@ -30,7 +30,7 @@ BEGIN_FTR_SECTION; \
mfspr ra,SPRN_PURR; /* get processor util. reg */ \ mfspr ra,SPRN_PURR; /* get processor util. reg */ \
END_FTR_SECTION_IFSET(CPU_FTR_PURR); \ END_FTR_SECTION_IFSET(CPU_FTR_PURR); \
BEGIN_FTR_SECTION; \ BEGIN_FTR_SECTION; \
mftb ra; /* or get TB if no PURR */ \ MFTB(ra); /* or get TB if no PURR */ \
END_FTR_SECTION_IFCLR(CPU_FTR_PURR); \ END_FTR_SECTION_IFCLR(CPU_FTR_PURR); \
ld rb,PACA_STARTPURR(r13); \ ld rb,PACA_STARTPURR(r13); \
std ra,PACA_STARTPURR(r13); \ std ra,PACA_STARTPURR(r13); \
...@@ -45,7 +45,7 @@ BEGIN_FTR_SECTION; \ ...@@ -45,7 +45,7 @@ BEGIN_FTR_SECTION; \
mfspr ra,SPRN_PURR; /* get processor util. reg */ \ mfspr ra,SPRN_PURR; /* get processor util. reg */ \
END_FTR_SECTION_IFSET(CPU_FTR_PURR); \ END_FTR_SECTION_IFSET(CPU_FTR_PURR); \
BEGIN_FTR_SECTION; \ BEGIN_FTR_SECTION; \
mftb ra; /* or get TB if no PURR */ \ MFTB(ra); /* or get TB if no PURR */ \
END_FTR_SECTION_IFCLR(CPU_FTR_PURR); \ END_FTR_SECTION_IFCLR(CPU_FTR_PURR); \
ld rb,PACA_STARTPURR(r13); \ ld rb,PACA_STARTPURR(r13); \
std ra,PACA_STARTPURR(r13); \ std ra,PACA_STARTPURR(r13); \
...@@ -274,6 +274,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_601) ...@@ -274,6 +274,16 @@ END_FTR_SECTION_IFSET(CPU_FTR_601)
#define ISYNC_601 #define ISYNC_601
#endif #endif
#ifdef CONFIG_PPC_CELL
#define MFTB(dest) \
90: mftb dest; \
BEGIN_FTR_SECTION_NESTED(96); \
cmpwi dest,0; \
beq- 90b; \
END_FTR_SECTION_NESTED(CPU_FTR_CELL_TB_BUG, CPU_FTR_CELL_TB_BUG, 96)
#else
#define MFTB(dest) mftb dest
#endif
#ifndef CONFIG_SMP #ifndef CONFIG_SMP
#define TLBSYNC #define TLBSYNC
......
...@@ -619,10 +619,35 @@ ...@@ -619,10 +619,35 @@
: "=r" (rval)); rval;}) : "=r" (rval)); rval;})
#define mtspr(rn, v) asm volatile("mtspr " __stringify(rn) ",%0" : : "r" (v)) #define mtspr(rn, v) asm volatile("mtspr " __stringify(rn) ",%0" : : "r" (v))
#ifdef __powerpc64__
#ifdef CONFIG_PPC_CELL
#define mftb() ({unsigned long rval; \
asm volatile( \
"90: mftb %0;\n" \
"97: cmpwi %0,0;\n" \
" beq- 90b;\n" \
"99:\n" \
".section __ftr_fixup,\"a\"\n" \
".align 3\n" \
"98:\n" \
" .llong %1\n" \
" .llong %1\n" \
" .llong 97b-98b\n" \
" .llong 99b-98b\n" \
".previous" \
: "=r" (rval) : "i" (CPU_FTR_CELL_TB_BUG)); rval;})
#else
#define mftb() ({unsigned long rval; \ #define mftb() ({unsigned long rval; \
asm volatile("mftb %0" : "=r" (rval)); rval;}) asm volatile("mftb %0" : "=r" (rval)); rval;})
#endif /* !CONFIG_PPC_CELL */
#else /* __powerpc64__ */
#define mftbl() ({unsigned long rval; \ #define mftbl() ({unsigned long rval; \
asm volatile("mftbl %0" : "=r" (rval)); rval;}) asm volatile("mftbl %0" : "=r" (rval)); rval;})
#define mftbu() ({unsigned long rval; \
asm volatile("mftbu %0" : "=r" (rval)); rval;})
#endif /* !__powerpc64__ */
#define mttbl(v) asm volatile("mttbl %0":: "r"(v)) #define mttbl(v) asm volatile("mttbl %0":: "r"(v))
#define mttbu(v) asm volatile("mttbu %0":: "r"(v)) #define mttbu(v) asm volatile("mttbu %0":: "r"(v))
......
...@@ -82,30 +82,35 @@ struct div_result { ...@@ -82,30 +82,35 @@ struct div_result {
#define __USE_RTC() 0 #define __USE_RTC() 0
#endif #endif
/* On ppc64 this gets us the whole timebase; on ppc32 just the lower half */ #ifdef CONFIG_PPC64
/* For compatibility, get_tbl() is defined as get_tb() on ppc64 */
#define get_tbl get_tb
#else
static inline unsigned long get_tbl(void) static inline unsigned long get_tbl(void)
{ {
unsigned long tbl;
#if defined(CONFIG_403GCX) #if defined(CONFIG_403GCX)
unsigned long tbl;
asm volatile("mfspr %0, 0x3dd" : "=r" (tbl)); asm volatile("mfspr %0, 0x3dd" : "=r" (tbl));
return tbl;
#else #else
asm volatile("mftb %0" : "=r" (tbl)); return mftbl();
#endif #endif
return tbl;
} }
static inline unsigned int get_tbu(void) static inline unsigned int get_tbu(void)
{ {
#ifdef CONFIG_403GCX
unsigned int tbu; unsigned int tbu;
#if defined(CONFIG_403GCX)
asm volatile("mfspr %0, 0x3dc" : "=r" (tbu)); asm volatile("mfspr %0, 0x3dc" : "=r" (tbu));
return tbu;
#else #else
asm volatile("mftbu %0" : "=r" (tbu)); return mftbu();
#endif #endif
return tbu;
} }
#endif /* !CONFIG_PPC64 */
static inline unsigned int get_rtcl(void) static inline unsigned int get_rtcl(void)
{ {
...@@ -131,7 +136,7 @@ static inline u64 get_tb(void) ...@@ -131,7 +136,7 @@ static inline u64 get_tb(void)
{ {
return mftb(); return mftb();
} }
#else #else /* CONFIG_PPC64 */
static inline u64 get_tb(void) static inline u64 get_tb(void)
{ {
unsigned int tbhi, tblo, tbhi2; unsigned int tbhi, tblo, tbhi2;
...@@ -144,7 +149,7 @@ static inline u64 get_tb(void) ...@@ -144,7 +149,7 @@ static inline u64 get_tb(void)
return ((u64)tbhi << 32) | tblo; return ((u64)tbhi << 32) | tblo;
} }
#endif #endif /* !CONFIG_PPC64 */
static inline void set_tb(unsigned int upper, unsigned int lower) static inline void set_tb(unsigned int upper, unsigned int lower)
{ {
......
...@@ -8,6 +8,7 @@ ...@@ -8,6 +8,7 @@
*/ */
#include <asm/cputable.h> #include <asm/cputable.h>
#include <asm/reg.h>
#define CLOCK_TICK_RATE 1024000 /* Underlying HZ */ #define CLOCK_TICK_RATE 1024000 /* Underlying HZ */
...@@ -15,13 +16,11 @@ typedef unsigned long cycles_t; ...@@ -15,13 +16,11 @@ typedef unsigned long cycles_t;
static inline cycles_t get_cycles(void) static inline cycles_t get_cycles(void)
{ {
cycles_t ret;
#ifdef __powerpc64__ #ifdef __powerpc64__
return mftb();
__asm__ __volatile__("mftb %0" : "=r" (ret) : );
#else #else
cycles_t ret;
/* /*
* For the "cycle" counter we use the timebase lower half. * For the "cycle" counter we use the timebase lower half.
* Currently only used on SMP. * Currently only used on SMP.
...@@ -41,9 +40,8 @@ static inline cycles_t get_cycles(void) ...@@ -41,9 +40,8 @@ static inline cycles_t get_cycles(void)
" .long 99b-98b\n" " .long 99b-98b\n"
".previous" ".previous"
: "=r" (ret) : "i" (CPU_FTR_601)); : "=r" (ret) : "i" (CPU_FTR_601));
#endif
return ret; return ret;
#endif
} }
#endif /* __KERNEL__ */ #endif /* __KERNEL__ */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment