Commit 22999300 authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86/mm changes from Ingo Molnar:
 "The biggest change is new TLB partial flushing code for AMD CPUs.
  (The v3.6 kernel had the Intel CPU side code, see commits
  e0ba94f1..effee4b9.)

  There's also various other refinements around the TLB flush code"

* 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86: Distinguish TLB shootdown interrupts from other functions call interrupts
  x86/mm: Fix range check in tlbflush debugfs interface
  x86, cpu: Preset default tlb_flushall_shift on AMD
  x86, cpu: Add AMD TLB size detection
  x86, cpu: Push TLB detection CPUID check down
  x86, cpu: Fixup tlb_flushall_shift formatting
parents 7687b80a fd0f5869
...@@ -18,6 +18,10 @@ typedef struct { ...@@ -18,6 +18,10 @@ typedef struct {
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
unsigned int irq_resched_count; unsigned int irq_resched_count;
unsigned int irq_call_count; unsigned int irq_call_count;
/*
* irq_tlb_count is double-counted in irq_call_count, so it must be
* subtracted from irq_call_count when displaying irq_call_count
*/
unsigned int irq_tlb_count; unsigned int irq_tlb_count;
#endif #endif
#ifdef CONFIG_X86_THERMAL_VECTOR #ifdef CONFIG_X86_THERMAL_VECTOR
......
...@@ -737,6 +737,72 @@ static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c, ...@@ -737,6 +737,72 @@ static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c,
} }
#endif #endif
static void __cpuinit cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c)
{
if (!cpu_has_invlpg)
return;
tlb_flushall_shift = 5;
if (c->x86 <= 0x11)
tlb_flushall_shift = 4;
}
static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
{
u32 ebx, eax, ecx, edx;
u16 mask = 0xfff;
if (c->x86 < 0xf)
return;
if (c->extended_cpuid_level < 0x80000006)
return;
cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
tlb_lli_4k[ENTRIES] = ebx & mask;
/*
* K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
* characteristics from the CPUID function 0x80000005 instead.
*/
if (c->x86 == 0xf) {
cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
mask = 0xff;
}
/* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
if (!((eax >> 16) & mask)) {
u32 a, b, c, d;
cpuid(0x80000005, &a, &b, &c, &d);
tlb_lld_2m[ENTRIES] = (a >> 16) & 0xff;
} else {
tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;
}
/* a 4M entry uses two 2M entries */
tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;
/* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
if (!(eax & mask)) {
/* Erratum 658 */
if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
tlb_lli_2m[ENTRIES] = 1024;
} else {
cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
tlb_lli_2m[ENTRIES] = eax & 0xff;
}
} else
tlb_lli_2m[ENTRIES] = eax & mask;
tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
cpu_set_tlb_flushall_shift(c);
}
static const struct cpu_dev __cpuinitconst amd_cpu_dev = { static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
.c_vendor = "AMD", .c_vendor = "AMD",
.c_ident = { "AuthenticAMD" }, .c_ident = { "AuthenticAMD" },
...@@ -756,6 +822,7 @@ static const struct cpu_dev __cpuinitconst amd_cpu_dev = { ...@@ -756,6 +822,7 @@ static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
.c_size_cache = amd_size_cache, .c_size_cache = amd_size_cache,
#endif #endif
.c_early_init = early_init_amd, .c_early_init = early_init_amd,
.c_detect_tlb = cpu_detect_tlb_amd,
.c_bsp_init = bsp_init_amd, .c_bsp_init = bsp_init_amd,
.c_init = init_amd, .c_init = init_amd,
.c_x86_vendor = X86_VENDOR_AMD, .c_x86_vendor = X86_VENDOR_AMD,
......
...@@ -476,7 +476,7 @@ void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c) ...@@ -476,7 +476,7 @@ void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \ printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
"Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \ "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
"tlb_flushall_shift is 0x%x\n", "tlb_flushall_shift: %d\n",
tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES], tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES], tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES], tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
...@@ -942,8 +942,7 @@ void __init identify_boot_cpu(void) ...@@ -942,8 +942,7 @@ void __init identify_boot_cpu(void)
#else #else
vgetcpu_set_mode(); vgetcpu_set_mode();
#endif #endif
if (boot_cpu_data.cpuid_level >= 2) cpu_detect_tlb(&boot_cpu_data);
cpu_detect_tlb(&boot_cpu_data);
} }
void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c) void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
......
...@@ -648,6 +648,10 @@ static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c) ...@@ -648,6 +648,10 @@ static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c)
int i, j, n; int i, j, n;
unsigned int regs[4]; unsigned int regs[4];
unsigned char *desc = (unsigned char *)regs; unsigned char *desc = (unsigned char *)regs;
if (c->cpuid_level < 2)
return;
/* Number of times to iterate */ /* Number of times to iterate */
n = cpuid_eax(2) & 0xFF; n = cpuid_eax(2) & 0xFF;
......
...@@ -92,7 +92,8 @@ int arch_show_interrupts(struct seq_file *p, int prec) ...@@ -92,7 +92,8 @@ int arch_show_interrupts(struct seq_file *p, int prec)
seq_printf(p, " Rescheduling interrupts\n"); seq_printf(p, " Rescheduling interrupts\n");
seq_printf(p, "%*s: ", prec, "CAL"); seq_printf(p, "%*s: ", prec, "CAL");
for_each_online_cpu(j) for_each_online_cpu(j)
seq_printf(p, "%10u ", irq_stats(j)->irq_call_count); seq_printf(p, "%10u ", irq_stats(j)->irq_call_count -
irq_stats(j)->irq_tlb_count);
seq_printf(p, " Function call interrupts\n"); seq_printf(p, " Function call interrupts\n");
seq_printf(p, "%*s: ", prec, "TLB"); seq_printf(p, "%*s: ", prec, "TLB");
for_each_online_cpu(j) for_each_online_cpu(j)
...@@ -147,7 +148,6 @@ u64 arch_irq_stat_cpu(unsigned int cpu) ...@@ -147,7 +148,6 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
sum += irq_stats(cpu)->irq_resched_count; sum += irq_stats(cpu)->irq_resched_count;
sum += irq_stats(cpu)->irq_call_count; sum += irq_stats(cpu)->irq_call_count;
sum += irq_stats(cpu)->irq_tlb_count;
#endif #endif
#ifdef CONFIG_X86_THERMAL_VECTOR #ifdef CONFIG_X86_THERMAL_VECTOR
sum += irq_stats(cpu)->irq_thermal_count; sum += irq_stats(cpu)->irq_thermal_count;
......
...@@ -98,6 +98,8 @@ static void flush_tlb_func(void *info) ...@@ -98,6 +98,8 @@ static void flush_tlb_func(void *info)
{ {
struct flush_tlb_info *f = info; struct flush_tlb_info *f = info;
inc_irq_stat(irq_tlb_count);
if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
return; return;
...@@ -320,7 +322,7 @@ static ssize_t tlbflush_write_file(struct file *file, ...@@ -320,7 +322,7 @@ static ssize_t tlbflush_write_file(struct file *file,
if (kstrtos8(buf, 0, &shift)) if (kstrtos8(buf, 0, &shift))
return -EINVAL; return -EINVAL;
if (shift > 64) if (shift < -1 || shift >= BITS_PER_LONG)
return -EINVAL; return -EINVAL;
tlb_flushall_shift = shift; tlb_flushall_shift = shift;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment