Commit b73522e0 authored by Toshi Kani's avatar Toshi Kani Committed by Ingo Molnar

x86/mm/mtrr: Enhance MTRR checks in kernel mapping helpers

This patch adds the argument 'uniform' to mtrr_type_lookup(),
which gets set to 1 when a given range is covered uniformly by
MTRRs, i.e. the range is fully covered by a single MTRR entry or
the default type.

Change pud_set_huge() and pmd_set_huge() to honor the 'uniform'
flag to see if it is safe to create a huge page mapping in the
range.

This allows them to create a huge page mapping in a range
covered by a single MTRR entry of any memory type. It also
detects a non-optimal request properly. They continue to check
with the WB type since it does not effectively change the
uniform mapping even if a request spans multiple MTRR entries.

pmd_set_huge() logs a warning message to a non-optimal request
so that driver writers will be aware of such a case. Drivers
should make a mapping request aligned to a single MTRR entry
when the range is covered by MTRRs.
Signed-off-by: default avatarToshi Kani <toshi.kani@hp.com>
[ Realign, flesh out comments, improve warning message. ]
Signed-off-by: default avatarBorislav Petkov <bp@suse.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Elliott@hp.com
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Luis R. Rodriguez <mcgrof@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: dave.hansen@intel.com
Cc: linux-mm <linux-mm@kvack.org>
Cc: pebolle@tiscali.nl
Link: http://lkml.kernel.org/r/1431714237-880-7-git-send-email-toshi.kani@hp.com
Link: http://lkml.kernel.org/r/1432628901-18044-8-git-send-email-bp@alien8.deSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 0cc705f5
...@@ -31,7 +31,7 @@ ...@@ -31,7 +31,7 @@
* arch_phys_wc_add and arch_phys_wc_del. * arch_phys_wc_add and arch_phys_wc_del.
*/ */
# ifdef CONFIG_MTRR # ifdef CONFIG_MTRR
extern u8 mtrr_type_lookup(u64 addr, u64 end); extern u8 mtrr_type_lookup(u64 addr, u64 end, u8 *uniform);
extern void mtrr_save_fixed_ranges(void *); extern void mtrr_save_fixed_ranges(void *);
extern void mtrr_save_state(void); extern void mtrr_save_state(void);
extern int mtrr_add(unsigned long base, unsigned long size, extern int mtrr_add(unsigned long base, unsigned long size,
...@@ -50,7 +50,7 @@ extern int mtrr_trim_uncached_memory(unsigned long end_pfn); ...@@ -50,7 +50,7 @@ extern int mtrr_trim_uncached_memory(unsigned long end_pfn);
extern int amd_special_default_mtrr(void); extern int amd_special_default_mtrr(void);
extern int phys_wc_to_mtrr_index(int handle); extern int phys_wc_to_mtrr_index(int handle);
# else # else
static inline u8 mtrr_type_lookup(u64 addr, u64 end) static inline u8 mtrr_type_lookup(u64 addr, u64 end, u8 *uniform)
{ {
/* /*
* Return no-MTRRs: * Return no-MTRRs:
......
...@@ -147,19 +147,24 @@ static u8 mtrr_type_lookup_fixed(u64 start, u64 end) ...@@ -147,19 +147,24 @@ static u8 mtrr_type_lookup_fixed(u64 start, u64 end)
* Return Value: * Return Value:
* MTRR_TYPE_(type) - Matched memory type or default memory type (unmatched) * MTRR_TYPE_(type) - Matched memory type or default memory type (unmatched)
* *
* Output Argument: * Output Arguments:
* repeat - Set to 1 when [start:end] spanned across MTRR range and type * repeat - Set to 1 when [start:end] spanned across MTRR range and type
* returned corresponds only to [start:*partial_end]. Caller has * returned corresponds only to [start:*partial_end]. Caller has
* to lookup again for [*partial_end:end]. * to lookup again for [*partial_end:end].
*
* uniform - Set to 1 when an MTRR covers the region uniformly, i.e. the
* region is fully covered by a single MTRR entry or the default
* type.
*/ */
static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end, static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end,
int *repeat) int *repeat, u8 *uniform)
{ {
int i; int i;
u64 base, mask; u64 base, mask;
u8 prev_match, curr_match; u8 prev_match, curr_match;
*repeat = 0; *repeat = 0;
*uniform = 1;
/* Make end inclusive instead of exclusive */ /* Make end inclusive instead of exclusive */
end--; end--;
...@@ -214,6 +219,7 @@ static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end, ...@@ -214,6 +219,7 @@ static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end,
end = *partial_end - 1; /* end is inclusive */ end = *partial_end - 1; /* end is inclusive */
*repeat = 1; *repeat = 1;
*uniform = 0;
} }
if ((start & mask) != (base & mask)) if ((start & mask) != (base & mask))
...@@ -225,6 +231,7 @@ static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end, ...@@ -225,6 +231,7 @@ static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end,
continue; continue;
} }
*uniform = 0;
if (check_type_overlap(&prev_match, &curr_match)) if (check_type_overlap(&prev_match, &curr_match))
return curr_match; return curr_match;
} }
...@@ -241,10 +248,15 @@ static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end, ...@@ -241,10 +248,15 @@ static u8 mtrr_type_lookup_variable(u64 start, u64 end, u64 *partial_end,
* Return Values: * Return Values:
* MTRR_TYPE_(type) - The effective MTRR type for the region * MTRR_TYPE_(type) - The effective MTRR type for the region
* MTRR_TYPE_INVALID - MTRR is disabled * MTRR_TYPE_INVALID - MTRR is disabled
*
* Output Argument:
* uniform - Set to 1 when an MTRR covers the region uniformly, i.e. the
* region is fully covered by a single MTRR entry or the default
* type.
*/ */
u8 mtrr_type_lookup(u64 start, u64 end) u8 mtrr_type_lookup(u64 start, u64 end, u8 *uniform)
{ {
u8 type, prev_type; u8 type, prev_type, is_uniform = 1, dummy;
int repeat; int repeat;
u64 partial_end; u64 partial_end;
...@@ -260,14 +272,18 @@ u8 mtrr_type_lookup(u64 start, u64 end) ...@@ -260,14 +272,18 @@ u8 mtrr_type_lookup(u64 start, u64 end)
*/ */
if ((start < 0x100000) && if ((start < 0x100000) &&
(mtrr_state.have_fixed) && (mtrr_state.have_fixed) &&
(mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) (mtrr_state.enabled & MTRR_STATE_MTRR_FIXED_ENABLED)) {
return mtrr_type_lookup_fixed(start, end); is_uniform = 0;
type = mtrr_type_lookup_fixed(start, end);
goto out;
}
/* /*
* Look up the variable ranges. Look of multiple ranges matching * Look up the variable ranges. Look of multiple ranges matching
* this address and pick type as per MTRR precedence. * this address and pick type as per MTRR precedence.
*/ */
type = mtrr_type_lookup_variable(start, end, &partial_end, &repeat); type = mtrr_type_lookup_variable(start, end, &partial_end,
&repeat, &is_uniform);
/* /*
* Common path is with repeat = 0. * Common path is with repeat = 0.
...@@ -278,15 +294,19 @@ u8 mtrr_type_lookup(u64 start, u64 end) ...@@ -278,15 +294,19 @@ u8 mtrr_type_lookup(u64 start, u64 end)
while (repeat) { while (repeat) {
prev_type = type; prev_type = type;
start = partial_end; start = partial_end;
type = mtrr_type_lookup_variable(start, end, &partial_end, &repeat); is_uniform = 0;
type = mtrr_type_lookup_variable(start, end, &partial_end,
&repeat, &dummy);
if (check_type_overlap(&prev_type, &type)) if (check_type_overlap(&prev_type, &type))
return type; goto out;
} }
if (mtrr_tom2 && (start >= (1ULL<<32)) && (end < mtrr_tom2)) if (mtrr_tom2 && (start >= (1ULL<<32)) && (end < mtrr_tom2))
return MTRR_TYPE_WRBACK; type = MTRR_TYPE_WRBACK;
out:
*uniform = is_uniform;
return type; return type;
} }
......
...@@ -267,9 +267,9 @@ static unsigned long pat_x_mtrr_type(u64 start, u64 end, ...@@ -267,9 +267,9 @@ static unsigned long pat_x_mtrr_type(u64 start, u64 end,
* request is for WB. * request is for WB.
*/ */
if (req_type == _PAGE_CACHE_MODE_WB) { if (req_type == _PAGE_CACHE_MODE_WB) {
u8 mtrr_type; u8 mtrr_type, uniform;
mtrr_type = mtrr_type_lookup(start, end); mtrr_type = mtrr_type_lookup(start, end, &uniform);
if (mtrr_type != MTRR_TYPE_WRBACK) if (mtrr_type != MTRR_TYPE_WRBACK)
return _PAGE_CACHE_MODE_UC_MINUS; return _PAGE_CACHE_MODE_UC_MINUS;
......
...@@ -566,19 +566,28 @@ void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys, ...@@ -566,19 +566,28 @@ void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys,
/** /**
* pud_set_huge - setup kernel PUD mapping * pud_set_huge - setup kernel PUD mapping
* *
* MTRR can override PAT memory types with 4KiB granularity. Therefore, * MTRRs can override PAT memory types with 4KiB granularity. Therefore, this
* this function does not set up a huge page when the range is covered * function sets up a huge page only if any of the following conditions are met:
* by a non-WB type of MTRR. MTRR_TYPE_INVALID indicates that MTRR are *
* disabled. * - MTRRs are disabled, or
*
* - MTRRs are enabled and the range is completely covered by a single MTRR, or
*
* - MTRRs are enabled and the corresponding MTRR memory type is WB, which
* has no effect on the requested PAT memory type.
*
* Callers should try to decrease page size (1GB -> 2MB -> 4K) if the bigger
* page mapping attempt fails.
* *
* Returns 1 on success and 0 on failure. * Returns 1 on success and 0 on failure.
*/ */
int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
{ {
u8 mtrr; u8 mtrr, uniform;
mtrr = mtrr_type_lookup(addr, addr + PUD_SIZE); mtrr = mtrr_type_lookup(addr, addr + PUD_SIZE, &uniform);
if ((mtrr != MTRR_TYPE_WRBACK) && (mtrr != MTRR_TYPE_INVALID)) if ((mtrr != MTRR_TYPE_INVALID) && (!uniform) &&
(mtrr != MTRR_TYPE_WRBACK))
return 0; return 0;
prot = pgprot_4k_2_large(prot); prot = pgprot_4k_2_large(prot);
...@@ -593,20 +602,21 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) ...@@ -593,20 +602,21 @@ int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
/** /**
* pmd_set_huge - setup kernel PMD mapping * pmd_set_huge - setup kernel PMD mapping
* *
* MTRR can override PAT memory types with 4KiB granularity. Therefore, * See text over pud_set_huge() above.
* this function does not set up a huge page when the range is covered
* by a non-WB type of MTRR. MTRR_TYPE_INVALID indicates that MTRR are
* disabled.
* *
* Returns 1 on success and 0 on failure. * Returns 1 on success and 0 on failure.
*/ */
int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
{ {
u8 mtrr; u8 mtrr, uniform;
mtrr = mtrr_type_lookup(addr, addr + PMD_SIZE); mtrr = mtrr_type_lookup(addr, addr + PMD_SIZE, &uniform);
if ((mtrr != MTRR_TYPE_WRBACK) && (mtrr != MTRR_TYPE_INVALID)) if ((mtrr != MTRR_TYPE_INVALID) && (!uniform) &&
(mtrr != MTRR_TYPE_WRBACK)) {
pr_warn_once("%s: Cannot satisfy [mem %#010llx-%#010llx] with a huge-page mapping due to MTRR override.\n",
__func__, addr, addr + PMD_SIZE);
return 0; return 0;
}
prot = pgprot_4k_2_large(prot); prot = pgprot_4k_2_large(prot);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment