Commit c26a535b authored by Catalin Marinas's avatar Catalin Marinas

Merge tag 'for-3.20' of http://git.linaro.org/people/ard.biesheuvel/linux-arm into upstream

UEFI updates for arm64

This series consists of a reimplementation of the virtual remapping of
UEFI Runtime Services in a way that is stable across kexec, including
the required preparatory refactoring and other work to set the stage,
and some cleaning up afterwards to remove boot services memory and
identitity map handling that has now become redundant.

* tag 'for-3.20' of http://git.linaro.org/people/ard.biesheuvel/linux-arm:
  arm64/efi: remove idmap manipulations from UEFI code
  arm64/efi: remove free_boot_services() and friends
  arm64/efi: move SetVirtualAddressMap() to UEFI stub
  arm64/efi: set EFI_ALLOC_ALIGN to 64 KB
  efi: efistub: allow allocation alignment larger than EFI_PAGE_SIZE
  efi: split off remapping code from efi_config_init()
  arm64/mm: add create_pgd_mapping() to create private page tables
  arm64/mm: add explicit struct_mm argument to __create_mapping()
parents eaa27f34 9679be10
......@@ -6,29 +6,35 @@
#ifdef CONFIG_EFI
extern void efi_init(void);
extern void efi_idmap_init(void);
extern void efi_virtmap_init(void);
#else
#define efi_init()
#define efi_idmap_init()
#define efi_virtmap_init()
#endif
#define efi_call_virt(f, ...) \
({ \
efi_##f##_t *__f = efi.systab->runtime->f; \
efi_##f##_t *__f; \
efi_status_t __s; \
\
kernel_neon_begin(); \
efi_virtmap_load(); \
__f = efi.systab->runtime->f; \
__s = __f(__VA_ARGS__); \
efi_virtmap_unload(); \
kernel_neon_end(); \
__s; \
})
#define __efi_call_virt(f, ...) \
({ \
efi_##f##_t *__f = efi.systab->runtime->f; \
efi_##f##_t *__f; \
\
kernel_neon_begin(); \
efi_virtmap_load(); \
__f = efi.systab->runtime->f; \
__f(__VA_ARGS__); \
efi_virtmap_unload(); \
kernel_neon_end(); \
})
......@@ -44,4 +50,28 @@ extern void efi_idmap_init(void);
#define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__)
#define EFI_ALLOC_ALIGN SZ_64K
/*
* On ARM systems, virtually remapped UEFI runtime services are set up in three
* distinct stages:
* - The stub retrieves the final version of the memory map from UEFI, populates
* the virt_addr fields and calls the SetVirtualAddressMap() [SVAM] runtime
* service to communicate the new mapping to the firmware (Note that the new
* mapping is not live at this time)
* - During early boot, the page tables are allocated and populated based on the
* virt_addr fields in the memory map, but only if all descriptors with the
* EFI_MEMORY_RUNTIME attribute have a non-zero value for virt_addr. If this
* succeeds, the EFI_VIRTMAP flag is set to indicate that the virtual mappings
* have been installed successfully.
* - During an early initcall(), the UEFI Runtime Services are enabled and the
* EFI_RUNTIME_SERVICES bit set if some conditions are met, i.e., we need a
* non-early mapping of the UEFI system table, and we need to have the virtmap
* installed.
*/
#define EFI_VIRTMAP EFI_ARCH_1
void efi_virtmap_load(void);
void efi_virtmap_unload(void);
#endif /* _ASM_EFI_H */
......@@ -31,7 +31,8 @@ extern void paging_init(void);
extern void setup_mm_for_reboot(void);
extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
extern void init_mem_pgprot(void);
/* create an identity mapping for memory (or io if map_io is true) */
extern void create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io);
extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot);
#endif
......@@ -264,6 +264,11 @@ static inline pmd_t pte_pmd(pte_t pte)
return __pmd(pte_val(pte));
}
static inline pgprot_t mk_sect_prot(pgprot_t prot)
{
return __pgprot(pgprot_val(prot) & ~PTE_TABLE_BIT);
}
/*
* THP definitions.
*/
......
This diff is collapsed.
......@@ -401,7 +401,7 @@ void __init setup_arch(char **cmdline_p)
paging_init();
request_standard_resources();
efi_idmap_init();
efi_virtmap_init();
early_ioremap_reset();
unflatten_device_tree();
......
......@@ -156,29 +156,19 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
} while (pte++, addr += PAGE_SIZE, addr != end);
}
static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
unsigned long end, phys_addr_t phys,
int map_io)
static void __init alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot)
{
pmd_t *pmd;
unsigned long next;
pmdval_t prot_sect;
pgprot_t prot_pte;
if (map_io) {
prot_sect = PROT_SECT_DEVICE_nGnRE;
prot_pte = __pgprot(PROT_DEVICE_nGnRE);
} else {
prot_sect = PROT_SECT_NORMAL_EXEC;
prot_pte = PAGE_KERNEL_EXEC;
}
/*
* Check for initial section mappings in the pgd/pud and remove them.
*/
if (pud_none(*pud) || pud_bad(*pud)) {
pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t));
pud_populate(&init_mm, pud, pmd);
pud_populate(mm, pud, pmd);
}
pmd = pmd_offset(pud, addr);
......@@ -187,7 +177,8 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
/* try section mapping first */
if (((addr | next | phys) & ~SECTION_MASK) == 0) {
pmd_t old_pmd =*pmd;
set_pmd(pmd, __pmd(phys | prot_sect));
set_pmd(pmd, __pmd(phys |
pgprot_val(mk_sect_prot(prot))));
/*
* Check for previous table entries created during
* boot (__create_page_tables) and flush them.
......@@ -196,22 +187,22 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
flush_tlb_all();
} else {
alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
prot_pte);
prot);
}
phys += next - addr;
} while (pmd++, addr = next, addr != end);
}
static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
unsigned long end, phys_addr_t phys,
int map_io)
static void __init alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,
unsigned long addr, unsigned long end,
phys_addr_t phys, pgprot_t prot)
{
pud_t *pud;
unsigned long next;
if (pgd_none(*pgd)) {
pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t));
pgd_populate(&init_mm, pgd, pud);
pgd_populate(mm, pgd, pud);
}
BUG_ON(pgd_bad(*pgd));
......@@ -222,10 +213,11 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
/*
* For 4K granule only, attempt to put down a 1GB block
*/
if (!map_io && (PAGE_SHIFT == 12) &&
if ((PAGE_SHIFT == 12) &&
((addr | next | phys) & ~PUD_MASK) == 0) {
pud_t old_pud = *pud;
set_pud(pud, __pud(phys | PROT_SECT_NORMAL_EXEC));
set_pud(pud, __pud(phys |
pgprot_val(mk_sect_prot(prot))));
/*
* If we have an old value for a pud, it will
......@@ -240,7 +232,7 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
flush_tlb_all();
}
} else {
alloc_init_pmd(pud, addr, next, phys, map_io);
alloc_init_pmd(mm, pud, addr, next, phys, prot);
}
phys += next - addr;
} while (pud++, addr = next, addr != end);
......@@ -250,9 +242,9 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
* Create the page directory entries and any necessary page tables for the
* mapping specified by 'md'.
*/
static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
int map_io)
static void __init __create_mapping(struct mm_struct *mm, pgd_t *pgd,
phys_addr_t phys, unsigned long virt,
phys_addr_t size, pgprot_t prot)
{
unsigned long addr, length, end, next;
......@@ -262,7 +254,7 @@ static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
end = addr + length;
do {
next = pgd_addr_end(addr, end);
alloc_init_pud(pgd, addr, next, phys, map_io);
alloc_init_pud(mm, pgd, addr, next, phys, prot);
phys += next - addr;
} while (pgd++, addr = next, addr != end);
}
......@@ -275,17 +267,15 @@ static void __init create_mapping(phys_addr_t phys, unsigned long virt,
&phys, virt);
return;
}
__create_mapping(pgd_offset_k(virt & PAGE_MASK), phys, virt, size, 0);
__create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt,
size, PAGE_KERNEL_EXEC);
}
void __init create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io)
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
unsigned long virt, phys_addr_t size,
pgprot_t prot)
{
if ((addr >> PGDIR_SHIFT) >= ARRAY_SIZE(idmap_pg_dir)) {
pr_warn("BUG: not creating id mapping for %pa\n", &addr);
return;
}
__create_mapping(&idmap_pg_dir[pgd_index(addr)],
addr, addr, size, map_io);
__create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot);
}
static void __init map_mem(void)
......
......@@ -293,29 +293,15 @@ static __init int match_config_table(efi_guid_t *guid,
return 0;
}
int __init efi_config_init(efi_config_table_type_t *arch_tables)
int __init efi_config_parse_tables(void *config_tables, int count, int sz,
efi_config_table_type_t *arch_tables)
{
void *config_tables, *tablep;
int i, sz;
if (efi_enabled(EFI_64BIT))
sz = sizeof(efi_config_table_64_t);
else
sz = sizeof(efi_config_table_32_t);
/*
* Let's see what config tables the firmware passed to us.
*/
config_tables = early_memremap(efi.systab->tables,
efi.systab->nr_tables * sz);
if (config_tables == NULL) {
pr_err("Could not map Configuration table!\n");
return -ENOMEM;
}
void *tablep;
int i;
tablep = config_tables;
pr_info("");
for (i = 0; i < efi.systab->nr_tables; i++) {
for (i = 0; i < count; i++) {
efi_guid_t guid;
unsigned long table;
......@@ -328,8 +314,6 @@ int __init efi_config_init(efi_config_table_type_t *arch_tables)
if (table64 >> 32) {
pr_cont("\n");
pr_err("Table located above 4GB, disabling EFI.\n");
early_memunmap(config_tables,
efi.systab->nr_tables * sz);
return -EINVAL;
}
#endif
......@@ -344,13 +328,37 @@ int __init efi_config_init(efi_config_table_type_t *arch_tables)
tablep += sz;
}
pr_cont("\n");
early_memunmap(config_tables, efi.systab->nr_tables * sz);
set_bit(EFI_CONFIG_TABLES, &efi.flags);
return 0;
}
int __init efi_config_init(efi_config_table_type_t *arch_tables)
{
void *config_tables;
int sz, ret;
if (efi_enabled(EFI_64BIT))
sz = sizeof(efi_config_table_64_t);
else
sz = sizeof(efi_config_table_32_t);
/*
* Let's see what config tables the firmware passed to us.
*/
config_tables = early_memremap(efi.systab->tables,
efi.systab->nr_tables * sz);
if (config_tables == NULL) {
pr_err("Could not map Configuration table!\n");
return -ENOMEM;
}
ret = efi_config_parse_tables(config_tables, efi.systab->nr_tables, sz,
arch_tables);
early_memunmap(config_tables, efi.systab->nr_tables * sz);
return ret;
}
#ifdef CONFIG_EFI_VARS_MODULE
static int __init efi_load_efivars(void)
{
......
......@@ -295,3 +295,62 @@ unsigned long __init efi_entry(void *handle, efi_system_table_t *sys_table,
fail:
return EFI_ERROR;
}
/*
* This is the base address at which to start allocating virtual memory ranges
* for UEFI Runtime Services. This is in the low TTBR0 range so that we can use
* any allocation we choose, and eliminate the risk of a conflict after kexec.
* The value chosen is the largest non-zero power of 2 suitable for this purpose
* both on 32-bit and 64-bit ARM CPUs, to maximize the likelihood that it can
* be mapped efficiently.
*/
#define EFI_RT_VIRTUAL_BASE 0x40000000
/*
* efi_get_virtmap() - create a virtual mapping for the EFI memory map
*
* This function populates the virt_addr fields of all memory region descriptors
* in @memory_map whose EFI_MEMORY_RUNTIME attribute is set. Those descriptors
* are also copied to @runtime_map, and their total count is returned in @count.
*/
void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
unsigned long desc_size, efi_memory_desc_t *runtime_map,
int *count)
{
u64 efi_virt_base = EFI_RT_VIRTUAL_BASE;
efi_memory_desc_t *out = runtime_map;
int l;
for (l = 0; l < map_size; l += desc_size) {
efi_memory_desc_t *in = (void *)memory_map + l;
u64 paddr, size;
if (!(in->attribute & EFI_MEMORY_RUNTIME))
continue;
/*
* Make the mapping compatible with 64k pages: this allows
* a 4k page size kernel to kexec a 64k page size kernel and
* vice versa.
*/
paddr = round_down(in->phys_addr, SZ_64K);
size = round_up(in->num_pages * EFI_PAGE_SIZE +
in->phys_addr - paddr, SZ_64K);
/*
* Avoid wasting memory on PTEs by choosing a virtual base that
* is compatible with section mappings if this region has the
* appropriate size and physical alignment. (Sections are 2 MB
* on 4k granule kernels)
*/
if (IS_ALIGNED(in->phys_addr, SZ_2M) && size >= SZ_2M)
efi_virt_base = round_up(efi_virt_base, SZ_2M);
in->virt_addr = efi_virt_base + in->phys_addr - paddr;
efi_virt_base += size;
memcpy(out, in, desc_size);
out = (void *)out + desc_size;
++*count;
}
}
......@@ -32,6 +32,15 @@
static unsigned long __chunk_size = EFI_READ_CHUNK_SIZE;
/*
* Allow the platform to override the allocation granularity: this allows
* systems that have the capability to run with a larger page size to deal
* with the allocations for initrd and fdt more efficiently.
*/
#ifndef EFI_ALLOC_ALIGN
#define EFI_ALLOC_ALIGN EFI_PAGE_SIZE
#endif
struct file_info {
efi_file_handle_t *handle;
u64 size;
......@@ -150,10 +159,10 @@ efi_status_t efi_high_alloc(efi_system_table_t *sys_table_arg,
* a specific address. We are doing page-based allocations,
* so we must be aligned to a page.
*/
if (align < EFI_PAGE_SIZE)
align = EFI_PAGE_SIZE;
if (align < EFI_ALLOC_ALIGN)
align = EFI_ALLOC_ALIGN;
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
again:
for (i = 0; i < map_size / desc_size; i++) {
efi_memory_desc_t *desc;
......@@ -235,10 +244,10 @@ efi_status_t efi_low_alloc(efi_system_table_t *sys_table_arg,
* a specific address. We are doing page-based allocations,
* so we must be aligned to a page.
*/
if (align < EFI_PAGE_SIZE)
align = EFI_PAGE_SIZE;
if (align < EFI_ALLOC_ALIGN)
align = EFI_ALLOC_ALIGN;
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
for (i = 0; i < map_size / desc_size; i++) {
efi_memory_desc_t *desc;
unsigned long m = (unsigned long)map;
......@@ -292,7 +301,7 @@ void efi_free(efi_system_table_t *sys_table_arg, unsigned long size,
if (!size)
return;
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
efi_call_early(free_pages, addr, nr_pages);
}
......@@ -561,7 +570,7 @@ efi_status_t efi_relocate_kernel(efi_system_table_t *sys_table_arg,
* to the preferred address. If that fails, allocate as low
* as possible while respecting the required alignment.
*/
nr_pages = round_up(alloc_size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
nr_pages = round_up(alloc_size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
status = efi_call_early(allocate_pages,
EFI_ALLOCATE_ADDRESS, EFI_LOADER_DATA,
nr_pages, &efi_addr);
......
......@@ -39,4 +39,8 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
void *get_fdt(efi_system_table_t *sys_table);
void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
unsigned long desc_size, efi_memory_desc_t *runtime_map,
int *count);
#endif
......@@ -14,6 +14,8 @@
#include <linux/libfdt.h>
#include <asm/efi.h>
#include "efistub.h"
efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt,
unsigned long orig_fdt_size,
void *fdt, int new_fdt_size, char *cmdline_ptr,
......@@ -193,9 +195,26 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
unsigned long map_size, desc_size;
u32 desc_ver;
unsigned long mmap_key;
efi_memory_desc_t *memory_map;
efi_memory_desc_t *memory_map, *runtime_map;
unsigned long new_fdt_size;
efi_status_t status;
int runtime_entry_count = 0;
/*
* Get a copy of the current memory map that we will use to prepare
* the input for SetVirtualAddressMap(). We don't have to worry about
* subsequent allocations adding entries, since they could not affect
* the number of EFI_MEMORY_RUNTIME regions.
*/
status = efi_get_memory_map(sys_table, &runtime_map, &map_size,
&desc_size, &desc_ver, &mmap_key);
if (status != EFI_SUCCESS) {
pr_efi_err(sys_table, "Unable to retrieve UEFI memory map.\n");
return status;
}
pr_efi(sys_table,
"Exiting boot services and installing virtual address map...\n");
/*
* Estimate size of new FDT, and allocate memory for it. We
......@@ -248,12 +267,48 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
}
}
/*
* Update the memory map with virtual addresses. The function will also
* populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME
* entries so that we can pass it straight into SetVirtualAddressMap()
*/
efi_get_virtmap(memory_map, map_size, desc_size, runtime_map,
&runtime_entry_count);
/* Now we are ready to exit_boot_services.*/
status = sys_table->boottime->exit_boot_services(handle, mmap_key);
if (status == EFI_SUCCESS) {
efi_set_virtual_address_map_t *svam;
if (status == EFI_SUCCESS)
return status;
/* Install the new virtual address map */
svam = sys_table->runtime->set_virtual_address_map;
status = svam(runtime_entry_count * desc_size, desc_size,
desc_ver, runtime_map);
/*
* We are beyond the point of no return here, so if the call to
* SetVirtualAddressMap() failed, we need to signal that to the
* incoming kernel but proceed normally otherwise.
*/
if (status != EFI_SUCCESS) {
int l;
/*
* Set the virtual address field of all
* EFI_MEMORY_RUNTIME entries to 0. This will signal
* the incoming kernel that no virtual translation has
* been installed.
*/
for (l = 0; l < map_size; l += desc_size) {
efi_memory_desc_t *p = (void *)memory_map + l;
if (p->attribute & EFI_MEMORY_RUNTIME)
p->virt_addr = 0;
}
}
return EFI_SUCCESS;
}
pr_efi_err(sys_table, "Exit boot services failed.\n");
......@@ -264,6 +319,7 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
efi_free(sys_table, new_fdt_size, *new_fdt_addr);
fail:
sys_table->boottime->free_pool(runtime_map);
return EFI_LOAD_ERROR;
}
......
......@@ -875,6 +875,8 @@ static inline efi_status_t efi_query_variable_store(u32 attributes, unsigned lon
#endif
extern void __iomem *efi_lookup_mapped_addr(u64 phys_addr);
extern int efi_config_init(efi_config_table_type_t *arch_tables);
extern int efi_config_parse_tables(void *config_tables, int count, int sz,
efi_config_table_type_t *arch_tables);
extern u64 efi_get_iobase (void);
extern u32 efi_mem_type (unsigned long phys_addr);
extern u64 efi_mem_attributes (unsigned long phys_addr);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment