Commit f1c1174f authored by Heiko Carstens's avatar Heiko Carstens Committed by Martin Schwidefsky

s390/mm: use new mm defines instead of magic values

Reviewed-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
Signed-off-by: default avatarHeiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent c67da7c7
...@@ -191,7 +191,7 @@ struct arch_elf_state { ...@@ -191,7 +191,7 @@ struct arch_elf_state {
} while (0) } while (0)
#define CORE_DUMP_USE_REGSET #define CORE_DUMP_USE_REGSET
#define ELF_EXEC_PAGESIZE 4096 #define ELF_EXEC_PAGESIZE PAGE_SIZE
/* /*
* This is the base location for PIE (ET_DYN with INTERP) loads. On * This is the base location for PIE (ET_DYN with INTERP) loads. On
......
...@@ -81,7 +81,7 @@ struct ipl_parameter_block { ...@@ -81,7 +81,7 @@ struct ipl_parameter_block {
struct ipl_block_fcp fcp; struct ipl_block_fcp fcp;
struct ipl_block_ccw ccw; struct ipl_block_ccw ccw;
} ipl_info; } ipl_info;
} __attribute__((packed,aligned(4096))); } __packed __aligned(PAGE_SIZE);
/* /*
* IPL validity flags * IPL validity flags
......
...@@ -33,7 +33,7 @@ static inline int init_new_context(struct task_struct *tsk, ...@@ -33,7 +33,7 @@ static inline int init_new_context(struct task_struct *tsk,
mm->context.use_cmma = 0; mm->context.use_cmma = 0;
#endif #endif
switch (mm->context.asce_limit) { switch (mm->context.asce_limit) {
case 1UL << 42: case _REGION2_SIZE:
/* /*
* forked 3-level task, fall through to set new asce with new * forked 3-level task, fall through to set new asce with new
* mm->pgd * mm->pgd
...@@ -44,12 +44,12 @@ static inline int init_new_context(struct task_struct *tsk, ...@@ -44,12 +44,12 @@ static inline int init_new_context(struct task_struct *tsk,
mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
_ASCE_USER_BITS | _ASCE_TYPE_REGION3; _ASCE_USER_BITS | _ASCE_TYPE_REGION3;
break; break;
case 1UL << 53: case _REGION1_SIZE:
/* forked 4-level task, set new asce with new mm->pgd */ /* forked 4-level task, set new asce with new mm->pgd */
mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
_ASCE_USER_BITS | _ASCE_TYPE_REGION2; _ASCE_USER_BITS | _ASCE_TYPE_REGION2;
break; break;
case 1UL << 31: case _REGION3_SIZE:
/* forked 2-level compat task, set new asce with new mm->pgd */ /* forked 2-level compat task, set new asce with new mm->pgd */
mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
_ASCE_USER_BITS | _ASCE_TYPE_SEGMENT; _ASCE_USER_BITS | _ASCE_TYPE_SEGMENT;
......
...@@ -44,16 +44,16 @@ static inline void clear_table(unsigned long *s, unsigned long val, size_t n) ...@@ -44,16 +44,16 @@ static inline void clear_table(unsigned long *s, unsigned long val, size_t n)
static inline void crst_table_init(unsigned long *crst, unsigned long entry) static inline void crst_table_init(unsigned long *crst, unsigned long entry)
{ {
clear_table(crst, entry, sizeof(unsigned long)*2048); clear_table(crst, entry, _CRST_TABLE_SIZE);
} }
static inline unsigned long pgd_entry_type(struct mm_struct *mm) static inline unsigned long pgd_entry_type(struct mm_struct *mm)
{ {
if (mm->context.asce_limit <= (1UL << 31)) if (mm->context.asce_limit <= _REGION3_SIZE)
return _SEGMENT_ENTRY_EMPTY; return _SEGMENT_ENTRY_EMPTY;
if (mm->context.asce_limit <= (1UL << 42)) if (mm->context.asce_limit <= _REGION2_SIZE)
return _REGION3_ENTRY_EMPTY; return _REGION3_ENTRY_EMPTY;
if (mm->context.asce_limit <= (1UL << 53)) if (mm->context.asce_limit <= _REGION1_SIZE)
return _REGION2_ENTRY_EMPTY; return _REGION2_ENTRY_EMPTY;
return _REGION1_ENTRY_EMPTY; return _REGION1_ENTRY_EMPTY;
} }
...@@ -121,7 +121,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) ...@@ -121,7 +121,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
if (!table) if (!table)
return NULL; return NULL;
if (mm->context.asce_limit == (1UL << 31)) { if (mm->context.asce_limit == _REGION3_SIZE) {
/* Forking a compat process with 2 page table levels */ /* Forking a compat process with 2 page table levels */
if (!pgtable_pmd_page_ctor(virt_to_page(table))) { if (!pgtable_pmd_page_ctor(virt_to_page(table))) {
crst_table_free(mm, table); crst_table_free(mm, table);
...@@ -133,7 +133,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) ...@@ -133,7 +133,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{ {
if (mm->context.asce_limit == (1UL << 31)) if (mm->context.asce_limit == _REGION3_SIZE)
pgtable_pmd_page_dtor(virt_to_page(pgd)); pgtable_pmd_page_dtor(virt_to_page(pgd));
crst_table_free(mm, (unsigned long *) pgd); crst_table_free(mm, (unsigned long *) pgd);
} }
......
...@@ -80,7 +80,7 @@ struct qdr { ...@@ -80,7 +80,7 @@ struct qdr {
u32 qkey : 4; u32 qkey : 4;
u32 : 28; u32 : 28;
struct qdesfmt0 qdf0[126]; struct qdesfmt0 qdf0[126];
} __attribute__ ((packed, aligned(4096))); } __packed __aligned(PAGE_SIZE);
#define QIB_AC_OUTBOUND_PCI_SUPPORTED 0x40 #define QIB_AC_OUTBOUND_PCI_SUPPORTED 0x40
#define QIB_RFLAGS_ENABLE_QEBSM 0x80 #define QIB_RFLAGS_ENABLE_QEBSM 0x80
......
...@@ -130,7 +130,7 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, ...@@ -130,7 +130,7 @@ static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
unsigned long address) unsigned long address)
{ {
if (tlb->mm->context.asce_limit <= (1UL << 31)) if (tlb->mm->context.asce_limit <= _REGION3_SIZE)
return; return;
pgtable_pmd_page_dtor(virt_to_page(pmd)); pgtable_pmd_page_dtor(virt_to_page(pmd));
tlb_remove_table(tlb, pmd); tlb_remove_table(tlb, pmd);
...@@ -146,7 +146,7 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, ...@@ -146,7 +146,7 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
static inline void p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, static inline void p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
unsigned long address) unsigned long address)
{ {
if (tlb->mm->context.asce_limit <= (1UL << 53)) if (tlb->mm->context.asce_limit <= _REGION1_SIZE)
return; return;
tlb_remove_table(tlb, p4d); tlb_remove_table(tlb, p4d);
} }
...@@ -161,7 +161,7 @@ static inline void p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, ...@@ -161,7 +161,7 @@ static inline void p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d,
static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
unsigned long address) unsigned long address)
{ {
if (tlb->mm->context.asce_limit <= (1UL << 42)) if (tlb->mm->context.asce_limit <= _REGION2_SIZE)
return; return;
tlb_remove_table(tlb, pud); tlb_remove_table(tlb, pud);
} }
......
...@@ -76,7 +76,7 @@ void dump_trace(dump_trace_func_t func, void *data, struct task_struct *task, ...@@ -76,7 +76,7 @@ void dump_trace(dump_trace_func_t func, void *data, struct task_struct *task,
frame_size = STACK_FRAME_OVERHEAD + sizeof(struct pt_regs); frame_size = STACK_FRAME_OVERHEAD + sizeof(struct pt_regs);
#ifdef CONFIG_CHECK_STACK #ifdef CONFIG_CHECK_STACK
sp = __dump_trace(func, data, sp, sp = __dump_trace(func, data, sp,
S390_lowcore.panic_stack + frame_size - 4096, S390_lowcore.panic_stack + frame_size - PAGE_SIZE,
S390_lowcore.panic_stack + frame_size); S390_lowcore.panic_stack + frame_size);
#endif #endif
sp = __dump_trace(func, data, sp, sp = __dump_trace(func, data, sp,
......
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
*/ */
#include <linux/linkage.h> #include <linux/linkage.h>
#include <asm/page.h>
#include <asm/sigp.h> #include <asm/sigp.h>
/* /*
...@@ -55,8 +56,8 @@ ENTRY(relocate_kernel) ...@@ -55,8 +56,8 @@ ENTRY(relocate_kernel)
.back_pgm: .back_pgm:
lmg %r0,%r15,gprregs-.base(%r13) lmg %r0,%r15,gprregs-.base(%r13)
.top: .top:
lghi %r7,4096 # load PAGE_SIZE in r7 lghi %r7,PAGE_SIZE # load PAGE_SIZE in r7
lghi %r9,4096 # load PAGE_SIZE in r9 lghi %r9,PAGE_SIZE # load PAGE_SIZE in r9
lg %r5,0(%r2) # read another word for indirection page lg %r5,0(%r2) # read another word for indirection page
aghi %r2,8 # increment pointer aghi %r2,8 # increment pointer
tml %r5,0x1 # is it a destination page? tml %r5,0x1 # is it a destination page?
......
...@@ -305,7 +305,7 @@ static void __init setup_lowcore(void) ...@@ -305,7 +305,7 @@ static void __init setup_lowcore(void)
/* /*
* Setup lowcore for boot cpu * Setup lowcore for boot cpu
*/ */
BUILD_BUG_ON(sizeof(struct lowcore) != LC_PAGES * 4096); BUILD_BUG_ON(sizeof(struct lowcore) != LC_PAGES * PAGE_SIZE);
lc = memblock_virt_alloc_low(sizeof(*lc), sizeof(*lc)); lc = memblock_virt_alloc_low(sizeof(*lc), sizeof(*lc));
lc->restart_psw.mask = PSW_KERNEL_BITS; lc->restart_psw.mask = PSW_KERNEL_BITS;
lc->restart_psw.addr = (unsigned long) restart_int_handler; lc->restart_psw.addr = (unsigned long) restart_int_handler;
...@@ -469,10 +469,10 @@ static void __init setup_memory_end(void) ...@@ -469,10 +469,10 @@ static void __init setup_memory_end(void)
vmalloc_size = VMALLOC_END ?: (128UL << 30) - MODULES_LEN; vmalloc_size = VMALLOC_END ?: (128UL << 30) - MODULES_LEN;
tmp = (memory_end ?: max_physmem_end) / PAGE_SIZE; tmp = (memory_end ?: max_physmem_end) / PAGE_SIZE;
tmp = tmp * (sizeof(struct page) + PAGE_SIZE); tmp = tmp * (sizeof(struct page) + PAGE_SIZE);
if (tmp + vmalloc_size + MODULES_LEN <= (1UL << 42)) if (tmp + vmalloc_size + MODULES_LEN <= _REGION2_SIZE)
vmax = 1UL << 42; /* 3-level kernel page table */ vmax = _REGION2_SIZE; /* 3-level kernel page table */
else else
vmax = 1UL << 53; /* 4-level kernel page table */ vmax = _REGION1_SIZE; /* 4-level kernel page table */
/* module area is at the end of the kernel address space. */ /* module area is at the end of the kernel address space. */
MODULES_END = vmax; MODULES_END = vmax;
MODULES_VADDR = MODULES_END - MODULES_LEN; MODULES_VADDR = MODULES_END - MODULES_LEN;
......
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
* This is the infamous ld script for the 32 bits vdso * This is the infamous ld script for the 32 bits vdso
* library * library
*/ */
#include <asm/page.h>
#include <asm/vdso.h> #include <asm/vdso.h>
OUTPUT_FORMAT("elf32-s390", "elf32-s390", "elf32-s390") OUTPUT_FORMAT("elf32-s390", "elf32-s390", "elf32-s390")
...@@ -91,7 +93,7 @@ SECTIONS ...@@ -91,7 +93,7 @@ SECTIONS
.debug_ranges 0 : { *(.debug_ranges) } .debug_ranges 0 : { *(.debug_ranges) }
.gnu.attributes 0 : { KEEP (*(.gnu.attributes)) } .gnu.attributes 0 : { KEEP (*(.gnu.attributes)) }
. = ALIGN(4096); . = ALIGN(PAGE_SIZE);
PROVIDE(_vdso_data = .); PROVIDE(_vdso_data = .);
/DISCARD/ : { /DISCARD/ : {
......
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
* This is the infamous ld script for the 64 bits vdso * This is the infamous ld script for the 64 bits vdso
* library * library
*/ */
#include <asm/page.h>
#include <asm/vdso.h> #include <asm/vdso.h>
OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390") OUTPUT_FORMAT("elf64-s390", "elf64-s390", "elf64-s390")
...@@ -91,7 +93,7 @@ SECTIONS ...@@ -91,7 +93,7 @@ SECTIONS
.debug_ranges 0 : { *(.debug_ranges) } .debug_ranges 0 : { *(.debug_ranges) }
.gnu.attributes 0 : { KEEP (*(.gnu.attributes)) } .gnu.attributes 0 : { KEEP (*(.gnu.attributes)) }
. = ALIGN(4096); . = ALIGN(PAGE_SIZE);
PROVIDE(_vdso_data = .); PROVIDE(_vdso_data = .);
/DISCARD/ : { /DISCARD/ : {
......
...@@ -135,7 +135,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address) ...@@ -135,7 +135,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address)
pr_alert("AS:%016lx ", asce); pr_alert("AS:%016lx ", asce);
switch (asce & _ASCE_TYPE_MASK) { switch (asce & _ASCE_TYPE_MASK) {
case _ASCE_TYPE_REGION1: case _ASCE_TYPE_REGION1:
table = table + ((address >> 53) & 0x7ff); table += (address & _REGION1_INDEX) >> _REGION1_SHIFT;
if (bad_address(table)) if (bad_address(table))
goto bad; goto bad;
pr_cont("R1:%016lx ", *table); pr_cont("R1:%016lx ", *table);
...@@ -144,7 +144,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address) ...@@ -144,7 +144,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address)
table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
/* fallthrough */ /* fallthrough */
case _ASCE_TYPE_REGION2: case _ASCE_TYPE_REGION2:
table = table + ((address >> 42) & 0x7ff); table += (address & _REGION2_INDEX) >> _REGION2_SHIFT;
if (bad_address(table)) if (bad_address(table))
goto bad; goto bad;
pr_cont("R2:%016lx ", *table); pr_cont("R2:%016lx ", *table);
...@@ -153,7 +153,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address) ...@@ -153,7 +153,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address)
table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
/* fallthrough */ /* fallthrough */
case _ASCE_TYPE_REGION3: case _ASCE_TYPE_REGION3:
table = table + ((address >> 31) & 0x7ff); table += (address & _REGION3_INDEX) >> _REGION3_SHIFT;
if (bad_address(table)) if (bad_address(table))
goto bad; goto bad;
pr_cont("R3:%016lx ", *table); pr_cont("R3:%016lx ", *table);
...@@ -162,7 +162,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address) ...@@ -162,7 +162,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address)
table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
/* fallthrough */ /* fallthrough */
case _ASCE_TYPE_SEGMENT: case _ASCE_TYPE_SEGMENT:
table = table + ((address >> 20) & 0x7ff); table += (address & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
if (bad_address(table)) if (bad_address(table))
goto bad; goto bad;
pr_cont("S:%016lx ", *table); pr_cont("S:%016lx ", *table);
...@@ -170,7 +170,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address) ...@@ -170,7 +170,7 @@ static void dump_pagetable(unsigned long asce, unsigned long address)
goto out; goto out;
table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN); table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN);
} }
table = table + ((address >> 12) & 0xff); table += (address & _PAGE_INDEX) >> _PAGE_SHIFT;
if (bad_address(table)) if (bad_address(table))
goto bad; goto bad;
pr_cont("P:%016lx ", *table); pr_cont("P:%016lx ", *table);
......
...@@ -36,16 +36,16 @@ static struct gmap *gmap_alloc(unsigned long limit) ...@@ -36,16 +36,16 @@ static struct gmap *gmap_alloc(unsigned long limit)
unsigned long *table; unsigned long *table;
unsigned long etype, atype; unsigned long etype, atype;
if (limit < (1UL << 31)) { if (limit < _REGION3_SIZE) {
limit = (1UL << 31) - 1; limit = _REGION3_SIZE - 1;
atype = _ASCE_TYPE_SEGMENT; atype = _ASCE_TYPE_SEGMENT;
etype = _SEGMENT_ENTRY_EMPTY; etype = _SEGMENT_ENTRY_EMPTY;
} else if (limit < (1UL << 42)) { } else if (limit < _REGION2_SIZE) {
limit = (1UL << 42) - 1; limit = _REGION2_SIZE - 1;
atype = _ASCE_TYPE_REGION3; atype = _ASCE_TYPE_REGION3;
etype = _REGION3_ENTRY_EMPTY; etype = _REGION3_ENTRY_EMPTY;
} else if (limit < (1UL << 53)) { } else if (limit < _REGION1_SIZE) {
limit = (1UL << 53) - 1; limit = _REGION1_SIZE - 1;
atype = _ASCE_TYPE_REGION2; atype = _ASCE_TYPE_REGION2;
etype = _REGION2_ENTRY_EMPTY; etype = _REGION2_ENTRY_EMPTY;
} else { } else {
...@@ -65,7 +65,7 @@ static struct gmap *gmap_alloc(unsigned long limit) ...@@ -65,7 +65,7 @@ static struct gmap *gmap_alloc(unsigned long limit)
spin_lock_init(&gmap->guest_table_lock); spin_lock_init(&gmap->guest_table_lock);
spin_lock_init(&gmap->shadow_lock); spin_lock_init(&gmap->shadow_lock);
atomic_set(&gmap->ref_count, 1); atomic_set(&gmap->ref_count, 1);
page = alloc_pages(GFP_KERNEL, 2); page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
if (!page) if (!page)
goto out_free; goto out_free;
page->index = 0; page->index = 0;
...@@ -186,7 +186,7 @@ static void gmap_free(struct gmap *gmap) ...@@ -186,7 +186,7 @@ static void gmap_free(struct gmap *gmap)
gmap_flush_tlb(gmap); gmap_flush_tlb(gmap);
/* Free all segment & region tables. */ /* Free all segment & region tables. */
list_for_each_entry_safe(page, next, &gmap->crst_list, lru) list_for_each_entry_safe(page, next, &gmap->crst_list, lru)
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
gmap_radix_tree_free(&gmap->guest_to_host); gmap_radix_tree_free(&gmap->guest_to_host);
gmap_radix_tree_free(&gmap->host_to_guest); gmap_radix_tree_free(&gmap->host_to_guest);
...@@ -306,7 +306,7 @@ static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, ...@@ -306,7 +306,7 @@ static int gmap_alloc_table(struct gmap *gmap, unsigned long *table,
unsigned long *new; unsigned long *new;
/* since we dont free the gmap table until gmap_free we can unlock */ /* since we dont free the gmap table until gmap_free we can unlock */
page = alloc_pages(GFP_KERNEL, 2); page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
if (!page) if (!page)
return -ENOMEM; return -ENOMEM;
new = (unsigned long *) page_to_phys(page); new = (unsigned long *) page_to_phys(page);
...@@ -321,7 +321,7 @@ static int gmap_alloc_table(struct gmap *gmap, unsigned long *table, ...@@ -321,7 +321,7 @@ static int gmap_alloc_table(struct gmap *gmap, unsigned long *table,
} }
spin_unlock(&gmap->guest_table_lock); spin_unlock(&gmap->guest_table_lock);
if (page) if (page)
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
return 0; return 0;
} }
...@@ -546,30 +546,30 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) ...@@ -546,30 +546,30 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr)
/* Create higher level tables in the gmap page table */ /* Create higher level tables in the gmap page table */
table = gmap->table; table = gmap->table;
if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) { if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) {
table += (gaddr >> 53) & 0x7ff; table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;
if ((*table & _REGION_ENTRY_INVALID) && if ((*table & _REGION_ENTRY_INVALID) &&
gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY, gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY,
gaddr & 0xffe0000000000000UL)) gaddr & _REGION1_MASK))
return -ENOMEM; return -ENOMEM;
table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
} }
if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) { if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) {
table += (gaddr >> 42) & 0x7ff; table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;
if ((*table & _REGION_ENTRY_INVALID) && if ((*table & _REGION_ENTRY_INVALID) &&
gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY, gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY,
gaddr & 0xfffffc0000000000UL)) gaddr & _REGION2_MASK))
return -ENOMEM; return -ENOMEM;
table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
} }
if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) { if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) {
table += (gaddr >> 31) & 0x7ff; table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;
if ((*table & _REGION_ENTRY_INVALID) && if ((*table & _REGION_ENTRY_INVALID) &&
gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY, gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY,
gaddr & 0xffffffff80000000UL)) gaddr & _REGION3_MASK))
return -ENOMEM; return -ENOMEM;
table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
} }
table += (gaddr >> 20) & 0x7ff; table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
/* Walk the parent mm page table */ /* Walk the parent mm page table */
mm = gmap->mm; mm = gmap->mm;
pgd = pgd_offset(mm, vmaddr); pgd = pgd_offset(mm, vmaddr);
...@@ -771,7 +771,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap, ...@@ -771,7 +771,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap,
table = gmap->table; table = gmap->table;
switch (gmap->asce & _ASCE_TYPE_MASK) { switch (gmap->asce & _ASCE_TYPE_MASK) {
case _ASCE_TYPE_REGION1: case _ASCE_TYPE_REGION1:
table += (gaddr >> 53) & 0x7ff; table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;
if (level == 4) if (level == 4)
break; break;
if (*table & _REGION_ENTRY_INVALID) if (*table & _REGION_ENTRY_INVALID)
...@@ -779,7 +779,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap, ...@@ -779,7 +779,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap,
table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
/* Fallthrough */ /* Fallthrough */
case _ASCE_TYPE_REGION2: case _ASCE_TYPE_REGION2:
table += (gaddr >> 42) & 0x7ff; table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;
if (level == 3) if (level == 3)
break; break;
if (*table & _REGION_ENTRY_INVALID) if (*table & _REGION_ENTRY_INVALID)
...@@ -787,7 +787,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap, ...@@ -787,7 +787,7 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap,
table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
/* Fallthrough */ /* Fallthrough */
case _ASCE_TYPE_REGION3: case _ASCE_TYPE_REGION3:
table += (gaddr >> 31) & 0x7ff; table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;
if (level == 2) if (level == 2)
break; break;
if (*table & _REGION_ENTRY_INVALID) if (*table & _REGION_ENTRY_INVALID)
...@@ -795,13 +795,13 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap, ...@@ -795,13 +795,13 @@ static inline unsigned long *gmap_table_walk(struct gmap *gmap,
table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN); table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
/* Fallthrough */ /* Fallthrough */
case _ASCE_TYPE_SEGMENT: case _ASCE_TYPE_SEGMENT:
table += (gaddr >> 20) & 0x7ff; table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
if (level == 1) if (level == 1)
break; break;
if (*table & _REGION_ENTRY_INVALID) if (*table & _REGION_ENTRY_INVALID)
return NULL; return NULL;
table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN); table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN);
table += (gaddr >> 12) & 0xff; table += (gaddr & _PAGE_INDEX) >> _PAGE_SHIFT;
} }
return table; return table;
} }
...@@ -1126,7 +1126,7 @@ static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr) ...@@ -1126,7 +1126,7 @@ static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr)
table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */ table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */
if (!table || *table & _PAGE_INVALID) if (!table || *table & _PAGE_INVALID)
return; return;
gmap_call_notifier(sg, raddr, raddr + (1UL << 12) - 1); gmap_call_notifier(sg, raddr, raddr + _PAGE_SIZE - 1);
ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table); ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table);
} }
...@@ -1144,7 +1144,7 @@ static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr, ...@@ -1144,7 +1144,7 @@ static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr,
int i; int i;
BUG_ON(!gmap_is_shadow(sg)); BUG_ON(!gmap_is_shadow(sg));
for (i = 0; i < 256; i++, raddr += 1UL << 12) for (i = 0; i < _PAGE_ENTRIES; i++, raddr += _PAGE_SIZE)
pgt[i] = _PAGE_INVALID; pgt[i] = _PAGE_INVALID;
} }
...@@ -1164,8 +1164,8 @@ static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr) ...@@ -1164,8 +1164,8 @@ static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr)
ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */ ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */
if (!ste || !(*ste & _SEGMENT_ENTRY_ORIGIN)) if (!ste || !(*ste & _SEGMENT_ENTRY_ORIGIN))
return; return;
gmap_call_notifier(sg, raddr, raddr + (1UL << 20) - 1); gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1);
sto = (unsigned long) (ste - ((raddr >> 20) & 0x7ff)); sto = (unsigned long) (ste - ((raddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT));
gmap_idte_one(sto | _ASCE_TYPE_SEGMENT, raddr); gmap_idte_one(sto | _ASCE_TYPE_SEGMENT, raddr);
pgt = (unsigned long *)(*ste & _SEGMENT_ENTRY_ORIGIN); pgt = (unsigned long *)(*ste & _SEGMENT_ENTRY_ORIGIN);
*ste = _SEGMENT_ENTRY_EMPTY; *ste = _SEGMENT_ENTRY_EMPTY;
...@@ -1193,7 +1193,7 @@ static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr, ...@@ -1193,7 +1193,7 @@ static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr,
BUG_ON(!gmap_is_shadow(sg)); BUG_ON(!gmap_is_shadow(sg));
asce = (unsigned long) sgt | _ASCE_TYPE_SEGMENT; asce = (unsigned long) sgt | _ASCE_TYPE_SEGMENT;
for (i = 0; i < 2048; i++, raddr += 1UL << 20) { for (i = 0; i < _CRST_ENTRIES; i++, raddr += _SEGMENT_SIZE) {
if (!(sgt[i] & _SEGMENT_ENTRY_ORIGIN)) if (!(sgt[i] & _SEGMENT_ENTRY_ORIGIN))
continue; continue;
pgt = (unsigned long *)(sgt[i] & _REGION_ENTRY_ORIGIN); pgt = (unsigned long *)(sgt[i] & _REGION_ENTRY_ORIGIN);
...@@ -1222,8 +1222,8 @@ static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr) ...@@ -1222,8 +1222,8 @@ static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr)
r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */ r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */
if (!r3e || !(*r3e & _REGION_ENTRY_ORIGIN)) if (!r3e || !(*r3e & _REGION_ENTRY_ORIGIN))
return; return;
gmap_call_notifier(sg, raddr, raddr + (1UL << 31) - 1); gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1);
r3o = (unsigned long) (r3e - ((raddr >> 31) & 0x7ff)); r3o = (unsigned long) (r3e - ((raddr & _REGION3_INDEX) >> _REGION3_SHIFT));
gmap_idte_one(r3o | _ASCE_TYPE_REGION3, raddr); gmap_idte_one(r3o | _ASCE_TYPE_REGION3, raddr);
sgt = (unsigned long *)(*r3e & _REGION_ENTRY_ORIGIN); sgt = (unsigned long *)(*r3e & _REGION_ENTRY_ORIGIN);
*r3e = _REGION3_ENTRY_EMPTY; *r3e = _REGION3_ENTRY_EMPTY;
...@@ -1231,7 +1231,7 @@ static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr) ...@@ -1231,7 +1231,7 @@ static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr)
/* Free segment table */ /* Free segment table */
page = pfn_to_page(__pa(sgt) >> PAGE_SHIFT); page = pfn_to_page(__pa(sgt) >> PAGE_SHIFT);
list_del(&page->lru); list_del(&page->lru);
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
} }
/** /**
...@@ -1251,7 +1251,7 @@ static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr, ...@@ -1251,7 +1251,7 @@ static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr,
BUG_ON(!gmap_is_shadow(sg)); BUG_ON(!gmap_is_shadow(sg));
asce = (unsigned long) r3t | _ASCE_TYPE_REGION3; asce = (unsigned long) r3t | _ASCE_TYPE_REGION3;
for (i = 0; i < 2048; i++, raddr += 1UL << 31) { for (i = 0; i < _CRST_ENTRIES; i++, raddr += _REGION3_SIZE) {
if (!(r3t[i] & _REGION_ENTRY_ORIGIN)) if (!(r3t[i] & _REGION_ENTRY_ORIGIN))
continue; continue;
sgt = (unsigned long *)(r3t[i] & _REGION_ENTRY_ORIGIN); sgt = (unsigned long *)(r3t[i] & _REGION_ENTRY_ORIGIN);
...@@ -1260,7 +1260,7 @@ static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr, ...@@ -1260,7 +1260,7 @@ static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr,
/* Free segment table */ /* Free segment table */
page = pfn_to_page(__pa(sgt) >> PAGE_SHIFT); page = pfn_to_page(__pa(sgt) >> PAGE_SHIFT);
list_del(&page->lru); list_del(&page->lru);
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
} }
} }
...@@ -1280,8 +1280,8 @@ static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr) ...@@ -1280,8 +1280,8 @@ static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr)
r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */ r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */
if (!r2e || !(*r2e & _REGION_ENTRY_ORIGIN)) if (!r2e || !(*r2e & _REGION_ENTRY_ORIGIN))
return; return;
gmap_call_notifier(sg, raddr, raddr + (1UL << 42) - 1); gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1);
r2o = (unsigned long) (r2e - ((raddr >> 42) & 0x7ff)); r2o = (unsigned long) (r2e - ((raddr & _REGION2_INDEX) >> _REGION2_SHIFT));
gmap_idte_one(r2o | _ASCE_TYPE_REGION2, raddr); gmap_idte_one(r2o | _ASCE_TYPE_REGION2, raddr);
r3t = (unsigned long *)(*r2e & _REGION_ENTRY_ORIGIN); r3t = (unsigned long *)(*r2e & _REGION_ENTRY_ORIGIN);
*r2e = _REGION2_ENTRY_EMPTY; *r2e = _REGION2_ENTRY_EMPTY;
...@@ -1289,7 +1289,7 @@ static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr) ...@@ -1289,7 +1289,7 @@ static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr)
/* Free region 3 table */ /* Free region 3 table */
page = pfn_to_page(__pa(r3t) >> PAGE_SHIFT); page = pfn_to_page(__pa(r3t) >> PAGE_SHIFT);
list_del(&page->lru); list_del(&page->lru);
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
} }
/** /**
...@@ -1309,7 +1309,7 @@ static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr, ...@@ -1309,7 +1309,7 @@ static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr,
BUG_ON(!gmap_is_shadow(sg)); BUG_ON(!gmap_is_shadow(sg));
asce = (unsigned long) r2t | _ASCE_TYPE_REGION2; asce = (unsigned long) r2t | _ASCE_TYPE_REGION2;
for (i = 0; i < 2048; i++, raddr += 1UL << 42) { for (i = 0; i < _CRST_ENTRIES; i++, raddr += _REGION2_SIZE) {
if (!(r2t[i] & _REGION_ENTRY_ORIGIN)) if (!(r2t[i] & _REGION_ENTRY_ORIGIN))
continue; continue;
r3t = (unsigned long *)(r2t[i] & _REGION_ENTRY_ORIGIN); r3t = (unsigned long *)(r2t[i] & _REGION_ENTRY_ORIGIN);
...@@ -1318,7 +1318,7 @@ static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr, ...@@ -1318,7 +1318,7 @@ static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr,
/* Free region 3 table */ /* Free region 3 table */
page = pfn_to_page(__pa(r3t) >> PAGE_SHIFT); page = pfn_to_page(__pa(r3t) >> PAGE_SHIFT);
list_del(&page->lru); list_del(&page->lru);
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
} }
} }
...@@ -1338,8 +1338,8 @@ static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr) ...@@ -1338,8 +1338,8 @@ static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr)
r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */ r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */
if (!r1e || !(*r1e & _REGION_ENTRY_ORIGIN)) if (!r1e || !(*r1e & _REGION_ENTRY_ORIGIN))
return; return;
gmap_call_notifier(sg, raddr, raddr + (1UL << 53) - 1); gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1);
r1o = (unsigned long) (r1e - ((raddr >> 53) & 0x7ff)); r1o = (unsigned long) (r1e - ((raddr & _REGION1_INDEX) >> _REGION1_SHIFT));
gmap_idte_one(r1o | _ASCE_TYPE_REGION1, raddr); gmap_idte_one(r1o | _ASCE_TYPE_REGION1, raddr);
r2t = (unsigned long *)(*r1e & _REGION_ENTRY_ORIGIN); r2t = (unsigned long *)(*r1e & _REGION_ENTRY_ORIGIN);
*r1e = _REGION1_ENTRY_EMPTY; *r1e = _REGION1_ENTRY_EMPTY;
...@@ -1347,7 +1347,7 @@ static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr) ...@@ -1347,7 +1347,7 @@ static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr)
/* Free region 2 table */ /* Free region 2 table */
page = pfn_to_page(__pa(r2t) >> PAGE_SHIFT); page = pfn_to_page(__pa(r2t) >> PAGE_SHIFT);
list_del(&page->lru); list_del(&page->lru);
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
} }
/** /**
...@@ -1367,7 +1367,7 @@ static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr, ...@@ -1367,7 +1367,7 @@ static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr,
BUG_ON(!gmap_is_shadow(sg)); BUG_ON(!gmap_is_shadow(sg));
asce = (unsigned long) r1t | _ASCE_TYPE_REGION1; asce = (unsigned long) r1t | _ASCE_TYPE_REGION1;
for (i = 0; i < 2048; i++, raddr += 1UL << 53) { for (i = 0; i < _CRST_ENTRIES; i++, raddr += _REGION1_SIZE) {
if (!(r1t[i] & _REGION_ENTRY_ORIGIN)) if (!(r1t[i] & _REGION_ENTRY_ORIGIN))
continue; continue;
r2t = (unsigned long *)(r1t[i] & _REGION_ENTRY_ORIGIN); r2t = (unsigned long *)(r1t[i] & _REGION_ENTRY_ORIGIN);
...@@ -1378,7 +1378,7 @@ static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr, ...@@ -1378,7 +1378,7 @@ static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr,
/* Free region 2 table */ /* Free region 2 table */
page = pfn_to_page(__pa(r2t) >> PAGE_SHIFT); page = pfn_to_page(__pa(r2t) >> PAGE_SHIFT);
list_del(&page->lru); list_del(&page->lru);
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
} }
} }
...@@ -1535,7 +1535,7 @@ struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce, ...@@ -1535,7 +1535,7 @@ struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce,
/* protect after insertion, so it will get properly invalidated */ /* protect after insertion, so it will get properly invalidated */
down_read(&parent->mm->mmap_sem); down_read(&parent->mm->mmap_sem);
rc = gmap_protect_range(parent, asce & _ASCE_ORIGIN, rc = gmap_protect_range(parent, asce & _ASCE_ORIGIN,
((asce & _ASCE_TABLE_LENGTH) + 1) * 4096, ((asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE,
PROT_READ, PGSTE_VSIE_BIT); PROT_READ, PGSTE_VSIE_BIT);
up_read(&parent->mm->mmap_sem); up_read(&parent->mm->mmap_sem);
spin_lock(&parent->shadow_lock); spin_lock(&parent->shadow_lock);
...@@ -1578,7 +1578,7 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, ...@@ -1578,7 +1578,7 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
BUG_ON(!gmap_is_shadow(sg)); BUG_ON(!gmap_is_shadow(sg));
/* Allocate a shadow region second table */ /* Allocate a shadow region second table */
page = alloc_pages(GFP_KERNEL, 2); page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
if (!page) if (!page)
return -ENOMEM; return -ENOMEM;
page->index = r2t & _REGION_ENTRY_ORIGIN; page->index = r2t & _REGION_ENTRY_ORIGIN;
...@@ -1614,10 +1614,10 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, ...@@ -1614,10 +1614,10 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
} }
spin_unlock(&sg->guest_table_lock); spin_unlock(&sg->guest_table_lock);
/* Make r2t read-only in parent gmap page table */ /* Make r2t read-only in parent gmap page table */
raddr = (saddr & 0xffe0000000000000UL) | _SHADOW_RMAP_REGION1; raddr = (saddr & _REGION1_MASK) | _SHADOW_RMAP_REGION1;
origin = r2t & _REGION_ENTRY_ORIGIN; origin = r2t & _REGION_ENTRY_ORIGIN;
offset = ((r2t & _REGION_ENTRY_OFFSET) >> 6) * 4096; offset = ((r2t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
len = ((r2t & _REGION_ENTRY_LENGTH) + 1) * 4096 - offset; len = ((r2t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ); rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ);
spin_lock(&sg->guest_table_lock); spin_lock(&sg->guest_table_lock);
if (!rc) { if (!rc) {
...@@ -1634,7 +1634,7 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t, ...@@ -1634,7 +1634,7 @@ int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
return rc; return rc;
out_free: out_free:
spin_unlock(&sg->guest_table_lock); spin_unlock(&sg->guest_table_lock);
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
return rc; return rc;
} }
EXPORT_SYMBOL_GPL(gmap_shadow_r2t); EXPORT_SYMBOL_GPL(gmap_shadow_r2t);
...@@ -1662,7 +1662,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, ...@@ -1662,7 +1662,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
BUG_ON(!gmap_is_shadow(sg)); BUG_ON(!gmap_is_shadow(sg));
/* Allocate a shadow region second table */ /* Allocate a shadow region second table */
page = alloc_pages(GFP_KERNEL, 2); page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
if (!page) if (!page)
return -ENOMEM; return -ENOMEM;
page->index = r3t & _REGION_ENTRY_ORIGIN; page->index = r3t & _REGION_ENTRY_ORIGIN;
...@@ -1697,10 +1697,10 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, ...@@ -1697,10 +1697,10 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
} }
spin_unlock(&sg->guest_table_lock); spin_unlock(&sg->guest_table_lock);
/* Make r3t read-only in parent gmap page table */ /* Make r3t read-only in parent gmap page table */
raddr = (saddr & 0xfffffc0000000000UL) | _SHADOW_RMAP_REGION2; raddr = (saddr & _REGION2_MASK) | _SHADOW_RMAP_REGION2;
origin = r3t & _REGION_ENTRY_ORIGIN; origin = r3t & _REGION_ENTRY_ORIGIN;
offset = ((r3t & _REGION_ENTRY_OFFSET) >> 6) * 4096; offset = ((r3t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
len = ((r3t & _REGION_ENTRY_LENGTH) + 1) * 4096 - offset; len = ((r3t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ); rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ);
spin_lock(&sg->guest_table_lock); spin_lock(&sg->guest_table_lock);
if (!rc) { if (!rc) {
...@@ -1717,7 +1717,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t, ...@@ -1717,7 +1717,7 @@ int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
return rc; return rc;
out_free: out_free:
spin_unlock(&sg->guest_table_lock); spin_unlock(&sg->guest_table_lock);
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
return rc; return rc;
} }
EXPORT_SYMBOL_GPL(gmap_shadow_r3t); EXPORT_SYMBOL_GPL(gmap_shadow_r3t);
...@@ -1745,7 +1745,7 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, ...@@ -1745,7 +1745,7 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE)); BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE));
/* Allocate a shadow segment table */ /* Allocate a shadow segment table */
page = alloc_pages(GFP_KERNEL, 2); page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
if (!page) if (!page)
return -ENOMEM; return -ENOMEM;
page->index = sgt & _REGION_ENTRY_ORIGIN; page->index = sgt & _REGION_ENTRY_ORIGIN;
...@@ -1781,10 +1781,10 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, ...@@ -1781,10 +1781,10 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
} }
spin_unlock(&sg->guest_table_lock); spin_unlock(&sg->guest_table_lock);
/* Make sgt read-only in parent gmap page table */ /* Make sgt read-only in parent gmap page table */
raddr = (saddr & 0xffffffff80000000UL) | _SHADOW_RMAP_REGION3; raddr = (saddr & _REGION3_MASK) | _SHADOW_RMAP_REGION3;
origin = sgt & _REGION_ENTRY_ORIGIN; origin = sgt & _REGION_ENTRY_ORIGIN;
offset = ((sgt & _REGION_ENTRY_OFFSET) >> 6) * 4096; offset = ((sgt & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
len = ((sgt & _REGION_ENTRY_LENGTH) + 1) * 4096 - offset; len = ((sgt & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ); rc = gmap_protect_rmap(sg, raddr, origin + offset, len, PROT_READ);
spin_lock(&sg->guest_table_lock); spin_lock(&sg->guest_table_lock);
if (!rc) { if (!rc) {
...@@ -1801,7 +1801,7 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt, ...@@ -1801,7 +1801,7 @@ int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
return rc; return rc;
out_free: out_free:
spin_unlock(&sg->guest_table_lock); spin_unlock(&sg->guest_table_lock);
__free_pages(page, 2); __free_pages(page, CRST_ALLOC_ORDER);
return rc; return rc;
} }
EXPORT_SYMBOL_GPL(gmap_shadow_sgt); EXPORT_SYMBOL_GPL(gmap_shadow_sgt);
...@@ -1902,7 +1902,7 @@ int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt, ...@@ -1902,7 +1902,7 @@ int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,
} }
spin_unlock(&sg->guest_table_lock); spin_unlock(&sg->guest_table_lock);
/* Make pgt read-only in parent gmap page table (not the pgste) */ /* Make pgt read-only in parent gmap page table (not the pgste) */
raddr = (saddr & 0xfffffffffff00000UL) | _SHADOW_RMAP_SEGMENT; raddr = (saddr & _SEGMENT_MASK) | _SHADOW_RMAP_SEGMENT;
origin = pgt & _SEGMENT_ENTRY_ORIGIN & PAGE_MASK; origin = pgt & _SEGMENT_ENTRY_ORIGIN & PAGE_MASK;
rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE, PROT_READ); rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE, PROT_READ);
spin_lock(&sg->guest_table_lock); spin_lock(&sg->guest_table_lock);
...@@ -2021,7 +2021,7 @@ static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, ...@@ -2021,7 +2021,7 @@ static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,
} }
/* Check for top level table */ /* Check for top level table */
start = sg->orig_asce & _ASCE_ORIGIN; start = sg->orig_asce & _ASCE_ORIGIN;
end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * 4096; end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE;
if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start && if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start &&
gaddr < end) { gaddr < end) {
/* The complete shadow table has to go */ /* The complete shadow table has to go */
...@@ -2032,7 +2032,7 @@ static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr, ...@@ -2032,7 +2032,7 @@ static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,
return; return;
} }
/* Remove the page table tree from on specific entry */ /* Remove the page table tree from on specific entry */
head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> 12); head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);
gmap_for_each_rmap_safe(rmap, rnext, head) { gmap_for_each_rmap_safe(rmap, rnext, head) {
bits = rmap->raddr & _SHADOW_RMAP_MASK; bits = rmap->raddr & _SHADOW_RMAP_MASK;
raddr = rmap->raddr ^ bits; raddr = rmap->raddr ^ bits;
...@@ -2076,7 +2076,7 @@ void ptep_notify(struct mm_struct *mm, unsigned long vmaddr, ...@@ -2076,7 +2076,7 @@ void ptep_notify(struct mm_struct *mm, unsigned long vmaddr,
struct gmap *gmap, *sg, *next; struct gmap *gmap, *sg, *next;
offset = ((unsigned long) pte) & (255 * sizeof(pte_t)); offset = ((unsigned long) pte) & (255 * sizeof(pte_t));
offset = offset * (4096 / sizeof(pte_t)); offset = offset * (PAGE_SIZE / sizeof(pte_t));
rcu_read_lock(); rcu_read_lock();
list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) { list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
spin_lock(&gmap->guest_table_lock); spin_lock(&gmap->guest_table_lock);
......
...@@ -84,7 +84,7 @@ void __init paging_init(void) ...@@ -84,7 +84,7 @@ void __init paging_init(void)
psw_t psw; psw_t psw;
init_mm.pgd = swapper_pg_dir; init_mm.pgd = swapper_pg_dir;
if (VMALLOC_END > (1UL << 42)) { if (VMALLOC_END > _REGION2_SIZE) {
asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH; asce_bits = _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;
pgd_type = _REGION2_ENTRY_EMPTY; pgd_type = _REGION2_ENTRY_EMPTY;
} else { } else {
...@@ -93,8 +93,7 @@ void __init paging_init(void) ...@@ -93,8 +93,7 @@ void __init paging_init(void)
} }
init_mm.context.asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits; init_mm.context.asce = (__pa(init_mm.pgd) & PAGE_MASK) | asce_bits;
S390_lowcore.kernel_asce = init_mm.context.asce; S390_lowcore.kernel_asce = init_mm.context.asce;
clear_table((unsigned long *) init_mm.pgd, pgd_type, crst_table_init((unsigned long *) init_mm.pgd, pgd_type);
sizeof(unsigned long)*2048);
vmem_map_init(); vmem_map_init();
/* enable virtual mapping in kernel mode */ /* enable virtual mapping in kernel mode */
......
...@@ -83,7 +83,7 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long end) ...@@ -83,7 +83,7 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long end)
int rc, notify; int rc, notify;
/* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */ /* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */
BUG_ON(mm->context.asce_limit < (1UL << 42)); BUG_ON(mm->context.asce_limit < _REGION2_SIZE);
if (end >= TASK_SIZE_MAX) if (end >= TASK_SIZE_MAX)
return -ENOMEM; return -ENOMEM;
rc = 0; rc = 0;
...@@ -96,11 +96,11 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long end) ...@@ -96,11 +96,11 @@ int crst_table_upgrade(struct mm_struct *mm, unsigned long end)
} }
spin_lock_bh(&mm->page_table_lock); spin_lock_bh(&mm->page_table_lock);
pgd = (unsigned long *) mm->pgd; pgd = (unsigned long *) mm->pgd;
if (mm->context.asce_limit == (1UL << 42)) { if (mm->context.asce_limit == _REGION2_SIZE) {
crst_table_init(table, _REGION2_ENTRY_EMPTY); crst_table_init(table, _REGION2_ENTRY_EMPTY);
p4d_populate(mm, (p4d_t *) table, (pud_t *) pgd); p4d_populate(mm, (p4d_t *) table, (pud_t *) pgd);
mm->pgd = (pgd_t *) table; mm->pgd = (pgd_t *) table;
mm->context.asce_limit = 1UL << 53; mm->context.asce_limit = _REGION1_SIZE;
mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
_ASCE_USER_BITS | _ASCE_TYPE_REGION2; _ASCE_USER_BITS | _ASCE_TYPE_REGION2;
} else { } else {
...@@ -124,7 +124,7 @@ void crst_table_downgrade(struct mm_struct *mm) ...@@ -124,7 +124,7 @@ void crst_table_downgrade(struct mm_struct *mm)
pgd_t *pgd; pgd_t *pgd;
/* downgrade should only happen from 3 to 2 levels (compat only) */ /* downgrade should only happen from 3 to 2 levels (compat only) */
BUG_ON(mm->context.asce_limit != (1UL << 42)); BUG_ON(mm->context.asce_limit != _REGION2_SIZE);
if (current->active_mm == mm) { if (current->active_mm == mm) {
clear_user_asce(); clear_user_asce();
...@@ -133,7 +133,7 @@ void crst_table_downgrade(struct mm_struct *mm) ...@@ -133,7 +133,7 @@ void crst_table_downgrade(struct mm_struct *mm)
pgd = mm->pgd; pgd = mm->pgd;
mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN); mm->pgd = (pgd_t *) (pgd_val(*pgd) & _REGION_ENTRY_ORIGIN);
mm->context.asce_limit = 1UL << 31; mm->context.asce_limit = _REGION3_SIZE;
mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH | mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
_ASCE_USER_BITS | _ASCE_TYPE_SEGMENT; _ASCE_USER_BITS | _ASCE_TYPE_SEGMENT;
crst_table_free(mm, (unsigned long *) pgd); crst_table_free(mm, (unsigned long *) pgd);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment