Commit 934828ed authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Michael Ellerman

powerpc/mm: Make 4K and 64K use pte_t for pgtable_t

This patch switches 4K Linux page size config to use pte_t * type
instead of struct page * for pgtable_t. This simplifies the code a lot
and helps in consolidating both 64K and 4K page allocator routines. The
changes should not have any impact, because we already store physical
address in the upper level page table tree and that implies we already
do struct page * to physical address conversion.

One change to note here is we move the pgtable_page_dtor() call for
nohash to pte_fragment_free_mm(). The nohash related change is due to
the related changes in pgtable_64.c.
Signed-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: default avatarMichael Ellerman <mpe@ellerman.id.au>
parent 74701d59
...@@ -41,6 +41,15 @@ extern struct kmem_cache *pgtable_cache[]; ...@@ -41,6 +41,15 @@ extern struct kmem_cache *pgtable_cache[];
pgtable_cache[(shift) - 1]; \ pgtable_cache[(shift) - 1]; \
}) })
#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO
extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int);
extern void pte_fragment_free(unsigned long *, int);
extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
#ifdef CONFIG_SMP
extern void __tlb_remove_table(void *_table);
#endif
static inline pgd_t *pgd_alloc(struct mm_struct *mm) static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{ {
return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL); return kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), GFP_KERNEL);
...@@ -72,29 +81,47 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) ...@@ -72,29 +81,47 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
pud_set(pud, __pgtable_ptr_val(pmd)); pud_set(pud, __pgtable_ptr_val(pmd));
} }
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
unsigned long address)
{
pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE);
}
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
GFP_KERNEL|__GFP_REPEAT);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd);
}
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
unsigned long address)
{
return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX);
}
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
pte_t *pte) pte_t *pte)
{ {
pmd_set(pmd, __pgtable_ptr_val(pte)); pmd_set(pmd, __pgtable_ptr_val(pte));
} }
/*
* FIXME!!
* Between 4K and 64K pages, we differ in what is stored in pmd. ie.
* typedef pte_t *pgtable_t; -> 64K
* typedef struct page *pgtable_t; -> 4k
*/
#ifdef CONFIG_PPC_4K_PAGES
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
pgtable_t pte_page) pgtable_t pte_page)
{ {
pmd_set(pmd, __pgtable_ptr_val(page_address(pte_page))); pmd_set(pmd, __pgtable_ptr_val(pte_page));
} }
static inline pgtable_t pmd_pgtable(pmd_t pmd) static inline pgtable_t pmd_pgtable(pmd_t pmd)
{ {
return pmd_page(pmd); return (pgtable_t)pmd_page_vaddr(pmd);
} }
#ifdef CONFIG_PPC_4K_PAGES
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
unsigned long address) unsigned long address)
{ {
...@@ -115,83 +142,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, ...@@ -115,83 +142,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
__free_page(page); __free_page(page);
return NULL; return NULL;
} }
return page; return pte;
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
free_page((unsigned long)pte);
} }
static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
{
pgtable_page_dtor(ptepage);
__free_page(ptepage);
}
static inline void pgtable_free(void *table, unsigned index_size)
{
if (!index_size)
free_page((unsigned long)table);
else {
BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE);
kmem_cache_free(PGT_CACHE(index_size), table);
}
}
#ifdef CONFIG_SMP
static inline void pgtable_free_tlb(struct mmu_gather *tlb,
void *table, int shift)
{
unsigned long pgf = (unsigned long)table;
BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
pgf |= shift;
tlb_remove_table(tlb, (void *)pgf);
}
static inline void __tlb_remove_table(void *_table)
{
void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
pgtable_free(table, shift);
}
#else /* !CONFIG_SMP */
static inline void pgtable_free_tlb(struct mmu_gather *tlb,
void *table, int shift)
{
pgtable_free(table, shift);
}
#endif /* CONFIG_SMP */
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
unsigned long address)
{
tlb_flush_pgtable(tlb, address);
pgtable_page_dtor(table);
pgtable_free_tlb(tlb, page_address(table), 0);
}
#else /* if CONFIG_PPC_64K_PAGES */ #else /* if CONFIG_PPC_64K_PAGES */
extern pte_t *pte_fragment_alloc(struct mm_struct *, unsigned long, int);
extern void pte_fragment_free(unsigned long *, int);
extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
#ifdef CONFIG_SMP
extern void __tlb_remove_table(void *_table);
#endif
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
pgtable_t pte_page)
{
pmd_set(pmd, __pgtable_ptr_val(pte_page));
}
static inline pgtable_t pmd_pgtable(pmd_t pmd)
{
return (pgtable_t)pmd_page_vaddr(pmd);
}
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
unsigned long address) unsigned long address)
{ {
...@@ -203,6 +157,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, ...@@ -203,6 +157,7 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
{ {
return (pgtable_t)pte_fragment_alloc(mm, address, 0); return (pgtable_t)pte_fragment_alloc(mm, address, 0);
} }
#endif
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{ {
...@@ -220,30 +175,6 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, ...@@ -220,30 +175,6 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
tlb_flush_pgtable(tlb, address); tlb_flush_pgtable(tlb, address);
pgtable_free_tlb(tlb, table, 0); pgtable_free_tlb(tlb, table, 0);
} }
#endif /* CONFIG_PPC_4K_PAGES */
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{
return kmem_cache_alloc(PGT_CACHE(PMD_CACHE_INDEX),
GFP_KERNEL|__GFP_REPEAT);
}
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{
kmem_cache_free(PGT_CACHE(PMD_CACHE_INDEX), pmd);
}
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
unsigned long address)
{
return pgtable_free_tlb(tlb, pmd, PMD_CACHE_INDEX);
}
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
unsigned long address)
{
pgtable_free_tlb(tlb, pud, PUD_INDEX_SIZE);
}
#define check_pgt_cache() do { } while (0) #define check_pgt_cache() do { } while (0)
......
...@@ -119,46 +119,14 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage) ...@@ -119,46 +119,14 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
__free_page(ptepage); __free_page(ptepage);
} }
static inline void pgtable_free(void *table, unsigned index_size) extern void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift);
{
if (!index_size)
free_page((unsigned long)table);
else {
BUG_ON(index_size > MAX_PGTABLE_INDEX_SIZE);
kmem_cache_free(PGT_CACHE(index_size), table);
}
}
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
static inline void pgtable_free_tlb(struct mmu_gather *tlb, extern void __tlb_remove_table(void *_table);
void *table, int shift) #endif
{
unsigned long pgf = (unsigned long)table;
BUG_ON(shift > MAX_PGTABLE_INDEX_SIZE);
pgf |= shift;
tlb_remove_table(tlb, (void *)pgf);
}
static inline void __tlb_remove_table(void *_table)
{
void *table = (void *)((unsigned long)_table & ~MAX_PGTABLE_INDEX_SIZE);
unsigned shift = (unsigned long)_table & MAX_PGTABLE_INDEX_SIZE;
pgtable_free(table, shift);
}
#else /* !CONFIG_SMP */
static inline void pgtable_free_tlb(struct mmu_gather *tlb,
void *table, int shift)
{
pgtable_free(table, shift);
}
#endif /* CONFIG_SMP */
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table,
unsigned long address) unsigned long address)
{ {
tlb_flush_pgtable(tlb, address); tlb_flush_pgtable(tlb, address);
pgtable_page_dtor(table);
pgtable_free_tlb(tlb, page_address(table), 0); pgtable_free_tlb(tlb, page_address(table), 0);
} }
......
...@@ -316,12 +316,20 @@ void arch_free_page(struct page *page, int order); ...@@ -316,12 +316,20 @@ void arch_free_page(struct page *page, int order);
#endif #endif
struct vm_area_struct; struct vm_area_struct;
#ifdef CONFIG_PPC_BOOK3S_64
/*
* For BOOK3s 64 with 4k and 64K linux page size
* we want to use pointers, because the page table
* actually store pfn
*/
typedef pte_t *pgtable_t;
#else
#if defined(CONFIG_PPC_64K_PAGES) && defined(CONFIG_PPC64) #if defined(CONFIG_PPC_64K_PAGES) && defined(CONFIG_PPC64)
typedef pte_t *pgtable_t; typedef pte_t *pgtable_t;
#else #else
typedef struct page *pgtable_t; typedef struct page *pgtable_t;
#endif #endif
#endif
#include <asm-generic/memory_model.h> #include <asm-generic/memory_model.h>
#endif /* __ASSEMBLY__ */ #endif /* __ASSEMBLY__ */
......
...@@ -367,6 +367,7 @@ pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel ...@@ -367,6 +367,7 @@ pte_t *pte_fragment_alloc(struct mm_struct *mm, unsigned long vmaddr, int kernel
return __alloc_for_cache(mm, kernel); return __alloc_for_cache(mm, kernel);
} }
#endif /* CONFIG_PPC_64K_PAGES */
void pte_fragment_free(unsigned long *table, int kernel) void pte_fragment_free(unsigned long *table, int kernel)
{ {
...@@ -413,7 +414,6 @@ void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift) ...@@ -413,7 +414,6 @@ void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int shift)
} }
} }
#endif #endif
#endif /* CONFIG_PPC_64K_PAGES */
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGEPAGE
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment