Commit ceb86879 authored by Andi Kleen's avatar Andi Kleen Committed by Linus Torvalds

hugetlb: introduce pud_huge

Straight forward extensions for huge pages located in the PUD instead of
PMDs.
Signed-off-by: default avatarAndi Kleen <ak@suse.de>
Signed-off-by: default avatarNick Piggin <npiggin@suse.de>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 4abd32db
...@@ -107,6 +107,12 @@ int pmd_huge(pmd_t pmd) ...@@ -107,6 +107,12 @@ int pmd_huge(pmd_t pmd)
{ {
return 0; return 0;
} }
int pud_huge(pud_t pud)
{
return 0;
}
struct page * struct page *
follow_huge_pmd(struct mm_struct *mm, unsigned long address, pmd_t *pmd, int write) follow_huge_pmd(struct mm_struct *mm, unsigned long address, pmd_t *pmd, int write)
{ {
......
...@@ -369,6 +369,11 @@ int pmd_huge(pmd_t pmd) ...@@ -369,6 +369,11 @@ int pmd_huge(pmd_t pmd)
return 0; return 0;
} }
int pud_huge(pud_t pud)
{
return 0;
}
struct page * struct page *
follow_huge_pmd(struct mm_struct *mm, unsigned long address, follow_huge_pmd(struct mm_struct *mm, unsigned long address,
pmd_t *pmd, int write) pmd_t *pmd, int write)
......
...@@ -120,6 +120,11 @@ int pmd_huge(pmd_t pmd) ...@@ -120,6 +120,11 @@ int pmd_huge(pmd_t pmd)
return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE); return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
} }
int pud_huge(pud_t pud)
{
return 0;
}
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
pmd_t *pmdp, int write) pmd_t *pmdp, int write)
{ {
......
...@@ -79,6 +79,11 @@ int pmd_huge(pmd_t pmd) ...@@ -79,6 +79,11 @@ int pmd_huge(pmd_t pmd)
return 0; return 0;
} }
int pud_huge(pud_t pud)
{
return 0;
}
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
pmd_t *pmd, int write) pmd_t *pmd, int write)
{ {
......
...@@ -295,6 +295,11 @@ int pmd_huge(pmd_t pmd) ...@@ -295,6 +295,11 @@ int pmd_huge(pmd_t pmd)
return 0; return 0;
} }
int pud_huge(pud_t pud)
{
return 0;
}
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
pmd_t *pmd, int write) pmd_t *pmd, int write)
{ {
......
...@@ -189,6 +189,11 @@ int pmd_huge(pmd_t pmd) ...@@ -189,6 +189,11 @@ int pmd_huge(pmd_t pmd)
return 0; return 0;
} }
int pud_huge(pud_t pud)
{
return 0;
}
struct page * struct page *
follow_huge_pmd(struct mm_struct *mm, unsigned long address, follow_huge_pmd(struct mm_struct *mm, unsigned long address,
pmd_t *pmd, int write) pmd_t *pmd, int write)
...@@ -209,6 +214,11 @@ int pmd_huge(pmd_t pmd) ...@@ -209,6 +214,11 @@ int pmd_huge(pmd_t pmd)
return !!(pmd_val(pmd) & _PAGE_PSE); return !!(pmd_val(pmd) & _PAGE_PSE);
} }
int pud_huge(pud_t pud)
{
return 0;
}
struct page * struct page *
follow_huge_pmd(struct mm_struct *mm, unsigned long address, follow_huge_pmd(struct mm_struct *mm, unsigned long address,
pmd_t *pmd, int write) pmd_t *pmd, int write)
...@@ -217,9 +227,22 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address, ...@@ -217,9 +227,22 @@ follow_huge_pmd(struct mm_struct *mm, unsigned long address,
page = pte_page(*(pte_t *)pmd); page = pte_page(*(pte_t *)pmd);
if (page) if (page)
page += ((address & ~HPAGE_MASK) >> PAGE_SHIFT); page += ((address & ~PMD_MASK) >> PAGE_SHIFT);
return page; return page;
} }
struct page *
follow_huge_pud(struct mm_struct *mm, unsigned long address,
pud_t *pud, int write)
{
struct page *page;
page = pte_page(*(pte_t *)pud);
if (page)
page += ((address & ~PUD_MASK) >> PAGE_SHIFT);
return page;
}
#endif #endif
/* x86_64 also uses this file */ /* x86_64 also uses this file */
......
...@@ -50,7 +50,10 @@ struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address, ...@@ -50,7 +50,10 @@ struct page *follow_huge_addr(struct mm_struct *mm, unsigned long address,
int write); int write);
struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address, struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
pmd_t *pmd, int write); pmd_t *pmd, int write);
struct page *follow_huge_pud(struct mm_struct *mm, unsigned long address,
pud_t *pud, int write);
int pmd_huge(pmd_t pmd); int pmd_huge(pmd_t pmd);
int pud_huge(pud_t pmd);
void hugetlb_change_protection(struct vm_area_struct *vma, void hugetlb_change_protection(struct vm_area_struct *vma,
unsigned long address, unsigned long end, pgprot_t newprot); unsigned long address, unsigned long end, pgprot_t newprot);
...@@ -78,8 +81,10 @@ static inline unsigned long hugetlb_total_pages(void) ...@@ -78,8 +81,10 @@ static inline unsigned long hugetlb_total_pages(void)
#define hugetlb_report_meminfo(buf) 0 #define hugetlb_report_meminfo(buf) 0
#define hugetlb_report_node_meminfo(n, buf) 0 #define hugetlb_report_node_meminfo(n, buf) 0
#define follow_huge_pmd(mm, addr, pmd, write) NULL #define follow_huge_pmd(mm, addr, pmd, write) NULL
#define follow_huge_pud(mm, addr, pud, write) NULL
#define prepare_hugepage_range(file, addr, len) (-EINVAL) #define prepare_hugepage_range(file, addr, len) (-EINVAL)
#define pmd_huge(x) 0 #define pmd_huge(x) 0
#define pud_huge(x) 0
#define is_hugepage_only_range(mm, addr, len) 0 #define is_hugepage_only_range(mm, addr, len) 0
#define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; }) #define hugetlb_free_pgd_range(tlb, addr, end, floor, ceiling) ({BUG(); 0; })
#define hugetlb_fault(mm, vma, addr, write) ({ BUG(); 0; }) #define hugetlb_fault(mm, vma, addr, write) ({ BUG(); 0; })
......
...@@ -1996,6 +1996,15 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -1996,6 +1996,15 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
return ret; return ret;
} }
/* Can be overriden by architectures */
__attribute__((weak)) struct page *
follow_huge_pud(struct mm_struct *mm, unsigned long address,
pud_t *pud, int write)
{
BUG();
return NULL;
}
int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma, int follow_hugetlb_page(struct mm_struct *mm, struct vm_area_struct *vma,
struct page **pages, struct vm_area_struct **vmas, struct page **pages, struct vm_area_struct **vmas,
unsigned long *position, int *length, int i, unsigned long *position, int *length, int i,
......
...@@ -998,19 +998,24 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address, ...@@ -998,19 +998,24 @@ struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
goto no_page_table; goto no_page_table;
pud = pud_offset(pgd, address); pud = pud_offset(pgd, address);
if (pud_none(*pud) || unlikely(pud_bad(*pud))) if (pud_none(*pud))
goto no_page_table;
if (pud_huge(*pud)) {
BUG_ON(flags & FOLL_GET);
page = follow_huge_pud(mm, address, pud, flags & FOLL_WRITE);
goto out;
}
if (unlikely(pud_bad(*pud)))
goto no_page_table; goto no_page_table;
pmd = pmd_offset(pud, address); pmd = pmd_offset(pud, address);
if (pmd_none(*pmd)) if (pmd_none(*pmd))
goto no_page_table; goto no_page_table;
if (pmd_huge(*pmd)) { if (pmd_huge(*pmd)) {
BUG_ON(flags & FOLL_GET); BUG_ON(flags & FOLL_GET);
page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE); page = follow_huge_pmd(mm, address, pmd, flags & FOLL_WRITE);
goto out; goto out;
} }
if (unlikely(pmd_bad(*pmd))) if (unlikely(pmd_bad(*pmd)))
goto no_page_table; goto no_page_table;
...@@ -1567,6 +1572,8 @@ static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, ...@@ -1567,6 +1572,8 @@ static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud,
unsigned long next; unsigned long next;
int err; int err;
BUG_ON(pud_huge(*pud));
pmd = pmd_alloc(mm, pud, addr); pmd = pmd_alloc(mm, pud, addr);
if (!pmd) if (!pmd)
return -ENOMEM; return -ENOMEM;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment