Commit 10102459 authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

mm: introduce do_set_pmd()

With postponed page table allocation we have chance to setup huge pages.
do_set_pte() calls do_set_pmd() if following criteria met:

 - page is compound;
 - pmd entry in pmd_none();
 - vma has suitable size and alignment;

Link: http://lkml.kernel.org/r/1466021202-61880-12-git-send-email-kirill.shutemov@linux.intel.comSigned-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent dd78fedd
...@@ -143,6 +143,8 @@ static inline bool is_huge_zero_pmd(pmd_t pmd) ...@@ -143,6 +143,8 @@ static inline bool is_huge_zero_pmd(pmd_t pmd)
struct page *get_huge_zero_page(void); struct page *get_huge_zero_page(void);
void put_huge_zero_page(void); void put_huge_zero_page(void);
#define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))
#else /* CONFIG_TRANSPARENT_HUGEPAGE */ #else /* CONFIG_TRANSPARENT_HUGEPAGE */
#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; }) #define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; }) #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
......
...@@ -796,11 +796,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) ...@@ -796,11 +796,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
return pmd; return pmd;
} }
static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot)
{
return pmd_mkhuge(mk_pmd(page, prot));
}
static inline struct list_head *page_deferred_list(struct page *page) static inline struct list_head *page_deferred_list(struct page *page)
{ {
/* /*
......
...@@ -2920,6 +2920,66 @@ static int pte_alloc_one_map(struct fault_env *fe) ...@@ -2920,6 +2920,66 @@ static int pte_alloc_one_map(struct fault_env *fe)
return 0; return 0;
} }
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define HPAGE_CACHE_INDEX_MASK (HPAGE_PMD_NR - 1)
static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
unsigned long haddr)
{
if (((vma->vm_start >> PAGE_SHIFT) & HPAGE_CACHE_INDEX_MASK) !=
(vma->vm_pgoff & HPAGE_CACHE_INDEX_MASK))
return false;
if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
return false;
return true;
}
static int do_set_pmd(struct fault_env *fe, struct page *page)
{
struct vm_area_struct *vma = fe->vma;
bool write = fe->flags & FAULT_FLAG_WRITE;
unsigned long haddr = fe->address & HPAGE_PMD_MASK;
pmd_t entry;
int i, ret;
if (!transhuge_vma_suitable(vma, haddr))
return VM_FAULT_FALLBACK;
ret = VM_FAULT_FALLBACK;
page = compound_head(page);
fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
if (unlikely(!pmd_none(*fe->pmd)))
goto out;
for (i = 0; i < HPAGE_PMD_NR; i++)
flush_icache_page(vma, page + i);
entry = mk_huge_pmd(page, vma->vm_page_prot);
if (write)
entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
add_mm_counter(vma->vm_mm, MM_FILEPAGES, HPAGE_PMD_NR);
page_add_file_rmap(page, true);
set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
update_mmu_cache_pmd(vma, haddr, fe->pmd);
/* fault is handled */
ret = 0;
out:
spin_unlock(fe->ptl);
return ret;
}
#else
static int do_set_pmd(struct fault_env *fe, struct page *page)
{
BUILD_BUG();
return 0;
}
#endif
/** /**
* alloc_set_pte - setup new PTE entry for given page and add reverse page * alloc_set_pte - setup new PTE entry for given page and add reverse page
* mapping. If needed, the fucntion allocates page table or use pre-allocated. * mapping. If needed, the fucntion allocates page table or use pre-allocated.
...@@ -2939,9 +2999,19 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg, ...@@ -2939,9 +2999,19 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
struct vm_area_struct *vma = fe->vma; struct vm_area_struct *vma = fe->vma;
bool write = fe->flags & FAULT_FLAG_WRITE; bool write = fe->flags & FAULT_FLAG_WRITE;
pte_t entry; pte_t entry;
int ret;
if (pmd_none(*fe->pmd) && PageTransCompound(page)) {
/* THP on COW? */
VM_BUG_ON_PAGE(memcg, page);
ret = do_set_pmd(fe, page);
if (ret != VM_FAULT_FALLBACK)
return ret;
}
if (!fe->pte) { if (!fe->pte) {
int ret = pte_alloc_one_map(fe); ret = pte_alloc_one_map(fe);
if (ret) if (ret)
return ret; return ret;
} }
......
...@@ -1986,8 +1986,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm, ...@@ -1986,8 +1986,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
} }
orig_entry = *pmd; orig_entry = *pmd;
entry = mk_pmd(new_page, vma->vm_page_prot); entry = mk_huge_pmd(new_page, vma->vm_page_prot);
entry = pmd_mkhuge(entry);
entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment