Commit e496cf3d authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

thp: introduce CONFIG_TRANSPARENT_HUGE_PAGECACHE

For file mappings, we don't deposit page tables on THP allocation
because it's not strictly required to implement split_huge_pmd(): we can
just clear pmd and let following page faults to reconstruct the page
table.

But Power makes use of deposited page table to address MMU quirk.

Let's hide THP page cache, including huge tmpfs, under separate config
option, so it can be forbidden on Power.

We can revert the patch later once solution for Power found.

Link: http://lkml.kernel.org/r/1466021202-61880-36-git-send-email-kirill.shutemov@linux.intel.comSigned-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent f3f0e1d2
...@@ -54,7 +54,6 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr, ...@@ -54,7 +54,6 @@ extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr,
unsigned long len, unsigned long pgoff, unsigned long flags); unsigned long len, unsigned long pgoff, unsigned long flags);
extern int shmem_lock(struct file *file, int lock, struct user_struct *user); extern int shmem_lock(struct file *file, int lock, struct user_struct *user);
extern bool shmem_mapping(struct address_space *mapping); extern bool shmem_mapping(struct address_space *mapping);
extern bool shmem_huge_enabled(struct vm_area_struct *vma);
extern void shmem_unlock_mapping(struct address_space *mapping); extern void shmem_unlock_mapping(struct address_space *mapping);
extern struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, extern struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
pgoff_t index, gfp_t gfp_mask); pgoff_t index, gfp_t gfp_mask);
...@@ -112,4 +111,13 @@ static inline long shmem_fcntl(struct file *f, unsigned int c, unsigned long a) ...@@ -112,4 +111,13 @@ static inline long shmem_fcntl(struct file *f, unsigned int c, unsigned long a)
#endif #endif
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
extern bool shmem_huge_enabled(struct vm_area_struct *vma);
#else
static inline bool shmem_huge_enabled(struct vm_area_struct *vma)
{
return false;
}
#endif
#endif #endif
...@@ -439,6 +439,14 @@ choice ...@@ -439,6 +439,14 @@ choice
benefit. benefit.
endchoice endchoice
#
# We don't deposit page tables on file THP mapping,
# but Power makes use of them to address MMU quirk.
#
config TRANSPARENT_HUGE_PAGECACHE
def_bool y
depends on TRANSPARENT_HUGEPAGE && !PPC
# #
# UP and nommu archs use km based percpu allocator # UP and nommu archs use km based percpu allocator
# #
......
...@@ -287,7 +287,7 @@ static struct attribute *hugepage_attr[] = { ...@@ -287,7 +287,7 @@ static struct attribute *hugepage_attr[] = {
&enabled_attr.attr, &enabled_attr.attr,
&defrag_attr.attr, &defrag_attr.attr,
&use_zero_page_attr.attr, &use_zero_page_attr.attr,
#ifdef CONFIG_SHMEM #if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
&shmem_enabled_attr.attr, &shmem_enabled_attr.attr,
#endif #endif
#ifdef CONFIG_DEBUG_VM #ifdef CONFIG_DEBUG_VM
......
...@@ -819,6 +819,8 @@ static bool hugepage_vma_check(struct vm_area_struct *vma) ...@@ -819,6 +819,8 @@ static bool hugepage_vma_check(struct vm_area_struct *vma)
(vma->vm_flags & VM_NOHUGEPAGE)) (vma->vm_flags & VM_NOHUGEPAGE))
return false; return false;
if (shmem_file(vma->vm_file)) { if (shmem_file(vma->vm_file)) {
if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
return false;
return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff, return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
HPAGE_PMD_NR); HPAGE_PMD_NR);
} }
...@@ -1222,7 +1224,7 @@ static void collect_mm_slot(struct mm_slot *mm_slot) ...@@ -1222,7 +1224,7 @@ static void collect_mm_slot(struct mm_slot *mm_slot)
} }
} }
#ifdef CONFIG_SHMEM #if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
{ {
struct vm_area_struct *vma; struct vm_area_struct *vma;
...@@ -1681,8 +1683,6 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, ...@@ -1681,8 +1683,6 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
if (khugepaged_scan.address < hstart) if (khugepaged_scan.address < hstart)
khugepaged_scan.address = hstart; khugepaged_scan.address = hstart;
VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK); VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
if (shmem_file(vma->vm_file) && !shmem_huge_enabled(vma))
goto skip;
while (khugepaged_scan.address < hend) { while (khugepaged_scan.address < hend) {
int ret; int ret;
...@@ -1694,9 +1694,12 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages, ...@@ -1694,9 +1694,12 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
khugepaged_scan.address + HPAGE_PMD_SIZE > khugepaged_scan.address + HPAGE_PMD_SIZE >
hend); hend);
if (shmem_file(vma->vm_file)) { if (shmem_file(vma->vm_file)) {
struct file *file = get_file(vma->vm_file); struct file *file;
pgoff_t pgoff = linear_page_index(vma, pgoff_t pgoff = linear_page_index(vma,
khugepaged_scan.address); khugepaged_scan.address);
if (!shmem_huge_enabled(vma))
goto skip;
file = get_file(vma->vm_file);
up_read(&mm->mmap_sem); up_read(&mm->mmap_sem);
ret = 1; ret = 1;
khugepaged_scan_shmem(mm, file->f_mapping, khugepaged_scan_shmem(mm, file->f_mapping,
......
...@@ -2920,7 +2920,7 @@ static int pte_alloc_one_map(struct fault_env *fe) ...@@ -2920,7 +2920,7 @@ static int pte_alloc_one_map(struct fault_env *fe)
return 0; return 0;
} }
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
#define HPAGE_CACHE_INDEX_MASK (HPAGE_PMD_NR - 1) #define HPAGE_CACHE_INDEX_MASK (HPAGE_PMD_NR - 1)
static inline bool transhuge_vma_suitable(struct vm_area_struct *vma, static inline bool transhuge_vma_suitable(struct vm_area_struct *vma,
...@@ -3002,7 +3002,8 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg, ...@@ -3002,7 +3002,8 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
pte_t entry; pte_t entry;
int ret; int ret;
if (pmd_none(*fe->pmd) && PageTransCompound(page)) { if (pmd_none(*fe->pmd) && PageTransCompound(page) &&
IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
/* THP on COW? */ /* THP on COW? */
VM_BUG_ON_PAGE(memcg, page); VM_BUG_ON_PAGE(memcg, page);
......
...@@ -363,7 +363,7 @@ static bool shmem_confirm_swap(struct address_space *mapping, ...@@ -363,7 +363,7 @@ static bool shmem_confirm_swap(struct address_space *mapping,
#define SHMEM_HUGE_DENY (-1) #define SHMEM_HUGE_DENY (-1)
#define SHMEM_HUGE_FORCE (-2) #define SHMEM_HUGE_FORCE (-2)
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
/* ifdef here to avoid bloating shmem.o when not necessary */ /* ifdef here to avoid bloating shmem.o when not necessary */
int shmem_huge __read_mostly; int shmem_huge __read_mostly;
...@@ -406,11 +406,11 @@ static const char *shmem_format_huge(int huge) ...@@ -406,11 +406,11 @@ static const char *shmem_format_huge(int huge)
} }
} }
#else /* !CONFIG_TRANSPARENT_HUGEPAGE */ #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */
#define shmem_huge SHMEM_HUGE_DENY #define shmem_huge SHMEM_HUGE_DENY
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */
/* /*
* Like add_to_page_cache_locked, but error if expected item has gone. * Like add_to_page_cache_locked, but error if expected item has gone.
...@@ -1229,7 +1229,7 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp, ...@@ -1229,7 +1229,7 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp,
void __rcu **results; void __rcu **results;
struct page *page; struct page *page;
if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
return NULL; return NULL;
rcu_read_lock(); rcu_read_lock();
...@@ -1270,7 +1270,7 @@ static struct page *shmem_alloc_and_acct_page(gfp_t gfp, ...@@ -1270,7 +1270,7 @@ static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
int nr; int nr;
int err = -ENOSPC; int err = -ENOSPC;
if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
huge = false; huge = false;
nr = huge ? HPAGE_PMD_NR : 1; nr = huge ? HPAGE_PMD_NR : 1;
...@@ -1773,7 +1773,7 @@ unsigned long shmem_get_unmapped_area(struct file *file, ...@@ -1773,7 +1773,7 @@ unsigned long shmem_get_unmapped_area(struct file *file,
get_area = current->mm->get_unmapped_area; get_area = current->mm->get_unmapped_area;
addr = get_area(file, uaddr, len, pgoff, flags); addr = get_area(file, uaddr, len, pgoff, flags);
if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
return addr; return addr;
if (IS_ERR_VALUE(addr)) if (IS_ERR_VALUE(addr))
return addr; return addr;
...@@ -1890,7 +1890,7 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma) ...@@ -1890,7 +1890,7 @@ static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
{ {
file_accessed(file); file_accessed(file);
vma->vm_ops = &shmem_vm_ops; vma->vm_ops = &shmem_vm_ops;
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
(vma->vm_end & HPAGE_PMD_MASK)) { (vma->vm_end & HPAGE_PMD_MASK)) {
khugepaged_enter(vma, vma->vm_flags); khugepaged_enter(vma, vma->vm_flags);
...@@ -3287,7 +3287,7 @@ static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, ...@@ -3287,7 +3287,7 @@ static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
sbinfo->gid = make_kgid(current_user_ns(), gid); sbinfo->gid = make_kgid(current_user_ns(), gid);
if (!gid_valid(sbinfo->gid)) if (!gid_valid(sbinfo->gid))
goto bad_val; goto bad_val;
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
} else if (!strcmp(this_char, "huge")) { } else if (!strcmp(this_char, "huge")) {
int huge; int huge;
huge = shmem_parse_huge(value); huge = shmem_parse_huge(value);
...@@ -3384,7 +3384,7 @@ static int shmem_show_options(struct seq_file *seq, struct dentry *root) ...@@ -3384,7 +3384,7 @@ static int shmem_show_options(struct seq_file *seq, struct dentry *root)
if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
seq_printf(seq, ",gid=%u", seq_printf(seq, ",gid=%u",
from_kgid_munged(&init_user_ns, sbinfo->gid)); from_kgid_munged(&init_user_ns, sbinfo->gid));
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
/* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
if (sbinfo->huge) if (sbinfo->huge)
seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
...@@ -3730,7 +3730,7 @@ int __init shmem_init(void) ...@@ -3730,7 +3730,7 @@ int __init shmem_init(void)
goto out1; goto out1;
} }
#ifdef CONFIG_TRANSPARENT_HUGEPAGE #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY) if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY)
SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
else else
...@@ -3747,7 +3747,7 @@ int __init shmem_init(void) ...@@ -3747,7 +3747,7 @@ int __init shmem_init(void)
return error; return error;
} }
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS) #if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS)
static ssize_t shmem_enabled_show(struct kobject *kobj, static ssize_t shmem_enabled_show(struct kobject *kobj,
struct kobj_attribute *attr, char *buf) struct kobj_attribute *attr, char *buf)
{ {
...@@ -3830,7 +3830,7 @@ bool shmem_huge_enabled(struct vm_area_struct *vma) ...@@ -3830,7 +3830,7 @@ bool shmem_huge_enabled(struct vm_area_struct *vma)
return false; return false;
} }
} }
#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */ #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */
#else /* !CONFIG_SHMEM */ #else /* !CONFIG_SHMEM */
...@@ -4010,7 +4010,7 @@ int shmem_zero_setup(struct vm_area_struct *vma) ...@@ -4010,7 +4010,7 @@ int shmem_zero_setup(struct vm_area_struct *vma)
vma->vm_file = file; vma->vm_file = file;
vma->vm_ops = &shmem_vm_ops; vma->vm_ops = &shmem_vm_ops;
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
(vma->vm_end & HPAGE_PMD_MASK)) { (vma->vm_end & HPAGE_PMD_MASK)) {
khugepaged_enter(vma, vma->vm_flags); khugepaged_enter(vma, vma->vm_flags);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment