Commit dd78fedd authored by Kirill A. Shutemov's avatar Kirill A. Shutemov Committed by Linus Torvalds

rmap: support file thp

Naive approach: on mapping/unmapping the page as compound we update
->_mapcount on each 4k page.  That's not efficient, but it's not obvious
how we can optimize this.  We can look into optimization later.

PG_double_map optimization doesn't work for file pages since lifecycle
of file pages is different comparing to anon pages: file page can be
mapped again at any time.

Link: http://lkml.kernel.org/r/1466021202-61880-11-git-send-email-kirill.shutemov@linux.intel.comSigned-off-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 7267ec00
...@@ -165,7 +165,7 @@ void do_page_add_anon_rmap(struct page *, struct vm_area_struct *, ...@@ -165,7 +165,7 @@ void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,
unsigned long, int); unsigned long, int);
void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
unsigned long, bool); unsigned long, bool);
void page_add_file_rmap(struct page *); void page_add_file_rmap(struct page *, bool);
void page_remove_rmap(struct page *, bool); void page_remove_rmap(struct page *, bool);
void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *, void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,
......
...@@ -3301,18 +3301,22 @@ static void __split_huge_page(struct page *page, struct list_head *list) ...@@ -3301,18 +3301,22 @@ static void __split_huge_page(struct page *page, struct list_head *list)
int total_mapcount(struct page *page) int total_mapcount(struct page *page)
{ {
int i, ret; int i, compound, ret;
VM_BUG_ON_PAGE(PageTail(page), page); VM_BUG_ON_PAGE(PageTail(page), page);
if (likely(!PageCompound(page))) if (likely(!PageCompound(page)))
return atomic_read(&page->_mapcount) + 1; return atomic_read(&page->_mapcount) + 1;
ret = compound_mapcount(page); compound = compound_mapcount(page);
if (PageHuge(page)) if (PageHuge(page))
return ret; return compound;
ret = compound;
for (i = 0; i < HPAGE_PMD_NR; i++) for (i = 0; i < HPAGE_PMD_NR; i++)
ret += atomic_read(&page[i]._mapcount) + 1; ret += atomic_read(&page[i]._mapcount) + 1;
/* File pages has compound_mapcount included in _mapcount */
if (!PageAnon(page))
return ret - compound * HPAGE_PMD_NR;
if (PageDoubleMap(page)) if (PageDoubleMap(page))
ret -= HPAGE_PMD_NR; ret -= HPAGE_PMD_NR;
return ret; return ret;
......
...@@ -1494,7 +1494,7 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr, ...@@ -1494,7 +1494,7 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr,
/* Ok, finally just insert the thing.. */ /* Ok, finally just insert the thing.. */
get_page(page); get_page(page);
inc_mm_counter_fast(mm, mm_counter_file(page)); inc_mm_counter_fast(mm, mm_counter_file(page));
page_add_file_rmap(page); page_add_file_rmap(page, false);
set_pte_at(mm, addr, pte, mk_pte(page, prot)); set_pte_at(mm, addr, pte, mk_pte(page, prot));
retval = 0; retval = 0;
...@@ -2962,7 +2962,7 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg, ...@@ -2962,7 +2962,7 @@ int alloc_set_pte(struct fault_env *fe, struct mem_cgroup *memcg,
lru_cache_add_active_or_unevictable(page, vma); lru_cache_add_active_or_unevictable(page, vma);
} else { } else {
inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page)); inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
page_add_file_rmap(page); page_add_file_rmap(page, false);
} }
set_pte_at(vma->vm_mm, fe->address, fe->pte, entry); set_pte_at(vma->vm_mm, fe->address, fe->pte, entry);
......
...@@ -259,7 +259,7 @@ static int remove_migration_pte(struct page *new, struct vm_area_struct *vma, ...@@ -259,7 +259,7 @@ static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
} else if (PageAnon(new)) } else if (PageAnon(new))
page_add_anon_rmap(new, vma, addr, false); page_add_anon_rmap(new, vma, addr, false);
else else
page_add_file_rmap(new); page_add_file_rmap(new, false);
if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new)) if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
mlock_vma_page(new); mlock_vma_page(new);
......
...@@ -1270,18 +1270,34 @@ void page_add_new_anon_rmap(struct page *page, ...@@ -1270,18 +1270,34 @@ void page_add_new_anon_rmap(struct page *page,
* *
* The caller needs to hold the pte lock. * The caller needs to hold the pte lock.
*/ */
void page_add_file_rmap(struct page *page) void page_add_file_rmap(struct page *page, bool compound)
{ {
int i, nr = 1;
VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
lock_page_memcg(page); lock_page_memcg(page);
if (atomic_inc_and_test(&page->_mapcount)) { if (compound && PageTransHuge(page)) {
__inc_zone_page_state(page, NR_FILE_MAPPED); for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED); if (atomic_inc_and_test(&page[i]._mapcount))
nr++;
}
if (!atomic_inc_and_test(compound_mapcount_ptr(page)))
goto out;
} else {
if (!atomic_inc_and_test(&page->_mapcount))
goto out;
} }
__mod_zone_page_state(page_zone(page), NR_FILE_MAPPED, nr);
mem_cgroup_inc_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
out:
unlock_page_memcg(page); unlock_page_memcg(page);
} }
static void page_remove_file_rmap(struct page *page) static void page_remove_file_rmap(struct page *page, bool compound)
{ {
int i, nr = 1;
VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
lock_page_memcg(page); lock_page_memcg(page);
/* Hugepages are not counted in NR_FILE_MAPPED for now. */ /* Hugepages are not counted in NR_FILE_MAPPED for now. */
...@@ -1292,15 +1308,24 @@ static void page_remove_file_rmap(struct page *page) ...@@ -1292,15 +1308,24 @@ static void page_remove_file_rmap(struct page *page)
} }
/* page still mapped by someone else? */ /* page still mapped by someone else? */
if (!atomic_add_negative(-1, &page->_mapcount)) if (compound && PageTransHuge(page)) {
goto out; for (i = 0, nr = 0; i < HPAGE_PMD_NR; i++) {
if (atomic_add_negative(-1, &page[i]._mapcount))
nr++;
}
if (!atomic_add_negative(-1, compound_mapcount_ptr(page)))
goto out;
} else {
if (!atomic_add_negative(-1, &page->_mapcount))
goto out;
}
/* /*
* We use the irq-unsafe __{inc|mod}_zone_page_stat because * We use the irq-unsafe __{inc|mod}_zone_page_stat because
* these counters are not modified in interrupt context, and * these counters are not modified in interrupt context, and
* pte lock(a spinlock) is held, which implies preemption disabled. * pte lock(a spinlock) is held, which implies preemption disabled.
*/ */
__dec_zone_page_state(page, NR_FILE_MAPPED); __mod_zone_page_state(page_zone(page), NR_FILE_MAPPED, -nr);
mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED); mem_cgroup_dec_page_stat(page, MEM_CGROUP_STAT_FILE_MAPPED);
if (unlikely(PageMlocked(page))) if (unlikely(PageMlocked(page)))
...@@ -1356,11 +1381,8 @@ static void page_remove_anon_compound_rmap(struct page *page) ...@@ -1356,11 +1381,8 @@ static void page_remove_anon_compound_rmap(struct page *page)
*/ */
void page_remove_rmap(struct page *page, bool compound) void page_remove_rmap(struct page *page, bool compound)
{ {
if (!PageAnon(page)) { if (!PageAnon(page))
VM_BUG_ON_PAGE(compound && !PageHuge(page), page); return page_remove_file_rmap(page, compound);
page_remove_file_rmap(page);
return;
}
if (compound) if (compound)
return page_remove_anon_compound_rmap(page); return page_remove_anon_compound_rmap(page);
......
...@@ -412,6 +412,12 @@ int __page_mapcount(struct page *page) ...@@ -412,6 +412,12 @@ int __page_mapcount(struct page *page)
int ret; int ret;
ret = atomic_read(&page->_mapcount) + 1; ret = atomic_read(&page->_mapcount) + 1;
/*
* For file THP page->_mapcount contains total number of mapping
* of the page: no need to look into compound_mapcount.
*/
if (!PageAnon(page) && !PageHuge(page))
return ret;
page = compound_head(page); page = compound_head(page);
ret += atomic_read(compound_mapcount_ptr(page)) + 1; ret += atomic_read(compound_mapcount_ptr(page)) + 1;
if (PageDoubleMap(page)) if (PageDoubleMap(page))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment