Commit eca56ff9 authored by Jerome Marchand's avatar Jerome Marchand Committed by Linus Torvalds

mm, shmem: add internal shmem resident memory accounting

Currently looking at /proc/<pid>/status or statm, there is no way to
distinguish shmem pages from pages mapped to a regular file (shmem pages
are mapped to /dev/zero), even though their implication in actual memory
use is quite different.

The internal accounting currently counts shmem pages together with
regular files.  As a preparation to extend the userspace interfaces,
this patch adds MM_SHMEMPAGES counter to mm_rss_stat to account for
shmem pages separately from MM_FILEPAGES.  The next patch will expose it
to userspace - this patch doesn't change the exported values yet, by
adding up MM_SHMEMPAGES to MM_FILEPAGES at places where MM_FILEPAGES was
used before.  The only user-visible change after this patch is the OOM
killer message that separates the reported "shmem-rss" from "file-rss".

[vbabka@suse.cz: forward-porting, tweak changelog]
Signed-off-by: default avatarJerome Marchand <jmarchan@redhat.com>
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarKonstantin Khlebnikov <khlebnikov@yandex-team.ru>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Acked-by: default avatarHugh Dickins <hughd@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 48131e03
...@@ -603,10 +603,7 @@ static void gmap_zap_swap_entry(swp_entry_t entry, struct mm_struct *mm) ...@@ -603,10 +603,7 @@ static void gmap_zap_swap_entry(swp_entry_t entry, struct mm_struct *mm)
else if (is_migration_entry(entry)) { else if (is_migration_entry(entry)) {
struct page *page = migration_entry_to_page(entry); struct page *page = migration_entry_to_page(entry);
if (PageAnon(page)) dec_mm_counter(mm, mm_counter(page));
dec_mm_counter(mm, MM_ANONPAGES);
else
dec_mm_counter(mm, MM_FILEPAGES);
} }
free_swap_and_cache(entry); free_swap_and_cache(entry);
} }
......
...@@ -83,7 +83,8 @@ unsigned long task_statm(struct mm_struct *mm, ...@@ -83,7 +83,8 @@ unsigned long task_statm(struct mm_struct *mm,
unsigned long *shared, unsigned long *text, unsigned long *shared, unsigned long *text,
unsigned long *data, unsigned long *resident) unsigned long *data, unsigned long *resident)
{ {
*shared = get_mm_counter(mm, MM_FILEPAGES); *shared = get_mm_counter(mm, MM_FILEPAGES) +
get_mm_counter(mm, MM_SHMEMPAGES);
*text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
>> PAGE_SHIFT; >> PAGE_SHIFT;
*data = mm->total_vm - mm->shared_vm; *data = mm->total_vm - mm->shared_vm;
......
...@@ -1361,10 +1361,26 @@ static inline void dec_mm_counter(struct mm_struct *mm, int member) ...@@ -1361,10 +1361,26 @@ static inline void dec_mm_counter(struct mm_struct *mm, int member)
atomic_long_dec(&mm->rss_stat.count[member]); atomic_long_dec(&mm->rss_stat.count[member]);
} }
/* Optimized variant when page is already known not to be PageAnon */
static inline int mm_counter_file(struct page *page)
{
if (PageSwapBacked(page))
return MM_SHMEMPAGES;
return MM_FILEPAGES;
}
static inline int mm_counter(struct page *page)
{
if (PageAnon(page))
return MM_ANONPAGES;
return mm_counter_file(page);
}
static inline unsigned long get_mm_rss(struct mm_struct *mm) static inline unsigned long get_mm_rss(struct mm_struct *mm)
{ {
return get_mm_counter(mm, MM_FILEPAGES) + return get_mm_counter(mm, MM_FILEPAGES) +
get_mm_counter(mm, MM_ANONPAGES); get_mm_counter(mm, MM_ANONPAGES) +
get_mm_counter(mm, MM_SHMEMPAGES);
} }
static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm) static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
......
...@@ -369,9 +369,10 @@ struct core_state { ...@@ -369,9 +369,10 @@ struct core_state {
}; };
enum { enum {
MM_FILEPAGES, MM_FILEPAGES, /* Resident file mapping pages */
MM_ANONPAGES, MM_ANONPAGES, /* Resident anonymous pages */
MM_SWAPENTS, MM_SWAPENTS, /* Anonymous swap entries */
MM_SHMEMPAGES, /* Resident shared memory pages */
NR_MM_COUNTERS NR_MM_COUNTERS
}; };
......
...@@ -180,7 +180,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr, ...@@ -180,7 +180,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
lru_cache_add_active_or_unevictable(kpage, vma); lru_cache_add_active_or_unevictable(kpage, vma);
if (!PageAnon(page)) { if (!PageAnon(page)) {
dec_mm_counter(mm, MM_FILEPAGES); dec_mm_counter(mm, mm_counter_file(page));
inc_mm_counter(mm, MM_ANONPAGES); inc_mm_counter(mm, MM_ANONPAGES);
} }
......
...@@ -832,10 +832,7 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, ...@@ -832,10 +832,7 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
} else if (is_migration_entry(entry)) { } else if (is_migration_entry(entry)) {
page = migration_entry_to_page(entry); page = migration_entry_to_page(entry);
if (PageAnon(page)) rss[mm_counter(page)]++;
rss[MM_ANONPAGES]++;
else
rss[MM_FILEPAGES]++;
if (is_write_migration_entry(entry) && if (is_write_migration_entry(entry) &&
is_cow_mapping(vm_flags)) { is_cow_mapping(vm_flags)) {
...@@ -874,10 +871,7 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, ...@@ -874,10 +871,7 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
if (page) { if (page) {
get_page(page); get_page(page);
page_dup_rmap(page); page_dup_rmap(page);
if (PageAnon(page)) rss[mm_counter(page)]++;
rss[MM_ANONPAGES]++;
else
rss[MM_FILEPAGES]++;
} }
out_set_pte: out_set_pte:
...@@ -1113,9 +1107,8 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, ...@@ -1113,9 +1107,8 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
tlb_remove_tlb_entry(tlb, pte, addr); tlb_remove_tlb_entry(tlb, pte, addr);
if (unlikely(!page)) if (unlikely(!page))
continue; continue;
if (PageAnon(page))
rss[MM_ANONPAGES]--; if (!PageAnon(page)) {
else {
if (pte_dirty(ptent)) { if (pte_dirty(ptent)) {
force_flush = 1; force_flush = 1;
set_page_dirty(page); set_page_dirty(page);
...@@ -1123,8 +1116,8 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, ...@@ -1123,8 +1116,8 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
if (pte_young(ptent) && if (pte_young(ptent) &&
likely(!(vma->vm_flags & VM_SEQ_READ))) likely(!(vma->vm_flags & VM_SEQ_READ)))
mark_page_accessed(page); mark_page_accessed(page);
rss[MM_FILEPAGES]--;
} }
rss[mm_counter(page)]--;
page_remove_rmap(page); page_remove_rmap(page);
if (unlikely(page_mapcount(page) < 0)) if (unlikely(page_mapcount(page) < 0))
print_bad_pte(vma, addr, ptent, page); print_bad_pte(vma, addr, ptent, page);
...@@ -1146,11 +1139,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, ...@@ -1146,11 +1139,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb,
struct page *page; struct page *page;
page = migration_entry_to_page(entry); page = migration_entry_to_page(entry);
rss[mm_counter(page)]--;
if (PageAnon(page))
rss[MM_ANONPAGES]--;
else
rss[MM_FILEPAGES]--;
} }
if (unlikely(!free_swap_and_cache(entry))) if (unlikely(!free_swap_and_cache(entry)))
print_bad_pte(vma, addr, ptent, NULL); print_bad_pte(vma, addr, ptent, NULL);
...@@ -1460,7 +1449,7 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr, ...@@ -1460,7 +1449,7 @@ static int insert_page(struct vm_area_struct *vma, unsigned long addr,
/* Ok, finally just insert the thing.. */ /* Ok, finally just insert the thing.. */
get_page(page); get_page(page);
inc_mm_counter_fast(mm, MM_FILEPAGES); inc_mm_counter_fast(mm, mm_counter_file(page));
page_add_file_rmap(page); page_add_file_rmap(page);
set_pte_at(mm, addr, pte, mk_pte(page, prot)); set_pte_at(mm, addr, pte, mk_pte(page, prot));
...@@ -2097,7 +2086,8 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma, ...@@ -2097,7 +2086,8 @@ static int wp_page_copy(struct mm_struct *mm, struct vm_area_struct *vma,
if (likely(pte_same(*page_table, orig_pte))) { if (likely(pte_same(*page_table, orig_pte))) {
if (old_page) { if (old_page) {
if (!PageAnon(old_page)) { if (!PageAnon(old_page)) {
dec_mm_counter_fast(mm, MM_FILEPAGES); dec_mm_counter_fast(mm,
mm_counter_file(old_page));
inc_mm_counter_fast(mm, MM_ANONPAGES); inc_mm_counter_fast(mm, MM_ANONPAGES);
} }
} else { } else {
...@@ -2820,7 +2810,7 @@ void do_set_pte(struct vm_area_struct *vma, unsigned long address, ...@@ -2820,7 +2810,7 @@ void do_set_pte(struct vm_area_struct *vma, unsigned long address,
inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES); inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
page_add_new_anon_rmap(page, vma, address); page_add_new_anon_rmap(page, vma, address);
} else { } else {
inc_mm_counter_fast(vma->vm_mm, MM_FILEPAGES); inc_mm_counter_fast(vma->vm_mm, mm_counter_file(page));
page_add_file_rmap(page); page_add_file_rmap(page);
} }
set_pte_at(vma->vm_mm, address, pte, entry); set_pte_at(vma->vm_mm, address, pte, entry);
......
...@@ -585,10 +585,11 @@ void oom_kill_process(struct oom_control *oc, struct task_struct *p, ...@@ -585,10 +585,11 @@ void oom_kill_process(struct oom_control *oc, struct task_struct *p,
*/ */
do_send_sig_info(SIGKILL, SEND_SIG_FORCED, victim, true); do_send_sig_info(SIGKILL, SEND_SIG_FORCED, victim, true);
mark_oom_victim(victim); mark_oom_victim(victim);
pr_err("Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB\n", pr_err("Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n",
task_pid_nr(victim), victim->comm, K(victim->mm->total_vm), task_pid_nr(victim), victim->comm, K(victim->mm->total_vm),
K(get_mm_counter(victim->mm, MM_ANONPAGES)), K(get_mm_counter(victim->mm, MM_ANONPAGES)),
K(get_mm_counter(victim->mm, MM_FILEPAGES))); K(get_mm_counter(victim->mm, MM_FILEPAGES)),
K(get_mm_counter(victim->mm, MM_SHMEMPAGES)));
task_unlock(victim); task_unlock(victim);
/* /*
......
...@@ -1364,10 +1364,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1364,10 +1364,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
if (PageHuge(page)) { if (PageHuge(page)) {
hugetlb_count_sub(1 << compound_order(page), mm); hugetlb_count_sub(1 << compound_order(page), mm);
} else { } else {
if (PageAnon(page)) dec_mm_counter(mm, mm_counter(page));
dec_mm_counter(mm, MM_ANONPAGES);
else
dec_mm_counter(mm, MM_FILEPAGES);
} }
set_pte_at(mm, address, pte, set_pte_at(mm, address, pte,
swp_entry_to_pte(make_hwpoison_entry(page))); swp_entry_to_pte(make_hwpoison_entry(page)));
...@@ -1377,10 +1374,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1377,10 +1374,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
* interest anymore. Simply discard the pte, vmscan * interest anymore. Simply discard the pte, vmscan
* will take care of the rest. * will take care of the rest.
*/ */
if (PageAnon(page)) dec_mm_counter(mm, mm_counter(page));
dec_mm_counter(mm, MM_ANONPAGES);
else
dec_mm_counter(mm, MM_FILEPAGES);
} else if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION)) { } else if (IS_ENABLED(CONFIG_MIGRATION) && (flags & TTU_MIGRATION)) {
swp_entry_t entry; swp_entry_t entry;
pte_t swp_pte; pte_t swp_pte;
...@@ -1420,7 +1414,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma, ...@@ -1420,7 +1414,7 @@ static int try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
swp_pte = pte_swp_mksoft_dirty(swp_pte); swp_pte = pte_swp_mksoft_dirty(swp_pte);
set_pte_at(mm, address, pte, swp_pte); set_pte_at(mm, address, pte, swp_pte);
} else } else
dec_mm_counter(mm, MM_FILEPAGES); dec_mm_counter(mm, mm_counter_file(page));
page_remove_rmap(page); page_remove_rmap(page);
page_cache_release(page); page_cache_release(page);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment