Commit 1a5bae25 authored by Anshuman Khandual's avatar Anshuman Khandual Committed by Linus Torvalds

mm/vmstat: add events for THP migration without split

Add following new vmstat events which will help in validating THP
migration without split.  Statistics reported through these new VM events
will help in performance debugging.

1. THP_MIGRATION_SUCCESS
2. THP_MIGRATION_FAILURE
3. THP_MIGRATION_SPLIT

In addition, these new events also update normal page migration statistics
appropriately via PGMIGRATE_SUCCESS and PGMIGRATE_FAILURE.  While here,
this updates current trace event 'mm_migrate_pages' to accommodate now
available THP statistics.

[akpm@linux-foundation.org: s/hpage_nr_pages/thp_nr_pages/]
[ziy@nvidia.com: v2]
  Link: http://lkml.kernel.org/r/C5E3C65C-8253-4638-9D3C-71A61858BB8B@nvidia.com
[anshuman.khandual@arm.com: s/thp_nr_pages/hpage_nr_pages/]
  Link: http://lkml.kernel.org/r/1594287583-16568-1-git-send-email-anshuman.khandual@arm.comSigned-off-by: default avatarAnshuman Khandual <anshuman.khandual@arm.com>
Signed-off-by: default avatarZi Yan <ziy@nvidia.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Reviewed-by: default avatarDaniel Jordan <daniel.m.jordan@oracle.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Zi Yan <ziy@nvidia.com>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Link: http://lkml.kernel.org/r/1594080415-27924-1-git-send-email-anshuman.khandual@arm.comSigned-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 4958e4d8
...@@ -253,5 +253,32 @@ which are function pointers of struct address_space_operations. ...@@ -253,5 +253,32 @@ which are function pointers of struct address_space_operations.
PG_isolated is alias with PG_reclaim flag so driver shouldn't use the flag PG_isolated is alias with PG_reclaim flag so driver shouldn't use the flag
for own purpose. for own purpose.
Monitoring Migration
=====================
The following events (counters) can be used to monitor page migration.
1. PGMIGRATE_SUCCESS: Normal page migration success. Each count means that a
page was migrated. If the page was a non-THP page, then this counter is
increased by one. If the page was a THP, then this counter is increased by
the number of THP subpages. For example, migration of a single 2MB THP that
has 4KB-size base pages (subpages) will cause this counter to increase by
512.
2. PGMIGRATE_FAIL: Normal page migration failure. Same counting rules as for
_SUCCESS, above: this will be increased by the number of subpages, if it was
a THP.
3. THP_MIGRATION_SUCCESS: A THP was migrated without being split.
4. THP_MIGRATION_FAIL: A THP could not be migrated nor it could be split.
5. THP_MIGRATION_SPLIT: A THP was migrated, but not as such: first, the THP had
to be split. After splitting, a migration retry was used for it's sub-pages.
THP_MIGRATION_* events also update the appropriate PGMIGRATE_SUCCESS or
PGMIGRATE_FAIL events. For example, a THP migration failure will cause both
THP_MIGRATION_FAIL and PGMIGRATE_FAIL to increase.
Christoph Lameter, May 8, 2006. Christoph Lameter, May 8, 2006.
Minchan Kim, Mar 28, 2016. Minchan Kim, Mar 28, 2016.
...@@ -56,6 +56,9 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, ...@@ -56,6 +56,9 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
#endif #endif
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
PGMIGRATE_SUCCESS, PGMIGRATE_FAIL, PGMIGRATE_SUCCESS, PGMIGRATE_FAIL,
THP_MIGRATION_SUCCESS,
THP_MIGRATION_FAIL,
THP_MIGRATION_SPLIT,
#endif #endif
#ifdef CONFIG_COMPACTION #ifdef CONFIG_COMPACTION
COMPACTMIGRATE_SCANNED, COMPACTFREE_SCANNED, COMPACTMIGRATE_SCANNED, COMPACTFREE_SCANNED,
......
...@@ -46,13 +46,18 @@ MIGRATE_REASON ...@@ -46,13 +46,18 @@ MIGRATE_REASON
TRACE_EVENT(mm_migrate_pages, TRACE_EVENT(mm_migrate_pages,
TP_PROTO(unsigned long succeeded, unsigned long failed, TP_PROTO(unsigned long succeeded, unsigned long failed,
enum migrate_mode mode, int reason), unsigned long thp_succeeded, unsigned long thp_failed,
unsigned long thp_split, enum migrate_mode mode, int reason),
TP_ARGS(succeeded, failed, mode, reason), TP_ARGS(succeeded, failed, thp_succeeded, thp_failed,
thp_split, mode, reason),
TP_STRUCT__entry( TP_STRUCT__entry(
__field( unsigned long, succeeded) __field( unsigned long, succeeded)
__field( unsigned long, failed) __field( unsigned long, failed)
__field( unsigned long, thp_succeeded)
__field( unsigned long, thp_failed)
__field( unsigned long, thp_split)
__field( enum migrate_mode, mode) __field( enum migrate_mode, mode)
__field( int, reason) __field( int, reason)
), ),
...@@ -60,13 +65,19 @@ TRACE_EVENT(mm_migrate_pages, ...@@ -60,13 +65,19 @@ TRACE_EVENT(mm_migrate_pages,
TP_fast_assign( TP_fast_assign(
__entry->succeeded = succeeded; __entry->succeeded = succeeded;
__entry->failed = failed; __entry->failed = failed;
__entry->thp_succeeded = thp_succeeded;
__entry->thp_failed = thp_failed;
__entry->thp_split = thp_split;
__entry->mode = mode; __entry->mode = mode;
__entry->reason = reason; __entry->reason = reason;
), ),
TP_printk("nr_succeeded=%lu nr_failed=%lu mode=%s reason=%s", TP_printk("nr_succeeded=%lu nr_failed=%lu nr_thp_succeeded=%lu nr_thp_failed=%lu nr_thp_split=%lu mode=%s reason=%s",
__entry->succeeded, __entry->succeeded,
__entry->failed, __entry->failed,
__entry->thp_succeeded,
__entry->thp_failed,
__entry->thp_split,
__print_symbolic(__entry->mode, MIGRATE_MODE), __print_symbolic(__entry->mode, MIGRATE_MODE),
__print_symbolic(__entry->reason, MIGRATE_REASON)) __print_symbolic(__entry->reason, MIGRATE_REASON))
); );
......
...@@ -1418,22 +1418,35 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, ...@@ -1418,22 +1418,35 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page,
enum migrate_mode mode, int reason) enum migrate_mode mode, int reason)
{ {
int retry = 1; int retry = 1;
int thp_retry = 1;
int nr_failed = 0; int nr_failed = 0;
int nr_succeeded = 0; int nr_succeeded = 0;
int nr_thp_succeeded = 0;
int nr_thp_failed = 0;
int nr_thp_split = 0;
int pass = 0; int pass = 0;
bool is_thp = false;
struct page *page; struct page *page;
struct page *page2; struct page *page2;
int swapwrite = current->flags & PF_SWAPWRITE; int swapwrite = current->flags & PF_SWAPWRITE;
int rc; int rc, nr_subpages;
if (!swapwrite) if (!swapwrite)
current->flags |= PF_SWAPWRITE; current->flags |= PF_SWAPWRITE;
for(pass = 0; pass < 10 && retry; pass++) { for (pass = 0; pass < 10 && (retry || thp_retry); pass++) {
retry = 0; retry = 0;
thp_retry = 0;
list_for_each_entry_safe(page, page2, from, lru) { list_for_each_entry_safe(page, page2, from, lru) {
retry: retry:
/*
* THP statistics is based on the source huge page.
* Capture required information that might get lost
* during migration.
*/
is_thp = PageTransHuge(page);
nr_subpages = hpage_nr_pages(page);
cond_resched(); cond_resched();
if (PageHuge(page)) if (PageHuge(page))
...@@ -1464,15 +1477,30 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, ...@@ -1464,15 +1477,30 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page,
unlock_page(page); unlock_page(page);
if (!rc) { if (!rc) {
list_safe_reset_next(page, page2, lru); list_safe_reset_next(page, page2, lru);
nr_thp_split++;
goto retry; goto retry;
} }
} }
if (is_thp) {
nr_thp_failed++;
nr_failed += nr_subpages;
goto out;
}
nr_failed++; nr_failed++;
goto out; goto out;
case -EAGAIN: case -EAGAIN:
if (is_thp) {
thp_retry++;
break;
}
retry++; retry++;
break; break;
case MIGRATEPAGE_SUCCESS: case MIGRATEPAGE_SUCCESS:
if (is_thp) {
nr_thp_succeeded++;
nr_succeeded += nr_subpages;
break;
}
nr_succeeded++; nr_succeeded++;
break; break;
default: default:
...@@ -1482,19 +1510,27 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page, ...@@ -1482,19 +1510,27 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page,
* removed from migration page list and not * removed from migration page list and not
* retried in the next outer loop. * retried in the next outer loop.
*/ */
if (is_thp) {
nr_thp_failed++;
nr_failed += nr_subpages;
break;
}
nr_failed++; nr_failed++;
break; break;
} }
} }
} }
nr_failed += retry; nr_failed += retry + thp_retry;
nr_thp_failed += thp_retry;
rc = nr_failed; rc = nr_failed;
out: out:
if (nr_succeeded)
count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded); count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
if (nr_failed)
count_vm_events(PGMIGRATE_FAIL, nr_failed); count_vm_events(PGMIGRATE_FAIL, nr_failed);
trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason); count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded);
count_vm_events(THP_MIGRATION_FAIL, nr_thp_failed);
count_vm_events(THP_MIGRATION_SPLIT, nr_thp_split);
trace_mm_migrate_pages(nr_succeeded, nr_failed, nr_thp_succeeded,
nr_thp_failed, nr_thp_split, mode, reason);
if (!swapwrite) if (!swapwrite)
current->flags &= ~PF_SWAPWRITE; current->flags &= ~PF_SWAPWRITE;
......
...@@ -1277,6 +1277,9 @@ const char * const vmstat_text[] = { ...@@ -1277,6 +1277,9 @@ const char * const vmstat_text[] = {
#ifdef CONFIG_MIGRATION #ifdef CONFIG_MIGRATION
"pgmigrate_success", "pgmigrate_success",
"pgmigrate_fail", "pgmigrate_fail",
"thp_migration_success",
"thp_migration_fail",
"thp_migration_split",
#endif #endif
#ifdef CONFIG_COMPACTION #ifdef CONFIG_COMPACTION
"compact_migrate_scanned", "compact_migrate_scanned",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment