Commit 7dd80b8a authored by Vlastimil Babka's avatar Vlastimil Babka Committed by Linus Torvalds

mm, page_owner: convert page_owner_inited to static key

CONFIG_PAGE_OWNER attempts to impose negligible runtime overhead when
enabled during compilation, but not actually enabled during runtime by
boot param page_owner=on.  This overhead can be further reduced using
the static key mechanism, which this patch does.
Signed-off-by: default avatarVlastimil Babka <vbabka@suse.cz>
Acked-by: default avatarMichal Hocko <mhocko@suse.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Minchan Kim <minchan@kernel.org>
Cc: Sasha Levin <sasha.levin@oracle.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Mel Gorman <mgorman@suse.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 60f30350
...@@ -28,10 +28,11 @@ with page owner and page owner is disabled in runtime due to no enabling ...@@ -28,10 +28,11 @@ with page owner and page owner is disabled in runtime due to no enabling
boot option, runtime overhead is marginal. If disabled in runtime, it boot option, runtime overhead is marginal. If disabled in runtime, it
doesn't require memory to store owner information, so there is no runtime doesn't require memory to store owner information, so there is no runtime
memory overhead. And, page owner inserts just two unlikely branches into memory overhead. And, page owner inserts just two unlikely branches into
the page allocator hotpath and if it returns false then allocation is the page allocator hotpath and if not enabled, then allocation is done
done like as the kernel without page owner. These two unlikely branches like as the kernel without page owner. These two unlikely branches should
would not affect to allocation performance. Following is the kernel's not affect to allocation performance, especially if the static keys jump
code size change due to this facility. label patching functionality is available. Following is the kernel's code
size change due to this facility.
- Without page owner - Without page owner
text data bss dec hex filename text data bss dec hex filename
......
#ifndef __LINUX_PAGE_OWNER_H #ifndef __LINUX_PAGE_OWNER_H
#define __LINUX_PAGE_OWNER_H #define __LINUX_PAGE_OWNER_H
#include <linux/jump_label.h>
#ifdef CONFIG_PAGE_OWNER #ifdef CONFIG_PAGE_OWNER
extern bool page_owner_inited; extern struct static_key_false page_owner_inited;
extern struct page_ext_operations page_owner_ops; extern struct page_ext_operations page_owner_ops;
extern void __reset_page_owner(struct page *page, unsigned int order); extern void __reset_page_owner(struct page *page, unsigned int order);
...@@ -12,27 +14,23 @@ extern gfp_t __get_page_owner_gfp(struct page *page); ...@@ -12,27 +14,23 @@ extern gfp_t __get_page_owner_gfp(struct page *page);
static inline void reset_page_owner(struct page *page, unsigned int order) static inline void reset_page_owner(struct page *page, unsigned int order)
{ {
if (likely(!page_owner_inited)) if (static_branch_unlikely(&page_owner_inited))
return; __reset_page_owner(page, order);
__reset_page_owner(page, order);
} }
static inline void set_page_owner(struct page *page, static inline void set_page_owner(struct page *page,
unsigned int order, gfp_t gfp_mask) unsigned int order, gfp_t gfp_mask)
{ {
if (likely(!page_owner_inited)) if (static_branch_unlikely(&page_owner_inited))
return; __set_page_owner(page, order, gfp_mask);
__set_page_owner(page, order, gfp_mask);
} }
static inline gfp_t get_page_owner_gfp(struct page *page) static inline gfp_t get_page_owner_gfp(struct page *page)
{ {
if (likely(!page_owner_inited)) if (static_branch_unlikely(&page_owner_inited))
return __get_page_owner_gfp(page);
else
return 0; return 0;
return __get_page_owner_gfp(page);
} }
#else #else
static inline void reset_page_owner(struct page *page, unsigned int order) static inline void reset_page_owner(struct page *page, unsigned int order)
......
...@@ -5,10 +5,11 @@ ...@@ -5,10 +5,11 @@
#include <linux/bootmem.h> #include <linux/bootmem.h>
#include <linux/stacktrace.h> #include <linux/stacktrace.h>
#include <linux/page_owner.h> #include <linux/page_owner.h>
#include <linux/jump_label.h>
#include "internal.h" #include "internal.h"
static bool page_owner_disabled = true; static bool page_owner_disabled = true;
bool page_owner_inited __read_mostly; DEFINE_STATIC_KEY_FALSE(page_owner_inited);
static void init_early_allocated_pages(void); static void init_early_allocated_pages(void);
...@@ -37,7 +38,7 @@ static void init_page_owner(void) ...@@ -37,7 +38,7 @@ static void init_page_owner(void)
if (page_owner_disabled) if (page_owner_disabled)
return; return;
page_owner_inited = true; static_branch_enable(&page_owner_inited);
init_early_allocated_pages(); init_early_allocated_pages();
} }
...@@ -147,7 +148,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) ...@@ -147,7 +148,7 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
struct page *page; struct page *page;
struct page_ext *page_ext; struct page_ext *page_ext;
if (!page_owner_inited) if (!static_branch_unlikely(&page_owner_inited))
return -EINVAL; return -EINVAL;
page = NULL; page = NULL;
...@@ -295,7 +296,7 @@ static int __init pageowner_init(void) ...@@ -295,7 +296,7 @@ static int __init pageowner_init(void)
{ {
struct dentry *dentry; struct dentry *dentry;
if (!page_owner_inited) { if (!static_branch_unlikely(&page_owner_inited)) {
pr_info("page_owner is disabled\n"); pr_info("page_owner is disabled\n");
return 0; return 0;
} }
......
...@@ -1120,7 +1120,7 @@ static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat) ...@@ -1120,7 +1120,7 @@ static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat)
#ifdef CONFIG_PAGE_OWNER #ifdef CONFIG_PAGE_OWNER
int mtype; int mtype;
if (!page_owner_inited) if (!static_branch_unlikely(&page_owner_inited))
return; return;
drain_all_pages(NULL); drain_all_pages(NULL);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment