Commit cfa52c0c authored by Karol Herbst's avatar Karol Herbst Committed by Ingo Molnar

x86/mm/kmmio: Fix mmiotrace for hugepages

Because Linux might use bigger pages than the 4K pages to handle those mmio
ioremaps, the kmmio code shouldn't rely on the pade id as it currently does.

Using the memory address instead of the page id lets us look up how big the
page is and what its base address is, so that we won't get a page fault
within the same page twice anymore.
Tested-by: default avatarPierre Moreau <pierre.morrow@free.fr>
Signed-off-by: default avatarKarol Herbst <nouveau@karolherbst.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Denys Vlasenko <dvlasenk@redhat.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Luis R. Rodriguez <mcgrof@suse.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Toshi Kani <toshi.kani@hp.com>
Cc: linux-mm@kvack.org
Cc: linux-x86_64@vger.kernel.org
Cc: nouveau@lists.freedesktop.org
Cc: pq@iki.fi
Cc: rostedt@goodmis.org
Link: http://lkml.kernel.org/r/1456966991-6861-1-git-send-email-nouveau@karolherbst.deSigned-off-by: default avatarIngo Molnar <mingo@kernel.org>
parent 405e1133
...@@ -33,7 +33,7 @@ ...@@ -33,7 +33,7 @@
struct kmmio_fault_page { struct kmmio_fault_page {
struct list_head list; struct list_head list;
struct kmmio_fault_page *release_next; struct kmmio_fault_page *release_next;
unsigned long page; /* location of the fault page */ unsigned long addr; /* the requested address */
pteval_t old_presence; /* page presence prior to arming */ pteval_t old_presence; /* page presence prior to arming */
bool armed; bool armed;
...@@ -70,9 +70,16 @@ unsigned int kmmio_count; ...@@ -70,9 +70,16 @@ unsigned int kmmio_count;
static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE]; static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE];
static LIST_HEAD(kmmio_probes); static LIST_HEAD(kmmio_probes);
static struct list_head *kmmio_page_list(unsigned long page) static struct list_head *kmmio_page_list(unsigned long addr)
{ {
return &kmmio_page_table[hash_long(page, KMMIO_PAGE_HASH_BITS)]; unsigned int l;
pte_t *pte = lookup_address(addr, &l);
if (!pte)
return NULL;
addr &= page_level_mask(l);
return &kmmio_page_table[hash_long(addr, KMMIO_PAGE_HASH_BITS)];
} }
/* Accessed per-cpu */ /* Accessed per-cpu */
...@@ -98,15 +105,19 @@ static struct kmmio_probe *get_kmmio_probe(unsigned long addr) ...@@ -98,15 +105,19 @@ static struct kmmio_probe *get_kmmio_probe(unsigned long addr)
} }
/* You must be holding RCU read lock. */ /* You must be holding RCU read lock. */
static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page) static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long addr)
{ {
struct list_head *head; struct list_head *head;
struct kmmio_fault_page *f; struct kmmio_fault_page *f;
unsigned int l;
pte_t *pte = lookup_address(addr, &l);
page &= PAGE_MASK; if (!pte)
head = kmmio_page_list(page); return NULL;
addr &= page_level_mask(l);
head = kmmio_page_list(addr);
list_for_each_entry_rcu(f, head, list) { list_for_each_entry_rcu(f, head, list) {
if (f->page == page) if (f->addr == addr)
return f; return f;
} }
return NULL; return NULL;
...@@ -137,10 +148,10 @@ static void clear_pte_presence(pte_t *pte, bool clear, pteval_t *old) ...@@ -137,10 +148,10 @@ static void clear_pte_presence(pte_t *pte, bool clear, pteval_t *old)
static int clear_page_presence(struct kmmio_fault_page *f, bool clear) static int clear_page_presence(struct kmmio_fault_page *f, bool clear)
{ {
unsigned int level; unsigned int level;
pte_t *pte = lookup_address(f->page, &level); pte_t *pte = lookup_address(f->addr, &level);
if (!pte) { if (!pte) {
pr_err("no pte for page 0x%08lx\n", f->page); pr_err("no pte for addr 0x%08lx\n", f->addr);
return -1; return -1;
} }
...@@ -156,7 +167,7 @@ static int clear_page_presence(struct kmmio_fault_page *f, bool clear) ...@@ -156,7 +167,7 @@ static int clear_page_presence(struct kmmio_fault_page *f, bool clear)
return -1; return -1;
} }
__flush_tlb_one(f->page); __flush_tlb_one(f->addr);
return 0; return 0;
} }
...@@ -176,12 +187,12 @@ static int arm_kmmio_fault_page(struct kmmio_fault_page *f) ...@@ -176,12 +187,12 @@ static int arm_kmmio_fault_page(struct kmmio_fault_page *f)
int ret; int ret;
WARN_ONCE(f->armed, KERN_ERR pr_fmt("kmmio page already armed.\n")); WARN_ONCE(f->armed, KERN_ERR pr_fmt("kmmio page already armed.\n"));
if (f->armed) { if (f->armed) {
pr_warning("double-arm: page 0x%08lx, ref %d, old %d\n", pr_warning("double-arm: addr 0x%08lx, ref %d, old %d\n",
f->page, f->count, !!f->old_presence); f->addr, f->count, !!f->old_presence);
} }
ret = clear_page_presence(f, true); ret = clear_page_presence(f, true);
WARN_ONCE(ret < 0, KERN_ERR pr_fmt("arming 0x%08lx failed.\n"), WARN_ONCE(ret < 0, KERN_ERR pr_fmt("arming at 0x%08lx failed.\n"),
f->page); f->addr);
f->armed = true; f->armed = true;
return ret; return ret;
} }
...@@ -191,7 +202,7 @@ static void disarm_kmmio_fault_page(struct kmmio_fault_page *f) ...@@ -191,7 +202,7 @@ static void disarm_kmmio_fault_page(struct kmmio_fault_page *f)
{ {
int ret = clear_page_presence(f, false); int ret = clear_page_presence(f, false);
WARN_ONCE(ret < 0, WARN_ONCE(ret < 0,
KERN_ERR "kmmio disarming 0x%08lx failed.\n", f->page); KERN_ERR "kmmio disarming at 0x%08lx failed.\n", f->addr);
f->armed = false; f->armed = false;
} }
...@@ -215,6 +226,12 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) ...@@ -215,6 +226,12 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr)
struct kmmio_context *ctx; struct kmmio_context *ctx;
struct kmmio_fault_page *faultpage; struct kmmio_fault_page *faultpage;
int ret = 0; /* default to fault not handled */ int ret = 0; /* default to fault not handled */
unsigned long page_base = addr;
unsigned int l;
pte_t *pte = lookup_address(addr, &l);
if (!pte)
return -EINVAL;
page_base &= page_level_mask(l);
/* /*
* Preemption is now disabled to prevent process switch during * Preemption is now disabled to prevent process switch during
...@@ -227,7 +244,7 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) ...@@ -227,7 +244,7 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr)
preempt_disable(); preempt_disable();
rcu_read_lock(); rcu_read_lock();
faultpage = get_kmmio_fault_page(addr); faultpage = get_kmmio_fault_page(page_base);
if (!faultpage) { if (!faultpage) {
/* /*
* Either this page fault is not caused by kmmio, or * Either this page fault is not caused by kmmio, or
...@@ -239,7 +256,7 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) ...@@ -239,7 +256,7 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr)
ctx = &get_cpu_var(kmmio_ctx); ctx = &get_cpu_var(kmmio_ctx);
if (ctx->active) { if (ctx->active) {
if (addr == ctx->addr) { if (page_base == ctx->addr) {
/* /*
* A second fault on the same page means some other * A second fault on the same page means some other
* condition needs handling by do_page_fault(), the * condition needs handling by do_page_fault(), the
...@@ -267,9 +284,9 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr) ...@@ -267,9 +284,9 @@ int kmmio_handler(struct pt_regs *regs, unsigned long addr)
ctx->active++; ctx->active++;
ctx->fpage = faultpage; ctx->fpage = faultpage;
ctx->probe = get_kmmio_probe(addr); ctx->probe = get_kmmio_probe(page_base);
ctx->saved_flags = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF)); ctx->saved_flags = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
ctx->addr = addr; ctx->addr = page_base;
if (ctx->probe && ctx->probe->pre_handler) if (ctx->probe && ctx->probe->pre_handler)
ctx->probe->pre_handler(ctx->probe, regs, addr); ctx->probe->pre_handler(ctx->probe, regs, addr);
...@@ -354,12 +371,11 @@ static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs) ...@@ -354,12 +371,11 @@ static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs)
} }
/* You must be holding kmmio_lock. */ /* You must be holding kmmio_lock. */
static int add_kmmio_fault_page(unsigned long page) static int add_kmmio_fault_page(unsigned long addr)
{ {
struct kmmio_fault_page *f; struct kmmio_fault_page *f;
page &= PAGE_MASK; f = get_kmmio_fault_page(addr);
f = get_kmmio_fault_page(page);
if (f) { if (f) {
if (!f->count) if (!f->count)
arm_kmmio_fault_page(f); arm_kmmio_fault_page(f);
...@@ -372,26 +388,25 @@ static int add_kmmio_fault_page(unsigned long page) ...@@ -372,26 +388,25 @@ static int add_kmmio_fault_page(unsigned long page)
return -1; return -1;
f->count = 1; f->count = 1;
f->page = page; f->addr = addr;
if (arm_kmmio_fault_page(f)) { if (arm_kmmio_fault_page(f)) {
kfree(f); kfree(f);
return -1; return -1;
} }
list_add_rcu(&f->list, kmmio_page_list(f->page)); list_add_rcu(&f->list, kmmio_page_list(f->addr));
return 0; return 0;
} }
/* You must be holding kmmio_lock. */ /* You must be holding kmmio_lock. */
static void release_kmmio_fault_page(unsigned long page, static void release_kmmio_fault_page(unsigned long addr,
struct kmmio_fault_page **release_list) struct kmmio_fault_page **release_list)
{ {
struct kmmio_fault_page *f; struct kmmio_fault_page *f;
page &= PAGE_MASK; f = get_kmmio_fault_page(addr);
f = get_kmmio_fault_page(page);
if (!f) if (!f)
return; return;
...@@ -420,18 +435,27 @@ int register_kmmio_probe(struct kmmio_probe *p) ...@@ -420,18 +435,27 @@ int register_kmmio_probe(struct kmmio_probe *p)
int ret = 0; int ret = 0;
unsigned long size = 0; unsigned long size = 0;
const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK); const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK);
unsigned int l;
pte_t *pte;
spin_lock_irqsave(&kmmio_lock, flags); spin_lock_irqsave(&kmmio_lock, flags);
if (get_kmmio_probe(p->addr)) { if (get_kmmio_probe(p->addr)) {
ret = -EEXIST; ret = -EEXIST;
goto out; goto out;
} }
pte = lookup_address(p->addr, &l);
if (!pte) {
ret = -EINVAL;
goto out;
}
kmmio_count++; kmmio_count++;
list_add_rcu(&p->list, &kmmio_probes); list_add_rcu(&p->list, &kmmio_probes);
while (size < size_lim) { while (size < size_lim) {
if (add_kmmio_fault_page(p->addr + size)) if (add_kmmio_fault_page(p->addr + size))
pr_err("Unable to set page fault.\n"); pr_err("Unable to set page fault.\n");
size += PAGE_SIZE; size += page_level_size(l);
} }
out: out:
spin_unlock_irqrestore(&kmmio_lock, flags); spin_unlock_irqrestore(&kmmio_lock, flags);
...@@ -506,11 +530,17 @@ void unregister_kmmio_probe(struct kmmio_probe *p) ...@@ -506,11 +530,17 @@ void unregister_kmmio_probe(struct kmmio_probe *p)
const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK); const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK);
struct kmmio_fault_page *release_list = NULL; struct kmmio_fault_page *release_list = NULL;
struct kmmio_delayed_release *drelease; struct kmmio_delayed_release *drelease;
unsigned int l;
pte_t *pte;
pte = lookup_address(p->addr, &l);
if (!pte)
return;
spin_lock_irqsave(&kmmio_lock, flags); spin_lock_irqsave(&kmmio_lock, flags);
while (size < size_lim) { while (size < size_lim) {
release_kmmio_fault_page(p->addr + size, &release_list); release_kmmio_fault_page(p->addr + size, &release_list);
size += PAGE_SIZE; size += page_level_size(l);
} }
list_del_rcu(&p->list); list_del_rcu(&p->list);
kmmio_count--; kmmio_count--;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment