Commit 60c8aec6 authored by Marcelo Tosatti's avatar Marcelo Tosatti Committed by Avi Kivity

KVM: MMU: use page array in unsync walk

Instead of invoking the handler directly collect pages into
an array so the caller can work with it.

Simplifies TLB flush collapsing.
Signed-off-by: default avatarMarcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: default avatarAvi Kivity <avi@redhat.com>
parent fbce554e
...@@ -200,7 +200,7 @@ struct kvm_mmu_page { ...@@ -200,7 +200,7 @@ struct kvm_mmu_page {
int multimapped; /* More than one parent_pte? */ int multimapped; /* More than one parent_pte? */
int root_count; /* Currently serving as active root */ int root_count; /* Currently serving as active root */
bool unsync; bool unsync;
bool unsync_children; unsigned int unsync_children;
union { union {
u64 *parent_pte; /* !multimapped */ u64 *parent_pte; /* !multimapped */
struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */ struct hlist_head parent_ptes; /* multimapped, kvm_pte_chain */
......
...@@ -908,8 +908,9 @@ static void kvm_mmu_update_unsync_bitmap(u64 *spte) ...@@ -908,8 +908,9 @@ static void kvm_mmu_update_unsync_bitmap(u64 *spte)
struct kvm_mmu_page *sp = page_header(__pa(spte)); struct kvm_mmu_page *sp = page_header(__pa(spte));
index = spte - sp->spt; index = spte - sp->spt;
__set_bit(index, sp->unsync_child_bitmap); if (!__test_and_set_bit(index, sp->unsync_child_bitmap))
sp->unsync_children = 1; sp->unsync_children++;
WARN_ON(!sp->unsync_children);
} }
static void kvm_mmu_update_parents_unsync(struct kvm_mmu_page *sp) static void kvm_mmu_update_parents_unsync(struct kvm_mmu_page *sp)
...@@ -936,7 +937,6 @@ static void kvm_mmu_update_parents_unsync(struct kvm_mmu_page *sp) ...@@ -936,7 +937,6 @@ static void kvm_mmu_update_parents_unsync(struct kvm_mmu_page *sp)
static int unsync_walk_fn(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) static int unsync_walk_fn(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
{ {
sp->unsync_children = 1;
kvm_mmu_update_parents_unsync(sp); kvm_mmu_update_parents_unsync(sp);
return 1; return 1;
} }
...@@ -967,18 +967,41 @@ static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva) ...@@ -967,18 +967,41 @@ static void nonpaging_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{ {
} }
#define KVM_PAGE_ARRAY_NR 16
struct kvm_mmu_pages {
struct mmu_page_and_offset {
struct kvm_mmu_page *sp;
unsigned int idx;
} page[KVM_PAGE_ARRAY_NR];
unsigned int nr;
};
#define for_each_unsync_children(bitmap, idx) \ #define for_each_unsync_children(bitmap, idx) \
for (idx = find_first_bit(bitmap, 512); \ for (idx = find_first_bit(bitmap, 512); \
idx < 512; \ idx < 512; \
idx = find_next_bit(bitmap, 512, idx+1)) idx = find_next_bit(bitmap, 512, idx+1))
static int mmu_unsync_walk(struct kvm_mmu_page *sp, int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
struct kvm_unsync_walk *walker) int idx)
{ {
int i, ret; int i;
if (!sp->unsync_children) if (sp->unsync)
return 0; for (i=0; i < pvec->nr; i++)
if (pvec->page[i].sp == sp)
return 0;
pvec->page[pvec->nr].sp = sp;
pvec->page[pvec->nr].idx = idx;
pvec->nr++;
return (pvec->nr == KVM_PAGE_ARRAY_NR);
}
static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
struct kvm_mmu_pages *pvec)
{
int i, ret, nr_unsync_leaf = 0;
for_each_unsync_children(sp->unsync_child_bitmap, i) { for_each_unsync_children(sp->unsync_child_bitmap, i) {
u64 ent = sp->spt[i]; u64 ent = sp->spt[i];
...@@ -988,17 +1011,22 @@ static int mmu_unsync_walk(struct kvm_mmu_page *sp, ...@@ -988,17 +1011,22 @@ static int mmu_unsync_walk(struct kvm_mmu_page *sp,
child = page_header(ent & PT64_BASE_ADDR_MASK); child = page_header(ent & PT64_BASE_ADDR_MASK);
if (child->unsync_children) { if (child->unsync_children) {
ret = mmu_unsync_walk(child, walker); if (mmu_pages_add(pvec, child, i))
if (ret) return -ENOSPC;
ret = __mmu_unsync_walk(child, pvec);
if (!ret)
__clear_bit(i, sp->unsync_child_bitmap);
else if (ret > 0)
nr_unsync_leaf += ret;
else
return ret; return ret;
__clear_bit(i, sp->unsync_child_bitmap);
} }
if (child->unsync) { if (child->unsync) {
ret = walker->entry(child, walker); nr_unsync_leaf++;
__clear_bit(i, sp->unsync_child_bitmap); if (mmu_pages_add(pvec, child, i))
if (ret) return -ENOSPC;
return ret;
} }
} }
} }
...@@ -1006,7 +1034,17 @@ static int mmu_unsync_walk(struct kvm_mmu_page *sp, ...@@ -1006,7 +1034,17 @@ static int mmu_unsync_walk(struct kvm_mmu_page *sp,
if (find_first_bit(sp->unsync_child_bitmap, 512) == 512) if (find_first_bit(sp->unsync_child_bitmap, 512) == 512)
sp->unsync_children = 0; sp->unsync_children = 0;
return 0; return nr_unsync_leaf;
}
static int mmu_unsync_walk(struct kvm_mmu_page *sp,
struct kvm_mmu_pages *pvec)
{
if (!sp->unsync_children)
return 0;
mmu_pages_add(pvec, sp, 0);
return __mmu_unsync_walk(sp, pvec);
} }
static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn)
...@@ -1056,30 +1094,81 @@ static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) ...@@ -1056,30 +1094,81 @@ static int kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
return 0; return 0;
} }
struct sync_walker { struct mmu_page_path {
struct kvm_vcpu *vcpu; struct kvm_mmu_page *parent[PT64_ROOT_LEVEL-1];
struct kvm_unsync_walk walker; unsigned int idx[PT64_ROOT_LEVEL-1];
}; };
static int mmu_sync_fn(struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk) #define for_each_sp(pvec, sp, parents, i) \
for (i = mmu_pages_next(&pvec, &parents, -1), \
sp = pvec.page[i].sp; \
i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \
i = mmu_pages_next(&pvec, &parents, i))
int mmu_pages_next(struct kvm_mmu_pages *pvec, struct mmu_page_path *parents,
int i)
{ {
struct sync_walker *sync_walk = container_of(walk, struct sync_walker, int n;
walker);
struct kvm_vcpu *vcpu = sync_walk->vcpu; for (n = i+1; n < pvec->nr; n++) {
struct kvm_mmu_page *sp = pvec->page[n].sp;
if (sp->role.level == PT_PAGE_TABLE_LEVEL) {
parents->idx[0] = pvec->page[n].idx;
return n;
}
kvm_sync_page(vcpu, sp); parents->parent[sp->role.level-2] = sp;
return (need_resched() || spin_needbreak(&vcpu->kvm->mmu_lock)); parents->idx[sp->role.level-1] = pvec->page[n].idx;
}
return n;
} }
static void mmu_sync_children(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) void mmu_pages_clear_parents(struct mmu_page_path *parents)
{ {
struct sync_walker walker = { struct kvm_mmu_page *sp;
.walker = { .entry = mmu_sync_fn, }, unsigned int level = 0;
.vcpu = vcpu,
}; do {
unsigned int idx = parents->idx[level];
sp = parents->parent[level];
if (!sp)
return;
--sp->unsync_children;
WARN_ON((int)sp->unsync_children < 0);
__clear_bit(idx, sp->unsync_child_bitmap);
level++;
} while (level < PT64_ROOT_LEVEL-1 && !sp->unsync_children);
}
static void kvm_mmu_pages_init(struct kvm_mmu_page *parent,
struct mmu_page_path *parents,
struct kvm_mmu_pages *pvec)
{
parents->parent[parent->role.level-1] = NULL;
pvec->nr = 0;
}
while (mmu_unsync_walk(sp, &walker.walker)) static void mmu_sync_children(struct kvm_vcpu *vcpu,
struct kvm_mmu_page *parent)
{
int i;
struct kvm_mmu_page *sp;
struct mmu_page_path parents;
struct kvm_mmu_pages pages;
kvm_mmu_pages_init(parent, &parents, &pages);
while (mmu_unsync_walk(parent, &pages)) {
for_each_sp(pages, sp, parents, i) {
kvm_sync_page(vcpu, sp);
mmu_pages_clear_parents(&parents);
}
cond_resched_lock(&vcpu->kvm->mmu_lock); cond_resched_lock(&vcpu->kvm->mmu_lock);
kvm_mmu_pages_init(parent, &parents, &pages);
}
} }
static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
...@@ -1245,33 +1334,29 @@ static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp) ...@@ -1245,33 +1334,29 @@ static void kvm_mmu_unlink_parents(struct kvm *kvm, struct kvm_mmu_page *sp)
} }
} }
struct zap_walker { static int mmu_zap_unsync_children(struct kvm *kvm,
struct kvm_unsync_walk walker; struct kvm_mmu_page *parent)
struct kvm *kvm;
int zapped;
};
static int mmu_zap_fn(struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk)
{
struct zap_walker *zap_walk = container_of(walk, struct zap_walker,
walker);
kvm_mmu_zap_page(zap_walk->kvm, sp);
zap_walk->zapped = 1;
return 0;
}
static int mmu_zap_unsync_children(struct kvm *kvm, struct kvm_mmu_page *sp)
{ {
struct zap_walker walker = { int i, zapped = 0;
.walker = { .entry = mmu_zap_fn, }, struct mmu_page_path parents;
.kvm = kvm, struct kvm_mmu_pages pages;
.zapped = 0,
};
if (sp->role.level == PT_PAGE_TABLE_LEVEL) if (parent->role.level == PT_PAGE_TABLE_LEVEL)
return 0; return 0;
mmu_unsync_walk(sp, &walker.walker);
return walker.zapped; kvm_mmu_pages_init(parent, &parents, &pages);
while (mmu_unsync_walk(parent, &pages)) {
struct kvm_mmu_page *sp;
for_each_sp(pages, sp, parents, i) {
kvm_mmu_zap_page(kvm, sp);
mmu_pages_clear_parents(&parents);
}
zapped += pages.nr;
kvm_mmu_pages_init(parent, &parents, &pages);
}
return zapped;
} }
static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment