Commit f34d251d authored by Xiao Guangrong's avatar Xiao Guangrong Committed by Gleb Natapov

KVM: MMU: collapse TLB flushes when zap all pages

kvm_zap_obsolete_pages uses lock-break technique to zap pages,
it will flush tlb every time when it does lock-break

We can reload mmu on all vcpus after updating the generation
number so that the obsolete pages are not used on any vcpus,
after that we do not need to flush tlb when obsolete pages
are zapped

It will do kvm_mmu_prepare_zap_page many times and use one
kvm_mmu_commit_zap_page to collapse tlb flush, the side-effects
is that causes obsolete pages unlinked from active_list but leave
on hash-list, so we add the comment around the hash list walker

Note: kvm_mmu_commit_zap_page is still needed before free
the pages since other vcpus may be doing locklessly shadow
page walking
Signed-off-by: default avatarXiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Reviewed-by: default avatarMarcelo Tosatti <mtosatti@redhat.com>
Signed-off-by: default avatarGleb Natapov <gleb@redhat.com>
parent e7d11c7a
...@@ -1654,6 +1654,16 @@ static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp, ...@@ -1654,6 +1654,16 @@ static int kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
static void kvm_mmu_commit_zap_page(struct kvm *kvm, static void kvm_mmu_commit_zap_page(struct kvm *kvm,
struct list_head *invalid_list); struct list_head *invalid_list);
/*
* NOTE: we should pay more attention on the zapped-obsolete page
* (is_obsolete_sp(sp) && sp->role.invalid) when you do hash list walk
* since it has been deleted from active_mmu_pages but still can be found
* at hast list.
*
* for_each_gfn_indirect_valid_sp has skipped that kind of page and
* kvm_mmu_get_page(), the only user of for_each_gfn_sp(), has skipped
* all the obsolete pages.
*/
#define for_each_gfn_sp(_kvm, _sp, _gfn) \ #define for_each_gfn_sp(_kvm, _sp, _gfn) \
hlist_for_each_entry(_sp, \ hlist_for_each_entry(_sp, \
&(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \ &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)], hash_link) \
...@@ -4224,11 +4234,13 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm) ...@@ -4224,11 +4234,13 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm)
if (sp->role.invalid) if (sp->role.invalid)
continue; continue;
/*
* Need not flush tlb since we only zap the sp with invalid
* generation number.
*/
if (batch >= BATCH_ZAP_PAGES && if (batch >= BATCH_ZAP_PAGES &&
(need_resched() || spin_needbreak(&kvm->mmu_lock))) { cond_resched_lock(&kvm->mmu_lock)) {
batch = 0; batch = 0;
kvm_mmu_commit_zap_page(kvm, &invalid_list);
cond_resched_lock(&kvm->mmu_lock);
goto restart; goto restart;
} }
...@@ -4239,6 +4251,10 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm) ...@@ -4239,6 +4251,10 @@ static void kvm_zap_obsolete_pages(struct kvm *kvm)
goto restart; goto restart;
} }
/*
* Should flush tlb before free page tables since lockless-walking
* may use the pages.
*/
kvm_mmu_commit_zap_page(kvm, &invalid_list); kvm_mmu_commit_zap_page(kvm, &invalid_list);
} }
...@@ -4257,6 +4273,17 @@ void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm) ...@@ -4257,6 +4273,17 @@ void kvm_mmu_invalidate_zap_all_pages(struct kvm *kvm)
trace_kvm_mmu_invalidate_zap_all_pages(kvm); trace_kvm_mmu_invalidate_zap_all_pages(kvm);
kvm->arch.mmu_valid_gen++; kvm->arch.mmu_valid_gen++;
/*
* Notify all vcpus to reload its shadow page table
* and flush TLB. Then all vcpus will switch to new
* shadow page table with the new mmu_valid_gen.
*
* Note: we should do this under the protection of
* mmu-lock, otherwise, vcpu would purge shadow page
* but miss tlb flush.
*/
kvm_reload_remote_mmus(kvm);
kvm_zap_obsolete_pages(kvm); kvm_zap_obsolete_pages(kvm);
spin_unlock(&kvm->mmu_lock); spin_unlock(&kvm->mmu_lock);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment