mmu.c 170 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7 8 9 10
/*
 * Kernel-based Virtual Machine driver for Linux
 *
 * This module enables machines with Intel VT-x extensions to run virtual
 * machines without emulation or binary translation.
 *
 * MMU support
 *
 * Copyright (C) 2006 Qumranet, Inc.
11
 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
12 13 14 15 16
 *
 * Authors:
 *   Yaniv Kamay  <yaniv@qumranet.com>
 *   Avi Kivity   <avi@qumranet.com>
 */
Avi Kivity's avatar
Avi Kivity committed
17

18
#include "irq.h"
19
#include "ioapic.h"
20
#include "mmu.h"
21
#include "mmu_internal.h"
22
#include "tdp_mmu.h"
23
#include "x86.h"
Avi Kivity's avatar
Avi Kivity committed
24
#include "kvm_cache_regs.h"
25
#include "kvm_emulate.h"
26
#include "cpuid.h"
27
#include "spte.h"
Avi Kivity's avatar
Avi Kivity committed
28

29
#include <linux/kvm_host.h>
30 31 32 33
#include <linux/types.h>
#include <linux/string.h>
#include <linux/mm.h>
#include <linux/highmem.h>
34 35
#include <linux/moduleparam.h>
#include <linux/export.h>
36
#include <linux/swap.h>
37
#include <linux/hugetlb.h>
38
#include <linux/compiler.h>
39
#include <linux/srcu.h>
40
#include <linux/slab.h>
41
#include <linux/sched/signal.h>
42
#include <linux/uaccess.h>
43
#include <linux/hash.h>
44
#include <linux/kern_levels.h>
45
#include <linux/kthread.h>
46

Avi Kivity's avatar
Avi Kivity committed
47
#include <asm/page.h>
48
#include <asm/memtype.h>
Avi Kivity's avatar
Avi Kivity committed
49
#include <asm/cmpxchg.h>
50
#include <asm/io.h>
51
#include <asm/set_memory.h>
52
#include <asm/vmx.h>
53
#include <asm/kvm_page_track.h>
54
#include "trace.h"
55

56 57
#include "paging.h"

58 59
extern bool itlb_multihit_kvm_mitigation;

60
int __read_mostly nx_huge_pages = -1;
61
static uint __read_mostly nx_huge_pages_recovery_period_ms;
62 63 64 65
#ifdef CONFIG_PREEMPT_RT
/* Recovery can cause latency spikes, disable it for PREEMPT_RT.  */
static uint __read_mostly nx_huge_pages_recovery_ratio = 0;
#else
66
static uint __read_mostly nx_huge_pages_recovery_ratio = 60;
67
#endif
68 69

static int set_nx_huge_pages(const char *val, const struct kernel_param *kp);
70
static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp);
71

72
static const struct kernel_param_ops nx_huge_pages_ops = {
73 74 75 76
	.set = set_nx_huge_pages,
	.get = param_get_bool,
};

77 78
static const struct kernel_param_ops nx_huge_pages_recovery_param_ops = {
	.set = set_nx_huge_pages_recovery_param,
79 80 81
	.get = param_get_uint,
};

82 83
module_param_cb(nx_huge_pages, &nx_huge_pages_ops, &nx_huge_pages, 0644);
__MODULE_PARM_TYPE(nx_huge_pages, "bool");
84
module_param_cb(nx_huge_pages_recovery_ratio, &nx_huge_pages_recovery_param_ops,
85 86
		&nx_huge_pages_recovery_ratio, 0644);
__MODULE_PARM_TYPE(nx_huge_pages_recovery_ratio, "uint");
87 88 89
module_param_cb(nx_huge_pages_recovery_period_ms, &nx_huge_pages_recovery_param_ops,
		&nx_huge_pages_recovery_period_ms, 0644);
__MODULE_PARM_TYPE(nx_huge_pages_recovery_period_ms, "uint");
90

91 92 93
static bool __read_mostly force_flush_and_sync_on_reuse;
module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644);

94 95 96 97 98 99 100
/*
 * When setting this variable to true it enables Two-Dimensional-Paging
 * where the hardware walks 2 page tables:
 * 1. the guest-virtual to guest-physical
 * 2. while doing 1. it walks guest-physical to host-physical
 * If the hardware supports that we don't need to do shadow paging.
 */
101
bool tdp_enabled = false;
102

103
static int max_huge_page_level __read_mostly;
104
static int tdp_root_level __read_mostly;
105
static int max_tdp_level __read_mostly;
106

107 108 109 110
enum {
	AUDIT_PRE_PAGE_FAULT,
	AUDIT_POST_PAGE_FAULT,
	AUDIT_PRE_PTE_WRITE,
111 112 113
	AUDIT_POST_PTE_WRITE,
	AUDIT_PRE_SYNC,
	AUDIT_POST_SYNC
114
};
115 116

#ifdef MMU_DEBUG
117
bool dbg = 0;
118
module_param(dbg, bool, 0644);
119
#endif
120

121 122
#define PTE_PREFETCH_NUM		8

123 124 125
#define PT32_LEVEL_BITS 10

#define PT32_LEVEL_SHIFT(level) \
Mike Day's avatar
Mike Day committed
126
		(PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS)
127

128 129 130
#define PT32_LVL_OFFSET_MASK(level) \
	(PT32_BASE_ADDR_MASK & ((1ULL << (PAGE_SHIFT + (((level) - 1) \
						* PT32_LEVEL_BITS))) - 1))
131 132 133 134 135 136 137 138

#define PT32_INDEX(address, level)\
	(((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))


#define PT32_BASE_ADDR_MASK PAGE_MASK
#define PT32_DIR_BASE_ADDR_MASK \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
139 140 141
#define PT32_LVL_ADDR_MASK(level) \
	(PAGE_MASK & ~((1ULL << (PAGE_SHIFT + (((level) - 1) \
					    * PT32_LEVEL_BITS))) - 1))
142

143 144
#include <trace/events/kvm.h>

145
/* make pte_list_desc fit well in cache lines */
146
#define PTE_LIST_EXT 14
147

148 149 150 151 152
/*
 * Slight optimization of cacheline layout, by putting `more' and `spte_count'
 * at the start; then accessing it will only use one single cacheline for
 * either full (entries==PTE_LIST_EXT) case or entries<=6.
 */
153 154
struct pte_list_desc {
	struct pte_list_desc *more;
155 156 157 158 159 160
	/*
	 * Stores number of entries stored in the pte_list_desc.  No need to be
	 * u64 but just for easier alignment.  When PTE_LIST_EXT, means full.
	 */
	u64 spte_count;
	u64 *sptes[PTE_LIST_EXT];
161 162
};

163 164 165 166
struct kvm_shadow_walk_iterator {
	u64 addr;
	hpa_t shadow_addr;
	u64 *sptep;
167
	int level;
168 169 170
	unsigned index;
};

171 172 173 174 175 176 177
#define for_each_shadow_entry_using_root(_vcpu, _root, _addr, _walker)     \
	for (shadow_walk_init_using_root(&(_walker), (_vcpu),              \
					 (_root), (_addr));                \
	     shadow_walk_okay(&(_walker));			           \
	     shadow_walk_next(&(_walker)))

#define for_each_shadow_entry(_vcpu, _addr, _walker)            \
178 179 180 181
	for (shadow_walk_init(&(_walker), _vcpu, _addr);	\
	     shadow_walk_okay(&(_walker));			\
	     shadow_walk_next(&(_walker)))

182 183 184 185 186 187
#define for_each_shadow_entry_lockless(_vcpu, _addr, _walker, spte)	\
	for (shadow_walk_init(&(_walker), _vcpu, _addr);		\
	     shadow_walk_okay(&(_walker)) &&				\
		({ spte = mmu_spte_get_lockless(_walker.sptep); 1; });	\
	     __shadow_walk_next(&(_walker), spte))

188
static struct kmem_cache *pte_list_desc_cache;
189
struct kmem_cache *mmu_page_header_cache;
190
static struct percpu_counter kvm_total_used_mmu_pages;
191

192
static void mmu_spte_set(u64 *sptep, u64 spte);
193 194
static union kvm_mmu_page_role
kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu);
195

196 197 198 199 200 201
struct kvm_mmu_role_regs {
	const unsigned long cr0;
	const unsigned long cr4;
	const u64 efer;
};

202 203 204
#define CREATE_TRACE_POINTS
#include "mmutrace.h"

205 206 207 208 209 210
/*
 * Yes, lot's of underscores.  They're a hint that you probably shouldn't be
 * reading from the role_regs.  Once the mmu_role is constructed, it becomes
 * the single source of truth for the MMU's state.
 */
#define BUILD_MMU_ROLE_REGS_ACCESSOR(reg, name, flag)			\
211
static inline bool __maybe_unused ____is_##reg##_##name(struct kvm_mmu_role_regs *regs)\
212 213 214 215 216 217 218 219 220 221 222 223 224 225
{									\
	return !!(regs->reg & flag);					\
}
BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, pg, X86_CR0_PG);
BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, wp, X86_CR0_WP);
BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pse, X86_CR4_PSE);
BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pae, X86_CR4_PAE);
BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smep, X86_CR4_SMEP);
BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smap, X86_CR4_SMAP);
BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pke, X86_CR4_PKE);
BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, la57, X86_CR4_LA57);
BUILD_MMU_ROLE_REGS_ACCESSOR(efer, nx, EFER_NX);
BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA);

226 227 228 229 230 231 232
/*
 * The MMU itself (with a valid role) is the single source of truth for the
 * MMU.  Do not use the regs used to build the MMU/role, nor the vCPU.  The
 * regs don't account for dependencies, e.g. clearing CR4 bits if CR0.PG=1,
 * and the vCPU may be incorrect/irrelevant.
 */
#define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name)		\
233
static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu)	\
234 235 236 237 238 239 240 241 242 243 244 245 246
{								\
	return !!(mmu->mmu_role. base_or_ext . reg##_##name);	\
}
BUILD_MMU_ROLE_ACCESSOR(ext,  cr0, pg);
BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp);
BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pse);
BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pae);
BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smep);
BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smap);
BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pke);
BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, la57);
BUILD_MMU_ROLE_ACCESSOR(base, efer, nx);

247 248 249 250 251 252 253 254 255 256
static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_role_regs regs = {
		.cr0 = kvm_read_cr0_bits(vcpu, KVM_MMU_CR0_ROLE_BITS),
		.cr4 = kvm_read_cr4_bits(vcpu, KVM_MMU_CR4_ROLE_BITS),
		.efer = vcpu->arch.efer,
	};

	return regs;
}
257

258 259 260 261 262 263 264 265 266 267 268 269
static int role_regs_to_root_level(struct kvm_mmu_role_regs *regs)
{
	if (!____is_cr0_pg(regs))
		return 0;
	else if (____is_efer_lma(regs))
		return ____is_cr4_la57(regs) ? PT64_ROOT_5LEVEL :
					       PT64_ROOT_4LEVEL;
	else if (____is_cr4_pae(regs))
		return PT32E_ROOT_LEVEL;
	else
		return PT32_ROOT_LEVEL;
}
270 271 272

static inline bool kvm_available_flush_tlb_with_range(void)
{
273
	return kvm_x86_ops.tlb_remote_flush_with_range;
274 275 276 277 278 279 280
}

static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,
		struct kvm_tlb_range *range)
{
	int ret = -ENOTSUPP;

281
	if (range && kvm_x86_ops.tlb_remote_flush_with_range)
282
		ret = static_call(kvm_x86_tlb_remote_flush_with_range)(kvm, range);
283 284 285 286 287

	if (ret)
		kvm_flush_remote_tlbs(kvm);
}

288
void kvm_flush_remote_tlbs_with_address(struct kvm *kvm,
289 290 291 292 293 294 295 296 297 298
		u64 start_gfn, u64 pages)
{
	struct kvm_tlb_range range;

	range.start_gfn = start_gfn;
	range.pages = pages;

	kvm_flush_remote_tlbs_with_range(kvm, &range);
}

299 300 301
static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn,
			   unsigned int access)
{
302
	u64 spte = make_mmio_spte(vcpu, gfn, access);
303

304 305
	trace_mark_mmio_spte(sptep, gfn, spte);
	mmu_spte_set(sptep, spte);
306 307 308 309
}

static gfn_t get_mmio_spte_gfn(u64 spte)
{
310
	u64 gpa = spte & shadow_nonpresent_or_rsvd_lower_gfn_mask;
311

312
	gpa |= (spte >> SHADOW_NONPRESENT_OR_RSVD_MASK_LEN)
313 314 315
	       & shadow_nonpresent_or_rsvd_mask;

	return gpa >> PAGE_SHIFT;
316 317 318 319
}

static unsigned get_mmio_spte_access(u64 spte)
{
320
	return spte & shadow_mmio_access_mask;
321 322
}

323
static bool check_mmio_spte(struct kvm_vcpu *vcpu, u64 spte)
324
{
325
	u64 kvm_gen, spte_gen, gen;
326

327 328 329
	gen = kvm_vcpu_memslots(vcpu)->generation;
	if (unlikely(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS))
		return false;
330

331
	kvm_gen = gen & MMIO_SPTE_GEN_MASK;
332 333 334 335
	spte_gen = get_mmio_spte_generation(spte);

	trace_check_mmio_spte(spte, kvm_gen, spte_gen);
	return likely(kvm_gen == spte_gen);
336 337
}

338 339 340 341 342
static int is_cpuid_PSE36(void)
{
	return 1;
}

343 344 345 346 347 348 349
static gfn_t pse36_gfn_delta(u32 gpte)
{
	int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT;

	return (gpte & PT32_DIR_PSE36_MASK) << shift;
}

350
#ifdef CONFIG_X86_64
Avi Kivity's avatar
Avi Kivity committed
351
static void __set_spte(u64 *sptep, u64 spte)
352
{
353
	WRITE_ONCE(*sptep, spte);
354 355
}

356
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
357
{
358
	WRITE_ONCE(*sptep, spte);
359 360 361 362 363 364
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	return xchg(sptep, spte);
}
365 366 367

static u64 __get_spte_lockless(u64 *sptep)
{
368
	return READ_ONCE(*sptep);
369
}
370
#else
371 372 373 374 375 376 377
union split_spte {
	struct {
		u32 spte_low;
		u32 spte_high;
	};
	u64 spte;
};
378

379 380
static void count_spte_clear(u64 *sptep, u64 spte)
{
381
	struct kvm_mmu_page *sp =  sptep_to_sp(sptep);
382 383 384 385 386 387 388 389 390

	if (is_shadow_present_pte(spte))
		return;

	/* Ensure the spte is completely set before we increase the count */
	smp_wmb();
	sp->clear_spte_count++;
}

391 392 393
static void __set_spte(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;
394

395 396 397 398 399 400 401 402 403 404 405 406
	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	ssptep->spte_high = sspte.spte_high;

	/*
	 * If we map the spte from nonpresent to present, We should store
	 * the high bits firstly, then set present bit, so cpu can not
	 * fetch this spte while we are setting the spte.
	 */
	smp_wmb();

407
	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
408 409
}

410 411 412 413 414 415 416
static void __update_clear_spte_fast(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

417
	WRITE_ONCE(ssptep->spte_low, sspte.spte_low);
418 419 420 421 422 423 424 425

	/*
	 * If we map the spte from present to nonpresent, we should clear
	 * present bit firstly to avoid vcpu fetch the old high bits.
	 */
	smp_wmb();

	ssptep->spte_high = sspte.spte_high;
426
	count_spte_clear(sptep, spte);
427 428 429 430 431 432 433 434 435 436 437
}

static u64 __update_clear_spte_slow(u64 *sptep, u64 spte)
{
	union split_spte *ssptep, sspte, orig;

	ssptep = (union split_spte *)sptep;
	sspte = (union split_spte)spte;

	/* xchg acts as a barrier before the setting of the high bits */
	orig.spte_low = xchg(&ssptep->spte_low, sspte.spte_low);
438 439
	orig.spte_high = ssptep->spte_high;
	ssptep->spte_high = sspte.spte_high;
440
	count_spte_clear(sptep, spte);
441 442 443

	return orig.spte;
}
444 445 446

/*
 * The idea using the light way get the spte on x86_32 guest is from
447
 * gup_get_pte (mm/gup.c).
448 449 450 451 452 453 454 455 456 457 458 459 460 461
 *
 * An spte tlb flush may be pending, because kvm_set_pte_rmapp
 * coalesces them and we are running out of the MMU lock.  Therefore
 * we need to protect against in-progress updates of the spte.
 *
 * Reading the spte while an update is in progress may get the old value
 * for the high part of the spte.  The race is fine for a present->non-present
 * change (because the high part of the spte is ignored for non-present spte),
 * but for a present->present change we must reread the spte.
 *
 * All such changes are done in two steps (present->non-present and
 * non-present->present), hence it is enough to count the number of
 * present->non-present updates: if it changed while reading the spte,
 * we might have hit the race.  This is done using clear_spte_count.
462 463 464
 */
static u64 __get_spte_lockless(u64 *sptep)
{
465
	struct kvm_mmu_page *sp =  sptep_to_sp(sptep);
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484
	union split_spte spte, *orig = (union split_spte *)sptep;
	int count;

retry:
	count = sp->clear_spte_count;
	smp_rmb();

	spte.spte_low = orig->spte_low;
	smp_rmb();

	spte.spte_high = orig->spte_high;
	smp_rmb();

	if (unlikely(spte.spte_low != orig->spte_low ||
	      count != sp->clear_spte_count))
		goto retry;

	return spte.spte;
}
485 486
#endif

487 488
static bool spte_has_volatile_bits(u64 spte)
{
489 490 491
	if (!is_shadow_present_pte(spte))
		return false;

492
	/*
493
	 * Always atomically update spte if it can be updated
494 495 496 497
	 * out of mmu-lock, it can ensure dirty bit is not lost,
	 * also, it can help us to get a stable is_writable_pte()
	 * to ensure tlb flush is not missed.
	 */
498 499
	if (spte_can_locklessly_be_made_writable(spte) ||
	    is_access_track_spte(spte))
500 501
		return true;

502
	if (spte_ad_enabled(spte)) {
503 504 505 506
		if ((spte & shadow_accessed_mask) == 0 ||
	    	    (is_writable_pte(spte) && (spte & shadow_dirty_mask) == 0))
			return true;
	}
507

508
	return false;
509 510
}

511 512 513 514 515 516 517 518 519 520 521 522
/* Rules for using mmu_spte_set:
 * Set the sptep from nonpresent to present.
 * Note: the sptep being assigned *must* be either not present
 * or in a state where the hardware will not attempt to update
 * the spte.
 */
static void mmu_spte_set(u64 *sptep, u64 new_spte)
{
	WARN_ON(is_shadow_present_pte(*sptep));
	__set_spte(sptep, new_spte);
}

523 524 525
/*
 * Update the SPTE (excluding the PFN), but do not track changes in its
 * accessed/dirty status.
526
 */
527
static u64 mmu_spte_update_no_track(u64 *sptep, u64 new_spte)
528
{
529
	u64 old_spte = *sptep;
530

531
	WARN_ON(!is_shadow_present_pte(new_spte));
532
	check_spte_writable_invariants(new_spte);
533

534 535
	if (!is_shadow_present_pte(old_spte)) {
		mmu_spte_set(sptep, new_spte);
536
		return old_spte;
537
	}
538

539
	if (!spte_has_volatile_bits(old_spte))
540
		__update_clear_spte_fast(sptep, new_spte);
541
	else
542
		old_spte = __update_clear_spte_slow(sptep, new_spte);
543

544 545
	WARN_ON(spte_to_pfn(old_spte) != spte_to_pfn(new_spte));

546 547 548 549 550 551
	return old_spte;
}

/* Rules for using mmu_spte_update:
 * Update the state bits, it means the mapped pfn is not changed.
 *
552 553 554
 * Whenever an MMU-writable SPTE is overwritten with a read-only SPTE, remote
 * TLBs must be flushed. Otherwise rmap_write_protect will find a read-only
 * spte, even though the writable spte might be cached on a CPU's TLB.
555 556 557 558 559 560 561 562 563 564 565
 *
 * Returns true if the TLB needs to be flushed
 */
static bool mmu_spte_update(u64 *sptep, u64 new_spte)
{
	bool flush = false;
	u64 old_spte = mmu_spte_update_no_track(sptep, new_spte);

	if (!is_shadow_present_pte(old_spte))
		return false;

566 567
	/*
	 * For the spte updated out of mmu-lock is safe, since
568
	 * we always atomically update it, see the comments in
569 570
	 * spte_has_volatile_bits().
	 */
571
	if (spte_can_locklessly_be_made_writable(old_spte) &&
572
	      !is_writable_pte(new_spte))
573
		flush = true;
574

575
	/*
576
	 * Flush TLB when accessed/dirty states are changed in the page tables,
577 578 579
	 * to guarantee consistency between TLB and page tables.
	 */

580 581
	if (is_accessed_spte(old_spte) && !is_accessed_spte(new_spte)) {
		flush = true;
582
		kvm_set_pfn_accessed(spte_to_pfn(old_spte));
583 584 585 586
	}

	if (is_dirty_spte(old_spte) && !is_dirty_spte(new_spte)) {
		flush = true;
587
		kvm_set_pfn_dirty(spte_to_pfn(old_spte));
588
	}
589

590
	return flush;
591 592
}

593 594 595 596
/*
 * Rules for using mmu_spte_clear_track_bits:
 * It sets the sptep from present to nonpresent, and track the
 * state bits, it is used to clear the last level sptep.
597
 * Returns the old PTE.
598
 */
599
static int mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep)
600
{
601
	kvm_pfn_t pfn;
602
	u64 old_spte = *sptep;
603
	int level = sptep_to_sp(sptep)->role.level;
604 605

	if (!spte_has_volatile_bits(old_spte))
606
		__update_clear_spte_fast(sptep, 0ull);
607
	else
608
		old_spte = __update_clear_spte_slow(sptep, 0ull);
609

610
	if (!is_shadow_present_pte(old_spte))
611
		return old_spte;
612

613 614
	kvm_update_page_stats(kvm, level, -1);

615
	pfn = spte_to_pfn(old_spte);
616 617 618 619 620 621

	/*
	 * KVM does not hold the refcount of the page used by
	 * kvm mmu, before reclaiming the page, we should
	 * unmap it from mmu first.
	 */
622
	WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));
623

624
	if (is_accessed_spte(old_spte))
625
		kvm_set_pfn_accessed(pfn);
626 627

	if (is_dirty_spte(old_spte))
628
		kvm_set_pfn_dirty(pfn);
629

630
	return old_spte;
631 632 633 634 635 636 637 638 639
}

/*
 * Rules for using mmu_spte_clear_no_track:
 * Directly clear spte without caring the state bits of sptep,
 * it is used to set the upper level spte.
 */
static void mmu_spte_clear_no_track(u64 *sptep)
{
640
	__update_clear_spte_fast(sptep, 0ull);
641 642
}

643 644 645 646 647
static u64 mmu_spte_get_lockless(u64 *sptep)
{
	return __get_spte_lockless(sptep);
}

648 649 650 651 652 653 654 655
/* Returns the Accessed status of the PTE and resets it at the same time. */
static bool mmu_spte_age(u64 *sptep)
{
	u64 spte = mmu_spte_get_lockless(sptep);

	if (!is_accessed_spte(spte))
		return false;

656
	if (spte_ad_enabled(spte)) {
657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673
		clear_bit((ffs(shadow_accessed_mask) - 1),
			  (unsigned long *)sptep);
	} else {
		/*
		 * Capture the dirty status of the page, so that it doesn't get
		 * lost when the SPTE is marked for access tracking.
		 */
		if (is_writable_pte(spte))
			kvm_set_pfn_dirty(spte_to_pfn(spte));

		spte = mark_spte_for_access_track(spte);
		mmu_spte_update_no_track(sptep, spte);
	}

	return true;
}

674 675
static void walk_shadow_page_lockless_begin(struct kvm_vcpu *vcpu)
{
676 677 678 679 680 681 682 683
	if (is_tdp_mmu(vcpu->arch.mmu)) {
		kvm_tdp_mmu_walk_lockless_begin();
	} else {
		/*
		 * Prevent page table teardown by making any free-er wait during
		 * kvm_flush_remote_tlbs() IPI to all active vcpus.
		 */
		local_irq_disable();
684

685 686 687 688 689 690
		/*
		 * Make sure a following spte read is not reordered ahead of the write
		 * to vcpu->mode.
		 */
		smp_store_mb(vcpu->mode, READING_SHADOW_PAGE_TABLES);
	}
691 692 693 694
}

static void walk_shadow_page_lockless_end(struct kvm_vcpu *vcpu)
{
695 696 697 698 699 700 701 702 703 704 705
	if (is_tdp_mmu(vcpu->arch.mmu)) {
		kvm_tdp_mmu_walk_lockless_end();
	} else {
		/*
		 * Make sure the write to vcpu->mode is not reordered in front of
		 * reads to sptes.  If it does, kvm_mmu_commit_zap_page() can see us
		 * OUTSIDE_GUEST_MODE and proceed to free the shadow page table.
		 */
		smp_store_release(&vcpu->mode, OUTSIDE_GUEST_MODE);
		local_irq_enable();
	}
706 707
}

708
static int mmu_topup_memory_caches(struct kvm_vcpu *vcpu, bool maybe_indirect)
709
{
710 711
	int r;

712
	/* 1 rmap, 1 parent PTE per level, and the prefetched rmaps. */
713 714
	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache,
				       1 + PT64_ROOT_MAX_LEVEL + PTE_PREFETCH_NUM);
715
	if (r)
716
		return r;
717 718
	r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_shadow_page_cache,
				       PT64_ROOT_MAX_LEVEL);
719
	if (r)
720
		return r;
721
	if (maybe_indirect) {
722 723
		r = kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_gfn_array_cache,
					       PT64_ROOT_MAX_LEVEL);
724 725 726
		if (r)
			return r;
	}
727 728
	return kvm_mmu_topup_memory_cache(&vcpu->arch.mmu_page_header_cache,
					  PT64_ROOT_MAX_LEVEL);
729 730 731 732
}

static void mmu_free_memory_caches(struct kvm_vcpu *vcpu)
{
733 734 735 736
	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_pte_list_desc_cache);
	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_shadow_page_cache);
	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_gfn_array_cache);
	kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_header_cache);
737 738
}

739
static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
740
{
741
	return kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
742 743
}

744
static void mmu_free_pte_list_desc(struct pte_list_desc *pte_list_desc)
745
{
746
	kmem_cache_free(pte_list_desc_cache, pte_list_desc);
747 748
}

749 750 751 752 753 754 755 756 757 758
static gfn_t kvm_mmu_page_get_gfn(struct kvm_mmu_page *sp, int index)
{
	if (!sp->role.direct)
		return sp->gfns[index];

	return sp->gfn + (index << ((sp->role.level - 1) * PT64_LEVEL_BITS));
}

static void kvm_mmu_page_set_gfn(struct kvm_mmu_page *sp, int index, gfn_t gfn)
{
759
	if (!sp->role.direct) {
760
		sp->gfns[index] = gfn;
761 762 763 764 765 766 767 768
		return;
	}

	if (WARN_ON(gfn != kvm_mmu_page_get_gfn(sp, index)))
		pr_err_ratelimited("gfn mismatch under direct page %llx "
				   "(expected %llx, got %llx)\n",
				   sp->gfn,
				   kvm_mmu_page_get_gfn(sp, index), gfn);
769 770
}

771
/*
772 773
 * Return the pointer to the large page information for a given gfn,
 * handling slots that are not large page aligned.
774
 */
775
static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
776
		const struct kvm_memory_slot *slot, int level)
777 778 779
{
	unsigned long idx;

780
	idx = gfn_to_index(gfn, slot->base_gfn, level);
781
	return &slot->arch.lpage_info[level - 2][idx];
782 783
}

784
static void update_gfn_disallow_lpage_count(const struct kvm_memory_slot *slot,
785 786 787 788 789
					    gfn_t gfn, int count)
{
	struct kvm_lpage_info *linfo;
	int i;

790
	for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
791 792 793 794 795 796
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->disallow_lpage += count;
		WARN_ON(linfo->disallow_lpage < 0);
	}
}

797
void kvm_mmu_gfn_disallow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
798 799 800 801
{
	update_gfn_disallow_lpage_count(slot, gfn, 1);
}

802
void kvm_mmu_gfn_allow_lpage(const struct kvm_memory_slot *slot, gfn_t gfn)
803 804 805 806
{
	update_gfn_disallow_lpage_count(slot, gfn, -1);
}

807
static void account_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
808
{
809
	struct kvm_memslots *slots;
810
	struct kvm_memory_slot *slot;
811
	gfn_t gfn;
812

813
	kvm->arch.indirect_shadow_pages++;
814
	gfn = sp->gfn;
815 816
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
817 818

	/* the non-leaf shadow pages are keeping readonly. */
819
	if (sp->role.level > PG_LEVEL_4K)
820 821 822
		return kvm_slot_page_track_add_page(kvm, slot, gfn,
						    KVM_PAGE_TRACK_WRITE);

823
	kvm_mmu_gfn_disallow_lpage(slot, gfn);
824 825
}

826
void account_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp)
827 828 829 830 831
{
	if (sp->lpage_disallowed)
		return;

	++kvm->stat.nx_lpage_splits;
832 833
	list_add_tail(&sp->lpage_disallowed_link,
		      &kvm->arch.lpage_disallowed_mmu_pages);
834 835 836
	sp->lpage_disallowed = true;
}

837
static void unaccount_shadowed(struct kvm *kvm, struct kvm_mmu_page *sp)
838
{
839
	struct kvm_memslots *slots;
840
	struct kvm_memory_slot *slot;
841
	gfn_t gfn;
842

843
	kvm->arch.indirect_shadow_pages--;
844
	gfn = sp->gfn;
845 846
	slots = kvm_memslots_for_spte_role(kvm, sp->role);
	slot = __gfn_to_memslot(slots, gfn);
847
	if (sp->role.level > PG_LEVEL_4K)
848 849 850
		return kvm_slot_page_track_remove_page(kvm, slot, gfn,
						       KVM_PAGE_TRACK_WRITE);

851
	kvm_mmu_gfn_allow_lpage(slot, gfn);
852 853
}

854
void unaccount_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp)
855 856 857
{
	--kvm->stat.nx_lpage_splits;
	sp->lpage_disallowed = false;
858
	list_del(&sp->lpage_disallowed_link);
859 860
}

861 862 863
static struct kvm_memory_slot *
gfn_to_memslot_dirty_bitmap(struct kvm_vcpu *vcpu, gfn_t gfn,
			    bool no_dirty_log)
864 865
{
	struct kvm_memory_slot *slot;
866

867
	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
868 869
	if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
		return NULL;
870
	if (no_dirty_log && kvm_slot_dirty_track_enabled(slot))
871
		return NULL;
872 873 874 875

	return slot;
}

876
/*
877
 * About rmap_head encoding:
878
 *
879 880
 * If the bit zero of rmap_head->val is clear, then it points to the only spte
 * in this rmap chain. Otherwise, (rmap_head->val & ~1) points to a struct
881
 * pte_list_desc containing more mappings.
882 883 884 885
 */

/*
 * Returns the number of pointers in the rmap chain, not counting the new one.
886
 */
887
static int pte_list_add(struct kvm_vcpu *vcpu, u64 *spte,
888
			struct kvm_rmap_head *rmap_head)
889
{
890
	struct pte_list_desc *desc;
891
	int count = 0;
892

893
	if (!rmap_head->val) {
894
		rmap_printk("%p %llx 0->1\n", spte, *spte);
895 896
		rmap_head->val = (unsigned long)spte;
	} else if (!(rmap_head->val & 1)) {
897
		rmap_printk("%p %llx 1->many\n", spte, *spte);
898
		desc = mmu_alloc_pte_list_desc(vcpu);
899
		desc->sptes[0] = (u64 *)rmap_head->val;
Avi Kivity's avatar
Avi Kivity committed
900
		desc->sptes[1] = spte;
901
		desc->spte_count = 2;
902
		rmap_head->val = (unsigned long)desc | 1;
903
		++count;
904
	} else {
905
		rmap_printk("%p %llx many->many\n", spte, *spte);
906
		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
907
		while (desc->spte_count == PTE_LIST_EXT) {
908
			count += PTE_LIST_EXT;
909 910 911
			if (!desc->more) {
				desc->more = mmu_alloc_pte_list_desc(vcpu);
				desc = desc->more;
912
				desc->spte_count = 0;
913 914
				break;
			}
915 916
			desc = desc->more;
		}
917 918
		count += desc->spte_count;
		desc->sptes[desc->spte_count++] = spte;
919
	}
920
	return count;
921 922
}

923
static void
924 925 926
pte_list_desc_remove_entry(struct kvm_rmap_head *rmap_head,
			   struct pte_list_desc *desc, int i,
			   struct pte_list_desc *prev_desc)
927
{
928
	int j = desc->spte_count - 1;
929

Avi Kivity's avatar
Avi Kivity committed
930 931
	desc->sptes[i] = desc->sptes[j];
	desc->sptes[j] = NULL;
932 933
	desc->spte_count--;
	if (desc->spte_count)
934 935
		return;
	if (!prev_desc && !desc->more)
936
		rmap_head->val = 0;
937 938 939 940
	else
		if (prev_desc)
			prev_desc->more = desc->more;
		else
941
			rmap_head->val = (unsigned long)desc->more | 1;
942
	mmu_free_pte_list_desc(desc);
943 944
}

945
static void __pte_list_remove(u64 *spte, struct kvm_rmap_head *rmap_head)
946
{
947 948
	struct pte_list_desc *desc;
	struct pte_list_desc *prev_desc;
949 950
	int i;

951
	if (!rmap_head->val) {
952
		pr_err("%s: %p 0->BUG\n", __func__, spte);
953
		BUG();
954
	} else if (!(rmap_head->val & 1)) {
955
		rmap_printk("%p 1->0\n", spte);
956
		if ((u64 *)rmap_head->val != spte) {
957
			pr_err("%s:  %p 1->BUG\n", __func__, spte);
958 959
			BUG();
		}
960
		rmap_head->val = 0;
961
	} else {
962
		rmap_printk("%p many->many\n", spte);
963
		desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
964 965
		prev_desc = NULL;
		while (desc) {
966
			for (i = 0; i < desc->spte_count; ++i) {
Avi Kivity's avatar
Avi Kivity committed
967
				if (desc->sptes[i] == spte) {
968 969
					pte_list_desc_remove_entry(rmap_head,
							desc, i, prev_desc);
970 971
					return;
				}
972
			}
973 974 975
			prev_desc = desc;
			desc = desc->more;
		}
976
		pr_err("%s: %p many->many\n", __func__, spte);
977 978 979 980
		BUG();
	}
}

981 982
static void pte_list_remove(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
			    u64 *sptep)
983
{
984
	mmu_spte_clear_track_bits(kvm, sptep);
985 986 987
	__pte_list_remove(sptep, rmap_head);
}

Peter Xu's avatar
Peter Xu committed
988
/* Return true if rmap existed, false otherwise */
989
static bool pte_list_destroy(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
Peter Xu's avatar
Peter Xu committed
990 991 992 993 994 995 996 997
{
	struct pte_list_desc *desc, *next;
	int i;

	if (!rmap_head->val)
		return false;

	if (!(rmap_head->val & 1)) {
998
		mmu_spte_clear_track_bits(kvm, (u64 *)rmap_head->val);
Peter Xu's avatar
Peter Xu committed
999 1000 1001 1002 1003 1004 1005
		goto out;
	}

	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);

	for (; desc; desc = next) {
		for (i = 0; i < desc->spte_count; i++)
1006
			mmu_spte_clear_track_bits(kvm, desc->sptes[i]);
Peter Xu's avatar
Peter Xu committed
1007 1008 1009 1010 1011 1012 1013 1014 1015
		next = desc->more;
		mmu_free_pte_list_desc(desc);
	}
out:
	/* rmap_head is meaningless now, remember to reset it */
	rmap_head->val = 0;
	return true;
}

1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
unsigned int pte_list_count(struct kvm_rmap_head *rmap_head)
{
	struct pte_list_desc *desc;
	unsigned int count = 0;

	if (!rmap_head->val)
		return 0;
	else if (!(rmap_head->val & 1))
		return 1;

	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);

	while (desc) {
		count += desc->spte_count;
		desc = desc->more;
	}

	return count;
}

1036 1037
static struct kvm_rmap_head *gfn_to_rmap(gfn_t gfn, int level,
					 const struct kvm_memory_slot *slot)
1038
{
1039
	unsigned long idx;
1040

1041
	idx = gfn_to_index(gfn, slot->base_gfn, level);
1042
	return &slot->arch.rmap[level - PG_LEVEL_4K][idx];
1043 1044
}

1045 1046
static bool rmap_can_add(struct kvm_vcpu *vcpu)
{
1047
	struct kvm_mmu_memory_cache *mc;
1048

1049
	mc = &vcpu->arch.mmu_pte_list_desc_cache;
1050
	return kvm_mmu_memory_cache_nr_free_objects(mc);
1051 1052
}

1053 1054
static void rmap_remove(struct kvm *kvm, u64 *spte)
{
1055 1056
	struct kvm_memslots *slots;
	struct kvm_memory_slot *slot;
1057 1058
	struct kvm_mmu_page *sp;
	gfn_t gfn;
1059
	struct kvm_rmap_head *rmap_head;
1060

1061
	sp = sptep_to_sp(spte);
1062
	gfn = kvm_mmu_page_get_gfn(sp, spte - sp->spt);
1063 1064

	/*
1065 1066 1067
	 * Unlike rmap_add, rmap_remove does not run in the context of a vCPU
	 * so we have to determine which memslots to use based on context
	 * information in sp->role.
1068 1069 1070 1071
	 */
	slots = kvm_memslots_for_spte_role(kvm, sp->role);

	slot = __gfn_to_memslot(slots, gfn);
1072
	rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
1073

1074
	__pte_list_remove(spte, rmap_head);
1075 1076
}

1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089
/*
 * Used by the following functions to iterate through the sptes linked by a
 * rmap.  All fields are private and not assumed to be used outside.
 */
struct rmap_iterator {
	/* private fields */
	struct pte_list_desc *desc;	/* holds the sptep if not NULL */
	int pos;			/* index of the sptep */
};

/*
 * Iteration must be started by this function.  This should also be used after
 * removing/dropping sptes from the rmap link because in such cases the
1090
 * information in the iterator may not be valid.
1091 1092 1093
 *
 * Returns sptep if found, NULL otherwise.
 */
1094 1095
static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head,
			   struct rmap_iterator *iter)
1096
{
1097 1098
	u64 *sptep;

1099
	if (!rmap_head->val)
1100 1101
		return NULL;

1102
	if (!(rmap_head->val & 1)) {
1103
		iter->desc = NULL;
1104 1105
		sptep = (u64 *)rmap_head->val;
		goto out;
1106 1107
	}

1108
	iter->desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);
1109
	iter->pos = 0;
1110 1111 1112 1113
	sptep = iter->desc->sptes[iter->pos];
out:
	BUG_ON(!is_shadow_present_pte(*sptep));
	return sptep;
1114 1115 1116 1117 1118 1119 1120 1121 1122
}

/*
 * Must be used with a valid iterator: e.g. after rmap_get_first().
 *
 * Returns sptep if found, NULL otherwise.
 */
static u64 *rmap_get_next(struct rmap_iterator *iter)
{
1123 1124
	u64 *sptep;

1125 1126 1127 1128 1129
	if (iter->desc) {
		if (iter->pos < PTE_LIST_EXT - 1) {
			++iter->pos;
			sptep = iter->desc->sptes[iter->pos];
			if (sptep)
1130
				goto out;
1131 1132 1133 1134 1135 1136 1137
		}

		iter->desc = iter->desc->more;

		if (iter->desc) {
			iter->pos = 0;
			/* desc->sptes[0] cannot be NULL */
1138 1139
			sptep = iter->desc->sptes[iter->pos];
			goto out;
1140 1141 1142 1143
		}
	}

	return NULL;
1144 1145 1146
out:
	BUG_ON(!is_shadow_present_pte(*sptep));
	return sptep;
1147 1148
}

1149 1150
#define for_each_rmap_spte(_rmap_head_, _iter_, _spte_)			\
	for (_spte_ = rmap_get_first(_rmap_head_, _iter_);		\
1151
	     _spte_; _spte_ = rmap_get_next(_iter_))
1152

1153
static void drop_spte(struct kvm *kvm, u64 *sptep)
1154
{
1155
	u64 old_spte = mmu_spte_clear_track_bits(kvm, sptep);
1156 1157

	if (is_shadow_present_pte(old_spte))
1158
		rmap_remove(kvm, sptep);
Avi Kivity's avatar
Avi Kivity committed
1159 1160
}

1161 1162 1163 1164

static bool __drop_large_spte(struct kvm *kvm, u64 *sptep)
{
	if (is_large_pte(*sptep)) {
1165
		WARN_ON(sptep_to_sp(sptep)->role.level == PG_LEVEL_4K);
1166 1167 1168 1169 1170 1171 1172 1173 1174
		drop_spte(kvm, sptep);
		return true;
	}

	return false;
}

static void drop_large_spte(struct kvm_vcpu *vcpu, u64 *sptep)
{
1175
	if (__drop_large_spte(vcpu->kvm, sptep)) {
1176
		struct kvm_mmu_page *sp = sptep_to_sp(sptep);
1177 1178 1179 1180

		kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn,
			KVM_PAGES_PER_HPAGE(sp->role.level));
	}
1181 1182 1183
}

/*
1184
 * Write-protect on the specified @sptep, @pt_protect indicates whether
1185
 * spte write-protection is caused by protecting shadow page table.
1186
 *
1187
 * Note: write protection is difference between dirty logging and spte
1188 1189 1190 1191 1192
 * protection:
 * - for dirty logging, the spte can be set to writable at anytime if
 *   its dirty bitmap is properly set.
 * - for spte protection, the spte can be writable only after unsync-ing
 *   shadow page.
1193
 *
1194
 * Return true if tlb need be flushed.
1195
 */
1196
static bool spte_write_protect(u64 *sptep, bool pt_protect)
1197 1198 1199
{
	u64 spte = *sptep;

1200
	if (!is_writable_pte(spte) &&
1201
	      !(pt_protect && spte_can_locklessly_be_made_writable(spte)))
1202 1203
		return false;

1204
	rmap_printk("spte %p %llx\n", sptep, *sptep);
1205

1206
	if (pt_protect)
1207
		spte &= ~shadow_mmu_writable_mask;
1208
	spte = spte & ~PT_WRITABLE_MASK;
1209

1210
	return mmu_spte_update(sptep, spte);
1211 1212
}

1213 1214
static bool rmap_write_protect(struct kvm_rmap_head *rmap_head,
			       bool pt_protect)
1215
{
1216 1217
	u64 *sptep;
	struct rmap_iterator iter;
1218
	bool flush = false;
1219

1220
	for_each_rmap_spte(rmap_head, &iter, sptep)
1221
		flush |= spte_write_protect(sptep, pt_protect);
1222

1223
	return flush;
1224 1225
}

1226
static bool spte_clear_dirty(u64 *sptep)
1227 1228 1229
{
	u64 spte = *sptep;

1230
	rmap_printk("spte %p %llx\n", sptep, *sptep);
1231

1232
	MMU_WARN_ON(!spte_ad_enabled(spte));
1233 1234 1235 1236
	spte &= ~shadow_dirty_mask;
	return mmu_spte_update(sptep, spte);
}

1237
static bool spte_wrprot_for_clear_dirty(u64 *sptep)
1238 1239 1240
{
	bool was_writable = test_and_clear_bit(PT_WRITABLE_SHIFT,
					       (unsigned long *)sptep);
1241
	if (was_writable && !spte_ad_enabled(*sptep))
1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252
		kvm_set_pfn_dirty(spte_to_pfn(*sptep));

	return was_writable;
}

/*
 * Gets the GFN ready for another round of dirty logging by clearing the
 *	- D bit on ad-enabled SPTEs, and
 *	- W bit on ad-disabled SPTEs.
 * Returns true iff any D or W bits were cleared.
 */
1253
static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1254
			       const struct kvm_memory_slot *slot)
1255 1256 1257 1258 1259
{
	u64 *sptep;
	struct rmap_iterator iter;
	bool flush = false;

1260
	for_each_rmap_spte(rmap_head, &iter, sptep)
1261 1262
		if (spte_ad_need_write_protect(*sptep))
			flush |= spte_wrprot_for_clear_dirty(sptep);
1263
		else
1264
			flush |= spte_clear_dirty(sptep);
1265 1266 1267 1268

	return flush;
}

1269
/**
1270
 * kvm_mmu_write_protect_pt_masked - write protect selected PT level pages
1271 1272 1273 1274 1275
 * @kvm: kvm instance
 * @slot: slot to protect
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should protect
 *
1276
 * Used when we do not need to care about huge page mappings.
1277
 */
1278
static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1279 1280
				     struct kvm_memory_slot *slot,
				     gfn_t gfn_offset, unsigned long mask)
1281
{
1282
	struct kvm_rmap_head *rmap_head;
1283

1284
	if (is_tdp_mmu_enabled(kvm))
1285 1286
		kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
				slot->base_gfn + gfn_offset, mask, true);
1287 1288 1289 1290

	if (!kvm_memslots_have_rmaps(kvm))
		return;

1291
	while (mask) {
1292 1293
		rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
					PG_LEVEL_4K, slot);
1294
		rmap_write_protect(rmap_head, false);
1295

1296 1297 1298
		/* clear the first set bit */
		mask &= mask - 1;
	}
1299 1300
}

1301
/**
1302 1303
 * kvm_mmu_clear_dirty_pt_masked - clear MMU D-bit for PT level pages, or write
 * protect the page if the D-bit isn't supported.
1304 1305 1306 1307 1308 1309 1310
 * @kvm: kvm instance
 * @slot: slot to clear D-bit
 * @gfn_offset: start of the BITS_PER_LONG pages we care about
 * @mask: indicates which pages we should clear D-bit
 *
 * Used for PML to re-log the dirty GPAs after userspace querying dirty_bitmap.
 */
1311 1312 1313
static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm,
					 struct kvm_memory_slot *slot,
					 gfn_t gfn_offset, unsigned long mask)
1314
{
1315
	struct kvm_rmap_head *rmap_head;
1316

1317
	if (is_tdp_mmu_enabled(kvm))
1318 1319
		kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot,
				slot->base_gfn + gfn_offset, mask, false);
1320 1321 1322 1323

	if (!kvm_memslots_have_rmaps(kvm))
		return;

1324
	while (mask) {
1325 1326
		rmap_head = gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
					PG_LEVEL_4K, slot);
1327
		__rmap_clear_dirty(kvm, rmap_head, slot);
1328 1329 1330 1331 1332 1333

		/* clear the first set bit */
		mask &= mask - 1;
	}
}

1334 1335 1336 1337 1338 1339 1340
/**
 * kvm_arch_mmu_enable_log_dirty_pt_masked - enable dirty logging for selected
 * PT level pages.
 *
 * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to
 * enable dirty logging for them.
 *
1341 1342
 * We need to care about huge page mappings: e.g. during dirty logging we may
 * have such mappings.
1343 1344 1345 1346 1347
 */
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
				struct kvm_memory_slot *slot,
				gfn_t gfn_offset, unsigned long mask)
{
1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
	/*
	 * Huge pages are NOT write protected when we start dirty logging in
	 * initially-all-set mode; must write protect them here so that they
	 * are split to 4K on the first write.
	 *
	 * The gfn_offset is guaranteed to be aligned to 64, but the base_gfn
	 * of memslot has no such restriction, so the range can cross two large
	 * pages.
	 */
	if (kvm_dirty_log_manual_protect_and_init_set(kvm)) {
		gfn_t start = slot->base_gfn + gfn_offset + __ffs(mask);
		gfn_t end = slot->base_gfn + gfn_offset + __fls(mask);

1361 1362 1363
		if (READ_ONCE(eager_page_split))
			kvm_mmu_try_split_huge_pages(kvm, slot, start, end, PG_LEVEL_4K);

1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
		kvm_mmu_slot_gfn_write_protect(kvm, slot, start, PG_LEVEL_2M);

		/* Cross two large pages? */
		if (ALIGN(start << PAGE_SHIFT, PMD_SIZE) !=
		    ALIGN(end << PAGE_SHIFT, PMD_SIZE))
			kvm_mmu_slot_gfn_write_protect(kvm, slot, end,
						       PG_LEVEL_2M);
	}

	/* Now handle 4K PTEs.  */
1374 1375
	if (kvm_x86_ops.cpu_dirty_log_size)
		kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask);
1376 1377
	else
		kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask);
1378 1379
}

1380 1381
int kvm_cpu_dirty_log_size(void)
{
1382
	return kvm_x86_ops.cpu_dirty_log_size;
1383 1384
}

1385
bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
1386 1387
				    struct kvm_memory_slot *slot, u64 gfn,
				    int min_level)
1388
{
1389
	struct kvm_rmap_head *rmap_head;
1390
	int i;
1391
	bool write_protected = false;
1392

1393 1394
	if (kvm_memslots_have_rmaps(kvm)) {
		for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
1395
			rmap_head = gfn_to_rmap(gfn, i, slot);
1396
			write_protected |= rmap_write_protect(rmap_head, true);
1397
		}
1398 1399
	}

1400
	if (is_tdp_mmu_enabled(kvm))
1401
		write_protected |=
1402
			kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level);
1403

1404
	return write_protected;
1405 1406
}

1407
static bool kvm_vcpu_write_protect_gfn(struct kvm_vcpu *vcpu, u64 gfn)
1408 1409 1410 1411
{
	struct kvm_memory_slot *slot;

	slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
1412
	return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn, PG_LEVEL_4K);
1413 1414
}

1415
static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
1416
			  const struct kvm_memory_slot *slot)
1417
{
1418
	return pte_list_destroy(kvm, rmap_head);
1419 1420
}

1421 1422 1423
static bool kvm_unmap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
			    struct kvm_memory_slot *slot, gfn_t gfn, int level,
			    pte_t unused)
1424
{
1425
	return kvm_zap_rmapp(kvm, rmap_head, slot);
1426 1427
}

1428 1429 1430
static bool kvm_set_pte_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
			      struct kvm_memory_slot *slot, gfn_t gfn, int level,
			      pte_t pte)
1431
{
1432 1433
	u64 *sptep;
	struct rmap_iterator iter;
1434
	bool need_flush = false;
1435
	u64 new_spte;
1436
	kvm_pfn_t new_pfn;
1437

1438 1439
	WARN_ON(pte_huge(pte));
	new_pfn = pte_pfn(pte);
1440

1441
restart:
1442
	for_each_rmap_spte(rmap_head, &iter, sptep) {
1443
		rmap_printk("spte %p %llx gfn %llx (%d)\n",
1444
			    sptep, *sptep, gfn, level);
1445

1446
		need_flush = true;
1447

1448
		if (pte_write(pte)) {
1449
			pte_list_remove(kvm, rmap_head, sptep);
1450
			goto restart;
1451
		} else {
1452 1453
			new_spte = kvm_mmu_changed_pte_notifier_make_spte(
					*sptep, new_pfn);
1454

1455
			mmu_spte_clear_track_bits(kvm, sptep);
1456
			mmu_spte_set(sptep, new_spte);
1457 1458
		}
	}
1459

1460 1461
	if (need_flush && kvm_available_flush_tlb_with_range()) {
		kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);
1462
		return false;
1463 1464
	}

1465
	return need_flush;
1466 1467
}

1468 1469
struct slot_rmap_walk_iterator {
	/* input fields. */
1470
	const struct kvm_memory_slot *slot;
1471 1472 1473 1474 1475 1476 1477
	gfn_t start_gfn;
	gfn_t end_gfn;
	int start_level;
	int end_level;

	/* output fields. */
	gfn_t gfn;
1478
	struct kvm_rmap_head *rmap;
1479 1480 1481
	int level;

	/* private field. */
1482
	struct kvm_rmap_head *end_rmap;
1483 1484 1485 1486 1487 1488 1489
};

static void
rmap_walk_init_level(struct slot_rmap_walk_iterator *iterator, int level)
{
	iterator->level = level;
	iterator->gfn = iterator->start_gfn;
1490 1491
	iterator->rmap = gfn_to_rmap(iterator->gfn, level, iterator->slot);
	iterator->end_rmap = gfn_to_rmap(iterator->end_gfn, level, iterator->slot);
1492 1493 1494 1495
}

static void
slot_rmap_walk_init(struct slot_rmap_walk_iterator *iterator,
1496
		    const struct kvm_memory_slot *slot, int start_level,
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534
		    int end_level, gfn_t start_gfn, gfn_t end_gfn)
{
	iterator->slot = slot;
	iterator->start_level = start_level;
	iterator->end_level = end_level;
	iterator->start_gfn = start_gfn;
	iterator->end_gfn = end_gfn;

	rmap_walk_init_level(iterator, iterator->start_level);
}

static bool slot_rmap_walk_okay(struct slot_rmap_walk_iterator *iterator)
{
	return !!iterator->rmap;
}

static void slot_rmap_walk_next(struct slot_rmap_walk_iterator *iterator)
{
	if (++iterator->rmap <= iterator->end_rmap) {
		iterator->gfn += (1UL << KVM_HPAGE_GFN_SHIFT(iterator->level));
		return;
	}

	if (++iterator->level > iterator->end_level) {
		iterator->rmap = NULL;
		return;
	}

	rmap_walk_init_level(iterator, iterator->level);
}

#define for_each_slot_rmap_range(_slot_, _start_level_, _end_level_,	\
	   _start_gfn, _end_gfn, _iter_)				\
	for (slot_rmap_walk_init(_iter_, _slot_, _start_level_,		\
				 _end_level_, _start_gfn, _end_gfn);	\
	     slot_rmap_walk_okay(_iter_);				\
	     slot_rmap_walk_next(_iter_))

1535 1536 1537
typedef bool (*rmap_handler_t)(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
			       struct kvm_memory_slot *slot, gfn_t gfn,
			       int level, pte_t pte);
1538

1539 1540 1541
static __always_inline bool kvm_handle_gfn_range(struct kvm *kvm,
						 struct kvm_gfn_range *range,
						 rmap_handler_t handler)
1542
{
1543
	struct slot_rmap_walk_iterator iterator;
1544
	bool ret = false;
1545

1546 1547 1548 1549
	for_each_slot_rmap_range(range->slot, PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
				 range->start, range->end - 1, &iterator)
		ret |= handler(kvm, iterator.rmap, range->slot, iterator.gfn,
			       iterator.level, range->pte);
1550

1551
	return ret;
1552 1553
}

1554
bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
1555
{
1556
	bool flush = false;
1557

1558 1559
	if (kvm_memslots_have_rmaps(kvm))
		flush = kvm_handle_gfn_range(kvm, range, kvm_unmap_rmapp);
1560

1561
	if (is_tdp_mmu_enabled(kvm))
1562
		flush = kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush);
1563

1564
	return flush;
1565 1566
}

1567
bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1568
{
1569
	bool flush = false;
1570

1571 1572
	if (kvm_memslots_have_rmaps(kvm))
		flush = kvm_handle_gfn_range(kvm, range, kvm_set_pte_rmapp);
1573

1574
	if (is_tdp_mmu_enabled(kvm))
1575
		flush |= kvm_tdp_mmu_set_spte_gfn(kvm, range);
1576

1577
	return flush;
1578 1579
}

1580 1581 1582
static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
			  struct kvm_memory_slot *slot, gfn_t gfn, int level,
			  pte_t unused)
1583
{
1584
	u64 *sptep;
1585
	struct rmap_iterator iter;
1586 1587
	int young = 0;

1588 1589
	for_each_rmap_spte(rmap_head, &iter, sptep)
		young |= mmu_spte_age(sptep);
1590

1591 1592 1593
	return young;
}

1594 1595 1596
static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head,
			       struct kvm_memory_slot *slot, gfn_t gfn,
			       int level, pte_t unused)
1597
{
1598 1599
	u64 *sptep;
	struct rmap_iterator iter;
1600

1601 1602
	for_each_rmap_spte(rmap_head, &iter, sptep)
		if (is_accessed_spte(*sptep))
1603 1604
			return true;
	return false;
1605 1606
}

1607 1608
#define RMAP_RECYCLE_THRESHOLD 1000

1609 1610
static void rmap_add(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
		     u64 *spte, gfn_t gfn)
1611
{
1612
	struct kvm_mmu_page *sp;
1613 1614
	struct kvm_rmap_head *rmap_head;
	int rmap_count;
1615

1616
	sp = sptep_to_sp(spte);
1617
	kvm_mmu_page_set_gfn(sp, spte - sp->spt, gfn);
1618
	rmap_head = gfn_to_rmap(gfn, sp->role.level, slot);
1619
	rmap_count = pte_list_add(vcpu, spte, rmap_head);
1620

1621 1622 1623 1624 1625
	if (rmap_count > RMAP_RECYCLE_THRESHOLD) {
		kvm_unmap_rmapp(vcpu->kvm, rmap_head, NULL, gfn, sp->role.level, __pte(0));
		kvm_flush_remote_tlbs_with_address(
				vcpu->kvm, sp->gfn, KVM_PAGES_PER_HPAGE(sp->role.level));
	}
1626 1627
}

1628
bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1629
{
1630
	bool young = false;
1631

1632 1633
	if (kvm_memslots_have_rmaps(kvm))
		young = kvm_handle_gfn_range(kvm, range, kvm_age_rmapp);
1634

1635
	if (is_tdp_mmu_enabled(kvm))
1636
		young |= kvm_tdp_mmu_age_gfn_range(kvm, range);
1637 1638

	return young;
1639 1640
}

1641
bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
1642
{
1643
	bool young = false;
1644

1645 1646
	if (kvm_memslots_have_rmaps(kvm))
		young = kvm_handle_gfn_range(kvm, range, kvm_test_age_rmapp);
1647

1648
	if (is_tdp_mmu_enabled(kvm))
1649
		young |= kvm_tdp_mmu_test_age_gfn(kvm, range);
1650 1651

	return young;
1652 1653
}

1654
#ifdef MMU_DEBUG
1655
static int is_empty_shadow_page(u64 *spt)
1656
{
1657 1658 1659
	u64 *pos;
	u64 *end;

1660
	for (pos = spt, end = pos + PAGE_SIZE / sizeof(u64); pos != end; pos++)
1661
		if (is_shadow_present_pte(*pos)) {
1662
			printk(KERN_ERR "%s: %p %llx\n", __func__,
1663
			       pos, *pos);
1664
			return 0;
1665
		}
1666 1667
	return 1;
}
1668
#endif
1669

1670 1671 1672 1673 1674 1675
/*
 * This value is the sum of all of the kvm instances's
 * kvm->arch.n_used_mmu_pages values.  We need a global,
 * aggregate version in order to make the slab shrinker
 * faster
 */
1676
static inline void kvm_mod_used_mmu_pages(struct kvm *kvm, long nr)
1677 1678 1679 1680 1681
{
	kvm->arch.n_used_mmu_pages += nr;
	percpu_counter_add(&kvm_total_used_mmu_pages, nr);
}

1682
static void kvm_mmu_free_page(struct kvm_mmu_page *sp)
1683
{
1684
	MMU_WARN_ON(!is_empty_shadow_page(sp->spt));
1685
	hlist_del(&sp->hash_link);
1686 1687
	list_del(&sp->link);
	free_page((unsigned long)sp->spt);
1688 1689
	if (!sp->role.direct)
		free_page((unsigned long)sp->gfns);
1690
	kmem_cache_free(mmu_page_header_cache, sp);
1691 1692
}

1693 1694
static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
1695
	return hash_64(gfn, KVM_MMU_HASH_SHIFT);
1696 1697
}

1698
static void mmu_page_add_parent_pte(struct kvm_vcpu *vcpu,
1699
				    struct kvm_mmu_page *sp, u64 *parent_pte)
1700 1701 1702 1703
{
	if (!parent_pte)
		return;

1704
	pte_list_add(vcpu, parent_pte, &sp->parent_ptes);
1705 1706
}

1707
static void mmu_page_remove_parent_pte(struct kvm_mmu_page *sp,
1708 1709
				       u64 *parent_pte)
{
1710
	__pte_list_remove(parent_pte, &sp->parent_ptes);
1711 1712
}

1713 1714 1715 1716
static void drop_parent_pte(struct kvm_mmu_page *sp,
			    u64 *parent_pte)
{
	mmu_page_remove_parent_pte(sp, parent_pte);
1717
	mmu_spte_clear_no_track(parent_pte);
1718 1719
}

1720
static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, int direct)
Marcelo Tosatti's avatar
Marcelo Tosatti committed
1721
{
1722
	struct kvm_mmu_page *sp;
1723

1724 1725
	sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
	sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache);
1726
	if (!direct)
1727
		sp->gfns = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_gfn_array_cache);
1728
	set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
1729 1730 1731 1732 1733 1734

	/*
	 * active_mmu_pages must be a FIFO list, as kvm_zap_obsolete_pages()
	 * depends on valid pages being added to the head of the list.  See
	 * comments in kvm_zap_obsolete_pages().
	 */
1735
	sp->mmu_valid_gen = vcpu->kvm->arch.mmu_valid_gen;
1736 1737 1738
	list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages);
	kvm_mod_used_mmu_pages(vcpu->kvm, +1);
	return sp;
Marcelo Tosatti's avatar
Marcelo Tosatti committed
1739 1740
}

1741
static void mark_unsync(u64 *spte);
1742
static void kvm_mmu_mark_parents_unsync(struct kvm_mmu_page *sp)
1743
{
1744 1745 1746 1747 1748 1749
	u64 *sptep;
	struct rmap_iterator iter;

	for_each_rmap_spte(&sp->parent_ptes, &iter, sptep) {
		mark_unsync(sptep);
	}
1750 1751
}

1752
static void mark_unsync(u64 *spte)
1753
{
1754
	struct kvm_mmu_page *sp;
1755
	unsigned int index;
1756

1757
	sp = sptep_to_sp(spte);
1758 1759
	index = spte - sp->spt;
	if (__test_and_set_bit(index, sp->unsync_child_bitmap))
1760
		return;
1761
	if (sp->unsync_children++)
1762
		return;
1763
	kvm_mmu_mark_parents_unsync(sp);
1764 1765
}

1766
static int nonpaging_sync_page(struct kvm_vcpu *vcpu,
1767
			       struct kvm_mmu_page *sp)
1768
{
1769
	return -1;
1770 1771
}

1772 1773 1774 1775 1776 1777 1778 1779 1780 1781
#define KVM_PAGE_ARRAY_NR 16

struct kvm_mmu_pages {
	struct mmu_page_and_offset {
		struct kvm_mmu_page *sp;
		unsigned int idx;
	} page[KVM_PAGE_ARRAY_NR];
	unsigned int nr;
};

1782 1783
static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp,
			 int idx)
1784
{
1785
	int i;
1786

1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
	if (sp->unsync)
		for (i=0; i < pvec->nr; i++)
			if (pvec->page[i].sp == sp)
				return 0;

	pvec->page[pvec->nr].sp = sp;
	pvec->page[pvec->nr].idx = idx;
	pvec->nr++;
	return (pvec->nr == KVM_PAGE_ARRAY_NR);
}

1798 1799 1800 1801 1802 1803 1804
static inline void clear_unsync_child_bit(struct kvm_mmu_page *sp, int idx)
{
	--sp->unsync_children;
	WARN_ON((int)sp->unsync_children < 0);
	__clear_bit(idx, sp->unsync_child_bitmap);
}

1805 1806 1807 1808
static int __mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
	int i, ret, nr_unsync_leaf = 0;
1809

1810
	for_each_set_bit(i, sp->unsync_child_bitmap, 512) {
1811
		struct kvm_mmu_page *child;
1812 1813
		u64 ent = sp->spt[i];

1814 1815 1816 1817
		if (!is_shadow_present_pte(ent) || is_large_pte(ent)) {
			clear_unsync_child_bit(sp, i);
			continue;
		}
1818

1819
		child = to_shadow_page(ent & PT64_BASE_ADDR_MASK);
1820 1821 1822 1823 1824 1825

		if (child->unsync_children) {
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;

			ret = __mmu_unsync_walk(child, pvec);
1826 1827 1828 1829
			if (!ret) {
				clear_unsync_child_bit(sp, i);
				continue;
			} else if (ret > 0) {
1830
				nr_unsync_leaf += ret;
1831
			} else
1832 1833 1834 1835 1836 1837
				return ret;
		} else if (child->unsync) {
			nr_unsync_leaf++;
			if (mmu_pages_add(pvec, child, i))
				return -ENOSPC;
		} else
1838
			clear_unsync_child_bit(sp, i);
1839 1840
	}

1841 1842 1843
	return nr_unsync_leaf;
}

1844 1845
#define INVALID_INDEX (-1)

1846 1847 1848
static int mmu_unsync_walk(struct kvm_mmu_page *sp,
			   struct kvm_mmu_pages *pvec)
{
1849
	pvec->nr = 0;
1850 1851 1852
	if (!sp->unsync_children)
		return 0;

1853
	mmu_pages_add(pvec, sp, INVALID_INDEX);
1854
	return __mmu_unsync_walk(sp, pvec);
1855 1856 1857 1858 1859
}

static void kvm_unlink_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
{
	WARN_ON(!sp->unsync);
1860
	trace_kvm_mmu_sync_page(sp);
1861 1862 1863 1864
	sp->unsync = 0;
	--kvm->stat.mmu_unsync;
}

1865 1866
static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				     struct list_head *invalid_list);
1867 1868
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list);
1869

1870 1871
#define for_each_valid_sp(_kvm, _sp, _list)				\
	hlist_for_each_entry(_sp, _list, hash_link)			\
1872
		if (is_obsolete_sp((_kvm), (_sp))) {			\
1873
		} else
1874 1875

#define for_each_gfn_indirect_valid_sp(_kvm, _sp, _gfn)			\
1876 1877
	for_each_valid_sp(_kvm, _sp,					\
	  &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)])	\
1878
		if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else
1879

1880 1881
static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
			 struct list_head *invalid_list)
1882
{
1883 1884 1885
	int ret = vcpu->arch.mmu->sync_page(vcpu, sp);

	if (ret < 0) {
1886
		kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list);
1887
		return false;
1888 1889
	}

1890
	return !!ret;
1891 1892
}

1893 1894 1895 1896
static bool kvm_mmu_remote_flush_or_zap(struct kvm *kvm,
					struct list_head *invalid_list,
					bool remote_flush)
{
1897
	if (!remote_flush && list_empty(invalid_list))
1898 1899 1900 1901 1902 1903 1904 1905 1906
		return false;

	if (!list_empty(invalid_list))
		kvm_mmu_commit_zap_page(kvm, invalid_list);
	else
		kvm_flush_remote_tlbs(kvm);
	return true;
}

1907 1908 1909 1910 1911 1912 1913
#ifdef CONFIG_KVM_MMU_AUDIT
#include "mmu_audit.c"
#else
static void kvm_mmu_audit(struct kvm_vcpu *vcpu, int point) { }
static void mmu_audit_disable(void) { }
#endif

1914 1915
static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
{
1916 1917 1918 1919 1920
	if (sp->role.invalid)
		return true;

	/* TDP MMU pages due not use the MMU generation. */
	return !sp->tdp_mmu_page &&
1921
	       unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen);
1922 1923
}

1924
struct mmu_page_path {
1925 1926
	struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL];
	unsigned int idx[PT64_ROOT_MAX_LEVEL];
1927 1928
};

1929
#define for_each_sp(pvec, sp, parents, i)			\
1930
		for (i = mmu_pages_first(&pvec, &parents);	\
1931 1932 1933
			i < pvec.nr && ({ sp = pvec.page[i].sp; 1;});	\
			i = mmu_pages_next(&pvec, &parents, i))

1934 1935 1936
static int mmu_pages_next(struct kvm_mmu_pages *pvec,
			  struct mmu_page_path *parents,
			  int i)
1937 1938 1939 1940 1941
{
	int n;

	for (n = i+1; n < pvec->nr; n++) {
		struct kvm_mmu_page *sp = pvec->page[n].sp;
1942 1943
		unsigned idx = pvec->page[n].idx;
		int level = sp->role.level;
1944

1945
		parents->idx[level-1] = idx;
1946
		if (level == PG_LEVEL_4K)
1947
			break;
1948

1949
		parents->parent[level-2] = sp;
1950 1951 1952 1953 1954
	}

	return n;
}

1955 1956 1957 1958 1959 1960 1961 1962 1963
static int mmu_pages_first(struct kvm_mmu_pages *pvec,
			   struct mmu_page_path *parents)
{
	struct kvm_mmu_page *sp;
	int level;

	if (pvec->nr == 0)
		return 0;

1964 1965
	WARN_ON(pvec->page[0].idx != INVALID_INDEX);

1966 1967
	sp = pvec->page[0].sp;
	level = sp->role.level;
1968
	WARN_ON(level == PG_LEVEL_4K);
1969 1970 1971 1972 1973 1974 1975 1976 1977 1978

	parents->parent[level-2] = sp;

	/* Also set up a sentinel.  Further entries in pvec are all
	 * children of sp, so this element is never overwritten.
	 */
	parents->parent[level-1] = NULL;
	return mmu_pages_next(pvec, parents, 0);
}

1979
static void mmu_pages_clear_parents(struct mmu_page_path *parents)
1980
{
1981 1982 1983 1984 1985 1986 1987 1988 1989
	struct kvm_mmu_page *sp;
	unsigned int level = 0;

	do {
		unsigned int idx = parents->idx[level];
		sp = parents->parent[level];
		if (!sp)
			return;

1990
		WARN_ON(idx == INVALID_INDEX);
1991
		clear_unsync_child_bit(sp, idx);
1992
		level++;
1993
	} while (!sp->unsync_children);
1994
}
1995

1996 1997
static int mmu_sync_children(struct kvm_vcpu *vcpu,
			     struct kvm_mmu_page *parent, bool can_yield)
1998 1999 2000 2001 2002
{
	int i;
	struct kvm_mmu_page *sp;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2003
	LIST_HEAD(invalid_list);
2004
	bool flush = false;
2005 2006

	while (mmu_unsync_walk(parent, &pages)) {
2007
		bool protected = false;
2008 2009

		for_each_sp(pages, sp, parents, i)
2010
			protected |= kvm_vcpu_write_protect_gfn(vcpu, sp->gfn);
2011

2012
		if (protected) {
2013
			kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, true);
2014 2015
			flush = false;
		}
2016

2017
		for_each_sp(pages, sp, parents, i) {
2018
			kvm_unlink_unsync_page(vcpu->kvm, sp);
2019
			flush |= kvm_sync_page(vcpu, sp, &invalid_list);
2020 2021
			mmu_pages_clear_parents(&parents);
		}
2022
		if (need_resched() || rwlock_needbreak(&vcpu->kvm->mmu_lock)) {
2023
			kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
2024 2025 2026 2027 2028
			if (!can_yield) {
				kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
				return -EINTR;
			}

2029
			cond_resched_rwlock_write(&vcpu->kvm->mmu_lock);
2030 2031
			flush = false;
		}
2032
	}
2033

2034
	kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
2035
	return 0;
2036 2037
}

2038 2039
static void __clear_sp_write_flooding_count(struct kvm_mmu_page *sp)
{
2040
	atomic_set(&sp->write_flooding_count,  0);
2041 2042 2043 2044
}

static void clear_sp_write_flooding_count(u64 *spte)
{
2045
	__clear_sp_write_flooding_count(sptep_to_sp(spte));
2046 2047
}

2048 2049 2050 2051
static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu,
					     gfn_t gfn,
					     gva_t gaddr,
					     unsigned level,
2052
					     int direct,
2053
					     unsigned int access)
2054
{
2055
	bool direct_mmu = vcpu->arch.mmu->direct_map;
2056
	union kvm_mmu_page_role role;
2057
	struct hlist_head *sp_list;
2058
	unsigned quadrant;
2059
	struct kvm_mmu_page *sp;
2060
	int collisions = 0;
2061
	LIST_HEAD(invalid_list);
2062

2063
	role = vcpu->arch.mmu->mmu_role.base;
2064
	role.level = level;
2065
	role.direct = direct;
2066
	role.access = access;
2067
	if (role.has_4_byte_gpte) {
2068 2069 2070 2071
		quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level));
		quadrant &= (1 << ((PT32_PT_BITS - PT64_PT_BITS) * level)) - 1;
		role.quadrant = quadrant;
	}
2072 2073 2074

	sp_list = &vcpu->kvm->arch.mmu_page_hash[kvm_page_table_hashfn(gfn)];
	for_each_valid_sp(vcpu->kvm, sp, sp_list) {
2075 2076 2077 2078 2079
		if (sp->gfn != gfn) {
			collisions++;
			continue;
		}

2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092
		if (sp->role.word != role.word) {
			/*
			 * If the guest is creating an upper-level page, zap
			 * unsync pages for the same gfn.  While it's possible
			 * the guest is using recursive page tables, in all
			 * likelihood the guest has stopped using the unsync
			 * page and is installing a completely unrelated page.
			 * Unsync pages must not be left as is, because the new
			 * upper-level page will be write-protected.
			 */
			if (level > PG_LEVEL_4K && sp->unsync)
				kvm_mmu_prepare_zap_page(vcpu->kvm, sp,
							 &invalid_list);
2093
			continue;
2094
		}
2095

2096 2097 2098
		if (direct_mmu)
			goto trace_get_page;

2099
		if (sp->unsync) {
2100
			/*
2101
			 * The page is good, but is stale.  kvm_sync_page does
2102 2103 2104 2105 2106 2107 2108 2109 2110
			 * get the latest guest state, but (unlike mmu_unsync_children)
			 * it doesn't write-protect the page or mark it synchronized!
			 * This way the validity of the mapping is ensured, but the
			 * overhead of write protection is not incurred until the
			 * guest invalidates the TLB mapping.  This allows multiple
			 * SPs for a single gfn to be unsync.
			 *
			 * If the sync fails, the page is zapped.  If so, break
			 * in order to rebuild it.
2111
			 */
2112
			if (!kvm_sync_page(vcpu, sp, &invalid_list))
2113 2114 2115
				break;

			WARN_ON(!list_empty(&invalid_list));
2116
			kvm_flush_remote_tlbs(vcpu->kvm);
2117
		}
2118

2119
		__clear_sp_write_flooding_count(sp);
2120 2121

trace_get_page:
2122
		trace_kvm_mmu_get_page(sp, false);
2123
		goto out;
2124
	}
2125

2126
	++vcpu->kvm->stat.mmu_cache_miss;
2127 2128 2129

	sp = kvm_mmu_alloc_page(vcpu, direct);

2130 2131
	sp->gfn = gfn;
	sp->role = role;
2132
	hlist_add_head(&sp->hash_link, sp_list);
2133
	if (!direct) {
2134
		account_shadowed(vcpu->kvm, sp);
2135
		if (level == PG_LEVEL_4K && kvm_vcpu_write_protect_gfn(vcpu, gfn))
2136
			kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1);
2137
	}
2138
	trace_kvm_mmu_get_page(sp, true);
2139
out:
2140 2141
	kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list);

2142 2143
	if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions)
		vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions;
2144
	return sp;
2145 2146
}

2147 2148 2149
static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,
					struct kvm_vcpu *vcpu, hpa_t root,
					u64 addr)
2150 2151
{
	iterator->addr = addr;
2152
	iterator->shadow_addr = root;
2153
	iterator->level = vcpu->arch.mmu->shadow_root_level;
2154

2155
	if (iterator->level >= PT64_ROOT_4LEVEL &&
2156 2157
	    vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL &&
	    !vcpu->arch.mmu->direct_map)
2158
		iterator->level = PT32E_ROOT_LEVEL;
2159

2160
	if (iterator->level == PT32E_ROOT_LEVEL) {
2161 2162 2163 2164
		/*
		 * prev_root is currently only used for 64-bit hosts. So only
		 * the active root_hpa is valid here.
		 */
2165
		BUG_ON(root != vcpu->arch.mmu->root_hpa);
2166

2167
		iterator->shadow_addr
2168
			= vcpu->arch.mmu->pae_root[(addr >> 30) & 3];
2169 2170 2171 2172 2173 2174 2175
		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
		--iterator->level;
		if (!iterator->shadow_addr)
			iterator->level = 0;
	}
}

2176 2177 2178
static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
			     struct kvm_vcpu *vcpu, u64 addr)
{
2179
	shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root_hpa,
2180 2181 2182
				    addr);
}

2183 2184
static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator)
{
2185
	if (iterator->level < PG_LEVEL_4K)
2186
		return false;
2187

2188 2189 2190 2191 2192
	iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level);
	iterator->sptep	= ((u64 *)__va(iterator->shadow_addr)) + iterator->index;
	return true;
}

2193 2194
static void __shadow_walk_next(struct kvm_shadow_walk_iterator *iterator,
			       u64 spte)
2195
{
2196
	if (!is_shadow_present_pte(spte) || is_last_spte(spte, iterator->level)) {
2197 2198 2199 2200
		iterator->level = 0;
		return;
	}

2201
	iterator->shadow_addr = spte & PT64_BASE_ADDR_MASK;
2202 2203 2204
	--iterator->level;
}

2205 2206
static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator)
{
2207
	__shadow_walk_next(iterator, *iterator->sptep);
2208 2209
}

2210 2211 2212 2213 2214 2215 2216 2217 2218
static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
			     struct kvm_mmu_page *sp)
{
	u64 spte;

	BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);

	spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));

2219
	mmu_spte_set(sptep, spte);
2220 2221 2222 2223 2224

	mmu_page_add_parent_pte(vcpu, sp, sptep);

	if (sp->unsync_children || sp->unsync)
		mark_unsync(sptep);
2225 2226
}

2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239
static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
				   unsigned direct_access)
{
	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
		struct kvm_mmu_page *child;

		/*
		 * For the direct sp, if the guest pte's dirty bit
		 * changed form clean to dirty, it will corrupt the
		 * sp's access: allow writable in the read-only sp,
		 * so we should update the spte at this point to get
		 * a new sp with the correct access.
		 */
2240
		child = to_shadow_page(*sptep & PT64_BASE_ADDR_MASK);
2241 2242 2243
		if (child->role.access == direct_access)
			return;

2244
		drop_parent_pte(child, sptep);
2245
		kvm_flush_remote_tlbs_with_address(vcpu->kvm, child->gfn, 1);
2246 2247 2248
	}
}

2249 2250 2251
/* Returns the number of zapped non-leaf child shadow pages. */
static int mmu_page_zap_pte(struct kvm *kvm, struct kvm_mmu_page *sp,
			    u64 *spte, struct list_head *invalid_list)
2252 2253 2254 2255 2256 2257
{
	u64 pte;
	struct kvm_mmu_page *child;

	pte = *spte;
	if (is_shadow_present_pte(pte)) {
2258
		if (is_last_spte(pte, sp->role.level)) {
2259
			drop_spte(kvm, spte);
2260
		} else {
2261
			child = to_shadow_page(pte & PT64_BASE_ADDR_MASK);
2262
			drop_parent_pte(child, spte);
2263 2264 2265 2266 2267 2268 2269 2270 2271 2272

			/*
			 * Recursively zap nested TDP SPs, parentless SPs are
			 * unlikely to be used again in the near future.  This
			 * avoids retaining a large number of stale nested SPs.
			 */
			if (tdp_enabled && invalid_list &&
			    child->role.guest_mode && !child->parent_ptes.val)
				return kvm_mmu_prepare_zap_page(kvm, child,
								invalid_list);
2273
		}
2274
	} else if (is_mmio_spte(pte)) {
2275
		mmu_spte_clear_no_track(spte);
2276
	}
2277
	return 0;
2278 2279
}

2280 2281 2282
static int kvm_mmu_page_unlink_children(struct kvm *kvm,
					struct kvm_mmu_page *sp,
					struct list_head *invalid_list)
2283
{
2284
	int zapped = 0;
2285 2286
	unsigned i;

2287
	for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
2288 2289 2290
		zapped += mmu_page_zap_pte(kvm, sp, sp->spt + i, invalid_list);

	return zapped;
2291 2292
}

2293
static void kvm_mmu_unlink_parents(struct kvm_mmu_page *sp)
2294
{
2295 2296
	u64 *sptep;
	struct rmap_iterator iter;
2297

2298
	while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
2299
		drop_parent_pte(sp, sptep);
2300 2301
}

2302
static int mmu_zap_unsync_children(struct kvm *kvm,
2303 2304
				   struct kvm_mmu_page *parent,
				   struct list_head *invalid_list)
2305
{
2306 2307 2308
	int i, zapped = 0;
	struct mmu_page_path parents;
	struct kvm_mmu_pages pages;
2309

2310
	if (parent->role.level == PG_LEVEL_4K)
2311
		return 0;
2312 2313 2314 2315 2316

	while (mmu_unsync_walk(parent, &pages)) {
		struct kvm_mmu_page *sp;

		for_each_sp(pages, sp, parents, i) {
2317
			kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
2318
			mmu_pages_clear_parents(&parents);
2319
			zapped++;
2320 2321 2322 2323
		}
	}

	return zapped;
2324 2325
}

2326 2327 2328 2329
static bool __kvm_mmu_prepare_zap_page(struct kvm *kvm,
				       struct kvm_mmu_page *sp,
				       struct list_head *invalid_list,
				       int *nr_zapped)
2330
{
2331
	bool list_unstable;
2332

2333
	trace_kvm_mmu_prepare_zap_page(sp);
2334
	++kvm->stat.mmu_shadow_zapped;
2335
	*nr_zapped = mmu_zap_unsync_children(kvm, sp, invalid_list);
2336
	*nr_zapped += kvm_mmu_page_unlink_children(kvm, sp, invalid_list);
2337
	kvm_mmu_unlink_parents(sp);
2338

2339 2340 2341
	/* Zapping children means active_mmu_pages has become unstable. */
	list_unstable = *nr_zapped;

2342
	if (!sp->role.invalid && !sp->role.direct)
2343
		unaccount_shadowed(kvm, sp);
2344

2345 2346
	if (sp->unsync)
		kvm_unlink_unsync_page(kvm, sp);
2347
	if (!sp->root_count) {
2348
		/* Count self */
2349
		(*nr_zapped)++;
2350 2351 2352 2353 2354 2355 2356 2357 2358 2359

		/*
		 * Already invalid pages (previously active roots) are not on
		 * the active page list.  See list_del() in the "else" case of
		 * !sp->root_count.
		 */
		if (sp->role.invalid)
			list_add(&sp->link, invalid_list);
		else
			list_move(&sp->link, invalid_list);
2360
		kvm_mod_used_mmu_pages(kvm, -1);
2361
	} else {
2362 2363 2364 2365 2366
		/*
		 * Remove the active root from the active page list, the root
		 * will be explicitly freed when the root_count hits zero.
		 */
		list_del(&sp->link);
2367

2368 2369 2370 2371 2372 2373
		/*
		 * Obsolete pages cannot be used on any vCPUs, see the comment
		 * in kvm_mmu_zap_all_fast().  Note, is_obsolete_sp() also
		 * treats invalid shadow pages as being obsolete.
		 */
		if (!is_obsolete_sp(kvm, sp))
2374
			kvm_reload_remote_mmus(kvm);
2375
	}
2376

2377 2378 2379
	if (sp->lpage_disallowed)
		unaccount_huge_nx_page(kvm, sp);

2380
	sp->role.invalid = 1;
2381 2382 2383 2384 2385 2386 2387 2388 2389 2390
	return list_unstable;
}

static bool kvm_mmu_prepare_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp,
				     struct list_head *invalid_list)
{
	int nr_zapped;

	__kvm_mmu_prepare_zap_page(kvm, sp, invalid_list, &nr_zapped);
	return nr_zapped;
2391 2392
}

2393 2394 2395
static void kvm_mmu_commit_zap_page(struct kvm *kvm,
				    struct list_head *invalid_list)
{
2396
	struct kvm_mmu_page *sp, *nsp;
2397 2398 2399 2400

	if (list_empty(invalid_list))
		return;

2401
	/*
2402 2403 2404 2405 2406 2407 2408
	 * We need to make sure everyone sees our modifications to
	 * the page tables and see changes to vcpu->mode here. The barrier
	 * in the kvm_flush_remote_tlbs() achieves this. This pairs
	 * with vcpu_enter_guest and walk_shadow_page_lockless_begin/end.
	 *
	 * In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
	 * guest mode and/or lockless shadow page table walks.
2409 2410
	 */
	kvm_flush_remote_tlbs(kvm);
2411

2412
	list_for_each_entry_safe(sp, nsp, invalid_list, link) {
2413
		WARN_ON(!sp->role.invalid || sp->root_count);
2414
		kvm_mmu_free_page(sp);
2415
	}
2416 2417
}

2418 2419
static unsigned long kvm_mmu_zap_oldest_mmu_pages(struct kvm *kvm,
						  unsigned long nr_to_zap)
2420
{
2421 2422
	unsigned long total_zapped = 0;
	struct kvm_mmu_page *sp, *tmp;
2423
	LIST_HEAD(invalid_list);
2424 2425
	bool unstable;
	int nr_zapped;
2426 2427

	if (list_empty(&kvm->arch.active_mmu_pages))
2428 2429
		return 0;

2430
restart:
2431
	list_for_each_entry_safe_reverse(sp, tmp, &kvm->arch.active_mmu_pages, link) {
2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
		/*
		 * Don't zap active root pages, the page itself can't be freed
		 * and zapping it will just force vCPUs to realloc and reload.
		 */
		if (sp->root_count)
			continue;

		unstable = __kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list,
						      &nr_zapped);
		total_zapped += nr_zapped;
		if (total_zapped >= nr_to_zap)
2443 2444
			break;

2445 2446
		if (unstable)
			goto restart;
2447
	}
2448

2449 2450 2451 2452 2453 2454
	kvm_mmu_commit_zap_page(kvm, &invalid_list);

	kvm->stat.mmu_recycled += total_zapped;
	return total_zapped;
}

2455 2456 2457 2458 2459 2460 2461
static inline unsigned long kvm_mmu_available_pages(struct kvm *kvm)
{
	if (kvm->arch.n_max_mmu_pages > kvm->arch.n_used_mmu_pages)
		return kvm->arch.n_max_mmu_pages -
			kvm->arch.n_used_mmu_pages;

	return 0;
2462 2463
}

2464 2465
static int make_mmu_pages_available(struct kvm_vcpu *vcpu)
{
2466
	unsigned long avail = kvm_mmu_available_pages(vcpu->kvm);
2467

2468
	if (likely(avail >= KVM_MIN_FREE_MMU_PAGES))
2469 2470
		return 0;

2471
	kvm_mmu_zap_oldest_mmu_pages(vcpu->kvm, KVM_REFILL_PAGES - avail);
2472

2473 2474 2475 2476 2477
	/*
	 * Note, this check is intentionally soft, it only guarantees that one
	 * page is available, while the caller may end up allocating as many as
	 * four pages, e.g. for PAE roots or for 5-level paging.  Temporarily
	 * exceeding the (arbitrary by default) limit will not harm the host,
2478
	 * being too aggressive may unnecessarily kill the guest, and getting an
2479 2480 2481
	 * exact count is far more trouble than it's worth, especially in the
	 * page fault paths.
	 */
2482 2483 2484 2485 2486
	if (!kvm_mmu_available_pages(vcpu->kvm))
		return -ENOSPC;
	return 0;
}

2487 2488
/*
 * Changing the number of mmu pages allocated to the vm
2489
 * Note: if goal_nr_mmu_pages is too small, you will get dead lock
2490
 */
2491
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long goal_nr_mmu_pages)
2492
{
2493
	write_lock(&kvm->mmu_lock);
2494

2495
	if (kvm->arch.n_used_mmu_pages > goal_nr_mmu_pages) {
2496 2497
		kvm_mmu_zap_oldest_mmu_pages(kvm, kvm->arch.n_used_mmu_pages -
						  goal_nr_mmu_pages);
2498

2499
		goal_nr_mmu_pages = kvm->arch.n_used_mmu_pages;
2500 2501
	}

2502
	kvm->arch.n_max_mmu_pages = goal_nr_mmu_pages;
2503

2504
	write_unlock(&kvm->mmu_lock);
2505 2506
}

2507
int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn)
2508
{
2509
	struct kvm_mmu_page *sp;
2510
	LIST_HEAD(invalid_list);
2511 2512
	int r;

2513
	pgprintk("%s: looking for gfn %llx\n", __func__, gfn);
2514
	r = 0;
2515
	write_lock(&kvm->mmu_lock);
2516
	for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2517
		pgprintk("%s: gfn %llx role %x\n", __func__, gfn,
2518 2519
			 sp->role.word);
		r = 1;
2520
		kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
2521
	}
2522
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
2523
	write_unlock(&kvm->mmu_lock);
2524

2525
	return r;
2526
}
2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541

static int kvm_mmu_unprotect_page_virt(struct kvm_vcpu *vcpu, gva_t gva)
{
	gpa_t gpa;
	int r;

	if (vcpu->arch.mmu->direct_map)
		return 0;

	gpa = kvm_mmu_gva_to_gpa_read(vcpu, gva, NULL);

	r = kvm_mmu_unprotect_page(vcpu->kvm, gpa >> PAGE_SHIFT);

	return r;
}
2542

2543
static void kvm_unsync_page(struct kvm *kvm, struct kvm_mmu_page *sp)
2544 2545
{
	trace_kvm_mmu_unsync_page(sp);
2546
	++kvm->stat.mmu_unsync;
2547 2548 2549 2550 2551
	sp->unsync = 1;

	kvm_mmu_mark_parents_unsync(sp);
}

2552 2553 2554 2555 2556 2557
/*
 * Attempt to unsync any shadow pages that can be reached by the specified gfn,
 * KVM is creating a writable mapping for said gfn.  Returns 0 if all pages
 * were marked unsync (or if there is no shadow page), -EPERM if the SPTE must
 * be write-protected.
 */
2558
int mmu_try_to_unsync_pages(struct kvm *kvm, const struct kvm_memory_slot *slot,
2559
			    gfn_t gfn, bool can_unsync, bool prefetch)
2560
{
2561
	struct kvm_mmu_page *sp;
2562
	bool locked = false;
2563

2564 2565 2566 2567 2568
	/*
	 * Force write-protection if the page is being tracked.  Note, the page
	 * track machinery is used to write-protect upper-level shadow pages,
	 * i.e. this guards the role.level == 4K assertion below!
	 */
2569
	if (kvm_slot_page_track_is_active(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE))
2570
		return -EPERM;
2571

2572 2573 2574 2575 2576 2577
	/*
	 * The page is not write-tracked, mark existing shadow pages unsync
	 * unless KVM is synchronizing an unsync SP (can_unsync = false).  In
	 * that case, KVM must complete emulation of the guest TLB flush before
	 * allowing shadow pages to become unsync (writable by the guest).
	 */
2578
	for_each_gfn_indirect_valid_sp(kvm, sp, gfn) {
2579
		if (!can_unsync)
2580
			return -EPERM;
2581

2582 2583
		if (sp->unsync)
			continue;
2584

2585
		if (prefetch)
2586 2587
			return -EEXIST;

2588 2589 2590 2591 2592 2593 2594 2595 2596
		/*
		 * TDP MMU page faults require an additional spinlock as they
		 * run with mmu_lock held for read, not write, and the unsync
		 * logic is not thread safe.  Take the spinklock regardless of
		 * the MMU type to avoid extra conditionals/parameters, there's
		 * no meaningful penalty if mmu_lock is held for write.
		 */
		if (!locked) {
			locked = true;
2597
			spin_lock(&kvm->arch.mmu_unsync_pages_lock);
2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610

			/*
			 * Recheck after taking the spinlock, a different vCPU
			 * may have since marked the page unsync.  A false
			 * positive on the unprotected check above is not
			 * possible as clearing sp->unsync _must_ hold mmu_lock
			 * for write, i.e. unsync cannot transition from 0->1
			 * while this CPU holds mmu_lock for read (or write).
			 */
			if (READ_ONCE(sp->unsync))
				continue;
		}

2611
		WARN_ON(sp->role.level != PG_LEVEL_4K);
2612
		kvm_unsync_page(kvm, sp);
2613
	}
2614
	if (locked)
2615
		spin_unlock(&kvm->arch.mmu_unsync_pages_lock);
2616

2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638
	/*
	 * We need to ensure that the marking of unsync pages is visible
	 * before the SPTE is updated to allow writes because
	 * kvm_mmu_sync_roots() checks the unsync flags without holding
	 * the MMU lock and so can race with this. If the SPTE was updated
	 * before the page had been marked as unsync-ed, something like the
	 * following could happen:
	 *
	 * CPU 1                    CPU 2
	 * ---------------------------------------------------------------------
	 * 1.2 Host updates SPTE
	 *     to be writable
	 *                      2.1 Guest writes a GPTE for GVA X.
	 *                          (GPTE being in the guest page table shadowed
	 *                           by the SP from CPU 1.)
	 *                          This reads SPTE during the page table walk.
	 *                          Since SPTE.W is read as 1, there is no
	 *                          fault.
	 *
	 *                      2.2 Guest issues TLB flush.
	 *                          That causes a VM Exit.
	 *
2639 2640
	 *                      2.3 Walking of unsync pages sees sp->unsync is
	 *                          false and skips the page.
2641 2642 2643 2644 2645 2646 2647 2648 2649 2650
	 *
	 *                      2.4 Guest accesses GVA X.
	 *                          Since the mapping in the SP was not updated,
	 *                          so the old mapping for GVA X incorrectly
	 *                          gets used.
	 * 1.1 Host marks SP
	 *     as unsync
	 *     (sp->unsync = true)
	 *
	 * The write barrier below ensures that 1.1 happens before 1.2 and thus
2651 2652
	 * the situation in 2.4 does not arise.  It pairs with the read barrier
	 * in is_unsync_root(), placed between 2.1's load of SPTE.W and 2.3.
2653 2654 2655
	 */
	smp_wmb();

2656
	return 0;
2657 2658
}

2659 2660
static int mmu_set_spte(struct kvm_vcpu *vcpu, struct kvm_memory_slot *slot,
			u64 *sptep, unsigned int pte_access, gfn_t gfn,
2661
			kvm_pfn_t pfn, struct kvm_page_fault *fault)
2662
{
2663
	struct kvm_mmu_page *sp = sptep_to_sp(sptep);
2664
	int level = sp->role.level;
2665
	int was_rmapped = 0;
2666
	int ret = RET_PF_FIXED;
2667
	bool flush = false;
2668
	bool wrprot;
2669
	u64 spte;
2670

2671 2672
	/* Prefetching always gets a writable pfn.  */
	bool host_writable = !fault || fault->map_writable;
2673
	bool prefetch = !fault || fault->prefetch;
2674
	bool write_fault = fault && fault->write;
2675

2676 2677
	pgprintk("%s: spte %llx write_fault %d gfn %llx\n", __func__,
		 *sptep, write_fault, gfn);
2678

2679 2680 2681 2682 2683
	if (unlikely(is_noslot_pfn(pfn))) {
		mark_mmio_spte(vcpu, sptep, gfn, pte_access);
		return RET_PF_EMULATE;
	}

2684
	if (is_shadow_present_pte(*sptep)) {
2685 2686 2687 2688
		/*
		 * If we overwrite a PTE page pointer with a 2MB PMD, unlink
		 * the parent of the now unreachable PTE.
		 */
2689
		if (level > PG_LEVEL_4K && !is_large_pte(*sptep)) {
2690
			struct kvm_mmu_page *child;
Avi Kivity's avatar
Avi Kivity committed
2691
			u64 pte = *sptep;
2692

2693
			child = to_shadow_page(pte & PT64_BASE_ADDR_MASK);
2694
			drop_parent_pte(child, sptep);
2695
			flush = true;
Avi Kivity's avatar
Avi Kivity committed
2696
		} else if (pfn != spte_to_pfn(*sptep)) {
2697
			pgprintk("hfn old %llx new %llx\n",
Avi Kivity's avatar
Avi Kivity committed
2698
				 spte_to_pfn(*sptep), pfn);
2699
			drop_spte(vcpu->kvm, sptep);
2700
			flush = true;
2701 2702
		} else
			was_rmapped = 1;
2703
	}
2704

2705
	wrprot = make_spte(vcpu, sp, slot, pte_access, gfn, pfn, *sptep, prefetch,
2706
			   true, host_writable, &spte);
2707 2708 2709 2710 2711 2712 2713 2714

	if (*sptep == spte) {
		ret = RET_PF_SPURIOUS;
	} else {
		trace_kvm_mmu_set_spte(level, gfn, sptep);
		flush |= mmu_spte_update(sptep, spte);
	}

2715
	if (wrprot) {
2716
		if (write_fault)
2717
			ret = RET_PF_EMULATE;
2718
	}
2719

2720
	if (flush)
2721 2722
		kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn,
				KVM_PAGES_PER_HPAGE(level));
2723

Avi Kivity's avatar
Avi Kivity committed
2724
	pgprintk("%s: setting spte %llx\n", __func__, *sptep);
2725

2726
	if (!was_rmapped) {
2727
		WARN_ON_ONCE(ret == RET_PF_SPURIOUS);
2728
		kvm_update_page_stats(vcpu->kvm, level, 1);
2729
		rmap_add(vcpu, slot, sptep, gfn);
2730
	}
2731

2732
	return ret;
2733 2734
}

2735 2736 2737 2738 2739
static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
				    struct kvm_mmu_page *sp,
				    u64 *start, u64 *end)
{
	struct page *pages[PTE_PREFETCH_NUM];
2740
	struct kvm_memory_slot *slot;
2741
	unsigned int access = sp->role.access;
2742 2743 2744 2745
	int i, ret;
	gfn_t gfn;

	gfn = kvm_mmu_page_get_gfn(sp, start - sp->spt);
2746 2747
	slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, access & ACC_WRITE_MASK);
	if (!slot)
2748 2749
		return -1;

2750
	ret = gfn_to_page_many_atomic(slot, gfn, pages, end - start);
2751 2752 2753
	if (ret <= 0)
		return -1;

2754
	for (i = 0; i < ret; i++, gfn++, start++) {
2755
		mmu_set_spte(vcpu, slot, start, access, gfn,
2756
			     page_to_pfn(pages[i]), NULL);
2757 2758
		put_page(pages[i]);
	}
2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774

	return 0;
}

static void __direct_pte_prefetch(struct kvm_vcpu *vcpu,
				  struct kvm_mmu_page *sp, u64 *sptep)
{
	u64 *spte, *start = NULL;
	int i;

	WARN_ON(!sp->role.direct);

	i = (sptep - sp->spt) & ~(PTE_PREFETCH_NUM - 1);
	spte = sp->spt + i;

	for (i = 0; i < PTE_PREFETCH_NUM; i++, spte++) {
2775
		if (is_shadow_present_pte(*spte) || spte == sptep) {
2776 2777 2778
			if (!start)
				continue;
			if (direct_pte_prefetch_many(vcpu, sp, start, spte) < 0)
2779
				return;
2780 2781 2782 2783
			start = NULL;
		} else if (!start)
			start = spte;
	}
2784 2785
	if (start)
		direct_pte_prefetch_many(vcpu, sp, start, spte);
2786 2787 2788 2789 2790 2791
}

static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
{
	struct kvm_mmu_page *sp;

2792
	sp = sptep_to_sp(sptep);
2793

2794
	/*
2795 2796 2797
	 * Without accessed bits, there's no way to distinguish between
	 * actually accessed translations and prefetched, so disable pte
	 * prefetch if accessed bits aren't available.
2798
	 */
2799
	if (sp_ad_disabled(sp))
2800 2801
		return;

2802
	if (sp->role.level > PG_LEVEL_4K)
2803 2804
		return;

2805 2806 2807 2808 2809 2810 2811
	/*
	 * If addresses are being invalidated, skip prefetching to avoid
	 * accidentally prefetching those addresses.
	 */
	if (unlikely(vcpu->kvm->mmu_notifier_count))
		return;

2812 2813 2814
	__direct_pte_prefetch(vcpu, sp, sptep);
}

2815
static int host_pfn_mapping_level(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn,
2816
				  const struct kvm_memory_slot *slot)
2817 2818 2819 2820 2821
{
	unsigned long hva;
	pte_t *pte;
	int level;

2822
	if (!PageCompound(pfn_to_page(pfn)) && !kvm_is_zone_device_pfn(pfn))
2823
		return PG_LEVEL_4K;
2824

2825 2826 2827 2828 2829 2830 2831 2832
	/*
	 * Note, using the already-retrieved memslot and __gfn_to_hva_memslot()
	 * is not solely for performance, it's also necessary to avoid the
	 * "writable" check in __gfn_to_hva_many(), which will always fail on
	 * read-only memslots due to gfn_to_hva() assuming writes.  Earlier
	 * page fault steps have already verified the guest isn't writing a
	 * read-only memslot.
	 */
2833 2834
	hva = __gfn_to_hva_memslot(slot, gfn);

2835
	pte = lookup_address_in_mm(kvm->mm, hva, &level);
2836
	if (unlikely(!pte))
2837
		return PG_LEVEL_4K;
2838 2839 2840 2841

	return level;
}

2842 2843 2844
int kvm_mmu_max_mapping_level(struct kvm *kvm,
			      const struct kvm_memory_slot *slot, gfn_t gfn,
			      kvm_pfn_t pfn, int max_level)
2845 2846
{
	struct kvm_lpage_info *linfo;
2847
	int host_level;
2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858

	max_level = min(max_level, max_huge_page_level);
	for ( ; max_level > PG_LEVEL_4K; max_level--) {
		linfo = lpage_info_slot(gfn, slot, max_level);
		if (!linfo->disallow_lpage)
			break;
	}

	if (max_level == PG_LEVEL_4K)
		return PG_LEVEL_4K;

2859 2860
	host_level = host_pfn_mapping_level(kvm, gfn, pfn, slot);
	return min(host_level, max_level);
2861 2862
}

2863
void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
2864
{
2865
	struct kvm_memory_slot *slot = fault->slot;
2866 2867
	kvm_pfn_t mask;

2868
	fault->huge_page_disallowed = fault->exec && fault->nx_huge_page_workaround_enabled;
2869

2870 2871
	if (unlikely(fault->max_level == PG_LEVEL_4K))
		return;
2872

2873 2874
	if (is_error_noslot_pfn(fault->pfn) || kvm_is_reserved_pfn(fault->pfn))
		return;
2875

2876
	if (kvm_slot_dirty_track_enabled(slot))
2877
		return;
2878

2879 2880 2881 2882
	/*
	 * Enforce the iTLB multihit workaround after capturing the requested
	 * level, which will be used to do precise, accurate accounting.
	 */
2883 2884 2885 2886 2887
	fault->req_level = kvm_mmu_max_mapping_level(vcpu->kvm, slot,
						     fault->gfn, fault->pfn,
						     fault->max_level);
	if (fault->req_level == PG_LEVEL_4K || fault->huge_page_disallowed)
		return;
2888 2889

	/*
2890 2891
	 * mmu_notifier_retry() was successful and mmu_lock is held, so
	 * the pmd can't be split from under us.
2892
	 */
2893 2894 2895 2896
	fault->goal_level = fault->req_level;
	mask = KVM_PAGES_PER_HPAGE(fault->goal_level) - 1;
	VM_BUG_ON((fault->gfn & mask) != (fault->pfn & mask));
	fault->pfn &= ~mask;
2897 2898
}

2899
void disallowed_hugepage_adjust(struct kvm_page_fault *fault, u64 spte, int cur_level)
2900
{
2901 2902
	if (cur_level > PG_LEVEL_4K &&
	    cur_level == fault->goal_level &&
2903 2904 2905 2906 2907 2908 2909 2910 2911
	    is_shadow_present_pte(spte) &&
	    !is_large_pte(spte)) {
		/*
		 * A small SPTE exists for this pfn, but FNAME(fetch)
		 * and __direct_map would like to create a large PTE
		 * instead: just force them to go down another level,
		 * patching back for them into pfn the next 9 bits of
		 * the address.
		 */
2912 2913 2914 2915
		u64 page_mask = KVM_PAGES_PER_HPAGE(cur_level) -
				KVM_PAGES_PER_HPAGE(cur_level - 1);
		fault->pfn |= fault->gfn & page_mask;
		fault->goal_level--;
2916 2917 2918
	}
}

2919
static int __direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
2920
{
2921
	struct kvm_shadow_walk_iterator it;
2922
	struct kvm_mmu_page *sp;
2923
	int ret;
2924
	gfn_t base_gfn = fault->gfn;
2925

2926
	kvm_mmu_hugepage_adjust(vcpu, fault);
2927

2928
	trace_kvm_mmu_spte_requested(fault);
2929
	for_each_shadow_entry(vcpu, fault->addr, it) {
2930 2931 2932 2933
		/*
		 * We cannot overwrite existing page tables with an NX
		 * large page, as the leaf could be executable.
		 */
2934
		if (fault->nx_huge_page_workaround_enabled)
2935
			disallowed_hugepage_adjust(fault, *it.sptep, it.level);
2936

2937
		base_gfn = fault->gfn & ~(KVM_PAGES_PER_HPAGE(it.level) - 1);
2938
		if (it.level == fault->goal_level)
2939
			break;
2940

2941
		drop_large_spte(vcpu, it.sptep);
2942 2943 2944 2945 2946 2947 2948
		if (is_shadow_present_pte(*it.sptep))
			continue;

		sp = kvm_mmu_get_page(vcpu, base_gfn, it.addr,
				      it.level - 1, true, ACC_ALL);

		link_shadow_page(vcpu, it.sptep, sp);
2949 2950
		if (fault->is_tdp && fault->huge_page_disallowed &&
		    fault->req_level >= it.level)
2951
			account_huge_nx_page(vcpu->kvm, sp);
2952
	}
2953

2954 2955 2956
	if (WARN_ON_ONCE(it.level != fault->goal_level))
		return -EFAULT;

2957
	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL,
2958
			   base_gfn, fault->pfn, fault);
2959 2960 2961
	if (ret == RET_PF_SPURIOUS)
		return ret;

2962 2963 2964
	direct_pte_prefetch(vcpu, it.sptep);
	++vcpu->stat.pf_fixed;
	return ret;
2965 2966
}

2967
static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *tsk)
2968
{
2969
	send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, PAGE_SHIFT, tsk);
2970 2971
}

2972
static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn)
2973
{
2974 2975 2976 2977 2978 2979
	/*
	 * Do not cache the mmio info caused by writing the readonly gfn
	 * into the spte otherwise read access on readonly gfn also can
	 * caused mmio page fault and treat it as mmio access.
	 */
	if (pfn == KVM_PFN_ERR_RO_FAULT)
2980
		return RET_PF_EMULATE;
2981

2982
	if (pfn == KVM_PFN_ERR_HWPOISON) {
2983
		kvm_send_hwpoison_signal(kvm_vcpu_gfn_to_hva(vcpu, gfn), current);
2984
		return RET_PF_RETRY;
2985
	}
2986

2987
	return -EFAULT;
2988 2989
}

2990 2991
static bool handle_abnormal_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
				unsigned int access, int *ret_val)
2992 2993
{
	/* The pfn is invalid, report the error! */
2994 2995
	if (unlikely(is_error_pfn(fault->pfn))) {
		*ret_val = kvm_handle_bad_page(vcpu, fault->gfn, fault->pfn);
2996
		return true;
2997 2998
	}

2999
	if (unlikely(!fault->slot)) {
3000 3001 3002
		gva_t gva = fault->is_tdp ? 0 : fault->addr;

		vcpu_cache_mmio_info(vcpu, gva, fault->gfn,
3003
				     access & shadow_mmio_access_mask);
3004 3005 3006 3007 3008 3009 3010 3011 3012 3013
		/*
		 * If MMIO caching is disabled, emulate immediately without
		 * touching the shadow page tables as attempting to install an
		 * MMIO SPTE will just be an expensive nop.
		 */
		if (unlikely(!shadow_mmio_value)) {
			*ret_val = RET_PF_EMULATE;
			return true;
		}
	}
3014

3015
	return false;
3016 3017
}

3018
static bool page_fault_can_be_fast(struct kvm_page_fault *fault)
3019
{
3020 3021 3022 3023
	/*
	 * Do not fix the mmio spte with invalid generation number which
	 * need to be updated by slow page fault path.
	 */
3024
	if (fault->rsvd)
3025 3026
		return false;

3027
	/* See if the page fault is due to an NX violation */
3028
	if (unlikely(fault->exec && fault->present))
3029 3030
		return false;

3031
	/*
3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042
	 * #PF can be fast if:
	 * 1. The shadow page table entry is not present, which could mean that
	 *    the fault is potentially caused by access tracking (if enabled).
	 * 2. The shadow page table entry is present and the fault
	 *    is caused by write-protect, that means we just need change the W
	 *    bit of the spte which can be done out of mmu-lock.
	 *
	 * However, if access tracking is disabled we know that a non-present
	 * page must be a genuine page fault where we have to create a new SPTE.
	 * So, if access tracking is disabled, we return true only for write
	 * accesses to a present page.
3043 3044
	 */

3045
	return shadow_acc_track_mask != 0 || (fault->write && fault->present);
3046 3047
}

3048 3049 3050 3051
/*
 * Returns true if the SPTE was fixed successfully. Otherwise,
 * someone else modified the SPTE from its original value.
 */
3052
static bool
3053
fast_pf_fix_direct_spte(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
3054
			u64 *sptep, u64 old_spte, u64 new_spte)
3055
{
3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067
	/*
	 * Theoretically we could also set dirty bit (and flush TLB) here in
	 * order to eliminate unnecessary PML logging. See comments in
	 * set_spte. But fast_page_fault is very unlikely to happen with PML
	 * enabled, so we do not do this. This might result in the same GPA
	 * to be logged in PML buffer again when the write really happens, and
	 * eventually to be called by mark_page_dirty twice. But it's also no
	 * harm. This also avoids the TLB flush needed after setting dirty bit
	 * so non-PML cases won't be impacted.
	 *
	 * Compare with set_spte where instead shadow_dirty_mask is set.
	 */
3068
	if (cmpxchg64(sptep, old_spte, new_spte) != old_spte)
3069 3070
		return false;

3071 3072
	if (is_writable_pte(new_spte) && !is_writable_pte(old_spte))
		mark_page_dirty_in_slot(vcpu->kvm, fault->slot, fault->gfn);
3073 3074 3075 3076

	return true;
}

3077
static bool is_access_allowed(struct kvm_page_fault *fault, u64 spte)
3078
{
3079
	if (fault->exec)
3080 3081
		return is_executable_pte(spte);

3082
	if (fault->write)
3083 3084 3085 3086 3087 3088
		return is_writable_pte(spte);

	/* Fault was on Read access */
	return spte & PT_PRESENT_MASK;
}

3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111
/*
 * Returns the last level spte pointer of the shadow page walk for the given
 * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
 * walk could be performed, returns NULL and *spte does not contain valid data.
 *
 * Contract:
 *  - Must be called between walk_shadow_page_lockless_{begin,end}.
 *  - The returned sptep must not be used after walk_shadow_page_lockless_end.
 */
static u64 *fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, gpa_t gpa, u64 *spte)
{
	struct kvm_shadow_walk_iterator iterator;
	u64 old_spte;
	u64 *sptep = NULL;

	for_each_shadow_entry_lockless(vcpu, gpa, iterator, old_spte) {
		sptep = iterator.sptep;
		*spte = old_spte;
	}

	return sptep;
}

3112
/*
3113
 * Returns one of RET_PF_INVALID, RET_PF_FIXED or RET_PF_SPURIOUS.
3114
 */
3115
static int fast_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
3116
{
3117
	struct kvm_mmu_page *sp;
3118
	int ret = RET_PF_INVALID;
3119
	u64 spte = 0ull;
3120
	u64 *sptep = NULL;
3121
	uint retry_count = 0;
3122

3123
	if (!page_fault_can_be_fast(fault))
3124
		return ret;
3125 3126 3127

	walk_shadow_page_lockless_begin(vcpu);

3128
	do {
3129
		u64 new_spte;
3130

3131
		if (is_tdp_mmu(vcpu->arch.mmu))
3132
			sptep = kvm_tdp_mmu_fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
3133
		else
3134
			sptep = fast_pf_get_last_sptep(vcpu, fault->addr, &spte);
3135

3136 3137 3138
		if (!is_shadow_present_pte(spte))
			break;

3139
		sp = sptep_to_sp(sptep);
3140 3141
		if (!is_last_spte(spte, sp->role.level))
			break;
3142

3143
		/*
3144 3145 3146 3147 3148
		 * Check whether the memory access that caused the fault would
		 * still cause it if it were to be performed right now. If not,
		 * then this is a spurious fault caused by TLB lazily flushed,
		 * or some other CPU has already fixed the PTE after the
		 * current CPU took the fault.
3149 3150 3151 3152
		 *
		 * Need not check the access of upper level table entries since
		 * they are always ACC_ALL.
		 */
3153
		if (is_access_allowed(fault, spte)) {
3154
			ret = RET_PF_SPURIOUS;
3155 3156
			break;
		}
3157

3158 3159 3160 3161 3162 3163 3164 3165 3166 3167
		new_spte = spte;

		if (is_access_track_spte(spte))
			new_spte = restore_acc_track_spte(new_spte);

		/*
		 * Currently, to simplify the code, write-protection can
		 * be removed in the fast path only if the SPTE was
		 * write-protected for dirty-logging or access tracking.
		 */
3168
		if (fault->write &&
3169
		    spte_can_locklessly_be_made_writable(spte)) {
3170
			new_spte |= PT_WRITABLE_MASK;
3171 3172

			/*
3173 3174 3175
			 * Do not fix write-permission on the large spte when
			 * dirty logging is enabled. Since we only dirty the
			 * first page into the dirty-bitmap in
3176 3177 3178 3179 3180
			 * fast_pf_fix_direct_spte(), other pages are missed
			 * if its slot has dirty logging enabled.
			 *
			 * Instead, we let the slow page fault path create a
			 * normal spte to fix the access.
3181
			 */
3182 3183
			if (sp->role.level > PG_LEVEL_4K &&
			    kvm_slot_dirty_track_enabled(fault->slot))
3184
				break;
3185
		}
3186

3187
		/* Verify that the fault can be handled in the fast path */
3188
		if (new_spte == spte ||
3189
		    !is_access_allowed(fault, new_spte))
3190 3191 3192 3193 3194
			break;

		/*
		 * Currently, fast page fault only works for direct mapping
		 * since the gfn is not stable for indirect shadow page. See
3195
		 * Documentation/virt/kvm/locking.rst to get more detail.
3196
		 */
3197
		if (fast_pf_fix_direct_spte(vcpu, fault, sptep, spte, new_spte)) {
3198
			ret = RET_PF_FIXED;
3199
			break;
3200
		}
3201 3202 3203 3204 3205 3206 3207 3208

		if (++retry_count > 4) {
			printk_once(KERN_WARNING
				"kvm: Fast #PF retrying more than 4 times.\n");
			break;
		}

	} while (true);
3209

3210
	trace_fast_page_fault(vcpu, fault, sptep, spte, ret);
3211 3212
	walk_shadow_page_lockless_end(vcpu);

3213
	return ret;
3214 3215
}

3216 3217
static void mmu_free_root_page(struct kvm *kvm, hpa_t *root_hpa,
			       struct list_head *invalid_list)
3218
{
3219
	struct kvm_mmu_page *sp;
3220

3221
	if (!VALID_PAGE(*root_hpa))
3222
		return;
3223

3224
	sp = to_shadow_page(*root_hpa & PT64_BASE_ADDR_MASK);
3225

3226
	if (is_tdp_mmu_page(sp))
3227
		kvm_tdp_mmu_put_root(kvm, sp, false);
3228 3229
	else if (!--sp->root_count && sp->role.invalid)
		kvm_mmu_prepare_zap_page(kvm, sp, invalid_list);
3230

3231 3232 3233
	*root_hpa = INVALID_PAGE;
}

3234
/* roots_to_free must be some combination of the KVM_MMU_ROOT_* flags */
3235 3236
void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			ulong roots_to_free)
3237
{
3238
	struct kvm *kvm = vcpu->kvm;
3239 3240
	int i;
	LIST_HEAD(invalid_list);
3241
	bool free_active_root = roots_to_free & KVM_MMU_ROOT_CURRENT;
3242

3243
	BUILD_BUG_ON(KVM_MMU_NUM_PREV_ROOTS >= BITS_PER_LONG);
3244

3245
	/* Before acquiring the MMU lock, see if we need to do any real work. */
3246 3247 3248 3249 3250 3251 3252 3253 3254
	if (!(free_active_root && VALID_PAGE(mmu->root_hpa))) {
		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
			if ((roots_to_free & KVM_MMU_ROOT_PREVIOUS(i)) &&
			    VALID_PAGE(mmu->prev_roots[i].hpa))
				break;

		if (i == KVM_MMU_NUM_PREV_ROOTS)
			return;
	}
3255

3256
	write_lock(&kvm->mmu_lock);
3257

3258 3259
	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
		if (roots_to_free & KVM_MMU_ROOT_PREVIOUS(i))
3260
			mmu_free_root_page(kvm, &mmu->prev_roots[i].hpa,
3261
					   &invalid_list);
3262

3263 3264 3265
	if (free_active_root) {
		if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL &&
		    (mmu->root_level >= PT64_ROOT_4LEVEL || mmu->direct_map)) {
3266
			mmu_free_root_page(kvm, &mmu->root_hpa, &invalid_list);
3267
		} else if (mmu->pae_root) {
3268 3269 3270 3271 3272 3273 3274 3275
			for (i = 0; i < 4; ++i) {
				if (!IS_VALID_PAE_ROOT(mmu->pae_root[i]))
					continue;

				mmu_free_root_page(kvm, &mmu->pae_root[i],
						   &invalid_list);
				mmu->pae_root[i] = INVALID_PAE_ROOT;
			}
3276
		}
3277
		mmu->root_hpa = INVALID_PAGE;
3278
		mmu->root_pgd = 0;
3279
	}
3280

3281
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
3282
	write_unlock(&kvm->mmu_lock);
3283
}
3284
EXPORT_SYMBOL_GPL(kvm_mmu_free_roots);
3285

3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312
void kvm_mmu_free_guest_mode_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
{
	unsigned long roots_to_free = 0;
	hpa_t root_hpa;
	int i;

	/*
	 * This should not be called while L2 is active, L2 can't invalidate
	 * _only_ its own roots, e.g. INVVPID unconditionally exits.
	 */
	WARN_ON_ONCE(mmu->mmu_role.base.guest_mode);

	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
		root_hpa = mmu->prev_roots[i].hpa;
		if (!VALID_PAGE(root_hpa))
			continue;

		if (!to_shadow_page(root_hpa) ||
			to_shadow_page(root_hpa)->role.guest_mode)
			roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);
	}

	kvm_mmu_free_roots(vcpu, mmu, roots_to_free);
}
EXPORT_SYMBOL_GPL(kvm_mmu_free_guest_mode_roots);


3313 3314 3315 3316
static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
{
	int ret = 0;

3317
	if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
3318
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
3319 3320 3321 3322 3323 3324
		ret = 1;
	}

	return ret;
}

3325 3326
static hpa_t mmu_alloc_root(struct kvm_vcpu *vcpu, gfn_t gfn, gva_t gva,
			    u8 level, bool direct)
3327 3328
{
	struct kvm_mmu_page *sp;
3329 3330 3331 3332 3333 3334 3335 3336 3337

	sp = kvm_mmu_get_page(vcpu, gfn, gva, level, direct, ACC_ALL);
	++sp->root_count;

	return __pa(sp->spt);
}

static int mmu_alloc_direct_roots(struct kvm_vcpu *vcpu)
{
3338 3339
	struct kvm_mmu *mmu = vcpu->arch.mmu;
	u8 shadow_root_level = mmu->shadow_root_level;
3340
	hpa_t root;
3341
	unsigned i;
3342 3343 3344 3345 3346 3347
	int r;

	write_lock(&vcpu->kvm->mmu_lock);
	r = make_mmu_pages_available(vcpu);
	if (r < 0)
		goto out_unlock;
3348

3349
	if (is_tdp_mmu_enabled(vcpu->kvm)) {
3350
		root = kvm_tdp_mmu_get_vcpu_root_hpa(vcpu);
3351
		mmu->root_hpa = root;
3352
	} else if (shadow_root_level >= PT64_ROOT_4LEVEL) {
3353
		root = mmu_alloc_root(vcpu, 0, 0, shadow_root_level, true);
3354
		mmu->root_hpa = root;
3355
	} else if (shadow_root_level == PT32E_ROOT_LEVEL) {
3356 3357 3358 3359
		if (WARN_ON_ONCE(!mmu->pae_root)) {
			r = -EIO;
			goto out_unlock;
		}
3360

3361
		for (i = 0; i < 4; ++i) {
3362
			WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
3363

3364 3365
			root = mmu_alloc_root(vcpu, i << (30 - PAGE_SHIFT),
					      i << 30, PT32_ROOT_LEVEL, true);
3366 3367
			mmu->pae_root[i] = root | PT_PRESENT_MASK |
					   shadow_me_mask;
3368
		}
3369
		mmu->root_hpa = __pa(mmu->pae_root);
3370 3371
	} else {
		WARN_ONCE(1, "Bad TDP root level = %d\n", shadow_root_level);
3372 3373
		r = -EIO;
		goto out_unlock;
3374
	}
3375

3376
	/* root_pgd is ignored for direct MMUs. */
3377
	mmu->root_pgd = 0;
3378 3379 3380
out_unlock:
	write_unlock(&vcpu->kvm->mmu_lock);
	return r;
3381 3382
}

3383 3384 3385 3386
static int mmu_first_shadow_root_alloc(struct kvm *kvm)
{
	struct kvm_memslots *slots;
	struct kvm_memory_slot *slot;
3387
	int r = 0, i, bkt;
3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411

	/*
	 * Check if this is the first shadow root being allocated before
	 * taking the lock.
	 */
	if (kvm_shadow_root_allocated(kvm))
		return 0;

	mutex_lock(&kvm->slots_arch_lock);

	/* Recheck, under the lock, whether this is the first shadow root. */
	if (kvm_shadow_root_allocated(kvm))
		goto out_unlock;

	/*
	 * Check if anything actually needs to be allocated, e.g. all metadata
	 * will be allocated upfront if TDP is disabled.
	 */
	if (kvm_memslots_have_rmaps(kvm) &&
	    kvm_page_track_write_tracking_enabled(kvm))
		goto out_success;

	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
3412
		kvm_for_each_memslot(slot, bkt, slots) {
3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443
			/*
			 * Both of these functions are no-ops if the target is
			 * already allocated, so unconditionally calling both
			 * is safe.  Intentionally do NOT free allocations on
			 * failure to avoid having to track which allocations
			 * were made now versus when the memslot was created.
			 * The metadata is guaranteed to be freed when the slot
			 * is freed, and will be kept/used if userspace retries
			 * KVM_RUN instead of killing the VM.
			 */
			r = memslot_rmap_alloc(slot, slot->npages);
			if (r)
				goto out_unlock;
			r = kvm_page_track_write_tracking_alloc(slot);
			if (r)
				goto out_unlock;
		}
	}

	/*
	 * Ensure that shadow_root_allocated becomes true strictly after
	 * all the related pointers are set.
	 */
out_success:
	smp_store_release(&kvm->arch.shadow_root_allocated, true);

out_unlock:
	mutex_unlock(&kvm->slots_arch_lock);
	return r;
}

3444
static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
3445
{
3446
	struct kvm_mmu *mmu = vcpu->arch.mmu;
3447
	u64 pdptrs[4], pm_mask;
3448
	gfn_t root_gfn, root_pgd;
3449
	hpa_t root;
3450 3451
	unsigned i;
	int r;
3452

3453
	root_pgd = mmu->get_guest_pgd(vcpu);
3454
	root_gfn = root_pgd >> PAGE_SHIFT;
3455

3456 3457 3458
	if (mmu_check_root(vcpu, root_gfn))
		return 1;

3459 3460 3461 3462
	/*
	 * On SVM, reading PDPTRs might access guest memory, which might fault
	 * and thus might sleep.  Grab the PDPTRs before acquiring mmu_lock.
	 */
3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473
	if (mmu->root_level == PT32E_ROOT_LEVEL) {
		for (i = 0; i < 4; ++i) {
			pdptrs[i] = mmu->get_pdptr(vcpu, i);
			if (!(pdptrs[i] & PT_PRESENT_MASK))
				continue;

			if (mmu_check_root(vcpu, pdptrs[i] >> PAGE_SHIFT))
				return 1;
		}
	}

3474
	r = mmu_first_shadow_root_alloc(vcpu->kvm);
3475 3476 3477
	if (r)
		return r;

3478 3479 3480 3481 3482
	write_lock(&vcpu->kvm->mmu_lock);
	r = make_mmu_pages_available(vcpu);
	if (r < 0)
		goto out_unlock;

3483 3484 3485 3486
	/*
	 * Do we shadow a long mode page table? If so we need to
	 * write-protect the guests page table root.
	 */
3487
	if (mmu->root_level >= PT64_ROOT_4LEVEL) {
3488
		root = mmu_alloc_root(vcpu, root_gfn, 0,
3489 3490
				      mmu->shadow_root_level, false);
		mmu->root_hpa = root;
3491
		goto set_root_pgd;
3492
	}
3493

3494 3495 3496 3497
	if (WARN_ON_ONCE(!mmu->pae_root)) {
		r = -EIO;
		goto out_unlock;
	}
3498

3499 3500
	/*
	 * We shadow a 32 bit page table. This may be a legacy 2-level
3501 3502
	 * or a PAE 3-level page table. In either case we need to be aware that
	 * the shadow page table may be a PAE or a long mode page table.
3503
	 */
3504
	pm_mask = PT_PRESENT_MASK | shadow_me_mask;
3505
	if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL) {
3506 3507
		pm_mask |= PT_ACCESSED_MASK | PT_WRITABLE_MASK | PT_USER_MASK;

3508
		if (WARN_ON_ONCE(!mmu->pml4_root)) {
3509 3510 3511
			r = -EIO;
			goto out_unlock;
		}
3512
		mmu->pml4_root[0] = __pa(mmu->pae_root) | pm_mask;
3513 3514 3515 3516 3517 3518 3519 3520

		if (mmu->shadow_root_level == PT64_ROOT_5LEVEL) {
			if (WARN_ON_ONCE(!mmu->pml5_root)) {
				r = -EIO;
				goto out_unlock;
			}
			mmu->pml5_root[0] = __pa(mmu->pml4_root) | pm_mask;
		}
3521 3522
	}

3523
	for (i = 0; i < 4; ++i) {
3524
		WARN_ON_ONCE(IS_VALID_PAE_ROOT(mmu->pae_root[i]));
3525

3526
		if (mmu->root_level == PT32E_ROOT_LEVEL) {
3527
			if (!(pdptrs[i] & PT_PRESENT_MASK)) {
3528
				mmu->pae_root[i] = INVALID_PAE_ROOT;
Avi Kivity's avatar
Avi Kivity committed
3529 3530
				continue;
			}
3531
			root_gfn = pdptrs[i] >> PAGE_SHIFT;
3532
		}
3533

3534 3535
		root = mmu_alloc_root(vcpu, root_gfn, i << 30,
				      PT32_ROOT_LEVEL, false);
3536
		mmu->pae_root[i] = root | pm_mask;
3537
	}
3538

3539 3540 3541
	if (mmu->shadow_root_level == PT64_ROOT_5LEVEL)
		mmu->root_hpa = __pa(mmu->pml5_root);
	else if (mmu->shadow_root_level == PT64_ROOT_4LEVEL)
3542
		mmu->root_hpa = __pa(mmu->pml4_root);
3543 3544
	else
		mmu->root_hpa = __pa(mmu->pae_root);
3545

3546
set_root_pgd:
3547
	mmu->root_pgd = root_pgd;
3548 3549
out_unlock:
	write_unlock(&vcpu->kvm->mmu_lock);
3550

3551
	return 0;
3552 3553
}

3554 3555 3556
static int mmu_alloc_special_roots(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu *mmu = vcpu->arch.mmu;
3557
	bool need_pml5 = mmu->shadow_root_level > PT64_ROOT_4LEVEL;
3558 3559 3560
	u64 *pml5_root = NULL;
	u64 *pml4_root = NULL;
	u64 *pae_root;
3561 3562

	/*
3563 3564 3565 3566
	 * When shadowing 32-bit or PAE NPT with 64-bit NPT, the PML4 and PDP
	 * tables are allocated and initialized at root creation as there is no
	 * equivalent level in the guest's NPT to shadow.  Allocate the tables
	 * on demand, as running a 32-bit L1 VMM on 64-bit KVM is very rare.
3567
	 */
3568 3569 3570
	if (mmu->direct_map || mmu->root_level >= PT64_ROOT_4LEVEL ||
	    mmu->shadow_root_level < PT64_ROOT_4LEVEL)
		return 0;
3571

3572 3573 3574 3575 3576 3577 3578 3579
	/*
	 * NPT, the only paging mode that uses this horror, uses a fixed number
	 * of levels for the shadow page tables, e.g. all MMUs are 4-level or
	 * all MMus are 5-level.  Thus, this can safely require that pml5_root
	 * is allocated if the other roots are valid and pml5 is needed, as any
	 * prior MMU would also have required pml5.
	 */
	if (mmu->pae_root && mmu->pml4_root && (!need_pml5 || mmu->pml5_root))
3580
		return 0;
3581

3582 3583 3584 3585
	/*
	 * The special roots should always be allocated in concert.  Yell and
	 * bail if KVM ends up in a state where only one of the roots is valid.
	 */
3586
	if (WARN_ON_ONCE(!tdp_enabled || mmu->pae_root || mmu->pml4_root ||
3587
			 (need_pml5 && mmu->pml5_root)))
3588
		return -EIO;
3589

3590 3591 3592 3593
	/*
	 * Unlike 32-bit NPT, the PDP table doesn't need to be in low mem, and
	 * doesn't need to be decrypted.
	 */
3594 3595 3596
	pae_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
	if (!pae_root)
		return -ENOMEM;
3597

3598
#ifdef CONFIG_X86_64
3599
	pml4_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3600 3601 3602
	if (!pml4_root)
		goto err_pml4;

3603
	if (need_pml5) {
3604 3605 3606
		pml5_root = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
		if (!pml5_root)
			goto err_pml5;
3607
	}
3608
#endif
3609

3610
	mmu->pae_root = pae_root;
3611
	mmu->pml4_root = pml4_root;
3612
	mmu->pml5_root = pml5_root;
3613

3614
	return 0;
3615 3616 3617 3618 3619 3620 3621 3622

#ifdef CONFIG_X86_64
err_pml5:
	free_page((unsigned long)pml4_root);
err_pml4:
	free_page((unsigned long)pae_root);
	return -ENOMEM;
#endif
3623 3624
}

3625 3626 3627 3628
static bool is_unsync_root(hpa_t root)
{
	struct kvm_mmu_page *sp;

3629 3630 3631
	if (!VALID_PAGE(root))
		return false;

3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651
	/*
	 * The read barrier orders the CPU's read of SPTE.W during the page table
	 * walk before the reads of sp->unsync/sp->unsync_children here.
	 *
	 * Even if another CPU was marking the SP as unsync-ed simultaneously,
	 * any guest page table changes are not guaranteed to be visible anyway
	 * until this VCPU issues a TLB flush strictly after those changes are
	 * made.  We only need to ensure that the other CPU sets these flags
	 * before any actual changes to the page tables are made.  The comments
	 * in mmu_try_to_unsync_pages() describe what could go wrong if this
	 * requirement isn't satisfied.
	 */
	smp_rmb();
	sp = to_shadow_page(root);
	if (sp->unsync || sp->unsync_children)
		return true;

	return false;
}

3652
void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
3653 3654 3655 3656
{
	int i;
	struct kvm_mmu_page *sp;

3657
	if (vcpu->arch.mmu->direct_map)
3658 3659
		return;

3660
	if (!VALID_PAGE(vcpu->arch.mmu->root_hpa))
3661
		return;
3662

3663
	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);
3664

3665 3666
	if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) {
		hpa_t root = vcpu->arch.mmu->root_hpa;
3667
		sp = to_shadow_page(root);
3668

3669
		if (!is_unsync_root(root))
3670 3671
			return;

3672
		write_lock(&vcpu->kvm->mmu_lock);
3673 3674
		kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);

3675
		mmu_sync_children(vcpu, sp, true);
3676

3677
		kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3678
		write_unlock(&vcpu->kvm->mmu_lock);
3679 3680
		return;
	}
3681

3682
	write_lock(&vcpu->kvm->mmu_lock);
3683 3684
	kvm_mmu_audit(vcpu, AUDIT_PRE_SYNC);

3685
	for (i = 0; i < 4; ++i) {
3686
		hpa_t root = vcpu->arch.mmu->pae_root[i];
3687

3688
		if (IS_VALID_PAE_ROOT(root)) {
3689
			root &= PT64_BASE_ADDR_MASK;
3690
			sp = to_shadow_page(root);
3691
			mmu_sync_children(vcpu, sp, true);
3692 3693 3694
		}
	}

3695
	kvm_mmu_audit(vcpu, AUDIT_POST_SYNC);
3696
	write_unlock(&vcpu->kvm->mmu_lock);
3697 3698
}

3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711
void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu)
{
	unsigned long roots_to_free = 0;
	int i;

	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
		if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa))
			roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);

	/* sync prev_roots by simply freeing them */
	kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free);
}

3712 3713 3714
static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
				  gpa_t vaddr, u32 access,
				  struct x86_exception *exception)
3715
{
3716 3717
	if (exception)
		exception->error_code = 0;
3718
	return kvm_translate_gpa(vcpu, mmu, vaddr, access, exception);
3719 3720
}

3721
static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3722
{
3723 3724 3725 3726 3727 3728 3729
	/*
	 * A nested guest cannot use the MMIO cache if it is using nested
	 * page tables, because cr2 is a nGPA while the cache stores GPAs.
	 */
	if (mmu_is_nested(vcpu))
		return false;

3730 3731 3732 3733 3734 3735
	if (direct)
		return vcpu_match_mmio_gpa(vcpu, addr);

	return vcpu_match_mmio_gva(vcpu, addr);
}

3736 3737 3738
/*
 * Return the level of the lowest level SPTE added to sptes.
 * That SPTE may be non-present.
3739 3740
 *
 * Must be called between walk_shadow_page_lockless_{begin,end}.
3741
 */
3742
static int get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, int *root_level)
3743 3744
{
	struct kvm_shadow_walk_iterator iterator;
3745
	int leaf = -1;
3746
	u64 spte;
3747

3748 3749
	for (shadow_walk_init(&iterator, vcpu, addr),
	     *root_level = iterator.level;
3750 3751
	     shadow_walk_okay(&iterator);
	     __shadow_walk_next(&iterator, spte)) {
3752
		leaf = iterator.level;
3753 3754
		spte = mmu_spte_get_lockless(iterator.sptep);

3755
		sptes[leaf] = spte;
3756 3757 3758 3759 3760
	}

	return leaf;
}

3761
/* return true if reserved bit(s) are detected on a valid, non-MMIO SPTE. */
3762 3763
static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep)
{
3764
	u64 sptes[PT64_ROOT_MAX_LEVEL + 1];
3765
	struct rsvd_bits_validate *rsvd_check;
3766
	int root, leaf, level;
3767 3768
	bool reserved = false;

3769 3770
	walk_shadow_page_lockless_begin(vcpu);

3771
	if (is_tdp_mmu(vcpu->arch.mmu))
3772
		leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes, &root);
3773
	else
3774
		leaf = get_walk(vcpu, addr, sptes, &root);
3775

3776 3777
	walk_shadow_page_lockless_end(vcpu);

3778 3779 3780 3781 3782
	if (unlikely(leaf < 0)) {
		*sptep = 0ull;
		return reserved;
	}

3783 3784 3785 3786 3787 3788 3789 3790 3791 3792
	*sptep = sptes[leaf];

	/*
	 * Skip reserved bits checks on the terminal leaf if it's not a valid
	 * SPTE.  Note, this also (intentionally) skips MMIO SPTEs, which, by
	 * design, always have reserved bits set.  The purpose of the checks is
	 * to detect reserved bits on non-MMIO SPTEs. i.e. buggy SPTEs.
	 */
	if (!is_shadow_present_pte(sptes[leaf]))
		leaf++;
3793 3794 3795

	rsvd_check = &vcpu->arch.mmu->shadow_zero_check;

3796
	for (level = root; level >= leaf; level--)
3797
		reserved |= is_rsvd_spte(rsvd_check, sptes[level], level);
3798 3799

	if (reserved) {
3800
		pr_err("%s: reserved bits set on MMU-present spte, addr 0x%llx, hierarchy:\n",
3801
		       __func__, addr);
3802
		for (level = root; level >= leaf; level--)
3803 3804
			pr_err("------ spte = 0x%llx level = %d, rsvd bits = 0x%llx",
			       sptes[level], level,
3805
			       get_rsvd_bits(rsvd_check, sptes[level], level));
3806
	}
3807

3808
	return reserved;
3809 3810
}

3811
static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
3812 3813
{
	u64 spte;
3814
	bool reserved;
3815

3816
	if (mmio_info_in_cache(vcpu, addr, direct))
3817
		return RET_PF_EMULATE;
3818

3819
	reserved = get_mmio_spte(vcpu, addr, &spte);
3820
	if (WARN_ON(reserved))
3821
		return -EINVAL;
3822 3823 3824

	if (is_mmio_spte(spte)) {
		gfn_t gfn = get_mmio_spte_gfn(spte);
3825
		unsigned int access = get_mmio_spte_access(spte);
3826

3827
		if (!check_mmio_spte(vcpu, spte))
3828
			return RET_PF_INVALID;
3829

3830 3831
		if (direct)
			addr = 0;
3832 3833

		trace_handle_mmio_page_fault(addr, gfn, access);
3834
		vcpu_cache_mmio_info(vcpu, addr, gfn, access);
3835
		return RET_PF_EMULATE;
3836 3837 3838 3839 3840 3841
	}

	/*
	 * If the page table is zapped by other cpus, let CPU fault again on
	 * the address.
	 */
3842
	return RET_PF_RETRY;
3843 3844
}

3845
static bool page_fault_handle_page_track(struct kvm_vcpu *vcpu,
3846
					 struct kvm_page_fault *fault)
3847
{
3848
	if (unlikely(fault->rsvd))
3849 3850
		return false;

3851
	if (!fault->present || !fault->write)
3852 3853 3854 3855 3856 3857
		return false;

	/*
	 * guest is writing the page which is write tracked which can
	 * not be fixed by page fault handler.
	 */
3858
	if (kvm_slot_page_track_is_active(vcpu->kvm, fault->slot, fault->gfn, KVM_PAGE_TRACK_WRITE))
3859 3860 3861 3862 3863
		return true;

	return false;
}

3864 3865 3866 3867 3868 3869
static void shadow_page_table_clear_flood(struct kvm_vcpu *vcpu, gva_t addr)
{
	struct kvm_shadow_walk_iterator iterator;
	u64 spte;

	walk_shadow_page_lockless_begin(vcpu);
3870
	for_each_shadow_entry_lockless(vcpu, addr, iterator, spte)
3871 3872 3873 3874
		clear_sp_write_flooding_count(iterator.sptep);
	walk_shadow_page_lockless_end(vcpu);
}

3875 3876
static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
				    gfn_t gfn)
3877 3878
{
	struct kvm_arch_async_pf arch;
3879

3880
	arch.token = (vcpu->arch.apf.id++ << 12) | vcpu->vcpu_id;
3881
	arch.gfn = gfn;
3882
	arch.direct_map = vcpu->arch.mmu->direct_map;
3883
	arch.cr3 = vcpu->arch.mmu->get_guest_pgd(vcpu);
3884

3885 3886
	return kvm_setup_async_pf(vcpu, cr2_or_gpa,
				  kvm_vcpu_gfn_to_hva(vcpu, gfn), &arch);
3887 3888
}

3889
static bool kvm_faultin_pfn(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault, int *r)
3890
{
3891
	struct kvm_memory_slot *slot = fault->slot;
3892 3893
	bool async;

3894 3895 3896 3897 3898 3899
	/*
	 * Retry the page fault if the gfn hit a memslot that is being deleted
	 * or moved.  This ensures any existing SPTEs for the old memslot will
	 * be zapped before KVM inserts a new MMIO SPTE for the gfn.
	 */
	if (slot && (slot->flags & KVM_MEMSLOT_INVALID))
3900
		goto out_retry;
3901

3902 3903 3904
	if (!kvm_is_visible_memslot(slot)) {
		/* Don't expose private memslots to L2. */
		if (is_guest_mode(vcpu)) {
3905
			fault->slot = NULL;
3906 3907
			fault->pfn = KVM_PFN_NOSLOT;
			fault->map_writable = false;
3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920
			return false;
		}
		/*
		 * If the APIC access page exists but is disabled, go directly
		 * to emulation without caching the MMIO access or creating a
		 * MMIO SPTE.  That way the cache doesn't need to be purged
		 * when the AVIC is re-enabled.
		 */
		if (slot && slot->id == APIC_ACCESS_PAGE_PRIVATE_MEMSLOT &&
		    !kvm_apicv_activated(vcpu->kvm)) {
			*r = RET_PF_EMULATE;
			return true;
		}
3921 3922
	}

3923
	async = false;
3924 3925 3926
	fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, &async,
					  fault->write, &fault->map_writable,
					  &fault->hva);
3927 3928 3929
	if (!async)
		return false; /* *pfn has correct page already */

3930
	if (!fault->prefetch && kvm_can_do_async_pf(vcpu)) {
3931 3932 3933
		trace_kvm_try_async_get_page(fault->addr, fault->gfn);
		if (kvm_find_async_pf_gfn(vcpu, fault->gfn)) {
			trace_kvm_async_pf_doublefault(fault->addr, fault->gfn);
3934
			kvm_make_request(KVM_REQ_APF_HALT, vcpu);
3935
			goto out_retry;
3936
		} else if (kvm_arch_setup_async_pf(vcpu, fault->addr, fault->gfn))
3937
			goto out_retry;
3938 3939
	}

3940 3941 3942
	fault->pfn = __gfn_to_pfn_memslot(slot, fault->gfn, false, NULL,
					  fault->write, &fault->map_writable,
					  &fault->hva);
3943
	return false;
3944 3945 3946 3947

out_retry:
	*r = RET_PF_RETRY;
	return true;
3948 3949
}

3950 3951 3952 3953 3954 3955 3956
/*
 * Returns true if the page fault is stale and needs to be retried, i.e. if the
 * root was invalidated by a memslot update or a relevant mmu_notifier fired.
 */
static bool is_page_fault_stale(struct kvm_vcpu *vcpu,
				struct kvm_page_fault *fault, int mmu_seq)
{
3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971
	struct kvm_mmu_page *sp = to_shadow_page(vcpu->arch.mmu->root_hpa);

	/* Special roots, e.g. pae_root, are not backed by shadow pages. */
	if (sp && is_obsolete_sp(vcpu->kvm, sp))
		return true;

	/*
	 * Roots without an associated shadow page are considered invalid if
	 * there is a pending request to free obsolete roots.  The request is
	 * only a hint that the current root _may_ be obsolete and needs to be
	 * reloaded, e.g. if the guest frees a PGD that KVM is tracking as a
	 * previous root, then __kvm_mmu_prepare_zap_page() signals all vCPUs
	 * to reload even if no vCPU is actively using the root.
	 */
	if (!sp && kvm_test_request(KVM_REQ_MMU_RELOAD, vcpu))
3972 3973 3974 3975 3976 3977
		return true;

	return fault->slot &&
	       mmu_notifier_retry_hva(vcpu->kvm, mmu_seq, fault->hva);
}

3978
static int direct_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
3979
{
3980
	bool is_tdp_mmu_fault = is_tdp_mmu(vcpu->arch.mmu);
3981

3982
	unsigned long mmu_seq;
3983
	int r;
3984

3985
	fault->gfn = fault->addr >> PAGE_SHIFT;
3986 3987
	fault->slot = kvm_vcpu_gfn_to_memslot(vcpu, fault->gfn);

3988
	if (page_fault_handle_page_track(vcpu, fault))
3989
		return RET_PF_EMULATE;
3990

3991
	r = fast_page_fault(vcpu, fault);
3992 3993
	if (r != RET_PF_INVALID)
		return r;
3994

3995
	r = mmu_topup_memory_caches(vcpu, false);
3996 3997
	if (r)
		return r;
3998

3999 4000 4001
	mmu_seq = vcpu->kvm->mmu_notifier_seq;
	smp_rmb();

4002
	if (kvm_faultin_pfn(vcpu, fault, &r))
4003
		return r;
4004

4005
	if (handle_abnormal_pfn(vcpu, fault, ACC_ALL, &r))
4006
		return r;
4007

4008
	r = RET_PF_RETRY;
4009

4010
	if (is_tdp_mmu_fault)
4011 4012 4013 4014
		read_lock(&vcpu->kvm->mmu_lock);
	else
		write_lock(&vcpu->kvm->mmu_lock);

4015
	if (is_page_fault_stale(vcpu, fault, mmu_seq))
4016
		goto out_unlock;
4017

4018 4019
	r = make_mmu_pages_available(vcpu);
	if (r)
4020
		goto out_unlock;
4021

4022
	if (is_tdp_mmu_fault)
4023
		r = kvm_tdp_mmu_map(vcpu, fault);
4024
	else
4025
		r = __direct_map(vcpu, fault);
4026

4027
out_unlock:
4028
	if (is_tdp_mmu_fault)
4029 4030 4031
		read_unlock(&vcpu->kvm->mmu_lock);
	else
		write_unlock(&vcpu->kvm->mmu_lock);
4032
	kvm_release_pfn_clean(fault->pfn);
4033
	return r;
4034 4035
}

4036 4037
static int nonpaging_page_fault(struct kvm_vcpu *vcpu,
				struct kvm_page_fault *fault)
4038
{
4039
	pgprintk("%s: gva %lx error %x\n", __func__, fault->addr, fault->error_code);
4040 4041

	/* This path builds a PAE pagetable, we can map 2mb pages at maximum. */
4042 4043
	fault->max_level = PG_LEVEL_2M;
	return direct_page_fault(vcpu, fault);
4044 4045
}

4046
int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
4047
				u64 fault_address, char *insn, int insn_len)
4048 4049
{
	int r = 1;
4050
	u32 flags = vcpu->arch.apf.host_apf_flags;
4051

4052 4053 4054 4055 4056 4057
#ifndef CONFIG_X86_64
	/* A 64-bit CR2 should be impossible on 32-bit KVM. */
	if (WARN_ON_ONCE(fault_address >> 32))
		return -EFAULT;
#endif

4058
	vcpu->arch.l1tf_flush_l1d = true;
4059
	if (!flags) {
4060 4061
		trace_kvm_page_fault(fault_address, error_code);

4062
		if (kvm_event_needs_reinjection(vcpu))
4063 4064 4065
			kvm_mmu_unprotect_page_virt(vcpu, fault_address);
		r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
				insn_len);
4066
	} else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
4067
		vcpu->arch.apf.host_apf_flags = 0;
4068
		local_irq_disable();
4069
		kvm_async_pf_task_wait_schedule(fault_address);
4070
		local_irq_enable();
4071 4072
	} else {
		WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
4073
	}
4074

4075 4076 4077 4078
	return r;
}
EXPORT_SYMBOL_GPL(kvm_handle_page_fault);

4079
int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
4080
{
4081 4082 4083
	while (fault->max_level > PG_LEVEL_4K) {
		int page_num = KVM_PAGES_PER_HPAGE(fault->max_level);
		gfn_t base = (fault->addr >> PAGE_SHIFT) & ~(page_num - 1);
4084

4085 4086
		if (kvm_mtrr_check_gfn_range_consistency(vcpu, base, page_num))
			break;
4087 4088

		--fault->max_level;
4089
	}
4090

4091
	return direct_page_fault(vcpu, fault);
4092 4093
}

4094
static void nonpaging_init_context(struct kvm_mmu *context)
4095 4096 4097
{
	context->page_fault = nonpaging_page_fault;
	context->gva_to_gpa = nonpaging_gva_to_gpa;
4098
	context->sync_page = nonpaging_sync_page;
4099
	context->invlpg = NULL;
4100
	context->direct_map = true;
4101 4102
}

4103
static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd,
4104 4105
				  union kvm_mmu_page_role role)
{
4106
	return (role.direct || pgd == root->pgd) &&
4107 4108
	       VALID_PAGE(root->hpa) && to_shadow_page(root->hpa) &&
	       role.word == to_shadow_page(root->hpa)->role.word;
4109 4110
}

4111
/*
4112
 * Find out if a previously cached root matching the new pgd/role is available.
4113 4114 4115 4116 4117 4118
 * The current root is also inserted into the cache.
 * If a matching root was found, it is assigned to kvm_mmu->root_hpa and true is
 * returned.
 * Otherwise, the LRU root from the cache is assigned to kvm_mmu->root_hpa and
 * false is returned. This root should now be freed by the caller.
 */
4119
static bool cached_root_available(struct kvm_vcpu *vcpu, gpa_t new_pgd,
4120 4121 4122 4123
				  union kvm_mmu_page_role new_role)
{
	uint i;
	struct kvm_mmu_root_info root;
4124
	struct kvm_mmu *mmu = vcpu->arch.mmu;
4125

4126
	root.pgd = mmu->root_pgd;
4127 4128
	root.hpa = mmu->root_hpa;

4129
	if (is_root_usable(&root, new_pgd, new_role))
4130 4131
		return true;

4132 4133 4134
	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
		swap(root, mmu->prev_roots[i]);

4135
		if (is_root_usable(&root, new_pgd, new_role))
4136 4137 4138 4139
			break;
	}

	mmu->root_hpa = root.hpa;
4140
	mmu->root_pgd = root.pgd;
4141 4142 4143 4144

	return i < KVM_MMU_NUM_PREV_ROOTS;
}

4145
static bool fast_pgd_switch(struct kvm_vcpu *vcpu, gpa_t new_pgd,
4146
			    union kvm_mmu_page_role new_role)
4147
{
4148
	struct kvm_mmu *mmu = vcpu->arch.mmu;
4149 4150 4151 4152 4153 4154 4155

	/*
	 * For now, limit the fast switch to 64-bit hosts+VMs in order to avoid
	 * having to deal with PDPTEs. We may add support for 32-bit hosts/VMs
	 * later if necessary.
	 */
	if (mmu->shadow_root_level >= PT64_ROOT_4LEVEL &&
4156
	    mmu->root_level >= PT64_ROOT_4LEVEL)
4157
		return cached_root_available(vcpu, new_pgd, new_role);
4158 4159

	return false;
4160 4161
}

4162
static void __kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd,
4163
			      union kvm_mmu_page_role new_role)
4164
{
4165
	if (!fast_pgd_switch(vcpu, new_pgd, new_role)) {
4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177
		kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOT_CURRENT);
		return;
	}

	/*
	 * It's possible that the cached previous root page is obsolete because
	 * of a change in the MMU generation number. However, changing the
	 * generation number is accompanied by KVM_REQ_MMU_RELOAD, which will
	 * free the root set here and allocate a new one.
	 */
	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);

4178
	if (force_flush_and_sync_on_reuse) {
4179 4180
		kvm_make_request(KVM_REQ_MMU_SYNC, vcpu);
		kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu);
4181
	}
4182 4183 4184 4185 4186 4187 4188 4189 4190

	/*
	 * The last MMIO access's GVA and GPA are cached in the VCPU. When
	 * switching to a new CR3, that GVA->GPA mapping may no longer be
	 * valid. So clear any cached MMIO info even when we don't need to sync
	 * the shadow page tables.
	 */
	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);

4191 4192 4193 4194 4195 4196 4197
	/*
	 * If this is a direct root page, it doesn't have a write flooding
	 * count. Otherwise, clear the write flooding count.
	 */
	if (!new_role.direct)
		__clear_sp_write_flooding_count(
				to_shadow_page(vcpu->arch.mmu->root_hpa));
4198 4199
}

4200
void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd)
4201
{
4202
	__kvm_mmu_new_pgd(vcpu, new_pgd, kvm_mmu_calc_root_page_role(vcpu));
4203
}
4204
EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd);
4205

4206 4207
static unsigned long get_cr3(struct kvm_vcpu *vcpu)
{
4208
	return kvm_read_cr3(vcpu);
4209 4210
}

4211
static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn,
4212
			   unsigned int access)
4213 4214 4215 4216 4217 4218 4219
{
	if (unlikely(is_mmio_spte(*sptep))) {
		if (gfn != get_mmio_spte_gfn(*sptep)) {
			mmu_spte_clear_no_track(sptep);
			return true;
		}

4220
		mark_mmio_spte(vcpu, sptep, gfn, access);
4221 4222 4223 4224 4225 4226
		return true;
	}

	return false;
}

4227 4228 4229 4230 4231
#define PTTYPE_EPT 18 /* arbitrary */
#define PTTYPE PTTYPE_EPT
#include "paging_tmpl.h"
#undef PTTYPE

4232 4233 4234 4235 4236 4237 4238 4239
#define PTTYPE 64
#include "paging_tmpl.h"
#undef PTTYPE

#define PTTYPE 32
#include "paging_tmpl.h"
#undef PTTYPE

4240
static void
4241
__reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check,
4242
			u64 pa_bits_rsvd, int level, bool nx, bool gbpages,
4243
			bool pse, bool amd)
4244
{
4245
	u64 gbpages_bit_rsvd = 0;
4246
	u64 nonleaf_bit8_rsvd = 0;
4247
	u64 high_bits_rsvd;
4248

4249
	rsvd_check->bad_mt_xwr = 0;
4250

4251
	if (!gbpages)
4252
		gbpages_bit_rsvd = rsvd_bits(7, 7);
4253

4254 4255 4256 4257 4258 4259 4260 4261 4262
	if (level == PT32E_ROOT_LEVEL)
		high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 62);
	else
		high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);

	/* Note, NX doesn't exist in PDPTEs, this is handled below. */
	if (!nx)
		high_bits_rsvd |= rsvd_bits(63, 63);

4263 4264 4265 4266
	/*
	 * Non-leaf PML4Es and PDPEs reserve bit 8 (which would be the G bit for
	 * leaf entries) on AMD CPUs only.
	 */
4267
	if (amd)
4268 4269
		nonleaf_bit8_rsvd = rsvd_bits(8, 8);

4270
	switch (level) {
4271 4272
	case PT32_ROOT_LEVEL:
		/* no rsvd bits for 2 level 4K page table entries */
4273 4274 4275 4276
		rsvd_check->rsvd_bits_mask[0][1] = 0;
		rsvd_check->rsvd_bits_mask[0][0] = 0;
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4277

4278
		if (!pse) {
4279
			rsvd_check->rsvd_bits_mask[1][1] = 0;
4280 4281 4282
			break;
		}

4283 4284
		if (is_cpuid_PSE36())
			/* 36bits PSE 4MB page */
4285
			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(17, 21);
4286 4287
		else
			/* 32 bits PSE 4MB page */
4288
			rsvd_check->rsvd_bits_mask[1][1] = rsvd_bits(13, 21);
4289 4290
		break;
	case PT32E_ROOT_LEVEL:
4291 4292 4293 4294 4295 4296 4297 4298
		rsvd_check->rsvd_bits_mask[0][2] = rsvd_bits(63, 63) |
						   high_bits_rsvd |
						   rsvd_bits(5, 8) |
						   rsvd_bits(1, 2);	/* PDPTE */
		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;	/* PDE */
		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;	/* PTE */
		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
						   rsvd_bits(13, 20);	/* large page */
4299 4300
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4301
		break;
4302
	case PT64_ROOT_5LEVEL:
4303 4304 4305
		rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd |
						   nonleaf_bit8_rsvd |
						   rsvd_bits(7, 7);
4306 4307
		rsvd_check->rsvd_bits_mask[1][4] =
			rsvd_check->rsvd_bits_mask[0][4];
4308
		fallthrough;
4309
	case PT64_ROOT_4LEVEL:
4310 4311 4312 4313 4314 4315 4316
		rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd |
						   nonleaf_bit8_rsvd |
						   rsvd_bits(7, 7);
		rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd |
						   gbpages_bit_rsvd;
		rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd;
		rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4317 4318
		rsvd_check->rsvd_bits_mask[1][3] =
			rsvd_check->rsvd_bits_mask[0][3];
4319 4320 4321 4322 4323
		rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd |
						   gbpages_bit_rsvd |
						   rsvd_bits(13, 29);
		rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd |
						   rsvd_bits(13, 20); /* large page */
4324 4325
		rsvd_check->rsvd_bits_mask[1][0] =
			rsvd_check->rsvd_bits_mask[0][0];
4326 4327 4328 4329
		break;
	}
}

4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344
static bool guest_can_use_gbpages(struct kvm_vcpu *vcpu)
{
	/*
	 * If TDP is enabled, let the guest use GBPAGES if they're supported in
	 * hardware.  The hardware page walker doesn't let KVM disable GBPAGES,
	 * i.e. won't treat them as reserved, and KVM doesn't redo the GVA->GPA
	 * walk for performance and complexity reasons.  Not to mention KVM
	 * _can't_ solve the problem because GVA->GPA walks aren't visible to
	 * KVM once a TDP translation is installed.  Mimic hardware behavior so
	 * that KVM's is at least consistent, i.e. doesn't randomly inject #PF.
	 */
	return tdp_enabled ? boot_cpu_has(X86_FEATURE_GBPAGES) :
			     guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES);
}

4345 4346 4347
static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
				  struct kvm_mmu *context)
{
4348
	__reset_rsvds_bits_mask(&context->guest_rsvd_check,
4349
				vcpu->arch.reserved_gpa_bits,
4350
				context->root_level, is_efer_nx(context),
4351
				guest_can_use_gbpages(vcpu),
4352
				is_cr4_pse(context),
4353
				guest_cpuid_is_amd_or_hygon(vcpu));
4354 4355
}

4356 4357
static void
__reset_rsvds_bits_mask_ept(struct rsvd_bits_validate *rsvd_check,
4358
			    u64 pa_bits_rsvd, bool execonly, int huge_page_level)
4359
{
4360
	u64 high_bits_rsvd = pa_bits_rsvd & rsvd_bits(0, 51);
4361
	u64 large_1g_rsvd = 0, large_2m_rsvd = 0;
4362
	u64 bad_mt_xwr;
4363

4364 4365 4366 4367 4368
	if (huge_page_level < PG_LEVEL_1G)
		large_1g_rsvd = rsvd_bits(7, 7);
	if (huge_page_level < PG_LEVEL_2M)
		large_2m_rsvd = rsvd_bits(7, 7);

4369 4370
	rsvd_check->rsvd_bits_mask[0][4] = high_bits_rsvd | rsvd_bits(3, 7);
	rsvd_check->rsvd_bits_mask[0][3] = high_bits_rsvd | rsvd_bits(3, 7);
4371 4372
	rsvd_check->rsvd_bits_mask[0][2] = high_bits_rsvd | rsvd_bits(3, 6) | large_1g_rsvd;
	rsvd_check->rsvd_bits_mask[0][1] = high_bits_rsvd | rsvd_bits(3, 6) | large_2m_rsvd;
4373
	rsvd_check->rsvd_bits_mask[0][0] = high_bits_rsvd;
4374 4375

	/* large page */
4376
	rsvd_check->rsvd_bits_mask[1][4] = rsvd_check->rsvd_bits_mask[0][4];
4377
	rsvd_check->rsvd_bits_mask[1][3] = rsvd_check->rsvd_bits_mask[0][3];
4378 4379
	rsvd_check->rsvd_bits_mask[1][2] = high_bits_rsvd | rsvd_bits(12, 29) | large_1g_rsvd;
	rsvd_check->rsvd_bits_mask[1][1] = high_bits_rsvd | rsvd_bits(12, 20) | large_2m_rsvd;
4380
	rsvd_check->rsvd_bits_mask[1][0] = rsvd_check->rsvd_bits_mask[0][0];
4381

4382 4383 4384 4385 4386 4387 4388 4389
	bad_mt_xwr = 0xFFull << (2 * 8);	/* bits 3..5 must not be 2 */
	bad_mt_xwr |= 0xFFull << (3 * 8);	/* bits 3..5 must not be 3 */
	bad_mt_xwr |= 0xFFull << (7 * 8);	/* bits 3..5 must not be 7 */
	bad_mt_xwr |= REPEAT_BYTE(1ull << 2);	/* bits 0..2 must not be 010 */
	bad_mt_xwr |= REPEAT_BYTE(1ull << 6);	/* bits 0..2 must not be 110 */
	if (!execonly) {
		/* bits 0..2 must not be 100 unless VMX capabilities allow it */
		bad_mt_xwr |= REPEAT_BYTE(1ull << 4);
4390
	}
4391
	rsvd_check->bad_mt_xwr = bad_mt_xwr;
4392 4393
}

4394
static void reset_rsvds_bits_mask_ept(struct kvm_vcpu *vcpu,
4395
		struct kvm_mmu *context, bool execonly, int huge_page_level)
4396 4397
{
	__reset_rsvds_bits_mask_ept(&context->guest_rsvd_check,
4398 4399
				    vcpu->arch.reserved_gpa_bits, execonly,
				    huge_page_level);
4400 4401
}

4402 4403 4404 4405 4406
static inline u64 reserved_hpa_bits(void)
{
	return rsvd_bits(shadow_phys_bits, 63);
}

4407 4408 4409 4410 4411
/*
 * the page table on host is the shadow page table for the page
 * table in guest or amd nested guest, its mmu features completely
 * follow the features in guest.
 */
4412 4413
static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
					struct kvm_mmu *context)
4414
{
4415 4416 4417 4418 4419 4420 4421 4422
	/*
	 * KVM uses NX when TDP is disabled to handle a variety of scenarios,
	 * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and
	 * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0.
	 * The iTLB multi-hit workaround can be toggled at any time, so assume
	 * NX can be used by any non-nested shadow MMU to avoid having to reset
	 * MMU contexts.  Note, KVM forces EFER.NX=1 when TDP is disabled.
	 */
4423
	bool uses_nx = is_efer_nx(context) || !tdp_enabled;
4424 4425 4426 4427 4428

	/* @amd adds a check on bit of SPTEs, which KVM shouldn't use anyways. */
	bool is_amd = true;
	/* KVM doesn't use 2-level page tables for the shadow MMU. */
	bool is_pse = false;
4429 4430
	struct rsvd_bits_validate *shadow_zero_check;
	int i;
4431

4432 4433
	WARN_ON_ONCE(context->shadow_root_level < PT32E_ROOT_LEVEL);

4434
	shadow_zero_check = &context->shadow_zero_check;
4435
	__reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
4436
				context->shadow_root_level, uses_nx,
4437
				guest_can_use_gbpages(vcpu), is_pse, is_amd);
4438 4439 4440 4441 4442 4443 4444 4445 4446

	if (!shadow_me_mask)
		return;

	for (i = context->shadow_root_level; --i >= 0;) {
		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
	}

4447 4448
}

4449 4450 4451 4452 4453 4454
static inline bool boot_cpu_is_amd(void)
{
	WARN_ON_ONCE(!tdp_enabled);
	return shadow_x_mask == 0;
}

4455 4456 4457 4458 4459
/*
 * the direct page table on host, use as much mmu features as
 * possible, however, kvm currently does not do execution-protection.
 */
static void
4460
reset_tdp_shadow_zero_bits_mask(struct kvm_mmu *context)
4461
{
4462 4463 4464 4465 4466
	struct rsvd_bits_validate *shadow_zero_check;
	int i;

	shadow_zero_check = &context->shadow_zero_check;

4467
	if (boot_cpu_is_amd())
4468
		__reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(),
4469
					context->shadow_root_level, false,
4470
					boot_cpu_has(X86_FEATURE_GBPAGES),
4471
					false, true);
4472
	else
4473
		__reset_rsvds_bits_mask_ept(shadow_zero_check,
4474 4475
					    reserved_hpa_bits(), false,
					    max_huge_page_level);
4476

4477 4478 4479 4480 4481 4482 4483
	if (!shadow_me_mask)
		return;

	for (i = context->shadow_root_level; --i >= 0;) {
		shadow_zero_check->rsvd_bits_mask[0][i] &= ~shadow_me_mask;
		shadow_zero_check->rsvd_bits_mask[1][i] &= ~shadow_me_mask;
	}
4484 4485 4486 4487 4488 4489 4490
}

/*
 * as the comments in reset_shadow_zero_bits_mask() except it
 * is the shadow page table for intel nested guest.
 */
static void
4491
reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
4492 4493
{
	__reset_rsvds_bits_mask_ept(&context->shadow_zero_check,
4494 4495
				    reserved_hpa_bits(), execonly,
				    max_huge_page_level);
4496 4497
}

4498 4499 4500 4501 4502 4503 4504 4505 4506 4507
#define BYTE_MASK(access) \
	((1 & (access) ? 2 : 0) | \
	 (2 & (access) ? 4 : 0) | \
	 (3 & (access) ? 8 : 0) | \
	 (4 & (access) ? 16 : 0) | \
	 (5 & (access) ? 32 : 0) | \
	 (6 & (access) ? 64 : 0) | \
	 (7 & (access) ? 128 : 0))


4508
static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept)
4509
{
4510 4511 4512 4513 4514 4515
	unsigned byte;

	const u8 x = BYTE_MASK(ACC_EXEC_MASK);
	const u8 w = BYTE_MASK(ACC_WRITE_MASK);
	const u8 u = BYTE_MASK(ACC_USER_MASK);

4516 4517 4518
	bool cr4_smep = is_cr4_smep(mmu);
	bool cr4_smap = is_cr4_smap(mmu);
	bool cr0_wp = is_cr0_wp(mmu);
4519
	bool efer_nx = is_efer_nx(mmu);
4520 4521

	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
4522 4523
		unsigned pfec = byte << 1;

4524
		/*
4525 4526
		 * Each "*f" variable has a 1 bit for each UWX value
		 * that causes a fault with the given PFEC.
4527
		 */
4528

4529
		/* Faults from writes to non-writable pages */
4530
		u8 wf = (pfec & PFERR_WRITE_MASK) ? (u8)~w : 0;
4531
		/* Faults from user mode accesses to supervisor pages */
4532
		u8 uf = (pfec & PFERR_USER_MASK) ? (u8)~u : 0;
4533
		/* Faults from fetches of non-executable pages*/
4534
		u8 ff = (pfec & PFERR_FETCH_MASK) ? (u8)~x : 0;
4535 4536 4537 4538 4539 4540 4541 4542 4543 4544
		/* Faults from kernel mode fetches of user pages */
		u8 smepf = 0;
		/* Faults from kernel mode accesses of user pages */
		u8 smapf = 0;

		if (!ept) {
			/* Faults from kernel mode accesses to user pages */
			u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u;

			/* Not really needed: !nx will cause pte.nx to fault */
4545
			if (!efer_nx)
4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559
				ff = 0;

			/* Allow supervisor writes if !cr0.wp */
			if (!cr0_wp)
				wf = (pfec & PFERR_USER_MASK) ? wf : 0;

			/* Disallow supervisor fetches of user code if cr4.smep */
			if (cr4_smep)
				smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0;

			/*
			 * SMAP:kernel-mode data accesses from user-mode
			 * mappings should fault. A fault is considered
			 * as a SMAP violation if all of the following
Peng Hao's avatar
Peng Hao committed
4560
			 * conditions are true:
4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573
			 *   - X86_CR4_SMAP is set in CR4
			 *   - A user page is accessed
			 *   - The access is not a fetch
			 *   - Page fault in kernel mode
			 *   - if CPL = 3 or X86_EFLAGS_AC is clear
			 *
			 * Here, we cover the first three conditions.
			 * The fourth is computed dynamically in permission_fault();
			 * PFERR_RSVD_MASK bit will be set in PFEC if the access is
			 * *not* subject to SMAP restrictions.
			 */
			if (cr4_smap)
				smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
4574
		}
4575 4576

		mmu->permissions[byte] = ff | uf | wf | smepf | smapf;
4577 4578 4579
	}
}

4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603
/*
* PKU is an additional mechanism by which the paging controls access to
* user-mode addresses based on the value in the PKRU register.  Protection
* key violations are reported through a bit in the page fault error code.
* Unlike other bits of the error code, the PK bit is not known at the
* call site of e.g. gva_to_gpa; it must be computed directly in
* permission_fault based on two bits of PKRU, on some machine state (CR4,
* CR0, EFER, CPL), and on other bits of the error code and the page tables.
*
* In particular the following conditions come from the error code, the
* page tables and the machine state:
* - PK is always zero unless CR4.PKE=1 and EFER.LMA=1
* - PK is always zero if RSVD=1 (reserved bit set) or F=1 (instruction fetch)
* - PK is always zero if U=0 in the page tables
* - PKRU.WD is ignored if CR0.WP=0 and the access is a supervisor access.
*
* The PKRU bitmask caches the result of these four conditions.  The error
* code (minus the P bit) and the page table's U bit form an index into the
* PKRU bitmask.  Two bits of the PKRU bitmask are then extracted and ANDed
* with the two bits of the PKRU register corresponding to the protection key.
* For the first three conditions above the bits will be 00, thus masking
* away both AD and WD.  For all reads or if the last condition holds, WD
* only will be masked away.
*/
4604
static void update_pkru_bitmask(struct kvm_mmu *mmu)
4605 4606 4607 4608
{
	unsigned bit;
	bool wp;

4609 4610 4611
	mmu->pkru_mask = 0;

	if (!is_cr4_pke(mmu))
4612 4613
		return;

4614
	wp = is_cr0_wp(mmu);
4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647

	for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) {
		unsigned pfec, pkey_bits;
		bool check_pkey, check_write, ff, uf, wf, pte_user;

		pfec = bit << 1;
		ff = pfec & PFERR_FETCH_MASK;
		uf = pfec & PFERR_USER_MASK;
		wf = pfec & PFERR_WRITE_MASK;

		/* PFEC.RSVD is replaced by ACC_USER_MASK. */
		pte_user = pfec & PFERR_RSVD_MASK;

		/*
		 * Only need to check the access which is not an
		 * instruction fetch and is to a user page.
		 */
		check_pkey = (!ff && pte_user);
		/*
		 * write access is controlled by PKRU if it is a
		 * user access or CR0.WP = 1.
		 */
		check_write = check_pkey && wf && (uf || wp);

		/* PKRU.AD stops both read and write access. */
		pkey_bits = !!check_pkey;
		/* PKRU.WD stops write access. */
		pkey_bits |= (!!check_write) << 1;

		mmu->pkru_mask |= (pkey_bits & 3) << pfec;
	}
}

4648 4649
static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
					struct kvm_mmu *mmu)
4650
{
4651 4652
	if (!is_cr0_pg(mmu))
		return;
4653

4654 4655 4656
	reset_rsvds_bits_mask(vcpu, mmu);
	update_permission_bitmask(mmu, false);
	update_pkru_bitmask(mmu);
4657 4658
}

4659
static void paging64_init_context(struct kvm_mmu *context)
4660 4661 4662
{
	context->page_fault = paging64_page_fault;
	context->gva_to_gpa = paging64_gva_to_gpa;
4663
	context->sync_page = paging64_sync_page;
Marcelo Tosatti's avatar
Marcelo Tosatti committed
4664
	context->invlpg = paging64_invlpg;
4665
	context->direct_map = false;
4666 4667
}

4668
static void paging32_init_context(struct kvm_mmu *context)
4669 4670 4671
{
	context->page_fault = paging32_page_fault;
	context->gva_to_gpa = paging32_gva_to_gpa;
4672
	context->sync_page = paging32_sync_page;
Marcelo Tosatti's avatar
Marcelo Tosatti committed
4673
	context->invlpg = paging32_invlpg;
4674
	context->direct_map = false;
4675 4676
}

4677 4678
static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu,
							 struct kvm_mmu_role_regs *regs)
4679 4680 4681
{
	union kvm_mmu_extended_role ext = {0};

4682 4683 4684 4685 4686 4687
	if (____is_cr0_pg(regs)) {
		ext.cr0_pg = 1;
		ext.cr4_pae = ____is_cr4_pae(regs);
		ext.cr4_smep = ____is_cr4_smep(regs);
		ext.cr4_smap = ____is_cr4_smap(regs);
		ext.cr4_pse = ____is_cr4_pse(regs);
4688 4689 4690 4691

		/* PKEY and LA57 are active iff long mode is active. */
		ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs);
		ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs);
4692
		ext.efer_lma = ____is_efer_lma(regs);
4693
	}
4694 4695 4696 4697 4698 4699

	ext.valid = 1;

	return ext;
}

4700
static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu,
4701
						   struct kvm_mmu_role_regs *regs,
4702 4703 4704 4705 4706
						   bool base_only)
{
	union kvm_mmu_role role = {0};

	role.base.access = ACC_ALL;
4707 4708 4709 4710
	if (____is_cr0_pg(regs)) {
		role.base.efer_nx = ____is_efer_nx(regs);
		role.base.cr0_wp = ____is_cr0_wp(regs);
	}
4711 4712 4713 4714 4715 4716
	role.base.smm = is_smm(vcpu);
	role.base.guest_mode = is_guest_mode(vcpu);

	if (base_only)
		return role;

4717
	role.ext = kvm_calc_mmu_role_ext(vcpu, regs);
4718 4719 4720 4721

	return role;
}

4722 4723
static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu)
{
4724 4725 4726 4727
	/* tdp_root_level is architecture forced level, use it if nonzero */
	if (tdp_root_level)
		return tdp_root_level;

4728
	/* Use 5-level TDP if and only if it's useful/necessary. */
4729
	if (max_tdp_level == 5 && cpuid_maxphyaddr(vcpu) <= 48)
4730 4731
		return 4;

4732
	return max_tdp_level;
4733 4734
}

4735
static union kvm_mmu_role
4736 4737
kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu,
				struct kvm_mmu_role_regs *regs, bool base_only)
4738
{
4739
	union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only);
4740

4741
	role.base.ad_disabled = (shadow_accessed_mask == 0);
4742
	role.base.level = kvm_mmu_get_tdp_level(vcpu);
4743
	role.base.direct = true;
4744
	role.base.has_4_byte_gpte = false;
4745 4746 4747 4748

	return role;
}

4749
static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
4750
{
4751
	struct kvm_mmu *context = &vcpu->arch.root_mmu;
4752
	struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
4753
	union kvm_mmu_role new_role =
4754
		kvm_calc_tdp_mmu_root_page_role(vcpu, &regs, false);
4755

4756 4757 4758 4759
	if (new_role.as_u64 == context->mmu_role.as_u64)
		return;

	context->mmu_role.as_u64 = new_role.as_u64;
4760
	context->page_fault = kvm_tdp_page_fault;
4761
	context->sync_page = nonpaging_sync_page;
4762
	context->invlpg = NULL;
4763
	context->shadow_root_level = kvm_mmu_get_tdp_level(vcpu);
4764
	context->direct_map = true;
4765
	context->get_guest_pgd = get_cr3;
4766
	context->get_pdptr = kvm_pdptr_read;
4767
	context->inject_page_fault = kvm_inject_page_fault;
4768
	context->root_level = role_regs_to_root_level(&regs);
4769

4770
	if (!is_cr0_pg(context))
4771
		context->gva_to_gpa = nonpaging_gva_to_gpa;
4772
	else if (is_cr4_pae(context))
4773
		context->gva_to_gpa = paging64_gva_to_gpa;
4774
	else
4775
		context->gva_to_gpa = paging32_gva_to_gpa;
4776

4777
	reset_guest_paging_metadata(vcpu, context);
4778
	reset_tdp_shadow_zero_bits_mask(context);
4779 4780
}

4781
static union kvm_mmu_role
4782 4783
kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu,
				      struct kvm_mmu_role_regs *regs, bool base_only)
4784
{
4785
	union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only);
4786

4787 4788
	role.base.smep_andnot_wp = role.ext.cr4_smep && !____is_cr0_wp(regs);
	role.base.smap_andnot_wp = role.ext.cr4_smap && !____is_cr0_wp(regs);
4789
	role.base.has_4_byte_gpte = ____is_cr0_pg(regs) && !____is_cr4_pae(regs);
4790

4791 4792 4793 4794
	return role;
}

static union kvm_mmu_role
4795 4796
kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu,
				   struct kvm_mmu_role_regs *regs, bool base_only)
4797 4798
{
	union kvm_mmu_role role =
4799
		kvm_calc_shadow_root_page_role_common(vcpu, regs, base_only);
4800

4801
	role.base.direct = !____is_cr0_pg(regs);
4802

4803
	if (!____is_efer_lma(regs))
4804
		role.base.level = PT32E_ROOT_LEVEL;
4805
	else if (____is_cr4_la57(regs))
4806
		role.base.level = PT64_ROOT_5LEVEL;
4807
	else
4808
		role.base.level = PT64_ROOT_4LEVEL;
4809 4810 4811 4812

	return role;
}

4813
static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context,
4814
				    struct kvm_mmu_role_regs *regs,
4815
				    union kvm_mmu_role new_role)
4816
{
4817 4818
	if (new_role.as_u64 == context->mmu_role.as_u64)
		return;
4819

4820
	context->mmu_role.as_u64 = new_role.as_u64;
4821

4822
	if (!is_cr0_pg(context))
4823
		nonpaging_init_context(context);
4824
	else if (is_cr4_pae(context))
4825
		paging64_init_context(context);
4826
	else
4827
		paging32_init_context(context);
4828
	context->root_level = role_regs_to_root_level(regs);
4829

4830
	reset_guest_paging_metadata(vcpu, context);
4831 4832
	context->shadow_root_level = new_role.base.level;

4833
	reset_shadow_zero_bits_mask(vcpu, context);
4834
}
4835

4836 4837
static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
				struct kvm_mmu_role_regs *regs)
4838
{
4839
	struct kvm_mmu *context = &vcpu->arch.root_mmu;
4840
	union kvm_mmu_role new_role =
4841
		kvm_calc_shadow_mmu_root_page_role(vcpu, regs, false);
4842

4843
	shadow_mmu_init_context(vcpu, context, regs, new_role);
4844 4845
}

4846
static union kvm_mmu_role
4847 4848
kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu,
				   struct kvm_mmu_role_regs *regs)
4849 4850
{
	union kvm_mmu_role role =
4851
		kvm_calc_shadow_root_page_role_common(vcpu, regs, false);
4852 4853

	role.base.direct = false;
4854
	role.base.level = kvm_mmu_get_tdp_level(vcpu);
4855 4856 4857 4858

	return role;
}

4859 4860
void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
			     unsigned long cr4, u64 efer, gpa_t nested_cr3)
4861
{
4862
	struct kvm_mmu *context = &vcpu->arch.guest_mmu;
4863 4864
	struct kvm_mmu_role_regs regs = {
		.cr0 = cr0,
4865
		.cr4 = cr4 & ~X86_CR4_PKE,
4866 4867
		.efer = efer,
	};
4868
	union kvm_mmu_role new_role;
4869

4870
	new_role = kvm_calc_shadow_npt_root_page_role(vcpu, &regs);
4871

4872
	__kvm_mmu_new_pgd(vcpu, nested_cr3, new_role.base);
4873

4874
	shadow_mmu_init_context(vcpu, context, &regs, new_role);
4875 4876
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu);
4877

4878 4879
static union kvm_mmu_role
kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty,
4880
				   bool execonly, u8 level)
4881
{
4882
	union kvm_mmu_role role = {0};
4883

4884 4885
	/* SMM flag is inherited from root_mmu */
	role.base.smm = vcpu->arch.root_mmu.mmu_role.base.smm;
4886

4887
	role.base.level = level;
4888
	role.base.has_4_byte_gpte = false;
4889 4890 4891 4892
	role.base.direct = false;
	role.base.ad_disabled = !accessed_dirty;
	role.base.guest_mode = true;
	role.base.access = ACC_ALL;
4893

4894 4895
	/* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
	role.ext.word = 0;
4896
	role.ext.execonly = execonly;
4897
	role.ext.valid = 1;
4898 4899 4900 4901

	return role;
}

4902
void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
4903 4904
			     int huge_page_level, bool accessed_dirty,
			     gpa_t new_eptp)
4905
{
4906
	struct kvm_mmu *context = &vcpu->arch.guest_mmu;
4907
	u8 level = vmx_eptp_page_walk_level(new_eptp);
4908 4909
	union kvm_mmu_role new_role =
		kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
4910
						   execonly, level);
4911

4912
	__kvm_mmu_new_pgd(vcpu, new_eptp, new_role.base);
4913 4914 4915

	if (new_role.as_u64 == context->mmu_role.as_u64)
		return;
4916

4917 4918
	context->mmu_role.as_u64 = new_role.as_u64;

4919
	context->shadow_root_level = level;
4920

4921
	context->ept_ad = accessed_dirty;
4922 4923 4924 4925
	context->page_fault = ept_page_fault;
	context->gva_to_gpa = ept_gva_to_gpa;
	context->sync_page = ept_sync_page;
	context->invlpg = ept_invlpg;
4926
	context->root_level = level;
4927
	context->direct_map = false;
4928

4929
	update_permission_bitmask(context, true);
4930
	context->pkru_mask = 0;
4931
	reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level);
4932
	reset_ept_shadow_zero_bits_mask(context, execonly);
4933 4934 4935
}
EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu);

4936
static void init_kvm_softmmu(struct kvm_vcpu *vcpu)
4937
{
4938
	struct kvm_mmu *context = &vcpu->arch.root_mmu;
4939
	struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
4940

4941
	kvm_init_shadow_mmu(vcpu, &regs);
4942

4943
	context->get_guest_pgd     = get_cr3;
4944 4945
	context->get_pdptr         = kvm_pdptr_read;
	context->inject_page_fault = kvm_inject_page_fault;
4946 4947
}

4948 4949
static union kvm_mmu_role
kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu, struct kvm_mmu_role_regs *regs)
4950
{
4951 4952 4953
	union kvm_mmu_role role;

	role = kvm_calc_shadow_root_page_role_common(vcpu, regs, false);
4954 4955 4956 4957 4958 4959 4960

	/*
	 * Nested MMUs are used only for walking L2's gva->gpa, they never have
	 * shadow pages of their own and so "direct" has no meaning.   Set it
	 * to "true" to try to detect bogus usage of the nested MMU.
	 */
	role.base.direct = true;
4961
	role.base.level = role_regs_to_root_level(regs);
4962 4963 4964
	return role;
}

4965
static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
4966
{
4967 4968
	struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
	union kvm_mmu_role new_role = kvm_calc_nested_mmu_role(vcpu, &regs);
4969 4970
	struct kvm_mmu *g_context = &vcpu->arch.nested_mmu;

4971 4972 4973 4974
	if (new_role.as_u64 == g_context->mmu_role.as_u64)
		return;

	g_context->mmu_role.as_u64 = new_role.as_u64;
4975
	g_context->get_guest_pgd     = get_cr3;
4976
	g_context->get_pdptr         = kvm_pdptr_read;
4977
	g_context->inject_page_fault = kvm_inject_page_fault;
4978
	g_context->root_level        = new_role.base.level;
4979

4980 4981 4982 4983 4984 4985
	/*
	 * L2 page tables are never shadowed, so there is no need to sync
	 * SPTEs.
	 */
	g_context->invlpg            = NULL;

4986
	/*
4987
	 * Note that arch.mmu->gva_to_gpa translates l2_gpa to l1_gpa using
4988 4989 4990 4991 4992
	 * L1's nested page tables (e.g. EPT12). The nested translation
	 * of l2_gva to l1_gpa is done by arch.nested_mmu.gva_to_gpa using
	 * L2's page tables as the first level of translation and L1's
	 * nested page tables as the second level of translation. Basically
	 * the gva_to_gpa functions between mmu and nested_mmu are swapped.
4993
	 */
4994
	if (!is_paging(vcpu))
4995
		g_context->gva_to_gpa = nonpaging_gva_to_gpa;
4996
	else if (is_long_mode(vcpu))
4997
		g_context->gva_to_gpa = paging64_gva_to_gpa;
4998
	else if (is_pae(vcpu))
4999
		g_context->gva_to_gpa = paging64_gva_to_gpa;
5000
	else
5001
		g_context->gva_to_gpa = paging32_gva_to_gpa;
5002

5003
	reset_guest_paging_metadata(vcpu, g_context);
5004 5005
}

5006
void kvm_init_mmu(struct kvm_vcpu *vcpu)
5007
{
5008
	if (mmu_is_nested(vcpu))
5009
		init_kvm_nested_mmu(vcpu);
5010
	else if (tdp_enabled)
5011
		init_kvm_tdp_mmu(vcpu);
5012
	else
5013
		init_kvm_softmmu(vcpu);
5014
}
5015
EXPORT_SYMBOL_GPL(kvm_init_mmu);
5016

5017 5018 5019
static union kvm_mmu_page_role
kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu)
{
5020
	struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu);
5021 5022
	union kvm_mmu_role role;

5023
	if (tdp_enabled)
5024
		role = kvm_calc_tdp_mmu_root_page_role(vcpu, &regs, true);
5025
	else
5026
		role = kvm_calc_shadow_mmu_root_page_role(vcpu, &regs, true);
5027 5028

	return role.base;
5029
}
5030

5031 5032 5033 5034 5035
void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu)
{
	/*
	 * Invalidate all MMU roles to force them to reinitialize as CPUID
	 * information is factored into reserved bit calculations.
5036 5037 5038 5039 5040 5041 5042 5043
	 *
	 * Correctly handling multiple vCPU models with respect to paging and
	 * physical address properties) in a single VM would require tracking
	 * all relevant CPUID information in kvm_mmu_page_role. That is very
	 * undesirable as it would increase the memory requirements for
	 * gfn_track (see struct kvm_mmu_page_role comments).  For now that
	 * problem is swept under the rug; KVM's CPUID API is horrific and
	 * it's all but impossible to solve it without introducing a new API.
5044 5045 5046 5047 5048
	 */
	vcpu->arch.root_mmu.mmu_role.ext.valid = 0;
	vcpu->arch.guest_mmu.mmu_role.ext.valid = 0;
	vcpu->arch.nested_mmu.mmu_role.ext.valid = 0;
	kvm_mmu_reset_context(vcpu);
5049 5050

	/*
5051 5052
	 * Changing guest CPUID after KVM_RUN is forbidden, see the comment in
	 * kvm_arch_vcpu_ioctl().
5053
	 */
5054
	KVM_BUG_ON(vcpu->arch.last_vmentry_cpu != -1, vcpu->kvm);
5055 5056
}

5057
void kvm_mmu_reset_context(struct kvm_vcpu *vcpu)
5058
{
5059
	kvm_mmu_unload(vcpu);
5060
	kvm_init_mmu(vcpu);
Avi Kivity's avatar
Avi Kivity committed
5061
}
5062
EXPORT_SYMBOL_GPL(kvm_mmu_reset_context);
Avi Kivity's avatar
Avi Kivity committed
5063 5064

int kvm_mmu_load(struct kvm_vcpu *vcpu)
5065
{
5066 5067
	int r;

5068
	r = mmu_topup_memory_caches(vcpu, !vcpu->arch.mmu->direct_map);
Avi Kivity's avatar
Avi Kivity committed
5069 5070
	if (r)
		goto out;
5071
	r = mmu_alloc_special_roots(vcpu);
Avi Kivity's avatar
Avi Kivity committed
5072 5073
	if (r)
		goto out;
5074
	if (vcpu->arch.mmu->direct_map)
5075 5076 5077
		r = mmu_alloc_direct_roots(vcpu);
	else
		r = mmu_alloc_shadow_roots(vcpu);
5078 5079
	if (r)
		goto out;
5080 5081 5082

	kvm_mmu_sync_roots(vcpu);

5083
	kvm_mmu_load_pgd(vcpu);
5084
	static_call(kvm_x86_flush_tlb_current)(vcpu);
5085 5086
out:
	return r;
5087
}
Avi Kivity's avatar
Avi Kivity committed
5088 5089 5090

void kvm_mmu_unload(struct kvm_vcpu *vcpu)
{
5091 5092 5093 5094
	kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, KVM_MMU_ROOTS_ALL);
	WARN_ON(VALID_PAGE(vcpu->arch.root_mmu.root_hpa));
	kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
	WARN_ON(VALID_PAGE(vcpu->arch.guest_mmu.root_hpa));
Avi Kivity's avatar
Avi Kivity committed
5095
}
5096

5097 5098 5099 5100 5101 5102 5103 5104
static bool need_remote_flush(u64 old, u64 new)
{
	if (!is_shadow_present_pte(old))
		return false;
	if (!is_shadow_present_pte(new))
		return true;
	if ((old ^ new) & PT64_BASE_ADDR_MASK)
		return true;
5105 5106
	old ^= shadow_nx_mask;
	new ^= shadow_nx_mask;
5107 5108 5109
	return (old & ~new & PT64_PERM_MASK) != 0;
}

5110
static u64 mmu_pte_write_fetch_gpte(struct kvm_vcpu *vcpu, gpa_t *gpa,
5111
				    int *bytes)
5112
{
5113
	u64 gentry = 0;
5114
	int r;
5115 5116 5117

	/*
	 * Assume that the pte write on a page table of the same type
5118 5119
	 * as the current vcpu paging mode since we update the sptes only
	 * when they have the same mode.
5120
	 */
5121
	if (is_pae(vcpu) && *bytes == 4) {
5122
		/* Handle a 32-bit guest writing two halves of a 64-bit gpte */
5123 5124
		*gpa &= ~(gpa_t)7;
		*bytes = 8;
5125 5126
	}

5127 5128 5129 5130
	if (*bytes == 4 || *bytes == 8) {
		r = kvm_vcpu_read_guest_atomic(vcpu, *gpa, &gentry, *bytes);
		if (r)
			gentry = 0;
5131 5132
	}

5133 5134 5135 5136 5137 5138 5139
	return gentry;
}

/*
 * If we're seeing too many writes to a page, it may no longer be a page table,
 * or we may be forking, in which case it is better to unmap the page.
 */
5140
static bool detect_write_flooding(struct kvm_mmu_page *sp)
5141
{
5142 5143 5144 5145
	/*
	 * Skip write-flooding detected for the sp whose level is 1, because
	 * it can become unsync, then the guest page is not write-protected.
	 */
5146
	if (sp->role.level == PG_LEVEL_4K)
5147
		return false;
5148

5149 5150
	atomic_inc(&sp->write_flooding_count);
	return atomic_read(&sp->write_flooding_count) >= 3;
5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165
}

/*
 * Misaligned accesses are too much trouble to fix up; also, they usually
 * indicate a page is not used as a page table.
 */
static bool detect_write_misaligned(struct kvm_mmu_page *sp, gpa_t gpa,
				    int bytes)
{
	unsigned offset, pte_size, misaligned;

	pgprintk("misaligned: gpa %llx bytes %d role %x\n",
		 gpa, bytes, sp->role.word);

	offset = offset_in_page(gpa);
5166
	pte_size = sp->role.has_4_byte_gpte ? 4 : 8;
5167 5168 5169 5170 5171 5172 5173 5174

	/*
	 * Sometimes, the OS only writes the last one bytes to update status
	 * bits, for example, in linux, andb instruction is used in clear_bit().
	 */
	if (!(offset & (pte_size - 1)) && bytes == 1)
		return false;

5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189
	misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1);
	misaligned |= bytes < 4;

	return misaligned;
}

static u64 *get_written_sptes(struct kvm_mmu_page *sp, gpa_t gpa, int *nspte)
{
	unsigned page_offset, quadrant;
	u64 *spte;
	int level;

	page_offset = offset_in_page(gpa);
	level = sp->role.level;
	*nspte = 1;
5190
	if (sp->role.has_4_byte_gpte) {
5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211
		page_offset <<= 1;	/* 32->64 */
		/*
		 * A 32-bit pde maps 4MB while the shadow pdes map
		 * only 2MB.  So we need to double the offset again
		 * and zap two pdes instead of one.
		 */
		if (level == PT32_ROOT_LEVEL) {
			page_offset &= ~7; /* kill rounding error */
			page_offset <<= 1;
			*nspte = 2;
		}
		quadrant = page_offset >> PAGE_SHIFT;
		page_offset &= ~PAGE_MASK;
		if (quadrant != sp->role.quadrant)
			return NULL;
	}

	spte = &sp->spt[page_offset / sizeof(*spte)];
	return spte;
}

5212
static void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa,
5213 5214
			      const u8 *new, int bytes,
			      struct kvm_page_track_notifier_node *node)
5215 5216 5217 5218 5219 5220
{
	gfn_t gfn = gpa >> PAGE_SHIFT;
	struct kvm_mmu_page *sp;
	LIST_HEAD(invalid_list);
	u64 entry, gentry, *spte;
	int npte;
5221
	bool flush = false;
5222 5223 5224 5225 5226

	/*
	 * If we don't have indirect shadow pages, it means no page is
	 * write-protected, so we can exit simply.
	 */
5227
	if (!READ_ONCE(vcpu->kvm->arch.indirect_shadow_pages))
5228 5229 5230 5231 5232 5233
		return;

	pgprintk("%s: gpa %llx bytes %d\n", __func__, gpa, bytes);

	/*
	 * No need to care whether allocation memory is successful
5234
	 * or not since pte prefetch is skipped if it does not have
5235 5236
	 * enough objects in the cache.
	 */
5237
	mmu_topup_memory_caches(vcpu, true);
5238

5239
	write_lock(&vcpu->kvm->mmu_lock);
5240 5241 5242

	gentry = mmu_pte_write_fetch_gpte(vcpu, &gpa, &bytes);

5243
	++vcpu->kvm->stat.mmu_pte_write;
5244
	kvm_mmu_audit(vcpu, AUDIT_PRE_PTE_WRITE);
5245

5246
	for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) {
5247
		if (detect_write_misaligned(sp, gpa, bytes) ||
5248
		      detect_write_flooding(sp)) {
5249
			kvm_mmu_prepare_zap_page(vcpu->kvm, sp, &invalid_list);
5250
			++vcpu->kvm->stat.mmu_flooded;
5251 5252
			continue;
		}
5253 5254 5255 5256 5257

		spte = get_written_sptes(sp, gpa, &npte);
		if (!spte)
			continue;

5258
		while (npte--) {
5259
			entry = *spte;
5260
			mmu_page_zap_pte(vcpu->kvm, sp, spte, NULL);
5261 5262
			if (gentry && sp->role.level != PG_LEVEL_4K)
				++vcpu->kvm->stat.mmu_pde_zapped;
5263
			if (need_remote_flush(entry, *spte))
5264
				flush = true;
5265
			++spte;
5266 5267
		}
	}
5268
	kvm_mmu_remote_flush_or_zap(vcpu->kvm, &invalid_list, flush);
5269
	kvm_mmu_audit(vcpu, AUDIT_POST_PTE_WRITE);
5270
	write_unlock(&vcpu->kvm->mmu_lock);
5271 5272
}

5273
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
5274
		       void *insn, int insn_len)
5275
{
5276
	int r, emulation_type = EMULTYPE_PF;
5277
	bool direct = vcpu->arch.mmu->direct_map;
5278

5279
	if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa)))
5280 5281
		return RET_PF_RETRY;

5282
	r = RET_PF_INVALID;
5283
	if (unlikely(error_code & PFERR_RSVD_MASK)) {
5284
		r = handle_mmio_page_fault(vcpu, cr2_or_gpa, direct);
5285
		if (r == RET_PF_EMULATE)
5286 5287
			goto emulate;
	}
5288

5289
	if (r == RET_PF_INVALID) {
5290 5291
		r = kvm_mmu_do_page_fault(vcpu, cr2_or_gpa,
					  lower_32_bits(error_code), false);
5292
		if (KVM_BUG_ON(r == RET_PF_INVALID, vcpu->kvm))
5293
			return -EIO;
5294 5295
	}

5296
	if (r < 0)
5297
		return r;
5298 5299
	if (r != RET_PF_EMULATE)
		return 1;
5300

5301 5302 5303 5304 5305 5306 5307
	/*
	 * Before emulating the instruction, check if the error code
	 * was due to a RO violation while translating the guest page.
	 * This can occur when using nested virtualization with nested
	 * paging in both guests. If true, we simply unprotect the page
	 * and resume the guest.
	 */
5308
	if (vcpu->arch.mmu->direct_map &&
5309
	    (error_code & PFERR_NESTED_GUEST_PAGE) == PFERR_NESTED_GUEST_PAGE) {
5310
		kvm_mmu_unprotect_page(vcpu->kvm, gpa_to_gfn(cr2_or_gpa));
5311 5312 5313
		return 1;
	}

5314 5315 5316 5317 5318 5319
	/*
	 * vcpu->arch.mmu.page_fault returned RET_PF_EMULATE, but we can still
	 * optimistically try to just unprotect the page and let the processor
	 * re-execute the instruction that caused the page fault.  Do not allow
	 * retrying MMIO emulation, as it's not only pointless but could also
	 * cause us to enter an infinite loop because the processor will keep
5320 5321 5322 5323
	 * faulting on the non-existent MMIO address.  Retrying an instruction
	 * from a nested guest is also pointless and dangerous as we are only
	 * explicitly shadowing L1's page tables, i.e. unprotecting something
	 * for L1 isn't going to magically fix whatever issue cause L2 to fail.
5324
	 */
5325
	if (!mmio_info_in_cache(vcpu, cr2_or_gpa, direct) && !is_guest_mode(vcpu))
5326
		emulation_type |= EMULTYPE_ALLOW_RETRY_PF;
5327
emulate:
5328
	return x86_emulate_instruction(vcpu, cr2_or_gpa, emulation_type, insn,
5329
				       insn_len);
5330 5331 5332
}
EXPORT_SYMBOL_GPL(kvm_mmu_page_fault);

5333 5334
void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gva_t gva, hpa_t root_hpa)
Marcelo Tosatti's avatar
Marcelo Tosatti committed
5335
{
5336
	int i;
5337

5338 5339 5340 5341 5342 5343
	/* It's actually a GPA for vcpu->arch.guest_mmu.  */
	if (mmu != &vcpu->arch.guest_mmu) {
		/* INVLPG on a non-canonical address is a NOP according to the SDM.  */
		if (is_noncanonical_address(gva, vcpu))
			return;

5344
		static_call(kvm_x86_flush_tlb_gva)(vcpu, gva);
5345 5346 5347
	}

	if (!mmu->invlpg)
5348 5349
		return;

5350 5351
	if (root_hpa == INVALID_PAGE) {
		mmu->invlpg(vcpu, gva, mmu->root_hpa);
5352

5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370
		/*
		 * INVLPG is required to invalidate any global mappings for the VA,
		 * irrespective of PCID. Since it would take us roughly similar amount
		 * of work to determine whether any of the prev_root mappings of the VA
		 * is marked global, or to just sync it blindly, so we might as well
		 * just always sync it.
		 *
		 * Mappings not reachable via the current cr3 or the prev_roots will be
		 * synced when switching to that cr3, so nothing needs to be done here
		 * for them.
		 */
		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
			if (VALID_PAGE(mmu->prev_roots[i].hpa))
				mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
	} else {
		mmu->invlpg(vcpu, gva, root_hpa);
	}
}
5371

5372 5373
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva)
{
5374
	kvm_mmu_invalidate_gva(vcpu, vcpu->arch.walk_mmu, gva, INVALID_PAGE);
Marcelo Tosatti's avatar
Marcelo Tosatti committed
5375 5376 5377 5378
	++vcpu->stat.invlpg;
}
EXPORT_SYMBOL_GPL(kvm_mmu_invlpg);

5379

5380 5381
void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid)
{
5382
	struct kvm_mmu *mmu = vcpu->arch.mmu;
5383
	bool tlb_flush = false;
5384
	uint i;
5385 5386

	if (pcid == kvm_get_active_pcid(vcpu)) {
5387
		mmu->invlpg(vcpu, gva, mmu->root_hpa);
5388
		tlb_flush = true;
5389 5390
	}

5391 5392
	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
		if (VALID_PAGE(mmu->prev_roots[i].hpa) &&
5393
		    pcid == kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd)) {
5394 5395 5396
			mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
			tlb_flush = true;
		}
5397
	}
5398

5399
	if (tlb_flush)
5400
		static_call(kvm_x86_flush_tlb_gva)(vcpu, gva);
5401

5402 5403 5404
	++vcpu->stat.invlpg;

	/*
5405 5406 5407
	 * Mappings not reachable via the current cr3 or the prev_roots will be
	 * synced when switching to that cr3, so nothing needs to be done here
	 * for them.
5408 5409 5410
	 */
}

5411 5412
void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
		       int tdp_max_root_level, int tdp_huge_page_level)
5413
{
5414
	tdp_enabled = enable_tdp;
5415
	tdp_root_level = tdp_forced_root_level;
5416
	max_tdp_level = tdp_max_root_level;
5417 5418

	/*
5419
	 * max_huge_page_level reflects KVM's MMU capabilities irrespective
5420 5421 5422 5423 5424 5425
	 * of kernel support, e.g. KVM may be capable of using 1GB pages when
	 * the kernel is not.  But, KVM never creates a page size greater than
	 * what is used by the kernel for any given HVA, i.e. the kernel's
	 * capabilities are ultimately consulted by kvm_mmu_hugepage_adjust().
	 */
	if (tdp_enabled)
5426
		max_huge_page_level = tdp_huge_page_level;
5427
	else if (boot_cpu_has(X86_FEATURE_GBPAGES))
5428
		max_huge_page_level = PG_LEVEL_1G;
5429
	else
5430
		max_huge_page_level = PG_LEVEL_2M;
5431
}
5432
EXPORT_SYMBOL_GPL(kvm_configure_mmu);
5433 5434

/* The return value indicates if tlb flush on all vcpus is needed. */
5435 5436 5437
typedef bool (*slot_level_handler) (struct kvm *kvm,
				    struct kvm_rmap_head *rmap_head,
				    const struct kvm_memory_slot *slot);
5438 5439 5440

/* The caller should hold mmu-lock before calling this function. */
static __always_inline bool
5441
slot_handle_level_range(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5442
			slot_level_handler fn, int start_level, int end_level,
5443 5444
			gfn_t start_gfn, gfn_t end_gfn, bool flush_on_yield,
			bool flush)
5445 5446 5447 5448 5449 5450
{
	struct slot_rmap_walk_iterator iterator;

	for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
			end_gfn, &iterator) {
		if (iterator.rmap)
5451
			flush |= fn(kvm, iterator.rmap, memslot);
5452

5453
		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
5454
			if (flush && flush_on_yield) {
5455 5456 5457
				kvm_flush_remote_tlbs_with_address(kvm,
						start_gfn,
						iterator.gfn - start_gfn + 1);
5458 5459
				flush = false;
			}
5460
			cond_resched_rwlock_write(&kvm->mmu_lock);
5461 5462 5463 5464 5465 5466 5467
		}
	}

	return flush;
}

static __always_inline bool
5468
slot_handle_level(struct kvm *kvm, const struct kvm_memory_slot *memslot,
5469
		  slot_level_handler fn, int start_level, int end_level,
5470
		  bool flush_on_yield)
5471 5472 5473 5474
{
	return slot_handle_level_range(kvm, memslot, fn, start_level,
			end_level, memslot->base_gfn,
			memslot->base_gfn + memslot->npages - 1,
5475
			flush_on_yield, false);
5476 5477 5478
}

static __always_inline bool
5479 5480
slot_handle_level_4k(struct kvm *kvm, const struct kvm_memory_slot *memslot,
		     slot_level_handler fn, bool flush_on_yield)
5481
{
5482
	return slot_handle_level(kvm, memslot, fn, PG_LEVEL_4K,
5483
				 PG_LEVEL_4K, flush_on_yield);
5484 5485
}

5486
static void free_mmu_pages(struct kvm_mmu *mmu)
5487
{
5488 5489
	if (!tdp_enabled && mmu->pae_root)
		set_memory_encrypted((unsigned long)mmu->pae_root, 1);
5490
	free_page((unsigned long)mmu->pae_root);
5491
	free_page((unsigned long)mmu->pml4_root);
5492
	free_page((unsigned long)mmu->pml5_root);
5493 5494
}

5495
static int __kvm_mmu_create(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
5496
{
5497
	struct page *page;
5498 5499
	int i;

5500 5501 5502 5503 5504
	mmu->root_hpa = INVALID_PAGE;
	mmu->root_pgd = 0;
	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
		mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID;

5505 5506 5507 5508
	/* vcpu->arch.guest_mmu isn't used when !tdp_enabled. */
	if (!tdp_enabled && mmu == &vcpu->arch.guest_mmu)
		return 0;

5509
	/*
5510 5511 5512 5513
	 * When using PAE paging, the four PDPTEs are treated as 'root' pages,
	 * while the PDP table is a per-vCPU construct that's allocated at MMU
	 * creation.  When emulating 32-bit mode, cr3 is only 32 bits even on
	 * x86_64.  Therefore we need to allocate the PDP table in the first
5514 5515 5516 5517
	 * 4GB of memory, which happens to fit the DMA32 zone.  TDP paging
	 * generally doesn't use PAE paging and can skip allocating the PDP
	 * table.  The main exception, handled here, is SVM's 32-bit NPT.  The
	 * other exception is for shadowing L1's 32-bit or PAE NPT on 64-bit
5518
	 * KVM; that horror is handled on-demand by mmu_alloc_special_roots().
5519
	 */
5520
	if (tdp_enabled && kvm_mmu_get_tdp_level(vcpu) > PT32E_ROOT_LEVEL)
5521 5522
		return 0;

5523
	page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_DMA32);
5524
	if (!page)
5525 5526
		return -ENOMEM;

5527
	mmu->pae_root = page_address(page);
5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541

	/*
	 * CR3 is only 32 bits when PAE paging is used, thus it's impossible to
	 * get the CPU to treat the PDPTEs as encrypted.  Decrypt the page so
	 * that KVM's writes and the CPU's reads get along.  Note, this is
	 * only necessary when using shadow paging, as 64-bit NPT can get at
	 * the C-bit even when shadowing 32-bit NPT, and SME isn't supported
	 * by 32-bit kernels (when KVM itself uses 32-bit NPT).
	 */
	if (!tdp_enabled)
		set_memory_decrypted((unsigned long)mmu->pae_root, 1);
	else
		WARN_ON_ONCE(shadow_me_mask);

5542
	for (i = 0; i < 4; ++i)
5543
		mmu->pae_root[i] = INVALID_PAE_ROOT;
5544

5545 5546 5547
	return 0;
}

5548
int kvm_mmu_create(struct kvm_vcpu *vcpu)
5549
{
5550
	int ret;
5551

5552
	vcpu->arch.mmu_pte_list_desc_cache.kmem_cache = pte_list_desc_cache;
5553 5554
	vcpu->arch.mmu_pte_list_desc_cache.gfp_zero = __GFP_ZERO;

5555
	vcpu->arch.mmu_page_header_cache.kmem_cache = mmu_page_header_cache;
5556
	vcpu->arch.mmu_page_header_cache.gfp_zero = __GFP_ZERO;
5557

5558 5559
	vcpu->arch.mmu_shadow_page_cache.gfp_zero = __GFP_ZERO;

5560 5561
	vcpu->arch.mmu = &vcpu->arch.root_mmu;
	vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
5562

5563
	ret = __kvm_mmu_create(vcpu, &vcpu->arch.guest_mmu);
5564 5565 5566
	if (ret)
		return ret;

5567
	ret = __kvm_mmu_create(vcpu, &vcpu->arch.root_mmu);
5568 5569 5570 5571 5572 5573 5574
	if (ret)
		goto fail_allocate_root;

	return ret;
 fail_allocate_root:
	free_mmu_pages(&vcpu->arch.guest_mmu);
	return ret;
5575 5576
}

5577
#define BATCH_ZAP_PAGES	10
5578 5579 5580
static void kvm_zap_obsolete_pages(struct kvm *kvm)
{
	struct kvm_mmu_page *sp, *node;
5581
	int nr_zapped, batch = 0;
5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593

restart:
	list_for_each_entry_safe_reverse(sp, node,
	      &kvm->arch.active_mmu_pages, link) {
		/*
		 * No obsolete valid page exists before a newly created page
		 * since active_mmu_pages is a FIFO list.
		 */
		if (!is_obsolete_sp(kvm, sp))
			break;

		/*
5594 5595 5596
		 * Invalid pages should never land back on the list of active
		 * pages.  Skip the bogus page, otherwise we'll get stuck in an
		 * infinite loop if the page gets put back on the list (again).
5597
		 */
5598
		if (WARN_ON(sp->role.invalid))
5599 5600
			continue;

5601 5602 5603 5604 5605 5606
		/*
		 * No need to flush the TLB since we're only zapping shadow
		 * pages with an obsolete generation number and all vCPUS have
		 * loaded a new root, i.e. the shadow pages being zapped cannot
		 * be in active use by the guest.
		 */
5607
		if (batch >= BATCH_ZAP_PAGES &&
5608
		    cond_resched_rwlock_write(&kvm->mmu_lock)) {
5609
			batch = 0;
5610 5611 5612
			goto restart;
		}

5613 5614
		if (__kvm_mmu_prepare_zap_page(kvm, sp,
				&kvm->arch.zapped_obsolete_pages, &nr_zapped)) {
5615
			batch += nr_zapped;
5616
			goto restart;
5617
		}
5618 5619
	}

5620 5621 5622 5623 5624
	/*
	 * Trigger a remote TLB flush before freeing the page tables to ensure
	 * KVM is not in the middle of a lockless shadow page table walk, which
	 * may reference the pages.
	 */
5625
	kvm_mmu_commit_zap_page(kvm, &kvm->arch.zapped_obsolete_pages);
5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638
}

/*
 * Fast invalidate all shadow pages and use lock-break technique
 * to zap obsolete pages.
 *
 * It's required when memslot is being deleted or VM is being
 * destroyed, in these cases, we should ensure that KVM MMU does
 * not use any resource of the being-deleted slot or all slots
 * after calling the function.
 */
static void kvm_mmu_zap_all_fast(struct kvm *kvm)
{
5639 5640
	lockdep_assert_held(&kvm->slots_lock);

5641
	write_lock(&kvm->mmu_lock);
5642
	trace_kvm_mmu_zap_all_fast(kvm);
5643 5644 5645 5646 5647 5648 5649 5650 5651

	/*
	 * Toggle mmu_valid_gen between '0' and '1'.  Because slots_lock is
	 * held for the entire duration of zapping obsolete pages, it's
	 * impossible for there to be multiple invalid generations associated
	 * with *valid* shadow pages at any given time, i.e. there is exactly
	 * one valid generation and (at most) one invalid generation.
	 */
	kvm->arch.mmu_valid_gen = kvm->arch.mmu_valid_gen ? 0 : 1;
5652

5653 5654 5655 5656 5657 5658 5659 5660 5661
	/* In order to ensure all threads see this change when
	 * handling the MMU reload signal, this must happen in the
	 * same critical section as kvm_reload_remote_mmus, and
	 * before kvm_zap_obsolete_pages as kvm_zap_obsolete_pages
	 * could drop the MMU lock and yield.
	 */
	if (is_tdp_mmu_enabled(kvm))
		kvm_tdp_mmu_invalidate_all_roots(kvm);

5662 5663 5664 5665 5666 5667 5668 5669 5670 5671
	/*
	 * Notify all vcpus to reload its shadow page table and flush TLB.
	 * Then all vcpus will switch to new shadow page table with the new
	 * mmu_valid_gen.
	 *
	 * Note: we need to do this under the protection of mmu_lock,
	 * otherwise, vcpu would purge shadow page but miss tlb flush.
	 */
	kvm_reload_remote_mmus(kvm);

5672
	kvm_zap_obsolete_pages(kvm);
5673

5674
	write_unlock(&kvm->mmu_lock);
5675 5676 5677 5678 5679 5680

	if (is_tdp_mmu_enabled(kvm)) {
		read_lock(&kvm->mmu_lock);
		kvm_tdp_mmu_zap_invalidated_roots(kvm);
		read_unlock(&kvm->mmu_lock);
	}
5681 5682
}

5683 5684 5685 5686 5687
static bool kvm_has_zapped_obsolete_pages(struct kvm *kvm)
{
	return unlikely(!list_empty_careful(&kvm->arch.zapped_obsolete_pages));
}

5688
static void kvm_mmu_invalidate_zap_pages_in_memslot(struct kvm *kvm,
5689 5690
			struct kvm_memory_slot *slot,
			struct kvm_page_track_notifier_node *node)
5691
{
5692
	kvm_mmu_zap_all_fast(kvm);
5693 5694
}

5695
void kvm_mmu_init_vm(struct kvm *kvm)
5696
{
5697
	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5698

5699 5700
	spin_lock_init(&kvm->arch.mmu_unsync_pages_lock);

5701
	kvm_mmu_init_tdp_mmu(kvm);
5702

5703
	node->track_write = kvm_mmu_pte_write;
5704
	node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;
5705
	kvm_page_track_register_notifier(kvm, node);
5706 5707
}

5708
void kvm_mmu_uninit_vm(struct kvm *kvm)
5709
{
5710
	struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
5711

5712
	kvm_page_track_unregister_notifier(kvm, node);
5713 5714

	kvm_mmu_uninit_tdp_mmu(kvm);
5715 5716
}

5717 5718 5719 5720
static bool __kvm_zap_rmaps(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
{
	const struct kvm_memory_slot *memslot;
	struct kvm_memslots *slots;
5721
	struct kvm_memslot_iter iter;
5722 5723
	bool flush = false;
	gfn_t start, end;
5724
	int i;
5725 5726 5727 5728 5729 5730

	if (!kvm_memslots_have_rmaps(kvm))
		return flush;

	for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
		slots = __kvm_memslots(kvm, i);
5731 5732 5733

		kvm_for_each_memslot_in_gfn_range(&iter, slots, gfn_start, gfn_end) {
			memslot = iter.slot;
5734 5735
			start = max(gfn_start, memslot->base_gfn);
			end = min(gfn_end, memslot->base_gfn + memslot->npages);
5736
			if (WARN_ON_ONCE(start >= end))
5737 5738 5739
				continue;

			flush = slot_handle_level_range(kvm, memslot, kvm_zap_rmapp,
5740

5741 5742 5743 5744 5745 5746 5747 5748
							PG_LEVEL_4K, KVM_MAX_HUGEPAGE_LEVEL,
							start, end - 1, true, flush);
		}
	}

	return flush;
}

5749 5750 5751 5752
/*
 * Invalidate (zap) SPTEs that cover GFNs from gfn_start and up to gfn_end
 * (not including it)
 */
Xiao Guangrong's avatar
Xiao Guangrong committed
5753 5754
void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
{
5755
	bool flush;
5756
	int i;
Xiao Guangrong's avatar
Xiao Guangrong committed
5757

5758 5759 5760
	if (WARN_ON_ONCE(gfn_end <= gfn_start))
		return;

5761 5762
	write_lock(&kvm->mmu_lock);

5763 5764
	kvm_inc_notifier_count(kvm, gfn_start, gfn_end);

5765
	flush = __kvm_zap_rmaps(kvm, gfn_start, gfn_end);
Xiao Guangrong's avatar
Xiao Guangrong committed
5766

5767
	if (is_tdp_mmu_enabled(kvm)) {
5768 5769
		for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
			flush = kvm_tdp_mmu_zap_gfn_range(kvm, i, gfn_start,
5770
							  gfn_end, flush);
5771
	}
5772 5773

	if (flush)
5774 5775
		kvm_flush_remote_tlbs_with_address(kvm, gfn_start,
						   gfn_end - gfn_start);
5776

5777 5778
	kvm_dec_notifier_count(kvm, gfn_start, gfn_end);

5779
	write_unlock(&kvm->mmu_lock);
Xiao Guangrong's avatar
Xiao Guangrong committed
5780 5781
}

5782
static bool slot_rmap_write_protect(struct kvm *kvm,
5783
				    struct kvm_rmap_head *rmap_head,
5784
				    const struct kvm_memory_slot *slot)
5785
{
5786
	return rmap_write_protect(rmap_head, false);
5787 5788
}

5789
void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
5790
				      const struct kvm_memory_slot *memslot,
5791
				      int start_level)
5792
{
5793
	bool flush = false;
5794

5795 5796 5797 5798 5799 5800 5801
	if (kvm_memslots_have_rmaps(kvm)) {
		write_lock(&kvm->mmu_lock);
		flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect,
					  start_level, KVM_MAX_HUGEPAGE_LEVEL,
					  false);
		write_unlock(&kvm->mmu_lock);
	}
5802

5803 5804 5805 5806 5807 5808
	if (is_tdp_mmu_enabled(kvm)) {
		read_lock(&kvm->mmu_lock);
		flush |= kvm_tdp_mmu_wrprot_slot(kvm, memslot, start_level);
		read_unlock(&kvm->mmu_lock);
	}

5809
	/*
5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829
	 * Flush TLBs if any SPTEs had to be write-protected to ensure that
	 * guest writes are reflected in the dirty bitmap before the memslot
	 * update completes, i.e. before enabling dirty logging is visible to
	 * userspace.
	 *
	 * Perform the TLB flush outside the mmu_lock to reduce the amount of
	 * time the lock is held. However, this does mean that another CPU can
	 * now grab mmu_lock and encounter a write-protected SPTE while CPUs
	 * still have a writable mapping for the associated GFN in their TLB.
	 *
	 * This is safe but requires KVM to be careful when making decisions
	 * based on the write-protection status of an SPTE. Specifically, KVM
	 * also write-protects SPTEs to monitor changes to guest page tables
	 * during shadow paging, and must guarantee no CPUs can write to those
	 * page before the lock is dropped. As mentioned in the previous
	 * paragraph, a write-protected SPTE is no guarantee that CPU cannot
	 * perform writes. So to determine if a TLB flush is truly required, KVM
	 * will clear a separate software-only bit (MMU-writable) and skip the
	 * flush if-and-only-if this bit was already clear.
	 *
5830
	 * See is_writable_pte() for more details.
5831
	 */
5832
	if (flush)
5833
		kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
5834
}
5835

5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851
/* Must be called with the mmu_lock held in write-mode. */
void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
				   const struct kvm_memory_slot *memslot,
				   u64 start, u64 end,
				   int target_level)
{
	if (is_tdp_mmu_enabled(kvm))
		kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end,
						 target_level, false);

	/*
	 * A TLB flush is unnecessary at this point for the same resons as in
	 * kvm_mmu_slot_try_split_huge_pages().
	 */
}

5852
void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
5853 5854
					const struct kvm_memory_slot *memslot,
					int target_level)
5855 5856 5857 5858 5859 5860
{
	u64 start = memslot->base_gfn;
	u64 end = start + memslot->npages;

	if (is_tdp_mmu_enabled(kvm)) {
		read_lock(&kvm->mmu_lock);
5861
		kvm_tdp_mmu_try_split_huge_pages(kvm, memslot, start, end, target_level, true);
5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875
		read_unlock(&kvm->mmu_lock);
	}

	/*
	 * No TLB flush is necessary here. KVM will flush TLBs after
	 * write-protecting and/or clearing dirty on the newly split SPTEs to
	 * ensure that guest writes are reflected in the dirty log before the
	 * ioctl to enable dirty logging on this memslot completes. Since the
	 * split SPTEs retain the write and dirty bits of the huge SPTE, it is
	 * safe for KVM to decide if a TLB flush is necessary based on the split
	 * SPTEs.
	 */
}

5876
static bool kvm_mmu_zap_collapsible_spte(struct kvm *kvm,
5877
					 struct kvm_rmap_head *rmap_head,
5878
					 const struct kvm_memory_slot *slot)
5879 5880 5881 5882
{
	u64 *sptep;
	struct rmap_iterator iter;
	int need_tlb_flush = 0;
5883
	kvm_pfn_t pfn;
5884 5885
	struct kvm_mmu_page *sp;

5886
restart:
5887
	for_each_rmap_spte(rmap_head, &iter, sptep) {
5888
		sp = sptep_to_sp(sptep);
5889 5890 5891
		pfn = spte_to_pfn(*sptep);

		/*
5892 5893 5894 5895 5896
		 * We cannot do huge page mapping for indirect shadow pages,
		 * which are found on the last rmap (level = 1) when not using
		 * tdp; such shadow pages are synced with the page table in
		 * the guest, and the guest page table is using 4K page size
		 * mapping if the indirect sp has level = 1.
5897
		 */
5898
		if (sp->role.direct && !kvm_is_reserved_pfn(pfn) &&
5899 5900
		    sp->role.level < kvm_mmu_max_mapping_level(kvm, slot, sp->gfn,
							       pfn, PG_LEVEL_NUM)) {
5901
			pte_list_remove(kvm, rmap_head, sptep);
5902 5903 5904 5905 5906 5907 5908

			if (kvm_available_flush_tlb_with_range())
				kvm_flush_remote_tlbs_with_address(kvm, sp->gfn,
					KVM_PAGES_PER_HPAGE(sp->role.level));
			else
				need_tlb_flush = 1;

5909 5910
			goto restart;
		}
5911 5912 5913 5914 5915 5916
	}

	return need_tlb_flush;
}

void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm,
5917
				   const struct kvm_memory_slot *slot)
5918
{
5919 5920
	if (kvm_memslots_have_rmaps(kvm)) {
		write_lock(&kvm->mmu_lock);
5921 5922 5923 5924 5925
		/*
		 * Zap only 4k SPTEs since the legacy MMU only supports dirty
		 * logging at a 4k granularity and never creates collapsible
		 * 2m SPTEs during dirty logging.
		 */
5926
		if (slot_handle_level_4k(kvm, slot, kvm_mmu_zap_collapsible_spte, true))
5927 5928 5929
			kvm_arch_flush_remote_tlbs_memslot(kvm, slot);
		write_unlock(&kvm->mmu_lock);
	}
5930 5931 5932

	if (is_tdp_mmu_enabled(kvm)) {
		read_lock(&kvm->mmu_lock);
5933
		kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot);
5934 5935
		read_unlock(&kvm->mmu_lock);
	}
5936 5937
}

5938
void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm,
5939
					const struct kvm_memory_slot *memslot)
5940 5941
{
	/*
5942
	 * All current use cases for flushing the TLBs for a specific memslot
5943
	 * related to dirty logging, and many do the TLB flush out of mmu_lock.
5944 5945 5946
	 * The interaction between the various operations on memslot must be
	 * serialized by slots_locks to ensure the TLB flush from one operation
	 * is observed by any other operation on the same memslot.
5947 5948
	 */
	lockdep_assert_held(&kvm->slots_lock);
5949 5950
	kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
					   memslot->npages);
5951 5952
}

5953
void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
5954
				   const struct kvm_memory_slot *memslot)
5955
{
5956
	bool flush = false;
5957

5958 5959
	if (kvm_memslots_have_rmaps(kvm)) {
		write_lock(&kvm->mmu_lock);
5960 5961 5962 5963 5964
		/*
		 * Clear dirty bits only on 4k SPTEs since the legacy MMU only
		 * support dirty logging at a 4k granularity.
		 */
		flush = slot_handle_level_4k(kvm, memslot, __rmap_clear_dirty, false);
5965 5966
		write_unlock(&kvm->mmu_lock);
	}
5967

5968 5969 5970 5971 5972 5973
	if (is_tdp_mmu_enabled(kvm)) {
		read_lock(&kvm->mmu_lock);
		flush |= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);
		read_unlock(&kvm->mmu_lock);
	}

5974 5975 5976 5977 5978 5979 5980
	/*
	 * It's also safe to flush TLBs out of mmu lock here as currently this
	 * function is only used for dirty logging, in which case flushing TLB
	 * out of mmu lock also guarantees no dirty pages will be lost in
	 * dirty_bitmap.
	 */
	if (flush)
5981
		kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
5982 5983
}

5984
void kvm_mmu_zap_all(struct kvm *kvm)
5985 5986
{
	struct kvm_mmu_page *sp, *node;
5987
	LIST_HEAD(invalid_list);
5988
	int ign;
5989

5990
	write_lock(&kvm->mmu_lock);
5991
restart:
5992
	list_for_each_entry_safe(sp, node, &kvm->arch.active_mmu_pages, link) {
5993
		if (WARN_ON(sp->role.invalid))
5994
			continue;
5995
		if (__kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list, &ign))
5996
			goto restart;
5997
		if (cond_resched_rwlock_write(&kvm->mmu_lock))
5998 5999 6000
			goto restart;
	}

6001
	kvm_mmu_commit_zap_page(kvm, &invalid_list);
6002

6003
	if (is_tdp_mmu_enabled(kvm))
6004 6005
		kvm_tdp_mmu_zap_all(kvm);

6006
	write_unlock(&kvm->mmu_lock);
6007 6008
}

6009
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen)
6010
{
6011
	WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
6012

6013
	gen &= MMIO_SPTE_GEN_MASK;
6014

6015
	/*
6016 6017 6018 6019 6020 6021 6022 6023
	 * Generation numbers are incremented in multiples of the number of
	 * address spaces in order to provide unique generations across all
	 * address spaces.  Strip what is effectively the address space
	 * modifier prior to checking for a wrap of the MMIO generation so
	 * that a wrap in any address space is detected.
	 */
	gen &= ~((u64)KVM_ADDRESS_SPACE_NUM - 1);

6024
	/*
6025
	 * The very rare case: if the MMIO generation number has wrapped,
6026 6027
	 * zap all shadow pages.
	 */
6028
	if (unlikely(gen == 0)) {
6029
		kvm_debug_ratelimited("kvm: zapping shadow pages for mmio generation wraparound\n");
6030
		kvm_mmu_zap_all_fast(kvm);
6031
	}
6032 6033
}

6034 6035
static unsigned long
mmu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
6036 6037
{
	struct kvm *kvm;
6038
	int nr_to_scan = sc->nr_to_scan;
6039
	unsigned long freed = 0;
6040

6041
	mutex_lock(&kvm_lock);
6042 6043

	list_for_each_entry(kvm, &vm_list, vm_list) {
6044
		int idx;
6045
		LIST_HEAD(invalid_list);
6046

6047 6048 6049 6050 6051 6052 6053 6054
		/*
		 * Never scan more than sc->nr_to_scan VM instances.
		 * Will not hit this condition practically since we do not try
		 * to shrink more than one VM and it is very unlikely to see
		 * !n_used_mmu_pages so many times.
		 */
		if (!nr_to_scan--)
			break;
6055 6056 6057 6058 6059 6060
		/*
		 * n_used_mmu_pages is accessed without holding kvm->mmu_lock
		 * here. We may skip a VM instance errorneosly, but we do not
		 * want to shrink a VM that only started to populate its MMU
		 * anyway.
		 */
6061 6062
		if (!kvm->arch.n_used_mmu_pages &&
		    !kvm_has_zapped_obsolete_pages(kvm))
6063 6064
			continue;

6065
		idx = srcu_read_lock(&kvm->srcu);
6066
		write_lock(&kvm->mmu_lock);
6067

6068 6069 6070 6071 6072 6073
		if (kvm_has_zapped_obsolete_pages(kvm)) {
			kvm_mmu_commit_zap_page(kvm,
			      &kvm->arch.zapped_obsolete_pages);
			goto unlock;
		}

6074
		freed = kvm_mmu_zap_oldest_mmu_pages(kvm, sc->nr_to_scan);
6075

6076
unlock:
6077
		write_unlock(&kvm->mmu_lock);
6078
		srcu_read_unlock(&kvm->srcu, idx);
6079

6080 6081 6082 6083 6084
		/*
		 * unfair on small ones
		 * per-vm shrinkers cry out
		 * sadness comes quickly
		 */
6085 6086
		list_move_tail(&kvm->vm_list, &vm_list);
		break;
6087 6088
	}

6089
	mutex_unlock(&kvm_lock);
6090 6091 6092 6093 6094 6095
	return freed;
}

static unsigned long
mmu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
{
6096
	return percpu_counter_read_positive(&kvm_total_used_mmu_pages);
6097 6098 6099
}

static struct shrinker mmu_shrinker = {
6100 6101
	.count_objects = mmu_shrink_count,
	.scan_objects = mmu_shrink_scan,
6102 6103 6104
	.seeks = DEFAULT_SEEKS * 10,
};

6105
static void mmu_destroy_caches(void)
6106
{
6107 6108
	kmem_cache_destroy(pte_list_desc_cache);
	kmem_cache_destroy(mmu_page_header_cache);
6109 6110
}

6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144
static bool get_nx_auto_mode(void)
{
	/* Return true when CPU has the bug, and mitigations are ON */
	return boot_cpu_has_bug(X86_BUG_ITLB_MULTIHIT) && !cpu_mitigations_off();
}

static void __set_nx_huge_pages(bool val)
{
	nx_huge_pages = itlb_multihit_kvm_mitigation = val;
}

static int set_nx_huge_pages(const char *val, const struct kernel_param *kp)
{
	bool old_val = nx_huge_pages;
	bool new_val;

	/* In "auto" mode deploy workaround only if CPU has the bug. */
	if (sysfs_streq(val, "off"))
		new_val = 0;
	else if (sysfs_streq(val, "force"))
		new_val = 1;
	else if (sysfs_streq(val, "auto"))
		new_val = get_nx_auto_mode();
	else if (strtobool(val, &new_val) < 0)
		return -EINVAL;

	__set_nx_huge_pages(new_val);

	if (new_val != old_val) {
		struct kvm *kvm;

		mutex_lock(&kvm_lock);

		list_for_each_entry(kvm, &vm_list, vm_list) {
6145
			mutex_lock(&kvm->slots_lock);
6146
			kvm_mmu_zap_all_fast(kvm);
6147
			mutex_unlock(&kvm->slots_lock);
6148 6149

			wake_up_process(kvm->arch.nx_lpage_recovery_thread);
6150 6151 6152 6153 6154 6155 6156
		}
		mutex_unlock(&kvm_lock);
	}

	return 0;
}

6157 6158
int kvm_mmu_module_init(void)
{
6159 6160
	int ret = -ENOMEM;

6161 6162 6163
	if (nx_huge_pages == -1)
		__set_nx_huge_pages(get_nx_auto_mode());

6164 6165 6166 6167 6168 6169 6170 6171 6172 6173
	/*
	 * MMU roles use union aliasing which is, generally speaking, an
	 * undefined behavior. However, we supposedly know how compilers behave
	 * and the current status quo is unlikely to change. Guardians below are
	 * supposed to let us know if the assumption becomes false.
	 */
	BUILD_BUG_ON(sizeof(union kvm_mmu_page_role) != sizeof(u32));
	BUILD_BUG_ON(sizeof(union kvm_mmu_extended_role) != sizeof(u32));
	BUILD_BUG_ON(sizeof(union kvm_mmu_role) != sizeof(u64));

6174
	kvm_mmu_reset_all_pte_masks();
6175

6176 6177
	pte_list_desc_cache = kmem_cache_create("pte_list_desc",
					    sizeof(struct pte_list_desc),
6178
					    0, SLAB_ACCOUNT, NULL);
6179
	if (!pte_list_desc_cache)
6180
		goto out;
6181

6182 6183
	mmu_page_header_cache = kmem_cache_create("kvm_mmu_page_header",
						  sizeof(struct kvm_mmu_page),
6184
						  0, SLAB_ACCOUNT, NULL);
6185
	if (!mmu_page_header_cache)
6186
		goto out;
6187

6188
	if (percpu_counter_init(&kvm_total_used_mmu_pages, 0, GFP_KERNEL))
6189
		goto out;
6190

6191 6192 6193
	ret = register_shrinker(&mmu_shrinker);
	if (ret)
		goto out;
6194

6195 6196
	return 0;

6197
out:
6198
	mmu_destroy_caches();
6199
	return ret;
6200 6201
}

6202 6203
void kvm_mmu_destroy(struct kvm_vcpu *vcpu)
{
6204
	kvm_mmu_unload(vcpu);
6205 6206
	free_mmu_pages(&vcpu->arch.root_mmu);
	free_mmu_pages(&vcpu->arch.guest_mmu);
6207
	mmu_free_memory_caches(vcpu);
6208 6209 6210 6211 6212 6213 6214
}

void kvm_mmu_module_exit(void)
{
	mmu_destroy_caches();
	percpu_counter_destroy(&kvm_total_used_mmu_pages);
	unregister_shrinker(&mmu_shrinker);
6215 6216
	mmu_audit_disable();
}
6217

6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242
/*
 * Calculate the effective recovery period, accounting for '0' meaning "let KVM
 * select a halving time of 1 hour".  Returns true if recovery is enabled.
 */
static bool calc_nx_huge_pages_recovery_period(uint *period)
{
	/*
	 * Use READ_ONCE to get the params, this may be called outside of the
	 * param setters, e.g. by the kthread to compute its next timeout.
	 */
	bool enabled = READ_ONCE(nx_huge_pages);
	uint ratio = READ_ONCE(nx_huge_pages_recovery_ratio);

	if (!enabled || !ratio)
		return false;

	*period = READ_ONCE(nx_huge_pages_recovery_period_ms);
	if (!*period) {
		/* Make sure the period is not less than one second.  */
		ratio = min(ratio, 3600u);
		*period = 60 * 60 * 1000 / ratio;
	}
	return true;
}

6243
static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp)
6244
{
6245 6246
	bool was_recovery_enabled, is_recovery_enabled;
	uint old_period, new_period;
6247 6248
	int err;

6249
	was_recovery_enabled = calc_nx_huge_pages_recovery_period(&old_period);
6250

6251 6252 6253 6254
	err = param_set_uint(val, kp);
	if (err)
		return err;

6255
	is_recovery_enabled = calc_nx_huge_pages_recovery_period(&new_period);
6256

6257
	if (is_recovery_enabled &&
6258
	    (!was_recovery_enabled || old_period > new_period)) {
6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273
		struct kvm *kvm;

		mutex_lock(&kvm_lock);

		list_for_each_entry(kvm, &vm_list, vm_list)
			wake_up_process(kvm->arch.nx_lpage_recovery_thread);

		mutex_unlock(&kvm_lock);
	}

	return err;
}

static void kvm_recover_nx_lpages(struct kvm *kvm)
{
6274
	unsigned long nx_lpage_splits = kvm->stat.nx_lpage_splits;
6275 6276 6277 6278
	int rcu_idx;
	struct kvm_mmu_page *sp;
	unsigned int ratio;
	LIST_HEAD(invalid_list);
6279
	bool flush = false;
6280 6281 6282
	ulong to_zap;

	rcu_idx = srcu_read_lock(&kvm->srcu);
6283
	write_lock(&kvm->mmu_lock);
6284 6285

	ratio = READ_ONCE(nx_huge_pages_recovery_ratio);
6286
	to_zap = ratio ? DIV_ROUND_UP(nx_lpage_splits, ratio) : 0;
6287 6288 6289 6290
	for ( ; to_zap; --to_zap) {
		if (list_empty(&kvm->arch.lpage_disallowed_mmu_pages))
			break;

6291 6292 6293 6294 6295 6296 6297 6298 6299
		/*
		 * We use a separate list instead of just using active_mmu_pages
		 * because the number of lpage_disallowed pages is expected to
		 * be relatively small compared to the total.
		 */
		sp = list_first_entry(&kvm->arch.lpage_disallowed_mmu_pages,
				      struct kvm_mmu_page,
				      lpage_disallowed_link);
		WARN_ON_ONCE(!sp->lpage_disallowed);
6300
		if (is_tdp_mmu_page(sp)) {
6301
			flush |= kvm_tdp_mmu_zap_sp(kvm, sp);
6302
		} else {
6303 6304 6305
			kvm_mmu_prepare_zap_page(kvm, sp, &invalid_list);
			WARN_ON_ONCE(sp->lpage_disallowed);
		}
6306

6307
		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
6308
			kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
6309
			cond_resched_rwlock_write(&kvm->mmu_lock);
6310
			flush = false;
6311 6312
		}
	}
6313
	kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush);
6314

6315
	write_unlock(&kvm->mmu_lock);
6316 6317 6318 6319 6320
	srcu_read_unlock(&kvm->srcu, rcu_idx);
}

static long get_nx_lpage_recovery_timeout(u64 start_time)
{
6321 6322
	bool enabled;
	uint period;
6323

6324
	enabled = calc_nx_huge_pages_recovery_period(&period);
6325

6326 6327
	return enabled ? start_time + msecs_to_jiffies(period) - get_jiffies_64()
		       : MAX_SCHEDULE_TIMEOUT;
6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372
}

static int kvm_nx_lpage_recovery_worker(struct kvm *kvm, uintptr_t data)
{
	u64 start_time;
	long remaining_time;

	while (true) {
		start_time = get_jiffies_64();
		remaining_time = get_nx_lpage_recovery_timeout(start_time);

		set_current_state(TASK_INTERRUPTIBLE);
		while (!kthread_should_stop() && remaining_time > 0) {
			schedule_timeout(remaining_time);
			remaining_time = get_nx_lpage_recovery_timeout(start_time);
			set_current_state(TASK_INTERRUPTIBLE);
		}

		set_current_state(TASK_RUNNING);

		if (kthread_should_stop())
			return 0;

		kvm_recover_nx_lpages(kvm);
	}
}

int kvm_mmu_post_init_vm(struct kvm *kvm)
{
	int err;

	err = kvm_vm_create_worker_thread(kvm, kvm_nx_lpage_recovery_worker, 0,
					  "kvm-nx-lpage-recovery",
					  &kvm->arch.nx_lpage_recovery_thread);
	if (!err)
		kthread_unpark(kvm->arch.nx_lpage_recovery_thread);

	return err;
}

void kvm_mmu_pre_destroy_vm(struct kvm *kvm)
{
	if (kvm->arch.nx_lpage_recovery_thread)
		kthread_stop(kvm->arch.nx_lpage_recovery_thread);
}