processor.c 14.2 KB
Newer Older
1 2 3 4 5 6 7
// SPDX-License-Identifier: GPL-2.0
/*
 * AArch64 code
 *
 * Copyright (C) 2018, Red Hat, Inc.
 */

8
#include <linux/compiler.h>
9
#include <assert.h>
10

11
#include "guest_modes.h"
12
#include "kvm_util.h"
13
#include "processor.h"
14

15
#define DEFAULT_ARM64_GUEST_STACK_VADDR_MIN	0xac0000
16

17 18
static vm_vaddr_t exception_handlers;

19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
static uint64_t page_align(struct kvm_vm *vm, uint64_t v)
{
	return (v + vm->page_size) & ~(vm->page_size - 1);
}

static uint64_t pgd_index(struct kvm_vm *vm, vm_vaddr_t gva)
{
	unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;
	uint64_t mask = (1UL << (vm->va_bits - shift)) - 1;

	return (gva >> shift) & mask;
}

static uint64_t pud_index(struct kvm_vm *vm, vm_vaddr_t gva)
{
	unsigned int shift = 2 * (vm->page_shift - 3) + vm->page_shift;
	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;

	TEST_ASSERT(vm->pgtable_levels == 4,
		"Mode %d does not have 4 page table levels", vm->mode);

	return (gva >> shift) & mask;
}

static uint64_t pmd_index(struct kvm_vm *vm, vm_vaddr_t gva)
{
	unsigned int shift = (vm->page_shift - 3) + vm->page_shift;
	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;

	TEST_ASSERT(vm->pgtable_levels >= 3,
		"Mode %d does not have >= 3 page table levels", vm->mode);

	return (gva >> shift) & mask;
}

static uint64_t pte_index(struct kvm_vm *vm, vm_vaddr_t gva)
{
	uint64_t mask = (1UL << (vm->page_shift - 3)) - 1;
	return (gva >> vm->page_shift) & mask;
}

static uint64_t pte_addr(struct kvm_vm *vm, uint64_t entry)
{
	uint64_t mask = ((1UL << (vm->va_bits - vm->page_shift)) - 1) << vm->page_shift;
	return entry & mask;
}

static uint64_t ptrs_per_pgd(struct kvm_vm *vm)
{
	unsigned int shift = (vm->pgtable_levels - 1) * (vm->page_shift - 3) + vm->page_shift;
	return 1 << (vm->va_bits - shift);
}

72
static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm)
73 74 75 76
{
	return 1 << (vm->page_shift - 3);
}

77
void virt_arch_pgd_alloc(struct kvm_vm *vm)
78 79 80 81
{
	if (!vm->pgd_created) {
		vm_paddr_t paddr = vm_phy_pages_alloc(vm,
			page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size,
82
			KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0);
83 84 85 86 87
		vm->pgd = paddr;
		vm->pgd_created = true;
	}
}

88 89
static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
			 uint64_t flags)
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
{
	uint8_t attr_idx = flags & 7;
	uint64_t *ptep;

	TEST_ASSERT((vaddr % vm->page_size) == 0,
		"Virtual address not on page boundary,\n"
		"  vaddr: 0x%lx vm->page_size: 0x%x", vaddr, vm->page_size);
	TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
		(vaddr >> vm->page_shift)),
		"Invalid virtual address, vaddr: 0x%lx", vaddr);
	TEST_ASSERT((paddr % vm->page_size) == 0,
		"Physical address not on page boundary,\n"
		"  paddr: 0x%lx vm->page_size: 0x%x", paddr, vm->page_size);
	TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
		"Physical address beyond beyond maximum supported,\n"
		"  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
		paddr, vm->max_gfn, vm->page_size);

	ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8;
109 110
	if (!*ptep)
		*ptep = vm_alloc_page_table(vm) | 3;
111 112 113 114

	switch (vm->pgtable_levels) {
	case 4:
		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8;
115 116
		if (!*ptep)
			*ptep = vm_alloc_page_table(vm) | 3;
117 118 119
		/* fall through */
	case 3:
		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8;
120 121
		if (!*ptep)
			*ptep = vm_alloc_page_table(vm) | 3;
122 123 124 125 126
		/* fall through */
	case 2:
		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8;
		break;
	default:
127
		TEST_FAIL("Page table levels must be 2, 3, or 4");
128 129 130 131 132 133
	}

	*ptep = paddr | 3;
	*ptep |= (attr_idx << 2) | (1 << 10) /* Access Flag */;
}

134
void virt_arch_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr)
135 136 137
{
	uint64_t attr_idx = 4; /* NORMAL (See DEFAULT_MAIR_EL1) */

138
	_virt_pg_map(vm, vaddr, paddr, attr_idx);
139 140
}

141
vm_paddr_t addr_arch_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
{
	uint64_t *ptep;

	if (!vm->pgd_created)
		goto unmapped_gva;

	ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, gva) * 8;
	if (!ptep)
		goto unmapped_gva;

	switch (vm->pgtable_levels) {
	case 4:
		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, gva) * 8;
		if (!ptep)
			goto unmapped_gva;
		/* fall through */
	case 3:
		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, gva) * 8;
		if (!ptep)
			goto unmapped_gva;
		/* fall through */
	case 2:
		ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, gva) * 8;
		if (!ptep)
			goto unmapped_gva;
		break;
	default:
169
		TEST_FAIL("Page table levels must be 2, 3, or 4");
170 171 172 173 174
	}

	return pte_addr(vm, *ptep) + (gva & (vm->page_size - 1));

unmapped_gva:
175
	TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva);
176
	exit(1);
177 178 179 180
}

static void pte_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent, uint64_t page, int level)
{
181
#ifdef DEBUG
182 183 184 185 186 187 188 189 190 191
	static const char * const type[] = { "", "pud", "pmd", "pte" };
	uint64_t pte, *ptep;

	if (level == 4)
		return;

	for (pte = page; pte < page + ptrs_per_pte(vm) * 8; pte += 8) {
		ptep = addr_gpa2hva(vm, pte);
		if (!*ptep)
			continue;
192
		fprintf(stream, "%*s%s: %lx: %lx at %p\n", indent, "", type[level], pte, *ptep, ptep);
193 194 195 196 197
		pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level + 1);
	}
#endif
}

198
void virt_arch_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
199 200 201 202 203 204 205 206 207 208 209
{
	int level = 4 - (vm->pgtable_levels - 1);
	uint64_t pgd, *ptep;

	if (!vm->pgd_created)
		return;

	for (pgd = vm->pgd; pgd < vm->pgd + ptrs_per_pgd(vm) * 8; pgd += 8) {
		ptep = addr_gpa2hva(vm, pgd);
		if (!*ptep)
			continue;
210
		fprintf(stream, "%*spgd: %lx: %lx at %p\n", indent, "", pgd, *ptep, ptep);
211 212 213
		pte_dump(stream, vm, indent + 1, pte_addr(vm, *ptep), level);
	}
}
214

215
void aarch64_vcpu_setup(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_vcpu_init *init)
216
{
217
	struct kvm_vcpu_init default_init = { .target = -1, };
218 219
	uint64_t sctlr_el1, tcr_el1;

220 221 222 223 224 225 226 227 228 229
	if (!init)
		init = &default_init;

	if (init->target == -1) {
		struct kvm_vcpu_init preferred;
		vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &preferred);
		init->target = preferred.target;
	}

	vcpu_ioctl(vm, vcpuid, KVM_ARM_VCPU_INIT, init);
230 231 232 233 234

	/*
	 * Enable FP/ASIMD to avoid trapping when accessing Q0-Q15
	 * registers, which the variable argument list macros do.
	 */
235
	set_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_CPACR_EL1), 3 << 20);
236

237 238
	get_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), &sctlr_el1);
	get_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_TCR_EL1), &tcr_el1);
239

240
	/* Configure base granule size */
241
	switch (vm->mode) {
242
	case VM_MODE_P52V48_4K:
243 244
		TEST_FAIL("AArch64 does not support 4K sized pages "
			  "with 52-bit physical address ranges");
245
	case VM_MODE_PXXV48_4K:
246 247
		TEST_FAIL("AArch64 does not support 4K sized pages "
			  "with ANY-bit physical address ranges");
248
	case VM_MODE_P52V48_64K:
249 250
	case VM_MODE_P48V48_64K:
	case VM_MODE_P40V48_64K:
251
	case VM_MODE_P36V48_64K:
252 253
		tcr_el1 |= 1ul << 14; /* TG0 = 64KB */
		break;
254 255 256 257 258 259
	case VM_MODE_P48V48_16K:
	case VM_MODE_P40V48_16K:
	case VM_MODE_P36V48_16K:
	case VM_MODE_P36V47_16K:
		tcr_el1 |= 2ul << 14; /* TG0 = 16KB */
		break;
260
	case VM_MODE_P48V48_4K:
261
	case VM_MODE_P40V48_4K:
262
	case VM_MODE_P36V48_4K:
263 264
		tcr_el1 |= 0ul << 14; /* TG0 = 4KB */
		break;
265 266 267 268 269 270 271 272 273 274
	default:
		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
	}

	/* Configure output size */
	switch (vm->mode) {
	case VM_MODE_P52V48_64K:
		tcr_el1 |= 6ul << 32; /* IPS = 52 bits */
		break;
	case VM_MODE_P48V48_4K:
275
	case VM_MODE_P48V48_16K:
276 277 278
	case VM_MODE_P48V48_64K:
		tcr_el1 |= 5ul << 32; /* IPS = 48 bits */
		break;
279
	case VM_MODE_P40V48_4K:
280
	case VM_MODE_P40V48_16K:
281 282 283
	case VM_MODE_P40V48_64K:
		tcr_el1 |= 2ul << 32; /* IPS = 40 bits */
		break;
284
	case VM_MODE_P36V48_4K:
285
	case VM_MODE_P36V48_16K:
286
	case VM_MODE_P36V48_64K:
287
	case VM_MODE_P36V47_16K:
288 289
		tcr_el1 |= 1ul << 32; /* IPS = 36 bits */
		break;
290
	default:
291
		TEST_FAIL("Unknown guest mode, mode: 0x%x", vm->mode);
292 293 294 295 296 297 298
	}

	sctlr_el1 |= (1 << 0) | (1 << 2) | (1 << 12) /* M | C | I */;
	/* TCR_EL1 |= IRGN0:WBWA | ORGN0:WBWA | SH0:Inner-Shareable */;
	tcr_el1 |= (1 << 8) | (1 << 10) | (3 << 12);
	tcr_el1 |= (64 - vm->va_bits) /* T0SZ */;

299 300 301 302
	set_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_SCTLR_EL1), sctlr_el1);
	set_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_TCR_EL1), tcr_el1);
	set_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_MAIR_EL1), DEFAULT_MAIR_EL1);
	set_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_TTBR0_EL1), vm->pgd);
303
	set_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_TPIDR_EL1), vcpuid);
304 305
}

306
void vcpu_arch_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)
307 308 309 310 311 312
{
	uint64_t pstate, pc;

	get_reg(vm, vcpuid, ARM64_CORE_REG(regs.pstate), &pstate);
	get_reg(vm, vcpuid, ARM64_CORE_REG(regs.pc), &pc);

313
	fprintf(stream, "%*spstate: 0x%.16lx pc: 0x%.16lx\n",
314
		indent, "", pstate, pc);
315
}
316

317 318 319
struct kvm_vcpu *aarch64_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpu_id,
					  struct kvm_vcpu_init *init,
					  void *guest_code)
320 321 322 323 324
{
	size_t stack_size = vm->page_size == 4096 ?
					DEFAULT_STACK_PGS * vm->page_size :
					vm->page_size;
	uint64_t stack_vaddr = vm_vaddr_alloc(vm, stack_size,
325
					      DEFAULT_ARM64_GUEST_STACK_VADDR_MIN);
326
	struct kvm_vcpu *vcpu = vm_vcpu_add(vm, vcpu_id);
327

328
	aarch64_vcpu_setup(vm, vcpu_id, init);
329

330 331 332 333
	set_reg(vm, vcpu_id, ARM64_CORE_REG(sp_el1), stack_vaddr + stack_size);
	set_reg(vm, vcpu_id, ARM64_CORE_REG(regs.pc), (uint64_t)guest_code);

	return vcpu;
334 335
}

336 337
struct kvm_vcpu *vm_arch_vcpu_add(struct kvm_vm *vm, uint32_t vcpu_id,
				  void *guest_code)
338
{
339
	return aarch64_vcpu_add_default(vm, vcpu_id, NULL, guest_code);
340
}
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358

void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...)
{
	va_list ap;
	int i;

	TEST_ASSERT(num >= 1 && num <= 8, "Unsupported number of args,\n"
		    "  num: %u\n", num);

	va_start(ap, num);

	for (i = 0; i < num; i++) {
		set_reg(vm, vcpuid, ARM64_CORE_REG(regs.regs[i]),
			va_arg(ap, uint64_t));
	}

	va_end(ap);
}
359

360 361 362 363 364 365 366
void kvm_exit_unexpected_exception(int vector, uint64_t ec, bool valid_ec)
{
	ucall(UCALL_UNHANDLED, 3, vector, ec, valid_ec);
	while (1)
		;
}

367 368
void assert_on_unhandled_exception(struct kvm_vm *vm, uint32_t vcpuid)
{
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392
	struct ucall uc;

	if (get_ucall(vm, vcpuid, &uc) != UCALL_UNHANDLED)
		return;

	if (uc.args[2]) /* valid_ec */ {
		assert(VECTOR_IS_SYNC(uc.args[0]));
		TEST_FAIL("Unexpected exception (vector:0x%lx, ec:0x%lx)",
			  uc.args[0], uc.args[1]);
	} else {
		assert(!VECTOR_IS_SYNC(uc.args[0]));
		TEST_FAIL("Unexpected exception (vector:0x%lx)",
			  uc.args[0]);
	}
}

struct handlers {
	handler_fn exception_handlers[VECTOR_NUM][ESR_EC_NUM];
};

void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid)
{
	extern char vectors;

393
	set_reg(vm, vcpuid, KVM_ARM64_SYS_REG(SYS_VBAR_EL1), (uint64_t)&vectors);
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431
}

void route_exception(struct ex_regs *regs, int vector)
{
	struct handlers *handlers = (struct handlers *)exception_handlers;
	bool valid_ec;
	int ec = 0;

	switch (vector) {
	case VECTOR_SYNC_CURRENT:
	case VECTOR_SYNC_LOWER_64:
		ec = (read_sysreg(esr_el1) >> ESR_EC_SHIFT) & ESR_EC_MASK;
		valid_ec = true;
		break;
	case VECTOR_IRQ_CURRENT:
	case VECTOR_IRQ_LOWER_64:
	case VECTOR_FIQ_CURRENT:
	case VECTOR_FIQ_LOWER_64:
	case VECTOR_ERROR_CURRENT:
	case VECTOR_ERROR_LOWER_64:
		ec = 0;
		valid_ec = false;
		break;
	default:
		valid_ec = false;
		goto unexpected_exception;
	}

	if (handlers && handlers->exception_handlers[vector][ec])
		return handlers->exception_handlers[vector][ec](regs);

unexpected_exception:
	kvm_exit_unexpected_exception(vector, ec, valid_ec);
}

void vm_init_descriptor_tables(struct kvm_vm *vm)
{
	vm->handlers = vm_vaddr_alloc(vm, sizeof(struct handlers),
432
			vm->page_size);
433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455

	*(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers;
}

void vm_install_sync_handler(struct kvm_vm *vm, int vector, int ec,
			 void (*handler)(struct ex_regs *))
{
	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);

	assert(VECTOR_IS_SYNC(vector));
	assert(vector < VECTOR_NUM);
	assert(ec < ESR_EC_NUM);
	handlers->exception_handlers[vector][ec] = handler;
}

void vm_install_exception_handler(struct kvm_vm *vm, int vector,
			 void (*handler)(struct ex_regs *))
{
	struct handlers *handlers = addr_gva2hva(vm, vm->handlers);

	assert(!VECTOR_IS_SYNC(vector));
	assert(vector < VECTOR_NUM);
	handlers->exception_handlers[vector][0] = handler;
456
}
457 458 459 460 461

uint32_t guest_get_vcpuid(void)
{
	return read_sysreg(tpidr_el1);
}
462

463 464 465 466 467 468 469 470 471 472 473 474
void aarch64_get_supported_page_sizes(uint32_t ipa,
				      bool *ps4k, bool *ps16k, bool *ps64k)
{
	struct kvm_vcpu_init preferred_init;
	int kvm_fd, vm_fd, vcpu_fd, err;
	uint64_t val;
	struct kvm_one_reg reg = {
		.id	= KVM_ARM64_SYS_REG(SYS_ID_AA64MMFR0_EL1),
		.addr	= (uint64_t)&val,
	};

	kvm_fd = open_kvm_dev_path_or_exit();
475 476
	vm_fd = __kvm_ioctl(kvm_fd, KVM_CREATE_VM, ipa);
	TEST_ASSERT(vm_fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VM, vm_fd));
477 478

	vcpu_fd = ioctl(vm_fd, KVM_CREATE_VCPU, 0);
479
	TEST_ASSERT(vcpu_fd >= 0, KVM_IOCTL_ERROR(KVM_CREATE_VCPU, vcpu_fd));
480 481

	err = ioctl(vm_fd, KVM_ARM_PREFERRED_TARGET, &preferred_init);
482
	TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_ARM_PREFERRED_TARGET, err));
483
	err = ioctl(vcpu_fd, KVM_ARM_VCPU_INIT, &preferred_init);
484
	TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_ARM_VCPU_INIT, err));
485 486

	err = ioctl(vcpu_fd, KVM_GET_ONE_REG, &reg);
487
	TEST_ASSERT(err == 0, KVM_IOCTL_ERROR(KVM_GET_ONE_REG, vcpu_fd));
488 489 490 491 492 493 494 495 496 497

	*ps4k = ((val >> 28) & 0xf) != 0xf;
	*ps64k = ((val >> 24) & 0xf) == 0;
	*ps16k = ((val >> 20) & 0xf) != 0;

	close(vcpu_fd);
	close(vm_fd);
	close(kvm_fd);
}

498 499 500 501 502 503 504 505
/*
 * arm64 doesn't have a true default mode, so start by computing the
 * available IPA space and page sizes early.
 */
void __attribute__((constructor)) init_guest_modes(void)
{
       guest_modes_append_default();
}
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530

void smccc_hvc(uint32_t function_id, uint64_t arg0, uint64_t arg1,
	       uint64_t arg2, uint64_t arg3, uint64_t arg4, uint64_t arg5,
	       uint64_t arg6, struct arm_smccc_res *res)
{
	asm volatile("mov   w0, %w[function_id]\n"
		     "mov   x1, %[arg0]\n"
		     "mov   x2, %[arg1]\n"
		     "mov   x3, %[arg2]\n"
		     "mov   x4, %[arg3]\n"
		     "mov   x5, %[arg4]\n"
		     "mov   x6, %[arg5]\n"
		     "mov   x7, %[arg6]\n"
		     "hvc   #0\n"
		     "mov   %[res0], x0\n"
		     "mov   %[res1], x1\n"
		     "mov   %[res2], x2\n"
		     "mov   %[res3], x3\n"
		     : [res0] "=r"(res->a0), [res1] "=r"(res->a1),
		       [res2] "=r"(res->a2), [res3] "=r"(res->a3)
		     : [function_id] "r"(function_id), [arg0] "r"(arg0),
		       [arg1] "r"(arg1), [arg2] "r"(arg2), [arg3] "r"(arg3),
		       [arg4] "r"(arg4), [arg5] "r"(arg5), [arg6] "r"(arg6)
		     : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7");
}