arm_arch_timer.c 42.6 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7
/*
 *  linux/drivers/clocksource/arm_arch_timer.c
 *
 *  Copyright (C) 2011 ARM Ltd.
 *  All Rights Reserved
 */
8

9
#define pr_fmt(fmt) 	"arch_timer: " fmt
10

11 12 13 14 15
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/device.h>
#include <linux/smp.h>
#include <linux/cpu.h>
16
#include <linux/cpu_pm.h>
17
#include <linux/clockchips.h>
18
#include <linux/clocksource.h>
19 20
#include <linux/interrupt.h>
#include <linux/of_irq.h>
21
#include <linux/of_address.h>
22
#include <linux/io.h>
23
#include <linux/slab.h>
24
#include <linux/sched/clock.h>
25
#include <linux/sched_clock.h>
26
#include <linux/acpi.h>
27 28

#include <asm/arch_timer.h>
29
#include <asm/virt.h>
30 31 32

#include <clocksource/arm_arch_timer.h>

33 34 35
#define CNTTIDR		0x08
#define CNTTIDR_VIRT(n)	(BIT(1) << ((n) * 4))

36 37 38 39 40 41 42 43
#define CNTACR(n)	(0x40 + ((n) * 4))
#define CNTACR_RPCT	BIT(0)
#define CNTACR_RVCT	BIT(1)
#define CNTACR_RFRQ	BIT(2)
#define CNTACR_RVOFF	BIT(3)
#define CNTACR_RWVT	BIT(4)
#define CNTACR_RWPT	BIT(5)

44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62
#define CNTVCT_LO	0x08
#define CNTVCT_HI	0x0c
#define CNTFRQ		0x10
#define CNTP_TVAL	0x28
#define CNTP_CTL	0x2c
#define CNTV_TVAL	0x38
#define CNTV_CTL	0x3c

static unsigned arch_timers_present __initdata;

static void __iomem *arch_counter_base;

struct arch_timer {
	void __iomem *base;
	struct clock_event_device evt;
};

#define to_arch_timer(e) container_of(e, struct arch_timer, evt)

63
static u32 arch_timer_rate;
64
static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI];
65 66 67

static struct clock_event_device __percpu *arch_timer_evt;

68
static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI;
69
static bool arch_timer_c3stop;
70
static bool arch_timer_mem_use_virtual;
71
static bool arch_counter_suspend_stop;
72
#ifdef CONFIG_GENERIC_GETTIMEOFDAY
73
static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_ARCHTIMER;
74 75 76
#else
static enum vdso_clock_mode vdso_default = VDSO_CLOCKMODE_NONE;
#endif /* CONFIG_GENERIC_GETTIMEOFDAY */
77

78
static cpumask_t evtstrm_available = CPU_MASK_NONE;
79 80 81 82 83 84 85 86
static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);

static int __init early_evtstrm_cfg(char *buf)
{
	return strtobool(buf, &evtstrm_enable);
}
early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);

87 88 89 90
/*
 * Architected system timer support.
 */

91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
static __always_inline
void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
			  struct clock_event_device *clk)
{
	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
		struct arch_timer *timer = to_arch_timer(clk);
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			writel_relaxed(val, timer->base + CNTP_CTL);
			break;
		case ARCH_TIMER_REG_TVAL:
			writel_relaxed(val, timer->base + CNTP_TVAL);
			break;
		}
	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
		struct arch_timer *timer = to_arch_timer(clk);
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			writel_relaxed(val, timer->base + CNTV_CTL);
			break;
		case ARCH_TIMER_REG_TVAL:
			writel_relaxed(val, timer->base + CNTV_TVAL);
			break;
		}
	} else {
		arch_timer_reg_write_cp15(access, reg, val);
	}
}

static __always_inline
u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
			struct clock_event_device *clk)
{
	u32 val;

	if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
		struct arch_timer *timer = to_arch_timer(clk);
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			val = readl_relaxed(timer->base + CNTP_CTL);
			break;
		case ARCH_TIMER_REG_TVAL:
			val = readl_relaxed(timer->base + CNTP_TVAL);
			break;
		}
	} else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
		struct arch_timer *timer = to_arch_timer(clk);
		switch (reg) {
		case ARCH_TIMER_REG_CTRL:
			val = readl_relaxed(timer->base + CNTV_CTL);
			break;
		case ARCH_TIMER_REG_TVAL:
			val = readl_relaxed(timer->base + CNTV_TVAL);
			break;
		}
	} else {
		val = arch_timer_reg_read_cp15(access, reg);
	}

	return val;
}

153
static notrace u64 arch_counter_get_cntpct_stable(void)
154 155 156 157
{
	return __arch_counter_get_cntpct_stable();
}

158
static notrace u64 arch_counter_get_cntpct(void)
159 160 161 162
{
	return __arch_counter_get_cntpct();
}

163
static notrace u64 arch_counter_get_cntvct_stable(void)
164 165 166 167
{
	return __arch_counter_get_cntvct_stable();
}

168
static notrace u64 arch_counter_get_cntvct(void)
169 170 171 172
{
	return __arch_counter_get_cntvct();
}

173 174 175 176 177 178 179
/*
 * Default to cp15 based access because arm64 uses this function for
 * sched_clock() before DT is probed and the cp15 method is guaranteed
 * to exist on arm64. arm doesn't use this before DT is probed so even
 * if we don't have the cp15 accessors we won't have a problem.
 */
u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
180
EXPORT_SYMBOL_GPL(arch_timer_read_counter);
181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204

static u64 arch_counter_read(struct clocksource *cs)
{
	return arch_timer_read_counter();
}

static u64 arch_counter_read_cc(const struct cyclecounter *cc)
{
	return arch_timer_read_counter();
}

static struct clocksource clocksource_counter = {
	.name	= "arch_sys_counter",
	.rating	= 400,
	.read	= arch_counter_read,
	.mask	= CLOCKSOURCE_MASK(56),
	.flags	= CLOCK_SOURCE_IS_CONTINUOUS,
};

static struct cyclecounter cyclecounter __ro_after_init = {
	.read	= arch_counter_read_cc,
	.mask	= CLOCKSOURCE_MASK(56),
};

205 206 207 208 209 210
struct ate_acpi_oem_info {
	char oem_id[ACPI_OEM_ID_SIZE + 1];
	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
	u32 oem_revision;
};

211
#ifdef CONFIG_FSL_ERRATUM_A008585
212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230
/*
 * The number of retries is an arbitrary value well beyond the highest number
 * of iterations the loop has been observed to take.
 */
#define __fsl_a008585_read_reg(reg) ({			\
	u64 _old, _new;					\
	int _retries = 200;				\
							\
	do {						\
		_old = read_sysreg(reg);		\
		_new = read_sysreg(reg);		\
		_retries--;				\
	} while (unlikely(_old != _new) && _retries);	\
							\
	WARN_ON_ONCE(!_retries);			\
	_new;						\
})

static u32 notrace fsl_a008585_read_cntp_tval_el0(void)
231 232 233 234
{
	return __fsl_a008585_read_reg(cntp_tval_el0);
}

235
static u32 notrace fsl_a008585_read_cntv_tval_el0(void)
236 237 238 239
{
	return __fsl_a008585_read_reg(cntv_tval_el0);
}

240 241 242 243 244
static u64 notrace fsl_a008585_read_cntpct_el0(void)
{
	return __fsl_a008585_read_reg(cntpct_el0);
}

245
static u64 notrace fsl_a008585_read_cntvct_el0(void)
246 247 248
{
	return __fsl_a008585_read_reg(cntvct_el0);
}
249 250
#endif

251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285
#ifdef CONFIG_HISILICON_ERRATUM_161010101
/*
 * Verify whether the value of the second read is larger than the first by
 * less than 32 is the only way to confirm the value is correct, so clear the
 * lower 5 bits to check whether the difference is greater than 32 or not.
 * Theoretically the erratum should not occur more than twice in succession
 * when reading the system counter, but it is possible that some interrupts
 * may lead to more than twice read errors, triggering the warning, so setting
 * the number of retries far beyond the number of iterations the loop has been
 * observed to take.
 */
#define __hisi_161010101_read_reg(reg) ({				\
	u64 _old, _new;						\
	int _retries = 50;					\
								\
	do {							\
		_old = read_sysreg(reg);			\
		_new = read_sysreg(reg);			\
		_retries--;					\
	} while (unlikely((_new - _old) >> 5) && _retries);	\
								\
	WARN_ON_ONCE(!_retries);				\
	_new;							\
})

static u32 notrace hisi_161010101_read_cntp_tval_el0(void)
{
	return __hisi_161010101_read_reg(cntp_tval_el0);
}

static u32 notrace hisi_161010101_read_cntv_tval_el0(void)
{
	return __hisi_161010101_read_reg(cntv_tval_el0);
}

286 287 288 289 290
static u64 notrace hisi_161010101_read_cntpct_el0(void)
{
	return __hisi_161010101_read_reg(cntpct_el0);
}

291 292 293 294
static u64 notrace hisi_161010101_read_cntvct_el0(void)
{
	return __hisi_161010101_read_reg(cntvct_el0);
}
295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317

static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
	/*
	 * Note that trailing spaces are required to properly match
	 * the OEM table information.
	 */
	{
		.oem_id		= "HISI  ",
		.oem_table_id	= "HIP05   ",
		.oem_revision	= 0,
	},
	{
		.oem_id		= "HISI  ",
		.oem_table_id	= "HIP06   ",
		.oem_revision	= 0,
	},
	{
		.oem_id		= "HISI  ",
		.oem_table_id	= "HIP07   ",
		.oem_revision	= 0,
	},
	{ /* Sentinel indicating the end of the OEM array */ },
};
318 319
#endif

320
#ifdef CONFIG_ARM64_ERRATUM_858921
321 322 323 324 325 326 327 328 329
static u64 notrace arm64_858921_read_cntpct_el0(void)
{
	u64 old, new;

	old = read_sysreg(cntpct_el0);
	new = read_sysreg(cntpct_el0);
	return (((old ^ new) >> 32) & 1) ? old : new;
}

330 331 332 333 334 335 336 337 338 339
static u64 notrace arm64_858921_read_cntvct_el0(void)
{
	u64 old, new;

	old = read_sysreg(cntvct_el0);
	new = read_sysreg(cntvct_el0);
	return (((old ^ new) >> 32) & 1) ? old : new;
}
#endif

340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
#ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
/*
 * The low bits of the counter registers are indeterminate while bit 10 or
 * greater is rolling over. Since the counter value can jump both backward
 * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values
 * with all ones or all zeros in the low bits. Bound the loop by the maximum
 * number of CPU cycles in 3 consecutive 24 MHz counter periods.
 */
#define __sun50i_a64_read_reg(reg) ({					\
	u64 _val;							\
	int _retries = 150;						\
									\
	do {								\
		_val = read_sysreg(reg);				\
		_retries--;						\
	} while (((_val + 1) & GENMASK(9, 0)) <= 1 && _retries);	\
									\
	WARN_ON_ONCE(!_retries);					\
	_val;								\
})

static u64 notrace sun50i_a64_read_cntpct_el0(void)
{
	return __sun50i_a64_read_reg(cntpct_el0);
}

static u64 notrace sun50i_a64_read_cntvct_el0(void)
{
	return __sun50i_a64_read_reg(cntvct_el0);
}

static u32 notrace sun50i_a64_read_cntp_tval_el0(void)
{
	return read_sysreg(cntp_cval_el0) - sun50i_a64_read_cntpct_el0();
}

static u32 notrace sun50i_a64_read_cntv_tval_el0(void)
{
	return read_sysreg(cntv_cval_el0) - sun50i_a64_read_cntvct_el0();
}
#endif

382
#ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
383
DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
384 385
EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);

386
static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0);
387

388 389 390 391
static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,
						struct clock_event_device *clk)
{
	unsigned long ctrl;
392
	u64 cval;
393 394 395 396 397

	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
	ctrl |= ARCH_TIMER_CTRL_ENABLE;
	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;

398
	if (access == ARCH_TIMER_PHYS_ACCESS) {
399
		cval = evt + arch_counter_get_cntpct_stable();
400
		write_sysreg(cval, cntp_cval_el0);
401
	} else {
402
		cval = evt + arch_counter_get_cntvct_stable();
403
		write_sysreg(cval, cntv_cval_el0);
404
	}
405 406 407 408

	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
}

409
static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt,
410 411 412 413 414 415
					    struct clock_event_device *clk)
{
	erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
	return 0;
}

416
static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt,
417 418 419 420 421 422
					    struct clock_event_device *clk)
{
	erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
	return 0;
}

423 424 425
static const struct arch_timer_erratum_workaround ool_workarounds[] = {
#ifdef CONFIG_FSL_ERRATUM_A008585
	{
426
		.match_type = ate_match_dt,
427
		.id = "fsl,erratum-a008585",
428
		.desc = "Freescale erratum a005858",
429 430
		.read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,
		.read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,
431
		.read_cntpct_el0 = fsl_a008585_read_cntpct_el0,
432
		.read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
433 434
		.set_next_event_phys = erratum_set_next_event_tval_phys,
		.set_next_event_virt = erratum_set_next_event_tval_virt,
435 436
	},
#endif
437 438
#ifdef CONFIG_HISILICON_ERRATUM_161010101
	{
439
		.match_type = ate_match_dt,
440
		.id = "hisilicon,erratum-161010101",
441
		.desc = "HiSilicon erratum 161010101",
442 443
		.read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
		.read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
444
		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
445
		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
446 447
		.set_next_event_phys = erratum_set_next_event_tval_phys,
		.set_next_event_virt = erratum_set_next_event_tval_virt,
448 449 450 451 452 453 454
	},
	{
		.match_type = ate_match_acpi_oem_info,
		.id = hisi_161010101_oem_info,
		.desc = "HiSilicon erratum 161010101",
		.read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
		.read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
455
		.read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
456 457 458
		.read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
		.set_next_event_phys = erratum_set_next_event_tval_phys,
		.set_next_event_virt = erratum_set_next_event_tval_virt,
459 460
	},
#endif
461 462 463 464 465
#ifdef CONFIG_ARM64_ERRATUM_858921
	{
		.match_type = ate_match_local_cap_id,
		.id = (void *)ARM64_WORKAROUND_858921,
		.desc = "ARM erratum 858921",
466
		.read_cntpct_el0 = arm64_858921_read_cntpct_el0,
467 468 469
		.read_cntvct_el0 = arm64_858921_read_cntvct_el0,
	},
#endif
470 471 472 473 474 475 476 477 478 479 480 481 482
#ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
	{
		.match_type = ate_match_dt,
		.id = "allwinner,erratum-unknown1",
		.desc = "Allwinner erratum UNKNOWN1",
		.read_cntp_tval_el0 = sun50i_a64_read_cntp_tval_el0,
		.read_cntv_tval_el0 = sun50i_a64_read_cntv_tval_el0,
		.read_cntpct_el0 = sun50i_a64_read_cntpct_el0,
		.read_cntvct_el0 = sun50i_a64_read_cntvct_el0,
		.set_next_event_phys = erratum_set_next_event_tval_phys,
		.set_next_event_virt = erratum_set_next_event_tval_virt,
	},
#endif
483 484 485 486 487 488 489 490
#ifdef CONFIG_ARM64_ERRATUM_1418040
	{
		.match_type = ate_match_local_cap_id,
		.id = (void *)ARM64_WORKAROUND_1418040,
		.desc = "ARM erratum 1418040",
		.disable_compat_vdso = true,
	},
#endif
491
};
492 493 494 495 496 497 498 499 500 501 502 503 504

typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
			       const void *);

static
bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
				 const void *arg)
{
	const struct device_node *np = arg;

	return of_property_read_bool(np, wa->id);
}

505 506 507 508 509 510 511
static
bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
					const void *arg)
{
	return this_cpu_has_cap((uintptr_t)wa->id);
}

512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533

static
bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
				       const void *arg)
{
	static const struct ate_acpi_oem_info empty_oem_info = {};
	const struct ate_acpi_oem_info *info = wa->id;
	const struct acpi_table_header *table = arg;

	/* Iterate over the ACPI OEM info array, looking for a match */
	while (memcmp(info, &empty_oem_info, sizeof(*info))) {
		if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
		    !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
		    info->oem_revision == table->oem_revision)
			return true;

		info++;
	}

	return false;
}

534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552
static const struct arch_timer_erratum_workaround *
arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
			  ate_match_fn_t match_fn,
			  void *arg)
{
	int i;

	for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
		if (ool_workarounds[i].match_type != type)
			continue;

		if (match_fn(&ool_workarounds[i], arg))
			return &ool_workarounds[i];
	}

	return NULL;
}

static
553 554
void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
				  bool local)
555
{
556 557 558 559 560 561 562 563 564
	int i;

	if (local) {
		__this_cpu_write(timer_unstable_counter_workaround, wa);
	} else {
		for_each_possible_cpu(i)
			per_cpu(timer_unstable_counter_workaround, i) = wa;
	}

565 566
	if (wa->read_cntvct_el0 || wa->read_cntpct_el0)
		atomic_set(&timer_unstable_counter_workaround_in_use, 1);
567 568 569 570 571 572 573 574

	/*
	 * Don't use the vdso fastpath if errata require using the
	 * out-of-line counter accessor. We may change our mind pretty
	 * late in the game (with a per-CPU erratum, for example), so
	 * change both the default value and the vdso itself.
	 */
	if (wa->read_cntvct_el0) {
575 576
		clocksource_counter.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
		vdso_default = VDSO_CLOCKMODE_NONE;
577 578 579
	} else if (wa->disable_compat_vdso && vdso_default != VDSO_CLOCKMODE_NONE) {
		vdso_default = VDSO_CLOCKMODE_ARCHTIMER_NOCOMPAT;
		clocksource_counter.vdso_clock_mode = vdso_default;
580
	}
581 582 583 584 585
}

static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
					    void *arg)
{
586
	const struct arch_timer_erratum_workaround *wa, *__wa;
587
	ate_match_fn_t match_fn = NULL;
588
	bool local = false;
589 590 591 592 593

	switch (type) {
	case ate_match_dt:
		match_fn = arch_timer_check_dt_erratum;
		break;
594 595 596 597
	case ate_match_local_cap_id:
		match_fn = arch_timer_check_local_cap_erratum;
		local = true;
		break;
598 599 600
	case ate_match_acpi_oem_info:
		match_fn = arch_timer_check_acpi_oem_erratum;
		break;
601 602 603 604 605 606 607 608 609
	default:
		WARN_ON(1);
		return;
	}

	wa = arch_timer_iterate_errata(type, match_fn, arg);
	if (!wa)
		return;

610 611 612 613
	__wa = __this_cpu_read(timer_unstable_counter_workaround);
	if (__wa && wa != __wa)
		pr_warn("Can't enable workaround for %s (clashes with %s\n)",
			wa->desc, __wa->desc);
614

615 616
	if (__wa)
		return;
617

618
	arch_timer_enable_workaround(wa, local);
619 620
	pr_info("Enabling %s workaround for %s\n",
		local ? "local" : "global", wa->desc);
621 622
}

623 624
static bool arch_timer_this_cpu_has_cntvct_wa(void)
{
625
	return has_erratum_handler(read_cntvct_el0);
626 627
}

628 629 630
static bool arch_timer_counter_has_wa(void)
{
	return atomic_read(&timer_unstable_counter_workaround_in_use);
631
}
632 633
#else
#define arch_timer_check_ool_workaround(t,a)		do { } while(0)
634
#define arch_timer_this_cpu_has_cntvct_wa()		({false;})
635
#define arch_timer_counter_has_wa()			({false;})
636
#endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
637

638
static __always_inline irqreturn_t timer_handler(const int access,
639 640 641
					struct clock_event_device *evt)
{
	unsigned long ctrl;
642

643
	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
644 645
	if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
		ctrl |= ARCH_TIMER_CTRL_IT_MASK;
646
		arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667
		evt->event_handler(evt);
		return IRQ_HANDLED;
	}

	return IRQ_NONE;
}

static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
{
	struct clock_event_device *evt = dev_id;

	return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
}

static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
{
	struct clock_event_device *evt = dev_id;

	return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
}

668 669 670 671 672 673 674 675 676 677 678 679 680 681
static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
{
	struct clock_event_device *evt = dev_id;

	return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
}

static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
{
	struct clock_event_device *evt = dev_id;

	return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
}

682 683
static __always_inline int timer_shutdown(const int access,
					  struct clock_event_device *clk)
684 685
{
	unsigned long ctrl;
686 687 688 689 690 691

	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
	ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);

	return 0;
692 693
}

694
static int arch_timer_shutdown_virt(struct clock_event_device *clk)
695
{
696
	return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
697 698
}

699
static int arch_timer_shutdown_phys(struct clock_event_device *clk)
700
{
701
	return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
702 703
}

704
static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
705
{
706
	return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
707 708
}

709
static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
710
{
711
	return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
712 713
}

714
static __always_inline void set_next_event(const int access, unsigned long evt,
715
					   struct clock_event_device *clk)
716 717
{
	unsigned long ctrl;
718
	ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
719 720
	ctrl |= ARCH_TIMER_CTRL_ENABLE;
	ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
721 722
	arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
	arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
723 724 725
}

static int arch_timer_set_next_event_virt(unsigned long evt,
726
					  struct clock_event_device *clk)
727
{
728
	set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
729 730 731 732
	return 0;
}

static int arch_timer_set_next_event_phys(unsigned long evt,
733
					  struct clock_event_device *clk)
734
{
735
	set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
736 737 738
	return 0;
}

739 740
static int arch_timer_set_next_event_virt_mem(unsigned long evt,
					      struct clock_event_device *clk)
741
{
742 743 744 745 746 747 748 749 750 751 752
	set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
	return 0;
}

static int arch_timer_set_next_event_phys_mem(unsigned long evt,
					      struct clock_event_device *clk)
{
	set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
	return 0;
}

753 754
static void __arch_timer_setup(unsigned type,
			       struct clock_event_device *clk)
755 756 757
{
	clk->features = CLOCK_EVT_FEAT_ONESHOT;

758
	if (type == ARCH_TIMER_TYPE_CP15) {
759 760 761 762
		typeof(clk->set_next_event) sne;

		arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);

763 764
		if (arch_timer_c3stop)
			clk->features |= CLOCK_EVT_FEAT_C3STOP;
765 766 767
		clk->name = "arch_sys_timer";
		clk->rating = 450;
		clk->cpumask = cpumask_of(smp_processor_id());
768 769
		clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
		switch (arch_timer_uses_ppi) {
770
		case ARCH_TIMER_VIRT_PPI:
771
			clk->set_state_shutdown = arch_timer_shutdown_virt;
772
			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
773
			sne = erratum_handler(set_next_event_virt);
774
			break;
775 776 777
		case ARCH_TIMER_PHYS_SECURE_PPI:
		case ARCH_TIMER_PHYS_NONSECURE_PPI:
		case ARCH_TIMER_HYP_PPI:
778
			clk->set_state_shutdown = arch_timer_shutdown_phys;
779
			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
780
			sne = erratum_handler(set_next_event_phys);
781 782 783
			break;
		default:
			BUG();
784
		}
785

786
		clk->set_next_event = sne;
787
	} else {
788
		clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
789 790
		clk->name = "arch_mem_timer";
		clk->rating = 400;
791
		clk->cpumask = cpu_possible_mask;
792
		if (arch_timer_mem_use_virtual) {
793
			clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
794
			clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
795 796 797
			clk->set_next_event =
				arch_timer_set_next_event_virt_mem;
		} else {
798
			clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
799
			clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
800 801 802
			clk->set_next_event =
				arch_timer_set_next_event_phys_mem;
		}
803 804
	}

805
	clk->set_state_shutdown(clk);
806

807 808
	clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
}
809

810 811 812 813 814 815 816 817 818
static void arch_timer_evtstrm_enable(int divider)
{
	u32 cntkctl = arch_timer_get_cntkctl();

	cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
	/* Set the divider and enable virtual event stream */
	cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
			| ARCH_TIMER_VIRT_EVT_EN;
	arch_timer_set_cntkctl(cntkctl);
819
	arch_timer_set_evtstrm_feature();
820
	cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
821 822
}

823 824 825 826 827 828 829 830 831 832 833 834 835
static void arch_timer_configure_evtstream(void)
{
	int evt_stream_div, pos;

	/* Find the closest power of two to the divisor */
	evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
	pos = fls(evt_stream_div);
	if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
		pos--;
	/* enable event stream */
	arch_timer_evtstrm_enable(min(pos, 15));
}

836 837 838 839
static void arch_counter_set_user_access(void)
{
	u32 cntkctl = arch_timer_get_cntkctl();

840
	/* Disable user access to the timers and both counters */
841 842 843
	/* Also disable virtual event stream */
	cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
			| ARCH_TIMER_USR_VT_ACCESS_EN
844
		        | ARCH_TIMER_USR_VCT_ACCESS_EN
845 846 847
			| ARCH_TIMER_VIRT_EVT_EN
			| ARCH_TIMER_USR_PCT_ACCESS_EN);

848 849 850 851 852 853 854 855 856
	/*
	 * Enable user access to the virtual counter if it doesn't
	 * need to be workaround. The vdso may have been already
	 * disabled though.
	 */
	if (arch_timer_this_cpu_has_cntvct_wa())
		pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
	else
		cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
857 858 859 860

	arch_timer_set_cntkctl(cntkctl);
}

861 862
static bool arch_timer_has_nonsecure_ppi(void)
{
863 864
	return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
		arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
865 866
}

867 868 869 870 871 872 873 874 875 876 877 878 879
static u32 check_ppi_trigger(int irq)
{
	u32 flags = irq_get_trigger_type(irq);

	if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
		pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
		pr_warn("WARNING: Please fix your firmware\n");
		flags = IRQF_TRIGGER_LOW;
	}

	return flags;
}

880
static int arch_timer_starting_cpu(unsigned int cpu)
881
{
882
	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
883
	u32 flags;
884

885
	__arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
886

887 888
	flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
	enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
889

890
	if (arch_timer_has_nonsecure_ppi()) {
891 892 893
		flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
		enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
				  flags);
894
	}
895 896

	arch_counter_set_user_access();
897
	if (evtstrm_enable)
898
		arch_timer_configure_evtstream();
899 900 901 902

	return 0;
}

903 904 905 906 907 908 909 910 911 912 913
static int validate_timer_rate(void)
{
	if (!arch_timer_rate)
		return -EINVAL;

	/* Arch timer frequency < 1MHz can cause trouble */
	WARN_ON(arch_timer_rate < 1000000);

	return 0;
}

914 915 916 917 918 919
/*
 * For historical reasons, when probing with DT we use whichever (non-zero)
 * rate was probed first, and don't verify that others match. If the first node
 * probed has a clock-frequency property, this overrides the HW register.
 */
static void arch_timer_of_configure_rate(u32 rate, struct device_node *np)
920
{
921 922 923
	/* Who has more than one independent system counter? */
	if (arch_timer_rate)
		return;
924

925 926
	if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
		arch_timer_rate = rate;
927

928
	/* Check the timer frequency. */
929
	if (validate_timer_rate())
930
		pr_warn("frequency not available\n");
931 932 933 934
}

static void arch_timer_banner(unsigned type)
{
935
	pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
936 937 938 939
		type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
		type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
			" and " : "",
		type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
940 941
		(unsigned long)arch_timer_rate / 1000000,
		(unsigned long)(arch_timer_rate / 10000) % 100,
942
		type & ARCH_TIMER_TYPE_CP15 ?
943
			(arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
944
			"",
945 946
		type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
		type & ARCH_TIMER_TYPE_MEM ?
947 948
			arch_timer_mem_use_virtual ? "virt" : "phys" :
			"");
949 950 951 952 953 954 955
}

u32 arch_timer_get_rate(void)
{
	return arch_timer_rate;
}

956 957 958 959 960 961 962 963 964 965
bool arch_timer_evtstrm_available(void)
{
	/*
	 * We might get called from a preemptible context. This is fine
	 * because availability of the event stream should be always the same
	 * for a preemptible context and context where we might resume a task.
	 */
	return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
}

966
static u64 arch_counter_get_cntvct_mem(void)
967
{
968 969 970 971 972 973 974 975 976
	u32 vct_lo, vct_hi, tmp_hi;

	do {
		vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
		vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
		tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
	} while (vct_hi != tmp_hi);

	return ((u64) vct_hi << 32) | vct_lo;
977 978
}

979 980 981 982 983 984
static struct arch_timer_kvm_info arch_timer_kvm_info;

struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
{
	return &arch_timer_kvm_info;
}
985

986 987 988 989 990
static void __init arch_counter_register(unsigned type)
{
	u64 start_count;

	/* Register the CP15 based counter if we have one */
991
	if (type & ARCH_TIMER_TYPE_CP15) {
992 993
		u64 (*rd)(void);

994
		if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
995 996 997 998 999 1000 1001 1002 1003 1004 1005
		    arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) {
			if (arch_timer_counter_has_wa())
				rd = arch_counter_get_cntvct_stable;
			else
				rd = arch_counter_get_cntvct;
		} else {
			if (arch_timer_counter_has_wa())
				rd = arch_counter_get_cntpct_stable;
			else
				rd = arch_counter_get_cntpct;
		}
1006

1007
		arch_timer_read_counter = rd;
1008
		clocksource_counter.vdso_clock_mode = vdso_default;
1009
	} else {
1010
		arch_timer_read_counter = arch_counter_get_cntvct_mem;
1011 1012
	}

1013 1014
	if (!arch_counter_suspend_stop)
		clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
1015 1016 1017 1018
	start_count = arch_timer_read_counter();
	clocksource_register_hz(&clocksource_counter, arch_timer_rate);
	cyclecounter.mult = clocksource_counter.mult;
	cyclecounter.shift = clocksource_counter.shift;
1019 1020
	timecounter_init(&arch_timer_kvm_info.timecounter,
			 &cyclecounter, start_count);
1021 1022 1023

	/* 56 bits minimum, so we assume worst case rollover */
	sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
1024 1025
}

1026
static void arch_timer_stop(struct clock_event_device *clk)
1027
{
1028
	pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
1029

1030 1031
	disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
	if (arch_timer_has_nonsecure_ppi())
1032
		disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1033

1034
	clk->set_state_shutdown(clk);
1035 1036
}

1037
static int arch_timer_dying_cpu(unsigned int cpu)
1038
{
1039
	struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1040

1041 1042
	cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);

1043 1044
	arch_timer_stop(clk);
	return 0;
1045 1046
}

1047
#ifdef CONFIG_CPU_PM
1048
static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
1049 1050 1051
static int arch_timer_cpu_pm_notify(struct notifier_block *self,
				    unsigned long action, void *hcpu)
{
1052
	if (action == CPU_PM_ENTER) {
1053
		__this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
1054 1055 1056

		cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
	} else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
1057
		arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
1058

1059
		if (arch_timer_have_evtstrm_feature())
1060 1061
			cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
	}
1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072
	return NOTIFY_OK;
}

static struct notifier_block arch_timer_cpu_pm_notifier = {
	.notifier_call = arch_timer_cpu_pm_notify,
};

static int __init arch_timer_cpu_pm_init(void)
{
	return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
}
1073 1074 1075 1076 1077 1078

static void __init arch_timer_cpu_pm_deinit(void)
{
	WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
}

1079 1080 1081 1082 1083
#else
static int __init arch_timer_cpu_pm_init(void)
{
	return 0;
}
1084 1085 1086 1087

static void __init arch_timer_cpu_pm_deinit(void)
{
}
1088 1089
#endif

1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
static int __init arch_timer_register(void)
{
	int err;
	int ppi;

	arch_timer_evt = alloc_percpu(struct clock_event_device);
	if (!arch_timer_evt) {
		err = -ENOMEM;
		goto out;
	}

1101 1102
	ppi = arch_timer_ppi[arch_timer_uses_ppi];
	switch (arch_timer_uses_ppi) {
1103
	case ARCH_TIMER_VIRT_PPI:
1104 1105
		err = request_percpu_irq(ppi, arch_timer_handler_virt,
					 "arch_timer", arch_timer_evt);
1106
		break;
1107 1108
	case ARCH_TIMER_PHYS_SECURE_PPI:
	case ARCH_TIMER_PHYS_NONSECURE_PPI:
1109 1110
		err = request_percpu_irq(ppi, arch_timer_handler_phys,
					 "arch_timer", arch_timer_evt);
1111
		if (!err && arch_timer_has_nonsecure_ppi()) {
1112
			ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1113 1114 1115
			err = request_percpu_irq(ppi, arch_timer_handler_phys,
						 "arch_timer", arch_timer_evt);
			if (err)
1116
				free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1117 1118
						arch_timer_evt);
		}
1119
		break;
1120
	case ARCH_TIMER_HYP_PPI:
1121 1122 1123 1124 1125
		err = request_percpu_irq(ppi, arch_timer_handler_phys,
					 "arch_timer", arch_timer_evt);
		break;
	default:
		BUG();
1126 1127 1128
	}

	if (err) {
1129
		pr_err("can't register interrupt %d (%d)\n", ppi, err);
1130 1131 1132
		goto out_free;
	}

1133 1134 1135 1136
	err = arch_timer_cpu_pm_init();
	if (err)
		goto out_unreg_notify;

1137 1138
	/* Register and immediately configure the timer on the boot CPU */
	err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1139
				"clockevents/arm/arch_timer:starting",
1140 1141 1142
				arch_timer_starting_cpu, arch_timer_dying_cpu);
	if (err)
		goto out_unreg_cpupm;
1143 1144
	return 0;

1145 1146 1147
out_unreg_cpupm:
	arch_timer_cpu_pm_deinit();

1148
out_unreg_notify:
1149 1150
	free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
	if (arch_timer_has_nonsecure_ppi())
1151
		free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1152 1153 1154 1155 1156 1157 1158 1159
				arch_timer_evt);

out_free:
	free_percpu(arch_timer_evt);
out:
	return err;
}

1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171
static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
{
	int ret;
	irq_handler_t func;
	struct arch_timer *t;

	t = kzalloc(sizeof(*t), GFP_KERNEL);
	if (!t)
		return -ENOMEM;

	t->base = base;
	t->evt.irq = irq;
1172
	__arch_timer_setup(ARCH_TIMER_TYPE_MEM, &t->evt);
1173 1174 1175 1176 1177 1178 1179 1180

	if (arch_timer_mem_use_virtual)
		func = arch_timer_handler_virt_mem;
	else
		func = arch_timer_handler_phys_mem;

	ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
	if (ret) {
1181
		pr_err("Failed to request mem timer irq\n");
1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
		kfree(t);
	}

	return ret;
}

static const struct of_device_id arch_timer_of_match[] __initconst = {
	{ .compatible   = "arm,armv7-timer",    },
	{ .compatible   = "arm,armv8-timer",    },
	{},
};

static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
	{ .compatible   = "arm,armv7-timer-mem", },
	{},
};

1199
static bool __init arch_timer_needs_of_probing(void)
1200 1201
{
	struct device_node *dn;
1202
	bool needs_probing = false;
1203
	unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1204

1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
	/* We have two timers, and both device-tree nodes are probed. */
	if ((arch_timers_present & mask) == mask)
		return false;

	/*
	 * Only one type of timer is probed,
	 * check if we have another type of timer node in device-tree.
	 */
	if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
		dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
	else
		dn = of_find_matching_node(NULL, arch_timer_of_match);

	if (dn && of_device_is_available(dn))
1219
		needs_probing = true;
1220

1221 1222
	of_node_put(dn);

1223
	return needs_probing;
1224 1225
}

1226
static int __init arch_timer_common_init(void)
1227 1228 1229
{
	arch_timer_banner(arch_timers_present);
	arch_counter_register(arch_timers_present);
1230
	return arch_timer_arch_init();
1231 1232
}

1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251
/**
 * arch_timer_select_ppi() - Select suitable PPI for the current system.
 *
 * If HYP mode is available, we know that the physical timer
 * has been configured to be accessible from PL1. Use it, so
 * that a guest can use the virtual timer instead.
 *
 * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
 * accesses to CNTP_*_EL1 registers are silently redirected to
 * their CNTHP_*_EL2 counterparts, and use a different PPI
 * number.
 *
 * If no interrupt provided for virtual timer, we'll have to
 * stick to the physical timer. It'd better be accessible...
 * For arm64 we never use the secure interrupt.
 *
 * Return: a suitable PPI type for the current system.
 */
static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1252
{
1253 1254
	if (is_kernel_in_hyp_mode())
		return ARCH_TIMER_HYP_PPI;
1255

1256 1257
	if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
		return ARCH_TIMER_VIRT_PPI;
1258

1259 1260 1261 1262 1263 1264
	if (IS_ENABLED(CONFIG_ARM64))
		return ARCH_TIMER_PHYS_NONSECURE_PPI;

	return ARCH_TIMER_PHYS_SECURE_PPI;
}

1265 1266 1267 1268 1269 1270 1271
static void __init arch_timer_populate_kvm_info(void)
{
	arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
	if (is_kernel_in_hyp_mode())
		arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
}

1272
static int __init arch_timer_of_init(struct device_node *np)
1273
{
1274
	int i, ret;
1275
	u32 rate;
1276

1277
	if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1278
		pr_warn("multiple nodes in dt, skipping\n");
1279
		return 0;
1280 1281
	}

1282
	arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1283
	for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++)
1284 1285
		arch_timer_ppi[i] = irq_of_parse_and_map(np, i);

1286
	arch_timer_populate_kvm_info();
1287

1288
	rate = arch_timer_get_cntfrq();
1289
	arch_timer_of_configure_rate(rate, np);
1290 1291 1292

	arch_timer_c3stop = !of_property_read_bool(np, "always-on");

1293 1294
	/* Check for globally applicable workarounds */
	arch_timer_check_ool_workaround(ate_match_dt, np);
1295

1296 1297 1298 1299 1300 1301
	/*
	 * If we cannot rely on firmware initializing the timer registers then
	 * we should use the physical timers instead.
	 */
	if (IS_ENABLED(CONFIG_ARM) &&
	    of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1302
		arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1303 1304 1305 1306 1307 1308 1309
	else
		arch_timer_uses_ppi = arch_timer_select_ppi();

	if (!arch_timer_ppi[arch_timer_uses_ppi]) {
		pr_err("No interrupt available, giving up\n");
		return -EINVAL;
	}
1310

1311 1312 1313 1314
	/* On some systems, the counter stops ticking when in suspend. */
	arch_counter_suspend_stop = of_property_read_bool(np,
							 "arm,no-tick-in-suspend");

1315 1316 1317 1318 1319 1320 1321 1322
	ret = arch_timer_register();
	if (ret)
		return ret;

	if (arch_timer_needs_of_probing())
		return 0;

	return arch_timer_common_init();
1323
}
1324 1325
TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1326

1327 1328
static u32 __init
arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1329
{
1330 1331
	void __iomem *base;
	u32 rate;
1332

1333 1334 1335 1336 1337 1338
	base = ioremap(frame->cntbase, frame->size);
	if (!base) {
		pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
		return 0;
	}

1339
	rate = readl_relaxed(base + CNTFRQ);
1340

1341
	iounmap(base);
1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354

	return rate;
}

static struct arch_timer_mem_frame * __init
arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
{
	struct arch_timer_mem_frame *frame, *best_frame = NULL;
	void __iomem *cntctlbase;
	u32 cnttidr;
	int i;

	cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1355
	if (!cntctlbase) {
1356 1357 1358
		pr_err("Can't map CNTCTLBase @ %pa\n",
			&timer_mem->cntctlbase);
		return NULL;
1359 1360 1361 1362 1363 1364 1365 1366
	}

	cnttidr = readl_relaxed(cntctlbase + CNTTIDR);

	/*
	 * Try to find a virtual capable frame. Otherwise fall back to a
	 * physical capable frame.
	 */
1367 1368 1369
	for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
		u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
			     CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1370

1371 1372 1373
		frame = &timer_mem->frame[i];
		if (!frame->valid)
			continue;
1374

1375
		/* Try enabling everything, and see what sticks */
1376 1377
		writel_relaxed(cntacr, cntctlbase + CNTACR(i));
		cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1378

1379
		if ((cnttidr & CNTTIDR_VIRT(i)) &&
1380
		    !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1381 1382 1383 1384
			best_frame = frame;
			arch_timer_mem_use_virtual = true;
			break;
		}
1385 1386 1387 1388

		if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
			continue;

1389
		best_frame = frame;
1390 1391
	}

1392 1393
	iounmap(cntctlbase);

1394
	return best_frame;
1395 1396 1397 1398 1399 1400 1401
}

static int __init
arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
{
	void __iomem *base;
	int ret, irq = 0;
1402 1403

	if (arch_timer_mem_use_virtual)
1404
		irq = frame->virt_irq;
1405
	else
1406
		irq = frame->phys_irq;
1407

1408
	if (!irq) {
1409
		pr_err("Frame missing %s irq.\n",
1410
		       arch_timer_mem_use_virtual ? "virt" : "phys");
1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
		return -EINVAL;
	}

	if (!request_mem_region(frame->cntbase, frame->size,
				"arch_mem_timer"))
		return -EBUSY;

	base = ioremap(frame->cntbase, frame->size);
	if (!base) {
		pr_err("Can't map frame's registers\n");
		return -ENXIO;
1422 1423
	}

1424
	ret = arch_timer_mem_register(base, irq);
1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
	if (ret) {
		iounmap(base);
		return ret;
	}

	arch_counter_base = base;
	arch_timers_present |= ARCH_TIMER_TYPE_MEM;

	return 0;
}

static int __init arch_timer_mem_of_init(struct device_node *np)
{
	struct arch_timer_mem *timer_mem;
	struct arch_timer_mem_frame *frame;
	struct device_node *frame_node;
	struct resource res;
	int ret = -EINVAL;
	u32 rate;

	timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
	if (!timer_mem)
		return -ENOMEM;

	if (of_address_to_resource(np, 0, &res))
1450
		goto out;
1451 1452
	timer_mem->cntctlbase = res.start;
	timer_mem->size = resource_size(&res);
1453

1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493
	for_each_available_child_of_node(np, frame_node) {
		u32 n;
		struct arch_timer_mem_frame *frame;

		if (of_property_read_u32(frame_node, "frame-number", &n)) {
			pr_err(FW_BUG "Missing frame-number.\n");
			of_node_put(frame_node);
			goto out;
		}
		if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
			pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
			       ARCH_TIMER_MEM_MAX_FRAMES - 1);
			of_node_put(frame_node);
			goto out;
		}
		frame = &timer_mem->frame[n];

		if (frame->valid) {
			pr_err(FW_BUG "Duplicated frame-number.\n");
			of_node_put(frame_node);
			goto out;
		}

		if (of_address_to_resource(frame_node, 0, &res)) {
			of_node_put(frame_node);
			goto out;
		}
		frame->cntbase = res.start;
		frame->size = resource_size(&res);

		frame->virt_irq = irq_of_parse_and_map(frame_node,
						       ARCH_TIMER_VIRT_SPI);
		frame->phys_irq = irq_of_parse_and_map(frame_node,
						       ARCH_TIMER_PHYS_SPI);

		frame->valid = true;
	}

	frame = arch_timer_mem_find_best_frame(timer_mem);
	if (!frame) {
1494 1495
		pr_err("Unable to find a suitable frame in timer @ %pa\n",
			&timer_mem->cntctlbase);
1496 1497 1498 1499 1500 1501 1502 1503 1504
		ret = -EINVAL;
		goto out;
	}

	rate = arch_timer_mem_frame_get_cntfrq(frame);
	arch_timer_of_configure_rate(rate, np);

	ret = arch_timer_mem_frame_register(frame);
	if (!ret && !arch_timer_needs_of_probing())
1505
		ret = arch_timer_common_init();
1506
out:
1507
	kfree(timer_mem);
1508
	return ret;
1509
}
1510
TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1511
		       arch_timer_mem_of_init);
1512

1513
#ifdef CONFIG_ACPI_GTDT
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
static int __init
arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
{
	struct arch_timer_mem_frame *frame;
	u32 rate;
	int i;

	for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
		frame = &timer_mem->frame[i];

		if (!frame->valid)
			continue;

		rate = arch_timer_mem_frame_get_cntfrq(frame);
		if (rate == arch_timer_rate)
			continue;

		pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
			&frame->cntbase,
			(unsigned long)rate, (unsigned long)arch_timer_rate);

		return -EINVAL;
	}

	return 0;
}

static int __init arch_timer_mem_acpi_init(int platform_timer_count)
{
	struct arch_timer_mem *timers, *timer;
1544
	struct arch_timer_mem_frame *frame, *best_frame = NULL;
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559
	int timer_count, i, ret = 0;

	timers = kcalloc(platform_timer_count, sizeof(*timers),
			    GFP_KERNEL);
	if (!timers)
		return -ENOMEM;

	ret = acpi_arch_timer_mem_init(timers, &timer_count);
	if (ret || !timer_count)
		goto out;

	/*
	 * While unlikely, it's theoretically possible that none of the frames
	 * in a timer expose the combination of feature we want.
	 */
1560
	for (i = 0; i < timer_count; i++) {
1561 1562 1563
		timer = &timers[i];

		frame = arch_timer_mem_find_best_frame(timer);
1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
		if (!best_frame)
			best_frame = frame;

		ret = arch_timer_mem_verify_cntfrq(timer);
		if (ret) {
			pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
			goto out;
		}

		if (!best_frame) /* implies !frame */
			/*
			 * Only complain about missing suitable frames if we
			 * haven't already found one in a previous iteration.
			 */
			pr_err("Unable to find a suitable frame in timer @ %pa\n",
				&timer->cntctlbase);
1580 1581
	}

1582 1583
	if (best_frame)
		ret = arch_timer_mem_frame_register(best_frame);
1584 1585 1586 1587 1588 1589
out:
	kfree(timers);
	return ret;
}

/* Initialize per-processor generic timer and memory-mapped timer(if present) */
1590 1591
static int __init arch_timer_acpi_init(struct acpi_table_header *table)
{
1592
	int ret, platform_timer_count;
1593

1594
	if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1595
		pr_warn("already initialized, skipping\n");
1596 1597 1598
		return -EINVAL;
	}

1599
	arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1600

1601
	ret = acpi_gtdt_init(table, &platform_timer_count);
1602
	if (ret)
1603
		return ret;
1604

1605
	arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1606
		acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1607

1608
	arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1609
		acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1610

1611
	arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1612
		acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1613

1614
	arch_timer_populate_kvm_info();
1615

1616 1617 1618 1619 1620
	/*
	 * When probing via ACPI, we have no mechanism to override the sysreg
	 * CNTFRQ value. This *must* be correct.
	 */
	arch_timer_rate = arch_timer_get_cntfrq();
1621 1622
	ret = validate_timer_rate();
	if (ret) {
1623
		pr_err(FW_BUG "frequency not available.\n");
1624
		return ret;
1625
	}
1626

1627 1628 1629 1630 1631 1632
	arch_timer_uses_ppi = arch_timer_select_ppi();
	if (!arch_timer_ppi[arch_timer_uses_ppi]) {
		pr_err("No interrupt available, giving up\n");
		return -EINVAL;
	}

1633
	/* Always-on capability */
1634
	arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1635

1636 1637 1638
	/* Check for globally applicable workarounds */
	arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);

1639 1640 1641 1642
	ret = arch_timer_register();
	if (ret)
		return ret;

1643 1644 1645 1646
	if (platform_timer_count &&
	    arch_timer_mem_acpi_init(platform_timer_count))
		pr_err("Failed to initialize memory-mapped timer.\n");

1647
	return arch_timer_common_init();
1648
}
1649
TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1650
#endif