tegra-smmu.c 22 KB
Newer Older
1
/*
2
 * Copyright (C) 2011-2014 NVIDIA CORPORATION.  All rights reserved.
3
 *
4 5 6
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
7 8
 */

9
#include <linux/bitops.h>
10
#include <linux/debugfs.h>
11
#include <linux/err.h>
12
#include <linux/iommu.h>
13
#include <linux/kernel.h>
14
#include <linux/of.h>
15 16 17
#include <linux/of_device.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
18
#include <linux/dma-mapping.h>
19 20

#include <soc/tegra/ahb.h>
21
#include <soc/tegra/mc.h>
22

23 24 25
struct tegra_smmu {
	void __iomem *regs;
	struct device *dev;
26

27 28
	struct tegra_mc *mc;
	const struct tegra_smmu_soc *soc;
29

30
	unsigned long pfn_mask;
31
	unsigned long tlb_mask;
32

33 34
	unsigned long *asids;
	struct mutex lock;
35

36
	struct list_head list;
37 38

	struct dentry *debugfs;
39 40
};

41
struct tegra_smmu_as {
42
	struct iommu_domain domain;
43 44
	struct tegra_smmu *smmu;
	unsigned int use_count;
45
	u32 *count;
46
	struct page **pts;
47
	struct page *pd;
48
	dma_addr_t pd_dma;
49 50
	unsigned id;
	u32 attr;
51 52
};

53 54 55 56 57
static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom)
{
	return container_of(dom, struct tegra_smmu_as, domain);
}

58 59 60 61 62
static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,
			       unsigned long offset)
{
	writel(value, smmu->regs + offset);
}
63

64 65 66 67
static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)
{
	return readl(smmu->regs + offset);
}
68

69 70
#define SMMU_CONFIG 0x010
#define  SMMU_CONFIG_ENABLE (1 << 0)
71

72 73 74
#define SMMU_TLB_CONFIG 0x14
#define  SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29)
#define  SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28)
75 76
#define  SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \
	((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
77

78 79 80 81
#define SMMU_PTC_CONFIG 0x18
#define  SMMU_PTC_CONFIG_ENABLE (1 << 29)
#define  SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24)
#define  SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f)
82

83 84
#define SMMU_PTB_ASID 0x01c
#define  SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)
85

86
#define SMMU_PTB_DATA 0x020
87
#define  SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr))
88

89
#define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr))
90

91 92 93 94 95 96 97 98 99 100
#define SMMU_TLB_FLUSH 0x030
#define  SMMU_TLB_FLUSH_VA_MATCH_ALL     (0 << 0)
#define  SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0)
#define  SMMU_TLB_FLUSH_VA_MATCH_GROUP   (3 << 0)
#define  SMMU_TLB_FLUSH_ASID(x)          (((x) & 0x7f) << 24)
#define  SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \
					  SMMU_TLB_FLUSH_VA_MATCH_SECTION)
#define  SMMU_TLB_FLUSH_VA_GROUP(addr)   ((((addr) & 0xffffc000) >> 12) | \
					  SMMU_TLB_FLUSH_VA_MATCH_GROUP)
#define  SMMU_TLB_FLUSH_ASID_MATCH       (1 << 31)
101

102 103 104
#define SMMU_PTC_FLUSH 0x034
#define  SMMU_PTC_FLUSH_TYPE_ALL (0 << 0)
#define  SMMU_PTC_FLUSH_TYPE_ADR (1 << 0)
105

106 107
#define SMMU_PTC_FLUSH_HI 0x9b8
#define  SMMU_PTC_FLUSH_HI_MASK 0x3
108

109 110 111 112
/* per-SWGROUP SMMU_*_ASID register */
#define SMMU_ASID_ENABLE (1 << 31)
#define SMMU_ASID_MASK 0x7f
#define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK)
113

114 115 116
/* page table definitions */
#define SMMU_NUM_PDE 1024
#define SMMU_NUM_PTE 1024
117

118 119
#define SMMU_SIZE_PD (SMMU_NUM_PDE * 4)
#define SMMU_SIZE_PT (SMMU_NUM_PTE * 4)
120

121 122
#define SMMU_PDE_SHIFT 22
#define SMMU_PTE_SHIFT 12
123

124 125 126
#define SMMU_PD_READABLE	(1 << 31)
#define SMMU_PD_WRITABLE	(1 << 30)
#define SMMU_PD_NONSECURE	(1 << 29)
127

128 129 130 131
#define SMMU_PDE_READABLE	(1 << 31)
#define SMMU_PDE_WRITABLE	(1 << 30)
#define SMMU_PDE_NONSECURE	(1 << 29)
#define SMMU_PDE_NEXT		(1 << 28)
132

133 134 135
#define SMMU_PTE_READABLE	(1 << 31)
#define SMMU_PTE_WRITABLE	(1 << 30)
#define SMMU_PTE_NONSECURE	(1 << 29)
136

137 138 139 140
#define SMMU_PDE_ATTR		(SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \
				 SMMU_PDE_NONSECURE)
#define SMMU_PTE_ATTR		(SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \
				 SMMU_PTE_NONSECURE)
141

142 143 144 145 146 147 148 149 150 151
static unsigned int iova_pd_index(unsigned long iova)
{
	return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1);
}

static unsigned int iova_pt_index(unsigned long iova)
{
	return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1);
}

152
static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)
153
{
154 155 156
	addr >>= 12;
	return (addr & smmu->pfn_mask) == addr;
}
157

158 159 160
static dma_addr_t smmu_pde_to_dma(u32 pde)
{
	return pde << 12;
161 162
}

163 164 165 166 167
static void smmu_flush_ptc_all(struct tegra_smmu *smmu)
{
	smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
}

168
static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,
169
				  unsigned long offset)
170
{
171 172
	u32 value;

173
	offset &= ~(smmu->mc->soc->atom_size - 1);
174

175
	if (smmu->mc->soc->num_address_bits > 32) {
176 177
#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
		value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK;
178
#else
179
		value = 0;
180
#endif
181
		smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
182
	}
183

184
	value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
185
	smmu_writel(smmu, value, SMMU_PTC_FLUSH);
186 187
}

188
static inline void smmu_flush_tlb(struct tegra_smmu *smmu)
189
{
190
	smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);
191 192
}

193 194
static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,
				       unsigned long asid)
195
{
196
	u32 value;
197

198 199 200
	value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) |
		SMMU_TLB_FLUSH_VA_MATCH_ALL;
	smmu_writel(smmu, value, SMMU_TLB_FLUSH);
201 202
}

203 204 205
static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,
					  unsigned long asid,
					  unsigned long iova)
206
{
207
	u32 value;
208

209 210 211
	value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) |
		SMMU_TLB_FLUSH_VA_SECTION(iova);
	smmu_writel(smmu, value, SMMU_TLB_FLUSH);
212 213
}

214 215 216
static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,
					unsigned long asid,
					unsigned long iova)
217
{
218
	u32 value;
219

220 221 222
	value = SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_ASID(asid) |
		SMMU_TLB_FLUSH_VA_GROUP(iova);
	smmu_writel(smmu, value, SMMU_TLB_FLUSH);
223 224
}

225
static inline void smmu_flush(struct tegra_smmu *smmu)
226
{
227
	smmu_readl(smmu, SMMU_CONFIG);
228 229
}

230
static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)
231
{
232
	unsigned long id;
233

234
	mutex_lock(&smmu->lock);
235

236 237 238 239
	id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);
	if (id >= smmu->soc->num_asids) {
		mutex_unlock(&smmu->lock);
		return -ENOSPC;
240 241
	}

242 243 244 245 246
	set_bit(id, smmu->asids);
	*idp = id;

	mutex_unlock(&smmu->lock);
	return 0;
247 248
}

249
static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)
250
{
251 252 253
	mutex_lock(&smmu->lock);
	clear_bit(id, smmu->asids);
	mutex_unlock(&smmu->lock);
254
}
255 256

static bool tegra_smmu_capable(enum iommu_cap cap)
257
{
258
	return false;
259 260
}

261
static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
262
{
263
	struct tegra_smmu_as *as;
264

265 266 267
	if (type != IOMMU_DOMAIN_UNMANAGED)
		return NULL;

268 269
	as = kzalloc(sizeof(*as), GFP_KERNEL);
	if (!as)
270
		return NULL;
271

272
	as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;
273

274
	as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
275 276
	if (!as->pd) {
		kfree(as);
277
		return NULL;
278
	}
279

280
	as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);
281 282 283
	if (!as->count) {
		__free_page(as->pd);
		kfree(as);
284
		return NULL;
285
	}
286

287 288
	as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);
	if (!as->pts) {
289
		kfree(as->count);
290 291 292 293 294
		__free_page(as->pd);
		kfree(as);
		return NULL;
	}

295
	/* setup aperture */
296 297 298
	as->domain.geometry.aperture_start = 0;
	as->domain.geometry.aperture_end = 0xffffffff;
	as->domain.geometry.force_aperture = true;
299

300
	return &as->domain;
301 302
}

303
static void tegra_smmu_domain_free(struct iommu_domain *domain)
304
{
305
	struct tegra_smmu_as *as = to_smmu_as(domain);
306

307
	/* TODO: free page directory and page tables */
308

309
	kfree(as);
310 311
}

312 313
static const struct tegra_smmu_swgroup *
tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)
314
{
315 316
	const struct tegra_smmu_swgroup *group = NULL;
	unsigned int i;
317

318 319 320 321 322 323
	for (i = 0; i < smmu->soc->num_swgroups; i++) {
		if (smmu->soc->swgroups[i].swgroup == swgroup) {
			group = &smmu->soc->swgroups[i];
			break;
		}
	}
324

325
	return group;
326 327
}

328 329
static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,
			      unsigned int asid)
330
{
331 332 333
	const struct tegra_smmu_swgroup *group;
	unsigned int i;
	u32 value;
334

335 336
	for (i = 0; i < smmu->soc->num_clients; i++) {
		const struct tegra_mc_client *client = &smmu->soc->clients[i];
337

338 339
		if (client->swgroup != swgroup)
			continue;
340

341 342 343 344
		value = smmu_readl(smmu, client->smmu.reg);
		value |= BIT(client->smmu.bit);
		smmu_writel(smmu, value, client->smmu.reg);
	}
345

346 347 348 349 350 351 352 353
	group = tegra_smmu_find_swgroup(smmu, swgroup);
	if (group) {
		value = smmu_readl(smmu, group->reg);
		value &= ~SMMU_ASID_MASK;
		value |= SMMU_ASID_VALUE(asid);
		value |= SMMU_ASID_ENABLE;
		smmu_writel(smmu, value, group->reg);
	}
354 355
}

356 357
static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,
			       unsigned int asid)
358
{
359 360 361
	const struct tegra_smmu_swgroup *group;
	unsigned int i;
	u32 value;
362

363 364 365 366 367 368 369 370
	group = tegra_smmu_find_swgroup(smmu, swgroup);
	if (group) {
		value = smmu_readl(smmu, group->reg);
		value &= ~SMMU_ASID_MASK;
		value |= SMMU_ASID_VALUE(asid);
		value &= ~SMMU_ASID_ENABLE;
		smmu_writel(smmu, value, group->reg);
	}
371

372 373
	for (i = 0; i < smmu->soc->num_clients; i++) {
		const struct tegra_mc_client *client = &smmu->soc->clients[i];
374

375 376
		if (client->swgroup != swgroup)
			continue;
377

378 379 380 381
		value = smmu_readl(smmu, client->smmu.reg);
		value &= ~BIT(client->smmu.bit);
		smmu_writel(smmu, value, client->smmu.reg);
	}
382 383
}

384 385
static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,
				 struct tegra_smmu_as *as)
386
{
387
	u32 value;
388 389
	int err;

390 391 392
	if (as->use_count > 0) {
		as->use_count++;
		return 0;
393 394
	}

395 396 397 398 399 400 401 402 403 404 405
	as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,
				  DMA_TO_DEVICE);
	if (dma_mapping_error(smmu->dev, as->pd_dma))
		return -ENOMEM;

	/* We can't handle 64-bit DMA addresses */
	if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {
		err = -ENOMEM;
		goto err_unmap;
	}

406 407
	err = tegra_smmu_alloc_asid(smmu, &as->id);
	if (err < 0)
408
		goto err_unmap;
409

410
	smmu_flush_ptc(smmu, as->pd_dma, 0);
411
	smmu_flush_tlb_asid(smmu, as->id);
412

413
	smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);
414
	value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);
415 416
	smmu_writel(smmu, value, SMMU_PTB_DATA);
	smmu_flush(smmu);
417

418 419
	as->smmu = smmu;
	as->use_count++;
420

421
	return 0;
422 423 424 425

err_unmap:
	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
	return err;
426 427
}

428 429
static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
				    struct tegra_smmu_as *as)
430
{
431 432 433 434
	if (--as->use_count > 0)
		return;

	tegra_smmu_free_asid(smmu, as->id);
435 436 437

	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);

438
	as->smmu = NULL;
439 440
}

441 442
static int tegra_smmu_attach_dev(struct iommu_domain *domain,
				 struct device *dev)
443
{
444
	struct tegra_smmu *smmu = dev->archdata.iommu;
445
	struct tegra_smmu_as *as = to_smmu_as(domain);
446 447 448 449
	struct device_node *np = dev->of_node;
	struct of_phandle_args args;
	unsigned int index = 0;
	int err = 0;
450

451 452 453
	while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
					   &args)) {
		unsigned int swgroup = args.args[0];
454

455 456
		if (args.np != smmu->dev->of_node) {
			of_node_put(args.np);
457
			continue;
458
		}
459

460
		of_node_put(args.np);
461

462 463 464 465 466 467
		err = tegra_smmu_as_prepare(smmu, as);
		if (err < 0)
			return err;

		tegra_smmu_enable(smmu, swgroup, as->id);
		index++;
468 469
	}

470 471
	if (index == 0)
		return -ENODEV;
472

473 474
	return 0;
}
475

476 477
static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev)
{
478
	struct tegra_smmu_as *as = to_smmu_as(domain);
479 480 481 482
	struct device_node *np = dev->of_node;
	struct tegra_smmu *smmu = as->smmu;
	struct of_phandle_args args;
	unsigned int index = 0;
483

484 485 486
	while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
					   &args)) {
		unsigned int swgroup = args.args[0];
487

488 489 490 491
		if (args.np != smmu->dev->of_node) {
			of_node_put(args.np);
			continue;
		}
492

493
		of_node_put(args.np);
494

495 496 497 498
		tegra_smmu_disable(smmu, swgroup, as->id);
		tegra_smmu_as_unprepare(smmu, as);
		index++;
	}
499 500
}

501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,
			       u32 value)
{
	unsigned int pd_index = iova_pd_index(iova);
	struct tegra_smmu *smmu = as->smmu;
	u32 *pd = page_address(as->pd);
	unsigned long offset = pd_index * sizeof(*pd);

	/* Set the page directory entry first */
	pd[pd_index] = value;

	/* The flush the page directory entry from caches */
	dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,
					 sizeof(*pd), DMA_TO_DEVICE);

	/* And flush the iommu */
	smmu_flush_ptc(smmu, as->pd_dma, offset);
	smmu_flush_tlb_section(smmu, as->id, iova);
	smmu_flush(smmu);
}

522 523 524 525 526 527 528 529
static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova)
{
	u32 *pt = page_address(pt_page);

	return pt + iova_pt_index(iova);
}

static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,
530
				  dma_addr_t *dmap)
531 532 533
{
	unsigned int pd_index = iova_pd_index(iova);
	struct page *pt_page;
534
	u32 *pd;
535

536 537
	pt_page = as->pts[pd_index];
	if (!pt_page)
538 539
		return NULL;

540 541
	pd = page_address(as->pd);
	*dmap = smmu_pde_to_dma(pd[pd_index]);
542 543 544 545

	return tegra_smmu_pte_offset(pt_page, iova);
}

546
static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
547
		       dma_addr_t *dmap)
548
{
549
	unsigned int pde = iova_pd_index(iova);
550 551
	struct tegra_smmu *smmu = as->smmu;

552
	if (!as->pts[pde]) {
553 554 555
		struct page *page;
		dma_addr_t dma;

556
		page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
557 558
		if (!page)
			return NULL;
559

560 561 562 563 564 565 566 567 568 569 570 571 572 573
		dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,
				   DMA_TO_DEVICE);
		if (dma_mapping_error(smmu->dev, dma)) {
			__free_page(page);
			return NULL;
		}

		if (!smmu_dma_addr_valid(smmu, dma)) {
			dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,
				       DMA_TO_DEVICE);
			__free_page(page);
			return NULL;
		}

574 575
		as->pts[pde] = page;

576 577
		tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |
							      SMMU_PDE_NEXT));
578 579

		*dmap = dma;
580
	} else {
581 582
		u32 *pd = page_address(as->pd);

583
		*dmap = smmu_pde_to_dma(pd[pde]);
584 585
	}

586 587
	return tegra_smmu_pte_offset(as->pts[pde], iova);
}
588

589 590 591
static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)
{
	unsigned int pd_index = iova_pd_index(iova);
592

593
	as->count[pd_index]++;
594
}
595

596
static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)
597
{
598
	unsigned int pde = iova_pd_index(iova);
599
	struct page *page = as->pts[pde];
600

601 602 603 604
	/*
	 * When no entries in this page table are used anymore, return the
	 * memory page to the system.
	 */
605
	if (--as->count[pde] == 0) {
606 607
		struct tegra_smmu *smmu = as->smmu;
		u32 *pd = page_address(as->pd);
608
		dma_addr_t pte_dma = smmu_pde_to_dma(pd[pde]);
609

610
		tegra_smmu_set_pde(as, iova, 0);
611

612
		dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);
613
		__free_page(page);
614
		as->pts[pde] = NULL;
615 616 617
	}
}

618
static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,
619
			       u32 *pte, dma_addr_t pte_dma, u32 val)
620 621 622 623 624 625
{
	struct tegra_smmu *smmu = as->smmu;
	unsigned long offset = offset_in_page(pte);

	*pte = val;

626 627 628
	dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,
					 4, DMA_TO_DEVICE);
	smmu_flush_ptc(smmu, pte_dma, offset);
629 630 631 632
	smmu_flush_tlb_group(smmu, as->id, iova);
	smmu_flush(smmu);
}

633 634
static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
			  phys_addr_t paddr, size_t size, int prot)
635
{
636
	struct tegra_smmu_as *as = to_smmu_as(domain);
637
	dma_addr_t pte_dma;
638
	u32 *pte;
639

640
	pte = as_get_pte(as, iova, &pte_dma);
641 642
	if (!pte)
		return -ENOMEM;
643

644 645 646 647
	/* If we aren't overwriting a pre-existing entry, increment use */
	if (*pte == 0)
		tegra_smmu_pte_get_use(as, iova);

648
	tegra_smmu_set_pte(as, iova, pte, pte_dma,
649
			   __phys_to_pfn(paddr) | SMMU_PTE_ATTR);
650 651 652 653

	return 0;
}

654 655
static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
			       size_t size)
656
{
657
	struct tegra_smmu_as *as = to_smmu_as(domain);
658
	dma_addr_t pte_dma;
659
	u32 *pte;
660

661
	pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
662
	if (!pte || !*pte)
663
		return 0;
664

665
	tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);
666 667
	tegra_smmu_pte_put_use(as, iova);

668
	return size;
669 670
}

671 672
static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain,
					   dma_addr_t iova)
673
{
674
	struct tegra_smmu_as *as = to_smmu_as(domain);
675
	unsigned long pfn;
676
	dma_addr_t pte_dma;
677
	u32 *pte;
678

679
	pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
680 681 682
	if (!pte || !*pte)
		return 0;

683
	pfn = *pte & as->smmu->pfn_mask;
684

685
	return PFN_PHYS(pfn);
686 687
}

688
static struct tegra_smmu *tegra_smmu_find(struct device_node *np)
689
{
690 691
	struct platform_device *pdev;
	struct tegra_mc *mc;
692

693 694 695 696 697 698 699 700 701
	pdev = of_find_device_by_node(np);
	if (!pdev)
		return NULL;

	mc = platform_get_drvdata(pdev);
	if (!mc)
		return NULL;

	return mc->smmu;
702 703
}

704
static int tegra_smmu_add_device(struct device *dev)
705
{
706
	struct device_node *np = dev->of_node;
707
	struct iommu_group *group;
708 709
	struct of_phandle_args args;
	unsigned int index = 0;
710

711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
	while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,
					  &args) == 0) {
		struct tegra_smmu *smmu;

		smmu = tegra_smmu_find(args.np);
		if (smmu) {
			/*
			 * Only a single IOMMU master interface is currently
			 * supported by the Linux kernel, so abort after the
			 * first match.
			 */
			dev->archdata.iommu = smmu;
			break;
		}

		index++;
	}

729 730 731 732 733 734
	group = iommu_group_get_for_dev(dev);
	if (IS_ERR(group))
		return PTR_ERR(group);

	iommu_group_put(group);

735
	return 0;
736 737
}

738
static void tegra_smmu_remove_device(struct device *dev)
739
{
740
	dev->archdata.iommu = NULL;
741
	iommu_group_remove_device(dev);
742
}
743

744 745
static const struct iommu_ops tegra_smmu_ops = {
	.capable = tegra_smmu_capable,
746 747
	.domain_alloc = tegra_smmu_domain_alloc,
	.domain_free = tegra_smmu_domain_free,
748 749 750 751
	.attach_dev = tegra_smmu_attach_dev,
	.detach_dev = tegra_smmu_detach_dev,
	.add_device = tegra_smmu_add_device,
	.remove_device = tegra_smmu_remove_device,
752
	.device_group = generic_device_group,
753 754 755 756
	.map = tegra_smmu_map,
	.unmap = tegra_smmu_unmap,
	.map_sg = default_iommu_map_sg,
	.iova_to_phys = tegra_smmu_iova_to_phys,
757

758 759
	.pgsize_bitmap = SZ_4K,
};
760

761 762 763 764 765 766 767
static void tegra_smmu_ahb_enable(void)
{
	static const struct of_device_id ahb_match[] = {
		{ .compatible = "nvidia,tegra30-ahb", },
		{ }
	};
	struct device_node *ahb;
768

769 770 771 772
	ahb = of_find_matching_node(NULL, ahb_match);
	if (ahb) {
		tegra_ahb_enable_smmu(ahb);
		of_node_put(ahb);
773
	}
774
}
775

776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872
static int tegra_smmu_swgroups_show(struct seq_file *s, void *data)
{
	struct tegra_smmu *smmu = s->private;
	unsigned int i;
	u32 value;

	seq_printf(s, "swgroup    enabled  ASID\n");
	seq_printf(s, "------------------------\n");

	for (i = 0; i < smmu->soc->num_swgroups; i++) {
		const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];
		const char *status;
		unsigned int asid;

		value = smmu_readl(smmu, group->reg);

		if (value & SMMU_ASID_ENABLE)
			status = "yes";
		else
			status = "no";

		asid = value & SMMU_ASID_MASK;

		seq_printf(s, "%-9s  %-7s  %#04x\n", group->name, status,
			   asid);
	}

	return 0;
}

static int tegra_smmu_swgroups_open(struct inode *inode, struct file *file)
{
	return single_open(file, tegra_smmu_swgroups_show, inode->i_private);
}

static const struct file_operations tegra_smmu_swgroups_fops = {
	.open = tegra_smmu_swgroups_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static int tegra_smmu_clients_show(struct seq_file *s, void *data)
{
	struct tegra_smmu *smmu = s->private;
	unsigned int i;
	u32 value;

	seq_printf(s, "client       enabled\n");
	seq_printf(s, "--------------------\n");

	for (i = 0; i < smmu->soc->num_clients; i++) {
		const struct tegra_mc_client *client = &smmu->soc->clients[i];
		const char *status;

		value = smmu_readl(smmu, client->smmu.reg);

		if (value & BIT(client->smmu.bit))
			status = "yes";
		else
			status = "no";

		seq_printf(s, "%-12s %s\n", client->name, status);
	}

	return 0;
}

static int tegra_smmu_clients_open(struct inode *inode, struct file *file)
{
	return single_open(file, tegra_smmu_clients_show, inode->i_private);
}

static const struct file_operations tegra_smmu_clients_fops = {
	.open = tegra_smmu_clients_open,
	.read = seq_read,
	.llseek = seq_lseek,
	.release = single_release,
};

static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)
{
	smmu->debugfs = debugfs_create_dir("smmu", NULL);
	if (!smmu->debugfs)
		return;

	debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,
			    &tegra_smmu_swgroups_fops);
	debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,
			    &tegra_smmu_clients_fops);
}

static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)
{
	debugfs_remove_recursive(smmu->debugfs);
}

873 874 875 876 877 878 879 880
struct tegra_smmu *tegra_smmu_probe(struct device *dev,
				    const struct tegra_smmu_soc *soc,
				    struct tegra_mc *mc)
{
	struct tegra_smmu *smmu;
	size_t size;
	u32 value;
	int err;
881

882 883 884
	/* This can happen on Tegra20 which doesn't have an SMMU */
	if (!soc)
		return NULL;
885

886 887 888
	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
	if (!smmu)
		return ERR_PTR(-ENOMEM);
889

890 891 892 893 894 895 896 897 898
	/*
	 * This is a bit of a hack. Ideally we'd want to simply return this
	 * value. However the IOMMU registration process will attempt to add
	 * all devices to the IOMMU when bus_set_iommu() is called. In order
	 * not to rely on global variables to track the IOMMU instance, we
	 * set it here so that it can be looked up from the .add_device()
	 * callback via the IOMMU device's .drvdata field.
	 */
	mc->smmu = smmu;
899

900
	size = BITS_TO_LONGS(soc->num_asids) * sizeof(long);
901

902 903 904
	smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);
	if (!smmu->asids)
		return ERR_PTR(-ENOMEM);
905

906
	mutex_init(&smmu->lock);
907

908 909 910 911
	smmu->regs = mc->regs;
	smmu->soc = soc;
	smmu->dev = dev;
	smmu->mc = mc;
912

913 914 915
	smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1;
	dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n",
		mc->soc->num_address_bits, smmu->pfn_mask);
916 917 918
	smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1;
	dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,
		smmu->tlb_mask);
919

920
	value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f);
921

922 923
	if (soc->supports_request_limit)
		value |= SMMU_PTC_CONFIG_REQ_LIMIT(8);
924

925
	smmu_writel(smmu, value, SMMU_PTC_CONFIG);
926

927
	value = SMMU_TLB_CONFIG_HIT_UNDER_MISS |
928
		SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);
929

930 931
	if (soc->supports_round_robin_arbitration)
		value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION;
932

933
	smmu_writel(smmu, value, SMMU_TLB_CONFIG);
934

935
	smmu_flush_ptc_all(smmu);
936 937 938 939 940
	smmu_flush_tlb(smmu);
	smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
	smmu_flush(smmu);

	tegra_smmu_ahb_enable();
941

942 943 944
	err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops);
	if (err < 0)
		return ERR_PTR(err);
945

946 947 948
	if (IS_ENABLED(CONFIG_DEBUG_FS))
		tegra_smmu_debugfs_init(smmu);

949 950
	return smmu;
}
951 952 953 954 955 956

void tegra_smmu_remove(struct tegra_smmu *smmu)
{
	if (IS_ENABLED(CONFIG_DEBUG_FS))
		tegra_smmu_debugfs_exit(smmu);
}