rockchip-iommu.c 33 KB
Newer Older
1 2 3 4 5 6 7 8 9
/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/compiler.h>
#include <linux/delay.h>
#include <linux/device.h>
10
#include <linux/dma-iommu.h>
11
#include <linux/dma-mapping.h>
12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
#include <linux/errno.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/iommu.h>
#include <linux/jiffies.h>
#include <linux/list.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/spinlock.h>

/** MMU register offsets */
#define RK_MMU_DTE_ADDR		0x00	/* Directory table address */
#define RK_MMU_STATUS		0x04
#define RK_MMU_COMMAND		0x08
#define RK_MMU_PAGE_FAULT_ADDR	0x0C	/* IOVA of last page fault */
#define RK_MMU_ZAP_ONE_LINE	0x10	/* Shootdown one IOTLB entry */
#define RK_MMU_INT_RAWSTAT	0x14	/* IRQ status ignoring mask */
#define RK_MMU_INT_CLEAR	0x18	/* Acknowledge and re-arm irq */
#define RK_MMU_INT_MASK		0x1C	/* IRQ enable */
#define RK_MMU_INT_STATUS	0x20	/* IRQ status after masking */
#define RK_MMU_AUTO_GATING	0x24

#define DTE_ADDR_DUMMY		0xCAFEBABE
#define FORCE_RESET_TIMEOUT	100	/* ms */

/* RK_MMU_STATUS fields */
#define RK_MMU_STATUS_PAGING_ENABLED       BIT(0)
#define RK_MMU_STATUS_PAGE_FAULT_ACTIVE    BIT(1)
#define RK_MMU_STATUS_STALL_ACTIVE         BIT(2)
#define RK_MMU_STATUS_IDLE                 BIT(3)
#define RK_MMU_STATUS_REPLAY_BUFFER_EMPTY  BIT(4)
#define RK_MMU_STATUS_PAGE_FAULT_IS_WRITE  BIT(5)
#define RK_MMU_STATUS_STALL_NOT_ACTIVE     BIT(31)

/* RK_MMU_COMMAND command values */
#define RK_MMU_CMD_ENABLE_PAGING    0  /* Enable memory translation */
#define RK_MMU_CMD_DISABLE_PAGING   1  /* Disable memory translation */
#define RK_MMU_CMD_ENABLE_STALL     2  /* Stall paging to allow other cmds */
#define RK_MMU_CMD_DISABLE_STALL    3  /* Stop stall re-enables paging */
#define RK_MMU_CMD_ZAP_CACHE        4  /* Shoot down entire IOTLB */
#define RK_MMU_CMD_PAGE_FAULT_DONE  5  /* Clear page fault */
#define RK_MMU_CMD_FORCE_RESET      6  /* Reset all registers */

/* RK_MMU_INT_* register fields */
#define RK_MMU_IRQ_PAGE_FAULT    0x01  /* page fault */
#define RK_MMU_IRQ_BUS_ERROR     0x02  /* bus read error */
#define RK_MMU_IRQ_MASK          (RK_MMU_IRQ_PAGE_FAULT | RK_MMU_IRQ_BUS_ERROR)

#define NUM_DT_ENTRIES 1024
#define NUM_PT_ENTRIES 1024

#define SPAGE_ORDER 12
#define SPAGE_SIZE (1 << SPAGE_ORDER)

 /*
  * Support mapping any size that fits in one page table:
  *   4 KiB to 4 MiB
  */
#define RK_IOMMU_PGSIZE_BITMAP 0x007ff000

#define IOMMU_REG_POLL_COUNT_FAST 1000

struct rk_iommu_domain {
	struct list_head iommus;
80
	struct platform_device *pdev;
81
	u32 *dt; /* page directory table */
82
	dma_addr_t dt_dma;
83 84
	spinlock_t iommus_lock; /* lock for iommus list */
	spinlock_t dt_lock; /* lock for modifying page directory table */
85 86

	struct iommu_domain domain;
87 88 89 90
};

struct rk_iommu {
	struct device *dev;
91 92
	void __iomem **bases;
	int num_mmu;
93
	int irq;
94
	struct iommu_device iommu;
95 96 97 98
	struct list_head node; /* entry in rk_iommu_domain.iommus */
	struct iommu_domain *domain; /* domain to which iommu is attached */
};

99 100
static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma,
				  unsigned int count)
101
{
102
	size_t size = count * sizeof(u32); /* count of u32 entry */
103

104
	dma_sync_single_for_device(&dom->pdev->dev, dma, size, DMA_TO_DEVICE);
105 106
}

107 108 109 110 111
static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom)
{
	return container_of(dom, struct rk_iommu_domain, domain);
}

112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
/**
 * Inspired by _wait_for in intel_drv.h
 * This is NOT safe for use in interrupt context.
 *
 * Note that it's important that we check the condition again after having
 * timed out, since the timeout could be due to preemption or similar and
 * we've never had a chance to check the condition before the timeout.
 */
#define rk_wait_for(COND, MS) ({ \
	unsigned long timeout__ = jiffies + msecs_to_jiffies(MS) + 1;	\
	int ret__ = 0;							\
	while (!(COND)) {						\
		if (time_after(jiffies, timeout__)) {			\
			ret__ = (COND) ? 0 : -ETIMEDOUT;		\
			break;						\
		}							\
		usleep_range(50, 100);					\
	}								\
	ret__;								\
})

/*
 * The Rockchip rk3288 iommu uses a 2-level page table.
 * The first level is the "Directory Table" (DT).
 * The DT consists of 1024 4-byte Directory Table Entries (DTEs), each pointing
 * to a "Page Table".
 * The second level is the 1024 Page Tables (PT).
 * Each PT consists of 1024 4-byte Page Table Entries (PTEs), each pointing to
 * a 4 KB page of physical memory.
 *
 * The DT and each PT fits in a single 4 KB page (4-bytes * 1024 entries).
 * Each iommu device has a MMU_DTE_ADDR register that contains the physical
 * address of the start of the DT page.
 *
 * The structure of the page table is as follows:
 *
 *                   DT
 * MMU_DTE_ADDR -> +-----+
 *                 |     |
 *                 +-----+     PT
 *                 | DTE | -> +-----+
 *                 +-----+    |     |     Memory
 *                 |     |    +-----+     Page
 *                 |     |    | PTE | -> +-----+
 *                 +-----+    +-----+    |     |
 *                            |     |    |     |
 *                            |     |    |     |
 *                            +-----+    |     |
 *                                       |     |
 *                                       |     |
 *                                       +-----+
 */

/*
 * Each DTE has a PT address and a valid bit:
 * +---------------------+-----------+-+
 * | PT address          | Reserved  |V|
 * +---------------------+-----------+-+
 *  31:12 - PT address (PTs always starts on a 4 KB boundary)
 *  11: 1 - Reserved
 *      0 - 1 if PT @ PT address is valid
 */
#define RK_DTE_PT_ADDRESS_MASK    0xfffff000
#define RK_DTE_PT_VALID           BIT(0)

static inline phys_addr_t rk_dte_pt_address(u32 dte)
{
	return (phys_addr_t)dte & RK_DTE_PT_ADDRESS_MASK;
}

static inline bool rk_dte_is_pt_valid(u32 dte)
{
	return dte & RK_DTE_PT_VALID;
}

187
static inline u32 rk_mk_dte(dma_addr_t pt_dma)
188
{
189
	return (pt_dma & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID;
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
}

/*
 * Each PTE has a Page address, some flags and a valid bit:
 * +---------------------+---+-------+-+
 * | Page address        |Rsv| Flags |V|
 * +---------------------+---+-------+-+
 *  31:12 - Page address (Pages always start on a 4 KB boundary)
 *  11: 9 - Reserved
 *   8: 1 - Flags
 *      8 - Read allocate - allocate cache space on read misses
 *      7 - Read cache - enable cache & prefetch of data
 *      6 - Write buffer - enable delaying writes on their way to memory
 *      5 - Write allocate - allocate cache space on write misses
 *      4 - Write cache - different writes can be merged together
 *      3 - Override cache attributes
 *          if 1, bits 4-8 control cache attributes
 *          if 0, the system bus defaults are used
 *      2 - Writable
 *      1 - Readable
 *      0 - 1 if Page @ Page address is valid
 */
#define RK_PTE_PAGE_ADDRESS_MASK  0xfffff000
#define RK_PTE_PAGE_FLAGS_MASK    0x000001fe
#define RK_PTE_PAGE_WRITABLE      BIT(2)
#define RK_PTE_PAGE_READABLE      BIT(1)
#define RK_PTE_PAGE_VALID         BIT(0)

static inline phys_addr_t rk_pte_page_address(u32 pte)
{
	return (phys_addr_t)pte & RK_PTE_PAGE_ADDRESS_MASK;
}

static inline bool rk_pte_is_page_valid(u32 pte)
{
	return pte & RK_PTE_PAGE_VALID;
}

/* TODO: set cache flags per prot IOMMU_CACHE */
static u32 rk_mk_pte(phys_addr_t page, int prot)
{
	u32 flags = 0;
	flags |= (prot & IOMMU_READ) ? RK_PTE_PAGE_READABLE : 0;
	flags |= (prot & IOMMU_WRITE) ? RK_PTE_PAGE_WRITABLE : 0;
	page &= RK_PTE_PAGE_ADDRESS_MASK;
	return page | flags | RK_PTE_PAGE_VALID;
}

static u32 rk_mk_pte_invalid(u32 pte)
{
	return pte & ~RK_PTE_PAGE_VALID;
}

/*
 * rk3288 iova (IOMMU Virtual Address) format
 *  31       22.21       12.11          0
 * +-----------+-----------+-------------+
 * | DTE index | PTE index | Page offset |
 * +-----------+-----------+-------------+
 *  31:22 - DTE index   - index of DTE in DT
 *  21:12 - PTE index   - index of PTE in PT @ DTE.pt_address
 *  11: 0 - Page offset - offset into page @ PTE.page_address
 */
#define RK_IOVA_DTE_MASK    0xffc00000
#define RK_IOVA_DTE_SHIFT   22
#define RK_IOVA_PTE_MASK    0x003ff000
#define RK_IOVA_PTE_SHIFT   12
#define RK_IOVA_PAGE_MASK   0x00000fff
#define RK_IOVA_PAGE_SHIFT  0

static u32 rk_iova_dte_index(dma_addr_t iova)
{
	return (u32)(iova & RK_IOVA_DTE_MASK) >> RK_IOVA_DTE_SHIFT;
}

static u32 rk_iova_pte_index(dma_addr_t iova)
{
	return (u32)(iova & RK_IOVA_PTE_MASK) >> RK_IOVA_PTE_SHIFT;
}

static u32 rk_iova_page_offset(dma_addr_t iova)
{
	return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT;
}

275
static u32 rk_iommu_read(void __iomem *base, u32 offset)
276
{
277
	return readl(base + offset);
278 279
}

280
static void rk_iommu_write(void __iomem *base, u32 offset, u32 value)
281
{
282
	writel(value, base + offset);
283 284 285 286
}

static void rk_iommu_command(struct rk_iommu *iommu, u32 command)
{
287 288 289 290
	int i;

	for (i = 0; i < iommu->num_mmu; i++)
		writel(command, iommu->bases[i] + RK_MMU_COMMAND);
291 292
}

293 294 295 296
static void rk_iommu_base_command(void __iomem *base, u32 command)
{
	writel(command, base + RK_MMU_COMMAND);
}
297 298 299
static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova,
			       size_t size)
{
300 301
	int i;

302 303 304 305 306
	dma_addr_t iova_end = iova + size;
	/*
	 * TODO(djkurtz): Figure out when it is more efficient to shootdown the
	 * entire iotlb rather than iterate over individual iovas.
	 */
307 308 309
	for (i = 0; i < iommu->num_mmu; i++)
		for (; iova < iova_end; iova += SPAGE_SIZE)
			rk_iommu_write(iommu->bases[i], RK_MMU_ZAP_ONE_LINE, iova);
310 311 312 313
}

static bool rk_iommu_is_stall_active(struct rk_iommu *iommu)
{
314 315 316 317
	bool active = true;
	int i;

	for (i = 0; i < iommu->num_mmu; i++)
318 319
		active &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) &
					   RK_MMU_STATUS_STALL_ACTIVE);
320 321

	return active;
322 323 324 325
}

static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu)
{
326 327 328 329
	bool enable = true;
	int i;

	for (i = 0; i < iommu->num_mmu; i++)
330 331
		enable &= !!(rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) &
					   RK_MMU_STATUS_PAGING_ENABLED);
332 333

	return enable;
334 335 336 337
}

static int rk_iommu_enable_stall(struct rk_iommu *iommu)
{
338
	int ret, i;
339 340 341 342 343 344 345 346 347 348 349 350

	if (rk_iommu_is_stall_active(iommu))
		return 0;

	/* Stall can only be enabled if paging is enabled */
	if (!rk_iommu_is_paging_enabled(iommu))
		return 0;

	rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_STALL);

	ret = rk_wait_for(rk_iommu_is_stall_active(iommu), 1);
	if (ret)
351 352 353
		for (i = 0; i < iommu->num_mmu; i++)
			dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n",
				rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));
354 355 356 357 358 359

	return ret;
}

static int rk_iommu_disable_stall(struct rk_iommu *iommu)
{
360
	int ret, i;
361 362 363 364 365 366 367 368

	if (!rk_iommu_is_stall_active(iommu))
		return 0;

	rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_STALL);

	ret = rk_wait_for(!rk_iommu_is_stall_active(iommu), 1);
	if (ret)
369 370 371
		for (i = 0; i < iommu->num_mmu; i++)
			dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n",
				rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));
372 373 374 375 376 377

	return ret;
}

static int rk_iommu_enable_paging(struct rk_iommu *iommu)
{
378
	int ret, i;
379 380 381 382 383 384 385 386

	if (rk_iommu_is_paging_enabled(iommu))
		return 0;

	rk_iommu_command(iommu, RK_MMU_CMD_ENABLE_PAGING);

	ret = rk_wait_for(rk_iommu_is_paging_enabled(iommu), 1);
	if (ret)
387 388 389
		for (i = 0; i < iommu->num_mmu; i++)
			dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n",
				rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));
390 391 392 393 394 395

	return ret;
}

static int rk_iommu_disable_paging(struct rk_iommu *iommu)
{
396
	int ret, i;
397 398 399 400 401 402 403 404

	if (!rk_iommu_is_paging_enabled(iommu))
		return 0;

	rk_iommu_command(iommu, RK_MMU_CMD_DISABLE_PAGING);

	ret = rk_wait_for(!rk_iommu_is_paging_enabled(iommu), 1);
	if (ret)
405 406 407
		for (i = 0; i < iommu->num_mmu; i++)
			dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n",
				rk_iommu_read(iommu->bases[i], RK_MMU_STATUS));
408 409 410 411 412 413

	return ret;
}

static int rk_iommu_force_reset(struct rk_iommu *iommu)
{
414
	int ret, i;
415 416 417 418 419 420
	u32 dte_addr;

	/*
	 * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY
	 * and verifying that upper 5 nybbles are read back.
	 */
421 422
	for (i = 0; i < iommu->num_mmu; i++) {
		rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY);
423

424 425 426 427 428
		dte_addr = rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR);
		if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) {
			dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n");
			return -EFAULT;
		}
429 430 431 432
	}

	rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET);

433 434 435 436 437 438 439 440
	for (i = 0; i < iommu->num_mmu; i++) {
		ret = rk_wait_for(rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR) == 0x00000000,
				  FORCE_RESET_TIMEOUT);
		if (ret) {
			dev_err(iommu->dev, "FORCE_RESET command timed out\n");
			return ret;
		}
	}
441

442
	return 0;
443 444
}

445
static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova)
446
{
447
	void __iomem *base = iommu->bases[index];
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462
	u32 dte_index, pte_index, page_offset;
	u32 mmu_dte_addr;
	phys_addr_t mmu_dte_addr_phys, dte_addr_phys;
	u32 *dte_addr;
	u32 dte;
	phys_addr_t pte_addr_phys = 0;
	u32 *pte_addr = NULL;
	u32 pte = 0;
	phys_addr_t page_addr_phys = 0;
	u32 page_flags = 0;

	dte_index = rk_iova_dte_index(iova);
	pte_index = rk_iova_pte_index(iova);
	page_offset = rk_iova_page_offset(iova);

463
	mmu_dte_addr = rk_iommu_read(base, RK_MMU_DTE_ADDR);
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
	mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr;

	dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index);
	dte_addr = phys_to_virt(dte_addr_phys);
	dte = *dte_addr;

	if (!rk_dte_is_pt_valid(dte))
		goto print_it;

	pte_addr_phys = rk_dte_pt_address(dte) + (pte_index * 4);
	pte_addr = phys_to_virt(pte_addr_phys);
	pte = *pte_addr;

	if (!rk_pte_is_page_valid(pte))
		goto print_it;

	page_addr_phys = rk_pte_page_address(pte) + page_offset;
	page_flags = pte & RK_PTE_PAGE_FLAGS_MASK;

print_it:
	dev_err(iommu->dev, "iova = %pad: dte_index: %#03x pte_index: %#03x page_offset: %#03x\n",
		&iova, dte_index, pte_index, page_offset);
	dev_err(iommu->dev, "mmu_dte_addr: %pa dte@%pa: %#08x valid: %u pte@%pa: %#08x valid: %u page@%pa flags: %#03x\n",
		&mmu_dte_addr_phys, &dte_addr_phys, dte,
		rk_dte_is_pt_valid(dte), &pte_addr_phys, pte,
		rk_pte_is_page_valid(pte), &page_addr_phys, page_flags);
}

static irqreturn_t rk_iommu_irq(int irq, void *dev_id)
{
	struct rk_iommu *iommu = dev_id;
	u32 status;
	u32 int_status;
	dma_addr_t iova;
498 499
	irqreturn_t ret = IRQ_NONE;
	int i;
500

501 502 503 504
	for (i = 0; i < iommu->num_mmu; i++) {
		int_status = rk_iommu_read(iommu->bases[i], RK_MMU_INT_STATUS);
		if (int_status == 0)
			continue;
505

506 507
		ret = IRQ_HANDLED;
		iova = rk_iommu_read(iommu->bases[i], RK_MMU_PAGE_FAULT_ADDR);
508

509 510
		if (int_status & RK_MMU_IRQ_PAGE_FAULT) {
			int flags;
511

512 513 514
			status = rk_iommu_read(iommu->bases[i], RK_MMU_STATUS);
			flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ?
					IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
515

516 517 518
			dev_err(iommu->dev, "Page fault at %pad of type %s\n",
				&iova,
				(flags == IOMMU_FAULT_WRITE) ? "write" : "read");
519

520
			log_iova(iommu, i, iova);
521

522 523 524 525 526 527 528 529 530 531
			/*
			 * Report page fault to any installed handlers.
			 * Ignore the return code, though, since we always zap cache
			 * and clear the page fault anyway.
			 */
			if (iommu->domain)
				report_iommu_fault(iommu->domain, iommu->dev, iova,
						   flags);
			else
				dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n");
532

533 534 535
			rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);
			rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_PAGE_FAULT_DONE);
		}
536

537 538
		if (int_status & RK_MMU_IRQ_BUS_ERROR)
			dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova);
539

540 541 542
		if (int_status & ~RK_MMU_IRQ_MASK)
			dev_err(iommu->dev, "unexpected int_status: %#08x\n",
				int_status);
543

544 545
		rk_iommu_write(iommu->bases[i], RK_MMU_INT_CLEAR, int_status);
	}
546

547
	return ret;
548 549 550 551 552
}

static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain,
					 dma_addr_t iova)
{
553
	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593
	unsigned long flags;
	phys_addr_t pt_phys, phys = 0;
	u32 dte, pte;
	u32 *page_table;

	spin_lock_irqsave(&rk_domain->dt_lock, flags);

	dte = rk_domain->dt[rk_iova_dte_index(iova)];
	if (!rk_dte_is_pt_valid(dte))
		goto out;

	pt_phys = rk_dte_pt_address(dte);
	page_table = (u32 *)phys_to_virt(pt_phys);
	pte = page_table[rk_iova_pte_index(iova)];
	if (!rk_pte_is_page_valid(pte))
		goto out;

	phys = rk_pte_page_address(pte) + rk_iova_page_offset(iova);
out:
	spin_unlock_irqrestore(&rk_domain->dt_lock, flags);

	return phys;
}

static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain,
			      dma_addr_t iova, size_t size)
{
	struct list_head *pos;
	unsigned long flags;

	/* shootdown these iova from all iommus using this domain */
	spin_lock_irqsave(&rk_domain->iommus_lock, flags);
	list_for_each(pos, &rk_domain->iommus) {
		struct rk_iommu *iommu;
		iommu = list_entry(pos, struct rk_iommu, node);
		rk_iommu_zap_lines(iommu, iova, size);
	}
	spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);
}

594 595 596 597 598 599 600 601 602
static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain,
					 dma_addr_t iova, size_t size)
{
	rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE);
	if (size > SPAGE_SIZE)
		rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE,
					SPAGE_SIZE);
}

603 604 605
static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain,
				  dma_addr_t iova)
{
606
	struct device *dev = &rk_domain->pdev->dev;
607
	u32 *page_table, *dte_addr;
608
	u32 dte_index, dte;
609
	phys_addr_t pt_phys;
610
	dma_addr_t pt_dma;
611 612 613

	assert_spin_locked(&rk_domain->dt_lock);

614 615
	dte_index = rk_iova_dte_index(iova);
	dte_addr = &rk_domain->dt[dte_index];
616 617 618 619 620 621 622 623
	dte = *dte_addr;
	if (rk_dte_is_pt_valid(dte))
		goto done;

	page_table = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
	if (!page_table)
		return ERR_PTR(-ENOMEM);

624 625 626 627 628 629
	pt_dma = dma_map_single(dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE);
	if (dma_mapping_error(dev, pt_dma)) {
		dev_err(dev, "DMA mapping error while allocating page table\n");
		free_page((unsigned long)page_table);
		return ERR_PTR(-ENOMEM);
	}
630

631 632
	dte = rk_mk_dte(pt_dma);
	*dte_addr = dte;
633

634 635 636
	rk_table_flush(rk_domain, pt_dma, NUM_PT_ENTRIES);
	rk_table_flush(rk_domain,
		       rk_domain->dt_dma + dte_index * sizeof(u32), 1);
637 638 639 640 641 642
done:
	pt_phys = rk_dte_pt_address(dte);
	return (u32 *)phys_to_virt(pt_phys);
}

static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain,
643 644
				  u32 *pte_addr, dma_addr_t pte_dma,
				  size_t size)
645 646 647 648 649 650 651 652 653 654 655 656 657 658
{
	unsigned int pte_count;
	unsigned int pte_total = size / SPAGE_SIZE;

	assert_spin_locked(&rk_domain->dt_lock);

	for (pte_count = 0; pte_count < pte_total; pte_count++) {
		u32 pte = pte_addr[pte_count];
		if (!rk_pte_is_page_valid(pte))
			break;

		pte_addr[pte_count] = rk_mk_pte_invalid(pte);
	}

659
	rk_table_flush(rk_domain, pte_dma, pte_count);
660 661 662 663 664

	return pte_count * SPAGE_SIZE;
}

static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr,
665 666
			     dma_addr_t pte_dma, dma_addr_t iova,
			     phys_addr_t paddr, size_t size, int prot)
667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684
{
	unsigned int pte_count;
	unsigned int pte_total = size / SPAGE_SIZE;
	phys_addr_t page_phys;

	assert_spin_locked(&rk_domain->dt_lock);

	for (pte_count = 0; pte_count < pte_total; pte_count++) {
		u32 pte = pte_addr[pte_count];

		if (rk_pte_is_page_valid(pte))
			goto unwind;

		pte_addr[pte_count] = rk_mk_pte(paddr, prot);

		paddr += SPAGE_SIZE;
	}

685
	rk_table_flush(rk_domain, pte_dma, pte_total);
686

687 688 689 690 691 692 693 694
	/*
	 * Zap the first and last iova to evict from iotlb any previously
	 * mapped cachelines holding stale values for its dte and pte.
	 * We only zap the first and last iova, since only they could have
	 * dte or pte shared with an existing mapping.
	 */
	rk_iommu_zap_iova_first_last(rk_domain, iova, size);

695 696 697
	return 0;
unwind:
	/* Unmap the range of iovas that we just mapped */
698 699
	rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma,
			    pte_count * SPAGE_SIZE);
700 701 702 703 704 705 706 707 708 709 710 711

	iova += pte_count * SPAGE_SIZE;
	page_phys = rk_pte_page_address(pte_addr[pte_count]);
	pr_err("iova: %pad already mapped to %pa cannot remap to phys: %pa prot: %#x\n",
	       &iova, &page_phys, &paddr, prot);

	return -EADDRINUSE;
}

static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
			phys_addr_t paddr, size_t size, int prot)
{
712
	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
713
	unsigned long flags;
714
	dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
715
	u32 *page_table, *pte_addr;
716
	u32 dte_index, pte_index;
717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
	int ret;

	spin_lock_irqsave(&rk_domain->dt_lock, flags);

	/*
	 * pgsize_bitmap specifies iova sizes that fit in one page table
	 * (1024 4-KiB pages = 4 MiB).
	 * So, size will always be 4096 <= size <= 4194304.
	 * Since iommu_map() guarantees that both iova and size will be
	 * aligned, we will always only be mapping from a single dte here.
	 */
	page_table = rk_dte_get_page_table(rk_domain, iova);
	if (IS_ERR(page_table)) {
		spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
		return PTR_ERR(page_table);
	}

734 735 736 737 738 739 740
	dte_index = rk_domain->dt[rk_iova_dte_index(iova)];
	pte_index = rk_iova_pte_index(iova);
	pte_addr = &page_table[pte_index];
	pte_dma = rk_dte_pt_address(dte_index) + pte_index * sizeof(u32);
	ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova,
				paddr, size, prot);

741 742 743 744 745 746 747 748
	spin_unlock_irqrestore(&rk_domain->dt_lock, flags);

	return ret;
}

static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
			     size_t size)
{
749
	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
750
	unsigned long flags;
751
	dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774
	phys_addr_t pt_phys;
	u32 dte;
	u32 *pte_addr;
	size_t unmap_size;

	spin_lock_irqsave(&rk_domain->dt_lock, flags);

	/*
	 * pgsize_bitmap specifies iova sizes that fit in one page table
	 * (1024 4-KiB pages = 4 MiB).
	 * So, size will always be 4096 <= size <= 4194304.
	 * Since iommu_unmap() guarantees that both iova and size will be
	 * aligned, we will always only be unmapping from a single dte here.
	 */
	dte = rk_domain->dt[rk_iova_dte_index(iova)];
	/* Just return 0 if iova is unmapped */
	if (!rk_dte_is_pt_valid(dte)) {
		spin_unlock_irqrestore(&rk_domain->dt_lock, flags);
		return 0;
	}

	pt_phys = rk_dte_pt_address(dte);
	pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova);
775 776
	pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32);
	unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size);
777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805

	spin_unlock_irqrestore(&rk_domain->dt_lock, flags);

	/* Shootdown iotlb entries for iova range that was just unmapped */
	rk_iommu_zap_iova(rk_domain, iova, unmap_size);

	return unmap_size;
}

static struct rk_iommu *rk_iommu_from_dev(struct device *dev)
{
	struct iommu_group *group;
	struct device *iommu_dev;
	struct rk_iommu *rk_iommu;

	group = iommu_group_get(dev);
	if (!group)
		return NULL;
	iommu_dev = iommu_group_get_iommudata(group);
	rk_iommu = dev_get_drvdata(iommu_dev);
	iommu_group_put(group);

	return rk_iommu;
}

static int rk_iommu_attach_device(struct iommu_domain *domain,
				  struct device *dev)
{
	struct rk_iommu *iommu;
806
	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
807
	unsigned long flags;
808
	int ret, i;
809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827

	/*
	 * Allow 'virtual devices' (e.g., drm) to attach to domain.
	 * Such a device does not belong to an iommu group.
	 */
	iommu = rk_iommu_from_dev(dev);
	if (!iommu)
		return 0;

	ret = rk_iommu_enable_stall(iommu);
	if (ret)
		return ret;

	ret = rk_iommu_force_reset(iommu);
	if (ret)
		return ret;

	iommu->domain = domain;

828
	ret = devm_request_irq(iommu->dev, iommu->irq, rk_iommu_irq,
829 830 831 832
			       IRQF_SHARED, dev_name(dev), iommu);
	if (ret)
		return ret;

833
	for (i = 0; i < iommu->num_mmu; i++) {
834 835
		rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR,
			       rk_domain->dt_dma);
836
		rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE);
837 838
		rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK);
	}
839 840 841 842 843 844 845 846 847

	ret = rk_iommu_enable_paging(iommu);
	if (ret)
		return ret;

	spin_lock_irqsave(&rk_domain->iommus_lock, flags);
	list_add_tail(&iommu->node, &rk_domain->iommus);
	spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);

848
	dev_dbg(dev, "Attached to iommu domain\n");
849 850 851 852 853 854 855 856 857 858

	rk_iommu_disable_stall(iommu);

	return 0;
}

static void rk_iommu_detach_device(struct iommu_domain *domain,
				   struct device *dev)
{
	struct rk_iommu *iommu;
859
	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
860
	unsigned long flags;
861
	int i;
862 863 864 865 866 867 868 869 870 871 872 873 874

	/* Allow 'virtual devices' (eg drm) to detach from domain */
	iommu = rk_iommu_from_dev(dev);
	if (!iommu)
		return;

	spin_lock_irqsave(&rk_domain->iommus_lock, flags);
	list_del_init(&iommu->node);
	spin_unlock_irqrestore(&rk_domain->iommus_lock, flags);

	/* Ignore error while disabling, just keep going */
	rk_iommu_enable_stall(iommu);
	rk_iommu_disable_paging(iommu);
875 876 877 878
	for (i = 0; i < iommu->num_mmu; i++) {
		rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, 0);
		rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0);
	}
879 880
	rk_iommu_disable_stall(iommu);

881
	devm_free_irq(iommu->dev, iommu->irq, iommu);
882 883 884

	iommu->domain = NULL;

885
	dev_dbg(dev, "Detached from iommu domain\n");
886 887
}

888
static struct iommu_domain *rk_iommu_domain_alloc(unsigned type)
889 890
{
	struct rk_iommu_domain *rk_domain;
891 892
	struct platform_device *pdev;
	struct device *iommu_dev;
893

894
	if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
895 896
		return NULL;

897 898 899 900 901 902
	/* Register a pdev per domain, so DMA API can base on this *dev
	 * even some virtual master doesn't have an iommu slave
	 */
	pdev = platform_device_register_simple("rk_iommu_domain",
					       PLATFORM_DEVID_AUTO, NULL, 0);
	if (IS_ERR(pdev))
903
		return NULL;
904

905 906 907 908 909 910
	rk_domain = devm_kzalloc(&pdev->dev, sizeof(*rk_domain), GFP_KERNEL);
	if (!rk_domain)
		goto err_unreg_pdev;

	rk_domain->pdev = pdev;

911 912
	if (type == IOMMU_DOMAIN_DMA &&
	    iommu_get_dma_cookie(&rk_domain->domain))
913 914
		goto err_unreg_pdev;

915 916 917 918 919 920 921
	/*
	 * rk32xx iommus use a 2 level pagetable.
	 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries.
	 * Allocate one 4 KiB page for each table.
	 */
	rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);
	if (!rk_domain->dt)
922 923 924 925 926 927 928 929 930
		goto err_put_cookie;

	iommu_dev = &pdev->dev;
	rk_domain->dt_dma = dma_map_single(iommu_dev, rk_domain->dt,
					   SPAGE_SIZE, DMA_TO_DEVICE);
	if (dma_mapping_error(iommu_dev, rk_domain->dt_dma)) {
		dev_err(iommu_dev, "DMA map error for DT\n");
		goto err_free_dt;
	}
931

932
	rk_table_flush(rk_domain, rk_domain->dt_dma, NUM_DT_ENTRIES);
933 934 935 936 937

	spin_lock_init(&rk_domain->iommus_lock);
	spin_lock_init(&rk_domain->dt_lock);
	INIT_LIST_HEAD(&rk_domain->iommus);

938 939 940 941
	rk_domain->domain.geometry.aperture_start = 0;
	rk_domain->domain.geometry.aperture_end   = DMA_BIT_MASK(32);
	rk_domain->domain.geometry.force_aperture = true;

942
	return &rk_domain->domain;
943

944 945 946
err_free_dt:
	free_page((unsigned long)rk_domain->dt);
err_put_cookie:
947 948
	if (type == IOMMU_DOMAIN_DMA)
		iommu_put_dma_cookie(&rk_domain->domain);
949 950 951
err_unreg_pdev:
	platform_device_unregister(pdev);

952
	return NULL;
953 954
}

955
static void rk_iommu_domain_free(struct iommu_domain *domain)
956
{
957
	struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
958 959 960 961 962 963 964 965 966
	int i;

	WARN_ON(!list_empty(&rk_domain->iommus));

	for (i = 0; i < NUM_DT_ENTRIES; i++) {
		u32 dte = rk_domain->dt[i];
		if (rk_dte_is_pt_valid(dte)) {
			phys_addr_t pt_phys = rk_dte_pt_address(dte);
			u32 *page_table = phys_to_virt(pt_phys);
967 968
			dma_unmap_single(&rk_domain->pdev->dev, pt_phys,
					 SPAGE_SIZE, DMA_TO_DEVICE);
969 970 971 972
			free_page((unsigned long)page_table);
		}
	}

973 974
	dma_unmap_single(&rk_domain->pdev->dev, rk_domain->dt_dma,
			 SPAGE_SIZE, DMA_TO_DEVICE);
975
	free_page((unsigned long)rk_domain->dt);
976

977 978
	if (domain->type == IOMMU_DOMAIN_DMA)
		iommu_put_dma_cookie(&rk_domain->domain);
979 980

	platform_device_unregister(rk_domain->pdev);
981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
}

static bool rk_iommu_is_dev_iommu_master(struct device *dev)
{
	struct device_node *np = dev->of_node;
	int ret;

	/*
	 * An iommu master has an iommus property containing a list of phandles
	 * to iommu nodes, each with an #iommu-cells property with value 0.
	 */
	ret = of_count_phandle_with_args(np, "iommus", "#iommu-cells");
	return (ret > 0);
}

static int rk_iommu_group_set_iommudata(struct iommu_group *group,
					struct device *dev)
{
	struct device_node *np = dev->of_node;
	struct platform_device *pd;
	int ret;
	struct of_phandle_args args;

	/*
	 * An iommu master has an iommus property containing a list of phandles
	 * to iommu nodes, each with an #iommu-cells property with value 0.
	 */
	ret = of_parse_phandle_with_args(np, "iommus", "#iommu-cells", 0,
					 &args);
	if (ret) {
1011 1012
		dev_err(dev, "of_parse_phandle_with_args(%pOF) => %d\n",
			np, ret);
1013 1014 1015
		return ret;
	}
	if (args.args_count != 0) {
1016 1017
		dev_err(dev, "incorrect number of iommu params found for %pOF (found %d, expected 0)\n",
			args.np, args.args_count);
1018 1019 1020 1021 1022 1023
		return -EINVAL;
	}

	pd = of_find_device_by_node(args.np);
	of_node_put(args.np);
	if (!pd) {
1024
		dev_err(dev, "iommu %pOF not found\n", args.np);
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036
		return -EPROBE_DEFER;
	}

	/* TODO(djkurtz): handle multiple slave iommus for a single master */
	iommu_group_set_iommudata(group, &pd->dev, NULL);

	return 0;
}

static int rk_iommu_add_device(struct device *dev)
{
	struct iommu_group *group;
1037
	struct rk_iommu *iommu;
1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
	int ret;

	if (!rk_iommu_is_dev_iommu_master(dev))
		return -ENODEV;

	group = iommu_group_get(dev);
	if (!group) {
		group = iommu_group_alloc();
		if (IS_ERR(group)) {
			dev_err(dev, "Failed to allocate IOMMU group\n");
			return PTR_ERR(group);
		}
	}

	ret = iommu_group_add_device(group, dev);
	if (ret)
		goto err_put_group;

	ret = rk_iommu_group_set_iommudata(group, dev);
	if (ret)
		goto err_remove_device;

1060 1061 1062 1063
	iommu = rk_iommu_from_dev(dev);
	if (iommu)
		iommu_device_link(&iommu->iommu, dev);

1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
	iommu_group_put(group);

	return 0;

err_remove_device:
	iommu_group_remove_device(dev);
err_put_group:
	iommu_group_put(group);
	return ret;
}

static void rk_iommu_remove_device(struct device *dev)
{
1077 1078
	struct rk_iommu *iommu;

1079 1080 1081
	if (!rk_iommu_is_dev_iommu_master(dev))
		return;

1082 1083 1084 1085
	iommu = rk_iommu_from_dev(dev);
	if (iommu)
		iommu_device_unlink(&iommu->iommu, dev);

1086 1087 1088 1089
	iommu_group_remove_device(dev);
}

static const struct iommu_ops rk_iommu_ops = {
1090 1091
	.domain_alloc = rk_iommu_domain_alloc,
	.domain_free = rk_iommu_domain_free,
1092 1093 1094 1095
	.attach_dev = rk_iommu_attach_device,
	.detach_dev = rk_iommu_detach_device,
	.map = rk_iommu_map,
	.unmap = rk_iommu_unmap,
1096
	.map_sg = default_iommu_map_sg,
1097 1098 1099 1100 1101 1102
	.add_device = rk_iommu_add_device,
	.remove_device = rk_iommu_remove_device,
	.iova_to_phys = rk_iommu_iova_to_phys,
	.pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP,
};

1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
static int rk_iommu_domain_probe(struct platform_device *pdev)
{
	struct device *dev = &pdev->dev;

	dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL);
	if (!dev->dma_parms)
		return -ENOMEM;

	/* Set dma_ops for dev, otherwise it would be dummy_dma_ops */
	arch_setup_dma_ops(dev, 0, DMA_BIT_MASK(32), NULL, false);

	dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
	dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32));

	return 0;
}

static struct platform_driver rk_iommu_domain_driver = {
	.probe = rk_iommu_domain_probe,
	.driver = {
		   .name = "rk_iommu_domain",
	},
};

1127 1128 1129 1130 1131
static int rk_iommu_probe(struct platform_device *pdev)
{
	struct device *dev = &pdev->dev;
	struct rk_iommu *iommu;
	struct resource *res;
1132
	int num_res = pdev->num_resources;
1133
	int err, i;
1134 1135 1136 1137 1138 1139 1140

	iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
	if (!iommu)
		return -ENOMEM;

	platform_set_drvdata(pdev, iommu);
	iommu->dev = dev;
1141
	iommu->num_mmu = 0;
1142 1143

	iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * num_res,
1144 1145 1146
				    GFP_KERNEL);
	if (!iommu->bases)
		return -ENOMEM;
1147

1148
	for (i = 0; i < num_res; i++) {
1149
		res = platform_get_resource(pdev, IORESOURCE_MEM, i);
1150 1151
		if (!res)
			continue;
1152 1153 1154 1155 1156 1157 1158
		iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res);
		if (IS_ERR(iommu->bases[i]))
			continue;
		iommu->num_mmu++;
	}
	if (iommu->num_mmu == 0)
		return PTR_ERR(iommu->bases[0]);
1159 1160 1161 1162 1163 1164 1165

	iommu->irq = platform_get_irq(pdev, 0);
	if (iommu->irq < 0) {
		dev_err(dev, "Failed to get IRQ, %d\n", iommu->irq);
		return -ENXIO;
	}

1166 1167 1168 1169 1170 1171 1172 1173
	err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));
	if (err)
		return err;

	iommu_device_set_ops(&iommu->iommu, &rk_iommu_ops);
	err = iommu_device_register(&iommu->iommu);

	return err;
1174 1175 1176 1177
}

static int rk_iommu_remove(struct platform_device *pdev)
{
1178 1179 1180 1181 1182 1183 1184
	struct rk_iommu *iommu = platform_get_drvdata(pdev);

	if (iommu) {
		iommu_device_sysfs_remove(&iommu->iommu);
		iommu_device_unregister(&iommu->iommu);
	}

1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
	return 0;
}

static const struct of_device_id rk_iommu_dt_ids[] = {
	{ .compatible = "rockchip,iommu" },
	{ /* sentinel */ }
};
MODULE_DEVICE_TABLE(of, rk_iommu_dt_ids);

static struct platform_driver rk_iommu_driver = {
	.probe = rk_iommu_probe,
	.remove = rk_iommu_remove,
	.driver = {
		   .name = "rk_iommu",
1199
		   .of_match_table = rk_iommu_dt_ids,
1200 1201 1202 1203 1204
	},
};

static int __init rk_iommu_init(void)
{
1205
	struct device_node *np;
1206 1207
	int ret;

1208 1209 1210 1211 1212 1213
	np = of_find_matching_node(NULL, rk_iommu_dt_ids);
	if (!np)
		return 0;

	of_node_put(np);

1214 1215 1216 1217
	ret = bus_set_iommu(&platform_bus_type, &rk_iommu_ops);
	if (ret)
		return ret;

1218 1219 1220 1221 1222 1223 1224 1225
	ret = platform_driver_register(&rk_iommu_domain_driver);
	if (ret)
		return ret;

	ret = platform_driver_register(&rk_iommu_driver);
	if (ret)
		platform_driver_unregister(&rk_iommu_domain_driver);
	return ret;
1226 1227 1228 1229
}
static void __exit rk_iommu_exit(void)
{
	platform_driver_unregister(&rk_iommu_driver);
1230
	platform_driver_unregister(&rk_iommu_domain_driver);
1231 1232 1233 1234 1235 1236 1237 1238 1239
}

subsys_initcall(rk_iommu_init);
module_exit(rk_iommu_exit);

MODULE_DESCRIPTION("IOMMU API for Rockchip");
MODULE_AUTHOR("Simon Xue <xxm@rock-chips.com> and Daniel Kurtz <djkurtz@chromium.org>");
MODULE_ALIAS("platform:rockchip-iommu");
MODULE_LICENSE("GPL v2");