dmar.c 49.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * Copyright (c) 2006, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
 * Place - Suite 330, Boston, MA 02111-1307 USA.
 *
17 18 19 20
 * Copyright (C) 2006-2008 Intel Corporation
 * Author: Ashok Raj <ashok.raj@intel.com>
 * Author: Shaohua Li <shaohua.li@intel.com>
 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21
 *
22
 * This file implements early detection/parsing of Remapping Devices
23 24
 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
 * tables.
25 26
 *
 * These routines are used by both DMA-remapping and Interrupt-remapping
27 28
 */

29
#define pr_fmt(fmt)     "DMAR: " fmt
30

31 32
#include <linux/pci.h>
#include <linux/dmar.h>
Kay, Allen M's avatar
Kay, Allen M committed
33 34
#include <linux/iova.h>
#include <linux/intel-iommu.h>
35
#include <linux/timer.h>
36 37
#include <linux/irq.h>
#include <linux/interrupt.h>
38
#include <linux/tboot.h>
39
#include <linux/dmi.h>
40
#include <linux/slab.h>
41
#include <linux/iommu.h>
42
#include <asm/irq_remapping.h>
43
#include <asm/iommu_table.h>
44

45 46
#include "irq_remapping.h"

47 48 49 50 51 52 53 54
typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
struct dmar_res_callback {
	dmar_res_handler_t	cb[ACPI_DMAR_TYPE_RESERVED];
	void			*arg[ACPI_DMAR_TYPE_RESERVED];
	bool			ignore_unhandled;
	bool			print_entry;
};

55 56 57 58 59 60 61 62 63 64 65
/*
 * Assumptions:
 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
 *    before IO devices managed by that unit.
 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
 *    after IO devices managed by that unit.
 * 3) Hotplug events are rare.
 *
 * Locking rules for DMA and interrupt remapping related global data structures:
 * 1) Use dmar_global_lock in process context
 * 2) Use RCU in interrupt context
66
 */
67
DECLARE_RWSEM(dmar_global_lock);
68 69
LIST_HEAD(dmar_drhd_units);

70
struct acpi_table_header * __initdata dmar_tbl;
71
static int dmar_dev_scope_status = 1;
72
static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
73

74
static int alloc_iommu(struct dmar_drhd_unit *drhd);
75
static void free_iommu(struct intel_iommu *iommu);
76

77 78
extern const struct iommu_ops intel_iommu_ops;

79
static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
80 81 82 83 84 85
{
	/*
	 * add INCLUDE_ALL at the tail, so scan the list will find it at
	 * the very end.
	 */
	if (drhd->include_all)
86
		list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
87
	else
88
		list_add_rcu(&drhd->list, &dmar_drhd_units);
89 90
}

91
void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
92 93 94 95 96 97
{
	struct acpi_dmar_device_scope *scope;

	*cnt = 0;
	while (start < end) {
		scope = start;
98
		if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
99
		    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
100 101
		    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
			(*cnt)++;
102 103
		else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
			scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
104
			pr_warn("Unsupported device scope\n");
105
		}
106 107 108
		start += scope->length;
	}
	if (*cnt == 0)
109 110
		return NULL;

111
	return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
112 113
}

114
void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
115
{
116
	int i;
117
	struct device *tmp_dev;
118

119
	if (*devices && *cnt) {
120
		for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
121
			put_device(tmp_dev);
122 123
		kfree(*devices);
	}
124 125 126

	*devices = NULL;
	*cnt = 0;
127 128
}

129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154
/* Optimize out kzalloc()/kfree() for normal cases */
static char dmar_pci_notify_info_buf[64];

static struct dmar_pci_notify_info *
dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
{
	int level = 0;
	size_t size;
	struct pci_dev *tmp;
	struct dmar_pci_notify_info *info;

	BUG_ON(dev->is_virtfn);

	/* Only generate path[] for device addition event */
	if (event == BUS_NOTIFY_ADD_DEVICE)
		for (tmp = dev; tmp; tmp = tmp->bus->self)
			level++;

	size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
	if (size <= sizeof(dmar_pci_notify_info_buf)) {
		info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
	} else {
		info = kzalloc(size, GFP_KERNEL);
		if (!info) {
			pr_warn("Out of memory when allocating notify_info "
				"for %s.\n", pci_name(dev));
155 156
			if (dmar_dev_scope_status == 0)
				dmar_dev_scope_status = -ENOMEM;
157 158 159 160 161 162 163 164 165
			return NULL;
		}
	}

	info->event = event;
	info->dev = dev;
	info->seg = pci_domain_nr(dev->bus);
	info->level = level;
	if (event == BUS_NOTIFY_ADD_DEVICE) {
166 167
		for (tmp = dev; tmp; tmp = tmp->bus->self) {
			level--;
168
			info->path[level].bus = tmp->bus->number;
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
			info->path[level].device = PCI_SLOT(tmp->devfn);
			info->path[level].function = PCI_FUNC(tmp->devfn);
			if (pci_is_root_bus(tmp->bus))
				info->bus = tmp->bus->number;
		}
	}

	return info;
}

static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
{
	if ((void *)info != dmar_pci_notify_info_buf)
		kfree(info);
}

static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
				struct acpi_dmar_pci_path *path, int count)
{
	int i;

	if (info->bus != bus)
191
		goto fallback;
192
	if (info->level != count)
193
		goto fallback;
194 195 196 197

	for (i = 0; i < count; i++) {
		if (path[i].device != info->path[i].device ||
		    path[i].function != info->path[i].function)
198
			goto fallback;
199 200 201
	}

	return true;
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217

fallback:

	if (count != 1)
		return false;

	i = info->level - 1;
	if (bus              == info->path[i].bus &&
	    path[0].device   == info->path[i].device &&
	    path[0].function == info->path[i].function) {
		pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
			bus, path[0].device, path[0].function);
		return true;
	}

	return false;
218 219 220 221 222
}

/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
			  void *start, void*end, u16 segment,
223 224
			  struct dmar_dev_scope *devices,
			  int devices_cnt)
225 226
{
	int i, level;
227
	struct device *tmp, *dev = &info->dev->dev;
228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
	struct acpi_dmar_device_scope *scope;
	struct acpi_dmar_pci_path *path;

	if (segment != info->seg)
		return 0;

	for (; start < end; start += scope->length) {
		scope = start;
		if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
		    scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
			continue;

		path = (struct acpi_dmar_pci_path *)(scope + 1);
		level = (scope->length - sizeof(*scope)) / sizeof(*path);
		if (!dmar_match_pci_path(info, scope->bus, path, level))
			continue;

245 246 247 248 249 250 251 252 253 254 255 256 257 258
		/*
		 * We expect devices with endpoint scope to have normal PCI
		 * headers, and devices with bridge scope to have bridge PCI
		 * headers.  However PCI NTB devices may be listed in the
		 * DMAR table with bridge scope, even though they have a
		 * normal PCI header.  NTB devices are identified by class
		 * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
		 * for this special case.
		 */
		if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
		     info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
		    (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
		     (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
		      info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) {
259
			pr_warn("Device scope type does not match for %s\n",
260
				pci_name(info->dev));
261 262 263 264 265
			return -EINVAL;
		}

		for_each_dev_scope(devices, devices_cnt, i, tmp)
			if (tmp == NULL) {
266 267 268 269
				devices[i].bus = info->dev->bus->number;
				devices[i].devfn = info->dev->devfn;
				rcu_assign_pointer(devices[i].dev,
						   get_device(dev));
270 271 272 273 274 275 276 277 278
				return 1;
			}
		BUG_ON(i >= devices_cnt);
	}

	return 0;
}

int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
279
			  struct dmar_dev_scope *devices, int count)
280 281
{
	int index;
282
	struct device *tmp;
283 284 285 286 287

	if (info->seg != segment)
		return 0;

	for_each_active_dev_scope(devices, count, index, tmp)
288
		if (tmp == &info->dev->dev) {
289
			RCU_INIT_POINTER(devices[index].dev, NULL);
290
			synchronize_rcu();
291
			put_device(tmp);
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
			return 1;
		}

	return 0;
}

static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
{
	int ret = 0;
	struct dmar_drhd_unit *dmaru;
	struct acpi_dmar_hardware_unit *drhd;

	for_each_drhd_unit(dmaru) {
		if (dmaru->include_all)
			continue;

		drhd = container_of(dmaru->hdr,
				    struct acpi_dmar_hardware_unit, header);
		ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
				((void *)drhd) + drhd->header.length,
				dmaru->segment,
				dmaru->devices, dmaru->devices_cnt);
314
		if (ret)
315 316 317 318
			break;
	}
	if (ret >= 0)
		ret = dmar_iommu_notify_scope_dev(info);
319 320
	if (ret < 0 && dmar_dev_scope_status == 0)
		dmar_dev_scope_status = ret;
321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341

	return ret;
}

static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
{
	struct dmar_drhd_unit *dmaru;

	for_each_drhd_unit(dmaru)
		if (dmar_remove_dev_scope(info, dmaru->segment,
			dmaru->devices, dmaru->devices_cnt))
			break;
	dmar_iommu_notify_scope_dev(info);
}

static int dmar_pci_bus_notifier(struct notifier_block *nb,
				 unsigned long action, void *data)
{
	struct pci_dev *pdev = to_pci_dev(data);
	struct dmar_pci_notify_info *info;

342 343 344
	/* Only care about add/remove events for physical functions.
	 * For VFs we actually do the lookup based on the corresponding
	 * PF in device_to_iommu() anyway. */
345 346
	if (pdev->is_virtfn)
		return NOTIFY_DONE;
347 348
	if (action != BUS_NOTIFY_ADD_DEVICE &&
	    action != BUS_NOTIFY_REMOVED_DEVICE)
349 350 351 352 353 354 355 356 357
		return NOTIFY_DONE;

	info = dmar_alloc_pci_notify_info(pdev, action);
	if (!info)
		return NOTIFY_DONE;

	down_write(&dmar_global_lock);
	if (action == BUS_NOTIFY_ADD_DEVICE)
		dmar_pci_bus_add_dev(info);
358
	else if (action == BUS_NOTIFY_REMOVED_DEVICE)
359 360 361 362 363 364 365 366 367 368 369 370 371
		dmar_pci_bus_del_dev(info);
	up_write(&dmar_global_lock);

	dmar_free_pci_notify_info(info);

	return NOTIFY_OK;
}

static struct notifier_block dmar_pci_bus_nb = {
	.notifier_call = dmar_pci_bus_notifier,
	.priority = INT_MIN,
};

372 373 374 375 376 377 378 379 380 381 382 383 384
static struct dmar_drhd_unit *
dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
{
	struct dmar_drhd_unit *dmaru;

	list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
		if (dmaru->segment == drhd->segment &&
		    dmaru->reg_base_addr == drhd->address)
			return dmaru;

	return NULL;
}

385 386 387 388 389
/**
 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 * structure which uniquely represent one DMA remapping hardware unit
 * present in the platform
 */
390
static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
391 392 393
{
	struct acpi_dmar_hardware_unit *drhd;
	struct dmar_drhd_unit *dmaru;
394
	int ret;
395

396
	drhd = (struct acpi_dmar_hardware_unit *)header;
397 398 399 400 401
	dmaru = dmar_find_dmaru(drhd);
	if (dmaru)
		goto out;

	dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
402 403 404
	if (!dmaru)
		return -ENOMEM;

405 406 407 408 409 410
	/*
	 * If header is allocated from slab by ACPI _DSM method, we need to
	 * copy the content because the memory buffer will be freed on return.
	 */
	dmaru->hdr = (void *)(dmaru + 1);
	memcpy(dmaru->hdr, header, header->length);
411
	dmaru->reg_base_addr = drhd->address;
412
	dmaru->segment = drhd->segment;
413
	dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
414 415 416 417 418 419
	dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
					      ((void *)drhd) + drhd->header.length,
					      &dmaru->devices_cnt);
	if (dmaru->devices_cnt && dmaru->devices == NULL) {
		kfree(dmaru);
		return -ENOMEM;
420
	}
421

422 423
	ret = alloc_iommu(dmaru);
	if (ret) {
424 425
		dmar_free_dev_scope(&dmaru->devices,
				    &dmaru->devices_cnt);
426 427 428 429
		kfree(dmaru);
		return ret;
	}
	dmar_register_drhd_unit(dmaru);
430

431
out:
432 433 434
	if (arg)
		(*(int *)arg)++;

435 436 437
	return 0;
}

438 439 440 441 442 443 444 445 446
static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
{
	if (dmaru->devices && dmaru->devices_cnt)
		dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
	if (dmaru->iommu)
		free_iommu(dmaru->iommu);
	kfree(dmaru);
}

447 448
static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
				      void *arg)
449 450 451 452
{
	struct acpi_dmar_andd *andd = (void *)header;

	/* Check for NUL termination within the designated length */
453
	if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
454 455 456 457 458 459 460 461 462
		WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
			   "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
			   "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
			   dmi_get_system_info(DMI_BIOS_VENDOR),
			   dmi_get_system_info(DMI_BIOS_VERSION),
			   dmi_get_system_info(DMI_PRODUCT_VERSION));
		return -EINVAL;
	}
	pr_info("ANDD device: %x name: %s\n", andd->device_number,
463
		andd->device_name);
464 465 466 467

	return 0;
}

468
#ifdef CONFIG_ACPI_NUMA
469
static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
470 471 472 473 474
{
	struct acpi_dmar_rhsa *rhsa;
	struct dmar_drhd_unit *drhd;

	rhsa = (struct acpi_dmar_rhsa *)header;
475
	for_each_drhd_unit(drhd) {
476 477 478 479 480 481
		if (drhd->reg_base_addr == rhsa->base_address) {
			int node = acpi_map_pxm_to_node(rhsa->proximity_domain);

			if (!node_online(node))
				node = -1;
			drhd->iommu->node = node;
482 483
			return 0;
		}
484
	}
485 486 487 488 489 490 491 492
	WARN_TAINT(
		1, TAINT_FIRMWARE_WORKAROUND,
		"Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
		"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
		drhd->reg_base_addr,
		dmi_get_system_info(DMI_BIOS_VENDOR),
		dmi_get_system_info(DMI_BIOS_VERSION),
		dmi_get_system_info(DMI_PRODUCT_VERSION));
493

494
	return 0;
495
}
496 497
#else
#define	dmar_parse_one_rhsa		dmar_res_noop
498
#endif
499

500 501 502 503 504
static void __init
dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
{
	struct acpi_dmar_hardware_unit *drhd;
	struct acpi_dmar_reserved_memory *rmrr;
505
	struct acpi_dmar_atsr *atsr;
506
	struct acpi_dmar_rhsa *rhsa;
507 508 509

	switch (header->type) {
	case ACPI_DMAR_TYPE_HARDWARE_UNIT:
510 511
		drhd = container_of(header, struct acpi_dmar_hardware_unit,
				    header);
512
		pr_info("DRHD base: %#016Lx flags: %#x\n",
513
			(unsigned long long)drhd->address, drhd->flags);
514 515
		break;
	case ACPI_DMAR_TYPE_RESERVED_MEMORY:
516 517
		rmrr = container_of(header, struct acpi_dmar_reserved_memory,
				    header);
518
		pr_info("RMRR base: %#016Lx end: %#016Lx\n",
Fenghua Yu's avatar
Fenghua Yu committed
519 520
			(unsigned long long)rmrr->base_address,
			(unsigned long long)rmrr->end_address);
521
		break;
522
	case ACPI_DMAR_TYPE_ROOT_ATS:
523
		atsr = container_of(header, struct acpi_dmar_atsr, header);
524
		pr_info("ATSR flags: %#x\n", atsr->flags);
525
		break;
526
	case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
527
		rhsa = container_of(header, struct acpi_dmar_rhsa, header);
528
		pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
529 530 531
		       (unsigned long long)rhsa->base_address,
		       rhsa->proximity_domain);
		break;
532
	case ACPI_DMAR_TYPE_NAMESPACE:
533 534 535
		/* We don't print this here because we need to sanity-check
		   it first. So print it in dmar_parse_one_andd() instead. */
		break;
536 537 538
	}
}

539 540 541 542 543 544 545 546
/**
 * dmar_table_detect - checks to see if the platform supports DMAR devices
 */
static int __init dmar_table_detect(void)
{
	acpi_status status = AE_OK;

	/* if we could find DMAR table, then there are DMAR devices */
547
	status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
548 549

	if (ACPI_SUCCESS(status) && !dmar_tbl) {
550
		pr_warn("Unable to map DMAR\n");
551 552 553
		status = AE_NOT_FOUND;
	}

554
	return ACPI_SUCCESS(status) ? 0 : -ENOENT;
555
}
556

557 558 559 560 561 562
static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
				       size_t len, struct dmar_res_callback *cb)
{
	struct acpi_dmar_header *iter, *next;
	struct acpi_dmar_header *end = ((void *)start) + len;

563
	for (iter = start; iter < end; iter = next) {
564 565 566 567 568 569 570
		next = (void *)iter + iter->length;
		if (iter->length == 0) {
			/* Avoid looping forever on bad ACPI tables */
			pr_debug(FW_BUG "Invalid 0-length structure\n");
			break;
		} else if (next > end) {
			/* Avoid passing table end */
571
			pr_warn(FW_BUG "Record passes table end\n");
572
			return -EINVAL;
573 574 575 576 577 578 579 580 581 582
		}

		if (cb->print_entry)
			dmar_table_print_dmar_entry(iter);

		if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
			/* continue for forward compatibility */
			pr_debug("Unknown DMAR structure type %d\n",
				 iter->type);
		} else if (cb->cb[iter->type]) {
583 584
			int ret;

585
			ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
586 587
			if (ret)
				return ret;
588 589 590
		} else if (!cb->ignore_unhandled) {
			pr_warn("No handler for DMAR structure type %d\n",
				iter->type);
591
			return -EINVAL;
592 593 594
		}
	}

595
	return 0;
596 597 598 599 600 601 602 603 604
}

static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
				       struct dmar_res_callback *cb)
{
	return dmar_walk_remapping_entries((void *)(dmar + 1),
			dmar->header.length - sizeof(*dmar), cb);
}

605 606 607 608 609 610 611
/**
 * parse_dmar_table - parses the DMA reporting table
 */
static int __init
parse_dmar_table(void)
{
	struct acpi_table_dmar *dmar;
612
	int drhd_count = 0;
613
	int ret;
614 615 616 617 618 619 620 621 622 623
	struct dmar_res_callback cb = {
		.print_entry = true,
		.ignore_unhandled = true,
		.arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
		.cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
		.cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
		.cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
		.cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
		.cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
	};
624

625 626 627 628 629 630
	/*
	 * Do it again, earlier dmar_tbl mapping could be mapped with
	 * fixed map.
	 */
	dmar_table_detect();

631 632 633 634 635 636
	/*
	 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
	 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
	 */
	dmar_tbl = tboot_get_dmar_table(dmar_tbl);

637 638 639 640
	dmar = (struct acpi_table_dmar *)dmar_tbl;
	if (!dmar)
		return -ENODEV;

Fenghua Yu's avatar
Fenghua Yu committed
641
	if (dmar->width < PAGE_SHIFT - 1) {
642
		pr_warn("Invalid DMAR haw\n");
643 644 645
		return -EINVAL;
	}

646
	pr_info("Host address width %d\n", dmar->width + 1);
647 648
	ret = dmar_walk_dmar_table(dmar, &cb);
	if (ret == 0 && drhd_count == 0)
649
		pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
650

651 652 653
	return ret;
}

654 655
static int dmar_pci_device_match(struct dmar_dev_scope devices[],
				 int cnt, struct pci_dev *dev)
656 657
{
	int index;
658
	struct device *tmp;
659 660

	while (dev) {
661
		for_each_active_dev_scope(devices, cnt, index, tmp)
662
			if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
663 664 665 666 667 668 669 670 671 672 673 674
				return 1;

		/* Check our parent */
		dev = dev->bus->self;
	}

	return 0;
}

struct dmar_drhd_unit *
dmar_find_matched_drhd_unit(struct pci_dev *dev)
{
675
	struct dmar_drhd_unit *dmaru;
676 677
	struct acpi_dmar_hardware_unit *drhd;

678 679
	dev = pci_physfn(dev);

680
	rcu_read_lock();
681
	for_each_drhd_unit(dmaru) {
682 683 684 685 686 687
		drhd = container_of(dmaru->hdr,
				    struct acpi_dmar_hardware_unit,
				    header);

		if (dmaru->include_all &&
		    drhd->segment == pci_domain_nr(dev->bus))
688
			goto out;
689

690 691
		if (dmar_pci_device_match(dmaru->devices,
					  dmaru->devices_cnt, dev))
692
			goto out;
693
	}
694 695 696
	dmaru = NULL;
out:
	rcu_read_unlock();
697

698
	return dmaru;
699 700
}

701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718
static void __init dmar_acpi_insert_dev_scope(u8 device_number,
					      struct acpi_device *adev)
{
	struct dmar_drhd_unit *dmaru;
	struct acpi_dmar_hardware_unit *drhd;
	struct acpi_dmar_device_scope *scope;
	struct device *tmp;
	int i;
	struct acpi_dmar_pci_path *path;

	for_each_drhd_unit(dmaru) {
		drhd = container_of(dmaru->hdr,
				    struct acpi_dmar_hardware_unit,
				    header);

		for (scope = (void *)(drhd + 1);
		     (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
		     scope = ((void *)scope) + scope->length) {
719
			if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
				continue;
			if (scope->enumeration_id != device_number)
				continue;

			path = (void *)(scope + 1);
			pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
				dev_name(&adev->dev), dmaru->reg_base_addr,
				scope->bus, path->device, path->function);
			for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
				if (tmp == NULL) {
					dmaru->devices[i].bus = scope->bus;
					dmaru->devices[i].devfn = PCI_DEVFN(path->device,
									    path->function);
					rcu_assign_pointer(dmaru->devices[i].dev,
							   get_device(&adev->dev));
					return;
				}
			BUG_ON(i >= dmaru->devices_cnt);
		}
	}
	pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
		device_number, dev_name(&adev->dev));
}

static int __init dmar_acpi_dev_scope_init(void)
{
746 747 748 749 750
	struct acpi_dmar_andd *andd;

	if (dmar_tbl == NULL)
		return -ENODEV;

751 752 753
	for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
	     ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
	     andd = ((void *)andd) + andd->header.length) {
754
		if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
755 756 757 758
			acpi_handle h;
			struct acpi_device *adev;

			if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
759
							  andd->device_name,
760 761
							  &h))) {
				pr_err("Failed to find handle for ACPI object %s\n",
762
				       andd->device_name);
763 764
				continue;
			}
765
			if (acpi_bus_get_device(h, &adev)) {
766
				pr_err("Failed to get device for ACPI object %s\n",
767
				       andd->device_name);
768 769 770 771 772 773 774 775
				continue;
			}
			dmar_acpi_insert_dev_scope(andd->device_number, adev);
		}
	}
	return 0;
}

776 777
int __init dmar_dev_scope_init(void)
{
778 779
	struct pci_dev *dev = NULL;
	struct dmar_pci_notify_info *info;
780

781 782
	if (dmar_dev_scope_status != 1)
		return dmar_dev_scope_status;
783

784 785 786 787 788
	if (list_empty(&dmar_drhd_units)) {
		dmar_dev_scope_status = -ENODEV;
	} else {
		dmar_dev_scope_status = 0;

789 790
		dmar_acpi_dev_scope_init();

791 792 793 794 795 796 797 798 799 800 801 802 803
		for_each_pci_dev(dev) {
			if (dev->is_virtfn)
				continue;

			info = dmar_alloc_pci_notify_info(dev,
					BUS_NOTIFY_ADD_DEVICE);
			if (!info) {
				return dmar_dev_scope_status;
			} else {
				dmar_pci_bus_add_dev(info);
				dmar_free_pci_notify_info(info);
			}
		}
804

805
		bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
806 807
	}

808
	return dmar_dev_scope_status;
809 810
}

811 812 813

int __init dmar_table_init(void)
{
814
	static int dmar_table_initialized;
815 816
	int ret;

817 818 819 820
	if (dmar_table_initialized == 0) {
		ret = parse_dmar_table();
		if (ret < 0) {
			if (ret != -ENODEV)
821
				pr_info("Parse DMAR table failure.\n");
822 823 824 825
		} else  if (list_empty(&dmar_drhd_units)) {
			pr_info("No DMAR devices found\n");
			ret = -ENODEV;
		}
826

827 828 829 830
		if (ret < 0)
			dmar_table_initialized = ret;
		else
			dmar_table_initialized = 1;
831
	}
832

833
	return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
834 835
}

836 837
static void warn_invalid_dmar(u64 addr, const char *message)
{
838 839 840 841 842 843 844 845
	WARN_TAINT_ONCE(
		1, TAINT_FIRMWARE_WORKAROUND,
		"Your BIOS is broken; DMAR reported at address %llx%s!\n"
		"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
		addr, message,
		dmi_get_system_info(DMI_BIOS_VENDOR),
		dmi_get_system_info(DMI_BIOS_VERSION),
		dmi_get_system_info(DMI_PRODUCT_VERSION));
846
}
847

848 849
static int __ref
dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
850 851
{
	struct acpi_dmar_hardware_unit *drhd;
852 853
	void __iomem *addr;
	u64 cap, ecap;
854

855 856 857 858 859
	drhd = (void *)entry;
	if (!drhd->address) {
		warn_invalid_dmar(0, "");
		return -EINVAL;
	}
860

861 862 863 864
	if (arg)
		addr = ioremap(drhd->address, VTD_PAGE_SIZE);
	else
		addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
865
	if (!addr) {
866
		pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
867 868
		return -EINVAL;
	}
869

870 871
	cap = dmar_readq(addr + DMAR_CAP_REG);
	ecap = dmar_readq(addr + DMAR_ECAP_REG);
872 873 874 875 876

	if (arg)
		iounmap(addr);
	else
		early_iounmap(addr, VTD_PAGE_SIZE);
877

878 879 880
	if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
		warn_invalid_dmar(drhd->address, " returns all ones");
		return -EINVAL;
881
	}
882 883

	return 0;
884 885
}

886
int __init detect_intel_iommu(void)
887 888
{
	int ret;
889 890 891 892
	struct dmar_res_callback validate_drhd_cb = {
		.cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
		.ignore_unhandled = true,
	};
893

894
	down_write(&dmar_global_lock);
895
	ret = dmar_table_detect();
896 897 898 899
	if (!ret)
		ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
					   &validate_drhd_cb);
	if (!ret && !no_iommu && !iommu_detected && !dmar_disabled) {
900 901 902 903
		iommu_detected = 1;
		/* Make sure ACS will be enabled */
		pci_request_acs();
	}
904

905
#ifdef CONFIG_X86
906
	if (!ret)
907
		x86_init.iommu.iommu_init = intel_iommu_init;
908
#endif
909

910 911 912 913
	if (dmar_tbl) {
		acpi_put_table(dmar_tbl);
		dmar_tbl = NULL;
	}
914
	up_write(&dmar_global_lock);
915

916
	return ret ? ret : 1;
917 918
}

919 920 921 922 923 924 925 926 927 928
static void unmap_iommu(struct intel_iommu *iommu)
{
	iounmap(iommu->reg);
	release_mem_region(iommu->reg_phys, iommu->reg_size);
}

/**
 * map_iommu: map the iommu's registers
 * @iommu: the iommu to map
 * @phys_addr: the physical address of the base resgister
929
 *
930
 * Memory map the iommu's registers.  Start w/ a single page, and
931
 * possibly expand if that turns out to be insufficent.
932 933 934 935 936 937 938 939 940
 */
static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
{
	int map_size, err=0;

	iommu->reg_phys = phys_addr;
	iommu->reg_size = VTD_PAGE_SIZE;

	if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
941
		pr_err("Can't reserve memory\n");
942 943 944 945 946 947
		err = -EBUSY;
		goto out;
	}

	iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
	if (!iommu->reg) {
948
		pr_err("Can't map the region\n");
949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971
		err = -ENOMEM;
		goto release;
	}

	iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
	iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);

	if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
		err = -EINVAL;
		warn_invalid_dmar(phys_addr, " returns all ones");
		goto unmap;
	}

	/* the registers might be more than one page */
	map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
			 cap_max_fault_reg_offset(iommu->cap));
	map_size = VTD_PAGE_ALIGN(map_size);
	if (map_size > iommu->reg_size) {
		iounmap(iommu->reg);
		release_mem_region(iommu->reg_phys, iommu->reg_size);
		iommu->reg_size = map_size;
		if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
					iommu->name)) {
972
			pr_err("Can't reserve memory\n");
973 974 975 976 977
			err = -EBUSY;
			goto out;
		}
		iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
		if (!iommu->reg) {
978
			pr_err("Can't map the region\n");
979 980 981 982 983 984 985 986 987 988 989 990 991 992 993
			err = -ENOMEM;
			goto release;
		}
	}
	err = 0;
	goto out;

unmap:
	iounmap(iommu->reg);
release:
	release_mem_region(iommu->reg_phys, iommu->reg_size);
out:
	return err;
}

994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
static int dmar_alloc_seq_id(struct intel_iommu *iommu)
{
	iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
					    DMAR_UNITS_SUPPORTED);
	if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
		iommu->seq_id = -1;
	} else {
		set_bit(iommu->seq_id, dmar_seq_ids);
		sprintf(iommu->name, "dmar%d", iommu->seq_id);
	}

	return iommu->seq_id;
}

static void dmar_free_seq_id(struct intel_iommu *iommu)
{
	if (iommu->seq_id >= 0) {
		clear_bit(iommu->seq_id, dmar_seq_ids);
		iommu->seq_id = -1;
	}
}

1016
static int alloc_iommu(struct dmar_drhd_unit *drhd)
1017
{
1018
	struct intel_iommu *iommu;
1019
	u32 ver, sts;
1020
	int agaw = 0;
1021
	int msagaw = 0;
1022
	int err;
1023

1024
	if (!drhd->reg_base_addr) {
1025
		warn_invalid_dmar(0, "");
1026 1027 1028
		return -EINVAL;
	}

1029 1030
	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
	if (!iommu)
1031
		return -ENOMEM;
1032

1033
	if (dmar_alloc_seq_id(iommu) < 0) {
1034
		pr_err("Failed to allocate seq_id\n");
1035 1036 1037
		err = -ENOSPC;
		goto error;
	}
1038

1039 1040
	err = map_iommu(iommu, drhd->reg_base_addr);
	if (err) {
1041
		pr_err("Failed to map %s\n", iommu->name);
1042
		goto error_free_seq_id;
1043
	}
1044

1045
	err = -EINVAL;
Weidong Han's avatar
Weidong Han committed
1046 1047
	agaw = iommu_calculate_agaw(iommu);
	if (agaw < 0) {
1048 1049
		pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
			iommu->seq_id);
1050
		goto err_unmap;
1051 1052 1053
	}
	msagaw = iommu_calculate_max_sagaw(iommu);
	if (msagaw < 0) {
1054
		pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
Weidong Han's avatar
Weidong Han committed
1055
			iommu->seq_id);
1056
		goto err_unmap;
Weidong Han's avatar
Weidong Han committed
1057 1058
	}
	iommu->agaw = agaw;
1059
	iommu->msagaw = msagaw;
1060
	iommu->segment = drhd->segment;
Weidong Han's avatar
Weidong Han committed
1061

1062 1063
	iommu->node = -1;

1064
	ver = readl(iommu->reg + DMAR_VER_REG);
1065 1066
	pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
		iommu->name,
Fenghua Yu's avatar
Fenghua Yu committed
1067 1068 1069 1070
		(unsigned long long)drhd->reg_base_addr,
		DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
		(unsigned long long)iommu->cap,
		(unsigned long long)iommu->ecap);
1071

1072 1073 1074 1075 1076 1077 1078 1079 1080
	/* Reflect status in gcmd */
	sts = readl(iommu->reg + DMAR_GSTS_REG);
	if (sts & DMA_GSTS_IRES)
		iommu->gcmd |= DMA_GCMD_IRE;
	if (sts & DMA_GSTS_TES)
		iommu->gcmd |= DMA_GCMD_TE;
	if (sts & DMA_GSTS_QIES)
		iommu->gcmd |= DMA_GCMD_QIE;

1081
	raw_spin_lock_init(&iommu->register_lock);
1082

1083
	if (intel_iommu_enabled) {
1084 1085 1086 1087
		err = iommu_device_sysfs_add(&iommu->iommu, NULL,
					     intel_iommu_groups,
					     "%s", iommu->name);
		if (err)
1088
			goto err_unmap;
1089

1090 1091 1092 1093
		iommu_device_set_ops(&iommu->iommu, &intel_iommu_ops);

		err = iommu_device_register(&iommu->iommu);
		if (err)
1094
			goto err_unmap;
1095 1096
	}

1097 1098
	drhd->iommu = iommu;

1099
	return 0;
1100

1101
err_unmap:
1102
	unmap_iommu(iommu);
1103 1104 1105
error_free_seq_id:
	dmar_free_seq_id(iommu);
error:
1106
	kfree(iommu);
1107
	return err;
1108 1109
}

1110
static void free_iommu(struct intel_iommu *iommu)
1111
{
1112 1113 1114 1115
	if (intel_iommu_enabled) {
		iommu_device_unregister(&iommu->iommu);
		iommu_device_sysfs_remove(&iommu->iommu);
	}
1116

1117
	if (iommu->irq) {
1118 1119 1120 1121 1122
		if (iommu->pr_irq) {
			free_irq(iommu->pr_irq, iommu);
			dmar_free_hwirq(iommu->pr_irq);
			iommu->pr_irq = 0;
		}
1123
		free_irq(iommu->irq, iommu);
1124
		dmar_free_hwirq(iommu->irq);
1125
		iommu->irq = 0;
1126
	}
1127

1128 1129 1130 1131 1132 1133
	if (iommu->qi) {
		free_page((unsigned long)iommu->qi->desc);
		kfree(iommu->qi->desc_status);
		kfree(iommu->qi);
	}

1134
	if (iommu->reg)
1135 1136
		unmap_iommu(iommu);

1137
	dmar_free_seq_id(iommu);
1138 1139
	kfree(iommu);
}
1140 1141 1142 1143 1144 1145

/*
 * Reclaim all the submitted descriptors which have completed its work.
 */
static inline void reclaim_free_desc(struct q_inval *qi)
{
1146 1147
	while (qi->desc_status[qi->free_tail] == QI_DONE ||
	       qi->desc_status[qi->free_tail] == QI_ABORT) {
1148 1149 1150 1151 1152 1153
		qi->desc_status[qi->free_tail] = QI_FREE;
		qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
		qi->free_cnt++;
	}
}

1154 1155 1156
static int qi_check_fault(struct intel_iommu *iommu, int index)
{
	u32 fault;
1157
	int head, tail;
1158 1159 1160
	struct q_inval *qi = iommu->qi;
	int wait_index = (index + 1) % QI_LENGTH;

1161 1162 1163
	if (qi->desc_status[wait_index] == QI_ABORT)
		return -EAGAIN;

1164 1165 1166 1167 1168 1169 1170 1171 1172
	fault = readl(iommu->reg + DMAR_FSTS_REG);

	/*
	 * If IQE happens, the head points to the descriptor associated
	 * with the error. No new descriptors are fetched until the IQE
	 * is cleared.
	 */
	if (fault & DMA_FSTS_IQE) {
		head = readl(iommu->reg + DMAR_IQH_REG);
1173
		if ((head >> DMAR_IQ_SHIFT) == index) {
1174
			pr_err("VT-d detected invalid descriptor: "
1175 1176 1177
				"low=%llx, high=%llx\n",
				(unsigned long long)qi->desc[index].low,
				(unsigned long long)qi->desc[index].high);
1178 1179 1180 1181 1182 1183 1184
			memcpy(&qi->desc[index], &qi->desc[wait_index],
					sizeof(struct qi_desc));
			writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
			return -EINVAL;
		}
	}

1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210
	/*
	 * If ITE happens, all pending wait_desc commands are aborted.
	 * No new descriptors are fetched until the ITE is cleared.
	 */
	if (fault & DMA_FSTS_ITE) {
		head = readl(iommu->reg + DMAR_IQH_REG);
		head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
		head |= 1;
		tail = readl(iommu->reg + DMAR_IQT_REG);
		tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;

		writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);

		do {
			if (qi->desc_status[head] == QI_IN_USE)
				qi->desc_status[head] = QI_ABORT;
			head = (head - 2 + QI_LENGTH) % QI_LENGTH;
		} while (head != tail);

		if (qi->desc_status[wait_index] == QI_ABORT)
			return -EAGAIN;
	}

	if (fault & DMA_FSTS_ICE)
		writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);

1211 1212 1213
	return 0;
}

1214 1215 1216 1217
/*
 * Submit the queued invalidation descriptor to the remapping
 * hardware unit and wait for its completion.
 */
1218
int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1219
{
1220
	int rc;
1221 1222 1223 1224 1225 1226
	struct q_inval *qi = iommu->qi;
	struct qi_desc *hw, wait_desc;
	int wait_index, index;
	unsigned long flags;

	if (!qi)
1227
		return 0;
1228 1229 1230

	hw = qi->desc;

1231 1232 1233
restart:
	rc = 0;

1234
	raw_spin_lock_irqsave(&qi->q_lock, flags);
1235
	while (qi->free_cnt < 3) {
1236
		raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1237
		cpu_relax();
1238
		raw_spin_lock_irqsave(&qi->q_lock, flags);
1239 1240 1241 1242 1243 1244 1245 1246 1247
	}

	index = qi->free_head;
	wait_index = (index + 1) % QI_LENGTH;

	qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;

	hw[index] = *desc;

1248 1249
	wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
			QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
	wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);

	hw[wait_index] = wait_desc;

	qi->free_head = (qi->free_head + 2) % QI_LENGTH;
	qi->free_cnt -= 2;

	/*
	 * update the HW tail register indicating the presence of
	 * new descriptors.
	 */
1261
	writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1262 1263

	while (qi->desc_status[wait_index] != QI_DONE) {
1264 1265 1266 1267 1268 1269 1270
		/*
		 * We will leave the interrupts disabled, to prevent interrupt
		 * context to queue another cmd while a cmd is already submitted
		 * and waiting for completion on this cpu. This is to avoid
		 * a deadlock where the interrupt context can wait indefinitely
		 * for free slots in the queue.
		 */
1271 1272
		rc = qi_check_fault(iommu, index);
		if (rc)
1273
			break;
1274

1275
		raw_spin_unlock(&qi->q_lock);
1276
		cpu_relax();
1277
		raw_spin_lock(&qi->q_lock);
1278
	}
1279 1280

	qi->desc_status[index] = QI_DONE;
1281 1282

	reclaim_free_desc(qi);
1283
	raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1284

1285 1286 1287
	if (rc == -EAGAIN)
		goto restart;

1288
	return rc;
1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300
}

/*
 * Flush the global interrupt entry cache.
 */
void qi_global_iec(struct intel_iommu *iommu)
{
	struct qi_desc desc;

	desc.low = QI_IEC_TYPE;
	desc.high = 0;

1301
	/* should never fail */
1302 1303 1304
	qi_submit_sync(&desc, iommu);
}

1305 1306
void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
		      u64 type)
1307 1308 1309 1310 1311 1312 1313
{
	struct qi_desc desc;

	desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
			| QI_CC_GRAN(type) | QI_CC_TYPE;
	desc.high = 0;

1314
	qi_submit_sync(&desc, iommu);
1315 1316
}

1317 1318
void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
		    unsigned int size_order, u64 type)
1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335
{
	u8 dw = 0, dr = 0;

	struct qi_desc desc;
	int ih = 0;

	if (cap_write_drain(iommu->cap))
		dw = 1;

	if (cap_read_drain(iommu->cap))
		dr = 1;

	desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
		| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
	desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
		| QI_IOTLB_AM(size_order);

1336
	qi_submit_sync(&desc, iommu);
1337 1338
}

1339 1340 1341 1342 1343 1344 1345
void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
			u64 addr, unsigned mask)
{
	struct qi_desc desc;

	if (mask) {
		BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1346
		addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
		desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
	} else
		desc.high = QI_DEV_IOTLB_ADDR(addr);

	if (qdep >= QI_DEV_IOTLB_MAX_INVS)
		qdep = 0;

	desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
		   QI_DIOTLB_TYPE;

	qi_submit_sync(&desc, iommu);
}

1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
/*
 * Disable Queued Invalidation interface.
 */
void dmar_disable_qi(struct intel_iommu *iommu)
{
	unsigned long flags;
	u32 sts;
	cycles_t start_time = get_cycles();

	if (!ecap_qis(iommu->ecap))
		return;

1372
	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1373

1374
	sts =  readl(iommu->reg + DMAR_GSTS_REG);
1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391
	if (!(sts & DMA_GSTS_QIES))
		goto end;

	/*
	 * Give a chance to HW to complete the pending invalidation requests.
	 */
	while ((readl(iommu->reg + DMAR_IQT_REG) !=
		readl(iommu->reg + DMAR_IQH_REG)) &&
		(DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
		cpu_relax();

	iommu->gcmd &= ~DMA_GCMD_QIE;
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
		      !(sts & DMA_GSTS_QIES), sts);
end:
1392
	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1393 1394
}

1395 1396 1397 1398 1399
/*
 * Enable queued invalidation.
 */
static void __dmar_enable_qi(struct intel_iommu *iommu)
{
1400
	u32 sts;
1401 1402 1403 1404 1405 1406
	unsigned long flags;
	struct q_inval *qi = iommu->qi;

	qi->free_head = qi->free_tail = 0;
	qi->free_cnt = QI_LENGTH;

1407
	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1408 1409 1410 1411 1412 1413 1414

	/* write zero to the tail reg */
	writel(0, iommu->reg + DMAR_IQT_REG);

	dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));

	iommu->gcmd |= DMA_GCMD_QIE;
1415
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1416 1417 1418 1419

	/* Make sure hardware complete it */
	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);

1420
	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1421 1422
}

1423 1424 1425 1426 1427 1428 1429 1430
/*
 * Enable Queued Invalidation interface. This is a must to support
 * interrupt-remapping. Also used by DMA-remapping, which replaces
 * register based IOTLB invalidation.
 */
int dmar_enable_qi(struct intel_iommu *iommu)
{
	struct q_inval *qi;
1431
	struct page *desc_page;
1432 1433 1434 1435 1436 1437 1438 1439 1440 1441

	if (!ecap_qis(iommu->ecap))
		return -ENOENT;

	/*
	 * queued invalidation is already setup and enabled.
	 */
	if (iommu->qi)
		return 0;

1442
	iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1443 1444 1445 1446 1447
	if (!iommu->qi)
		return -ENOMEM;

	qi = iommu->qi;

1448 1449 1450

	desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
	if (!desc_page) {
1451
		kfree(qi);
1452
		iommu->qi = NULL;
1453 1454 1455
		return -ENOMEM;
	}

1456 1457
	qi->desc = page_address(desc_page);

1458
	qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1459 1460 1461
	if (!qi->desc_status) {
		free_page((unsigned long) qi->desc);
		kfree(qi);
1462
		iommu->qi = NULL;
1463 1464 1465
		return -ENOMEM;
	}

1466
	raw_spin_lock_init(&qi->q_lock);
1467

1468
	__dmar_enable_qi(iommu);
1469 1470 1471

	return 0;
}
1472 1473 1474

/* iommu interrupt handling. Most stuff are MSI-like. */

1475 1476 1477 1478 1479 1480 1481
enum faulttype {
	DMA_REMAP,
	INTR_REMAP,
	UNKNOWN,
};

static const char *dma_remap_fault_reasons[] =
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495
{
	"Software",
	"Present bit in root entry is clear",
	"Present bit in context entry is clear",
	"Invalid context entry",
	"Access beyond MGAW",
	"PTE Write access is not set",
	"PTE Read access is not set",
	"Next page table ptr is invalid",
	"Root table address invalid",
	"Context table ptr is invalid",
	"non-zero reserved fields in RTP",
	"non-zero reserved fields in CTP",
	"non-zero reserved fields in PTE",
1496
	"PCE for translation request specifies blocking",
1497
};
1498

1499
static const char *irq_remap_fault_reasons[] =
1500 1501 1502 1503 1504 1505 1506 1507 1508 1509
{
	"Detected reserved fields in the decoded interrupt-remapped request",
	"Interrupt index exceeded the interrupt-remapping table size",
	"Present field in the IRTE entry is clear",
	"Error accessing interrupt-remapping table pointed by IRTA_REG",
	"Detected reserved fields in the IRTE entry",
	"Blocked a compatibility format interrupt request",
	"Blocked an interrupt request due to source-id verification failure",
};

1510
static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1511
{
1512 1513
	if (fault_reason >= 0x20 && (fault_reason - 0x20 <
					ARRAY_SIZE(irq_remap_fault_reasons))) {
1514
		*fault_type = INTR_REMAP;
1515
		return irq_remap_fault_reasons[fault_reason - 0x20];
1516 1517 1518 1519 1520
	} else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
		*fault_type = DMA_REMAP;
		return dma_remap_fault_reasons[fault_reason];
	} else {
		*fault_type = UNKNOWN;
1521
		return "Unknown";
1522
	}
1523 1524
}

1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535

static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
{
	if (iommu->irq == irq)
		return DMAR_FECTL_REG;
	else if (iommu->pr_irq == irq)
		return DMAR_PECTL_REG;
	else
		BUG();
}

1536
void dmar_msi_unmask(struct irq_data *data)
1537
{
1538
	struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1539
	int reg = dmar_msi_reg(iommu, data->irq);
1540 1541 1542
	unsigned long flag;

	/* unmask it */
1543
	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1544
	writel(0, iommu->reg + reg);
1545
	/* Read a reg to force flush the post write */
1546
	readl(iommu->reg + reg);
1547
	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1548 1549
}

1550
void dmar_msi_mask(struct irq_data *data)
1551
{
1552
	struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1553 1554
	int reg = dmar_msi_reg(iommu, data->irq);
	unsigned long flag;
1555 1556

	/* mask it */
1557
	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1558
	writel(DMA_FECTL_IM, iommu->reg + reg);
1559
	/* Read a reg to force flush the post write */
1560
	readl(iommu->reg + reg);
1561
	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1562 1563 1564 1565
}

void dmar_msi_write(int irq, struct msi_msg *msg)
{
1566
	struct intel_iommu *iommu = irq_get_handler_data(irq);
1567
	int reg = dmar_msi_reg(iommu, irq);
1568 1569
	unsigned long flag;

1570
	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1571 1572 1573
	writel(msg->data, iommu->reg + reg + 4);
	writel(msg->address_lo, iommu->reg + reg + 8);
	writel(msg->address_hi, iommu->reg + reg + 12);
1574
	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1575 1576 1577 1578
}

void dmar_msi_read(int irq, struct msi_msg *msg)
{
1579
	struct intel_iommu *iommu = irq_get_handler_data(irq);
1580
	int reg = dmar_msi_reg(iommu, irq);
1581 1582
	unsigned long flag;

1583
	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1584 1585 1586
	msg->data = readl(iommu->reg + reg + 4);
	msg->address_lo = readl(iommu->reg + reg + 8);
	msg->address_hi = readl(iommu->reg + reg + 12);
1587
	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1588 1589 1590 1591 1592 1593
}

static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
		u8 fault_reason, u16 source_id, unsigned long long addr)
{
	const char *reason;
1594
	int fault_type;
1595

1596
	reason = dmar_get_fault_reason(fault_reason, &fault_type);
1597

1598
	if (fault_type == INTR_REMAP)
1599 1600
		pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index %llx [fault reason %02d] %s\n",
			source_id >> 8, PCI_SLOT(source_id & 0xFF),
1601 1602 1603
			PCI_FUNC(source_id & 0xFF), addr >> 48,
			fault_reason, reason);
	else
1604 1605 1606
		pr_err("[%s] Request device [%02x:%02x.%d] fault addr %llx [fault reason %02d] %s\n",
		       type ? "DMA Read" : "DMA Write",
		       source_id >> 8, PCI_SLOT(source_id & 0xFF),
1607
		       PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1608 1609 1610 1611
	return 0;
}

#define PRIMARY_FAULT_REG_LEN (16)
1612
irqreturn_t dmar_fault(int irq, void *dev_id)
1613 1614 1615 1616 1617
{
	struct intel_iommu *iommu = dev_id;
	int reg, fault_index;
	u32 fault_status;
	unsigned long flag;
1618 1619 1620 1621 1622 1623 1624
	bool ratelimited;
	static DEFINE_RATELIMIT_STATE(rs,
				      DEFAULT_RATELIMIT_INTERVAL,
				      DEFAULT_RATELIMIT_BURST);

	/* Disable printing, simply clear the fault when ratelimited */
	ratelimited = !__ratelimit(&rs);
1625

1626
	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1627
	fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1628
	if (fault_status && !ratelimited)
1629
		pr_err("DRHD: handling fault status reg %x\n", fault_status);
1630 1631 1632

	/* TBD: ignore advanced fault log currently */
	if (!(fault_status & DMA_FSTS_PPF))
1633
		goto unlock_exit;
1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649

	fault_index = dma_fsts_fault_record_index(fault_status);
	reg = cap_fault_reg_offset(iommu->cap);
	while (1) {
		u8 fault_reason;
		u16 source_id;
		u64 guest_addr;
		int type;
		u32 data;

		/* highest 32 bits */
		data = readl(iommu->reg + reg +
				fault_index * PRIMARY_FAULT_REG_LEN + 12);
		if (!(data & DMA_FRCD_F))
			break;

1650 1651 1652
		if (!ratelimited) {
			fault_reason = dma_frcd_fault_reason(data);
			type = dma_frcd_type(data);
1653

1654 1655 1656 1657 1658 1659 1660 1661
			data = readl(iommu->reg + reg +
				     fault_index * PRIMARY_FAULT_REG_LEN + 8);
			source_id = dma_frcd_source_id(data);

			guest_addr = dmar_readq(iommu->reg + reg +
					fault_index * PRIMARY_FAULT_REG_LEN);
			guest_addr = dma_frcd_page_addr(guest_addr);
		}
1662 1663 1664 1665 1666

		/* clear the fault */
		writel(DMA_FRCD_F, iommu->reg + reg +
			fault_index * PRIMARY_FAULT_REG_LEN + 12);

1667
		raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1668

1669 1670 1671
		if (!ratelimited)
			dmar_fault_do_one(iommu, type, fault_reason,
					  source_id, guest_addr);
1672 1673

		fault_index++;
1674
		if (fault_index >= cap_num_fault_regs(iommu->cap))
1675
			fault_index = 0;
1676
		raw_spin_lock_irqsave(&iommu->register_lock, flag);
1677 1678
	}

1679 1680 1681
	writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);

unlock_exit:
1682
	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1683 1684 1685 1686 1687 1688 1689
	return IRQ_HANDLED;
}

int dmar_set_interrupt(struct intel_iommu *iommu)
{
	int irq, ret;

1690 1691 1692 1693 1694 1695
	/*
	 * Check if the fault interrupt is already initialized.
	 */
	if (iommu->irq)
		return 0;

1696 1697 1698 1699
	irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
	if (irq > 0) {
		iommu->irq = irq;
	} else {
1700
		pr_err("No free IRQ vectors\n");
1701 1702 1703
		return -EINVAL;
	}

1704
	ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1705
	if (ret)
1706
		pr_err("Can't request irq\n");
1707 1708
	return ret;
}
1709 1710 1711 1712

int __init enable_drhd_fault_handling(void)
{
	struct dmar_drhd_unit *drhd;
1713
	struct intel_iommu *iommu;
1714 1715 1716 1717

	/*
	 * Enable fault control interrupt.
	 */
1718
	for_each_iommu(iommu, drhd) {
1719
		u32 fault_status;
1720
		int ret = dmar_set_interrupt(iommu);
1721 1722

		if (ret) {
1723
			pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1724 1725 1726
			       (unsigned long long)drhd->reg_base_addr, ret);
			return -1;
		}
1727 1728 1729 1730 1731

		/*
		 * Clear any previous faults.
		 */
		dmar_fault(iommu->irq, iommu);
1732 1733
		fault_status = readl(iommu->reg + DMAR_FSTS_REG);
		writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1734 1735 1736 1737
	}

	return 0;
}
1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762

/*
 * Re-enable Queued Invalidation interface.
 */
int dmar_reenable_qi(struct intel_iommu *iommu)
{
	if (!ecap_qis(iommu->ecap))
		return -ENOENT;

	if (!iommu->qi)
		return -ENOENT;

	/*
	 * First disable queued invalidation.
	 */
	dmar_disable_qi(iommu);
	/*
	 * Then enable queued invalidation again. Since there is no pending
	 * invalidation requests now, it's safe to re-enable queued
	 * invalidation.
	 */
	__dmar_enable_qi(iommu);

	return 0;
}
1763 1764 1765 1766

/*
 * Check interrupt remapping support in DMAR table description.
 */
1767
int __init dmar_ir_support(void)
1768 1769 1770
{
	struct acpi_table_dmar *dmar;
	dmar = (struct acpi_table_dmar *)dmar_tbl;
1771 1772
	if (!dmar)
		return 0;
1773 1774
	return dmar->flags & 0x1;
}
1775

1776 1777 1778 1779 1780 1781
/* Check whether DMAR units are in use */
static inline bool dmar_in_use(void)
{
	return irq_remapping_enabled || intel_iommu_enabled;
}

1782 1783 1784 1785
static int __init dmar_free_unused_resources(void)
{
	struct dmar_drhd_unit *dmaru, *dmaru_n;

1786
	if (dmar_in_use())
1787 1788
		return 0;

1789 1790
	if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
		bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1791

1792
	down_write(&dmar_global_lock);
1793 1794 1795 1796
	list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
		list_del(&dmaru->list);
		dmar_free_drhd(dmaru);
	}
1797
	up_write(&dmar_global_lock);
1798 1799 1800 1801 1802

	return 0;
}

late_initcall(dmar_free_unused_resources);
1803
IOMMU_INIT_POST(detect_intel_iommu);
1804 1805 1806 1807 1808 1809 1810

/*
 * DMAR Hotplug Support
 * For more details, please refer to Intel(R) Virtualization Technology
 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
 * "Remapping Hardware Unit Hot Plug".
 */
1811 1812 1813
static guid_t dmar_hp_guid =
	GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
		  0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825

/*
 * Currently there's only one revision and BIOS will not check the revision id,
 * so use 0 for safety.
 */
#define	DMAR_DSM_REV_ID			0
#define	DMAR_DSM_FUNC_DRHD		1
#define	DMAR_DSM_FUNC_ATSR		2
#define	DMAR_DSM_FUNC_RHSA		3

static inline bool dmar_detect_dsm(acpi_handle handle, int func)
{
1826
	return acpi_check_dsm(handle, &dmar_hp_guid, DMAR_DSM_REV_ID, 1 << func);
1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
}

static int dmar_walk_dsm_resource(acpi_handle handle, int func,
				  dmar_res_handler_t handler, void *arg)
{
	int ret = -ENODEV;
	union acpi_object *obj;
	struct acpi_dmar_header *start;
	struct dmar_res_callback callback;
	static int res_type[] = {
		[DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
		[DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
		[DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
	};

	if (!dmar_detect_dsm(handle, func))
		return 0;

1845
	obj = acpi_evaluate_dsm_typed(handle, &dmar_hp_guid, DMAR_DSM_REV_ID,
1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889
				      func, NULL, ACPI_TYPE_BUFFER);
	if (!obj)
		return -ENODEV;

	memset(&callback, 0, sizeof(callback));
	callback.cb[res_type[func]] = handler;
	callback.arg[res_type[func]] = arg;
	start = (struct acpi_dmar_header *)obj->buffer.pointer;
	ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);

	ACPI_FREE(obj);

	return ret;
}

static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
{
	int ret;
	struct dmar_drhd_unit *dmaru;

	dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
	if (!dmaru)
		return -ENODEV;

	ret = dmar_ir_hotplug(dmaru, true);
	if (ret == 0)
		ret = dmar_iommu_hotplug(dmaru, true);

	return ret;
}

static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
{
	int i, ret;
	struct device *dev;
	struct dmar_drhd_unit *dmaru;

	dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
	if (!dmaru)
		return 0;

	/*
	 * All PCI devices managed by this unit should have been destroyed.
	 */
1890
	if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
1891 1892 1893
		for_each_active_dev_scope(dmaru->devices,
					  dmaru->devices_cnt, i, dev)
			return -EBUSY;
1894
	}
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986

	ret = dmar_ir_hotplug(dmaru, false);
	if (ret == 0)
		ret = dmar_iommu_hotplug(dmaru, false);

	return ret;
}

static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
{
	struct dmar_drhd_unit *dmaru;

	dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
	if (dmaru) {
		list_del_rcu(&dmaru->list);
		synchronize_rcu();
		dmar_free_drhd(dmaru);
	}

	return 0;
}

static int dmar_hotplug_insert(acpi_handle handle)
{
	int ret;
	int drhd_count = 0;

	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
				     &dmar_validate_one_drhd, (void *)1);
	if (ret)
		goto out;

	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
				     &dmar_parse_one_drhd, (void *)&drhd_count);
	if (ret == 0 && drhd_count == 0) {
		pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
		goto out;
	} else if (ret) {
		goto release_drhd;
	}

	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
				     &dmar_parse_one_rhsa, NULL);
	if (ret)
		goto release_drhd;

	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
				     &dmar_parse_one_atsr, NULL);
	if (ret)
		goto release_atsr;

	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
				     &dmar_hp_add_drhd, NULL);
	if (!ret)
		return 0;

	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
			       &dmar_hp_remove_drhd, NULL);
release_atsr:
	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
			       &dmar_release_one_atsr, NULL);
release_drhd:
	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
			       &dmar_hp_release_drhd, NULL);
out:
	return ret;
}

static int dmar_hotplug_remove(acpi_handle handle)
{
	int ret;

	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
				     &dmar_check_one_atsr, NULL);
	if (ret)
		return ret;

	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
				     &dmar_hp_remove_drhd, NULL);
	if (ret == 0) {
		WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
					       &dmar_release_one_atsr, NULL));
		WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
					       &dmar_hp_release_drhd, NULL));
	} else {
		dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
				       &dmar_hp_add_drhd, NULL);
	}

	return ret;
}

1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
				       void *context, void **retval)
{
	acpi_handle *phdl = retval;

	if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
		*phdl = handle;
		return AE_CTRL_TERMINATE;
	}

	return AE_OK;
}

2000 2001 2002
static int dmar_device_hotplug(acpi_handle handle, bool insert)
{
	int ret;
2003 2004
	acpi_handle tmp = NULL;
	acpi_status status;
2005 2006 2007 2008

	if (!dmar_in_use())
		return 0;

2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
	if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
		tmp = handle;
	} else {
		status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
					     ACPI_UINT32_MAX,
					     dmar_get_dsm_handle,
					     NULL, NULL, &tmp);
		if (ACPI_FAILURE(status)) {
			pr_warn("Failed to locate _DSM method.\n");
			return -ENXIO;
		}
	}
	if (tmp == NULL)
2022 2023 2024 2025
		return 0;

	down_write(&dmar_global_lock);
	if (insert)
2026
		ret = dmar_hotplug_insert(tmp);
2027
	else
2028
		ret = dmar_hotplug_remove(tmp);
2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042
	up_write(&dmar_global_lock);

	return ret;
}

int dmar_device_add(acpi_handle handle)
{
	return dmar_device_hotplug(handle, true);
}

int dmar_device_remove(acpi_handle handle)
{
	return dmar_device_hotplug(handle, false);
}