bpmp.c 19.2 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7 8
/*
 * Copyright (c) 2016, NVIDIA CORPORATION.  All rights reserved.
 */

#include <linux/clk/tegra.h>
#include <linux/genalloc.h>
#include <linux/mailbox_client.h>
9
#include <linux/module.h>
10 11 12 13
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_device.h>
#include <linux/platform_device.h>
14
#include <linux/pm.h>
15
#include <linux/semaphore.h>
16
#include <linux/sched/clock.h>
17 18 19 20 21

#include <soc/tegra/bpmp.h>
#include <soc/tegra/bpmp-abi.h>
#include <soc/tegra/ivc.h>

22 23
#include "bpmp-private.h"

24 25
#define MSG_ACK		BIT(0)
#define MSG_RING	BIT(1)
26
#define TAG_SZ		32
27 28 29 30 31 32 33

static inline struct tegra_bpmp *
mbox_client_to_bpmp(struct mbox_client *client)
{
	return container_of(client, struct tegra_bpmp, mbox.client);
}

34 35 36 37 38 39 40 41
static inline const struct tegra_bpmp_ops *
channel_to_ops(struct tegra_bpmp_channel *channel)
{
	struct tegra_bpmp *bpmp = channel->bpmp;

	return bpmp->soc->ops;
}

42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
struct tegra_bpmp *tegra_bpmp_get(struct device *dev)
{
	struct platform_device *pdev;
	struct tegra_bpmp *bpmp;
	struct device_node *np;

	np = of_parse_phandle(dev->of_node, "nvidia,bpmp", 0);
	if (!np)
		return ERR_PTR(-ENOENT);

	pdev = of_find_device_by_node(np);
	if (!pdev) {
		bpmp = ERR_PTR(-ENODEV);
		goto put;
	}

	bpmp = platform_get_drvdata(pdev);
	if (!bpmp) {
		bpmp = ERR_PTR(-EPROBE_DEFER);
		put_device(&pdev->dev);
		goto put;
	}

put:
	of_node_put(np);
	return bpmp;
}
EXPORT_SYMBOL_GPL(tegra_bpmp_get);

void tegra_bpmp_put(struct tegra_bpmp *bpmp)
{
	if (bpmp)
		put_device(bpmp->dev);
}
EXPORT_SYMBOL_GPL(tegra_bpmp_put);

static int
tegra_bpmp_channel_get_thread_index(struct tegra_bpmp_channel *channel)
{
	struct tegra_bpmp *bpmp = channel->bpmp;
82
	unsigned int count;
83 84 85 86
	int index;

	count = bpmp->soc->channels.thread.count;

87 88
	index = channel - channel->bpmp->threaded_channels;
	if (index < 0 || index >= count)
89 90
		return -EINVAL;

91
	return index;
92 93 94 95 96 97 98 99 100 101
}

static bool tegra_bpmp_message_valid(const struct tegra_bpmp_message *msg)
{
	return (msg->tx.size <= MSG_DATA_MIN_SZ) &&
	       (msg->rx.size <= MSG_DATA_MIN_SZ) &&
	       (msg->tx.size == 0 || msg->tx.data) &&
	       (msg->rx.size == 0 || msg->rx.data);
}

102
static bool tegra_bpmp_is_response_ready(struct tegra_bpmp_channel *channel)
103
{
104
	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
105

106
	return ops->is_response_ready(channel);
107 108
}

109 110
static bool tegra_bpmp_is_request_ready(struct tegra_bpmp_channel *channel)
{
111 112 113
	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);

	return ops->is_request_ready(channel);
114 115 116
}

static int tegra_bpmp_wait_response(struct tegra_bpmp_channel *channel)
117 118 119 120 121 122 123
{
	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
	ktime_t end;

	end = ktime_add_us(ktime_get(), timeout);

	do {
124
		if (tegra_bpmp_is_response_ready(channel))
125 126 127 128 129 130
			return 0;
	} while (ktime_before(ktime_get(), end));

	return -ETIMEDOUT;
}

131 132
static int tegra_bpmp_ack_response(struct tegra_bpmp_channel *channel)
{
133 134 135
	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);

	return ops->ack_response(channel);
136 137 138 139
}

static int tegra_bpmp_ack_request(struct tegra_bpmp_channel *channel)
{
140 141 142
	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);

	return ops->ack_request(channel);
143 144 145 146
}

static bool
tegra_bpmp_is_request_channel_free(struct tegra_bpmp_channel *channel)
147
{
148
	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);
149

150
	return ops->is_request_channel_free(channel);
151 152
}

153 154 155
static bool
tegra_bpmp_is_response_channel_free(struct tegra_bpmp_channel *channel)
{
156 157 158
	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);

	return ops->is_response_channel_free(channel);
159 160 161 162
}

static int
tegra_bpmp_wait_request_channel_free(struct tegra_bpmp_channel *channel)
163 164 165 166 167 168 169
{
	unsigned long timeout = channel->bpmp->soc->channels.cpu_tx.timeout;
	ktime_t start, now;

	start = ns_to_ktime(local_clock());

	do {
170
		if (tegra_bpmp_is_request_channel_free(channel))
171 172 173 174 175 176 177 178
			return 0;

		now = ns_to_ktime(local_clock());
	} while (ktime_us_delta(now, start) < timeout);

	return -ETIMEDOUT;
}

179 180
static int tegra_bpmp_post_request(struct tegra_bpmp_channel *channel)
{
181 182 183
	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);

	return ops->post_request(channel);
184 185 186 187
}

static int tegra_bpmp_post_response(struct tegra_bpmp_channel *channel)
{
188 189 190
	const struct tegra_bpmp_ops *ops = channel_to_ops(channel);

	return ops->post_response(channel);
191 192 193 194
}

static int tegra_bpmp_ring_doorbell(struct tegra_bpmp *bpmp)
{
195
	return bpmp->soc->ops->ring_doorbell(bpmp);
196 197
}

198
static ssize_t __tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
199
					 void *data, size_t size, int *ret)
200
{
201 202
	int err;

203 204 205
	if (data && size > 0)
		memcpy(data, channel->ib->data, size);

206
	err = tegra_bpmp_ack_response(channel);
207 208 209 210 211 212
	if (err < 0)
		return err;

	*ret = channel->ib->code;

	return 0;
213 214 215
}

static ssize_t tegra_bpmp_channel_read(struct tegra_bpmp_channel *channel,
216
				       void *data, size_t size, int *ret)
217 218 219 220 221 222 223
{
	struct tegra_bpmp *bpmp = channel->bpmp;
	unsigned long flags;
	ssize_t err;
	int index;

	index = tegra_bpmp_channel_get_thread_index(channel);
224 225 226 227
	if (index < 0) {
		err = index;
		goto unlock;
	}
228 229

	spin_lock_irqsave(&bpmp->lock, flags);
230
	err = __tegra_bpmp_channel_read(channel, data, size, ret);
231 232 233
	clear_bit(index, bpmp->threaded.allocated);
	spin_unlock_irqrestore(&bpmp->lock, flags);

234
unlock:
235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
	up(&bpmp->threaded.lock);

	return err;
}

static ssize_t __tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
					  unsigned int mrq, unsigned long flags,
					  const void *data, size_t size)
{
	channel->ob->code = mrq;
	channel->ob->flags = flags;

	if (data && size > 0)
		memcpy(channel->ob->data, data, size);

250
	return tegra_bpmp_post_request(channel);
251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
}

static struct tegra_bpmp_channel *
tegra_bpmp_write_threaded(struct tegra_bpmp *bpmp, unsigned int mrq,
			  const void *data, size_t size)
{
	unsigned long timeout = bpmp->soc->channels.thread.timeout;
	unsigned int count = bpmp->soc->channels.thread.count;
	struct tegra_bpmp_channel *channel;
	unsigned long flags;
	unsigned int index;
	int err;

	err = down_timeout(&bpmp->threaded.lock, usecs_to_jiffies(timeout));
	if (err < 0)
		return ERR_PTR(err);

	spin_lock_irqsave(&bpmp->lock, flags);

	index = find_first_zero_bit(bpmp->threaded.allocated, count);
	if (index == count) {
272
		err = -EBUSY;
273 274 275
		goto unlock;
	}

276
	channel = &bpmp->threaded_channels[index];
277

278
	if (!tegra_bpmp_is_request_channel_free(channel)) {
279
		err = -EBUSY;
280 281 282 283 284 285 286
		goto unlock;
	}

	set_bit(index, bpmp->threaded.allocated);

	err = __tegra_bpmp_channel_write(channel, mrq, MSG_ACK | MSG_RING,
					 data, size);
287 288
	if (err < 0)
		goto clear_allocated;
289 290 291 292 293

	set_bit(index, bpmp->threaded.busy);

	spin_unlock_irqrestore(&bpmp->lock, flags);
	return channel;
294 295 296 297 298 299 300 301

clear_allocated:
	clear_bit(index, bpmp->threaded.allocated);
unlock:
	spin_unlock_irqrestore(&bpmp->lock, flags);
	up(&bpmp->threaded.lock);

	return ERR_PTR(err);
302 303 304 305 306 307 308 309
}

static ssize_t tegra_bpmp_channel_write(struct tegra_bpmp_channel *channel,
					unsigned int mrq, unsigned long flags,
					const void *data, size_t size)
{
	int err;

310
	err = tegra_bpmp_wait_request_channel_free(channel);
311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
	if (err < 0)
		return err;

	return __tegra_bpmp_channel_write(channel, mrq, flags, data, size);
}

int tegra_bpmp_transfer_atomic(struct tegra_bpmp *bpmp,
			       struct tegra_bpmp_message *msg)
{
	struct tegra_bpmp_channel *channel;
	int err;

	if (WARN_ON(!irqs_disabled()))
		return -EPERM;

	if (!tegra_bpmp_message_valid(msg))
		return -EINVAL;

329 330 331
	channel = bpmp->tx_channel;

	spin_lock(&bpmp->atomic_tx_lock);
332 333 334

	err = tegra_bpmp_channel_write(channel, msg->mrq, MSG_ACK,
				       msg->tx.data, msg->tx.size);
335 336
	if (err < 0) {
		spin_unlock(&bpmp->atomic_tx_lock);
337
		return err;
338 339 340
	}

	spin_unlock(&bpmp->atomic_tx_lock);
341

342
	err = tegra_bpmp_ring_doorbell(bpmp);
343 344 345
	if (err < 0)
		return err;

346
	err = tegra_bpmp_wait_response(channel);
347 348 349
	if (err < 0)
		return err;

350 351
	return __tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
					 &msg->rx.ret);
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372
}
EXPORT_SYMBOL_GPL(tegra_bpmp_transfer_atomic);

int tegra_bpmp_transfer(struct tegra_bpmp *bpmp,
			struct tegra_bpmp_message *msg)
{
	struct tegra_bpmp_channel *channel;
	unsigned long timeout;
	int err;

	if (WARN_ON(irqs_disabled()))
		return -EPERM;

	if (!tegra_bpmp_message_valid(msg))
		return -EINVAL;

	channel = tegra_bpmp_write_threaded(bpmp, msg->mrq, msg->tx.data,
					    msg->tx.size);
	if (IS_ERR(channel))
		return PTR_ERR(channel);

373
	err = tegra_bpmp_ring_doorbell(bpmp);
374 375 376 377 378 379 380 381 382
	if (err < 0)
		return err;

	timeout = usecs_to_jiffies(bpmp->soc->channels.thread.timeout);

	err = wait_for_completion_timeout(&channel->completion, timeout);
	if (err == 0)
		return -ETIMEDOUT;

383 384
	return tegra_bpmp_channel_read(channel, msg->rx.data, msg->rx.size,
				       &msg->rx.ret);
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399
}
EXPORT_SYMBOL_GPL(tegra_bpmp_transfer);

static struct tegra_bpmp_mrq *tegra_bpmp_find_mrq(struct tegra_bpmp *bpmp,
						  unsigned int mrq)
{
	struct tegra_bpmp_mrq *entry;

	list_for_each_entry(entry, &bpmp->mrqs, list)
		if (entry->mrq == mrq)
			return entry;

	return NULL;
}

400 401
void tegra_bpmp_mrq_return(struct tegra_bpmp_channel *channel, int code,
			   const void *data, size_t size)
402 403 404 405 406 407 408 409
{
	unsigned long flags = channel->ib->flags;
	struct tegra_bpmp *bpmp = channel->bpmp;
	int err;

	if (WARN_ON(size > MSG_DATA_MIN_SZ))
		return;

410
	err = tegra_bpmp_ack_request(channel);
411 412 413 414 415 416
	if (WARN_ON(err < 0))
		return;

	if ((flags & MSG_ACK) == 0)
		return;

417
	if (WARN_ON(!tegra_bpmp_is_response_channel_free(channel)))
418 419
		return;

420
	channel->ob->code = code;
421 422

	if (data && size > 0)
423
		memcpy(channel->ob->data, data, size);
424

425
	err = tegra_bpmp_post_response(channel);
426 427 428 429
	if (WARN_ON(err < 0))
		return;

	if (flags & MSG_RING) {
430
		err = tegra_bpmp_ring_doorbell(bpmp);
431 432 433 434
		if (WARN_ON(err < 0))
			return;
	}
}
435
EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_return);
436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502

static void tegra_bpmp_handle_mrq(struct tegra_bpmp *bpmp,
				  unsigned int mrq,
				  struct tegra_bpmp_channel *channel)
{
	struct tegra_bpmp_mrq *entry;
	u32 zero = 0;

	spin_lock(&bpmp->lock);

	entry = tegra_bpmp_find_mrq(bpmp, mrq);
	if (!entry) {
		spin_unlock(&bpmp->lock);
		tegra_bpmp_mrq_return(channel, -EINVAL, &zero, sizeof(zero));
		return;
	}

	entry->handler(mrq, channel, entry->data);

	spin_unlock(&bpmp->lock);
}

int tegra_bpmp_request_mrq(struct tegra_bpmp *bpmp, unsigned int mrq,
			   tegra_bpmp_mrq_handler_t handler, void *data)
{
	struct tegra_bpmp_mrq *entry;
	unsigned long flags;

	if (!handler)
		return -EINVAL;

	entry = devm_kzalloc(bpmp->dev, sizeof(*entry), GFP_KERNEL);
	if (!entry)
		return -ENOMEM;

	spin_lock_irqsave(&bpmp->lock, flags);

	entry->mrq = mrq;
	entry->handler = handler;
	entry->data = data;
	list_add(&entry->list, &bpmp->mrqs);

	spin_unlock_irqrestore(&bpmp->lock, flags);

	return 0;
}
EXPORT_SYMBOL_GPL(tegra_bpmp_request_mrq);

void tegra_bpmp_free_mrq(struct tegra_bpmp *bpmp, unsigned int mrq, void *data)
{
	struct tegra_bpmp_mrq *entry;
	unsigned long flags;

	spin_lock_irqsave(&bpmp->lock, flags);

	entry = tegra_bpmp_find_mrq(bpmp, mrq);
	if (!entry)
		goto unlock;

	list_del(&entry->list);
	devm_kfree(bpmp->dev, entry);

unlock:
	spin_unlock_irqrestore(&bpmp->lock, flags);
}
EXPORT_SYMBOL_GPL(tegra_bpmp_free_mrq);

503 504 505 506 507 508 509 510 511 512 513 514 515 516 517
bool tegra_bpmp_mrq_is_supported(struct tegra_bpmp *bpmp, unsigned int mrq)
{
	struct mrq_query_abi_request req = { .mrq = cpu_to_le32(mrq) };
	struct mrq_query_abi_response resp;
	struct tegra_bpmp_message msg = {
		.mrq = MRQ_QUERY_ABI,
		.tx = {
			.data = &req,
			.size = sizeof(req),
		},
		.rx = {
			.data = &resp,
			.size = sizeof(resp),
		},
	};
518
	int err;
519

520 521
	err = tegra_bpmp_transfer(bpmp, &msg);
	if (err || msg.rx.ret)
522 523 524 525 526 527
		return false;

	return resp.status == 0;
}
EXPORT_SYMBOL_GPL(tegra_bpmp_mrq_is_supported);

528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578
static void tegra_bpmp_mrq_handle_ping(unsigned int mrq,
				       struct tegra_bpmp_channel *channel,
				       void *data)
{
	struct mrq_ping_request *request;
	struct mrq_ping_response response;

	request = (struct mrq_ping_request *)channel->ib->data;

	memset(&response, 0, sizeof(response));
	response.reply = request->challenge << 1;

	tegra_bpmp_mrq_return(channel, 0, &response, sizeof(response));
}

static int tegra_bpmp_ping(struct tegra_bpmp *bpmp)
{
	struct mrq_ping_response response;
	struct mrq_ping_request request;
	struct tegra_bpmp_message msg;
	unsigned long flags;
	ktime_t start, end;
	int err;

	memset(&request, 0, sizeof(request));
	request.challenge = 1;

	memset(&response, 0, sizeof(response));

	memset(&msg, 0, sizeof(msg));
	msg.mrq = MRQ_PING;
	msg.tx.data = &request;
	msg.tx.size = sizeof(request);
	msg.rx.data = &response;
	msg.rx.size = sizeof(response);

	local_irq_save(flags);
	start = ktime_get();
	err = tegra_bpmp_transfer_atomic(bpmp, &msg);
	end = ktime_get();
	local_irq_restore(flags);

	if (!err)
		dev_dbg(bpmp->dev,
			"ping ok: challenge: %u, response: %u, time: %lld\n",
			request.challenge, response.reply,
			ktime_to_us(ktime_sub(end, start)));

	return err;
}

579 580 581
/* deprecated version of tag query */
static int tegra_bpmp_get_firmware_tag_old(struct tegra_bpmp *bpmp, char *tag,
					   size_t size)
582 583 584 585 586 587 588 589
{
	struct mrq_query_tag_request request;
	struct tegra_bpmp_message msg;
	unsigned long flags;
	dma_addr_t phys;
	void *virt;
	int err;

590 591 592 593
	if (size != TAG_SZ)
		return -EINVAL;

	virt = dma_alloc_coherent(bpmp->dev, TAG_SZ, &phys,
594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610
				  GFP_KERNEL | GFP_DMA32);
	if (!virt)
		return -ENOMEM;

	memset(&request, 0, sizeof(request));
	request.addr = phys;

	memset(&msg, 0, sizeof(msg));
	msg.mrq = MRQ_QUERY_TAG;
	msg.tx.data = &request;
	msg.tx.size = sizeof(request);

	local_irq_save(flags);
	err = tegra_bpmp_transfer_atomic(bpmp, &msg);
	local_irq_restore(flags);

	if (err == 0)
611
		memcpy(tag, virt, TAG_SZ);
612

613
	dma_free_coherent(bpmp->dev, TAG_SZ, virt, phys);
614 615 616 617

	return err;
}

618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
static int tegra_bpmp_get_firmware_tag(struct tegra_bpmp *bpmp, char *tag,
				       size_t size)
{
	if (tegra_bpmp_mrq_is_supported(bpmp, MRQ_QUERY_FW_TAG)) {
		struct mrq_query_fw_tag_response resp;
		struct tegra_bpmp_message msg = {
			.mrq = MRQ_QUERY_FW_TAG,
			.rx = {
				.data = &resp,
				.size = sizeof(resp),
			},
		};
		int err;

		if (size != sizeof(resp.tag))
			return -EINVAL;

		err = tegra_bpmp_transfer(bpmp, &msg);

		if (err)
			return err;
		if (msg.rx.ret < 0)
			return -EINVAL;

		memcpy(tag, resp.tag, sizeof(resp.tag));
		return 0;
	}

	return tegra_bpmp_get_firmware_tag_old(bpmp, tag, size);
}

649 650 651 652 653 654 655 656 657 658
static void tegra_bpmp_channel_signal(struct tegra_bpmp_channel *channel)
{
	unsigned long flags = channel->ob->flags;

	if ((flags & MSG_RING) == 0)
		return;

	complete(&channel->completion);
}

659
void tegra_bpmp_handle_rx(struct tegra_bpmp *bpmp)
660 661 662 663 664
{
	struct tegra_bpmp_channel *channel;
	unsigned int i, count;
	unsigned long *busy;

665
	channel = bpmp->rx_channel;
666 667 668
	count = bpmp->soc->channels.thread.count;
	busy = bpmp->threaded.busy;

669
	if (tegra_bpmp_is_request_ready(channel))
670 671 672 673 674 675 676
		tegra_bpmp_handle_mrq(bpmp, channel->ib->code, channel);

	spin_lock(&bpmp->lock);

	for_each_set_bit(i, busy, count) {
		struct tegra_bpmp_channel *channel;

677
		channel = &bpmp->threaded_channels[i];
678

679
		if (tegra_bpmp_is_response_ready(channel)) {
680 681 682 683 684 685 686 687 688 689 690
			tegra_bpmp_channel_signal(channel);
			clear_bit(i, busy);
		}
	}

	spin_unlock(&bpmp->lock);
}

static int tegra_bpmp_probe(struct platform_device *pdev)
{
	struct tegra_bpmp *bpmp;
691
	char tag[TAG_SZ];
692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
	size_t size;
	int err;

	bpmp = devm_kzalloc(&pdev->dev, sizeof(*bpmp), GFP_KERNEL);
	if (!bpmp)
		return -ENOMEM;

	bpmp->soc = of_device_get_match_data(&pdev->dev);
	bpmp->dev = &pdev->dev;

	INIT_LIST_HEAD(&bpmp->mrqs);
	spin_lock_init(&bpmp->lock);

	bpmp->threaded.count = bpmp->soc->channels.thread.count;
	sema_init(&bpmp->threaded.lock, bpmp->threaded.count);

	size = BITS_TO_LONGS(bpmp->threaded.count) * sizeof(long);

	bpmp->threaded.allocated = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
711 712
	if (!bpmp->threaded.allocated)
		return -ENOMEM;
713 714

	bpmp->threaded.busy = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
715 716
	if (!bpmp->threaded.busy)
		return -ENOMEM;
717

718 719 720
	spin_lock_init(&bpmp->atomic_tx_lock);
	bpmp->tx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->tx_channel),
					GFP_KERNEL);
721 722
	if (!bpmp->tx_channel)
		return -ENOMEM;
723

724 725
	bpmp->rx_channel = devm_kzalloc(&pdev->dev, sizeof(*bpmp->rx_channel),
	                                GFP_KERNEL);
726 727
	if (!bpmp->rx_channel)
		return -ENOMEM;
728

729 730 731
	bpmp->threaded_channels = devm_kcalloc(&pdev->dev, bpmp->threaded.count,
					       sizeof(*bpmp->threaded_channels),
					       GFP_KERNEL);
732 733
	if (!bpmp->threaded_channels)
		return -ENOMEM;
734

735
	err = bpmp->soc->ops->init(bpmp);
736
	if (err < 0)
737
		return err;
738 739 740 741

	err = tegra_bpmp_request_mrq(bpmp, MRQ_PING,
				     tegra_bpmp_mrq_handle_ping, bpmp);
	if (err < 0)
742
		goto deinit;
743 744 745 746 747 748 749

	err = tegra_bpmp_ping(bpmp);
	if (err < 0) {
		dev_err(&pdev->dev, "failed to ping BPMP: %d\n", err);
		goto free_mrq;
	}

750
	err = tegra_bpmp_get_firmware_tag(bpmp, tag, sizeof(tag));
751 752 753 754 755
	if (err < 0) {
		dev_err(&pdev->dev, "failed to get firmware tag: %d\n", err);
		goto free_mrq;
	}

756
	dev_info(&pdev->dev, "firmware: %.*s\n", (int)sizeof(tag), tag);
757

758 759
	platform_set_drvdata(pdev, bpmp);

760 761 762 763
	err = of_platform_default_populate(pdev->dev.of_node, NULL, &pdev->dev);
	if (err < 0)
		goto free_mrq;

764 765 766 767 768
	if (of_find_property(pdev->dev.of_node, "#clock-cells", NULL)) {
		err = tegra_bpmp_init_clocks(bpmp);
		if (err < 0)
			goto free_mrq;
	}
769

770 771 772 773 774
	if (of_find_property(pdev->dev.of_node, "#reset-cells", NULL)) {
		err = tegra_bpmp_init_resets(bpmp);
		if (err < 0)
			goto free_mrq;
	}
775

776 777 778 779 780
	if (of_find_property(pdev->dev.of_node, "#power-domain-cells", NULL)) {
		err = tegra_bpmp_init_powergates(bpmp);
		if (err < 0)
			goto free_mrq;
	}
781

782 783 784 785
	err = tegra_bpmp_init_debugfs(bpmp);
	if (err < 0)
		dev_err(&pdev->dev, "debugfs initialization failed: %d\n", err);

786 787 788 789
	return 0;

free_mrq:
	tegra_bpmp_free_mrq(bpmp, MRQ_PING, bpmp);
790 791 792
deinit:
	if (bpmp->soc->ops->deinit)
		bpmp->soc->ops->deinit(bpmp);
793

794 795 796
	return err;
}

797 798 799 800
static int __maybe_unused tegra_bpmp_resume(struct device *dev)
{
	struct tegra_bpmp *bpmp = dev_get_drvdata(dev);

801 802 803 804
	if (bpmp->soc->ops->resume)
		return bpmp->soc->ops->resume(bpmp);
	else
		return 0;
805 806
}

807
static const struct dev_pm_ops tegra_bpmp_pm_ops = {
808
	.resume_noirq = tegra_bpmp_resume,
809
};
810

811 812
#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
    IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC)
813 814 815
static const struct tegra_bpmp_soc tegra186_soc = {
	.channels = {
		.cpu_tx = {
816
			.offset = 3,
817 818 819
			.timeout = 60 * USEC_PER_SEC,
		},
		.thread = {
820 821
			.offset = 0,
			.count = 3,
822 823 824 825 826 827 828
			.timeout = 600 * USEC_PER_SEC,
		},
		.cpu_rx = {
			.offset = 13,
			.timeout = 0,
		},
	},
829
	.ops = &tegra186_bpmp_ops,
830 831
	.num_resets = 193,
};
832
#endif
833

834
#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854
static const struct tegra_bpmp_soc tegra210_soc = {
	.channels = {
		.cpu_tx = {
			.offset = 0,
			.count = 1,
			.timeout = 60 * USEC_PER_SEC,
		},
		.thread = {
			.offset = 4,
			.count = 1,
			.timeout = 600 * USEC_PER_SEC,
		},
		.cpu_rx = {
			.offset = 8,
			.count = 1,
			.timeout = 0,
		},
	},
	.ops = &tegra210_bpmp_ops,
};
855
#endif
856

857
static const struct of_device_id tegra_bpmp_match[] = {
858
#if IS_ENABLED(CONFIG_ARCH_TEGRA_186_SOC) || \
859 860
    IS_ENABLED(CONFIG_ARCH_TEGRA_194_SOC) || \
    IS_ENABLED(CONFIG_ARCH_TEGRA_234_SOC)
861
	{ .compatible = "nvidia,tegra186-bpmp", .data = &tegra186_soc },
862 863
#endif
#if IS_ENABLED(CONFIG_ARCH_TEGRA_210_SOC)
864
	{ .compatible = "nvidia,tegra210-bpmp", .data = &tegra210_soc },
865
#endif
866 867 868 869 870 871 872
	{ }
};

static struct platform_driver tegra_bpmp_driver = {
	.driver = {
		.name = "tegra-bpmp",
		.of_match_table = tegra_bpmp_match,
873
		.pm = &tegra_bpmp_pm_ops,
874
		.suppress_bind_attrs = true,
875 876 877
	},
	.probe = tegra_bpmp_probe,
};
878
builtin_platform_driver(tegra_bpmp_driver);