gadget.c 71 KB
Newer Older
1 2 3 4 5 6 7 8
/**
 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
 *
 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
 *
 * Authors: Felipe Balbi <balbi@ti.com>,
 *	    Sebastian Andrzej Siewior <bigeasy@linutronix.de>
 *
9 10 11
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2  of
 * the License as published by the Free Software Foundation.
12
 *
13 14 15 16
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
 */

#include <linux/kernel.h>
#include <linux/delay.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/platform_device.h>
#include <linux/pm_runtime.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/list.h>
#include <linux/dma-mapping.h>

#include <linux/usb/ch9.h>
#include <linux/usb/gadget.h>

33
#include "debug.h"
34 35 36 37
#include "core.h"
#include "gadget.h"
#include "io.h"

38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
/**
 * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
 * @dwc: pointer to our context structure
 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
 *
 * Caller should take care of locking. This function will
 * return 0 on success or -EINVAL if wrong Test Selector
 * is passed
 */
int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
{
	u32		reg;

	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
	reg &= ~DWC3_DCTL_TSTCTRL_MASK;

	switch (mode) {
	case TEST_J:
	case TEST_K:
	case TEST_SE0_NAK:
	case TEST_PACKET:
	case TEST_FORCE_EN:
		reg |= mode << 1;
		break;
	default:
		return -EINVAL;
	}

	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

	return 0;
}

71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86
/**
 * dwc3_gadget_get_link_state - Gets current state of USB Link
 * @dwc: pointer to our context structure
 *
 * Caller should take care of locking. This function will
 * return the link state on success (>= 0) or -ETIMEDOUT.
 */
int dwc3_gadget_get_link_state(struct dwc3 *dwc)
{
	u32		reg;

	reg = dwc3_readl(dwc->regs, DWC3_DSTS);

	return DWC3_DSTS_USBLNKST(reg);
}

87 88 89 90 91 92
/**
 * dwc3_gadget_set_link_state - Sets USB Link to a particular State
 * @dwc: pointer to our context structure
 * @state: the state to put link into
 *
 * Caller should take care of locking. This function will
93
 * return 0 on success or -ETIMEDOUT.
94 95 96
 */
int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
{
97
	int		retries = 10000;
98 99
	u32		reg;

100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
	/*
	 * Wait until device controller is ready. Only applies to 1.94a and
	 * later RTL.
	 */
	if (dwc->revision >= DWC3_REVISION_194A) {
		while (--retries) {
			reg = dwc3_readl(dwc->regs, DWC3_DSTS);
			if (reg & DWC3_DSTS_DCNRD)
				udelay(5);
			else
				break;
		}

		if (retries <= 0)
			return -ETIMEDOUT;
	}

117 118 119 120 121 122 123
	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;

	/* set requested state */
	reg |= DWC3_DCTL_ULSTCHNGREQ(state);
	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

124 125 126 127 128 129 130
	/*
	 * The following code is racy when called from dwc3_gadget_wakeup,
	 * and is not needed, at least on newer versions
	 */
	if (dwc->revision >= DWC3_REVISION_194A)
		return 0;

131
	/* wait for a change in DSTS */
132
	retries = 10000;
133 134 135 136 137 138
	while (--retries) {
		reg = dwc3_readl(dwc->regs, DWC3_DSTS);

		if (DWC3_DSTS_USBLNKST(reg) == state)
			return 0;

139
		udelay(5);
140 141
	}

142 143
	dwc3_trace(trace_dwc3_gadget,
			"link state change request timed out");
144 145 146 147

	return -ETIMEDOUT;
}

148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191
/**
 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
 * @dwc: pointer to our context structure
 *
 * This function will a best effort FIFO allocation in order
 * to improve FIFO usage and throughput, while still allowing
 * us to enable as many endpoints as possible.
 *
 * Keep in mind that this operation will be highly dependent
 * on the configured size for RAM1 - which contains TxFifo -,
 * the amount of endpoints enabled on coreConsultant tool, and
 * the width of the Master Bus.
 *
 * In the ideal world, we would always be able to satisfy the
 * following equation:
 *
 * ((512 + 2 * MDWIDTH-Bytes) + (Number of IN Endpoints - 1) * \
 * (3 * (1024 + MDWIDTH-Bytes) + MDWIDTH-Bytes)) / MDWIDTH-Bytes
 *
 * Unfortunately, due to many variables that's not always the case.
 */
int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
{
	int		last_fifo_depth = 0;
	int		ram1_depth;
	int		fifo_size;
	int		mdwidth;
	int		num;

	if (!dwc->needs_fifo_resize)
		return 0;

	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
	mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);

	/* MDWIDTH is represented in bits, we need it in bytes */
	mdwidth >>= 3;

	/*
	 * FIXME For now we will only allocate 1 wMaxPacketSize space
	 * for each enabled endpoint, later patches will come to
	 * improve this algorithm so that we better use the internal
	 * FIFO space
	 */
192 193 194
	for (num = 0; num < dwc->num_in_eps; num++) {
		/* bit0 indicates direction; 1 means IN ep */
		struct dwc3_ep	*dep = dwc->eps[(num << 1) | 1];
195
		int		mult = 1;
196 197 198 199 200
		int		tmp;

		if (!(dep->flags & DWC3_EP_ENABLED))
			continue;

201 202
		if (usb_endpoint_xfer_bulk(dep->endpoint.desc)
				|| usb_endpoint_xfer_isoc(dep->endpoint.desc))
203 204 205 206 207 208 209 210 211 212 213 214 215 216
			mult = 3;

		/*
		 * REVISIT: the following assumes we will always have enough
		 * space available on the FIFO RAM for all possible use cases.
		 * Make sure that's true somehow and change FIFO allocation
		 * accordingly.
		 *
		 * If we have Bulk or Isochronous endpoints, we want
		 * them to be able to be very, very fast. So we're giving
		 * those endpoints a fifo_size which is enough for 3 full
		 * packets
		 */
		tmp = mult * (dep->endpoint.maxpacket + mdwidth);
217 218 219
		tmp += mdwidth;

		fifo_size = DIV_ROUND_UP(tmp, mdwidth);
220

221 222
		fifo_size |= (last_fifo_depth << 16);

223
		dwc3_trace(trace_dwc3_gadget, "%s: Fifo Addr %04x Size %d",
224 225
				dep->name, last_fifo_depth, fifo_size & 0xffff);

226
		dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num), fifo_size);
227 228 229 230 231 232 233

		last_fifo_depth += (fifo_size & 0xffff);
	}

	return 0;
}

234 235 236 237
void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
		int status)
{
	struct dwc3			*dwc = dep->dwc;
238
	int				i;
239 240

	if (req->queued) {
241 242
		i = 0;
		do {
243
			dep->busy_slot++;
244 245 246 247 248 249 250
			/*
			 * Skip LINK TRB. We can't use req->trb and check for
			 * DWC3_TRBCTL_LINK_TRB because it points the TRB we
			 * just completed (not the LINK TRB).
			 */
			if (((dep->busy_slot & DWC3_TRB_MASK) ==
				DWC3_TRB_NUM- 1) &&
251
				usb_endpoint_xfer_isoc(dep->endpoint.desc))
252 253
				dep->busy_slot++;
		} while(++i < req->request.num_mapped_sgs);
254
		req->queued = false;
255 256
	}
	list_del(&req->list);
257
	req->trb = NULL;
258 259 260 261

	if (req->request.status == -EINPROGRESS)
		req->request.status = status;

262 263 264 265 266
	if (dwc->ep0_bounced && dep->number == 0)
		dwc->ep0_bounced = false;
	else
		usb_gadget_unmap_request(&dwc->gadget, &req->request,
				req->direction);
267 268 269 270

	dev_dbg(dwc->dev, "request %p from %s completed %d/%d ===> %d\n",
			req, dep->name, req->request.actual,
			req->request.length, status);
271
	trace_dwc3_gadget_giveback(req);
272 273

	spin_unlock(&dwc->lock);
274
	usb_gadget_giveback_request(&dep->endpoint, &req->request);
275 276 277
	spin_lock(&dwc->lock);
}

278
int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
279 280 281 282
{
	u32		timeout = 500;
	u32		reg;

283
	trace_dwc3_gadget_generic_cmd(cmd, param);
284

285 286 287 288 289 290
	dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
	dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);

	do {
		reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
		if (!(reg & DWC3_DGCMD_CMDACT)) {
291 292
			dwc3_trace(trace_dwc3_gadget,
					"Command Complete --> %d",
293
					DWC3_DGCMD_STATUS(reg));
294 295
			if (DWC3_DGCMD_STATUS(reg))
				return -EINVAL;
296 297 298 299 300 301 302 303
			return 0;
		}

		/*
		 * We can't sleep here, because it's also called from
		 * interrupt context.
		 */
		timeout--;
304 305 306
		if (!timeout) {
			dwc3_trace(trace_dwc3_gadget,
					"Command Timed Out");
307
			return -ETIMEDOUT;
308
		}
309 310 311 312
		udelay(1);
	} while (1);
}

313 314 315 316
int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
		unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
{
	struct dwc3_ep		*dep = dwc->eps[ep];
317
	u32			timeout = 500;
318 319
	u32			reg;

320
	trace_dwc3_gadget_ep_cmd(dep, cmd, params);
321

322 323 324
	dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
	dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
	dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
325 326 327 328 329

	dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
	do {
		reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
		if (!(reg & DWC3_DEPCMD_CMDACT)) {
330 331
			dwc3_trace(trace_dwc3_gadget,
					"Command Complete --> %d",
332
					DWC3_DEPCMD_STATUS(reg));
333 334
			if (DWC3_DEPCMD_STATUS(reg))
				return -EINVAL;
335 336 337 338 339 340 341 342
			return 0;
		}

		/*
		 * We can't sleep here, because it is also called from
		 * interrupt context.
		 */
		timeout--;
343 344 345
		if (!timeout) {
			dwc3_trace(trace_dwc3_gadget,
					"Command Timed Out");
346
			return -ETIMEDOUT;
347
		}
348

349
		udelay(1);
350 351 352 353
	} while (1);
}

static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
354
		struct dwc3_trb *trb)
355
{
356
	u32		offset = (char *) trb - (char *) dep->trb_pool;
357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390

	return dep->trb_pool_dma + offset;
}

static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
{
	struct dwc3		*dwc = dep->dwc;

	if (dep->trb_pool)
		return 0;

	dep->trb_pool = dma_alloc_coherent(dwc->dev,
			sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
			&dep->trb_pool_dma, GFP_KERNEL);
	if (!dep->trb_pool) {
		dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
				dep->name);
		return -ENOMEM;
	}

	return 0;
}

static void dwc3_free_trb_pool(struct dwc3_ep *dep)
{
	struct dwc3		*dwc = dep->dwc;

	dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
			dep->trb_pool, dep->trb_pool_dma);

	dep->trb_pool = NULL;
	dep->trb_pool_dma = 0;
}

391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep);

/**
 * dwc3_gadget_start_config - Configure EP resources
 * @dwc: pointer to our controller context structure
 * @dep: endpoint that is being enabled
 *
 * The assignment of transfer resources cannot perfectly follow the
 * data book due to the fact that the controller driver does not have
 * all knowledge of the configuration in advance. It is given this
 * information piecemeal by the composite gadget framework after every
 * SET_CONFIGURATION and SET_INTERFACE. Trying to follow the databook
 * programming model in this scenario can cause errors. For two
 * reasons:
 *
 * 1) The databook says to do DEPSTARTCFG for every SET_CONFIGURATION
 * and SET_INTERFACE (8.1.5). This is incorrect in the scenario of
 * multiple interfaces.
 *
 * 2) The databook does not mention doing more DEPXFERCFG for new
 * endpoint on alt setting (8.1.6).
 *
 * The following simplified method is used instead:
 *
 * All hardware endpoints can be assigned a transfer resource and this
 * setting will stay persistent until either a core reset or
 * hibernation. So whenever we do a DEPSTARTCFG(0) we can go ahead and
 * do DEPXFERCFG for every hardware endpoint as well. We are
 * guaranteed that there are as many transfer resources as endpoints.
 *
 * This function is called for each endpoint when it is being enabled
 * but is triggered only when called for EP0-out, which always happens
 * first, and which should only happen in one of the above conditions.
 */
425 426 427 428
static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
{
	struct dwc3_gadget_ep_cmd_params params;
	u32			cmd;
429 430 431 432 433
	int			i;
	int			ret;

	if (dep->number)
		return 0;
434 435

	memset(&params, 0x00, sizeof(params));
436
	cmd = DWC3_DEPCMD_DEPSTARTCFG;
437

438 439 440 441 442 443
	ret = dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
	if (ret)
		return ret;

	for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
		struct dwc3_ep *dep = dwc->eps[i];
444

445 446 447 448 449 450
		if (!dep)
			continue;

		ret = dwc3_gadget_set_xfer_resource(dwc, dep);
		if (ret)
			return ret;
451 452 453 454 455 456
	}

	return 0;
}

static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
457
		const struct usb_endpoint_descriptor *desc,
458
		const struct usb_ss_ep_comp_descriptor *comp_desc,
459
		bool ignore, bool restore)
460 461 462 463 464
{
	struct dwc3_gadget_ep_cmd_params params;

	memset(&params, 0x00, sizeof(params));

465
	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
466 467 468 469 470 471 472 473
		| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));

	/* Burst size is only needed in SuperSpeed mode */
	if (dwc->gadget.speed == USB_SPEED_SUPER) {
		u32 burst = dep->endpoint.maxburst - 1;

		params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst);
	}
474

475 476 477
	if (ignore)
		params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;

478 479 480 481 482
	if (restore) {
		params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
		params.param2 |= dep->saved_state;
	}

483 484
	params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
		| DWC3_DEPCFG_XFER_NOT_READY_EN;
485

486
	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
487 488
		params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
			| DWC3_DEPCFG_STREAM_EVENT_EN;
489 490 491
		dep->stream_capable = true;
	}

492
	if (!usb_endpoint_xfer_control(desc))
493
		params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
494 495 496 497 498 499 500

	/*
	 * We are doing 1:1 mapping for endpoints, meaning
	 * Physical Endpoints 2 maps to Logical Endpoint 2 and
	 * so on. We consider the direction bit as part of the physical
	 * endpoint number. So USB endpoint 0x81 is 0x03.
	 */
501
	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
502 503 504 505 506 507

	/*
	 * We must use the lower 16 TX FIFOs even though
	 * HW might have more
	 */
	if (dep->direction)
508
		params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
509 510

	if (desc->bInterval) {
511
		params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
512 513 514 515 516 517 518 519 520 521 522 523 524
		dep->interval = 1 << (desc->bInterval - 1);
	}

	return dwc3_send_gadget_ep_cmd(dwc, dep->number,
			DWC3_DEPCMD_SETEPCONFIG, &params);
}

static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
{
	struct dwc3_gadget_ep_cmd_params params;

	memset(&params, 0x00, sizeof(params));

525
	params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
526 527 528 529 530 531 532 533 534 535 536 537 538

	return dwc3_send_gadget_ep_cmd(dwc, dep->number,
			DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
}

/**
 * __dwc3_gadget_ep_enable - Initializes a HW endpoint
 * @dep: endpoint to be initialized
 * @desc: USB Endpoint Descriptor
 *
 * Caller should take care of locking
 */
static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
539
		const struct usb_endpoint_descriptor *desc,
540
		const struct usb_ss_ep_comp_descriptor *comp_desc,
541
		bool ignore, bool restore)
542 543 544
{
	struct dwc3		*dwc = dep->dwc;
	u32			reg;
545
	int			ret;
546

547
	dwc3_trace(trace_dwc3_gadget, "Enabling %s", dep->name);
548

549 550 551 552 553 554
	if (!(dep->flags & DWC3_EP_ENABLED)) {
		ret = dwc3_gadget_start_config(dwc, dep);
		if (ret)
			return ret;
	}

555 556
	ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore,
			restore);
557 558 559 560
	if (ret)
		return ret;

	if (!(dep->flags & DWC3_EP_ENABLED)) {
561 562
		struct dwc3_trb	*trb_st_hw;
		struct dwc3_trb	*trb_link;
563

564
		dep->endpoint.desc = desc;
565
		dep->comp_desc = comp_desc;
566 567 568 569 570 571 572 573 574 575
		dep->type = usb_endpoint_type(desc);
		dep->flags |= DWC3_EP_ENABLED;

		reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
		reg |= DWC3_DALEPENA_EP(dep->number);
		dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);

		if (!usb_endpoint_xfer_isoc(desc))
			return 0;

576
		/* Link TRB for ISOC. The HWO bit is never reset */
577 578
		trb_st_hw = &dep->trb_pool[0];

579
		trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
580
		memset(trb_link, 0, sizeof(*trb_link));
581

582 583 584 585
		trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
		trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
		trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
		trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
586 587
	}

588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604
	switch (usb_endpoint_type(desc)) {
	case USB_ENDPOINT_XFER_CONTROL:
		strlcat(dep->name, "-control", sizeof(dep->name));
		break;
	case USB_ENDPOINT_XFER_ISOC:
		strlcat(dep->name, "-isoc", sizeof(dep->name));
		break;
	case USB_ENDPOINT_XFER_BULK:
		strlcat(dep->name, "-bulk", sizeof(dep->name));
		break;
	case USB_ENDPOINT_XFER_INT:
		strlcat(dep->name, "-int", sizeof(dep->name));
		break;
	default:
		dev_err(dwc->dev, "invalid endpoint transfer type\n");
	}

605 606 607
	return 0;
}

608
static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
609
static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
610 611 612
{
	struct dwc3_request		*req;

613
	if (!list_empty(&dep->req_queued)) {
614
		dwc3_stop_active_transfer(dwc, dep->number, true);
615

616
		/* - giveback all requests to gadget driver */
617 618 619 620 621
		while (!list_empty(&dep->req_queued)) {
			req = next_request(&dep->req_queued);

			dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
		}
622 623
	}

624 625 626
	while (!list_empty(&dep->request_list)) {
		req = next_request(&dep->request_list);

627
		dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
628 629 630 631 632 633 634
	}
}

/**
 * __dwc3_gadget_ep_disable - Disables a HW endpoint
 * @dep: the endpoint to disable
 *
635 636 637
 * This function also removes requests which are currently processed ny the
 * hardware and those which are not yet scheduled.
 * Caller should take care of locking.
638 639 640 641 642 643
 */
static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
{
	struct dwc3		*dwc = dep->dwc;
	u32			reg;

644 645
	dwc3_trace(trace_dwc3_gadget, "Disabling %s", dep->name);

646
	dwc3_remove_requests(dwc, dep);
647

648 649
	/* make sure HW endpoint isn't stalled */
	if (dep->flags & DWC3_EP_STALL)
650
		__dwc3_gadget_ep_set_halt(dep, 0, false);
651

652 653 654 655
	reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
	reg &= ~DWC3_DALEPENA_EP(dep->number);
	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);

656
	dep->stream_capable = false;
657
	dep->endpoint.desc = NULL;
658
	dep->comp_desc = NULL;
659
	dep->type = 0;
660
	dep->flags = 0;
661

662 663 664 665
	snprintf(dep->name, sizeof(dep->name), "ep%d%s",
			dep->number >> 1,
			(dep->number & 1) ? "in" : "out");

666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
	return 0;
}

/* -------------------------------------------------------------------------- */

static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
		const struct usb_endpoint_descriptor *desc)
{
	return -EINVAL;
}

static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
{
	return -EINVAL;
}

/* -------------------------------------------------------------------------- */

static int dwc3_gadget_ep_enable(struct usb_ep *ep,
		const struct usb_endpoint_descriptor *desc)
{
	struct dwc3_ep			*dep;
	struct dwc3			*dwc;
	unsigned long			flags;
	int				ret;

	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
		pr_debug("dwc3: invalid parameters\n");
		return -EINVAL;
	}

	if (!desc->wMaxPacketSize) {
		pr_debug("dwc3: missing wMaxPacketSize\n");
		return -EINVAL;
	}

	dep = to_dwc3_ep(ep);
	dwc = dep->dwc;

705 706 707 708 709 710
	if (dep->flags & DWC3_EP_ENABLED) {
		dev_WARN_ONCE(dwc->dev, true, "%s is already enabled\n",
				dep->name);
		return 0;
	}

711
	spin_lock_irqsave(&dwc->lock, flags);
712
	ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);
713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static int dwc3_gadget_ep_disable(struct usb_ep *ep)
{
	struct dwc3_ep			*dep;
	struct dwc3			*dwc;
	unsigned long			flags;
	int				ret;

	if (!ep) {
		pr_debug("dwc3: invalid parameters\n");
		return -EINVAL;
	}

	dep = to_dwc3_ep(ep);
	dwc = dep->dwc;

	if (!(dep->flags & DWC3_EP_ENABLED)) {
		dev_WARN_ONCE(dwc->dev, true, "%s is already disabled\n",
				dep->name);
		return 0;
	}

	spin_lock_irqsave(&dwc->lock, flags);
	ret = __dwc3_gadget_ep_disable(dep);
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
	gfp_t gfp_flags)
{
	struct dwc3_request		*req;
	struct dwc3_ep			*dep = to_dwc3_ep(ep);

	req = kzalloc(sizeof(*req), gfp_flags);
753
	if (!req)
754 755 756 757 758
		return NULL;

	req->epnum	= dep->number;
	req->dep	= dep;

759 760
	trace_dwc3_alloc_request(req);

761 762 763 764 765 766 767 768
	return &req->request;
}

static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
		struct usb_request *request)
{
	struct dwc3_request		*req = to_dwc3_request(request);

769
	trace_dwc3_free_request(req);
770 771 772
	kfree(req);
}

773 774 775 776 777
/**
 * dwc3_prepare_one_trb - setup one TRB from one request
 * @dep: endpoint for which this request is prepared
 * @req: dwc3_request pointer
 */
778
static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
779
		struct dwc3_request *req, dma_addr_t dma,
780
		unsigned length, unsigned last, unsigned chain, unsigned node)
781
{
782
	struct dwc3_trb		*trb;
783

784
	dwc3_trace(trace_dwc3_gadget, "%s: req %p dma %08llx length %d%s%s",
785 786 787 788
			dep->name, req, (unsigned long long) dma,
			length, last ? " last" : "",
			chain ? " chain" : "");

789 790

	trb = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
791

792 793
	if (!req->trb) {
		dwc3_gadget_move_request_queued(req);
794 795
		req->trb = trb;
		req->trb_dma = dwc3_trb_dma_offset(dep, trb);
796
		req->start_slot = dep->free_slot & DWC3_TRB_MASK;
797
	}
798

799
	dep->free_slot++;
800 801 802 803
	/* Skip the LINK-TRB on ISOC */
	if (((dep->free_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
			usb_endpoint_xfer_isoc(dep->endpoint.desc))
		dep->free_slot++;
804

805 806 807
	trb->size = DWC3_TRB_SIZE_LENGTH(length);
	trb->bpl = lower_32_bits(dma);
	trb->bph = upper_32_bits(dma);
808

809
	switch (usb_endpoint_type(dep->endpoint.desc)) {
810
	case USB_ENDPOINT_XFER_CONTROL:
811
		trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
812 813 814
		break;

	case USB_ENDPOINT_XFER_ISOC:
815 816 817 818
		if (!node)
			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
		else
			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
819 820 821 822
		break;

	case USB_ENDPOINT_XFER_BULK:
	case USB_ENDPOINT_XFER_INT:
823
		trb->ctrl = DWC3_TRBCTL_NORMAL;
824 825 826 827 828 829 830 831 832
		break;
	default:
		/*
		 * This is only possible with faulty memory because we
		 * checked it already :)
		 */
		BUG();
	}

833 834 835
	if (!req->request.no_interrupt && !chain)
		trb->ctrl |= DWC3_TRB_CTRL_IOC;

836
	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
837 838
		trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
		trb->ctrl |= DWC3_TRB_CTRL_CSP;
839 840
	} else if (last) {
		trb->ctrl |= DWC3_TRB_CTRL_LST;
841
	}
842

843 844 845
	if (chain)
		trb->ctrl |= DWC3_TRB_CTRL_CHN;

846
	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
847
		trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
848

849
	trb->ctrl |= DWC3_TRB_CTRL_HWO;
850 851

	trace_dwc3_prepare_trb(dep, trb);
852 853
}

854 855 856 857 858
/*
 * dwc3_prepare_trbs - setup TRBs from requests
 * @dep: endpoint for which requests are being prepared
 * @starting: true if the endpoint is idle and no requests are queued.
 *
859 860 861
 * The function goes through the requests list and sets up TRBs for the
 * transfers. The function returns once there are no more TRBs available or
 * it runs out of requests.
862
 */
863
static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting)
864
{
865
	struct dwc3_request	*req, *n;
866
	u32			trbs_left;
867
	u32			max;
868
	unsigned int		last_one = 0;
869 870 871 872 873

	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);

	/* the first request must not be queued */
	trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;
874

875
	/* Can't wrap around on a non-isoc EP since there's no link TRB */
876
	if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
877 878 879 880 881
		max = DWC3_TRB_NUM - (dep->free_slot & DWC3_TRB_MASK);
		if (trbs_left > max)
			trbs_left = max;
	}

882
	/*
883 884
	 * If busy & slot are equal than it is either full or empty. If we are
	 * starting to process requests then we are empty. Otherwise we are
885 886 887 888
	 * full and don't do anything
	 */
	if (!trbs_left) {
		if (!starting)
889
			return;
890 891 892 893 894
		trbs_left = DWC3_TRB_NUM;
		/*
		 * In case we start from scratch, we queue the ISOC requests
		 * starting from slot 1. This is done because we use ring
		 * buffer and have no LST bit to stop us. Instead, we place
895
		 * IOC bit every TRB_NUM/4. We try to avoid having an interrupt
896 897 898 899 900 901
		 * after the first request so we start at slot 1 and have
		 * 7 requests proceed before we hit the first IOC.
		 * Other transfer types don't use the ring buffer and are
		 * processed from the first TRB until the last one. Since we
		 * don't wrap around we have to start at the beginning.
		 */
902
		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
903 904 905 906 907 908 909 910 911
			dep->busy_slot = 1;
			dep->free_slot = 1;
		} else {
			dep->busy_slot = 0;
			dep->free_slot = 0;
		}
	}

	/* The last TRB is a link TRB, not used for xfer */
912
	if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->endpoint.desc))
913
		return;
914 915

	list_for_each_entry_safe(req, n, &dep->request_list, list) {
916 917
		unsigned	length;
		dma_addr_t	dma;
918
		last_one = false;
919

920 921 922 923 924
		if (req->request.num_mapped_sgs > 0) {
			struct usb_request *request = &req->request;
			struct scatterlist *sg = request->sg;
			struct scatterlist *s;
			int		i;
925

926 927
			for_each_sg(sg, s, request->num_mapped_sgs, i) {
				unsigned chain = true;
928

929 930
				length = sg_dma_len(s);
				dma = sg_dma_address(s);
931

932 933
				if (i == (request->num_mapped_sgs - 1) ||
						sg_is_last(s)) {
934
					if (list_empty(&dep->request_list))
935
						last_one = true;
936 937
					chain = false;
				}
938

939 940 941
				trbs_left--;
				if (!trbs_left)
					last_one = true;
942

943 944
				if (last_one)
					chain = false;
945

946
				dwc3_prepare_one_trb(dep, req, dma, length,
947
						last_one, chain, i);
948

949 950 951
				if (last_one)
					break;
			}
952 953 954

			if (last_one)
				break;
955
		} else {
956 957 958
			dma = req->request.dma;
			length = req->request.length;
			trbs_left--;
959

960 961
			if (!trbs_left)
				last_one = 1;
962

963 964 965
			/* Is this the last request? */
			if (list_is_last(&req->list, &dep->request_list))
				last_one = 1;
966

967
			dwc3_prepare_one_trb(dep, req, dma, length,
968
					last_one, false, 0);
969

970 971
			if (last_one)
				break;
972 973 974 975 976 977 978 979 980 981 982 983 984 985
		}
	}
}

static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
		int start_new)
{
	struct dwc3_gadget_ep_cmd_params params;
	struct dwc3_request		*req;
	struct dwc3			*dwc = dep->dwc;
	int				ret;
	u32				cmd;

	if (start_new && (dep->flags & DWC3_EP_BUSY)) {
986
		dwc3_trace(trace_dwc3_gadget, "%s: endpoint busy", dep->name);
987 988 989 990 991 992 993 994 995 996 997 998 999 1000
		return -EBUSY;
	}

	/*
	 * If we are getting here after a short-out-packet we don't enqueue any
	 * new requests as we try to set the IOC bit only on the last request.
	 */
	if (start_new) {
		if (list_empty(&dep->req_queued))
			dwc3_prepare_trbs(dep, start_new);

		/* req points to the first request which will be sent */
		req = next_request(&dep->req_queued);
	} else {
1001 1002
		dwc3_prepare_trbs(dep, start_new);

1003
		/*
1004
		 * req points to the first request where HWO changed from 0 to 1
1005
		 */
1006
		req = next_request(&dep->req_queued);
1007 1008 1009 1010 1011 1012 1013 1014
	}
	if (!req) {
		dep->flags |= DWC3_EP_PENDING_REQUEST;
		return 0;
	}

	memset(&params, 0, sizeof(params));

1015 1016 1017
	if (start_new) {
		params.param0 = upper_32_bits(req->trb_dma);
		params.param1 = lower_32_bits(req->trb_dma);
1018
		cmd = DWC3_DEPCMD_STARTTRANSFER;
1019
	} else {
1020
		cmd = DWC3_DEPCMD_UPDATETRANSFER;
1021
	}
1022 1023 1024 1025 1026 1027 1028 1029 1030

	cmd |= DWC3_DEPCMD_PARAM(cmd_param);
	ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
	if (ret < 0) {
		dev_dbg(dwc->dev, "failed to send STARTTRANSFER command\n");

		/*
		 * FIXME we need to iterate over the list of requests
		 * here and stop, unmap, free and del each of the linked
1031
		 * requests instead of what we do now.
1032
		 */
1033 1034
		usb_gadget_unmap_request(&dwc->gadget, &req->request,
				req->direction);
1035 1036 1037 1038 1039
		list_del(&req->list);
		return ret;
	}

	dep->flags |= DWC3_EP_BUSY;
1040

1041
	if (start_new) {
1042
		dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc,
1043
				dep->number);
1044
		WARN_ON_ONCE(!dep->resource_index);
1045
	}
1046

1047 1048 1049
	return 0;
}

1050 1051 1052 1053 1054 1055
static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
		struct dwc3_ep *dep, u32 cur_uf)
{
	u32 uf;

	if (list_empty(&dep->request_list)) {
1056 1057 1058
		dwc3_trace(trace_dwc3_gadget,
				"ISOC ep %s run out for requests",
				dep->name);
1059
		dep->flags |= DWC3_EP_PENDING_REQUEST;
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
		return;
	}

	/* 4 micro frames in the future */
	uf = cur_uf + dep->interval * 4;

	__dwc3_gadget_kick_transfer(dep, uf, 1);
}

static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
		struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
{
	u32 cur_uf, mask;

	mask = ~(dep->interval - 1);
	cur_uf = event->parameters & mask;

	__dwc3_gadget_start_isoc(dwc, dep, cur_uf);
}

1080 1081
static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
{
1082 1083 1084
	struct dwc3		*dwc = dep->dwc;
	int			ret;

1085 1086 1087 1088 1089
	req->request.actual	= 0;
	req->request.status	= -EINPROGRESS;
	req->direction		= dep->direction;
	req->epnum		= dep->number;

1090 1091
	trace_dwc3_ep_queue(req);

1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
	/*
	 * We only add to our list of requests now and
	 * start consuming the list once we get XferNotReady
	 * IRQ.
	 *
	 * That way, we avoid doing anything that we don't need
	 * to do now and defer it until the point we receive a
	 * particular token from the Host side.
	 *
	 * This will also avoid Host cancelling URBs due to too
1102
	 * many NAKs.
1103
	 */
1104 1105 1106 1107 1108
	ret = usb_gadget_map_request(&dwc->gadget, &req->request,
			dep->direction);
	if (ret)
		return ret;

1109 1110
	list_add_tail(&req->list, &dep->request_list);

1111 1112 1113 1114 1115 1116 1117 1118
	/*
	 * If there are no pending requests and the endpoint isn't already
	 * busy, we will just start the request straight away.
	 *
	 * This will save one IRQ (XFER_NOT_READY) and possibly make it a
	 * little bit faster.
	 */
	if (!usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1119
			!usb_endpoint_xfer_int(dep->endpoint.desc) &&
1120 1121
			!(dep->flags & DWC3_EP_BUSY)) {
		ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1122
		goto out;
1123 1124
	}

1125
	/*
1126
	 * There are a few special cases:
1127
	 *
1128 1129 1130 1131 1132 1133
	 * 1. XferNotReady with empty list of requests. We need to kick the
	 *    transfer here in that situation, otherwise we will be NAKing
	 *    forever. If we get XferNotReady before gadget driver has a
	 *    chance to queue a request, we will ACK the IRQ but won't be
	 *    able to receive the data until the next request is queued.
	 *    The following code is handling exactly that.
1134 1135 1136
	 *
	 */
	if (dep->flags & DWC3_EP_PENDING_REQUEST) {
1137 1138 1139 1140 1141 1142 1143
		/*
		 * If xfernotready is already elapsed and it is a case
		 * of isoc transfer, then issue END TRANSFER, so that
		 * you can receive xfernotready again and can have
		 * notion of current microframe.
		 */
		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1144
			if (list_empty(&dep->req_queued)) {
1145
				dwc3_stop_active_transfer(dwc, dep->number, true);
1146 1147
				dep->flags = DWC3_EP_ENABLED;
			}
1148 1149 1150
			return 0;
		}

1151
		ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1152 1153 1154
		if (!ret)
			dep->flags &= ~DWC3_EP_PENDING_REQUEST;

1155
		goto out;
1156
	}
1157

1158 1159 1160 1161 1162 1163
	/*
	 * 2. XferInProgress on Isoc EP with an active transfer. We need to
	 *    kick the transfer here after queuing a request, otherwise the
	 *    core may not see the modified TRB(s).
	 */
	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1164 1165
			(dep->flags & DWC3_EP_BUSY) &&
			!(dep->flags & DWC3_EP_MISSED_ISOC)) {
1166 1167
		WARN_ON_ONCE(!dep->resource_index);
		ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index,
1168
				false);
1169
		goto out;
1170
	}
1171

1172 1173 1174 1175 1176
	/*
	 * 4. Stream Capable Bulk Endpoints. We need to start the transfer
	 * right away, otherwise host will not know we have streams to be
	 * handled.
	 */
1177
	if (dep->stream_capable)
1178 1179
		ret = __dwc3_gadget_kick_transfer(dep, 0, true);

1180 1181 1182 1183 1184 1185 1186 1187
out:
	if (ret && ret != -EBUSY)
		dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
				dep->name);
	if (ret == -EBUSY)
		ret = 0;

	return ret;
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200
}

static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
	gfp_t gfp_flags)
{
	struct dwc3_request		*req = to_dwc3_request(request);
	struct dwc3_ep			*dep = to_dwc3_ep(ep);
	struct dwc3			*dwc = dep->dwc;

	unsigned long			flags;

	int				ret;

1201
	spin_lock_irqsave(&dwc->lock, flags);
1202
	if (!dep->endpoint.desc) {
1203 1204
		dev_dbg(dwc->dev, "trying to queue request %p to disabled %s\n",
				request, ep->name);
1205 1206 1207 1208 1209 1210 1211 1212
		ret = -ESHUTDOWN;
		goto out;
	}

	if (WARN(req->dep != dep, "request %p belongs to '%s'\n",
				request, req->dep->name)) {
		ret = -EINVAL;
		goto out;
1213 1214 1215
	}

	ret = __dwc3_gadget_ep_queue(dep, req);
1216 1217

out:
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
		struct usb_request *request)
{
	struct dwc3_request		*req = to_dwc3_request(request);
	struct dwc3_request		*r = NULL;

	struct dwc3_ep			*dep = to_dwc3_ep(ep);
	struct dwc3			*dwc = dep->dwc;

	unsigned long			flags;
	int				ret = 0;

1235 1236
	trace_dwc3_ep_dequeue(req);

1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
	spin_lock_irqsave(&dwc->lock, flags);

	list_for_each_entry(r, &dep->request_list, list) {
		if (r == req)
			break;
	}

	if (r != req) {
		list_for_each_entry(r, &dep->req_queued, list) {
			if (r == req)
				break;
		}
		if (r == req) {
			/* wait until it is processed */
1251
			dwc3_stop_active_transfer(dwc, dep->number, true);
1252
			goto out1;
1253 1254 1255 1256 1257 1258 1259
		}
		dev_err(dwc->dev, "request %p was not queued to %s\n",
				request, ep->name);
		ret = -EINVAL;
		goto out0;
	}

1260
out1:
1261 1262 1263 1264 1265 1266 1267 1268 1269
	/* giveback the request */
	dwc3_gadget_giveback(dep, req, -ECONNRESET);

out0:
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

1270
int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1271 1272 1273 1274 1275
{
	struct dwc3_gadget_ep_cmd_params	params;
	struct dwc3				*dwc = dep->dwc;
	int					ret;

1276 1277 1278 1279 1280
	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
		dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
		return -EINVAL;
	}

1281 1282 1283
	memset(&params, 0x00, sizeof(params));

	if (value) {
1284 1285 1286 1287 1288 1289 1290 1291
		if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||
				(!list_empty(&dep->req_queued) ||
				 !list_empty(&dep->request_list)))) {
			dev_dbg(dwc->dev, "%s: pending request, cannot halt\n",
					dep->name);
			return -EAGAIN;
		}

1292 1293 1294
		ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
			DWC3_DEPCMD_SETSTALL, &params);
		if (ret)
1295
			dev_err(dwc->dev, "failed to set STALL on %s\n",
1296 1297 1298 1299 1300 1301 1302
					dep->name);
		else
			dep->flags |= DWC3_EP_STALL;
	} else {
		ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
			DWC3_DEPCMD_CLEARSTALL, &params);
		if (ret)
1303
			dev_err(dwc->dev, "failed to clear STALL on %s\n",
1304 1305
					dep->name);
		else
1306
			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1307
	}
1308

1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321
	return ret;
}

static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
{
	struct dwc3_ep			*dep = to_dwc3_ep(ep);
	struct dwc3			*dwc = dep->dwc;

	unsigned long			flags;

	int				ret;

	spin_lock_irqsave(&dwc->lock, flags);
1322
	ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1323 1324 1325 1326 1327 1328 1329 1330
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
{
	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1331 1332
	struct dwc3			*dwc = dep->dwc;
	unsigned long			flags;
1333
	int				ret;
1334

1335
	spin_lock_irqsave(&dwc->lock, flags);
1336 1337
	dep->flags |= DWC3_EP_WEDGE;

1338
	if (dep->number == 0 || dep->number == 1)
1339
		ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1340
	else
1341
		ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1342 1343 1344
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361
}

/* -------------------------------------------------------------------------- */

static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
	.bLength	= USB_DT_ENDPOINT_SIZE,
	.bDescriptorType = USB_DT_ENDPOINT,
	.bmAttributes	= USB_ENDPOINT_XFER_CONTROL,
};

static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
	.enable		= dwc3_gadget_ep0_enable,
	.disable	= dwc3_gadget_ep0_disable,
	.alloc_request	= dwc3_gadget_ep_alloc_request,
	.free_request	= dwc3_gadget_ep_free_request,
	.queue		= dwc3_gadget_ep0_queue,
	.dequeue	= dwc3_gadget_ep_dequeue,
1362
	.set_halt	= dwc3_gadget_ep0_set_halt,
1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431
	.set_wedge	= dwc3_gadget_ep_set_wedge,
};

static const struct usb_ep_ops dwc3_gadget_ep_ops = {
	.enable		= dwc3_gadget_ep_enable,
	.disable	= dwc3_gadget_ep_disable,
	.alloc_request	= dwc3_gadget_ep_alloc_request,
	.free_request	= dwc3_gadget_ep_free_request,
	.queue		= dwc3_gadget_ep_queue,
	.dequeue	= dwc3_gadget_ep_dequeue,
	.set_halt	= dwc3_gadget_ep_set_halt,
	.set_wedge	= dwc3_gadget_ep_set_wedge,
};

/* -------------------------------------------------------------------------- */

static int dwc3_gadget_get_frame(struct usb_gadget *g)
{
	struct dwc3		*dwc = gadget_to_dwc(g);
	u32			reg;

	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
	return DWC3_DSTS_SOFFN(reg);
}

static int dwc3_gadget_wakeup(struct usb_gadget *g)
{
	struct dwc3		*dwc = gadget_to_dwc(g);

	unsigned long		timeout;
	unsigned long		flags;

	u32			reg;

	int			ret = 0;

	u8			link_state;
	u8			speed;

	spin_lock_irqsave(&dwc->lock, flags);

	/*
	 * According to the Databook Remote wakeup request should
	 * be issued only when the device is in early suspend state.
	 *
	 * We can check that via USB Link State bits in DSTS register.
	 */
	reg = dwc3_readl(dwc->regs, DWC3_DSTS);

	speed = reg & DWC3_DSTS_CONNECTSPD;
	if (speed == DWC3_DSTS_SUPERSPEED) {
		dev_dbg(dwc->dev, "no wakeup on SuperSpeed\n");
		ret = -EINVAL;
		goto out;
	}

	link_state = DWC3_DSTS_USBLNKST(reg);

	switch (link_state) {
	case DWC3_LINK_STATE_RX_DET:	/* in HS, means Early Suspend */
	case DWC3_LINK_STATE_U3:	/* in HS, means SUSPEND */
		break;
	default:
		dev_dbg(dwc->dev, "can't wakeup from link state %d\n",
				link_state);
		ret = -EINVAL;
		goto out;
	}

1432 1433 1434 1435 1436
	ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
	if (ret < 0) {
		dev_err(dwc->dev, "failed to put link in Recovery\n");
		goto out;
	}
1437

1438 1439 1440
	/* Recent versions do this automatically */
	if (dwc->revision < DWC3_REVISION_194A) {
		/* write zeroes to Link Change Request */
1441
		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1442 1443 1444
		reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	}
1445

1446
	/* poll until Link State changes to ON */
1447 1448
	timeout = jiffies + msecs_to_jiffies(100);

1449
	while (!time_after(jiffies, timeout)) {
1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
		reg = dwc3_readl(dwc->regs, DWC3_DSTS);

		/* in HS, means ON */
		if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
			break;
	}

	if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
		dev_err(dwc->dev, "failed to send remote wakeup\n");
		ret = -EINVAL;
	}

out:
	spin_unlock_irqrestore(&dwc->lock, flags);

	return ret;
}

static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
		int is_selfpowered)
{
	struct dwc3		*dwc = gadget_to_dwc(g);
1472
	unsigned long		flags;
1473

1474
	spin_lock_irqsave(&dwc->lock, flags);
1475
	g->is_selfpowered = !!is_selfpowered;
1476
	spin_unlock_irqrestore(&dwc->lock, flags);
1477 1478 1479 1480

	return 0;
}

1481
static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1482 1483
{
	u32			reg;
1484
	u32			timeout = 500;
1485 1486

	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1487
	if (is_on) {
1488 1489 1490 1491 1492 1493 1494 1495
		if (dwc->revision <= DWC3_REVISION_187A) {
			reg &= ~DWC3_DCTL_TRGTULST_MASK;
			reg |= DWC3_DCTL_TRGTULST_RX_DET;
		}

		if (dwc->revision >= DWC3_REVISION_194A)
			reg &= ~DWC3_DCTL_KEEP_CONNECT;
		reg |= DWC3_DCTL_RUN_STOP;
1496 1497 1498 1499

		if (dwc->has_hibernation)
			reg |= DWC3_DCTL_KEEP_CONNECT;

1500
		dwc->pullups_connected = true;
1501
	} else {
1502
		reg &= ~DWC3_DCTL_RUN_STOP;
1503 1504 1505 1506

		if (dwc->has_hibernation && !suspend)
			reg &= ~DWC3_DCTL_KEEP_CONNECT;

1507
		dwc->pullups_connected = false;
1508
	}
1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522

	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

	do {
		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
		if (is_on) {
			if (!(reg & DWC3_DSTS_DEVCTRLHLT))
				break;
		} else {
			if (reg & DWC3_DSTS_DEVCTRLHLT)
				break;
		}
		timeout--;
		if (!timeout)
1523
			return -ETIMEDOUT;
1524
		udelay(1);
1525 1526
	} while (1);

1527
	dwc3_trace(trace_dwc3_gadget, "gadget %s data soft-%s",
1528 1529 1530
			dwc->gadget_driver
			? dwc->gadget_driver->function : "no-function",
			is_on ? "connect" : "disconnect");
1531 1532

	return 0;
1533 1534 1535 1536 1537 1538
}

static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
{
	struct dwc3		*dwc = gadget_to_dwc(g);
	unsigned long		flags;
1539
	int			ret;
1540 1541 1542 1543

	is_on = !!is_on;

	spin_lock_irqsave(&dwc->lock, flags);
1544
	ret = dwc3_gadget_run_stop(dwc, is_on, false);
1545 1546
	spin_unlock_irqrestore(&dwc->lock, flags);

1547
	return ret;
1548 1549
}

1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574
static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
{
	u32			reg;

	/* Enable all but Start and End of Frame IRQs */
	reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
			DWC3_DEVTEN_EVNTOVERFLOWEN |
			DWC3_DEVTEN_CMDCMPLTEN |
			DWC3_DEVTEN_ERRTICERREN |
			DWC3_DEVTEN_WKUPEVTEN |
			DWC3_DEVTEN_ULSTCNGEN |
			DWC3_DEVTEN_CONNECTDONEEN |
			DWC3_DEVTEN_USBRSTEN |
			DWC3_DEVTEN_DISCONNEVTEN);

	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
}

static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
{
	/* mask all interrupts */
	dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
}

static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1575
static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1576

1577 1578 1579 1580 1581 1582 1583
static int dwc3_gadget_start(struct usb_gadget *g,
		struct usb_gadget_driver *driver)
{
	struct dwc3		*dwc = gadget_to_dwc(g);
	struct dwc3_ep		*dep;
	unsigned long		flags;
	int			ret = 0;
1584
	int			irq;
1585 1586
	u32			reg;

1587 1588
	irq = platform_get_irq(to_platform_device(dwc->dev), 0);
	ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1589
			IRQF_SHARED, "dwc3", dwc);
1590 1591 1592 1593 1594 1595
	if (ret) {
		dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
				irq, ret);
		goto err0;
	}

1596 1597 1598 1599 1600 1601 1602
	spin_lock_irqsave(&dwc->lock, flags);

	if (dwc->gadget_driver) {
		dev_err(dwc->dev, "%s is already bound to %s\n",
				dwc->gadget.name,
				dwc->gadget_driver->driver.name);
		ret = -EBUSY;
1603
		goto err1;
1604 1605 1606 1607 1608 1609
	}

	dwc->gadget_driver	= driver;

	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	reg &= ~(DWC3_DCFG_SPEED_MASK);
1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623

	/**
	 * WORKAROUND: DWC3 revision < 2.20a have an issue
	 * which would cause metastability state on Run/Stop
	 * bit if we try to force the IP to USB2-only mode.
	 *
	 * Because of that, we cannot configure the IP to any
	 * speed other than the SuperSpeed
	 *
	 * Refers to:
	 *
	 * STAR#9000525659: Clock Domain Crossing on DCTL in
	 * USB 2.0 Mode
	 */
1624
	if (dwc->revision < DWC3_REVISION_220A) {
1625
		reg |= DWC3_DCFG_SUPERSPEED;
1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642
	} else {
		switch (dwc->maximum_speed) {
		case USB_SPEED_LOW:
			reg |= DWC3_DSTS_LOWSPEED;
			break;
		case USB_SPEED_FULL:
			reg |= DWC3_DSTS_FULLSPEED1;
			break;
		case USB_SPEED_HIGH:
			reg |= DWC3_DSTS_HIGHSPEED;
			break;
		case USB_SPEED_SUPER:	/* FALLTHROUGH */
		case USB_SPEED_UNKNOWN:	/* FALTHROUGH */
		default:
			reg |= DWC3_DSTS_SUPERSPEED;
		}
	}
1643 1644 1645 1646 1647 1648
	dwc3_writel(dwc->regs, DWC3_DCFG, reg);

	/* Start with SuperSpeed Default */
	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);

	dep = dwc->eps[0];
1649 1650
	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
			false);
1651 1652
	if (ret) {
		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1653
		goto err2;
1654 1655 1656
	}

	dep = dwc->eps[1];
1657 1658
	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
			false);
1659 1660
	if (ret) {
		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1661
		goto err3;
1662 1663 1664
	}

	/* begin to receive SETUP packets */
1665
	dwc->ep0state = EP0_SETUP_PHASE;
1666 1667
	dwc3_ep0_out_start(dwc);

1668 1669
	dwc3_gadget_enable_irq(dwc);

1670 1671 1672 1673
	spin_unlock_irqrestore(&dwc->lock, flags);

	return 0;

1674
err3:
1675 1676
	__dwc3_gadget_ep_disable(dwc->eps[0]);

1677
err2:
1678
	dwc->gadget_driver = NULL;
1679 1680

err1:
1681 1682
	spin_unlock_irqrestore(&dwc->lock, flags);

1683 1684 1685
	free_irq(irq, dwc);

err0:
1686 1687 1688
	return ret;
}

1689
static int dwc3_gadget_stop(struct usb_gadget *g)
1690 1691 1692
{
	struct dwc3		*dwc = gadget_to_dwc(g);
	unsigned long		flags;
1693
	int			irq;
1694 1695 1696

	spin_lock_irqsave(&dwc->lock, flags);

1697
	dwc3_gadget_disable_irq(dwc);
1698 1699 1700 1701 1702 1703 1704
	__dwc3_gadget_ep_disable(dwc->eps[0]);
	__dwc3_gadget_ep_disable(dwc->eps[1]);

	dwc->gadget_driver	= NULL;

	spin_unlock_irqrestore(&dwc->lock, flags);

1705 1706 1707
	irq = platform_get_irq(to_platform_device(dwc->dev), 0);
	free_irq(irq, dwc);

1708 1709
	return 0;
}
1710

1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721
static const struct usb_gadget_ops dwc3_gadget_ops = {
	.get_frame		= dwc3_gadget_get_frame,
	.wakeup			= dwc3_gadget_wakeup,
	.set_selfpowered	= dwc3_gadget_set_selfpowered,
	.pullup			= dwc3_gadget_pullup,
	.udc_start		= dwc3_gadget_start,
	.udc_stop		= dwc3_gadget_stop,
};

/* -------------------------------------------------------------------------- */

1722 1723
static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
		u8 num, u32 direction)
1724 1725
{
	struct dwc3_ep			*dep;
1726
	u8				i;
1727

1728 1729
	for (i = 0; i < num; i++) {
		u8 epnum = (i << 1) | (!!direction);
1730 1731

		dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1732
		if (!dep)
1733 1734 1735 1736
			return -ENOMEM;

		dep->dwc = dwc;
		dep->number = epnum;
1737
		dep->direction = !!direction;
1738 1739 1740 1741
		dwc->eps[epnum] = dep;

		snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
				(epnum & 1) ? "in" : "out");
1742

1743 1744
		dep->endpoint.name = dep->name;

1745
		dwc3_trace(trace_dwc3_gadget, "initializing %s", dep->name);
1746

1747
		if (epnum == 0 || epnum == 1) {
1748
			usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
1749
			dep->endpoint.maxburst = 1;
1750 1751 1752 1753 1754 1755
			dep->endpoint.ops = &dwc3_gadget_ep0_ops;
			if (!epnum)
				dwc->gadget.ep0 = &dep->endpoint;
		} else {
			int		ret;

1756
			usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
1757
			dep->endpoint.max_streams = 15;
1758 1759 1760 1761 1762
			dep->endpoint.ops = &dwc3_gadget_ep_ops;
			list_add_tail(&dep->endpoint.ep_list,
					&dwc->gadget.ep_list);

			ret = dwc3_alloc_trb_pool(dep);
1763
			if (ret)
1764 1765
				return ret;
		}
1766

1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
		if (epnum == 0 || epnum == 1) {
			dep->endpoint.caps.type_control = true;
		} else {
			dep->endpoint.caps.type_iso = true;
			dep->endpoint.caps.type_bulk = true;
			dep->endpoint.caps.type_int = true;
		}

		dep->endpoint.caps.dir_in = !!direction;
		dep->endpoint.caps.dir_out = !direction;

1778 1779 1780 1781 1782 1783 1784
		INIT_LIST_HEAD(&dep->request_list);
		INIT_LIST_HEAD(&dep->req_queued);
	}

	return 0;
}

1785 1786 1787 1788 1789 1790 1791 1792
static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
{
	int				ret;

	INIT_LIST_HEAD(&dwc->gadget.ep_list);

	ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
	if (ret < 0) {
1793 1794
		dwc3_trace(trace_dwc3_gadget,
				"failed to allocate OUT endpoints");
1795 1796 1797 1798 1799
		return ret;
	}

	ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
	if (ret < 0) {
1800 1801
		dwc3_trace(trace_dwc3_gadget,
				"failed to allocate IN endpoints");
1802 1803 1804 1805 1806 1807
		return ret;
	}

	return 0;
}

1808 1809 1810 1811 1812 1813 1814
static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
{
	struct dwc3_ep			*dep;
	u8				epnum;

	for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
		dep = dwc->eps[epnum];
1815 1816
		if (!dep)
			continue;
1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827
		/*
		 * Physical endpoints 0 and 1 are special; they form the
		 * bi-directional USB endpoint 0.
		 *
		 * For those two physical endpoints, we don't allocate a TRB
		 * pool nor do we add them the endpoints list. Due to that, we
		 * shouldn't do these two operations otherwise we would end up
		 * with all sorts of bugs when removing dwc3.ko.
		 */
		if (epnum != 0 && epnum != 1) {
			dwc3_free_trb_pool(dep);
1828
			list_del(&dep->endpoint.ep_list);
1829
		}
1830 1831 1832 1833 1834 1835

		kfree(dep);
	}
}

/* -------------------------------------------------------------------------- */
1836

1837 1838
static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
		struct dwc3_request *req, struct dwc3_trb *trb,
1839 1840 1841 1842
		const struct dwc3_event_depevt *event, int status)
{
	unsigned int		count;
	unsigned int		s_pkt = 0;
1843
	unsigned int		trb_status;
1844

1845 1846
	trace_dwc3_complete_trb(dep, trb);

1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923
	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
		/*
		 * We continue despite the error. There is not much we
		 * can do. If we don't clean it up we loop forever. If
		 * we skip the TRB then it gets overwritten after a
		 * while since we use them in a ring buffer. A BUG()
		 * would help. Lets hope that if this occurs, someone
		 * fixes the root cause instead of looking away :)
		 */
		dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
				dep->name, trb);
	count = trb->size & DWC3_TRB_SIZE_MASK;

	if (dep->direction) {
		if (count) {
			trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
			if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
				dev_dbg(dwc->dev, "incomplete IN transfer %s\n",
						dep->name);
				/*
				 * If missed isoc occurred and there is
				 * no request queued then issue END
				 * TRANSFER, so that core generates
				 * next xfernotready and we will issue
				 * a fresh START TRANSFER.
				 * If there are still queued request
				 * then wait, do not issue either END
				 * or UPDATE TRANSFER, just attach next
				 * request in request_list during
				 * giveback.If any future queued request
				 * is successfully transferred then we
				 * will issue UPDATE TRANSFER for all
				 * request in the request_list.
				 */
				dep->flags |= DWC3_EP_MISSED_ISOC;
			} else {
				dev_err(dwc->dev, "incomplete IN transfer %s\n",
						dep->name);
				status = -ECONNRESET;
			}
		} else {
			dep->flags &= ~DWC3_EP_MISSED_ISOC;
		}
	} else {
		if (count && (event->status & DEPEVT_STATUS_SHORT))
			s_pkt = 1;
	}

	/*
	 * We assume here we will always receive the entire data block
	 * which we should receive. Meaning, if we program RX to
	 * receive 4K but we receive only 2K, we assume that's all we
	 * should receive and we simply bounce the request back to the
	 * gadget driver for further processing.
	 */
	req->request.actual += req->request.length - count;
	if (s_pkt)
		return 1;
	if ((event->status & DEPEVT_STATUS_LST) &&
			(trb->ctrl & (DWC3_TRB_CTRL_LST |
				DWC3_TRB_CTRL_HWO)))
		return 1;
	if ((event->status & DEPEVT_STATUS_IOC) &&
			(trb->ctrl & DWC3_TRB_CTRL_IOC))
		return 1;
	return 0;
}

static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
		const struct dwc3_event_depevt *event, int status)
{
	struct dwc3_request	*req;
	struct dwc3_trb		*trb;
	unsigned int		slot;
	unsigned int		i;
	int			ret;

1924
	do {
1925 1926 1927 1928 1929 1930 1931 1932 1933
		req = next_request(&dep->req_queued);
		if (!req) {
			WARN_ON_ONCE(1);
			return 1;
		}
		i = 0;
		do {
			slot = req->start_slot + i;
			if ((slot == DWC3_TRB_NUM - 1) &&
1934
				usb_endpoint_xfer_isoc(dep->endpoint.desc))
1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945
				slot++;
			slot %= DWC3_TRB_NUM;
			trb = &dep->trb_pool[slot];

			ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
					event, status);
			if (ret)
				break;
		} while (++i < req->request.num_mapped_sgs);

		dwc3_gadget_giveback(dep, req, status);
1946 1947

		if (ret)
1948
			break;
1949
	} while (1);
1950

1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
			list_empty(&dep->req_queued)) {
		if (list_empty(&dep->request_list)) {
			/*
			 * If there is no entry in request list then do
			 * not issue END TRANSFER now. Just set PENDING
			 * flag, so that END TRANSFER is issued when an
			 * entry is added into request list.
			 */
			dep->flags = DWC3_EP_PENDING_REQUEST;
		} else {
1962
			dwc3_stop_active_transfer(dwc, dep->number, true);
1963 1964
			dep->flags = DWC3_EP_ENABLED;
		}
1965 1966 1967
		return 1;
	}

1968 1969 1970 1971
	return 1;
}

static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
1972
		struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1973 1974 1975
{
	unsigned		status = 0;
	int			clean_busy;
1976 1977 1978
	u32			is_xfer_complete;

	is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE);
1979 1980 1981 1982

	if (event->status & DEPEVT_STATUS_BUSERR)
		status = -ECONNRESET;

1983
	clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
1984 1985
	if (clean_busy && (is_xfer_complete ||
				usb_endpoint_xfer_isoc(dep->endpoint.desc)))
1986
		dep->flags &= ~DWC3_EP_BUSY;
1987 1988 1989 1990 1991 1992 1993 1994 1995 1996

	/*
	 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
	 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
	 */
	if (dwc->revision < DWC3_REVISION_183A) {
		u32		reg;
		int		i;

		for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
1997
			dep = dwc->eps[i];
1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011

			if (!(dep->flags & DWC3_EP_ENABLED))
				continue;

			if (!list_empty(&dep->req_queued))
				return;
		}

		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
		reg |= dwc->u1u2;
		dwc3_writel(dwc->regs, DWC3_DCTL, reg);

		dwc->u1u2 = 0;
	}
2012

2013
	if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2014 2015
		int ret;

2016
		ret = __dwc3_gadget_kick_transfer(dep, 0, is_xfer_complete);
2017 2018 2019
		if (!ret || ret == -EBUSY)
			return;
	}
2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
}

static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
		const struct dwc3_event_depevt *event)
{
	struct dwc3_ep		*dep;
	u8			epnum = event->endpoint_number;

	dep = dwc->eps[epnum];

2030 2031 2032
	if (!(dep->flags & DWC3_EP_ENABLED))
		return;

2033 2034 2035 2036 2037 2038 2039
	if (epnum == 0 || epnum == 1) {
		dwc3_ep0_interrupt(dwc, event);
		return;
	}

	switch (event->endpoint_event) {
	case DWC3_DEPEVT_XFERCOMPLETE:
2040
		dep->resource_index = 0;
2041

2042
		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2043 2044 2045 2046 2047
			dev_dbg(dwc->dev, "%s is an Isochronous endpoint\n",
					dep->name);
			return;
		}

2048
		dwc3_endpoint_transfer_complete(dwc, dep, event);
2049 2050
		break;
	case DWC3_DEPEVT_XFERINPROGRESS:
2051
		dwc3_endpoint_transfer_complete(dwc, dep, event);
2052 2053
		break;
	case DWC3_DEPEVT_XFERNOTREADY:
2054
		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2055 2056
			dwc3_gadget_start_isoc(dwc, dep, event);
		} else {
2057
			int active;
2058 2059
			int ret;

2060 2061
			active = event->status & DEPEVT_STATUS_TRANSFER_ACTIVE;

2062
			dwc3_trace(trace_dwc3_gadget, "%s: reason %s",
2063
					dep->name, active ? "Transfer Active"
2064 2065
					: "Transfer Not Active");

2066
			ret = __dwc3_gadget_kick_transfer(dep, 0, !active);
2067 2068 2069 2070 2071 2072 2073
			if (!ret || ret == -EBUSY)
				return;

			dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
					dep->name);
		}

2074 2075
		break;
	case DWC3_DEPEVT_STREAMEVT:
2076
		if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
2077 2078 2079 2080 2081 2082 2083
			dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
					dep->name);
			return;
		}

		switch (event->status) {
		case DEPEVT_STREAMEVT_FOUND:
2084 2085
			dwc3_trace(trace_dwc3_gadget,
					"Stream %d found and started",
2086 2087 2088 2089 2090 2091 2092 2093
					event->parameters);

			break;
		case DEPEVT_STREAMEVT_NOTFOUND:
			/* FALLTHROUGH */
		default:
			dev_dbg(dwc->dev, "Couldn't find suitable stream\n");
		}
2094 2095 2096 2097 2098
		break;
	case DWC3_DEPEVT_RXTXFIFOEVT:
		dev_dbg(dwc->dev, "%s FIFO Overrun\n", dep->name);
		break;
	case DWC3_DEPEVT_EPCMDCMPLT:
2099
		dwc3_trace(trace_dwc3_gadget, "Endpoint Command Complete");
2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112
		break;
	}
}

static void dwc3_disconnect_gadget(struct dwc3 *dwc)
{
	if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
		spin_unlock(&dwc->lock);
		dwc->gadget_driver->disconnect(&dwc->gadget);
		spin_lock(&dwc->lock);
	}
}

2113 2114
static void dwc3_suspend_gadget(struct dwc3 *dwc)
{
2115
	if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2116 2117 2118 2119 2120 2121 2122 2123
		spin_unlock(&dwc->lock);
		dwc->gadget_driver->suspend(&dwc->gadget);
		spin_lock(&dwc->lock);
	}
}

static void dwc3_resume_gadget(struct dwc3 *dwc)
{
2124
	if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2125 2126
		spin_unlock(&dwc->lock);
		dwc->gadget_driver->resume(&dwc->gadget);
2127
		spin_lock(&dwc->lock);
2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138
	}
}

static void dwc3_reset_gadget(struct dwc3 *dwc)
{
	if (!dwc->gadget_driver)
		return;

	if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
		spin_unlock(&dwc->lock);
		usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2139 2140 2141 2142
		spin_lock(&dwc->lock);
	}
}

2143
static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
2144 2145 2146 2147 2148 2149 2150 2151
{
	struct dwc3_ep *dep;
	struct dwc3_gadget_ep_cmd_params params;
	u32 cmd;
	int ret;

	dep = dwc->eps[epnum];

2152
	if (!dep->resource_index)
2153 2154
		return;

2155 2156 2157 2158 2159 2160 2161 2162 2163
	/*
	 * NOTICE: We are violating what the Databook says about the
	 * EndTransfer command. Ideally we would _always_ wait for the
	 * EndTransfer Command Completion IRQ, but that's causing too
	 * much trouble synchronizing between us and gadget driver.
	 *
	 * We have discussed this with the IP Provider and it was
	 * suggested to giveback all requests here, but give HW some
	 * extra time to synchronize with the interconnect. We're using
2164
	 * an arbitrary 100us delay for that.
2165 2166 2167 2168 2169 2170 2171 2172 2173
	 *
	 * Note also that a similar handling was tested by Synopsys
	 * (thanks a lot Paul) and nothing bad has come out of it.
	 * In short, what we're doing is:
	 *
	 * - Issue EndTransfer WITH CMDIOC bit set
	 * - Wait 100us
	 */

2174
	cmd = DWC3_DEPCMD_ENDTRANSFER;
2175 2176
	cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
	cmd |= DWC3_DEPCMD_CMDIOC;
2177
	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2178 2179 2180
	memset(&params, 0, sizeof(params));
	ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
	WARN_ON_ONCE(ret);
2181
	dep->resource_index = 0;
2182
	dep->flags &= ~DWC3_EP_BUSY;
2183
	udelay(100);
2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
}

static void dwc3_stop_active_transfers(struct dwc3 *dwc)
{
	u32 epnum;

	for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
		struct dwc3_ep *dep;

		dep = dwc->eps[epnum];
2194 2195 2196
		if (!dep)
			continue;

2197 2198 2199
		if (!(dep->flags & DWC3_EP_ENABLED))
			continue;

2200
		dwc3_remove_requests(dwc, dep);
2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213
	}
}

static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
{
	u32 epnum;

	for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
		struct dwc3_ep *dep;
		struct dwc3_gadget_ep_cmd_params params;
		int ret;

		dep = dwc->eps[epnum];
2214 2215
		if (!dep)
			continue;
2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230

		if (!(dep->flags & DWC3_EP_STALL))
			continue;

		dep->flags &= ~DWC3_EP_STALL;

		memset(&params, 0, sizeof(params));
		ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
				DWC3_DEPCMD_CLEARSTALL, &params);
		WARN_ON_ONCE(ret);
	}
}

static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
{
2231 2232
	int			reg;

2233 2234 2235 2236 2237 2238 2239 2240 2241 2242
	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
	reg &= ~DWC3_DCTL_INITU1ENA;
	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

	reg &= ~DWC3_DCTL_INITU2ENA;
	dwc3_writel(dwc->regs, DWC3_DCTL, reg);

	dwc3_disconnect_gadget(dwc);

	dwc->gadget.speed = USB_SPEED_UNKNOWN;
2243
	dwc->setup_packet_pending = false;
2244
	usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2245 2246 2247 2248 2249 2250
}

static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
{
	u32			reg;

2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281
	/*
	 * WORKAROUND: DWC3 revisions <1.88a have an issue which
	 * would cause a missing Disconnect Event if there's a
	 * pending Setup Packet in the FIFO.
	 *
	 * There's no suggested workaround on the official Bug
	 * report, which states that "unless the driver/application
	 * is doing any special handling of a disconnect event,
	 * there is no functional issue".
	 *
	 * Unfortunately, it turns out that we _do_ some special
	 * handling of a disconnect event, namely complete all
	 * pending transfers, notify gadget driver of the
	 * disconnection, and so on.
	 *
	 * Our suggested workaround is to follow the Disconnect
	 * Event steps here, instead, based on a setup_packet_pending
	 * flag. Such flag gets set whenever we have a XferNotReady
	 * event on EP0 and gets cleared on XferComplete for the
	 * same endpoint.
	 *
	 * Refers to:
	 *
	 * STAR#9000466709: RTL: Device : Disconnect event not
	 * generated if setup packet pending in FIFO
	 */
	if (dwc->revision < DWC3_REVISION_188A) {
		if (dwc->setup_packet_pending)
			dwc3_gadget_disconnect_interrupt(dwc);
	}

2282
	dwc3_reset_gadget(dwc);
2283 2284 2285 2286

	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2287
	dwc->test_mode = false;
2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337

	dwc3_stop_active_transfers(dwc);
	dwc3_clear_stall_all_ep(dwc);

	/* Reset device address to zero */
	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
	reg &= ~(DWC3_DCFG_DEVADDR_MASK);
	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
}

static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
{
	u32 reg;
	u32 usb30_clock = DWC3_GCTL_CLK_BUS;

	/*
	 * We change the clock only at SS but I dunno why I would want to do
	 * this. Maybe it becomes part of the power saving plan.
	 */

	if (speed != DWC3_DSTS_SUPERSPEED)
		return;

	/*
	 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
	 * each time on Connect Done.
	 */
	if (!usb30_clock)
		return;

	reg = dwc3_readl(dwc->regs, DWC3_GCTL);
	reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
	dwc3_writel(dwc->regs, DWC3_GCTL, reg);
}

static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
{
	struct dwc3_ep		*dep;
	int			ret;
	u32			reg;
	u8			speed;

	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
	speed = reg & DWC3_DSTS_CONNECTSPD;
	dwc->speed = speed;

	dwc3_update_ram_clk_sel(dwc, speed);

	switch (speed) {
	case DWC3_DCFG_SUPERSPEED:
2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353
		/*
		 * WORKAROUND: DWC3 revisions <1.90a have an issue which
		 * would cause a missing USB3 Reset event.
		 *
		 * In such situations, we should force a USB3 Reset
		 * event by calling our dwc3_gadget_reset_interrupt()
		 * routine.
		 *
		 * Refers to:
		 *
		 * STAR#9000483510: RTL: SS : USB3 reset event may
		 * not be generated always when the link enters poll
		 */
		if (dwc->revision < DWC3_REVISION_190A)
			dwc3_gadget_reset_interrupt(dwc);

2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375
		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
		dwc->gadget.ep0->maxpacket = 512;
		dwc->gadget.speed = USB_SPEED_SUPER;
		break;
	case DWC3_DCFG_HIGHSPEED:
		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
		dwc->gadget.ep0->maxpacket = 64;
		dwc->gadget.speed = USB_SPEED_HIGH;
		break;
	case DWC3_DCFG_FULLSPEED2:
	case DWC3_DCFG_FULLSPEED1:
		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
		dwc->gadget.ep0->maxpacket = 64;
		dwc->gadget.speed = USB_SPEED_FULL;
		break;
	case DWC3_DCFG_LOWSPEED:
		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
		dwc->gadget.ep0->maxpacket = 8;
		dwc->gadget.speed = USB_SPEED_LOW;
		break;
	}

2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386
	/* Enable USB2 LPM Capability */

	if ((dwc->revision > DWC3_REVISION_194A)
			&& (speed != DWC3_DCFG_SUPERSPEED)) {
		reg = dwc3_readl(dwc->regs, DWC3_DCFG);
		reg |= DWC3_DCFG_LPM_CAP;
		dwc3_writel(dwc->regs, DWC3_DCFG, reg);

		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
		reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);

2387
		reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2388

2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
		/*
		 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
		 * DCFG.LPMCap is set, core responses with an ACK and the
		 * BESL value in the LPM token is less than or equal to LPM
		 * NYET threshold.
		 */
		WARN_ONCE(dwc->revision < DWC3_REVISION_240A
				&& dwc->has_lpm_erratum,
				"LPM Erratum not available on dwc3 revisisions < 2.40a\n");

		if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
			reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);

2402 2403 2404 2405
		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	} else {
		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
		reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2406 2407 2408
		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
	}

2409
	dep = dwc->eps[0];
2410 2411
	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
			false);
2412 2413 2414 2415 2416 2417
	if (ret) {
		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
		return;
	}

	dep = dwc->eps[1];
2418 2419
	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
			false);
2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446
	if (ret) {
		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
		return;
	}

	/*
	 * Configure PHY via GUSB3PIPECTLn if required.
	 *
	 * Update GTXFIFOSIZn
	 *
	 * In both cases reset values should be sufficient.
	 */
}

static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
{
	/*
	 * TODO take core out of low power mode when that's
	 * implemented.
	 */

	dwc->gadget_driver->resume(&dwc->gadget);
}

static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
		unsigned int evtinfo)
{
2447
	enum dwc3_link_state	next = evtinfo & DWC3_LINK_STATE_MASK;
2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471
	unsigned int		pwropt;

	/*
	 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
	 * Hibernation mode enabled which would show up when device detects
	 * host-initiated U3 exit.
	 *
	 * In that case, device will generate a Link State Change Interrupt
	 * from U3 to RESUME which is only necessary if Hibernation is
	 * configured in.
	 *
	 * There are no functional changes due to such spurious event and we
	 * just need to ignore it.
	 *
	 * Refers to:
	 *
	 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
	 * operational mode
	 */
	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
	if ((dwc->revision < DWC3_REVISION_250A) &&
			(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
		if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
				(next == DWC3_LINK_STATE_RESUME)) {
2472 2473
			dwc3_trace(trace_dwc3_gadget,
					"ignoring transition U3 -> Resume");
2474 2475 2476
			return;
		}
	}
2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523

	/*
	 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
	 * on the link partner, the USB session might do multiple entry/exit
	 * of low power states before a transfer takes place.
	 *
	 * Due to this problem, we might experience lower throughput. The
	 * suggested workaround is to disable DCTL[12:9] bits if we're
	 * transitioning from U1/U2 to U0 and enable those bits again
	 * after a transfer completes and there are no pending transfers
	 * on any of the enabled endpoints.
	 *
	 * This is the first half of that workaround.
	 *
	 * Refers to:
	 *
	 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
	 * core send LGO_Ux entering U0
	 */
	if (dwc->revision < DWC3_REVISION_183A) {
		if (next == DWC3_LINK_STATE_U0) {
			u32	u1u2;
			u32	reg;

			switch (dwc->link_state) {
			case DWC3_LINK_STATE_U1:
			case DWC3_LINK_STATE_U2:
				reg = dwc3_readl(dwc->regs, DWC3_DCTL);
				u1u2 = reg & (DWC3_DCTL_INITU2ENA
						| DWC3_DCTL_ACCEPTU2ENA
						| DWC3_DCTL_INITU1ENA
						| DWC3_DCTL_ACCEPTU1ENA);

				if (!dwc->u1u2)
					dwc->u1u2 = reg & u1u2;

				reg &= ~u1u2;

				dwc3_writel(dwc->regs, DWC3_DCTL, reg);
				break;
			default:
				/* do nothing */
				break;
			}
		}
	}

2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540
	switch (next) {
	case DWC3_LINK_STATE_U1:
		if (dwc->speed == USB_SPEED_SUPER)
			dwc3_suspend_gadget(dwc);
		break;
	case DWC3_LINK_STATE_U2:
	case DWC3_LINK_STATE_U3:
		dwc3_suspend_gadget(dwc);
		break;
	case DWC3_LINK_STATE_RESUME:
		dwc3_resume_gadget(dwc);
		break;
	default:
		/* do nothing */
		break;
	}

2541
	dwc->link_state = next;
2542 2543
}

2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567
static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
		unsigned int evtinfo)
{
	unsigned int is_ss = evtinfo & BIT(4);

	/**
	 * WORKAROUND: DWC3 revison 2.20a with hibernation support
	 * have a known issue which can cause USB CV TD.9.23 to fail
	 * randomly.
	 *
	 * Because of this issue, core could generate bogus hibernation
	 * events which SW needs to ignore.
	 *
	 * Refers to:
	 *
	 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
	 * Device Fallback from SuperSpeed
	 */
	if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
		return;

	/* enter hibernation here */
}

2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583
static void dwc3_gadget_interrupt(struct dwc3 *dwc,
		const struct dwc3_event_devt *event)
{
	switch (event->type) {
	case DWC3_DEVICE_EVENT_DISCONNECT:
		dwc3_gadget_disconnect_interrupt(dwc);
		break;
	case DWC3_DEVICE_EVENT_RESET:
		dwc3_gadget_reset_interrupt(dwc);
		break;
	case DWC3_DEVICE_EVENT_CONNECT_DONE:
		dwc3_gadget_conndone_interrupt(dwc);
		break;
	case DWC3_DEVICE_EVENT_WAKEUP:
		dwc3_gadget_wakeup_interrupt(dwc);
		break;
2584 2585 2586 2587 2588 2589 2590
	case DWC3_DEVICE_EVENT_HIBER_REQ:
		if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
					"unexpected hibernation event\n"))
			break;

		dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
		break;
2591 2592 2593 2594
	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
		dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
		break;
	case DWC3_DEVICE_EVENT_EOPF:
2595
		dwc3_trace(trace_dwc3_gadget, "End of Periodic Frame");
2596 2597
		break;
	case DWC3_DEVICE_EVENT_SOF:
2598
		dwc3_trace(trace_dwc3_gadget, "Start of Periodic Frame");
2599 2600
		break;
	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2601
		dwc3_trace(trace_dwc3_gadget, "Erratic Error");
2602 2603
		break;
	case DWC3_DEVICE_EVENT_CMD_CMPL:
2604
		dwc3_trace(trace_dwc3_gadget, "Command Complete");
2605 2606
		break;
	case DWC3_DEVICE_EVENT_OVERFLOW:
2607
		dwc3_trace(trace_dwc3_gadget, "Overflow");
2608 2609
		break;
	default:
2610
		dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2611 2612 2613 2614 2615 2616
	}
}

static void dwc3_process_event_entry(struct dwc3 *dwc,
		const union dwc3_event *event)
{
2617 2618
	trace_dwc3_event(event->raw);

2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634
	/* Endpoint IRQ, handle it and return early */
	if (event->type.is_devspec == 0) {
		/* depevt */
		return dwc3_endpoint_interrupt(dwc, &event->depevt);
	}

	switch (event->type.type) {
	case DWC3_EVENT_TYPE_DEV:
		dwc3_gadget_interrupt(dwc, &event->devt);
		break;
	/* REVISIT what to do with Carkit and I2C events ? */
	default:
		dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
	}
}

2635
static irqreturn_t dwc3_process_event_buf(struct dwc3 *dwc, u32 buf)
2636
{
2637
	struct dwc3_event_buffer *evt;
2638
	irqreturn_t ret = IRQ_NONE;
2639
	int left;
2640
	u32 reg;
2641

2642 2643
	evt = dwc->ev_buffs[buf];
	left = evt->count;
2644

2645 2646
	if (!(evt->flags & DWC3_EVENT_PENDING))
		return IRQ_NONE;
2647

2648 2649
	while (left > 0) {
		union dwc3_event event;
2650

2651
		event.raw = *(u32 *) (evt->buf + evt->lpos);
2652

2653
		dwc3_process_event_entry(dwc, &event);
2654

2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665
		/*
		 * FIXME we wrap around correctly to the next entry as
		 * almost all entries are 4 bytes in size. There is one
		 * entry which has 12 bytes which is a regular entry
		 * followed by 8 bytes data. ATM I don't know how
		 * things are organized if we get next to the a
		 * boundary so I worry about that once we try to handle
		 * that.
		 */
		evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
		left -= 4;
2666

2667 2668
		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(buf), 4);
	}
2669

2670 2671 2672
	evt->count = 0;
	evt->flags &= ~DWC3_EVENT_PENDING;
	ret = IRQ_HANDLED;
2673

2674 2675 2676 2677
	/* Unmask interrupt */
	reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(buf));
	reg &= ~DWC3_GEVNTSIZ_INTMASK;
	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(buf), reg);
2678

2679 2680
	return ret;
}
2681

2682 2683 2684
static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc)
{
	struct dwc3 *dwc = _dwc;
2685
	unsigned long flags;
2686 2687 2688
	irqreturn_t ret = IRQ_NONE;
	int i;

2689
	spin_lock_irqsave(&dwc->lock, flags);
2690 2691 2692

	for (i = 0; i < dwc->num_event_buffers; i++)
		ret |= dwc3_process_event_buf(dwc, i);
2693

2694
	spin_unlock_irqrestore(&dwc->lock, flags);
2695 2696 2697 2698

	return ret;
}

2699
static irqreturn_t dwc3_check_event_buf(struct dwc3 *dwc, u32 buf)
2700 2701 2702
{
	struct dwc3_event_buffer *evt;
	u32 count;
2703
	u32 reg;
2704

2705 2706
	evt = dwc->ev_buffs[buf];

2707 2708 2709 2710 2711
	count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(buf));
	count &= DWC3_GEVNTCOUNT_MASK;
	if (!count)
		return IRQ_NONE;

2712 2713
	evt->count = count;
	evt->flags |= DWC3_EVENT_PENDING;
2714

2715 2716 2717 2718 2719
	/* Mask interrupt */
	reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(buf));
	reg |= DWC3_GEVNTSIZ_INTMASK;
	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(buf), reg);

2720
	return IRQ_WAKE_THREAD;
2721 2722 2723 2724 2725 2726 2727 2728
}

static irqreturn_t dwc3_interrupt(int irq, void *_dwc)
{
	struct dwc3			*dwc = _dwc;
	int				i;
	irqreturn_t			ret = IRQ_NONE;

2729
	for (i = 0; i < dwc->num_event_buffers; i++) {
2730 2731
		irqreturn_t status;

2732
		status = dwc3_check_event_buf(dwc, i);
2733
		if (status == IRQ_WAKE_THREAD)
2734 2735 2736 2737 2738 2739 2740 2741
			ret = status;
	}

	return ret;
}

/**
 * dwc3_gadget_init - Initializes gadget related registers
2742
 * @dwc: pointer to our controller context structure
2743 2744 2745
 *
 * Returns 0 on success otherwise negative errno.
 */
2746
int dwc3_gadget_init(struct dwc3 *dwc)
2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757
{
	int					ret;

	dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
			&dwc->ctrl_req_addr, GFP_KERNEL);
	if (!dwc->ctrl_req) {
		dev_err(dwc->dev, "failed to allocate ctrl request\n");
		ret = -ENOMEM;
		goto err0;
	}

2758
	dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb) * 2,
2759 2760 2761 2762 2763 2764 2765
			&dwc->ep0_trb_addr, GFP_KERNEL);
	if (!dwc->ep0_trb) {
		dev_err(dwc->dev, "failed to allocate ep0 trb\n");
		ret = -ENOMEM;
		goto err1;
	}

2766
	dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
2767 2768 2769 2770 2771
	if (!dwc->setup_buf) {
		ret = -ENOMEM;
		goto err2;
	}

2772
	dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
2773 2774
			DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
			GFP_KERNEL);
2775 2776 2777 2778 2779 2780
	if (!dwc->ep0_bounce) {
		dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
		ret = -ENOMEM;
		goto err3;
	}

2781 2782
	dwc->gadget.ops			= &dwc3_gadget_ops;
	dwc->gadget.speed		= USB_SPEED_UNKNOWN;
2783
	dwc->gadget.sg_supported	= true;
2784 2785
	dwc->gadget.name		= "dwc3-gadget";

2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808
	/*
	 * FIXME We might be setting max_speed to <SUPER, however versions
	 * <2.20a of dwc3 have an issue with metastability (documented
	 * elsewhere in this driver) which tells us we can't set max speed to
	 * anything lower than SUPER.
	 *
	 * Because gadget.max_speed is only used by composite.c and function
	 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
	 * to happen so we avoid sending SuperSpeed Capability descriptor
	 * together with our BOS descriptor as that could confuse host into
	 * thinking we can handle super speed.
	 *
	 * Note that, in fact, we won't even support GetBOS requests when speed
	 * is less than super speed because we don't have means, yet, to tell
	 * composite.c that we are USB 2.0 + LPM ECN.
	 */
	if (dwc->revision < DWC3_REVISION_220A)
		dwc3_trace(trace_dwc3_gadget,
				"Changing max_speed on rev %08x\n",
				dwc->revision);

	dwc->gadget.max_speed		= dwc->maximum_speed;

2809 2810 2811 2812 2813 2814
	/*
	 * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
	 * on ep out.
	 */
	dwc->gadget.quirk_ep_out_aligned_size = true;

2815 2816 2817 2818 2819 2820 2821
	/*
	 * REVISIT: Here we should clear all pending IRQs to be
	 * sure we're starting from a well known location.
	 */

	ret = dwc3_gadget_init_endpoints(dwc);
	if (ret)
2822
		goto err4;
2823 2824 2825 2826

	ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
	if (ret) {
		dev_err(dwc->dev, "failed to register udc\n");
2827
		goto err4;
2828 2829 2830 2831
	}

	return 0;

2832
err4:
2833
	dwc3_gadget_free_endpoints(dwc);
2834 2835
	dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
			dwc->ep0_bounce, dwc->ep0_bounce_addr);
2836

2837
err3:
2838
	kfree(dwc->setup_buf);
2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851

err2:
	dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
			dwc->ep0_trb, dwc->ep0_trb_addr);

err1:
	dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
			dwc->ctrl_req, dwc->ctrl_req_addr);

err0:
	return ret;
}

2852 2853
/* -------------------------------------------------------------------------- */

2854 2855 2856 2857 2858 2859
void dwc3_gadget_exit(struct dwc3 *dwc)
{
	usb_del_gadget_udc(&dwc->gadget);

	dwc3_gadget_free_endpoints(dwc);

2860 2861
	dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
			dwc->ep0_bounce, dwc->ep0_bounce_addr);
2862

2863
	kfree(dwc->setup_buf);
2864 2865 2866 2867 2868 2869 2870

	dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
			dwc->ep0_trb, dwc->ep0_trb_addr);

	dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
			dwc->ctrl_req, dwc->ctrl_req_addr);
}
2871

2872
int dwc3_gadget_suspend(struct dwc3 *dwc)
2873
{
2874
	if (dwc->pullups_connected) {
2875
		dwc3_gadget_disable_irq(dwc);
2876 2877
		dwc3_gadget_run_stop(dwc, true, true);
	}
2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895

	__dwc3_gadget_ep_disable(dwc->eps[0]);
	__dwc3_gadget_ep_disable(dwc->eps[1]);

	dwc->dcfg = dwc3_readl(dwc->regs, DWC3_DCFG);

	return 0;
}

int dwc3_gadget_resume(struct dwc3 *dwc)
{
	struct dwc3_ep		*dep;
	int			ret;

	/* Start with SuperSpeed Default */
	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);

	dep = dwc->eps[0];
2896 2897
	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
			false);
2898 2899 2900 2901
	if (ret)
		goto err0;

	dep = dwc->eps[1];
2902 2903
	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
			false);
2904 2905 2906 2907 2908 2909 2910 2911 2912
	if (ret)
		goto err1;

	/* begin to receive SETUP packets */
	dwc->ep0state = EP0_SETUP_PHASE;
	dwc3_ep0_out_start(dwc);

	dwc3_writel(dwc->regs, DWC3_DCFG, dwc->dcfg);

2913 2914 2915 2916 2917
	if (dwc->pullups_connected) {
		dwc3_gadget_enable_irq(dwc);
		dwc3_gadget_run_stop(dwc, true, false);
	}

2918 2919 2920 2921 2922 2923 2924 2925
	return 0;

err1:
	__dwc3_gadget_ep_disable(dwc->eps[0]);

err0:
	return ret;
}