hcd_intr.c 67.1 KB
Newer Older
1
// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
/*
 * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
 *
 * Copyright (C) 2004-2013 Synopsys, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions, and the following disclaimer,
 *    without modification.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The names of the above-listed copyright holders may not be used
 *    to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * ALTERNATIVELY, this software may be distributed under the terms of the
 * GNU General Public License ("GPL") as published by the Free Software
 * Foundation; either version 2 of the License, or (at your option) any
 * later version.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * This file contains the interrupt handlers for Host mode
 */
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/dma-mapping.h>
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/usb.h>

#include <linux/usb/hcd.h>
#include <linux/usb/ch11.h>

#include "core.h"
#include "hcd.h"

56 57 58 59 60 61
/*
 * If we get this many NAKs on a split transaction we'll slow down
 * retransmission.  A 1 here means delay after the first NAK.
 */
#define DWC2_NAKS_BEFORE_DELAY		3

62 63 64 65
/* This function is for debug only */
static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
{
	u16 curr_frame_number = hsotg->frame_number;
66 67 68 69
	u16 expected = dwc2_frame_num_inc(hsotg->last_frame_num, 1);

	if (expected != curr_frame_number)
		dwc2_sch_vdbg(hsotg, "MISSED SOF %04x != %04x\n",
70
			      expected, curr_frame_number);
71

72
#ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
73
	if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
74
		if (expected != curr_frame_number) {
75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
			hsotg->frame_num_array[hsotg->frame_num_idx] =
					curr_frame_number;
			hsotg->last_frame_num_array[hsotg->frame_num_idx] =
					hsotg->last_frame_num;
			hsotg->frame_num_idx++;
		}
	} else if (!hsotg->dumped_frame_num_array) {
		int i;

		dev_info(hsotg->dev, "Frame     Last Frame\n");
		dev_info(hsotg->dev, "-----     ----------\n");
		for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
			dev_info(hsotg->dev, "0x%04x    0x%04x\n",
				 hsotg->frame_num_array[i],
				 hsotg->last_frame_num_array[i]);
		}
		hsotg->dumped_frame_num_array = 1;
	}
#endif
94
	hsotg->last_frame_num = curr_frame_number;
95 96 97 98 99 100
}

static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
				    struct dwc2_host_chan *chan,
				    struct dwc2_qtd *qtd)
{
101
	struct usb_device *root_hub = dwc2_hsotg_to_hcd(hsotg)->self.root_hub;
102 103
	struct urb *usb_urb;

104 105 106 107 108 109 110
	if (!chan->qh)
		return;

	if (chan->qh->dev_speed == USB_SPEED_HIGH)
		return;

	if (!qtd->urb)
111 112 113
		return;

	usb_urb = qtd->urb->priv;
114
	if (!usb_urb || !usb_urb->dev || !usb_urb->dev->tt)
115 116
		return;

117 118 119 120 121 122 123 124 125
	/*
	 * The root hub doesn't really have a TT, but Linux thinks it
	 * does because how could you have a "high speed hub" that
	 * directly talks directly to low speed devices without a TT?
	 * It's all lies.  Lies, I tell you.
	 */
	if (usb_urb->dev->tt->hub == root_hub)
		return;

126
	if (qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
		chan->qh->tt_buffer_dirty = 1;
		if (usb_hub_clear_tt_buffer(usb_urb))
			/* Clear failed; let's hope things work anyway */
			chan->qh->tt_buffer_dirty = 0;
	}
}

/*
 * Handles the start-of-frame interrupt in host mode. Non-periodic
 * transactions may be queued to the DWC_otg controller for the current
 * (micro)frame. Periodic transactions may be queued to the controller
 * for the next (micro)frame.
 */
static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
{
	struct list_head *qh_entry;
	struct dwc2_qh *qh;
	enum dwc2_transaction_type tr_type;

146
	/* Clear interrupt */
147
	dwc2_writel(hsotg, GINTSTS_SOF, GINTSTS);
148

149 150 151 152
#ifdef DEBUG_SOF
	dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
#endif

153
	hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
154 155 156 157 158 159 160 161

	dwc2_track_missed_sofs(hsotg);

	/* Determine whether any periodic QHs should be executed */
	qh_entry = hsotg->periodic_sched_inactive.next;
	while (qh_entry != &hsotg->periodic_sched_inactive) {
		qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
		qh_entry = qh_entry->next;
162 163 164 165 166
		if (dwc2_frame_num_le(qh->next_active_frame,
				      hsotg->frame_number)) {
			dwc2_sch_vdbg(hsotg, "QH=%p ready fn=%04x, nxt=%04x\n",
				      qh, hsotg->frame_number,
				      qh->next_active_frame);
167

168 169 170 171
			/*
			 * Move QH to the ready list to be executed next
			 * (micro)frame
			 */
172
			list_move_tail(&qh->qh_list_entry,
173
				       &hsotg->periodic_sched_ready);
174
		}
175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190
	}
	tr_type = dwc2_hcd_select_transactions(hsotg);
	if (tr_type != DWC2_TRANSACTION_NONE)
		dwc2_hcd_queue_transactions(hsotg, tr_type);
}

/*
 * Handles the Rx FIFO Level Interrupt, which indicates that there is
 * at least one packet in the Rx FIFO. The packets are moved from the FIFO to
 * memory if the DWC_otg controller is operating in Slave mode.
 */
static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
{
	u32 grxsts, chnum, bcnt, dpid, pktsts;
	struct dwc2_host_chan *chan;

191 192
	if (dbg_perio())
		dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
193

194
	grxsts = dwc2_readl(hsotg, GRXSTSP);
195
	chnum = (grxsts & GRXSTS_HCHNUM_MASK) >> GRXSTS_HCHNUM_SHIFT;
196 197 198 199 200 201
	chan = hsotg->hc_ptr_array[chnum];
	if (!chan) {
		dev_err(hsotg->dev, "Unable to get corresponding channel\n");
		return;
	}

202 203
	bcnt = (grxsts & GRXSTS_BYTECNT_MASK) >> GRXSTS_BYTECNT_SHIFT;
	dpid = (grxsts & GRXSTS_DPID_MASK) >> GRXSTS_DPID_SHIFT;
204
	pktsts = (grxsts & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT;
205 206

	/* Packet Status */
207 208 209 210 211
	if (dbg_perio()) {
		dev_vdbg(hsotg->dev, "    Ch num = %d\n", chnum);
		dev_vdbg(hsotg->dev, "    Count = %d\n", bcnt);
		dev_vdbg(hsotg->dev, "    DPID = %d, chan.dpid = %d\n", dpid,
			 chan->data_pid_start);
212
		dev_vdbg(hsotg->dev, "    PStatus = %d\n", pktsts);
213
	}
214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257

	switch (pktsts) {
	case GRXSTS_PKTSTS_HCHIN:
		/* Read the data into the host buffer */
		if (bcnt > 0) {
			dwc2_read_packet(hsotg, chan->xfer_buf, bcnt);

			/* Update the HC fields for the next packet received */
			chan->xfer_count += bcnt;
			chan->xfer_buf += bcnt;
		}
		break;
	case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
	case GRXSTS_PKTSTS_DATATOGGLEERR:
	case GRXSTS_PKTSTS_HCHHALTED:
		/* Handled in interrupt, just ignore data */
		break;
	default:
		dev_err(hsotg->dev,
			"RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
		break;
	}
}

/*
 * This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
 * data packets may be written to the FIFO for OUT transfers. More requests
 * may be written to the non-periodic request queue for IN transfers. This
 * interrupt is enabled only in Slave mode.
 */
static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
{
	dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
	dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_NON_PERIODIC);
}

/*
 * This interrupt occurs when the periodic Tx FIFO is half-empty. More data
 * packets may be written to the FIFO for OUT transfers. More requests may be
 * written to the periodic request queue for IN transfers. This interrupt is
 * enabled only in Slave mode.
 */
static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
{
258 259
	if (dbg_perio())
		dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
260 261 262 263 264 265
	dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_PERIODIC);
}

static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
			      u32 *hprt0_modify)
{
266
	struct dwc2_core_params *params = &hsotg->params;
267 268 269 270
	int do_reset = 0;
	u32 usbcfg;
	u32 prtspd;
	u32 hcfg;
271
	u32 fslspclksel;
272 273 274 275 276
	u32 hfir;

	dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);

	/* Every time when port enables calculate HFIR.FrInterval */
277
	hfir = dwc2_readl(hsotg, HFIR);
278 279 280
	hfir &= ~HFIR_FRINT_MASK;
	hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
		HFIR_FRINT_MASK;
281
	dwc2_writel(hsotg, hfir, HFIR);
282 283 284 285 286 287 288 289

	/* Check if we need to adjust the PHY clock speed for low power */
	if (!params->host_support_fs_ls_low_power) {
		/* Port has been enabled, set the reset change flag */
		hsotg->flags.b.port_reset_change = 1;
		return;
	}

290
	usbcfg = dwc2_readl(hsotg, GUSBCFG);
291
	prtspd = (hprt0 & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
292 293 294 295 296 297

	if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
		/* Low power */
		if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
			/* Set PHY low power clock select for FS/LS devices */
			usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
298
			dwc2_writel(hsotg, usbcfg, GUSBCFG);
299 300 301
			do_reset = 1;
		}

302
		hcfg = dwc2_readl(hsotg, HCFG);
303 304
		fslspclksel = (hcfg & HCFG_FSLSPCLKSEL_MASK) >>
			      HCFG_FSLSPCLKSEL_SHIFT;
305 306

		if (prtspd == HPRT0_SPD_LOW_SPEED &&
307
		    params->host_ls_low_power_phy_clk) {
308 309 310
			/* 6 MHZ */
			dev_vdbg(hsotg->dev,
				 "FS_PHY programming HCFG to 6 MHz\n");
311
			if (fslspclksel != HCFG_FSLSPCLKSEL_6_MHZ) {
312
				fslspclksel = HCFG_FSLSPCLKSEL_6_MHZ;
313
				hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
314
				hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
315
				dwc2_writel(hsotg, hcfg, HCFG);
316 317 318 319 320 321
				do_reset = 1;
			}
		} else {
			/* 48 MHZ */
			dev_vdbg(hsotg->dev,
				 "FS_PHY programming HCFG to 48 MHz\n");
322
			if (fslspclksel != HCFG_FSLSPCLKSEL_48_MHZ) {
323
				fslspclksel = HCFG_FSLSPCLKSEL_48_MHZ;
324
				hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
325
				hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
326
				dwc2_writel(hsotg, hcfg, HCFG);
327 328 329 330 331 332 333
				do_reset = 1;
			}
		}
	} else {
		/* Not low power */
		if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
			usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
334
			dwc2_writel(hsotg, usbcfg, GUSBCFG);
335 336 337 338 339 340
			do_reset = 1;
		}
	}

	if (do_reset) {
		*hprt0_modify |= HPRT0_RST;
341
		dwc2_writel(hsotg, *hprt0_modify, HPRT0);
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
		queue_delayed_work(hsotg->wq_otg, &hsotg->reset_work,
				   msecs_to_jiffies(60));
	} else {
		/* Port has been enabled, set the reset change flag */
		hsotg->flags.b.port_reset_change = 1;
	}
}

/*
 * There are multiple conditions that can cause a port interrupt. This function
 * determines which interrupt conditions have occurred and handles them
 * appropriately.
 */
static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
{
	u32 hprt0;
	u32 hprt0_modify;

	dev_vdbg(hsotg->dev, "--Port Interrupt--\n");

362
	hprt0 = dwc2_readl(hsotg, HPRT0);
363 364 365 366 367 368 369 370 371 372 373 374 375 376
	hprt0_modify = hprt0;

	/*
	 * Clear appropriate bits in HPRT0 to clear the interrupt bit in
	 * GINTSTS
	 */
	hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
			  HPRT0_OVRCURRCHG);

	/*
	 * Port Connect Detected
	 * Set flag and clear if detected
	 */
	if (hprt0 & HPRT0_CONNDET) {
377
		dwc2_writel(hsotg, hprt0_modify | HPRT0_CONNDET, HPRT0);
378

379 380 381
		dev_vdbg(hsotg->dev,
			 "--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
			 hprt0);
382
		dwc2_hcd_connect(hsotg);
383 384 385 386 387 388 389 390 391 392 393 394

		/*
		 * The Hub driver asserts a reset when it sees port connect
		 * status change flag
		 */
	}

	/*
	 * Port Enable Changed
	 * Clear if detected - Set internal flag if disabled
	 */
	if (hprt0 & HPRT0_ENACHG) {
395
		dwc2_writel(hsotg, hprt0_modify | HPRT0_ENACHG, HPRT0);
396 397 398
		dev_vdbg(hsotg->dev,
			 "  --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
			 hprt0, !!(hprt0 & HPRT0_ENA));
399 400
		if (hprt0 & HPRT0_ENA) {
			hsotg->new_connection = true;
401
			dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
402
		} else {
403
			hsotg->flags.b.port_enable_change = 1;
404
			if (hsotg->params.dma_desc_fs_enable) {
405 406
				u32 hcfg;

407
				hsotg->params.dma_desc_enable = false;
408
				hsotg->new_connection = false;
409
				hcfg = dwc2_readl(hsotg, HCFG);
410
				hcfg &= ~HCFG_DESCDMA;
411
				dwc2_writel(hsotg, hcfg, HCFG);
412 413
			}
		}
414 415 416 417
	}

	/* Overcurrent Change Interrupt */
	if (hprt0 & HPRT0_OVRCURRCHG) {
418 419
		dwc2_writel(hsotg, hprt0_modify | HPRT0_OVRCURRCHG,
			    HPRT0);
420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
		dev_vdbg(hsotg->dev,
			 "  --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
			 hprt0);
		hsotg->flags.b.port_over_current_change = 1;
	}
}

/*
 * Gets the actual length of a transfer after the transfer halts. halt_status
 * holds the reason for the halt.
 *
 * For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
 * is set to 1 upon return if less than the requested number of bytes were
 * transferred. short_read may also be NULL on entry, in which case it remains
 * unchanged.
 */
static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
				       struct dwc2_host_chan *chan, int chnum,
				       struct dwc2_qtd *qtd,
				       enum dwc2_halt_status halt_status,
				       int *short_read)
{
	u32 hctsiz, count, length;

444
	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
445 446 447

	if (halt_status == DWC2_HC_XFER_COMPLETE) {
		if (chan->ep_is_in) {
448 449
			count = (hctsiz & TSIZ_XFERSIZE_MASK) >>
				TSIZ_XFERSIZE_SHIFT;
450
			length = chan->xfer_len - count;
451
			if (short_read)
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
				*short_read = (count != 0);
		} else if (chan->qh->do_split) {
			length = qtd->ssplit_out_xfer_count;
		} else {
			length = chan->xfer_len;
		}
	} else {
		/*
		 * Must use the hctsiz.pktcnt field to determine how much data
		 * has been transferred. This field reflects the number of
		 * packets that have been transferred via the USB. This is
		 * always an integral number of packets if the transfer was
		 * halted before its normal completion. (Can't use the
		 * hctsiz.xfersize field because that reflects the number of
		 * bytes transferred via the AHB, not the USB).
		 */
468
		count = (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT;
469 470 471 472 473 474 475 476 477 478 479 480
		length = (chan->start_pkt_count - count) * chan->max_packet;
	}

	return length;
}

/**
 * dwc2_update_urb_state() - Updates the state of the URB after a Transfer
 * Complete interrupt on the host channel. Updates the actual_length field
 * of the URB based on the number of bytes transferred via the host channel.
 * Sets the URB status if the data transfer is finished.
 *
481 482 483 484 485 486
 * @hsotg: Programming view of the DWC_otg controller
 * @chan: Programming view of host channel
 * @chnum: Channel number
 * @urb: Processing URB
 * @qtd: Queue transfer descriptor
 *
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
 * Return: 1 if the data transfer specified by the URB is completely finished,
 * 0 otherwise
 */
static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
				 struct dwc2_host_chan *chan, int chnum,
				 struct dwc2_hcd_urb *urb,
				 struct dwc2_qtd *qtd)
{
	u32 hctsiz;
	int xfer_done = 0;
	int short_read = 0;
	int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
						      DWC2_HC_XFER_COMPLETE,
						      &short_read);

	if (urb->actual_length + xfer_length > urb->length) {
503
		dev_dbg(hsotg->dev, "%s(): trimming xfer length\n", __func__);
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520
		xfer_length = urb->length - urb->actual_length;
	}

	dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
		 urb->actual_length, xfer_length);
	urb->actual_length += xfer_length;

	if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
	    (urb->flags & URB_SEND_ZERO_PACKET) &&
	    urb->actual_length >= urb->length &&
	    !(urb->length % chan->max_packet)) {
		xfer_done = 0;
	} else if (short_read || urb->actual_length >= urb->length) {
		xfer_done = 1;
		urb->status = 0;
	}

521
	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
522 523 524 525
	dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
		 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
	dev_vdbg(hsotg->dev, "  chan->xfer_len %d\n", chan->xfer_len);
	dev_vdbg(hsotg->dev, "  hctsiz.xfersize %d\n",
526
		 (hctsiz & TSIZ_XFERSIZE_MASK) >> TSIZ_XFERSIZE_SHIFT);
527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
	dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n", urb->length);
	dev_vdbg(hsotg->dev, "  urb->actual_length %d\n", urb->actual_length);
	dev_vdbg(hsotg->dev, "  short_read %d, xfer_done %d\n", short_read,
		 xfer_done);

	return xfer_done;
}

/*
 * Save the starting data toggle for the next transfer. The data toggle is
 * saved in the QH for non-control transfers and it's saved in the QTD for
 * control transfers.
 */
void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
			       struct dwc2_host_chan *chan, int chnum,
			       struct dwc2_qtd *qtd)
{
544
	u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
545
	u32 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
546 547

	if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
548 549 550 551
		if (WARN(!chan || !chan->qh,
			 "chan->qh must be specified for non-control eps\n"))
			return;

552 553 554 555 556
		if (pid == TSIZ_SC_MC_PID_DATA0)
			chan->qh->data_toggle = DWC2_HC_PID_DATA0;
		else
			chan->qh->data_toggle = DWC2_HC_PID_DATA1;
	} else {
557 558 559 560
		if (WARN(!qtd,
			 "qtd must be specified for control eps\n"))
			return;

561 562 563 564 565 566 567 568 569 570 571 572 573 574
		if (pid == TSIZ_SC_MC_PID_DATA0)
			qtd->data_toggle = DWC2_HC_PID_DATA0;
		else
			qtd->data_toggle = DWC2_HC_PID_DATA1;
	}
}

/**
 * dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
 * the transfer is stopped for any reason. The fields of the current entry in
 * the frame descriptor array are set based on the transfer state and the input
 * halt_status. Completes the Isochronous URB if all the URB frames have been
 * completed.
 *
575 576 577 578 579 580
 * @hsotg: Programming view of the DWC_otg controller
 * @chan: Programming view of host channel
 * @chnum: Channel number
 * @halt_status: Reason for halting a host channel
 * @qtd: Queue transfer descriptor
 *
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624
 * Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
 * transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
 */
static enum dwc2_halt_status dwc2_update_isoc_urb_state(
		struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
		int chnum, struct dwc2_qtd *qtd,
		enum dwc2_halt_status halt_status)
{
	struct dwc2_hcd_iso_packet_desc *frame_desc;
	struct dwc2_hcd_urb *urb = qtd->urb;

	if (!urb)
		return DWC2_HC_XFER_NO_HALT_STATUS;

	frame_desc = &urb->iso_descs[qtd->isoc_frame_index];

	switch (halt_status) {
	case DWC2_HC_XFER_COMPLETE:
		frame_desc->status = 0;
		frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
					chan, chnum, qtd, halt_status, NULL);
		break;
	case DWC2_HC_XFER_FRAME_OVERRUN:
		urb->error_count++;
		if (chan->ep_is_in)
			frame_desc->status = -ENOSR;
		else
			frame_desc->status = -ECOMM;
		frame_desc->actual_length = 0;
		break;
	case DWC2_HC_XFER_BABBLE_ERR:
		urb->error_count++;
		frame_desc->status = -EOVERFLOW;
		/* Don't need to update actual_length in this case */
		break;
	case DWC2_HC_XFER_XACT_ERR:
		urb->error_count++;
		frame_desc->status = -EPROTO;
		frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
					chan, chnum, qtd, halt_status, NULL);

		/* Skip whole frame */
		if (chan->qh->do_split &&
		    chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
625
		    hsotg->params.host_dma) {
626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
			qtd->complete_split = 0;
			qtd->isoc_split_offset = 0;
		}

		break;
	default:
		dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
			halt_status);
		break;
	}

	if (++qtd->isoc_frame_index == urb->packet_count) {
		/*
		 * urb->status is not used for isoc transfers. The individual
		 * frame_desc statuses are used instead.
		 */
642
		dwc2_host_complete(hsotg, qtd, 0);
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663
		halt_status = DWC2_HC_XFER_URB_COMPLETE;
	} else {
		halt_status = DWC2_HC_XFER_COMPLETE;
	}

	return halt_status;
}

/*
 * Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
 * QHs, removes the QH from the active non-periodic schedule. If any QTDs are
 * still linked to the QH, the QH is added to the end of the inactive
 * non-periodic schedule. For periodic QHs, removes the QH from the periodic
 * schedule if no more QTDs are linked to the QH.
 */
static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
			       int free_qtd)
{
	int continue_split = 0;
	struct dwc2_qtd *qtd;

664 665 666
	if (dbg_qh(qh))
		dev_vdbg(hsotg->dev, "  %s(%p,%p,%d)\n", __func__,
			 hsotg, qh, free_qtd);
667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712

	if (list_empty(&qh->qtd_list)) {
		dev_dbg(hsotg->dev, "## QTD list empty ##\n");
		goto no_qtd;
	}

	qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);

	if (qtd->complete_split)
		continue_split = 1;
	else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
		 qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
		continue_split = 1;

	if (free_qtd) {
		dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
		continue_split = 0;
	}

no_qtd:
	qh->channel = NULL;
	dwc2_hcd_qh_deactivate(hsotg, qh, continue_split);
}

/**
 * dwc2_release_channel() - Releases a host channel for use by other transfers
 *
 * @hsotg:       The HCD state structure
 * @chan:        The host channel to release
 * @qtd:         The QTD associated with the host channel. This QTD may be
 *               freed if the transfer is complete or an error has occurred.
 * @halt_status: Reason the channel is being released. This status
 *               determines the actions taken by this function.
 *
 * Also attempts to select and queue more transactions since at least one host
 * channel is available.
 */
static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
				 struct dwc2_host_chan *chan,
				 struct dwc2_qtd *qtd,
				 enum dwc2_halt_status halt_status)
{
	enum dwc2_transaction_type tr_type;
	u32 haintmsk;
	int free_qtd = 0;

713 714 715
	if (dbg_hc(chan))
		dev_vdbg(hsotg->dev, "  %s: channel %d, halt_status %d\n",
			 __func__, chan->hc_num, halt_status);
716 717 718 719 720 721 722 723 724 725 726

	switch (halt_status) {
	case DWC2_HC_XFER_URB_COMPLETE:
		free_qtd = 1;
		break;
	case DWC2_HC_XFER_AHB_ERR:
	case DWC2_HC_XFER_STALL:
	case DWC2_HC_XFER_BABBLE_ERR:
		free_qtd = 1;
		break;
	case DWC2_HC_XFER_XACT_ERR:
727
		if (qtd && qtd->error_count >= 3) {
728 729 730
			dev_vdbg(hsotg->dev,
				 "  Complete URB with transaction error\n");
			free_qtd = 1;
731
			dwc2_host_complete(hsotg, qtd, -EPROTO);
732 733 734 735 736 737 738 739 740 741 742 743
		}
		break;
	case DWC2_HC_XFER_URB_DEQUEUE:
		/*
		 * The QTD has already been removed and the QH has been
		 * deactivated. Don't want to do anything except release the
		 * host channel and try to queue more transfers.
		 */
		goto cleanup;
	case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
		dev_vdbg(hsotg->dev, "  Complete URB with I/O error\n");
		free_qtd = 1;
744
		dwc2_host_complete(hsotg, qtd, -EIO);
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763
		break;
	case DWC2_HC_XFER_NO_HALT_STATUS:
	default:
		break;
	}

	dwc2_deactivate_qh(hsotg, chan->qh, free_qtd);

cleanup:
	/*
	 * Release the host channel for use by other transfers. The cleanup
	 * function clears the channel interrupt enables and conditions, so
	 * there's no need to clear the Channel Halted interrupt separately.
	 */
	if (!list_empty(&chan->hc_list_entry))
		list_del(&chan->hc_list_entry);
	dwc2_hc_cleanup(hsotg, chan);
	list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);

764
	if (hsotg->params.uframe_sched) {
765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780
		hsotg->available_host_channels++;
	} else {
		switch (chan->ep_type) {
		case USB_ENDPOINT_XFER_CONTROL:
		case USB_ENDPOINT_XFER_BULK:
			hsotg->non_periodic_channels--;
			break;
		default:
			/*
			 * Don't release reservations for periodic channels
			 * here. That's done when a periodic transfer is
			 * descheduled (i.e. when the QH is removed from the
			 * periodic schedule).
			 */
			break;
		}
781 782
	}

783
	haintmsk = dwc2_readl(hsotg, HAINTMSK);
784
	haintmsk &= ~(1 << chan->hc_num);
785
	dwc2_writel(hsotg, haintmsk, HAINTMSK);
786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806

	/* Try to queue more transfers now that there's a free channel */
	tr_type = dwc2_hcd_select_transactions(hsotg);
	if (tr_type != DWC2_TRANSACTION_NONE)
		dwc2_hcd_queue_transactions(hsotg, tr_type);
}

/*
 * Halts a host channel. If the channel cannot be halted immediately because
 * the request queue is full, this function ensures that the FIFO empty
 * interrupt for the appropriate queue is enabled so that the halt request can
 * be queued when there is space in the request queue.
 *
 * This function may also be called in DMA mode. In that case, the channel is
 * simply released since the core always halts the channel automatically in
 * DMA mode.
 */
static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
			      struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
			      enum dwc2_halt_status halt_status)
{
807 808
	if (dbg_hc(chan))
		dev_vdbg(hsotg->dev, "%s()\n", __func__);
809

810
	if (hsotg->params.host_dma) {
811 812
		if (dbg_hc(chan))
			dev_vdbg(hsotg->dev, "DMA enabled\n");
813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
		dwc2_release_channel(hsotg, chan, qtd, halt_status);
		return;
	}

	/* Slave mode processing */
	dwc2_hc_halt(hsotg, chan, halt_status);

	if (chan->halt_on_queue) {
		u32 gintmsk;

		dev_vdbg(hsotg->dev, "Halt on queue\n");
		if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
		    chan->ep_type == USB_ENDPOINT_XFER_BULK) {
			dev_vdbg(hsotg->dev, "control/bulk\n");
			/*
			 * Make sure the Non-periodic Tx FIFO empty interrupt
			 * is enabled so that the non-periodic schedule will
			 * be processed
			 */
832
			gintmsk = dwc2_readl(hsotg, GINTMSK);
833
			gintmsk |= GINTSTS_NPTXFEMP;
834
			dwc2_writel(hsotg, gintmsk, GINTMSK);
835 836 837 838 839 840 841 842
		} else {
			dev_vdbg(hsotg->dev, "isoc/intr\n");
			/*
			 * Move the QH from the periodic queued schedule to
			 * the periodic assigned schedule. This allows the
			 * halt to be queued when the periodic schedule is
			 * processed.
			 */
843
			list_move_tail(&chan->qh->qh_list_entry,
844
				       &hsotg->periodic_sched_assigned);
845 846 847 848 849 850

			/*
			 * Make sure the Periodic Tx FIFO Empty interrupt is
			 * enabled so that the periodic schedule will be
			 * processed
			 */
851
			gintmsk = dwc2_readl(hsotg, GINTMSK);
852
			gintmsk |= GINTSTS_PTXFEMP;
853
			dwc2_writel(hsotg, gintmsk, GINTMSK);
854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917
		}
	}
}

/*
 * Performs common cleanup for non-periodic transfers after a Transfer
 * Complete interrupt. This function should be called after any endpoint type
 * specific handling is finished to release the host channel.
 */
static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
					    struct dwc2_host_chan *chan,
					    int chnum, struct dwc2_qtd *qtd,
					    enum dwc2_halt_status halt_status)
{
	dev_vdbg(hsotg->dev, "%s()\n", __func__);

	qtd->error_count = 0;

	if (chan->hcint & HCINTMSK_NYET) {
		/*
		 * Got a NYET on the last transaction of the transfer. This
		 * means that the endpoint should be in the PING state at the
		 * beginning of the next transfer.
		 */
		dev_vdbg(hsotg->dev, "got NYET\n");
		chan->qh->ping_state = 1;
	}

	/*
	 * Always halt and release the host channel to make it available for
	 * more transfers. There may still be more phases for a control
	 * transfer or more data packets for a bulk transfer at this point,
	 * but the host channel is still halted. A channel will be reassigned
	 * to the transfer when the non-periodic schedule is processed after
	 * the channel is released. This allows transactions to be queued
	 * properly via dwc2_hcd_queue_transactions, which also enables the
	 * Tx FIFO Empty interrupt if necessary.
	 */
	if (chan->ep_is_in) {
		/*
		 * IN transfers in Slave mode require an explicit disable to
		 * halt the channel. (In DMA mode, this call simply releases
		 * the channel.)
		 */
		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
	} else {
		/*
		 * The channel is automatically disabled by the core for OUT
		 * transfers in Slave mode
		 */
		dwc2_release_channel(hsotg, chan, qtd, halt_status);
	}
}

/*
 * Performs common cleanup for periodic transfers after a Transfer Complete
 * interrupt. This function should be called after any endpoint type specific
 * handling is finished to release the host channel.
 */
static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
					struct dwc2_host_chan *chan, int chnum,
					struct dwc2_qtd *qtd,
					enum dwc2_halt_status halt_status)
{
918
	u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935

	qtd->error_count = 0;

	if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
		/* Core halts channel in these cases */
		dwc2_release_channel(hsotg, chan, qtd, halt_status);
	else
		/* Flush any outstanding requests from the Tx queue */
		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
}

static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
				       struct dwc2_host_chan *chan, int chnum,
				       struct dwc2_qtd *qtd)
{
	struct dwc2_hcd_iso_packet_desc *frame_desc;
	u32 len;
936 937
	u32 hctsiz;
	u32 pid;
938 939 940 941 942 943 944

	if (!qtd->urb)
		return 0;

	frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
	len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
					  DWC2_HC_XFER_COMPLETE, NULL);
945
	if (!len && !qtd->isoc_split_offset) {
946 947 948 949 950 951
		qtd->complete_split = 0;
		return 0;
	}

	frame_desc->actual_length += len;

952 953 954 955 956 957 958 959
	if (chan->align_buf) {
		dev_vdbg(hsotg->dev, "non-aligned buffer\n");
		dma_unmap_single(hsotg->dev, chan->qh->dw_align_buf_dma,
				 DWC2_KMEM_UNALIGNED_BUF_SIZE, DMA_FROM_DEVICE);
		memcpy(qtd->urb->buf + (chan->xfer_dma - qtd->urb->dma),
		       chan->qh->dw_align_buf, len);
	}

960 961
	qtd->isoc_split_offset += len;

962
	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
963 964 965
	pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;

	if (frame_desc->actual_length >= frame_desc->length || pid == 0) {
966 967 968 969 970 971 972
		frame_desc->status = 0;
		qtd->isoc_frame_index++;
		qtd->complete_split = 0;
		qtd->isoc_split_offset = 0;
	}

	if (qtd->isoc_frame_index == qtd->urb->packet_count) {
973
		dwc2_host_complete(hsotg, qtd, 0);
974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993
		dwc2_release_channel(hsotg, chan, qtd,
				     DWC2_HC_XFER_URB_COMPLETE);
	} else {
		dwc2_release_channel(hsotg, chan, qtd,
				     DWC2_HC_XFER_NO_HALT_STATUS);
	}

	return 1;	/* Indicates that channel released */
}

/*
 * Handles a host channel Transfer Complete interrupt. This handler may be
 * called in either DMA mode or Slave mode.
 */
static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
				  struct dwc2_host_chan *chan, int chnum,
				  struct dwc2_qtd *qtd)
{
	struct dwc2_hcd_urb *urb = qtd->urb;
	enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
994
	int pipe_type;
995 996
	int urb_xfer_done;

997 998 999 1000
	if (dbg_hc(chan))
		dev_vdbg(hsotg->dev,
			 "--Host Channel %d Interrupt: Transfer Complete--\n",
			 chnum);
1001

1002 1003 1004 1005 1006
	if (!urb)
		goto handle_xfercomp_done;

	pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);

1007
	if (hsotg->params.dma_desc_enable) {
1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
		if (pipe_type == USB_ENDPOINT_XFER_ISOC)
			/* Do not disable the interrupt, just clear it */
			return;
		goto handle_xfercomp_done;
	}

	/* Handle xfer complete on CSPLIT */
	if (chan->qh->do_split) {
		if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
1018
		    hsotg->params.host_dma) {
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
			if (qtd->complete_split &&
			    dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
							qtd))
				goto handle_xfercomp_done;
		} else {
			qtd->complete_split = 0;
		}
	}

	/* Update the QTD and URB states */
	switch (pipe_type) {
	case USB_ENDPOINT_XFER_CONTROL:
		switch (qtd->control_phase) {
		case DWC2_CONTROL_SETUP:
			if (urb->length > 0)
				qtd->control_phase = DWC2_CONTROL_DATA;
			else
				qtd->control_phase = DWC2_CONTROL_STATUS;
			dev_vdbg(hsotg->dev,
				 "  Control setup transaction done\n");
			halt_status = DWC2_HC_XFER_COMPLETE;
			break;
		case DWC2_CONTROL_DATA:
			urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
							      chnum, urb, qtd);
			if (urb_xfer_done) {
				qtd->control_phase = DWC2_CONTROL_STATUS;
				dev_vdbg(hsotg->dev,
					 "  Control data transfer done\n");
			} else {
				dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
							  qtd);
			}
			halt_status = DWC2_HC_XFER_COMPLETE;
			break;
		case DWC2_CONTROL_STATUS:
			dev_vdbg(hsotg->dev, "  Control transfer complete\n");
			if (urb->status == -EINPROGRESS)
				urb->status = 0;
1058
			dwc2_host_complete(hsotg, qtd, urb->status);
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
			halt_status = DWC2_HC_XFER_URB_COMPLETE;
			break;
		}

		dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
						halt_status);
		break;
	case USB_ENDPOINT_XFER_BULK:
		dev_vdbg(hsotg->dev, "  Bulk transfer complete\n");
		urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
						      qtd);
		if (urb_xfer_done) {
1071
			dwc2_host_complete(hsotg, qtd, urb->status);
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090
			halt_status = DWC2_HC_XFER_URB_COMPLETE;
		} else {
			halt_status = DWC2_HC_XFER_COMPLETE;
		}

		dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
		dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
						halt_status);
		break;
	case USB_ENDPOINT_XFER_INT:
		dev_vdbg(hsotg->dev, "  Interrupt transfer complete\n");
		urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
						      qtd);

		/*
		 * Interrupt URB is done on the first transfer complete
		 * interrupt
		 */
		if (urb_xfer_done) {
1091 1092
			dwc2_host_complete(hsotg, qtd, urb->status);
			halt_status = DWC2_HC_XFER_URB_COMPLETE;
1093
		} else {
1094
			halt_status = DWC2_HC_XFER_COMPLETE;
1095 1096 1097 1098 1099 1100 1101
		}

		dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
		dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
					    halt_status);
		break;
	case USB_ENDPOINT_XFER_ISOC:
1102 1103
		if (dbg_perio())
			dev_vdbg(hsotg->dev, "  Isochronous transfer complete\n");
1104 1105
		if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
			halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1106 1107
							chnum, qtd,
							DWC2_HC_XFER_COMPLETE);
1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
		dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
					    halt_status);
		break;
	}

handle_xfercomp_done:
	disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
}

/*
 * Handles a host channel STALL interrupt. This handler may be called in
 * either DMA mode or Slave mode.
 */
static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
			       struct dwc2_host_chan *chan, int chnum,
			       struct dwc2_qtd *qtd)
{
	struct dwc2_hcd_urb *urb = qtd->urb;
1126
	int pipe_type;
1127 1128 1129 1130

	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
		chnum);

1131
	if (hsotg->params.dma_desc_enable) {
1132 1133 1134 1135 1136 1137 1138 1139
		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
					    DWC2_HC_XFER_STALL);
		goto handle_stall_done;
	}

	if (!urb)
		goto handle_stall_halt;

1140 1141
	pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);

1142
	if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1143
		dwc2_host_complete(hsotg, qtd, -EPIPE);
1144 1145 1146

	if (pipe_type == USB_ENDPOINT_XFER_BULK ||
	    pipe_type == USB_ENDPOINT_XFER_INT) {
1147
		dwc2_host_complete(hsotg, qtd, -EPIPE);
1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187
		/*
		 * USB protocol requires resetting the data toggle for bulk
		 * and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
		 * setup command is issued to the endpoint. Anticipate the
		 * CLEAR_FEATURE command since a STALL has occurred and reset
		 * the data toggle now.
		 */
		chan->qh->data_toggle = 0;
	}

handle_stall_halt:
	dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_STALL);

handle_stall_done:
	disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
}

/*
 * Updates the state of the URB when a transfer has been stopped due to an
 * abnormal condition before the transfer completes. Modifies the
 * actual_length field of the URB to reflect the number of bytes that have
 * actually been transferred via the host channel.
 */
static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
				      struct dwc2_host_chan *chan, int chnum,
				      struct dwc2_hcd_urb *urb,
				      struct dwc2_qtd *qtd,
				      enum dwc2_halt_status halt_status)
{
	u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
						      qtd, halt_status, NULL);
	u32 hctsiz;

	if (urb->actual_length + xfer_length > urb->length) {
		dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
		xfer_length = urb->length - urb->actual_length;
	}

	urb->actual_length += xfer_length;

1188
	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1189 1190 1191 1192 1193
	dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
		 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
	dev_vdbg(hsotg->dev, "  chan->start_pkt_count %d\n",
		 chan->start_pkt_count);
	dev_vdbg(hsotg->dev, "  hctsiz.pktcnt %d\n",
1194
		 (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT);
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
	dev_vdbg(hsotg->dev, "  chan->max_packet %d\n", chan->max_packet);
	dev_vdbg(hsotg->dev, "  bytes_transferred %d\n",
		 xfer_length);
	dev_vdbg(hsotg->dev, "  urb->actual_length %d\n",
		 urb->actual_length);
	dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n",
		 urb->length);
}

/*
 * Handles a host channel NAK interrupt. This handler may be called in either
 * DMA mode or Slave mode.
 */
static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
			     struct dwc2_host_chan *chan, int chnum,
			     struct dwc2_qtd *qtd)
{
1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
	if (!qtd) {
		dev_dbg(hsotg->dev, "%s: qtd is NULL\n", __func__);
		return;
	}

	if (!qtd->urb) {
		dev_dbg(hsotg->dev, "%s: qtd->urb is NULL\n", __func__);
		return;
	}

1222 1223 1224
	if (dbg_hc(chan))
		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
			 chnum);
1225 1226 1227 1228

	/*
	 * Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
	 * interrupt. Re-start the SSPLIT transfer.
1229 1230 1231 1232 1233
	 *
	 * Normally for non-periodic transfers we'll retry right away, but to
	 * avoid interrupt storms we'll wait before retrying if we've got
	 * several NAKs. If we didn't do this we'd retry directly from the
	 * interrupt handler and could end up quickly getting another
1234 1235 1236 1237
	 * interrupt (another NAK), which we'd retry. Note that we do not
	 * delay retries for IN parts of control requests, as those are expected
	 * to complete fairly quickly, and if we delay them we risk confusing
	 * the device and cause it issue STALL.
1238 1239 1240 1241 1242 1243
	 *
	 * Note that in DMA mode software only gets involved to re-send NAKed
	 * transfers for split transactions, so we only need to apply this
	 * delaying logic when handling splits. In non-DMA mode presumably we
	 * might want a similar delay if someone can demonstrate this problem
	 * affects that code path too.
1244 1245 1246 1247 1248
	 */
	if (chan->do_split) {
		if (chan->complete_split)
			qtd->error_count = 0;
		qtd->complete_split = 0;
1249
		qtd->num_naks++;
1250 1251 1252
		qtd->qh->want_wait = qtd->num_naks >= DWC2_NAKS_BEFORE_DELAY &&
				!(chan->ep_type == USB_ENDPOINT_XFER_CONTROL &&
				  chan->ep_is_in);
1253 1254 1255 1256 1257 1258 1259
		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
		goto handle_nak_done;
	}

	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
	case USB_ENDPOINT_XFER_CONTROL:
	case USB_ENDPOINT_XFER_BULK:
1260
		if (hsotg->params.host_dma && chan->ep_is_in) {
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
			/*
			 * NAK interrupts are enabled on bulk/control IN
			 * transfers in DMA mode for the sole purpose of
			 * resetting the error count after a transaction error
			 * occurs. The core will continue transferring data.
			 */
			qtd->error_count = 0;
			break;
		}

		/*
		 * NAK interrupts normally occur during OUT transfers in DMA
		 * or Slave mode. For IN transfers, more requests will be
		 * queued as request queue space is available.
		 */
		qtd->error_count = 0;

		if (!chan->qh->ping_state) {
			dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
						  qtd, DWC2_HC_XFER_NAK);
			dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);

			if (chan->speed == USB_SPEED_HIGH)
				chan->qh->ping_state = 1;
		}

		/*
		 * Halt the channel so the transfer can be re-started from
		 * the appropriate point or the PING protocol will
		 * start/continue
		 */
		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
		break;
	case USB_ENDPOINT_XFER_INT:
		qtd->error_count = 0;
		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
		break;
	case USB_ENDPOINT_XFER_ISOC:
		/* Should never get called for isochronous transfers */
		dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
		break;
	}

handle_nak_done:
	disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
}

/*
 * Handles a host channel ACK interrupt. This interrupt is enabled when
 * performing the PING protocol in Slave mode, when errors occur during
 * either Slave mode or DMA mode, and during Start Split transactions.
 */
static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
			     struct dwc2_host_chan *chan, int chnum,
			     struct dwc2_qtd *qtd)
{
	struct dwc2_hcd_iso_packet_desc *frame_desc;

1319 1320 1321
	if (dbg_hc(chan))
		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
			 chnum);
1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395

	if (chan->do_split) {
		/* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
		if (!chan->ep_is_in &&
		    chan->data_pid_start != DWC2_HC_PID_SETUP)
			qtd->ssplit_out_xfer_count = chan->xfer_len;

		if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
			qtd->complete_split = 1;
			dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
		} else {
			/* ISOC OUT */
			switch (chan->xact_pos) {
			case DWC2_HCSPLT_XACTPOS_ALL:
				break;
			case DWC2_HCSPLT_XACTPOS_END:
				qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
				qtd->isoc_split_offset = 0;
				break;
			case DWC2_HCSPLT_XACTPOS_BEGIN:
			case DWC2_HCSPLT_XACTPOS_MID:
				/*
				 * For BEGIN or MID, calculate the length for
				 * the next microframe to determine the correct
				 * SSPLIT token, either MID or END
				 */
				frame_desc = &qtd->urb->iso_descs[
						qtd->isoc_frame_index];
				qtd->isoc_split_offset += 188;

				if (frame_desc->length - qtd->isoc_split_offset
							<= 188)
					qtd->isoc_split_pos =
							DWC2_HCSPLT_XACTPOS_END;
				else
					qtd->isoc_split_pos =
							DWC2_HCSPLT_XACTPOS_MID;
				break;
			}
		}
	} else {
		qtd->error_count = 0;

		if (chan->qh->ping_state) {
			chan->qh->ping_state = 0;
			/*
			 * Halt the channel so the transfer can be re-started
			 * from the appropriate point. This only happens in
			 * Slave mode. In DMA mode, the ping_state is cleared
			 * when the transfer is started because the core
			 * automatically executes the PING, then the transfer.
			 */
			dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
		}
	}

	/*
	 * If the ACK occurred when _not_ in the PING state, let the channel
	 * continue transferring data after clearing the error count
	 */
	disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
}

/*
 * Handles a host channel NYET interrupt. This interrupt should only occur on
 * Bulk and Control OUT endpoints and for complete split transactions. If a
 * NYET occurs at the same time as a Transfer Complete interrupt, it is
 * handled in the xfercomp interrupt handler, not here. This handler may be
 * called in either DMA mode or Slave mode.
 */
static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
			      struct dwc2_host_chan *chan, int chnum,
			      struct dwc2_qtd *qtd)
{
1396 1397 1398
	if (dbg_hc(chan))
		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
			 chnum);
1399 1400 1401 1402 1403 1404 1405

	/*
	 * NYET on CSPLIT
	 * re-do the CSPLIT immediately on non-periodic
	 */
	if (chan->do_split && chan->complete_split) {
		if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1406
		    hsotg->params.host_dma) {
1407 1408
			qtd->complete_split = 0;
			qtd->isoc_split_offset = 0;
1409
			qtd->isoc_frame_index++;
1410
			if (qtd->urb &&
1411 1412
			    qtd->isoc_frame_index == qtd->urb->packet_count) {
				dwc2_host_complete(hsotg, qtd, 0);
1413
				dwc2_release_channel(hsotg, chan, qtd,
1414
						     DWC2_HC_XFER_URB_COMPLETE);
1415 1416
			} else {
				dwc2_release_channel(hsotg, chan, qtd,
1417
						DWC2_HC_XFER_NO_HALT_STATUS);
1418 1419 1420 1421 1422 1423
			}
			goto handle_nyet_done;
		}

		if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
		    chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1424 1425 1426
			struct dwc2_qh *qh = chan->qh;
			bool past_end;

1427
			if (!hsotg->params.uframe_sched) {
1428 1429 1430 1431 1432 1433 1434
				int frnum = dwc2_hcd_get_frame_number(hsotg);

				/* Don't have num_hs_transfers; simple logic */
				past_end = dwc2_full_frame_num(frnum) !=
				     dwc2_full_frame_num(qh->next_active_frame);
			} else {
				int end_frnum;
1435 1436

				/*
1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
				 * Figure out the end frame based on
				 * schedule.
				 *
				 * We don't want to go on trying again
				 * and again forever. Let's stop when
				 * we've done all the transfers that
				 * were scheduled.
				 *
				 * We're going to be comparing
				 * start_active_frame and
				 * next_active_frame, both of which
				 * are 1 before the time the packet
				 * goes on the wire, so that cancels
				 * out. Basically if had 1 transfer
				 * and we saw 1 NYET then we're done.
				 * We're getting a NYET here so if
				 * next >= (start + num_transfers)
				 * we're done. The complexity is that
				 * for all but ISOC_OUT we skip one
				 * slot.
				 */
1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472
				end_frnum = dwc2_frame_num_inc(
					qh->start_active_frame,
					qh->num_hs_transfers);

				if (qh->ep_type != USB_ENDPOINT_XFER_ISOC ||
				    qh->ep_is_in)
					end_frnum =
					       dwc2_frame_num_inc(end_frnum, 1);

				past_end = dwc2_frame_num_le(
					end_frnum, qh->next_active_frame);
			}

			if (past_end) {
				/* Treat this as a transaction error. */
1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522
#if 0
				/*
				 * Todo: Fix system performance so this can
				 * be treated as an error. Right now complete
				 * splits cannot be scheduled precisely enough
				 * due to other system activity, so this error
				 * occurs regularly in Slave mode.
				 */
				qtd->error_count++;
#endif
				qtd->complete_split = 0;
				dwc2_halt_channel(hsotg, chan, qtd,
						  DWC2_HC_XFER_XACT_ERR);
				/* Todo: add support for isoc release */
				goto handle_nyet_done;
			}
		}

		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
		goto handle_nyet_done;
	}

	chan->qh->ping_state = 1;
	qtd->error_count = 0;

	dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb, qtd,
				  DWC2_HC_XFER_NYET);
	dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);

	/*
	 * Halt the channel and re-start the transfer so the PING protocol
	 * will start
	 */
	dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);

handle_nyet_done:
	disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
}

/*
 * Handles a host channel babble interrupt. This handler may be called in
 * either DMA mode or Slave mode.
 */
static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
				struct dwc2_host_chan *chan, int chnum,
				struct dwc2_qtd *qtd)
{
	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
		chnum);

1523 1524
	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);

1525
	if (hsotg->params.dma_desc_enable) {
1526 1527
		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
					    DWC2_HC_XFER_BABBLE_ERR);
1528
		goto disable_int;
1529 1530 1531
	}

	if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1532
		dwc2_host_complete(hsotg, qtd, -EOVERFLOW);
1533 1534 1535 1536 1537
		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_BABBLE_ERR);
	} else {
		enum dwc2_halt_status halt_status;

		halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1538
						qtd, DWC2_HC_XFER_BABBLE_ERR);
1539 1540 1541
		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
	}

1542
disable_int:
1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566
	disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
}

/*
 * Handles a host channel AHB error interrupt. This handler is only called in
 * DMA mode.
 */
static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
				struct dwc2_host_chan *chan, int chnum,
				struct dwc2_qtd *qtd)
{
	struct dwc2_hcd_urb *urb = qtd->urb;
	char *pipetype, *speed;
	u32 hcchar;
	u32 hcsplt;
	u32 hctsiz;
	u32 hc_dma;

	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
		chnum);

	if (!urb)
		goto handle_ahberr_halt;

1567 1568
	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);

1569 1570 1571 1572
	hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
	hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
	hc_dma = dwc2_readl(hsotg, HCDMA(chnum));
1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619

	dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
	dev_err(hsotg->dev, "  hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
	dev_err(hsotg->dev, "  hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
	dev_err(hsotg->dev, "  Device address: %d\n",
		dwc2_hcd_get_dev_addr(&urb->pipe_info));
	dev_err(hsotg->dev, "  Endpoint: %d, %s\n",
		dwc2_hcd_get_ep_num(&urb->pipe_info),
		dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");

	switch (dwc2_hcd_get_pipe_type(&urb->pipe_info)) {
	case USB_ENDPOINT_XFER_CONTROL:
		pipetype = "CONTROL";
		break;
	case USB_ENDPOINT_XFER_BULK:
		pipetype = "BULK";
		break;
	case USB_ENDPOINT_XFER_INT:
		pipetype = "INTERRUPT";
		break;
	case USB_ENDPOINT_XFER_ISOC:
		pipetype = "ISOCHRONOUS";
		break;
	default:
		pipetype = "UNKNOWN";
		break;
	}

	dev_err(hsotg->dev, "  Endpoint type: %s\n", pipetype);

	switch (chan->speed) {
	case USB_SPEED_HIGH:
		speed = "HIGH";
		break;
	case USB_SPEED_FULL:
		speed = "FULL";
		break;
	case USB_SPEED_LOW:
		speed = "LOW";
		break;
	default:
		speed = "UNKNOWN";
		break;
	}

	dev_err(hsotg->dev, "  Speed: %s\n", speed);

1620 1621 1622
	dev_err(hsotg->dev, "  Max packet size: %d (mult %d)\n",
		dwc2_hcd_get_maxp(&urb->pipe_info),
		dwc2_hcd_get_maxp_mult(&urb->pipe_info));
1623
	dev_err(hsotg->dev, "  Data buffer length: %d\n", urb->length);
1624 1625 1626 1627
	dev_err(hsotg->dev, "  Transfer buffer: %p, Transfer DMA: %08lx\n",
		urb->buf, (unsigned long)urb->dma);
	dev_err(hsotg->dev, "  Setup buffer: %p, Setup DMA: %08lx\n",
		urb->setup_packet, (unsigned long)urb->setup_dma);
1628 1629 1630
	dev_err(hsotg->dev, "  Interval: %d\n", urb->interval);

	/* Core halts the channel for Descriptor DMA mode */
1631
	if (hsotg->params.dma_desc_enable) {
1632 1633 1634 1635 1636
		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
					    DWC2_HC_XFER_AHB_ERR);
		goto handle_ahberr_done;
	}

1637
	dwc2_host_complete(hsotg, qtd, -EIO);
1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660

handle_ahberr_halt:
	/*
	 * Force a channel halt. Don't call dwc2_halt_channel because that won't
	 * write to the HCCHARn register in DMA mode to force the halt.
	 */
	dwc2_hc_halt(hsotg, chan, DWC2_HC_XFER_AHB_ERR);

handle_ahberr_done:
	disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
}

/*
 * Handles a host channel transaction error interrupt. This handler may be
 * called in either DMA mode or Slave mode.
 */
static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
				 struct dwc2_host_chan *chan, int chnum,
				 struct dwc2_qtd *qtd)
{
	dev_dbg(hsotg->dev,
		"--Host Channel %d Interrupt: Transaction Error--\n", chnum);

1661 1662
	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);

1663
	if (hsotg->params.dma_desc_enable) {
1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697
		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
					    DWC2_HC_XFER_XACT_ERR);
		goto handle_xacterr_done;
	}

	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
	case USB_ENDPOINT_XFER_CONTROL:
	case USB_ENDPOINT_XFER_BULK:
		qtd->error_count++;
		if (!chan->qh->ping_state) {
			dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
						  qtd, DWC2_HC_XFER_XACT_ERR);
			dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
			if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
				chan->qh->ping_state = 1;
		}

		/*
		 * Halt the channel so the transfer can be re-started from
		 * the appropriate point or the PING protocol will start
		 */
		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
		break;
	case USB_ENDPOINT_XFER_INT:
		qtd->error_count++;
		if (chan->do_split && chan->complete_split)
			qtd->complete_split = 0;
		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
		break;
	case USB_ENDPOINT_XFER_ISOC:
		{
			enum dwc2_halt_status halt_status;

			halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1698
					 chnum, qtd, DWC2_HC_XFER_XACT_ERR);
1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
			dwc2_halt_channel(hsotg, chan, qtd, halt_status);
		}
		break;
	}

handle_xacterr_done:
	disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
}

/*
 * Handles a host channel frame overrun interrupt. This handler may be called
 * in either DMA mode or Slave mode.
 */
static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
				  struct dwc2_host_chan *chan, int chnum,
				  struct dwc2_qtd *qtd)
{
	enum dwc2_halt_status halt_status;

1718 1719 1720
	if (dbg_hc(chan))
		dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
			chnum);
1721

1722 1723
	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);

1724 1725 1726 1727 1728 1729 1730 1731 1732
	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
	case USB_ENDPOINT_XFER_CONTROL:
	case USB_ENDPOINT_XFER_BULK:
		break;
	case USB_ENDPOINT_XFER_INT:
		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_FRAME_OVERRUN);
		break;
	case USB_ENDPOINT_XFER_ISOC:
		halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1733
					qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784
		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
		break;
	}

	disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
}

/*
 * Handles a host channel data toggle error interrupt. This handler may be
 * called in either DMA mode or Slave mode.
 */
static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
				    struct dwc2_host_chan *chan, int chnum,
				    struct dwc2_qtd *qtd)
{
	dev_dbg(hsotg->dev,
		"--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);

	if (chan->ep_is_in)
		qtd->error_count = 0;
	else
		dev_err(hsotg->dev,
			"Data Toggle Error on OUT transfer, channel %d\n",
			chnum);

	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
	disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
}

/*
 * For debug only. It checks that a valid halt status is set and that
 * HCCHARn.chdis is clear. If there's a problem, corrective action is
 * taken and a warning is issued.
 *
 * Return: true if halt status is ok, false otherwise
 */
static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
				struct dwc2_host_chan *chan, int chnum,
				struct dwc2_qtd *qtd)
{
#ifdef DEBUG
	u32 hcchar;
	u32 hctsiz;
	u32 hcintmsk;
	u32 hcsplt;

	if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
		/*
		 * This code is here only as a check. This condition should
		 * never happen. Ignore the halt if it does occur.
		 */
1785 1786 1787 1788
		hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
		hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
		hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
		hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
1789 1790 1791 1792 1793 1794 1795 1796 1797
		dev_dbg(hsotg->dev,
			"%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
			 __func__);
		dev_dbg(hsotg->dev,
			"channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
			chnum, hcchar, hctsiz);
		dev_dbg(hsotg->dev,
			"hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
			chan->hcint, hcintmsk, hcsplt);
1798 1799 1800
		if (qtd)
			dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
				qtd->complete_split);
1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811
		dev_warn(hsotg->dev,
			 "%s: no halt status, channel %d, ignoring interrupt\n",
			 __func__, chnum);
		return false;
	}

	/*
	 * This code is here only as a check. hcchar.chdis should never be set
	 * when the halt interrupt occurs. Halt the channel again if it does
	 * occur.
	 */
1812
	hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836
	if (hcchar & HCCHAR_CHDIS) {
		dev_warn(hsotg->dev,
			 "%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
			 __func__, hcchar);
		chan->halt_pending = 0;
		dwc2_halt_channel(hsotg, chan, qtd, chan->halt_status);
		return false;
	}
#endif

	return true;
}

/*
 * Handles a host Channel Halted interrupt in DMA mode. This handler
 * determines the reason the channel halted and proceeds accordingly.
 */
static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
				    struct dwc2_host_chan *chan, int chnum,
				    struct dwc2_qtd *qtd)
{
	u32 hcintmsk;
	int out_nak_enh = 0;

1837 1838 1839 1840
	if (dbg_hc(chan))
		dev_vdbg(hsotg->dev,
			 "--Host Channel %d Interrupt: DMA Channel Halted--\n",
			 chnum);
1841 1842 1843 1844 1845

	/*
	 * For core with OUT NAK enhancement, the flow for high-speed
	 * CONTROL/BULK OUT is handled a little differently
	 */
1846
	if (hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_71a) {
1847 1848 1849 1850 1851 1852 1853 1854 1855
		if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
		    (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
		     chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
			out_nak_enh = 1;
		}
	}

	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
	    (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1856 1857
	     !hsotg->params.dma_desc_enable)) {
		if (hsotg->params.dma_desc_enable)
1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871
			dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
						    chan->halt_status);
		else
			/*
			 * Just release the channel. A dequeue can happen on a
			 * transfer timeout. In the case of an AHB Error, the
			 * channel was forced to halt because there's no way to
			 * gracefully recover.
			 */
			dwc2_release_channel(hsotg, chan, qtd,
					     chan->halt_status);
		return;
	}

1872
	hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887

	if (chan->hcint & HCINTMSK_XFERCOMPL) {
		/*
		 * Todo: This is here because of a possible hardware bug. Spec
		 * says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
		 * interrupt w/ACK bit set should occur, but I only see the
		 * XFERCOMP bit, even with it masked out. This is a workaround
		 * for that behavior. Should fix this when hardware is fixed.
		 */
		if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
			dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
		dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
	} else if (chan->hcint & HCINTMSK_STALL) {
		dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
	} else if ((chan->hcint & HCINTMSK_XACTERR) &&
1888
		   !hsotg->params.dma_desc_enable) {
1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907
		if (out_nak_enh) {
			if (chan->hcint &
			    (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
				dev_vdbg(hsotg->dev,
					 "XactErr with NYET/NAK/ACK\n");
				qtd->error_count = 0;
			} else {
				dev_vdbg(hsotg->dev,
					 "XactErr without NYET/NAK/ACK\n");
			}
		}

		/*
		 * Must handle xacterr before nak or ack. Could get a xacterr
		 * at the same time as either of these on a BULK/CONTROL OUT
		 * that started with a PING. The xacterr takes precedence.
		 */
		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
	} else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1908
		   hsotg->params.dma_desc_enable) {
1909 1910
		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
	} else if ((chan->hcint & HCINTMSK_AHBERR) &&
1911
		   hsotg->params.dma_desc_enable) {
1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958
		dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
	} else if (chan->hcint & HCINTMSK_BBLERR) {
		dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
	} else if (chan->hcint & HCINTMSK_FRMOVRUN) {
		dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
	} else if (!out_nak_enh) {
		if (chan->hcint & HCINTMSK_NYET) {
			/*
			 * Must handle nyet before nak or ack. Could get a nyet
			 * at the same time as either of those on a BULK/CONTROL
			 * OUT that started with a PING. The nyet takes
			 * precedence.
			 */
			dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
		} else if ((chan->hcint & HCINTMSK_NAK) &&
			   !(hcintmsk & HCINTMSK_NAK)) {
			/*
			 * If nak is not masked, it's because a non-split IN
			 * transfer is in an error state. In that case, the nak
			 * is handled by the nak interrupt handler, not here.
			 * Handle nak here for BULK/CONTROL OUT transfers, which
			 * halt on a NAK to allow rewinding the buffer pointer.
			 */
			dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
		} else if ((chan->hcint & HCINTMSK_ACK) &&
			   !(hcintmsk & HCINTMSK_ACK)) {
			/*
			 * If ack is not masked, it's because a non-split IN
			 * transfer is in an error state. In that case, the ack
			 * is handled by the ack interrupt handler, not here.
			 * Handle ack here for split transfers. Start splits
			 * halt on ACK.
			 */
			dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
		} else {
			if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
			    chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
				/*
				 * A periodic transfer halted with no other
				 * channel interrupts set. Assume it was halted
				 * by the core because it could not be completed
				 * in its scheduled (micro)frame.
				 */
				dev_dbg(hsotg->dev,
					"%s: Halt channel %d (assume incomplete periodic transfer)\n",
					__func__, chnum);
				dwc2_halt_channel(hsotg, chan, qtd,
1959
					DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1960 1961 1962 1963 1964 1965 1966
			} else {
				dev_err(hsotg->dev,
					"%s: Channel %d - ChHltd set, but reason is unknown\n",
					__func__, chnum);
				dev_err(hsotg->dev,
					"hcint 0x%08x, intsts 0x%08x\n",
					chan->hcint,
1967
					dwc2_readl(hsotg, GINTSTS));
1968
				goto error;
1969 1970 1971 1972 1973 1974
			}
		}
	} else {
		dev_info(hsotg->dev,
			 "NYET/NAK/ACK/other in non-error case, 0x%08x\n",
			 chan->hcint);
1975 1976 1977 1978 1979
error:
		/* Failthrough: use 3-strikes rule */
		qtd->error_count++;
		dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
					  qtd, DWC2_HC_XFER_XACT_ERR);
1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991
		/*
		 * We can get here after a completed transaction
		 * (urb->actual_length >= urb->length) which was not reported
		 * as completed. If that is the case, and we do not abort
		 * the transfer, a transfer of size 0 will be enqueued
		 * subsequently. If urb->actual_length is not DMA-aligned,
		 * the buffer will then point to an unaligned address, and
		 * the resulting behavior is undefined. Bail out in that
		 * situation.
		 */
		if (qtd->urb->actual_length >= qtd->urb->length)
			qtd->error_count = 3;
1992 1993
		dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011
	}
}

/*
 * Handles a host channel Channel Halted interrupt
 *
 * In slave mode, this handler is called only when the driver specifically
 * requests a halt. This occurs during handling other host channel interrupts
 * (e.g. nak, xacterr, stall, nyet, etc.).
 *
 * In DMA mode, this is the interrupt that occurs when the core has finished
 * processing a transfer on a channel. Other host channel interrupts (except
 * ahberr) are disabled in DMA mode.
 */
static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
				struct dwc2_host_chan *chan, int chnum,
				struct dwc2_qtd *qtd)
{
2012 2013 2014
	if (dbg_hc(chan))
		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
			 chnum);
2015

2016
	if (hsotg->params.host_dma) {
2017 2018 2019 2020 2021 2022 2023 2024
		dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
	} else {
		if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
			return;
		dwc2_release_channel(hsotg, chan, qtd, chan->halt_status);
	}
}

2025 2026 2027 2028 2029 2030 2031 2032 2033 2034
/*
 * Check if the given qtd is still the top of the list (and thus valid).
 *
 * If dwc2_hcd_qtd_unlink_and_free() has been called since we grabbed
 * the qtd from the top of the list, this will return false (otherwise true).
 */
static bool dwc2_check_qtd_still_ok(struct dwc2_qtd *qtd, struct dwc2_qh *qh)
{
	struct dwc2_qtd *cur_head;

2035
	if (!qh)
2036 2037 2038 2039 2040 2041 2042
		return false;

	cur_head = list_first_entry(&qh->qtd_list, struct dwc2_qtd,
				    qtd_list_entry);
	return (cur_head == qtd);
}

2043 2044 2045 2046 2047 2048 2049
/* Handles interrupt for a specific Host Channel */
static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
{
	struct dwc2_qtd *qtd;
	struct dwc2_host_chan *chan;
	u32 hcint, hcintmsk;

2050 2051
	chan = hsotg->hc_ptr_array[chnum];

2052 2053
	hcint = dwc2_readl(hsotg, HCINT(chnum));
	hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
2054 2055
	if (!chan) {
		dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
2056
		dwc2_writel(hsotg, hcint, HCINT(chnum));
2057 2058 2059
		return;
	}

2060 2061 2062 2063 2064 2065 2066 2067
	if (dbg_hc(chan)) {
		dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n",
			 chnum);
		dev_vdbg(hsotg->dev,
			 "  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
			 hcint, hcintmsk, hcint & hcintmsk);
	}

2068
	dwc2_writel(hsotg, hcint, HCINT(chnum));
2069 2070 2071 2072 2073 2074 2075 2076 2077 2078

	/*
	 * If we got an interrupt after someone called
	 * dwc2_hcd_endpoint_disable() we don't want to crash below
	 */
	if (!chan->qh) {
		dev_warn(hsotg->dev, "Interrupt on disabled channel\n");
		return;
	}

2079 2080 2081
	chan->hcint = hcint;
	hcint &= hcintmsk;

2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092
	/*
	 * If the channel was halted due to a dequeue, the qtd list might
	 * be empty or at least the first entry will not be the active qtd.
	 * In this case, take a shortcut and just release the channel.
	 */
	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
		/*
		 * If the channel was halted, this should be the only
		 * interrupt unmasked
		 */
		WARN_ON(hcint != HCINTMSK_CHHLTD);
2093
		if (hsotg->params.dma_desc_enable)
2094 2095 2096 2097 2098 2099 2100 2101
			dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
						    chan->halt_status);
		else
			dwc2_release_channel(hsotg, chan, NULL,
					     chan->halt_status);
		return;
	}

2102
	if (list_empty(&chan->qh->qtd_list)) {
2103 2104 2105 2106
		/*
		 * TODO: Will this ever happen with the
		 * DWC2_HC_XFER_URB_DEQUEUE handling above?
		 */
2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120
		dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
			chnum);
		dev_dbg(hsotg->dev,
			"  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
			chan->hcint, hcintmsk, hcint);
		chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
		disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
		chan->hcint = 0;
		return;
	}

	qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
			       qtd_list_entry);

2121
	if (!hsotg->params.host_dma) {
2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134
		if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
			hcint &= ~HCINTMSK_CHHLTD;
	}

	if (hcint & HCINTMSK_XFERCOMPL) {
		dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
		/*
		 * If NYET occurred at same time as Xfer Complete, the NYET is
		 * handled by the Xfer Complete interrupt handler. Don't want
		 * to call the NYET interrupt handler in this case.
		 */
		hcint &= ~HCINTMSK_NYET;
	}
2135 2136

	if (hcint & HCINTMSK_CHHLTD) {
2137
		dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
2138 2139 2140 2141
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
	if (hcint & HCINTMSK_AHBERR) {
2142
		dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
2143 2144 2145 2146
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
	if (hcint & HCINTMSK_STALL) {
2147
		dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
2148 2149 2150 2151
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
	if (hcint & HCINTMSK_NAK) {
2152
		dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
2153 2154 2155 2156
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
	if (hcint & HCINTMSK_ACK) {
2157
		dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
2158 2159 2160 2161
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
	if (hcint & HCINTMSK_NYET) {
2162
		dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
2163 2164 2165 2166
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
	if (hcint & HCINTMSK_XACTERR) {
2167
		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
2168 2169 2170 2171
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
	if (hcint & HCINTMSK_BBLERR) {
2172
		dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
2173 2174 2175 2176
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
	if (hcint & HCINTMSK_FRMOVRUN) {
2177
		dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
2178 2179 2180 2181
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
	if (hcint & HCINTMSK_DATATGLERR) {
2182
		dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
2183 2184 2185
		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
			goto exit;
	}
2186

2187
exit:
2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200
	chan->hcint = 0;
}

/*
 * This interrupt indicates that one or more host channels has a pending
 * interrupt. There are multiple conditions that can cause each host channel
 * interrupt. This function determines which conditions have occurred for each
 * host channel interrupt and handles them appropriately.
 */
static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
{
	u32 haint;
	int i;
2201
	struct dwc2_host_chan *chan, *chan_tmp;
2202

2203
	haint = dwc2_readl(hsotg, HAINT);
2204 2205 2206 2207 2208
	if (dbg_perio()) {
		dev_vdbg(hsotg->dev, "%s()\n", __func__);

		dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
	}
2209

2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225
	/*
	 * According to USB 2.0 spec section 11.18.8, a host must
	 * issue complete-split transactions in a microframe for a
	 * set of full-/low-speed endpoints in the same relative
	 * order as the start-splits were issued in a microframe for.
	 */
	list_for_each_entry_safe(chan, chan_tmp, &hsotg->split_order,
				 split_order_list_entry) {
		int hc_num = chan->hc_num;

		if (haint & (1 << hc_num)) {
			dwc2_hc_n_intr(hsotg, hc_num);
			haint &= ~(1 << hc_num);
		}
	}

2226
	for (i = 0; i < hsotg->params.host_channels; i++) {
2227 2228 2229 2230 2231 2232
		if (haint & (1 << i))
			dwc2_hc_n_intr(hsotg, i);
	}
}

/* This function handles interrupts for the HCD */
2233
irqreturn_t dwc2_handle_hcd_intr(struct dwc2_hsotg *hsotg)
2234
{
2235
	u32 gintsts, dbg_gintsts;
2236
	irqreturn_t retval = IRQ_NONE;
2237

2238
	if (!dwc2_is_controller_alive(hsotg)) {
2239
		dev_warn(hsotg->dev, "Controller is dead\n");
2240
		return retval;
2241 2242 2243 2244 2245 2246 2247 2248 2249
	}

	spin_lock(&hsotg->lock);

	/* Check if HOST Mode */
	if (dwc2_is_host_mode(hsotg)) {
		gintsts = dwc2_read_core_intr(hsotg);
		if (!gintsts) {
			spin_unlock(&hsotg->lock);
2250
			return retval;
2251 2252
		}

2253
		retval = IRQ_HANDLED;
2254

2255
		dbg_gintsts = gintsts;
2256
#ifndef DEBUG_SOF
2257
		dbg_gintsts &= ~GINTSTS_SOF;
2258
#endif
2259 2260 2261 2262 2263 2264
		if (!dbg_perio())
			dbg_gintsts &= ~(GINTSTS_HCHINT | GINTSTS_RXFLVL |
					 GINTSTS_PTXFEMP);

		/* Only print if there are any non-suppressed interrupts left */
		if (dbg_gintsts)
2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281
			dev_vdbg(hsotg->dev,
				 "DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
				 gintsts);

		if (gintsts & GINTSTS_SOF)
			dwc2_sof_intr(hsotg);
		if (gintsts & GINTSTS_RXFLVL)
			dwc2_rx_fifo_level_intr(hsotg);
		if (gintsts & GINTSTS_NPTXFEMP)
			dwc2_np_tx_fifo_empty_intr(hsotg);
		if (gintsts & GINTSTS_PRTINT)
			dwc2_port_intr(hsotg);
		if (gintsts & GINTSTS_HCHINT)
			dwc2_hc_intr(hsotg);
		if (gintsts & GINTSTS_PTXFEMP)
			dwc2_perio_tx_fifo_empty_intr(hsotg);

2282
		if (dbg_gintsts) {
2283 2284 2285 2286
			dev_vdbg(hsotg->dev,
				 "DWC OTG HCD Finished Servicing Interrupts\n");
			dev_vdbg(hsotg->dev,
				 "DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2287 2288
				 dwc2_readl(hsotg, GINTSTS),
				 dwc2_readl(hsotg, GINTMSK));
2289 2290 2291 2292 2293 2294 2295
		}
	}

	spin_unlock(&hsotg->lock);

	return retval;
}