trans.c 89.7 KB
Newer Older
1 2 3 4 5 6 7
/******************************************************************************
 *
 * This file is provided under a dual BSD/GPLv2 license.  When using or
 * redistributing this file, you may do so under either license.
 *
 * GPL LICENSE SUMMARY
 *
8 9
 * Copyright(c) 2007 - 2015 Intel Corporation. All rights reserved.
 * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
10
 * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of version 2 of the GNU General Public License as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
 * USA
 *
 * The full GNU General Public License is included in this distribution
27
 * in the file called COPYING.
28 29
 *
 * Contact Information:
30
 *  Intel Linux Wireless <linuxwifi@intel.com>
31 32 33 34
 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
 *
 * BSD LICENSE
 *
35 36
 * Copyright(c) 2005 - 2015 Intel Corporation. All rights reserved.
 * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
37
 * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 *  * Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *  * Neither the name Intel Corporation nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *****************************************************************************/
67 68
#include <linux/pci.h>
#include <linux/pci-aspm.h>
69
#include <linux/interrupt.h>
70
#include <linux/debugfs.h>
71
#include <linux/sched.h>
72 73
#include <linux/bitops.h>
#include <linux/gfp.h>
74
#include <linux/vmalloc.h>
75
#include <linux/pm_runtime.h>
76

77
#include "iwl-drv.h"
78
#include "iwl-trans.h"
79 80
#include "iwl-csr.h"
#include "iwl-prph.h"
81
#include "iwl-scd.h"
82
#include "iwl-agn-hw.h"
83
#include "fw/error-dump.h"
84
#include "internal.h"
85
#include "iwl-fh.h"
86

87 88 89 90
/* extended range in FW SRAM */
#define IWL_FW_MEM_EXTENDED_START	0x40000
#define IWL_FW_MEM_EXTENDED_END		0x57FFF

91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
static void iwl_pcie_free_fw_monitor(struct iwl_trans *trans)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	if (!trans_pcie->fw_mon_page)
		return;

	dma_unmap_page(trans->dev, trans_pcie->fw_mon_phys,
		       trans_pcie->fw_mon_size, DMA_FROM_DEVICE);
	__free_pages(trans_pcie->fw_mon_page,
		     get_order(trans_pcie->fw_mon_size));
	trans_pcie->fw_mon_page = NULL;
	trans_pcie->fw_mon_phys = 0;
	trans_pcie->fw_mon_size = 0;
}

107
static void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power)
108 109
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
110
	struct page *page = NULL;
111
	dma_addr_t phys;
112
	u32 size = 0;
113 114
	u8 power;

115 116 117 118 119 120 121 122 123 124 125 126
	if (!max_power) {
		/* default max_power is maximum */
		max_power = 26;
	} else {
		max_power += 11;
	}

	if (WARN(max_power > 26,
		 "External buffer size for monitor is too big %d, check the FW TLV\n",
		 max_power))
		return;

127 128 129 130 131 132 133 134
	if (trans_pcie->fw_mon_page) {
		dma_sync_single_for_device(trans->dev, trans_pcie->fw_mon_phys,
					   trans_pcie->fw_mon_size,
					   DMA_FROM_DEVICE);
		return;
	}

	phys = 0;
135
	for (power = max_power; power >= 11; power--) {
136 137 138 139 140 141 142 143 144 145 146 147 148
		int order;

		size = BIT(power);
		order = get_order(size);
		page = alloc_pages(__GFP_COMP | __GFP_NOWARN | __GFP_ZERO,
				   order);
		if (!page)
			continue;

		phys = dma_map_page(trans->dev, page, 0, PAGE_SIZE << order,
				    DMA_FROM_DEVICE);
		if (dma_mapping_error(trans->dev, phys)) {
			__free_pages(page, order);
149
			page = NULL;
150 151 152 153 154 155 156 157
			continue;
		}
		IWL_INFO(trans,
			 "Allocated 0x%08x bytes (order %d) for firmware monitor.\n",
			 size, order);
		break;
	}

158
	if (WARN_ON_ONCE(!page))
159 160
		return;

161 162 163 164 165 166
	if (power != max_power)
		IWL_ERR(trans,
			"Sorry - debug buffer is only %luK while you requested %luK\n",
			(unsigned long)BIT(power - 10),
			(unsigned long)BIT(max_power - 10));

167 168 169 170 171
	trans_pcie->fw_mon_page = page;
	trans_pcie->fw_mon_phys = phys;
	trans_pcie->fw_mon_size = size;
}

172 173 174 175 176 177 178 179 180 181 182 183 184 185
static u32 iwl_trans_pcie_read_shr(struct iwl_trans *trans, u32 reg)
{
	iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
		    ((reg & 0x0000ffff) | (2 << 28)));
	return iwl_read32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG);
}

static void iwl_trans_pcie_write_shr(struct iwl_trans *trans, u32 reg, u32 val)
{
	iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG, val);
	iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
		    ((reg & 0x0000ffff) | (3 << 28)));
}

186
static void iwl_pcie_set_pwr(struct iwl_trans *trans, bool vaux)
187
{
188
	if (trans->cfg->apmg_not_supported)
189 190
		return;

191 192 193 194 195 196 197 198
	if (vaux && pci_pme_capable(to_pci_dev(trans->dev), PCI_D3cold))
		iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
				       APMG_PS_CTRL_VAL_PWR_SRC_VAUX,
				       ~APMG_PS_CTRL_MSK_PWR_SRC);
	else
		iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
				       APMG_PS_CTRL_VAL_PWR_SRC_VMAIN,
				       ~APMG_PS_CTRL_MSK_PWR_SRC);
199 200
}

201 202 203
/* PCI registers */
#define PCI_CFG_RETRY_TIMEOUT	0x041

204
void iwl_pcie_apm_config(struct iwl_trans *trans)
205
{
206
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
207
	u16 lctl;
208
	u16 cap;
209 210 211 212 213 214 215 216 217

	/*
	 * HW bug W/A for instability in PCIe bus L0S->L1 transition.
	 * Check if BIOS (or OS) enabled L1-ASPM on this device.
	 * If so (likely), disable L0S, so device moves directly L0->L1;
	 *    costs negligible amount of power savings.
	 * If not (unlikely), enable L0S, so there is at least some
	 *    power savings, even without L1.
	 */
218
	pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_LNKCTL, &lctl);
219
	if (lctl & PCI_EXP_LNKCTL_ASPM_L1)
220
		iwl_set_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_ENABLED);
221
	else
222
		iwl_clear_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_ENABLED);
223
	trans->pm_support = !(lctl & PCI_EXP_LNKCTL_ASPM_L0S);
224 225 226

	pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_DEVCTL2, &cap);
	trans->ltr_enabled = cap & PCI_EXP_DEVCTL2_LTR_EN;
227 228 229
	IWL_DEBUG_POWER(trans, "L1 %sabled - LTR %sabled\n",
			(lctl & PCI_EXP_LNKCTL_ASPM_L1) ? "En" : "Dis",
			trans->ltr_enabled ? "En" : "Dis");
230 231
}

232 233
/*
 * Start up NIC's basic functionality after it has been reset
234
 * (e.g. after platform boot, or shutdown via iwl_pcie_apm_stop())
235 236
 * NOTE:  This does not load uCode nor start the embedded processor
 */
237
static int iwl_pcie_apm_init(struct iwl_trans *trans)
238
{
239 240
	int ret;

241 242 243 244 245 246 247 248
	IWL_DEBUG_INFO(trans, "Init card's basic functions\n");

	/*
	 * Use "set_bit" below rather than "write", to preserve any hardware
	 * bits already set by default after reset.
	 */

	/* Disable L0S exit timer (platform NMI Work/Around) */
249
	if (trans->cfg->device_family < IWL_DEVICE_FAMILY_8000)
250 251
		iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
			    CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER);
252 253 254 255 256 257

	/*
	 * Disable L0s without affecting L1;
	 *  don't wait for ICH L0s (ICH bug W/A)
	 */
	iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
258
		    CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX);
259 260 261 262 263 264 265 266 267

	/* Set FH wait threshold to maximum (HW error during stress W/A) */
	iwl_set_bit(trans, CSR_DBG_HPET_MEM_REG, CSR_DBG_HPET_MEM_REG_VAL);

	/*
	 * Enable HAP INTA (interrupt from management bus) to
	 * wake device's PCI Express link L1a -> L0s
	 */
	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
268
		    CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
269

270
	iwl_pcie_apm_config(trans);
271 272

	/* Configure analog phase-lock-loop before activating to D0A */
273 274
	if (trans->cfg->base_params->pll_cfg)
		iwl_set_bit(trans, CSR_ANA_PLL_CFG, CSR50_ANA_PLL_CFG_VAL);
275 276 277 278 279 280 281 282 283 284 285 286 287

	/*
	 * Set "initialization complete" bit to move adapter from
	 * D0U* --> D0A* (powered-up active) state.
	 */
	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);

	/*
	 * Wait for clock stabilization; once stabilized, access to
	 * device-internal resources is supported, e.g. iwl_write_prph()
	 * and accesses to uCode SRAM.
	 */
	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
288 289
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY, 25000);
290
	if (ret < 0) {
291 292
		IWL_ERR(trans, "Failed to init the card\n");
		return ret;
293 294
	}

295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316
	if (trans->cfg->host_interrupt_operation_mode) {
		/*
		 * This is a bit of an abuse - This is needed for 7260 / 3160
		 * only check host_interrupt_operation_mode even if this is
		 * not related to host_interrupt_operation_mode.
		 *
		 * Enable the oscillator to count wake up time for L1 exit. This
		 * consumes slightly more power (100uA) - but allows to be sure
		 * that we wake up from L1 on time.
		 *
		 * This looks weird: read twice the same register, discard the
		 * value, set a bit, and yet again, read that same register
		 * just to discard the value. But that's the way the hardware
		 * seems to like it.
		 */
		iwl_read_prph(trans, OSC_CLK);
		iwl_read_prph(trans, OSC_CLK);
		iwl_set_bits_prph(trans, OSC_CLK, OSC_CLK_FORCE_CONTROL);
		iwl_read_prph(trans, OSC_CLK);
		iwl_read_prph(trans, OSC_CLK);
	}

317 318 319
	/*
	 * Enable DMA clock and wait for it to stabilize.
	 *
320 321 322
	 * Write to "CLK_EN_REG"; "1" bits enable clocks, while "0"
	 * bits do not disable clocks.  This preserves any hardware
	 * bits already set by default in "CLK_CTRL_REG" after reset.
323
	 */
324
	if (!trans->cfg->apmg_not_supported) {
325 326 327 328 329 330 331 332 333 334 335 336
		iwl_write_prph(trans, APMG_CLK_EN_REG,
			       APMG_CLK_VAL_DMA_CLK_RQT);
		udelay(20);

		/* Disable L1-Active */
		iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
				  APMG_PCIDEV_STT_VAL_L1_ACT_DIS);

		/* Clear the interrupt in APMG if the NIC is in RFKILL */
		iwl_write_prph(trans, APMG_RTC_INT_STT_REG,
			       APMG_RTC_INT_STT_RFKILL);
	}
337

338
	set_bit(STATUS_DEVICE_ENABLED, &trans->status);
339

340
	return 0;
341 342
}

343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
/*
 * Enable LP XTAL to avoid HW bug where device may consume much power if
 * FW is not loaded after device reset. LP XTAL is disabled by default
 * after device HW reset. Do it only if XTAL is fed by internal source.
 * Configure device's "persistence" mode to avoid resetting XTAL again when
 * SHRD_HW_RST occurs in S3.
 */
static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
{
	int ret;
	u32 apmg_gp1_reg;
	u32 apmg_xtal_cfg_reg;
	u32 dl_cfg_reg;

	/* Force XTAL ON */
	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
				 CSR_GP_CNTRL_REG_FLAG_XTAL_ON);

	/* Reset entire device - do controller reset (results in SHRD_HW_RST) */
	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
363
	usleep_range(1000, 2000);
364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408

	/*
	 * Set "initialization complete" bit to move adapter from
	 * D0U* --> D0A* (powered-up active) state.
	 */
	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);

	/*
	 * Wait for clock stabilization; once stabilized, access to
	 * device-internal resources is possible.
	 */
	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
			   25000);
	if (WARN_ON(ret < 0)) {
		IWL_ERR(trans, "Access time out - failed to enable LP XTAL\n");
		/* Release XTAL ON request */
		__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
					   CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
		return;
	}

	/*
	 * Clear "disable persistence" to avoid LP XTAL resetting when
	 * SHRD_HW_RST is applied in S3.
	 */
	iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
				    APMG_PCIDEV_STT_VAL_PERSIST_DIS);

	/*
	 * Force APMG XTAL to be active to prevent its disabling by HW
	 * caused by APMG idle state.
	 */
	apmg_xtal_cfg_reg = iwl_trans_pcie_read_shr(trans,
						    SHR_APMG_XTAL_CFG_REG);
	iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
				 apmg_xtal_cfg_reg |
				 SHR_APMG_XTAL_CFG_XTAL_ON_REQ);

	/*
	 * Reset entire device again - do controller reset (results in
	 * SHRD_HW_RST). Turn MAC off before proceeding.
	 */
	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
409
	usleep_range(1000, 2000);
410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450

	/* Enable LP XTAL by indirect access through CSR */
	apmg_gp1_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_GP1_REG);
	iwl_trans_pcie_write_shr(trans, SHR_APMG_GP1_REG, apmg_gp1_reg |
				 SHR_APMG_GP1_WF_XTAL_LP_EN |
				 SHR_APMG_GP1_CHICKEN_BIT_SELECT);

	/* Clear delay line clock power up */
	dl_cfg_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_DL_CFG_REG);
	iwl_trans_pcie_write_shr(trans, SHR_APMG_DL_CFG_REG, dl_cfg_reg &
				 ~SHR_APMG_DL_CFG_DL_CLOCK_POWER_UP);

	/*
	 * Enable persistence mode to avoid LP XTAL resetting when
	 * SHRD_HW_RST is applied in S3.
	 */
	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
		    CSR_HW_IF_CONFIG_REG_PERSIST_MODE);

	/*
	 * Clear "initialization complete" bit to move adapter from
	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
	 */
	iwl_clear_bit(trans, CSR_GP_CNTRL,
		      CSR_GP_CNTRL_REG_FLAG_INIT_DONE);

	/* Activates XTAL resources monitor */
	__iwl_trans_pcie_set_bit(trans, CSR_MONITOR_CFG_REG,
				 CSR_MONITOR_XTAL_RESOURCES);

	/* Release XTAL ON request */
	__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
				   CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
	udelay(10);

	/* Release APMG XTAL */
	iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
				 apmg_xtal_cfg_reg &
				 ~SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
}

451
void iwl_pcie_apm_stop_master(struct iwl_trans *trans)
452
{
453
	int ret;
454 455 456 457 458

	/* stop device's busmaster DMA activity */
	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_STOP_MASTER);

	ret = iwl_poll_bit(trans, CSR_RESET,
459 460
			   CSR_RESET_REG_FLAG_MASTER_DISABLED,
			   CSR_RESET_REG_FLAG_MASTER_DISABLED, 100);
461
	if (ret < 0)
462 463 464 465 466
		IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n");

	IWL_DEBUG_INFO(trans, "stop master\n");
}

467
static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
468 469 470
{
	IWL_DEBUG_INFO(trans, "Stop card, put in low power state\n");

471 472 473 474 475 476 477 478
	if (op_mode_leave) {
		if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
			iwl_pcie_apm_init(trans);

		/* inform ME that we are leaving */
		if (trans->cfg->device_family == IWL_DEVICE_FAMILY_7000)
			iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
					  APMG_PCIDEV_STT_VAL_WAKE_ME);
479
		else if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000) {
480 481
			iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
				    CSR_RESET_LINK_PWR_MGMT_DISABLED);
482 483 484
			iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
				    CSR_HW_IF_CONFIG_REG_PREPARE |
				    CSR_HW_IF_CONFIG_REG_ENABLE_PME);
485 486 487 488
			mdelay(1);
			iwl_clear_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
				      CSR_RESET_LINK_PWR_MGMT_DISABLED);
		}
489 490 491
		mdelay(5);
	}

492
	clear_bit(STATUS_DEVICE_ENABLED, &trans->status);
493 494

	/* Stop device's DMA activity */
495
	iwl_pcie_apm_stop_master(trans);
496

497 498 499 500 501
	if (trans->cfg->lp_xtal_workaround) {
		iwl_pcie_apm_lp_xtal_enable(trans);
		return;
	}

502 503
	/* Reset the entire device */
	iwl_set_bit(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
504
	usleep_range(1000, 2000);
505 506 507 508 509 510 511 512 513

	/*
	 * Clear "initialization complete" bit to move adapter from
	 * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
	 */
	iwl_clear_bit(trans, CSR_GP_CNTRL,
		      CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
}

514
static int iwl_pcie_nic_init(struct iwl_trans *trans)
515
{
516
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
517
	int ret;
518 519

	/* nic_init */
520
	spin_lock(&trans_pcie->irq_lock);
521
	ret = iwl_pcie_apm_init(trans);
522
	spin_unlock(&trans_pcie->irq_lock);
523

524 525 526
	if (ret)
		return ret;

527
	iwl_pcie_set_pwr(trans, false);
528

529
	iwl_op_mode_nic_config(trans->op_mode);
530 531

	/* Allocate the RX queue, or reset if it is already allocated */
532
	iwl_pcie_rx_init(trans);
533 534

	/* Allocate or reset and init all Tx and Command queues */
535
	if (iwl_pcie_tx_init(trans))
536 537
		return -ENOMEM;

538
	if (trans->cfg->base_params->shadow_reg_enable) {
539
		/* enable shadow regs in HW */
540
		iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, 0x800FFFFF);
541
		IWL_DEBUG_INFO(trans, "Enabling shadow registers in device\n");
542 543 544 545 546 547 548 549
	}

	return 0;
}

#define HW_READY_TIMEOUT (50)

/* Note: returns poll_bit return value, which is >= 0 if success */
550
static int iwl_pcie_set_hw_ready(struct iwl_trans *trans)
551 552 553
{
	int ret;

554
	iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
555
		    CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);
556 557

	/* See if we got it */
558
	ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
559 560 561
			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
			   CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
			   HW_READY_TIMEOUT);
562

563 564 565
	if (ret >= 0)
		iwl_set_bit(trans, CSR_MBOX_SET_REG, CSR_MBOX_SET_REG_OS_ALIVE);

566
	IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : "");
567 568 569 570
	return ret;
}

/* Note: returns standard 0/-ERROR code */
571
int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
572 573
{
	int ret;
574
	int t = 0;
575
	int iter;
576

577
	IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n");
578

579
	ret = iwl_pcie_set_hw_ready(trans);
580
	/* If the card is ready, exit 0 */
581 582 583
	if (ret >= 0)
		return 0;

584 585
	iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
		    CSR_RESET_LINK_PWR_MGMT_DISABLED);
586
	usleep_range(1000, 2000);
587

588 589 590 591 592 593 594
	for (iter = 0; iter < 10; iter++) {
		/* If HW is not ready, prepare the conditions to check again */
		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
			    CSR_HW_IF_CONFIG_REG_PREPARE);

		do {
			ret = iwl_pcie_set_hw_ready(trans);
595 596
			if (ret >= 0)
				return 0;
597

598 599 600 601 602
			usleep_range(200, 1000);
			t += 200;
		} while (t < 150000);
		msleep(25);
	}
603

604
	IWL_ERR(trans, "Couldn't prepare the card\n");
605 606 607 608

	return ret;
}

609 610 611
/*
 * ucode
 */
612 613 614
static void iwl_pcie_load_firmware_chunk_fh(struct iwl_trans *trans,
					    u32 dst_addr, dma_addr_t phy_addr,
					    u32 byte_cnt)
615
{
616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637
	iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
		    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);

	iwl_write32(trans, FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL),
		    dst_addr);

	iwl_write32(trans, FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL),
		    phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);

	iwl_write32(trans, FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL),
		    (iwl_get_dma_hi_addr(phy_addr)
			<< FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);

	iwl_write32(trans, FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL),
		    BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM) |
		    BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX) |
		    FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);

	iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
		    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
		    FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
		    FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
638 639 640 641 642 643 644 645 646 647 648 649 650 651 652
}

static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans,
					u32 dst_addr, dma_addr_t phy_addr,
					u32 byte_cnt)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	unsigned long flags;
	int ret;

	trans_pcie->ucode_write_complete = false;

	if (!iwl_trans_grab_nic_access(trans, &flags))
		return -EIO;

653 654
	iwl_pcie_load_firmware_chunk_fh(trans, dst_addr, phy_addr,
					byte_cnt);
655
	iwl_trans_release_nic_access(trans, &flags);
656

657 658
	ret = wait_event_timeout(trans_pcie->ucode_write_waitq,
				 trans_pcie->ucode_write_complete, 5 * HZ);
659
	if (!ret) {
660
		IWL_ERR(trans, "Failed to load firmware chunk!\n");
661 662 663 664 665 666
		return -ETIMEDOUT;
	}

	return 0;
}

667
static int iwl_pcie_load_section(struct iwl_trans *trans, u8 section_num,
668
			    const struct fw_desc *section)
669
{
670 671
	u8 *v_addr;
	dma_addr_t p_addr;
672
	u32 offset, chunk_sz = min_t(u32, FH_MEM_TB_MAX_LENGTH, section->len);
673 674
	int ret = 0;

675 676 677
	IWL_DEBUG_FW(trans, "[%d] uCode section being loaded...\n",
		     section_num);

678 679 680 681 682 683 684 685 686 687
	v_addr = dma_alloc_coherent(trans->dev, chunk_sz, &p_addr,
				    GFP_KERNEL | __GFP_NOWARN);
	if (!v_addr) {
		IWL_DEBUG_INFO(trans, "Falling back to small chunks of DMA\n");
		chunk_sz = PAGE_SIZE;
		v_addr = dma_alloc_coherent(trans->dev, chunk_sz,
					    &p_addr, GFP_KERNEL);
		if (!v_addr)
			return -ENOMEM;
	}
688

689
	for (offset = 0; offset < section->len; offset += chunk_sz) {
690 691
		u32 copy_size, dst_addr;
		bool extended_addr = false;
692

693
		copy_size = min_t(u32, chunk_sz, section->len - offset);
694 695 696 697 698 699 700 701 702
		dst_addr = section->offset + offset;

		if (dst_addr >= IWL_FW_MEM_EXTENDED_START &&
		    dst_addr <= IWL_FW_MEM_EXTENDED_END)
			extended_addr = true;

		if (extended_addr)
			iwl_set_bits_prph(trans, LMPM_CHICK,
					  LMPM_CHICK_EXTENDED_ADDR_SPACE);
703

704
		memcpy(v_addr, (u8 *)section->data + offset, copy_size);
705 706 707 708 709 710 711
		ret = iwl_pcie_load_firmware_chunk(trans, dst_addr, p_addr,
						   copy_size);

		if (extended_addr)
			iwl_clear_bits_prph(trans, LMPM_CHICK,
					    LMPM_CHICK_EXTENDED_ADDR_SPACE);

712 713 714 715 716
		if (ret) {
			IWL_ERR(trans,
				"Could not load the [%d] uCode section\n",
				section_num);
			break;
717
		}
718 719
	}

720
	dma_free_coherent(trans->dev, chunk_sz, v_addr, p_addr);
721 722 723
	return ret;
}

724 725 726 727
static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans,
					   const struct fw_img *image,
					   int cpu,
					   int *first_ucode_section)
728 729
{
	int shift_param;
730 731
	int i, ret = 0, sec_num = 0x1;
	u32 val, last_read_idx = 0;
732 733 734

	if (cpu == 1) {
		shift_param = 0;
735
		*first_ucode_section = 0;
736 737
	} else {
		shift_param = 16;
738
		(*first_ucode_section)++;
739 740
	}

741
	for (i = *first_ucode_section; i < image->num_sec; i++) {
742 743
		last_read_idx = i;

744 745 746 747 748 749
		/*
		 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
		 * CPU1 to CPU2.
		 * PAGING_SEPARATOR_SECTION delimiter - separate between
		 * CPU2 non paged to CPU2 paging sec.
		 */
750
		if (!image->sec[i].data ||
751 752
		    image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
		    image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
753 754 755
			IWL_DEBUG_FW(trans,
				     "Break since Data not valid or Empty section, sec = %d\n",
				     i);
756
			break;
757 758
		}

759 760 761
		ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
		if (ret)
			return ret;
762

763
		/* Notify ucode of loaded section number and status */
764 765 766 767
		val = iwl_read_direct32(trans, FH_UCODE_LOAD_STATUS);
		val = val | (sec_num << shift_param);
		iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, val);

768
		sec_num = (sec_num << 1) | 0x1;
769 770
	}

771 772
	*first_ucode_section = last_read_idx;

773 774
	iwl_enable_interrupts(trans);

775 776 777 778 779 780 781 782 783 784 785 786 787 788 789
	if (trans->cfg->use_tfh) {
		if (cpu == 1)
			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
				       0xFFFF);
		else
			iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
				       0xFFFFFFFF);
	} else {
		if (cpu == 1)
			iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
					   0xFFFF);
		else
			iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
					   0xFFFFFFFF);
	}
790

791 792
	return 0;
}
793

794 795
static int iwl_pcie_load_cpu_sections(struct iwl_trans *trans,
				      const struct fw_img *image,
796 797
				      int cpu,
				      int *first_ucode_section)
798 799
{
	int i, ret = 0;
800
	u32 last_read_idx = 0;
801

802
	if (cpu == 1)
803
		*first_ucode_section = 0;
804
	else
805
		(*first_ucode_section)++;
806

807
	for (i = *first_ucode_section; i < image->num_sec; i++) {
808 809
		last_read_idx = i;

810 811 812 813 814 815
		/*
		 * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
		 * CPU1 to CPU2.
		 * PAGING_SEPARATOR_SECTION delimiter - separate between
		 * CPU2 non paged to CPU2 paging sec.
		 */
816
		if (!image->sec[i].data ||
817 818
		    image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
		    image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
819 820 821
			IWL_DEBUG_FW(trans,
				     "Break since Data not valid or Empty section, sec = %d\n",
				     i);
822
			break;
823 824
		}

825 826 827
		ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
		if (ret)
			return ret;
828 829
	}

830 831
	*first_ucode_section = last_read_idx;

832 833 834
	return 0;
}

835
void iwl_pcie_apply_destination(struct iwl_trans *trans)
836 837 838 839 840 841 842 843 844 845 846 847 848 849
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	const struct iwl_fw_dbg_dest_tlv *dest = trans->dbg_dest_tlv;
	int i;

	if (dest->version)
		IWL_ERR(trans,
			"DBG DEST version is %d - expect issues\n",
			dest->version);

	IWL_INFO(trans, "Applying debug destination %s\n",
		 get_fw_dbg_mode_string(dest->monitor_mode));

	if (dest->monitor_mode == EXTERNAL_MODE)
850
		iwl_pcie_alloc_fw_monitor(trans, dest->size_power);
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
	else
		IWL_WARN(trans, "PCI should have external buffer debug\n");

	for (i = 0; i < trans->dbg_dest_reg_num; i++) {
		u32 addr = le32_to_cpu(dest->reg_ops[i].addr);
		u32 val = le32_to_cpu(dest->reg_ops[i].val);

		switch (dest->reg_ops[i].op) {
		case CSR_ASSIGN:
			iwl_write32(trans, addr, val);
			break;
		case CSR_SETBIT:
			iwl_set_bit(trans, addr, BIT(val));
			break;
		case CSR_CLEARBIT:
			iwl_clear_bit(trans, addr, BIT(val));
			break;
		case PRPH_ASSIGN:
			iwl_write_prph(trans, addr, val);
			break;
		case PRPH_SETBIT:
			iwl_set_bits_prph(trans, addr, BIT(val));
			break;
		case PRPH_CLEARBIT:
			iwl_clear_bits_prph(trans, addr, BIT(val));
			break;
877 878 879 880 881 882 883 884
		case PRPH_BLOCKBIT:
			if (iwl_read_prph(trans, addr) & BIT(val)) {
				IWL_ERR(trans,
					"BIT(%u) in address 0x%x is 1, stopping FW configuration\n",
					val, addr);
				goto monitor;
			}
			break;
885 886 887 888 889 890 891
		default:
			IWL_ERR(trans, "FW debug - unknown OP %d\n",
				dest->reg_ops[i].op);
			break;
		}
	}

892
monitor:
893 894 895
	if (dest->monitor_mode == EXTERNAL_MODE && trans_pcie->fw_mon_size) {
		iwl_write_prph(trans, le32_to_cpu(dest->base_reg),
			       trans_pcie->fw_mon_phys >> dest->base_shift);
896
		if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
897 898 899 900 901 902 903 904 905
			iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
				       (trans_pcie->fw_mon_phys +
					trans_pcie->fw_mon_size - 256) >>
						dest->end_shift);
		else
			iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
				       (trans_pcie->fw_mon_phys +
					trans_pcie->fw_mon_size) >>
						dest->end_shift);
906 907 908
	}
}

909
static int iwl_pcie_load_given_ucode(struct iwl_trans *trans,
910
				const struct fw_img *image)
911
{
912
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
913
	int ret = 0;
914
	int first_ucode_section;
915

916
	IWL_DEBUG_FW(trans, "working with %s CPU\n",
917 918
		     image->is_dual_cpus ? "Dual" : "Single");

919 920 921 922
	/* load to FW the binary non secured sections of CPU1 */
	ret = iwl_pcie_load_cpu_sections(trans, image, 1, &first_ucode_section);
	if (ret)
		return ret;
923 924

	if (image->is_dual_cpus) {
925 926 927 928
		/* set CPU2 header address */
		iwl_write_prph(trans,
			       LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
			       LMPM_SECURE_CPU2_HDR_MEM_SPACE);
929

930
		/* load to FW the binary sections of CPU2 */
931 932
		ret = iwl_pcie_load_cpu_sections(trans, image, 2,
						 &first_ucode_section);
933 934
		if (ret)
			return ret;
935
	}
936

937 938 939
	/* supported for 7000 only for the moment */
	if (iwlwifi_mod_params.fw_monitor &&
	    trans->cfg->device_family == IWL_DEVICE_FAMILY_7000) {
940
		iwl_pcie_alloc_fw_monitor(trans, 0);
941 942 943 944 945 946 947 948

		if (trans_pcie->fw_mon_size) {
			iwl_write_prph(trans, MON_BUFF_BASE_ADDR,
				       trans_pcie->fw_mon_phys >> 4);
			iwl_write_prph(trans, MON_BUFF_END_ADDR,
				       (trans_pcie->fw_mon_phys +
					trans_pcie->fw_mon_size) >> 4);
		}
949 950
	} else if (trans->dbg_dest_tlv) {
		iwl_pcie_apply_destination(trans);
951 952
	}

953 954
	iwl_enable_interrupts(trans);

955
	/* release CPU reset */
956
	iwl_write32(trans, CSR_RESET, 0);
957

958 959
	return 0;
}
960

961 962
static int iwl_pcie_load_given_ucode_8000(struct iwl_trans *trans,
					  const struct fw_img *image)
963 964 965 966 967 968 969
{
	int ret = 0;
	int first_ucode_section;

	IWL_DEBUG_FW(trans, "working with %s CPU\n",
		     image->is_dual_cpus ? "Dual" : "Single");

970 971 972
	if (trans->dbg_dest_tlv)
		iwl_pcie_apply_destination(trans);

973 974 975 976 977 978 979 980 981 982
	IWL_DEBUG_POWER(trans, "Original WFPM value = 0x%08X\n",
			iwl_read_prph(trans, WFPM_GP2));

	/*
	 * Set default value. On resume reading the values that were
	 * zeored can provide debug data on the resume flow.
	 * This is for debugging only and has no functional impact.
	 */
	iwl_write_prph(trans, WFPM_GP2, 0x01010101);

983 984 985 986 987
	/* configure the ucode to be ready to get the secured image */
	/* release CPU reset */
	iwl_write_prph(trans, RELEASE_CPU_RESET, RELEASE_CPU_RESET_BIT);

	/* load to FW the binary Secured sections of CPU1 */
988 989
	ret = iwl_pcie_load_cpu_sections_8000(trans, image, 1,
					      &first_ucode_section);
990 991 992 993
	if (ret)
		return ret;

	/* load to FW the binary sections of CPU2 */
994 995
	return iwl_pcie_load_cpu_sections_8000(trans, image, 2,
					       &first_ucode_section);
996 997
}

998
bool iwl_trans_check_hw_rf_kill(struct iwl_trans *trans)
999
{
1000
	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1001
	bool hw_rfkill = iwl_is_rfkill_set(trans);
1002 1003
	bool prev = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
	bool report;
1004

1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
	if (hw_rfkill) {
		set_bit(STATUS_RFKILL_HW, &trans->status);
		set_bit(STATUS_RFKILL_OPMODE, &trans->status);
	} else {
		clear_bit(STATUS_RFKILL_HW, &trans->status);
		if (trans_pcie->opmode_down)
			clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
	}

	report = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1015

1016 1017
	if (prev != report)
		iwl_trans_pcie_rf_kill(trans, report);
1018 1019 1020 1021

	return hw_rfkill;
}

1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
struct iwl_causes_list {
	u32 cause_num;
	u32 mask_reg;
	u8 addr;
};

static struct iwl_causes_list causes_list[] = {
	{MSIX_FH_INT_CAUSES_D2S_CH0_NUM,	CSR_MSIX_FH_INT_MASK_AD, 0},
	{MSIX_FH_INT_CAUSES_D2S_CH1_NUM,	CSR_MSIX_FH_INT_MASK_AD, 0x1},
	{MSIX_FH_INT_CAUSES_S2D,		CSR_MSIX_FH_INT_MASK_AD, 0x3},
	{MSIX_FH_INT_CAUSES_FH_ERR,		CSR_MSIX_FH_INT_MASK_AD, 0x5},
	{MSIX_HW_INT_CAUSES_REG_ALIVE,		CSR_MSIX_HW_INT_MASK_AD, 0x10},
	{MSIX_HW_INT_CAUSES_REG_WAKEUP,		CSR_MSIX_HW_INT_MASK_AD, 0x11},
	{MSIX_HW_INT_CAUSES_REG_CT_KILL,	CSR_MSIX_HW_INT_MASK_AD, 0x16},
	{MSIX_HW_INT_CAUSES_REG_RF_KILL,	CSR_MSIX_HW_INT_MASK_AD, 0x17},
	{MSIX_HW_INT_CAUSES_REG_PERIODIC,	CSR_MSIX_HW_INT_MASK_AD, 0x18},
	{MSIX_HW_INT_CAUSES_REG_SW_ERR,		CSR_MSIX_HW_INT_MASK_AD, 0x29},
	{MSIX_HW_INT_CAUSES_REG_SCD,		CSR_MSIX_HW_INT_MASK_AD, 0x2A},
	{MSIX_HW_INT_CAUSES_REG_FH_TX,		CSR_MSIX_HW_INT_MASK_AD, 0x2B},
	{MSIX_HW_INT_CAUSES_REG_HW_ERR,		CSR_MSIX_HW_INT_MASK_AD, 0x2D},
	{MSIX_HW_INT_CAUSES_REG_HAP,		CSR_MSIX_HW_INT_MASK_AD, 0x2E},
};

static void iwl_pcie_map_non_rx_causes(struct iwl_trans *trans)
{
	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
	int val = trans_pcie->def_irq | MSIX_NON_AUTO_CLEAR_CAUSE;
	int i;

	/*
	 * Access all non RX causes and map them to the default irq.
	 * In case we are missing at least one interrupt vector,
	 * the first interrupt vector will serve non-RX and FBQ causes.
	 */
	for (i = 0; i < ARRAY_SIZE(causes_list); i++) {
		iwl_write8(trans, CSR_MSIX_IVAR(causes_list[i].addr), val);
		iwl_clear_bit(trans, causes_list[i].mask_reg,
			      causes_list[i].cause_num);
	}
}

static void iwl_pcie_map_rx_causes(struct iwl_trans *trans)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	u32 offset =
		trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 1 : 0;
	u32 val, idx;

	/*
	 * The first RX queue - fallback queue, which is designated for
	 * management frame, command responses etc, is always mapped to the
	 * first interrupt vector. The other RX queues are mapped to
	 * the other (N - 2) interrupt vectors.
	 */
	val = BIT(MSIX_FH_INT_CAUSES_Q(0));
	for (idx = 1; idx < trans->num_rx_queues; idx++) {
		iwl_write8(trans, CSR_MSIX_RX_IVAR(idx),
			   MSIX_FH_INT_CAUSES_Q(idx - offset));
		val |= BIT(MSIX_FH_INT_CAUSES_Q(idx));
	}
	iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~val);

	val = MSIX_FH_INT_CAUSES_Q(0);
	if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX)
		val |= MSIX_NON_AUTO_CLEAR_CAUSE;
	iwl_write8(trans, CSR_MSIX_RX_IVAR(0), val);

	if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS)
		iwl_write8(trans, CSR_MSIX_RX_IVAR(1), val);
}

1093
void iwl_pcie_conf_msix_hw(struct iwl_trans_pcie *trans_pcie)
1094 1095 1096 1097
{
	struct iwl_trans *trans = trans_pcie->trans;

	if (!trans_pcie->msix_enabled) {
1098 1099
		if (trans->cfg->mq_rx_supported &&
		    test_bit(STATUS_DEVICE_ENABLED, &trans->status))
1100 1101 1102 1103
			iwl_write_prph(trans, UREG_CHICK,
				       UREG_CHICK_MSI_ENABLE);
		return;
	}
1104 1105 1106 1107 1108 1109 1110
	/*
	 * The IVAR table needs to be configured again after reset,
	 * but if the device is disabled, we can't write to
	 * prph.
	 */
	if (test_bit(STATUS_DEVICE_ENABLED, &trans->status))
		iwl_write_prph(trans, UREG_CHICK, UREG_CHICK_MSIX_ENABLE);
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121

	/*
	 * Each cause from the causes list above and the RX causes is
	 * represented as a byte in the IVAR table. The first nibble
	 * represents the bound interrupt vector of the cause, the second
	 * represents no auto clear for this cause. This will be set if its
	 * interrupt vector is bound to serve other causes.
	 */
	iwl_pcie_map_rx_causes(trans);

	iwl_pcie_map_non_rx_causes(trans);
1122 1123 1124 1125 1126 1127 1128
}

static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie)
{
	struct iwl_trans *trans = trans_pcie->trans;

	iwl_pcie_conf_msix_hw(trans_pcie);
1129

1130 1131 1132 1133
	if (!trans_pcie->msix_enabled)
		return;

	trans_pcie->fh_init_mask = ~iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD);
1134
	trans_pcie->fh_mask = trans_pcie->fh_init_mask;
1135
	trans_pcie->hw_init_mask = ~iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD);
1136 1137 1138
	trans_pcie->hw_mask = trans_pcie->hw_init_mask;
}

1139
static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power)
1140
{
1141
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1142

1143 1144 1145 1146 1147 1148 1149
	lockdep_assert_held(&trans_pcie->mutex);

	if (trans_pcie->is_down)
		return;

	trans_pcie->is_down = true;

1150
	/* tell the device to stop sending interrupts */
1151 1152
	iwl_disable_interrupts(trans);

1153
	/* device going down, Stop using ICT table */
1154
	iwl_pcie_disable_ict(trans);
1155 1156 1157 1158 1159 1160 1161 1162

	/*
	 * If a HW restart happens during firmware loading,
	 * then the firmware loading might call this function
	 * and later it might be called again due to the
	 * restart. So don't process again if the device is
	 * already dead.
	 */
1163
	if (test_and_clear_bit(STATUS_DEVICE_ENABLED, &trans->status)) {
1164 1165
		IWL_DEBUG_INFO(trans,
			       "DEVICE_ENABLED bit was set and is now cleared\n");
1166
		iwl_pcie_tx_stop(trans);
1167
		iwl_pcie_rx_stop(trans);
1168

1169
		/* Power-down device's busmaster DMA clocks */
1170
		if (!trans->cfg->apmg_not_supported) {
1171 1172 1173 1174
			iwl_write_prph(trans, APMG_CLK_DIS_REG,
				       APMG_CLK_VAL_DMA_CLK_RQT);
			udelay(5);
		}
1175 1176 1177
	}

	/* Make sure (redundant) we've released our request to stay awake */
1178
	iwl_clear_bit(trans, CSR_GP_CNTRL,
1179
		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1180 1181

	/* Stop the device, and put it in low power state */
1182
	iwl_pcie_apm_stop(trans, false);
1183

1184 1185
	/* stop and reset the on-board processor */
	iwl_write32(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
1186
	usleep_range(1000, 2000);
1187

1188 1189 1190 1191 1192 1193 1194 1195 1196
	/*
	 * Upon stop, the IVAR table gets erased, so msi-x won't
	 * work. This causes a bug in RF-KILL flows, since the interrupt
	 * that enables radio won't fire on the correct irq, and the
	 * driver won't be able to handle the interrupt.
	 * Configure the IVAR table again after reset.
	 */
	iwl_pcie_conf_msix_hw(trans_pcie);

1197 1198 1199 1200 1201 1202
	/*
	 * Upon stop, the APM issues an interrupt if HW RF kill is set.
	 * This is a bug in certain verions of the hardware.
	 * Certain devices also keep sending HW RF kill interrupt all
	 * the time, unless the interrupt is ACKed even if the interrupt
	 * should be masked. Re-ACK all the interrupts here.
1203 1204 1205
	 */
	iwl_disable_interrupts(trans);

Don Fry's avatar
Don Fry committed
1206
	/* clear all status bits */
1207 1208 1209
	clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
	clear_bit(STATUS_INT_ENABLED, &trans->status);
	clear_bit(STATUS_TPOWER_PMI, &trans->status);
1210 1211 1212 1213 1214 1215 1216

	/*
	 * Even if we stop the HW, we still want the RF kill
	 * interrupt
	 */
	iwl_enable_rfkill_int(trans);

1217
	/* re-take ownership to prevent other users from stealing the device */
1218
	iwl_pcie_prepare_card_hw(trans);
1219 1220
}

1221
void iwl_pcie_synchronize_irqs(struct iwl_trans *trans)
1222 1223 1224 1225 1226 1227
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	if (trans_pcie->msix_enabled) {
		int i;

1228
		for (i = 0; i < trans_pcie->alloc_vecs; i++)
1229 1230 1231 1232 1233 1234
			synchronize_irq(trans_pcie->msix_entries[i].vector);
	} else {
		synchronize_irq(trans_pcie->pci_dev->irq);
	}
}

1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260
static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
				   const struct fw_img *fw, bool run_in_rfkill)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	bool hw_rfkill;
	int ret;

	/* This may fail if AMT took ownership of the device */
	if (iwl_pcie_prepare_card_hw(trans)) {
		IWL_WARN(trans, "Exit HW not ready\n");
		ret = -EIO;
		goto out;
	}

	iwl_enable_rfkill_int(trans);

	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);

	/*
	 * We enabled the RF-Kill interrupt and the handler may very
	 * well be running. Disable the interrupts to make sure no other
	 * interrupt can be fired.
	 */
	iwl_disable_interrupts(trans);

	/* Make sure it finished running */
1261
	iwl_pcie_synchronize_irqs(trans);
1262 1263 1264 1265

	mutex_lock(&trans_pcie->mutex);

	/* If platform's RF_KILL switch is NOT set to KILL */
1266
	hw_rfkill = iwl_trans_check_hw_rf_kill(trans);
1267 1268 1269 1270 1271 1272 1273 1274 1275
	if (hw_rfkill && !run_in_rfkill) {
		ret = -ERFKILL;
		goto out;
	}

	/* Someone called stop_device, don't try to start_fw */
	if (trans_pcie->is_down) {
		IWL_WARN(trans,
			 "Can't start_fw since the HW hasn't been started\n");
1276
		ret = -EIO;
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307
		goto out;
	}

	/* make sure rfkill handshake bits are cleared */
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR,
		    CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);

	/* clear (again), then enable host interrupts */
	iwl_write32(trans, CSR_INT, 0xFFFFFFFF);

	ret = iwl_pcie_nic_init(trans);
	if (ret) {
		IWL_ERR(trans, "Unable to init nic\n");
		goto out;
	}

	/*
	 * Now, we load the firmware and don't want to be interrupted, even
	 * by the RF-Kill interrupt (hence mask all the interrupt besides the
	 * FH_TX interrupt which is needed to load the firmware). If the
	 * RF-Kill switch is toggled, we will find out after having loaded
	 * the firmware and return the proper value to the caller.
	 */
	iwl_enable_fw_load_int(trans);

	/* really make sure rfkill handshake bits are cleared */
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
	iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);

	/* Load the given image to the HW */
1308
	if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
1309 1310 1311 1312 1313
		ret = iwl_pcie_load_given_ucode_8000(trans, fw);
	else
		ret = iwl_pcie_load_given_ucode(trans, fw);

	/* re-check RF-Kill state since we may have missed the interrupt */
1314
	hw_rfkill = iwl_trans_check_hw_rf_kill(trans);
1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328
	if (hw_rfkill && !run_in_rfkill)
		ret = -ERFKILL;

out:
	mutex_unlock(&trans_pcie->mutex);
	return ret;
}

static void iwl_trans_pcie_fw_alive(struct iwl_trans *trans, u32 scd_addr)
{
	iwl_pcie_reset_ict(trans);
	iwl_pcie_tx_start(trans, scd_addr);
}

1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357
void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans,
				       bool was_in_rfkill)
{
	bool hw_rfkill;

	/*
	 * Check again since the RF kill state may have changed while
	 * all the interrupts were disabled, in this case we couldn't
	 * receive the RF kill interrupt and update the state in the
	 * op_mode.
	 * Don't call the op_mode if the rkfill state hasn't changed.
	 * This allows the op_mode to call stop_device from the rfkill
	 * notification without endless recursion. Under very rare
	 * circumstances, we might have a small recursion if the rfkill
	 * state changed exactly now while we were called from stop_device.
	 * This is very unlikely but can happen and is supported.
	 */
	hw_rfkill = iwl_is_rfkill_set(trans);
	if (hw_rfkill) {
		set_bit(STATUS_RFKILL_HW, &trans->status);
		set_bit(STATUS_RFKILL_OPMODE, &trans->status);
	} else {
		clear_bit(STATUS_RFKILL_HW, &trans->status);
		clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
	}
	if (hw_rfkill != was_in_rfkill)
		iwl_trans_pcie_rf_kill(trans, hw_rfkill);
}

1358 1359 1360
static void iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1361
	bool was_in_rfkill;
1362 1363

	mutex_lock(&trans_pcie->mutex);
1364 1365
	trans_pcie->opmode_down = true;
	was_in_rfkill = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1366
	_iwl_trans_pcie_stop_device(trans, low_power);
1367
	iwl_trans_pcie_handle_stop_rfkill(trans, was_in_rfkill);
1368 1369 1370
	mutex_unlock(&trans_pcie->mutex);
}

1371 1372
void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state)
{
1373 1374 1375 1376 1377
	struct iwl_trans_pcie __maybe_unused *trans_pcie =
		IWL_TRANS_GET_PCIE_TRANS(trans);

	lockdep_assert_held(&trans_pcie->mutex);

1378 1379
	IWL_WARN(trans, "reporting RF_KILL (radio %s)\n",
		 state ? "disabled" : "enabled");
1380 1381 1382 1383 1384 1385
	if (iwl_op_mode_hw_rf_kill(trans->op_mode, state)) {
		if (trans->cfg->gen2)
			_iwl_trans_pcie_gen2_stop_device(trans, true);
		else
			_iwl_trans_pcie_stop_device(trans, true);
	}
1386 1387
}

1388 1389
static void iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test,
				      bool reset)
1390
{
1391
	if (!reset) {
1392 1393 1394 1395 1396
		/* Enable persistence mode to avoid reset */
		iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
			    CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
	}

1397
	iwl_disable_interrupts(trans);
1398 1399 1400 1401 1402 1403 1404 1405

	/*
	 * in testing mode, the host stays awake and the
	 * hardware won't be reset (not even partially)
	 */
	if (test)
		return;

1406 1407
	iwl_pcie_disable_ict(trans);

1408
	iwl_pcie_synchronize_irqs(trans);
1409

1410 1411
	iwl_clear_bit(trans, CSR_GP_CNTRL,
		      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1412 1413 1414
	iwl_clear_bit(trans, CSR_GP_CNTRL,
		      CSR_GP_CNTRL_REG_FLAG_INIT_DONE);

1415 1416
	iwl_pcie_enable_rx_wake(trans, false);

1417
	if (reset) {
1418 1419 1420 1421 1422 1423 1424
		/*
		 * reset TX queues -- some of their registers reset during S3
		 * so if we don't reset everything here the D3 image would try
		 * to execute some invalid memory upon resume
		 */
		iwl_trans_pcie_tx_reset(trans);
	}
1425 1426 1427 1428 1429

	iwl_pcie_set_pwr(trans, true);
}

static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
1430
				    enum iwl_d3_status *status,
1431
				    bool test,  bool reset)
1432
{
1433
	struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1434 1435 1436
	u32 val;
	int ret;

1437 1438 1439 1440 1441 1442
	if (test) {
		iwl_enable_interrupts(trans);
		*status = IWL_D3_STATUS_ALIVE;
		return 0;
	}

1443 1444
	iwl_pcie_enable_rx_wake(trans, true);

1445
	/*
1446 1447 1448 1449 1450
	 * Reconfigure IVAR table in case of MSIX or reset ict table in
	 * MSI mode since HW reset erased it.
	 * Also enables interrupts - none will happen as
	 * the device doesn't know we're waking it up, only when
	 * the opmode actually tells it after this call.
1451
	 */
1452 1453 1454
	iwl_pcie_conf_msix_hw(trans_pcie);
	if (!trans_pcie->msix_enabled)
		iwl_pcie_reset_ict(trans);
1455
	iwl_enable_interrupts(trans);
1456 1457 1458 1459

	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
	iwl_set_bit(trans, CSR_GP_CNTRL, CSR_GP_CNTRL_REG_FLAG_INIT_DONE);

1460
	if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
1461 1462
		udelay(2);

1463 1464 1465 1466
	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
			   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
			   25000);
1467
	if (ret < 0) {
1468 1469 1470 1471
		IWL_ERR(trans, "Failed to resume the device (mac ready)\n");
		return ret;
	}

1472 1473
	iwl_pcie_set_pwr(trans, false);

1474
	if (!reset) {
1475 1476 1477 1478
		iwl_clear_bit(trans, CSR_GP_CNTRL,
			      CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
	} else {
		iwl_trans_pcie_tx_reset(trans);
1479

1480 1481 1482 1483 1484 1485
		ret = iwl_pcie_rx_init(trans);
		if (ret) {
			IWL_ERR(trans,
				"Failed to resume the device (RX reset)\n");
			return ret;
		}
1486 1487
	}

1488 1489 1490
	IWL_DEBUG_POWER(trans, "WFPM value upon resume = 0x%08X\n",
			iwl_read_prph(trans, WFPM_GP2));

1491 1492 1493 1494 1495 1496
	val = iwl_read32(trans, CSR_RESET);
	if (val & CSR_RESET_REG_FLAG_NEVO_RESET)
		*status = IWL_D3_STATUS_RESET;
	else
		*status = IWL_D3_STATUS_ALIVE;

1497
	return 0;
1498 1499
}

1500 1501 1502 1503
static void iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
					struct iwl_trans *trans)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1504
	int max_irqs, num_irqs, i, ret, nr_online_cpus;
1505 1506
	u16 pci_cmd;

1507 1508 1509
	if (!trans->cfg->mq_rx_supported)
		goto enable_msi;

1510 1511
	nr_online_cpus = num_online_cpus();
	max_irqs = min_t(u32, nr_online_cpus + 2, IWL_MAX_RX_HW_QUEUES);
1512 1513
	for (i = 0; i < max_irqs; i++)
		trans_pcie->msix_entries[i].entry = i;
1514

1515 1516 1517 1518
	num_irqs = pci_enable_msix_range(pdev, trans_pcie->msix_entries,
					 MSIX_MIN_INTERRUPT_VECTORS,
					 max_irqs);
	if (num_irqs < 0) {
1519
		IWL_DEBUG_INFO(trans,
1520 1521 1522 1523 1524
			       "Failed to enable msi-x mode (ret %d). Moving to msi mode.\n",
			       num_irqs);
		goto enable_msi;
	}
	trans_pcie->def_irq = (num_irqs == max_irqs) ? num_irqs - 1 : 0;
1525

1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
	IWL_DEBUG_INFO(trans,
		       "MSI-X enabled. %d interrupt vectors were allocated\n",
		       num_irqs);

	/*
	 * In case the OS provides fewer interrupts than requested, different
	 * causes will share the same interrupt vector as follows:
	 * One interrupt less: non rx causes shared with FBQ.
	 * Two interrupts less: non rx causes shared with FBQ and RSS.
	 * More than two interrupts: we will use fewer RSS queues.
	 */
1537
	if (num_irqs <= nr_online_cpus) {
1538 1539 1540
		trans_pcie->trans->num_rx_queues = num_irqs + 1;
		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX |
			IWL_SHARED_IRQ_FIRST_RSS;
1541
	} else if (num_irqs == nr_online_cpus + 1) {
1542 1543 1544 1545
		trans_pcie->trans->num_rx_queues = num_irqs;
		trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX;
	} else {
		trans_pcie->trans->num_rx_queues = num_irqs - 1;
1546 1547
	}

1548 1549 1550 1551 1552 1553 1554 1555
	trans_pcie->alloc_vecs = num_irqs;
	trans_pcie->msix_enabled = true;
	return;

enable_msi:
	ret = pci_enable_msi(pdev);
	if (ret) {
		dev_err(&pdev->dev, "pci_enable_msi failed - %d\n", ret);
1556 1557 1558 1559 1560 1561 1562 1563 1564
		/* enable rfkill interrupt: hw bug w/a */
		pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
		if (pci_cmd & PCI_COMMAND_INTX_DISABLE) {
			pci_cmd &= ~PCI_COMMAND_INTX_DISABLE;
			pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
		}
	}
}

1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588
static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans)
{
	int iter_rx_q, i, ret, cpu, offset;
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	i = trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 0 : 1;
	iter_rx_q = trans_pcie->trans->num_rx_queues - 1 + i;
	offset = 1 + i;
	for (; i < iter_rx_q ; i++) {
		/*
		 * Get the cpu prior to the place to search
		 * (i.e. return will be > i - 1).
		 */
		cpu = cpumask_next(i - offset, cpu_online_mask);
		cpumask_set_cpu(cpu, &trans_pcie->affinity_mask[i]);
		ret = irq_set_affinity_hint(trans_pcie->msix_entries[i].vector,
					    &trans_pcie->affinity_mask[i]);
		if (ret)
			IWL_ERR(trans_pcie->trans,
				"Failed to set affinity mask for IRQ %d\n",
				i);
	}
}

1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
static const char *queue_name(struct device *dev,
			      struct iwl_trans_pcie *trans_p, int i)
{
	if (trans_p->shared_vec_mask) {
		int vec = trans_p->shared_vec_mask &
			  IWL_SHARED_IRQ_FIRST_RSS ? 1 : 0;

		if (i == 0)
			return DRV_NAME ": shared IRQ";

		return devm_kasprintf(dev, GFP_KERNEL,
				      DRV_NAME ": queue %d", i + vec);
	}
	if (i == 0)
		return DRV_NAME ": default queue";

	if (i == trans_p->alloc_vecs - 1)
		return DRV_NAME ": exception";

	return devm_kasprintf(dev, GFP_KERNEL,
			      DRV_NAME  ": queue %d", i);
}

1612 1613 1614
static int iwl_pcie_init_msix_handler(struct pci_dev *pdev,
				      struct iwl_trans_pcie *trans_pcie)
{
1615
	int i;
1616

1617
	for (i = 0; i < trans_pcie->alloc_vecs; i++) {
1618
		int ret;
1619
		struct msix_entry *msix_entry;
1620 1621 1622 1623
		const char *qname = queue_name(&pdev->dev, trans_pcie, i);

		if (!qname)
			return -ENOMEM;
1624 1625 1626 1627 1628 1629 1630 1631 1632

		msix_entry = &trans_pcie->msix_entries[i];
		ret = devm_request_threaded_irq(&pdev->dev,
						msix_entry->vector,
						iwl_pcie_msix_isr,
						(i == trans_pcie->def_irq) ?
						iwl_pcie_irq_msix_handler :
						iwl_pcie_irq_rx_msix_handler,
						IRQF_SHARED,
1633
						qname,
1634
						msix_entry);
1635 1636 1637
		if (ret) {
			IWL_ERR(trans_pcie->trans,
				"Error allocating IRQ %d\n", i);
1638

1639 1640 1641
			return ret;
		}
	}
1642
	iwl_pcie_irq_set_affinity(trans_pcie->trans);
1643 1644 1645 1646

	return 0;
}

1647
static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power)
1648
{
1649
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1650
	int err;
1651

1652 1653
	lockdep_assert_held(&trans_pcie->mutex);

1654
	err = iwl_pcie_prepare_card_hw(trans);
1655
	if (err) {
1656
		IWL_ERR(trans, "Error while preparing HW: %d\n", err);
1657
		return err;
1658
	}
1659

1660
	/* Reset the entire device */
1661
	iwl_write32(trans, CSR_RESET, CSR_RESET_REG_FLAG_SW_RESET);
1662
	usleep_range(1000, 2000);
1663

1664 1665 1666
	err = iwl_pcie_apm_init(trans);
	if (err)
		return err;
1667

1668
	iwl_pcie_init_msix(trans_pcie);
1669

1670 1671 1672
	/* From now on, the op_mode will be kept updated about RF kill state */
	iwl_enable_rfkill_int(trans);

1673 1674
	trans_pcie->opmode_down = false;

1675 1676 1677
	/* Set is_down to false here so that...*/
	trans_pcie->is_down = false;

1678 1679
	/* ...rfkill can call stop_device and set it false if needed */
	iwl_trans_check_hw_rf_kill(trans);
1680

1681 1682 1683 1684
	/* Make sure we sync here, because we'll need full access later */
	if (low_power)
		pm_runtime_resume(trans->dev);

1685
	return 0;
1686 1687
}

1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
static int iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	int ret;

	mutex_lock(&trans_pcie->mutex);
	ret = _iwl_trans_pcie_start_hw(trans, low_power);
	mutex_unlock(&trans_pcie->mutex);

	return ret;
}

1700
static void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans)
1701
{
1702
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1703

1704 1705
	mutex_lock(&trans_pcie->mutex);

1706
	/* disable interrupts - don't enable HW RF kill interrupt */
1707 1708
	iwl_disable_interrupts(trans);

1709
	iwl_pcie_apm_stop(trans, true);
1710

1711
	iwl_disable_interrupts(trans);
1712

1713
	iwl_pcie_disable_ict(trans);
1714

1715
	mutex_unlock(&trans_pcie->mutex);
1716

1717
	iwl_pcie_synchronize_irqs(trans);
1718 1719
}

1720 1721
static void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val)
{
1722
	writeb(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1723 1724 1725 1726
}

static void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val)
{
1727
	writel(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1728 1729 1730 1731
}

static u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs)
{
1732
	return readl(IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1733 1734
}

1735 1736
static u32 iwl_trans_pcie_read_prph(struct iwl_trans *trans, u32 reg)
{
Amnon Paz's avatar
Amnon Paz committed
1737 1738
	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_RADDR,
			       ((reg & 0x000FFFFF) | (3 << 24)));
1739 1740 1741 1742 1743 1744 1745
	return iwl_trans_pcie_read32(trans, HBUS_TARG_PRPH_RDAT);
}

static void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr,
				      u32 val)
{
	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WADDR,
Amnon Paz's avatar
Amnon Paz committed
1746
			       ((addr & 0x000FFFFF) | (3 << 24)));
1747 1748 1749
	iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WDAT, val);
}

1750
static void iwl_trans_pcie_configure(struct iwl_trans *trans,
1751
				     const struct iwl_trans_config *trans_cfg)
1752 1753 1754 1755
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	trans_pcie->cmd_queue = trans_cfg->cmd_queue;
1756
	trans_pcie->cmd_fifo = trans_cfg->cmd_fifo;
1757
	trans_pcie->cmd_q_wdg_timeout = trans_cfg->cmd_q_wdg_timeout;
1758 1759 1760 1761 1762 1763 1764
	if (WARN_ON(trans_cfg->n_no_reclaim_cmds > MAX_NO_RECLAIM_CMDS))
		trans_pcie->n_no_reclaim_cmds = 0;
	else
		trans_pcie->n_no_reclaim_cmds = trans_cfg->n_no_reclaim_cmds;
	if (trans_pcie->n_no_reclaim_cmds)
		memcpy(trans_pcie->no_reclaim_cmds, trans_cfg->no_reclaim_cmds,
		       trans_pcie->n_no_reclaim_cmds * sizeof(u8));
1765

1766 1767 1768
	trans_pcie->rx_buf_size = trans_cfg->rx_buf_size;
	trans_pcie->rx_page_order =
		iwl_trans_get_rb_size_order(trans_pcie->rx_buf_size);
1769

1770
	trans_pcie->bc_table_dword = trans_cfg->bc_table_dword;
1771
	trans_pcie->scd_set_active = trans_cfg->scd_set_active;
1772
	trans_pcie->sw_csum_tx = trans_cfg->sw_csum_tx;
1773

1774 1775 1776
	trans_pcie->page_offs = trans_cfg->cb_data_offs;
	trans_pcie->dev_cmd_offs = trans_cfg->cb_data_offs + sizeof(void *);

1777 1778 1779
	trans->command_groups = trans_cfg->command_groups;
	trans->command_groups_size = trans_cfg->command_groups_size;

1780 1781 1782 1783 1784
	/* Initialize NAPI here - it should be before registering to mac80211
	 * in the opmode but after the HW struct is allocated.
	 * As this function may be called again in some corner cases don't
	 * do anything if NAPI was already initialized.
	 */
1785
	if (trans_pcie->napi_dev.reg_state != NETREG_DUMMY)
1786
		init_dummy_netdev(&trans_pcie->napi_dev);
1787 1788
}

1789
void iwl_trans_pcie_free(struct iwl_trans *trans)
1790
{
1791
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1792
	int i;
1793

1794
	iwl_pcie_synchronize_irqs(trans);
1795

1796 1797 1798 1799
	if (trans->cfg->gen2)
		iwl_pcie_gen2_tx_free(trans);
	else
		iwl_pcie_tx_free(trans);
1800
	iwl_pcie_rx_free(trans);
1801

1802
	if (trans_pcie->msix_enabled) {
1803 1804 1805 1806 1807
		for (i = 0; i < trans_pcie->alloc_vecs; i++) {
			irq_set_affinity_hint(
				trans_pcie->msix_entries[i].vector,
				NULL);
		}
1808 1809 1810 1811 1812

		trans_pcie->msix_enabled = false;
	} else {
		iwl_pcie_free_ict(trans);
	}
1813

1814 1815
	iwl_pcie_free_fw_monitor(trans);

1816 1817 1818 1819 1820 1821 1822 1823 1824
	for_each_possible_cpu(i) {
		struct iwl_tso_hdr_page *p =
			per_cpu_ptr(trans_pcie->tso_hdr_page, i);

		if (p->page)
			__free_page(p->page);
	}

	free_percpu(trans_pcie->tso_hdr_page);
1825
	mutex_destroy(&trans_pcie->mutex);
1826
	iwl_trans_free(trans);
1827 1828
}

1829 1830 1831
static void iwl_trans_pcie_set_pmi(struct iwl_trans *trans, bool state)
{
	if (state)
1832
		set_bit(STATUS_TPOWER_PMI, &trans->status);
1833
	else
1834
		clear_bit(STATUS_TPOWER_PMI, &trans->status);
1835 1836
}

1837 1838
static bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans,
					   unsigned long *flags)
1839 1840
{
	int ret;
1841 1842 1843
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	spin_lock_irqsave(&trans_pcie->reg_lock, *flags);
1844

1845
	if (trans_pcie->cmd_hold_nic_awake)
1846 1847
		goto out;

1848
	/* this bit wakes up the NIC */
1849 1850
	__iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
				 CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1851
	if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
1852
		udelay(2);
1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878

	/*
	 * These bits say the device is running, and should keep running for
	 * at least a short while (at least as long as MAC_ACCESS_REQ stays 1),
	 * but they do not indicate that embedded SRAM is restored yet;
	 * 3945 and 4965 have volatile SRAM, and must save/restore contents
	 * to/from host DRAM when sleeping/waking for power-saving.
	 * Each direction takes approximately 1/4 millisecond; with this
	 * overhead, it's a good idea to grab and hold MAC_ACCESS_REQUEST if a
	 * series of register accesses are expected (e.g. reading Event Log),
	 * to keep device from sleeping.
	 *
	 * CSR_UCODE_DRV_GP1 register bit MAC_SLEEP == 0 indicates that
	 * SRAM is okay/restored.  We don't check that here because this call
	 * is just for hardware register access; but GP1 MAC_SLEEP check is a
	 * good idea before accessing 3945/4965 SRAM (e.g. reading Event Log).
	 *
	 * 5000 series and later (including 1000 series) have non-volatile SRAM,
	 * and do not save/restore SRAM when power cycling.
	 */
	ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
			   CSR_GP_CNTRL_REG_VAL_MAC_ACCESS_EN,
			   (CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
			    CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP), 15000);
	if (unlikely(ret < 0)) {
		iwl_write32(trans, CSR_RESET, CSR_RESET_REG_FLAG_FORCE_NMI);
1879 1880 1881 1882 1883
		WARN_ONCE(1,
			  "Timeout waiting for hardware access (CSR_GP_CNTRL 0x%08x)\n",
			  iwl_read32(trans, CSR_GP_CNTRL));
		spin_unlock_irqrestore(&trans_pcie->reg_lock, *flags);
		return false;
1884 1885
	}

1886
out:
1887 1888 1889 1890
	/*
	 * Fool sparse by faking we release the lock - sparse will
	 * track nic_access anyway.
	 */
1891
	__release(&trans_pcie->reg_lock);
1892 1893 1894
	return true;
}

1895 1896
static void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans,
					      unsigned long *flags)
1897
{
1898
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1899

1900
	lockdep_assert_held(&trans_pcie->reg_lock);
1901 1902 1903 1904 1905

	/*
	 * Fool sparse by faking we acquiring the lock - sparse will
	 * track nic_access anyway.
	 */
1906
	__acquire(&trans_pcie->reg_lock);
1907

1908
	if (trans_pcie->cmd_hold_nic_awake)
1909 1910
		goto out;

1911 1912
	__iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
				   CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1913 1914 1915 1916 1917 1918 1919
	/*
	 * Above we read the CSR_GP_CNTRL register, which will flush
	 * any previous writes, but we need the write that clears the
	 * MAC_ACCESS_REQ bit to be performed before any other writes
	 * scheduled on different CPUs (after we drop reg_lock).
	 */
	mmiowb();
1920
out:
1921
	spin_unlock_irqrestore(&trans_pcie->reg_lock, *flags);
1922 1923
}

1924 1925 1926 1927 1928 1929 1930
static int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
				   void *buf, int dwords)
{
	unsigned long flags;
	int offs, ret = 0;
	u32 *vals = buf;

1931
	if (iwl_trans_grab_nic_access(trans, &flags)) {
1932 1933 1934
		iwl_write32(trans, HBUS_TARG_MEM_RADDR, addr);
		for (offs = 0; offs < dwords; offs++)
			vals[offs] = iwl_read32(trans, HBUS_TARG_MEM_RDAT);
1935
		iwl_trans_release_nic_access(trans, &flags);
1936 1937 1938 1939 1940 1941 1942
	} else {
		ret = -EBUSY;
	}
	return ret;
}

static int iwl_trans_pcie_write_mem(struct iwl_trans *trans, u32 addr,
1943
				    const void *buf, int dwords)
1944 1945 1946
{
	unsigned long flags;
	int offs, ret = 0;
1947
	const u32 *vals = buf;
1948

1949
	if (iwl_trans_grab_nic_access(trans, &flags)) {
1950 1951
		iwl_write32(trans, HBUS_TARG_MEM_WADDR, addr);
		for (offs = 0; offs < dwords; offs++)
1952 1953
			iwl_write32(trans, HBUS_TARG_MEM_WDAT,
				    vals ? vals[offs] : 0);
1954
		iwl_trans_release_nic_access(trans, &flags);
1955 1956 1957 1958 1959
	} else {
		ret = -EBUSY;
	}
	return ret;
}
1960

1961 1962 1963 1964 1965 1966 1967 1968
static void iwl_trans_pcie_freeze_txq_timer(struct iwl_trans *trans,
					    unsigned long txqs,
					    bool freeze)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	int queue;

	for_each_set_bit(queue, &txqs, BITS_PER_LONG) {
1969
		struct iwl_txq *txq = trans_pcie->txq[queue];
1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983
		unsigned long now;

		spin_lock_bh(&txq->lock);

		now = jiffies;

		if (txq->frozen == freeze)
			goto next_queue;

		IWL_DEBUG_TX_QUEUES(trans, "%s TXQ %d\n",
				    freeze ? "Freezing" : "Waking", queue);

		txq->frozen = freeze;

1984
		if (txq->read_ptr == txq->write_ptr)
1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014
			goto next_queue;

		if (freeze) {
			if (unlikely(time_after(now,
						txq->stuck_timer.expires))) {
				/*
				 * The timer should have fired, maybe it is
				 * spinning right now on the lock.
				 */
				goto next_queue;
			}
			/* remember how long until the timer fires */
			txq->frozen_expiry_remainder =
				txq->stuck_timer.expires - now;
			del_timer(&txq->stuck_timer);
			goto next_queue;
		}

		/*
		 * Wake a non-empty queue -> arm timer with the
		 * remainder before it froze
		 */
		mod_timer(&txq->stuck_timer,
			  now + txq->frozen_expiry_remainder);

next_queue:
		spin_unlock_bh(&txq->lock);
	}
}

2015 2016 2017 2018 2019 2020
static void iwl_trans_pcie_block_txq_ptrs(struct iwl_trans *trans, bool block)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	int i;

	for (i = 0; i < trans->cfg->base_params->num_of_queues; i++) {
2021
		struct iwl_txq *txq = trans_pcie->txq[i];
2022 2023 2024 2025 2026 2027 2028 2029 2030 2031

		if (i == trans_pcie->cmd_queue)
			continue;

		spin_lock_bh(&txq->lock);

		if (!block && !(WARN_ON_ONCE(!txq->block))) {
			txq->block--;
			if (!txq->block) {
				iwl_write32(trans, HBUS_TARG_WRPTR,
2032
					    txq->write_ptr | (i << 8));
2033 2034 2035 2036 2037 2038 2039 2040 2041
			}
		} else if (block) {
			txq->block++;
		}

		spin_unlock_bh(&txq->lock);
	}
}

2042 2043
#define IWL_FLUSH_WAIT_MS	2000

2044 2045
void iwl_trans_pcie_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq)
{
2046 2047 2048 2049
	u32 txq_id = txq->id;
	u32 status;
	bool active;
	u8 fifo;
2050

2051 2052 2053
	if (trans->cfg->use_tfh) {
		IWL_ERR(trans, "Queue %d is stuck %d %d\n", txq_id,
			txq->read_ptr, txq->write_ptr);
2054 2055
		/* TODO: access new SCD registers and dump them */
		return;
2056
	}
2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071

	status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id));
	fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7;
	active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE));

	IWL_ERR(trans,
		"Queue %d is %sactive on fifo %d and stuck for %u ms. SW [%d, %d] HW [%d, %d] FH TRB=0x0%x\n",
		txq_id, active ? "" : "in", fifo,
		jiffies_to_msecs(txq->wd_timeout),
		txq->read_ptr, txq->write_ptr,
		iwl_read_prph(trans, SCD_QUEUE_RDPTR(txq_id)) &
			(TFD_QUEUE_SIZE_MAX - 1),
		iwl_read_prph(trans, SCD_QUEUE_WRPTR(txq_id)) &
			(TFD_QUEUE_SIZE_MAX - 1),
		iwl_read_direct32(trans, FH_TX_TRB_REG(fifo)));
2072 2073
}

2074
static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, int txq_idx)
2075
{
2076
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2077
	struct iwl_txq *txq;
2078
	unsigned long now = jiffies;
2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115
	u8 wr_ptr;

	if (!test_bit(txq_idx, trans_pcie->queue_used))
		return -EINVAL;

	IWL_DEBUG_TX_QUEUES(trans, "Emptying queue %d...\n", txq_idx);
	txq = trans_pcie->txq[txq_idx];
	wr_ptr = ACCESS_ONCE(txq->write_ptr);

	while (txq->read_ptr != ACCESS_ONCE(txq->write_ptr) &&
	       !time_after(jiffies,
			   now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS))) {
		u8 write_ptr = ACCESS_ONCE(txq->write_ptr);

		if (WARN_ONCE(wr_ptr != write_ptr,
			      "WR pointer moved while flushing %d -> %d\n",
			      wr_ptr, write_ptr))
			return -ETIMEDOUT;
		usleep_range(1000, 2000);
	}

	if (txq->read_ptr != txq->write_ptr) {
		IWL_ERR(trans,
			"fail to flush all tx fifo queues Q %d\n", txq_idx);
		iwl_trans_pcie_log_scd_error(trans, txq);
		return -ETIMEDOUT;
	}

	IWL_DEBUG_TX_QUEUES(trans, "Queue %d is now empty.\n", txq_idx);

	return 0;
}

static int iwl_trans_pcie_wait_txqs_empty(struct iwl_trans *trans, u32 txq_bm)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	int cnt;
2116 2117 2118
	int ret = 0;

	/* waiting for all the tx frames complete might take a while */
2119
	for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) {
2120

2121
		if (cnt == trans_pcie->cmd_queue)
2122
			continue;
2123 2124 2125 2126
		if (!test_bit(cnt, trans_pcie->queue_used))
			continue;
		if (!(BIT(cnt) & txq_bm))
			continue;
2127

2128 2129
		ret = iwl_trans_pcie_wait_txq_empty(trans, cnt);
		if (ret)
2130 2131
			break;
	}
2132

2133 2134 2135
	return ret;
}

2136 2137 2138
static void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg,
					 u32 mask, u32 value)
{
2139
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2140 2141
	unsigned long flags;

2142
	spin_lock_irqsave(&trans_pcie->reg_lock, flags);
2143
	__iwl_trans_pcie_set_bits_mask(trans, reg, mask, value);
2144
	spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
2145 2146
}

2147
static void iwl_trans_pcie_ref(struct iwl_trans *trans)
2148 2149 2150 2151 2152 2153
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	if (iwlwifi_mod_params.d0i3_disable)
		return;

2154
	pm_runtime_get(&trans_pcie->pci_dev->dev);
2155 2156 2157 2158 2159

#ifdef CONFIG_PM
	IWL_DEBUG_RPM(trans, "runtime usage count: %d\n",
		      atomic_read(&trans_pcie->pci_dev->dev.power.usage_count));
#endif /* CONFIG_PM */
2160 2161
}

2162
static void iwl_trans_pcie_unref(struct iwl_trans *trans)
2163 2164 2165 2166 2167 2168
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	if (iwlwifi_mod_params.d0i3_disable)
		return;

2169 2170 2171
	pm_runtime_mark_last_busy(&trans_pcie->pci_dev->dev);
	pm_runtime_put_autosuspend(&trans_pcie->pci_dev->dev);

2172 2173 2174 2175
#ifdef CONFIG_PM
	IWL_DEBUG_RPM(trans, "runtime usage count: %d\n",
		      atomic_read(&trans_pcie->pci_dev->dev.power.usage_count));
#endif /* CONFIG_PM */
2176 2177
}

2178 2179
static const char *get_csr_string(int cmd)
{
2180
#define IWL_CMD(x) case x: return #x
2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203
	switch (cmd) {
	IWL_CMD(CSR_HW_IF_CONFIG_REG);
	IWL_CMD(CSR_INT_COALESCING);
	IWL_CMD(CSR_INT);
	IWL_CMD(CSR_INT_MASK);
	IWL_CMD(CSR_FH_INT_STATUS);
	IWL_CMD(CSR_GPIO_IN);
	IWL_CMD(CSR_RESET);
	IWL_CMD(CSR_GP_CNTRL);
	IWL_CMD(CSR_HW_REV);
	IWL_CMD(CSR_EEPROM_REG);
	IWL_CMD(CSR_EEPROM_GP);
	IWL_CMD(CSR_OTP_GP_REG);
	IWL_CMD(CSR_GIO_REG);
	IWL_CMD(CSR_GP_UCODE_REG);
	IWL_CMD(CSR_GP_DRIVER_REG);
	IWL_CMD(CSR_UCODE_DRV_GP1);
	IWL_CMD(CSR_UCODE_DRV_GP2);
	IWL_CMD(CSR_LED_REG);
	IWL_CMD(CSR_DRAM_INT_TBL_REG);
	IWL_CMD(CSR_GIO_CHICKEN_BITS);
	IWL_CMD(CSR_ANA_PLL_CFG);
	IWL_CMD(CSR_HW_REV_WA_REG);
2204
	IWL_CMD(CSR_MONITOR_STATUS_REG);
2205 2206 2207 2208
	IWL_CMD(CSR_DBG_HPET_MEM_REG);
	default:
		return "UNKNOWN";
	}
2209
#undef IWL_CMD
2210 2211
}

2212
void iwl_pcie_dump_csr(struct iwl_trans *trans)
2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236
{
	int i;
	static const u32 csr_tbl[] = {
		CSR_HW_IF_CONFIG_REG,
		CSR_INT_COALESCING,
		CSR_INT,
		CSR_INT_MASK,
		CSR_FH_INT_STATUS,
		CSR_GPIO_IN,
		CSR_RESET,
		CSR_GP_CNTRL,
		CSR_HW_REV,
		CSR_EEPROM_REG,
		CSR_EEPROM_GP,
		CSR_OTP_GP_REG,
		CSR_GIO_REG,
		CSR_GP_UCODE_REG,
		CSR_GP_DRIVER_REG,
		CSR_UCODE_DRV_GP1,
		CSR_UCODE_DRV_GP2,
		CSR_LED_REG,
		CSR_DRAM_INT_TBL_REG,
		CSR_GIO_CHICKEN_BITS,
		CSR_ANA_PLL_CFG,
2237
		CSR_MONITOR_STATUS_REG,
2238 2239 2240 2241 2242 2243 2244 2245 2246
		CSR_HW_REV_WA_REG,
		CSR_DBG_HPET_MEM_REG
	};
	IWL_ERR(trans, "CSR values:\n");
	IWL_ERR(trans, "(2nd byte of CSR_INT_COALESCING is "
		"CSR_INT_PERIODIC_REG)\n");
	for (i = 0; i <  ARRAY_SIZE(csr_tbl); i++) {
		IWL_ERR(trans, "  %25s: 0X%08x\n",
			get_csr_string(csr_tbl[i]),
2247
			iwl_read32(trans, csr_tbl[i]));
2248 2249 2250
	}
}

2251 2252 2253
#ifdef CONFIG_IWLWIFI_DEBUGFS
/* create and remove of files */
#define DEBUGFS_ADD_FILE(name, parent, mode) do {			\
2254
	if (!debugfs_create_file(#name, mode, parent, trans,		\
2255
				 &iwl_dbgfs_##name##_ops))		\
2256
		goto err;						\
2257 2258 2259 2260 2261 2262
} while (0)

/* file operation */
#define DEBUGFS_READ_FILE_OPS(name)					\
static const struct file_operations iwl_dbgfs_##name##_ops = {		\
	.read = iwl_dbgfs_##name##_read,				\
2263
	.open = simple_open,						\
2264 2265 2266
	.llseek = generic_file_llseek,					\
};

2267 2268 2269
#define DEBUGFS_WRITE_FILE_OPS(name)                                    \
static const struct file_operations iwl_dbgfs_##name##_ops = {          \
	.write = iwl_dbgfs_##name##_write,                              \
2270
	.open = simple_open,						\
2271 2272 2273
	.llseek = generic_file_llseek,					\
};

2274 2275 2276 2277
#define DEBUGFS_READ_WRITE_FILE_OPS(name)				\
static const struct file_operations iwl_dbgfs_##name##_ops = {		\
	.write = iwl_dbgfs_##name##_write,				\
	.read = iwl_dbgfs_##name##_read,				\
2278
	.open = simple_open,						\
2279 2280 2281 2282
	.llseek = generic_file_llseek,					\
};

static ssize_t iwl_dbgfs_tx_queue_read(struct file *file,
2283 2284
				       char __user *user_buf,
				       size_t count, loff_t *ppos)
2285
{
2286
	struct iwl_trans *trans = file->private_data;
2287
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2288
	struct iwl_txq *txq;
2289 2290 2291 2292
	char *buf;
	int pos = 0;
	int cnt;
	int ret;
2293 2294
	size_t bufsz;

2295
	bufsz = sizeof(char) * 75 * trans->cfg->base_params->num_of_queues;
2296

2297
	if (!trans_pcie->txq_memory)
2298
		return -EAGAIN;
2299

2300 2301 2302 2303
	buf = kzalloc(bufsz, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

2304
	for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) {
2305
		txq = trans_pcie->txq[cnt];
2306
		pos += scnprintf(buf + pos, bufsz - pos,
2307
				"hwq %.2d: read=%u write=%u use=%d stop=%d need_update=%d frozen=%d%s\n",
2308
				cnt, txq->read_ptr, txq->write_ptr,
2309
				!!test_bit(cnt, trans_pcie->queue_used),
2310
				 !!test_bit(cnt, trans_pcie->queue_stopped),
2311
				 txq->need_update, txq->frozen,
2312
				 (cnt == trans_pcie->cmd_queue ? " HCMD" : ""));
2313 2314 2315 2316 2317 2318 2319
	}
	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
	kfree(buf);
	return ret;
}

static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
2320 2321 2322
				       char __user *user_buf,
				       size_t count, loff_t *ppos)
{
2323
	struct iwl_trans *trans = file->private_data;
2324
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360
	char *buf;
	int pos = 0, i, ret;
	size_t bufsz = sizeof(buf);

	bufsz = sizeof(char) * 121 * trans->num_rx_queues;

	if (!trans_pcie->rxq)
		return -EAGAIN;

	buf = kzalloc(bufsz, GFP_KERNEL);
	if (!buf)
		return -ENOMEM;

	for (i = 0; i < trans->num_rx_queues && pos < bufsz; i++) {
		struct iwl_rxq *rxq = &trans_pcie->rxq[i];

		pos += scnprintf(buf + pos, bufsz - pos, "queue#: %2d\n",
				 i);
		pos += scnprintf(buf + pos, bufsz - pos, "\tread: %u\n",
				 rxq->read);
		pos += scnprintf(buf + pos, bufsz - pos, "\twrite: %u\n",
				 rxq->write);
		pos += scnprintf(buf + pos, bufsz - pos, "\twrite_actual: %u\n",
				 rxq->write_actual);
		pos += scnprintf(buf + pos, bufsz - pos, "\tneed_update: %2d\n",
				 rxq->need_update);
		pos += scnprintf(buf + pos, bufsz - pos, "\tfree_count: %u\n",
				 rxq->free_count);
		if (rxq->rb_stts) {
			pos += scnprintf(buf + pos, bufsz - pos,
					 "\tclosed_rb_num: %u\n",
					 le16_to_cpu(rxq->rb_stts->closed_rb_num) &
					 0x0FFF);
		} else {
			pos += scnprintf(buf + pos, bufsz - pos,
					 "\tclosed_rb_num: Not Allocated\n");
2361
		}
2362
	}
2363 2364 2365 2366
	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
	kfree(buf);

	return ret;
2367 2368
}

2369 2370
static ssize_t iwl_dbgfs_interrupt_read(struct file *file,
					char __user *user_buf,
2371 2372
					size_t count, loff_t *ppos)
{
2373
	struct iwl_trans *trans = file->private_data;
2374
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2375 2376 2377 2378 2379 2380 2381 2382
	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;

	int pos = 0;
	char *buf;
	int bufsz = 24 * 64; /* 24 items * 64 char per item */
	ssize_t ret;

	buf = kzalloc(bufsz, GFP_KERNEL);
2383
	if (!buf)
2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431
		return -ENOMEM;

	pos += scnprintf(buf + pos, bufsz - pos,
			"Interrupt Statistics Report:\n");

	pos += scnprintf(buf + pos, bufsz - pos, "HW Error:\t\t\t %u\n",
		isr_stats->hw);
	pos += scnprintf(buf + pos, bufsz - pos, "SW Error:\t\t\t %u\n",
		isr_stats->sw);
	if (isr_stats->sw || isr_stats->hw) {
		pos += scnprintf(buf + pos, bufsz - pos,
			"\tLast Restarting Code:  0x%X\n",
			isr_stats->err_code);
	}
#ifdef CONFIG_IWLWIFI_DEBUG
	pos += scnprintf(buf + pos, bufsz - pos, "Frame transmitted:\t\t %u\n",
		isr_stats->sch);
	pos += scnprintf(buf + pos, bufsz - pos, "Alive interrupt:\t\t %u\n",
		isr_stats->alive);
#endif
	pos += scnprintf(buf + pos, bufsz - pos,
		"HW RF KILL switch toggled:\t %u\n", isr_stats->rfkill);

	pos += scnprintf(buf + pos, bufsz - pos, "CT KILL:\t\t\t %u\n",
		isr_stats->ctkill);

	pos += scnprintf(buf + pos, bufsz - pos, "Wakeup Interrupt:\t\t %u\n",
		isr_stats->wakeup);

	pos += scnprintf(buf + pos, bufsz - pos,
		"Rx command responses:\t\t %u\n", isr_stats->rx);

	pos += scnprintf(buf + pos, bufsz - pos, "Tx/FH interrupt:\t\t %u\n",
		isr_stats->tx);

	pos += scnprintf(buf + pos, bufsz - pos, "Unexpected INTA:\t\t %u\n",
		isr_stats->unhandled);

	ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
	kfree(buf);
	return ret;
}

static ssize_t iwl_dbgfs_interrupt_write(struct file *file,
					 const char __user *user_buf,
					 size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
2432
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2433 2434
	struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
	u32 reset_flag;
2435
	int ret;
2436

2437 2438 2439
	ret = kstrtou32_from_user(user_buf, count, 16, &reset_flag);
	if (ret)
		return ret;
2440 2441 2442 2443 2444 2445
	if (reset_flag == 0)
		memset(isr_stats, 0, sizeof(*isr_stats));

	return count;
}

2446
static ssize_t iwl_dbgfs_csr_write(struct file *file,
2447 2448
				   const char __user *user_buf,
				   size_t count, loff_t *ppos)
2449 2450 2451
{
	struct iwl_trans *trans = file->private_data;

2452
	iwl_pcie_dump_csr(trans);
2453 2454 2455 2456 2457

	return count;
}

static ssize_t iwl_dbgfs_fh_reg_read(struct file *file,
2458 2459
				     char __user *user_buf,
				     size_t count, loff_t *ppos)
2460 2461
{
	struct iwl_trans *trans = file->private_data;
2462
	char *buf = NULL;
2463
	ssize_t ret;
2464

2465 2466 2467 2468 2469 2470 2471
	ret = iwl_dump_fh(trans, &buf);
	if (ret < 0)
		return ret;
	if (!buf)
		return -EINVAL;
	ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret);
	kfree(buf);
2472 2473 2474
	return ret;
}

2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512
static ssize_t iwl_dbgfs_rfkill_read(struct file *file,
				     char __user *user_buf,
				     size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	char buf[100];
	int pos;

	pos = scnprintf(buf, sizeof(buf), "debug: %d\nhw: %d\n",
			trans_pcie->debug_rfkill,
			!(iwl_read32(trans, CSR_GP_CNTRL) &
				CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW));

	return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
}

static ssize_t iwl_dbgfs_rfkill_write(struct file *file,
				      const char __user *user_buf,
				      size_t count, loff_t *ppos)
{
	struct iwl_trans *trans = file->private_data;
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	bool old = trans_pcie->debug_rfkill;
	int ret;

	ret = kstrtobool_from_user(user_buf, count, &trans_pcie->debug_rfkill);
	if (ret)
		return ret;
	if (old == trans_pcie->debug_rfkill)
		return count;
	IWL_WARN(trans, "changing debug rfkill %d->%d\n",
		 old, trans_pcie->debug_rfkill);
	iwl_pcie_handle_rfkill_irq(trans);

	return count;
}

2513
DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
2514
DEBUGFS_READ_FILE_OPS(fh_reg);
2515 2516
DEBUGFS_READ_FILE_OPS(rx_queue);
DEBUGFS_READ_FILE_OPS(tx_queue);
2517
DEBUGFS_WRITE_FILE_OPS(csr);
2518
DEBUGFS_READ_WRITE_FILE_OPS(rfkill);
2519

2520 2521
/* Create the debugfs files and directories */
int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
2522
{
2523 2524
	struct dentry *dir = trans->dbgfs_dir;

2525 2526
	DEBUGFS_ADD_FILE(rx_queue, dir, S_IRUSR);
	DEBUGFS_ADD_FILE(tx_queue, dir, S_IRUSR);
2527
	DEBUGFS_ADD_FILE(interrupt, dir, S_IWUSR | S_IRUSR);
2528 2529
	DEBUGFS_ADD_FILE(csr, dir, S_IWUSR);
	DEBUGFS_ADD_FILE(fh_reg, dir, S_IRUSR);
2530
	DEBUGFS_ADD_FILE(rfkill, dir, S_IWUSR | S_IRUSR);
2531
	return 0;
2532 2533 2534 2535

err:
	IWL_ERR(trans, "failed to create the trans debugfs entry\n");
	return -ENOMEM;
2536
}
2537
#endif /*CONFIG_IWLWIFI_DEBUGFS */
2538

2539
static u32 iwl_trans_pcie_get_cmdlen(struct iwl_trans *trans, void *tfd)
2540
{
2541
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2542 2543 2544
	u32 cmdlen = 0;
	int i;

2545
	for (i = 0; i < trans_pcie->max_tbs; i++)
2546
		cmdlen += iwl_pcie_tfd_tb_get_len(trans, tfd, i);
2547 2548 2549 2550

	return cmdlen;
}

2551 2552 2553 2554 2555 2556
static u32 iwl_trans_pcie_dump_rbs(struct iwl_trans *trans,
				   struct iwl_fw_error_dump_data **data,
				   int allocated_rb_nums)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	int max_len = PAGE_SIZE << trans_pcie->rx_page_order;
2557 2558
	/* Dump RBs is supported only for pre-9000 devices (1 queue) */
	struct iwl_rxq *rxq = &trans_pcie->rxq[0];
2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592
	u32 i, r, j, rb_len = 0;

	spin_lock(&rxq->lock);

	r = le16_to_cpu(ACCESS_ONCE(rxq->rb_stts->closed_rb_num)) & 0x0FFF;

	for (i = rxq->read, j = 0;
	     i != r && j < allocated_rb_nums;
	     i = (i + 1) & RX_QUEUE_MASK, j++) {
		struct iwl_rx_mem_buffer *rxb = rxq->queue[i];
		struct iwl_fw_error_dump_rb *rb;

		dma_unmap_page(trans->dev, rxb->page_dma, max_len,
			       DMA_FROM_DEVICE);

		rb_len += sizeof(**data) + sizeof(*rb) + max_len;

		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RB);
		(*data)->len = cpu_to_le32(sizeof(*rb) + max_len);
		rb = (void *)(*data)->data;
		rb->index = cpu_to_le32(i);
		memcpy(rb->data, page_address(rxb->page), max_len);
		/* remap the page for the free benefit */
		rxb->page_dma = dma_map_page(trans->dev, rxb->page, 0,
						     max_len,
						     DMA_FROM_DEVICE);

		*data = iwl_fw_error_next_data(*data);
	}

	spin_unlock(&rxq->lock);

	return rb_len;
}
2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
#define IWL_CSR_TO_DUMP (0x250)

static u32 iwl_trans_pcie_dump_csr(struct iwl_trans *trans,
				   struct iwl_fw_error_dump_data **data)
{
	u32 csr_len = sizeof(**data) + IWL_CSR_TO_DUMP;
	__le32 *val;
	int i;

	(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_CSR);
	(*data)->len = cpu_to_le32(IWL_CSR_TO_DUMP);
	val = (void *)(*data)->data;

	for (i = 0; i < IWL_CSR_TO_DUMP; i += 4)
		*val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));

	*data = iwl_fw_error_next_data(*data);

	return csr_len;
}

2614 2615 2616 2617 2618 2619 2620 2621
static u32 iwl_trans_pcie_fh_regs_dump(struct iwl_trans *trans,
				       struct iwl_fw_error_dump_data **data)
{
	u32 fh_regs_len = FH_MEM_UPPER_BOUND - FH_MEM_LOWER_BOUND;
	unsigned long flags;
	__le32 *val;
	int i;

2622
	if (!iwl_trans_grab_nic_access(trans, &flags))
2623 2624 2625 2626 2627 2628
		return 0;

	(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FH_REGS);
	(*data)->len = cpu_to_le32(fh_regs_len);
	val = (void *)(*data)->data;

2629 2630 2631 2632 2633 2634 2635 2636 2637
	if (!trans->cfg->gen2)
		for (i = FH_MEM_LOWER_BOUND; i < FH_MEM_UPPER_BOUND;
		     i += sizeof(u32))
			*val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
	else
		for (i = FH_MEM_LOWER_BOUND_GEN2; i < FH_MEM_UPPER_BOUND_GEN2;
		     i += sizeof(u32))
			*val++ = cpu_to_le32(iwl_trans_pcie_read_prph(trans,
								      i));
2638 2639 2640 2641 2642 2643 2644 2645

	iwl_trans_release_nic_access(trans, &flags);

	*data = iwl_fw_error_next_data(*data);

	return sizeof(**data) + fh_regs_len;
}

2646 2647 2648 2649 2650 2651 2652 2653 2654 2655
static u32
iwl_trans_pci_dump_marbh_monitor(struct iwl_trans *trans,
				 struct iwl_fw_error_dump_fw_mon *fw_mon_data,
				 u32 monitor_len)
{
	u32 buf_size_in_dwords = (monitor_len >> 2);
	u32 *buffer = (u32 *)fw_mon_data->data;
	unsigned long flags;
	u32 i;

2656
	if (!iwl_trans_grab_nic_access(trans, &flags))
2657 2658
		return 0;

2659
	iwl_write_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x1);
2660
	for (i = 0; i < buf_size_in_dwords; i++)
2661 2662 2663
		buffer[i] = iwl_read_prph_no_grab(trans,
				MON_DMARB_RD_DATA_ADDR);
	iwl_write_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x0);
2664 2665 2666 2667 2668 2669

	iwl_trans_release_nic_access(trans, &flags);

	return monitor_len;
}

2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749
static u32
iwl_trans_pcie_dump_monitor(struct iwl_trans *trans,
			    struct iwl_fw_error_dump_data **data,
			    u32 monitor_len)
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	u32 len = 0;

	if ((trans_pcie->fw_mon_page &&
	     trans->cfg->device_family == IWL_DEVICE_FAMILY_7000) ||
	    trans->dbg_dest_tlv) {
		struct iwl_fw_error_dump_fw_mon *fw_mon_data;
		u32 base, write_ptr, wrap_cnt;

		/* If there was a dest TLV - use the values from there */
		if (trans->dbg_dest_tlv) {
			write_ptr =
				le32_to_cpu(trans->dbg_dest_tlv->write_ptr_reg);
			wrap_cnt = le32_to_cpu(trans->dbg_dest_tlv->wrap_count);
			base = le32_to_cpu(trans->dbg_dest_tlv->base_reg);
		} else {
			base = MON_BUFF_BASE_ADDR;
			write_ptr = MON_BUFF_WRPTR;
			wrap_cnt = MON_BUFF_CYCLE_CNT;
		}

		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FW_MONITOR);
		fw_mon_data = (void *)(*data)->data;
		fw_mon_data->fw_mon_wr_ptr =
			cpu_to_le32(iwl_read_prph(trans, write_ptr));
		fw_mon_data->fw_mon_cycle_cnt =
			cpu_to_le32(iwl_read_prph(trans, wrap_cnt));
		fw_mon_data->fw_mon_base_ptr =
			cpu_to_le32(iwl_read_prph(trans, base));

		len += sizeof(**data) + sizeof(*fw_mon_data);
		if (trans_pcie->fw_mon_page) {
			/*
			 * The firmware is now asserted, it won't write anything
			 * to the buffer. CPU can take ownership to fetch the
			 * data. The buffer will be handed back to the device
			 * before the firmware will be restarted.
			 */
			dma_sync_single_for_cpu(trans->dev,
						trans_pcie->fw_mon_phys,
						trans_pcie->fw_mon_size,
						DMA_FROM_DEVICE);
			memcpy(fw_mon_data->data,
			       page_address(trans_pcie->fw_mon_page),
			       trans_pcie->fw_mon_size);

			monitor_len = trans_pcie->fw_mon_size;
		} else if (trans->dbg_dest_tlv->monitor_mode == SMEM_MODE) {
			/*
			 * Update pointers to reflect actual values after
			 * shifting
			 */
			base = iwl_read_prph(trans, base) <<
			       trans->dbg_dest_tlv->base_shift;
			iwl_trans_read_mem(trans, base, fw_mon_data->data,
					   monitor_len / sizeof(u32));
		} else if (trans->dbg_dest_tlv->monitor_mode == MARBH_MODE) {
			monitor_len =
				iwl_trans_pci_dump_marbh_monitor(trans,
								 fw_mon_data,
								 monitor_len);
		} else {
			/* Didn't match anything - output no monitor data */
			monitor_len = 0;
		}

		len += monitor_len;
		(*data)->len = cpu_to_le32(monitor_len + sizeof(*fw_mon_data));
	}

	return len;
}

static struct iwl_trans_dump_data
*iwl_trans_pcie_dump_data(struct iwl_trans *trans,
2750
			  const struct iwl_fw_dbg_trigger_tlv *trigger)
2751 2752 2753
{
	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
	struct iwl_fw_error_dump_data *data;
2754
	struct iwl_txq *cmdq = trans_pcie->txq[trans_pcie->cmd_queue];
2755
	struct iwl_fw_error_dump_txcmd *txcmd;
2756
	struct iwl_trans_dump_data *dump_data;
2757
	u32 len, num_rbs;
2758
	u32 monitor_len;
2759
	int i, ptr;
2760 2761
	bool dump_rbs = test_bit(STATUS_FW_ERROR, &trans->status) &&
			!trans->cfg->mq_rx_supported;
2762

2763 2764 2765 2766 2767
	/* transport dump header */
	len = sizeof(*dump_data);

	/* host commands */
	len += sizeof(*data) +
2768
		cmdq->n_window * (sizeof(*txcmd) + TFD_MAX_PAYLOAD_SIZE);
2769

2770
	/* FW monitor */
2771
	if (trans_pcie->fw_mon_page) {
2772
		len += sizeof(*data) + sizeof(struct iwl_fw_error_dump_fw_mon) +
2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786
		       trans_pcie->fw_mon_size;
		monitor_len = trans_pcie->fw_mon_size;
	} else if (trans->dbg_dest_tlv) {
		u32 base, end;

		base = le32_to_cpu(trans->dbg_dest_tlv->base_reg);
		end = le32_to_cpu(trans->dbg_dest_tlv->end_reg);

		base = iwl_read_prph(trans, base) <<
		       trans->dbg_dest_tlv->base_shift;
		end = iwl_read_prph(trans, end) <<
		      trans->dbg_dest_tlv->end_shift;

		/* Make "end" point to the actual end */
2787
		if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000 ||
2788
		    trans->dbg_dest_tlv->monitor_mode == MARBH_MODE)
2789 2790 2791 2792 2793 2794 2795
			end += (1 << trans->dbg_dest_tlv->end_shift);
		monitor_len = end - base;
		len += sizeof(*data) + sizeof(struct iwl_fw_error_dump_fw_mon) +
		       monitor_len;
	} else {
		monitor_len = 0;
	}
2796

2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812
	if (trigger && (trigger->mode & IWL_FW_DBG_TRIGGER_MONITOR_ONLY)) {
		dump_data = vzalloc(len);
		if (!dump_data)
			return NULL;

		data = (void *)dump_data->data;
		len = iwl_trans_pcie_dump_monitor(trans, &data, monitor_len);
		dump_data->len = len;

		return dump_data;
	}

	/* CSR registers */
	len += sizeof(*data) + IWL_CSR_TO_DUMP;

	/* FH registers */
2813 2814 2815 2816 2817 2818
	if (trans->cfg->gen2)
		len += sizeof(*data) +
		       (FH_MEM_UPPER_BOUND_GEN2 - FH_MEM_LOWER_BOUND_GEN2);
	else
		len += sizeof(*data) +
		       (FH_MEM_UPPER_BOUND - FH_MEM_LOWER_BOUND);
2819 2820

	if (dump_rbs) {
2821 2822
		/* Dump RBs is supported only for pre-9000 devices (1 queue) */
		struct iwl_rxq *rxq = &trans_pcie->rxq[0];
2823
		/* RBs */
2824
		num_rbs = le16_to_cpu(ACCESS_ONCE(rxq->rb_stts->closed_rb_num))
2825
				      & 0x0FFF;
2826
		num_rbs = (num_rbs - rxq->read) & RX_QUEUE_MASK;
2827 2828 2829 2830 2831
		len += num_rbs * (sizeof(*data) +
				  sizeof(struct iwl_fw_error_dump_rb) +
				  (PAGE_SIZE << trans_pcie->rx_page_order));
	}

2832 2833 2834 2835 2836 2837 2838
	/* Paged memory for gen2 HW */
	if (trans->cfg->gen2)
		for (i = 0; i < trans_pcie->init_dram.paging_cnt; i++)
			len += sizeof(*data) +
			       sizeof(struct iwl_fw_error_dump_paging) +
			       trans_pcie->init_dram.paging[i].size;

2839 2840 2841
	dump_data = vzalloc(len);
	if (!dump_data)
		return NULL;
2842 2843

	len = 0;
2844
	data = (void *)dump_data->data;
2845 2846 2847
	data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXCMD);
	txcmd = (void *)data->data;
	spin_lock_bh(&cmdq->lock);
2848 2849 2850
	ptr = cmdq->write_ptr;
	for (i = 0; i < cmdq->n_window; i++) {
		u8 idx = get_cmd_index(cmdq, ptr);
2851 2852
		u32 caplen, cmdlen;

2853 2854
		cmdlen = iwl_trans_pcie_get_cmdlen(trans, cmdq->tfds +
						   trans_pcie->tfd_size * ptr);
2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869
		caplen = min_t(u32, TFD_MAX_PAYLOAD_SIZE, cmdlen);

		if (cmdlen) {
			len += sizeof(*txcmd) + caplen;
			txcmd->cmdlen = cpu_to_le32(cmdlen);
			txcmd->caplen = cpu_to_le32(caplen);
			memcpy(txcmd->data, cmdq->entries[idx].cmd, caplen);
			txcmd = (void *)((u8 *)txcmd->data + caplen);
		}

		ptr = iwl_queue_dec_wrap(ptr);
	}
	spin_unlock_bh(&cmdq->lock);

	data->len = cpu_to_le32(len);
2870
	len += sizeof(*data);
2871 2872
	data = iwl_fw_error_next_data(data);

2873
	len += iwl_trans_pcie_dump_csr(trans, &data);
2874
	len += iwl_trans_pcie_fh_regs_dump(trans, &data);
2875 2876
	if (dump_rbs)
		len += iwl_trans_pcie_dump_rbs(trans, &data, num_rbs);
2877

2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899
	/* Paged memory for gen2 HW */
	if (trans->cfg->gen2) {
		for (i = 0; i < trans_pcie->init_dram.paging_cnt; i++) {
			struct iwl_fw_error_dump_paging *paging;
			dma_addr_t addr =
				trans_pcie->init_dram.paging[i].physical;
			u32 page_len = trans_pcie->init_dram.paging[i].size;

			data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
			data->len = cpu_to_le32(sizeof(*paging) + page_len);
			paging = (void *)data->data;
			paging->index = cpu_to_le32(i);
			dma_sync_single_for_cpu(trans->dev, addr, page_len,
						DMA_BIDIRECTIONAL);
			memcpy(paging->data,
			       trans_pcie->init_dram.paging[i].block, page_len);
			data = iwl_fw_error_next_data(data);

			len += sizeof(*data) + sizeof(*paging) + page_len;
		}
	}

2900
	len += iwl_trans_pcie_dump_monitor(trans, &data, monitor_len);
2901

2902 2903 2904
	dump_data->len = len;

	return dump_data;
2905
}
2906

2907 2908 2909
#ifdef CONFIG_PM_SLEEP
static int iwl_trans_pcie_suspend(struct iwl_trans *trans)
{
2910 2911
	if (trans->runtime_pm_mode == IWL_PLAT_PM_MODE_D0I3 &&
	    (trans->system_pm_mode == IWL_PLAT_PM_MODE_D0I3))
2912 2913 2914 2915 2916 2917 2918
		return iwl_pci_fw_enter_d0i3(trans);

	return 0;
}

static void iwl_trans_pcie_resume(struct iwl_trans *trans)
{
2919 2920
	if (trans->runtime_pm_mode == IWL_PLAT_PM_MODE_D0I3 &&
	    (trans->system_pm_mode == IWL_PLAT_PM_MODE_D0I3))
2921 2922 2923 2924
		iwl_pci_fw_exit_d0i3(trans);
}
#endif /* CONFIG_PM_SLEEP */

2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952
#define IWL_TRANS_COMMON_OPS						\
	.op_mode_leave = iwl_trans_pcie_op_mode_leave,			\
	.write8 = iwl_trans_pcie_write8,				\
	.write32 = iwl_trans_pcie_write32,				\
	.read32 = iwl_trans_pcie_read32,				\
	.read_prph = iwl_trans_pcie_read_prph,				\
	.write_prph = iwl_trans_pcie_write_prph,			\
	.read_mem = iwl_trans_pcie_read_mem,				\
	.write_mem = iwl_trans_pcie_write_mem,				\
	.configure = iwl_trans_pcie_configure,				\
	.set_pmi = iwl_trans_pcie_set_pmi,				\
	.grab_nic_access = iwl_trans_pcie_grab_nic_access,		\
	.release_nic_access = iwl_trans_pcie_release_nic_access,	\
	.set_bits_mask = iwl_trans_pcie_set_bits_mask,			\
	.ref = iwl_trans_pcie_ref,					\
	.unref = iwl_trans_pcie_unref,					\
	.dump_data = iwl_trans_pcie_dump_data,				\
	.d3_suspend = iwl_trans_pcie_d3_suspend,			\
	.d3_resume = iwl_trans_pcie_d3_resume

#ifdef CONFIG_PM_SLEEP
#define IWL_TRANS_PM_OPS						\
	.suspend = iwl_trans_pcie_suspend,				\
	.resume = iwl_trans_pcie_resume,
#else
#define IWL_TRANS_PM_OPS
#endif /* CONFIG_PM_SLEEP */

2953
static const struct iwl_trans_ops trans_ops_pcie = {
2954 2955
	IWL_TRANS_COMMON_OPS,
	IWL_TRANS_PM_OPS
2956
	.start_hw = iwl_trans_pcie_start_hw,
2957
	.fw_alive = iwl_trans_pcie_fw_alive,
2958
	.start_fw = iwl_trans_pcie_start_fw,
2959
	.stop_device = iwl_trans_pcie_stop_device,
2960

2961
	.send_cmd = iwl_trans_pcie_send_hcmd,
2962

2963 2964 2965 2966 2967 2968 2969 2970
	.tx = iwl_trans_pcie_tx,
	.reclaim = iwl_trans_pcie_reclaim,

	.txq_disable = iwl_trans_pcie_txq_disable,
	.txq_enable = iwl_trans_pcie_txq_enable,

	.txq_set_shared_mode = iwl_trans_pcie_txq_set_shared_mode,

2971 2972
	.wait_tx_queues_empty = iwl_trans_pcie_wait_txqs_empty,

2973 2974 2975 2976 2977 2978 2979 2980
	.freeze_txq_timer = iwl_trans_pcie_freeze_txq_timer,
	.block_txq_ptrs = iwl_trans_pcie_block_txq_ptrs,
};

static const struct iwl_trans_ops trans_ops_pcie_gen2 = {
	IWL_TRANS_COMMON_OPS,
	IWL_TRANS_PM_OPS
	.start_hw = iwl_trans_pcie_start_hw,
2981 2982
	.fw_alive = iwl_trans_pcie_gen2_fw_alive,
	.start_fw = iwl_trans_pcie_gen2_start_fw,
2983
	.stop_device = iwl_trans_pcie_gen2_stop_device,
2984

2985
	.send_cmd = iwl_trans_pcie_gen2_send_hcmd,
2986

2987
	.tx = iwl_trans_pcie_gen2_tx,
2988
	.reclaim = iwl_trans_pcie_reclaim,
2989

2990 2991
	.txq_alloc = iwl_trans_pcie_dyn_txq_alloc,
	.txq_free = iwl_trans_pcie_dyn_txq_free,
2992
	.wait_txq_empty = iwl_trans_pcie_wait_txq_empty,
2993
};
2994

2995
struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
2996 2997
				       const struct pci_device_id *ent,
				       const struct iwl_cfg *cfg)
2998 2999 3000
{
	struct iwl_trans_pcie *trans_pcie;
	struct iwl_trans *trans;
3001
	int ret, addr_size;
3002

3003 3004 3005 3006
	ret = pcim_enable_device(pdev);
	if (ret)
		return ERR_PTR(ret);

3007 3008 3009 3010 3011 3012
	if (cfg->gen2)
		trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie),
					&pdev->dev, cfg, &trans_ops_pcie_gen2);
	else
		trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie),
					&pdev->dev, cfg, &trans_ops_pcie);
3013 3014
	if (!trans)
		return ERR_PTR(-ENOMEM);
3015 3016 3017 3018

	trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);

	trans_pcie->trans = trans;
3019
	trans_pcie->opmode_down = true;
3020
	spin_lock_init(&trans_pcie->irq_lock);
3021
	spin_lock_init(&trans_pcie->reg_lock);
3022
	mutex_init(&trans_pcie->mutex);
3023
	init_waitqueue_head(&trans_pcie->ucode_write_waitq);
3024 3025 3026 3027 3028
	trans_pcie->tso_hdr_page = alloc_percpu(struct iwl_tso_hdr_page);
	if (!trans_pcie->tso_hdr_page) {
		ret = -ENOMEM;
		goto out_no_pci;
	}
3029

3030

3031 3032 3033 3034 3035 3036 3037 3038 3039 3040
	if (!cfg->base_params->pcie_l1_allowed) {
		/*
		 * W/A - seems to solve weird behavior. We need to remove this
		 * if we don't want to stay in L1 all the time. This wastes a
		 * lot of power.
		 */
		pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S |
				       PCIE_LINK_STATE_L1 |
				       PCIE_LINK_STATE_CLKPM);
	}
3041

3042
	if (cfg->use_tfh) {
3043
		addr_size = 64;
3044
		trans_pcie->max_tbs = IWL_TFH_NUM_TBS;
3045
		trans_pcie->tfd_size = sizeof(struct iwl_tfh_tfd);
3046
	} else {
3047
		addr_size = 36;
3048
		trans_pcie->max_tbs = IWL_NUM_OF_TBS;
3049 3050
		trans_pcie->tfd_size = sizeof(struct iwl_tfd);
	}
3051 3052
	trans->max_skb_frags = IWL_PCIE_MAX_FRAGS(trans_pcie);

3053 3054
	pci_set_master(pdev);

3055
	ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(addr_size));
3056
	if (!ret)
3057 3058
		ret = pci_set_consistent_dma_mask(pdev,
						  DMA_BIT_MASK(addr_size));
3059 3060 3061 3062
	if (ret) {
		ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
		if (!ret)
			ret = pci_set_consistent_dma_mask(pdev,
3063
							  DMA_BIT_MASK(32));
3064
		/* both attempts failed: */
3065
		if (ret) {
3066
			dev_err(&pdev->dev, "No suitable DMA available\n");
3067
			goto out_no_pci;
3068 3069 3070
		}
	}

3071
	ret = pcim_iomap_regions_request_all(pdev, BIT(0), DRV_NAME);
3072
	if (ret) {
3073 3074
		dev_err(&pdev->dev, "pcim_iomap_regions_request_all failed\n");
		goto out_no_pci;
3075 3076
	}

3077
	trans_pcie->hw_base = pcim_iomap_table(pdev)[0];
3078
	if (!trans_pcie->hw_base) {
3079
		dev_err(&pdev->dev, "pcim_iomap_table failed\n");
3080
		ret = -ENODEV;
3081
		goto out_no_pci;
3082 3083 3084 3085 3086 3087
	}

	/* We disable the RETRY_TIMEOUT register (0x41) to keep
	 * PCI Tx retries from interfering with C3 CPU state */
	pci_write_config_byte(pdev, PCI_CFG_RETRY_TIMEOUT, 0x00);

3088 3089 3090
	trans_pcie->pci_dev = pdev;
	iwl_disable_interrupts(trans);

3091
	trans->hw_rev = iwl_read32(trans, CSR_HW_REV);
3092 3093 3094 3095 3096 3097
	/*
	 * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
	 * changed, and now the revision step also includes bit 0-1 (no more
	 * "dash" value). To keep hw_rev backwards compatible - we'll store it
	 * in the old format.
	 */
3098
	if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000) {
3099 3100
		unsigned long flags;

3101
		trans->hw_rev = (trans->hw_rev & 0xfff0) |
3102
				(CSR_HW_REV_STEP(trans->hw_rev << 2) << 2);
3103

3104 3105 3106
		ret = iwl_pcie_prepare_card_hw(trans);
		if (ret) {
			IWL_WARN(trans, "Exit HW not ready\n");
3107
			goto out_no_pci;
3108 3109
		}

3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
		/*
		 * in-order to recognize C step driver should read chip version
		 * id located at the AUX bus MISC address space.
		 */
		iwl_set_bit(trans, CSR_GP_CNTRL,
			    CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
		udelay(2);

		ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
				   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
				   CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
				   25000);
		if (ret < 0) {
			IWL_DEBUG_INFO(trans, "Failed to wake up the nic\n");
3124
			goto out_no_pci;
3125 3126
		}

3127
		if (iwl_trans_grab_nic_access(trans, &flags)) {
3128 3129
			u32 hw_step;

3130
			hw_step = iwl_read_prph_no_grab(trans, WFPM_CTRL_REG);
3131
			hw_step |= ENABLE_WFPM;
3132 3133
			iwl_write_prph_no_grab(trans, WFPM_CTRL_REG, hw_step);
			hw_step = iwl_read_prph_no_grab(trans, AUX_MISC_REG);
3134 3135 3136 3137 3138 3139 3140 3141
			hw_step = (hw_step >> HW_STEP_LOCATION_BITS) & 0xF;
			if (hw_step == 0x3)
				trans->hw_rev = (trans->hw_rev & 0xFFFFFFF3) |
						(SILICON_C_STEP << 2);
			iwl_trans_release_nic_access(trans, &flags);
		}
	}

3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152
	/*
	 * 9000-series integrated A-step has a problem with suspend/resume
	 * and sometimes even causes the whole platform to get stuck. This
	 * workaround makes the hardware not go into the problematic state.
	 */
	if (trans->cfg->integrated &&
	    trans->cfg->device_family == IWL_DEVICE_FAMILY_9000 &&
	    CSR_HW_REV_STEP(trans->hw_rev) == SILICON_A_STEP)
		iwl_set_bit(trans, CSR_HOST_CHICKEN,
			    CSR_HOST_CHICKEN_PM_IDLE_SRC_DIS_SB_PME);

3153 3154
	trans->hw_rf_id = iwl_read32(trans, CSR_HW_RF_ID);

3155
	iwl_pcie_set_interrupt_capa(pdev, trans);
3156
	trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
3157 3158
	snprintf(trans->hw_id_str, sizeof(trans->hw_id_str),
		 "PCI ID: 0x%04X:0x%04X", pdev->device, pdev->subsystem_device);
3159

3160
	/* Initialize the wait queue for commands */
3161
	init_waitqueue_head(&trans_pcie->wait_command_queue);
3162

3163 3164
	init_waitqueue_head(&trans_pcie->d0i3_waitq);

3165 3166
	if (trans_pcie->msix_enabled) {
		if (iwl_pcie_init_msix_handler(pdev, trans_pcie))
3167
			goto out_no_pci;
3168 3169 3170
	 } else {
		ret = iwl_pcie_alloc_ict(trans);
		if (ret)
3171
			goto out_no_pci;
3172

3173 3174 3175 3176
		ret = devm_request_threaded_irq(&pdev->dev, pdev->irq,
						iwl_pcie_isr,
						iwl_pcie_irq_handler,
						IRQF_SHARED, DRV_NAME, trans);
3177 3178 3179 3180 3181 3182
		if (ret) {
			IWL_ERR(trans, "Error allocating IRQ %d\n", pdev->irq);
			goto out_free_ict;
		}
		trans_pcie->inta_mask = CSR_INI_SET_MASK;
	 }
3183

3184 3185 3186 3187 3188 3189
#ifdef CONFIG_IWLWIFI_PCIE_RTPM
	trans->runtime_pm_mode = IWL_PLAT_PM_MODE_D0I3;
#else
	trans->runtime_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
#endif /* CONFIG_IWLWIFI_PCIE_RTPM */

3190 3191
	return trans;

3192 3193
out_free_ict:
	iwl_pcie_free_ict(trans);
3194
out_no_pci:
3195
	free_percpu(trans_pcie->tso_hdr_page);
3196
	iwl_trans_free(trans);
3197
	return ERR_PTR(ret);
3198
}