igb_main.c 209 KB
Newer Older
1 2 3
/*******************************************************************************

  Intel(R) Gigabit Ethernet Linux driver
4
  Copyright(c) 2007-2013 Intel Corporation.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27

  This program is free software; you can redistribute it and/or modify it
  under the terms and conditions of the GNU General Public License,
  version 2, as published by the Free Software Foundation.

  This program is distributed in the hope it will be useful, but WITHOUT
  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  more details.

  You should have received a copy of the GNU General Public License along with
  this program; if not, write to the Free Software Foundation, Inc.,
  51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.

  The full GNU General Public License is included in this distribution in
  the file called "COPYING".

  Contact Information:
  e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497

*******************************************************************************/

28 29
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

30 31 32
#include <linux/module.h>
#include <linux/types.h>
#include <linux/init.h>
Jiri Pirko's avatar
Jiri Pirko committed
33
#include <linux/bitops.h>
34 35 36 37
#include <linux/vmalloc.h>
#include <linux/pagemap.h>
#include <linux/netdevice.h>
#include <linux/ipv6.h>
38
#include <linux/slab.h>
39 40
#include <net/checksum.h>
#include <net/ip6_checksum.h>
41
#include <linux/net_tstamp.h>
42 43
#include <linux/mii.h>
#include <linux/ethtool.h>
44
#include <linux/if.h>
45 46
#include <linux/if_vlan.h>
#include <linux/pci.h>
47
#include <linux/pci-aspm.h>
48 49
#include <linux/delay.h>
#include <linux/interrupt.h>
50 51 52
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/sctp.h>
53
#include <linux/if_ether.h>
54
#include <linux/aer.h>
55
#include <linux/prefetch.h>
56
#include <linux/pm_runtime.h>
57
#ifdef CONFIG_IGB_DCA
Jeb Cramer's avatar
Jeb Cramer committed
58 59
#include <linux/dca.h>
#endif
60
#include <linux/i2c.h>
61 62
#include "igb.h"

63 64
#define MAJ 5
#define MIN 0
65
#define BUILD 5
Carolyn Wyborny's avatar
Carolyn Wyborny committed
66
#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
67
__stringify(BUILD) "-k"
68 69 70 71
char igb_driver_name[] = "igb";
char igb_driver_version[] = DRV_VERSION;
static const char igb_driver_string[] =
				"Intel(R) Gigabit Ethernet Network Driver";
72 73
static const char igb_copyright[] =
				"Copyright (c) 2007-2013 Intel Corporation.";
74 75 76 77 78

static const struct e1000_info *igb_info_tbl[] = {
	[board_82575] = &e1000_82575_info,
};

79
static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
80 81 82
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
83 84 85 86 87
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
88 89
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
90 91 92 93
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
94 95
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
96
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
97 98 99
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
100 101
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
Gasparakis, Joseph's avatar
Gasparakis, Joseph committed
102 103
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
Alexander Duyck's avatar
Alexander Duyck committed
104
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
105
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
106
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
Alexander Duyck's avatar
Alexander Duyck committed
107 108
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
109
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
110
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
111
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
112 113 114 115 116 117 118 119 120 121 122 123 124 125
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
	/* required last entry */
	{0, }
};

MODULE_DEVICE_TABLE(pci, igb_pci_tbl);

void igb_reset(struct igb_adapter *);
static int igb_setup_all_tx_resources(struct igb_adapter *);
static int igb_setup_all_rx_resources(struct igb_adapter *);
static void igb_free_all_tx_resources(struct igb_adapter *);
static void igb_free_all_rx_resources(struct igb_adapter *);
126
static void igb_setup_mrqc(struct igb_adapter *);
127
static int igb_probe(struct pci_dev *, const struct pci_device_id *);
128
static void igb_remove(struct pci_dev *pdev);
129 130 131
static int igb_sw_init(struct igb_adapter *);
static int igb_open(struct net_device *);
static int igb_close(struct net_device *);
132
static void igb_configure(struct igb_adapter *);
133 134 135 136
static void igb_configure_tx(struct igb_adapter *);
static void igb_configure_rx(struct igb_adapter *);
static void igb_clean_all_tx_rings(struct igb_adapter *);
static void igb_clean_all_rx_rings(struct igb_adapter *);
137 138
static void igb_clean_tx_ring(struct igb_ring *);
static void igb_clean_rx_ring(struct igb_ring *);
139
static void igb_set_rx_mode(struct net_device *);
140 141 142
static void igb_update_phy_info(unsigned long);
static void igb_watchdog(unsigned long);
static void igb_watchdog_task(struct work_struct *);
143
static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
Eric Dumazet's avatar
Eric Dumazet committed
144 145
static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
						 struct rtnl_link_stats64 *stats);
146 147
static int igb_change_mtu(struct net_device *, int);
static int igb_set_mac(struct net_device *, void *);
148
static void igb_set_uta(struct igb_adapter *adapter);
149 150 151
static irqreturn_t igb_intr(int irq, void *);
static irqreturn_t igb_intr_msi(int irq, void *);
static irqreturn_t igb_msix_other(int irq, void *);
152
static irqreturn_t igb_msix_ring(int irq, void *);
153
#ifdef CONFIG_IGB_DCA
154
static void igb_update_dca(struct igb_q_vector *);
Jeb Cramer's avatar
Jeb Cramer committed
155
static void igb_setup_dca(struct igb_adapter *);
156
#endif /* CONFIG_IGB_DCA */
157
static int igb_poll(struct napi_struct *, int);
158
static bool igb_clean_tx_irq(struct igb_q_vector *);
159
static bool igb_clean_rx_irq(struct igb_q_vector *, int);
160 161 162
static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
static void igb_tx_timeout(struct net_device *);
static void igb_reset_task(struct work_struct *);
163
static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
164 165
static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
166
static void igb_restore_vlan(struct igb_adapter *);
167
static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
168 169 170
static void igb_ping_all_vfs(struct igb_adapter *);
static void igb_msg_task(struct igb_adapter *);
static void igb_vmm_control(struct igb_adapter *);
171
static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
172
static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
173 174 175 176
static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
static int igb_ndo_set_vf_vlan(struct net_device *netdev,
			       int vf, u16 vlan, u8 qos);
static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
177 178
static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
				   bool setting);
179 180
static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
				 struct ifla_vf_info *ivi);
181
static void igb_check_vf_rate_limit(struct igb_adapter *);
RongQing Li's avatar
RongQing Li committed
182 183

#ifdef CONFIG_PCI_IOV
184
static int igb_vf_configure(struct igb_adapter *adapter, int vf);
RongQing Li's avatar
RongQing Li committed
185
#endif
186 187

#ifdef CONFIG_PM
188
#ifdef CONFIG_PM_SLEEP
189
static int igb_suspend(struct device *);
190
#endif
191 192 193 194 195 196 197 198 199 200 201
static int igb_resume(struct device *);
#ifdef CONFIG_PM_RUNTIME
static int igb_runtime_suspend(struct device *dev);
static int igb_runtime_resume(struct device *dev);
static int igb_runtime_idle(struct device *dev);
#endif
static const struct dev_pm_ops igb_pm_ops = {
	SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
	SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
			igb_runtime_idle)
};
202 203
#endif
static void igb_shutdown(struct pci_dev *);
204
static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
205
#ifdef CONFIG_IGB_DCA
Jeb Cramer's avatar
Jeb Cramer committed
206 207 208 209 210 211 212
static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
static struct notifier_block dca_notifier = {
	.notifier_call	= igb_notify_dca,
	.next		= NULL,
	.priority	= 0
};
#endif
213 214 215 216
#ifdef CONFIG_NET_POLL_CONTROLLER
/* for netdump / net console */
static void igb_netpoll(struct net_device *);
#endif
217
#ifdef CONFIG_PCI_IOV
218 219 220 221 222 223
static unsigned int max_vfs = 0;
module_param(max_vfs, uint, 0);
MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
                 "per physical function");
#endif /* CONFIG_PCI_IOV */

224 225 226 227 228
static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
		     pci_channel_state_t);
static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
static void igb_io_resume(struct pci_dev *);

229
static const struct pci_error_handlers igb_err_handler = {
230 231 232 233 234
	.error_detected = igb_io_error_detected,
	.slot_reset = igb_io_slot_reset,
	.resume = igb_io_resume,
};

235
static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
236 237 238 239 240

static struct pci_driver igb_driver = {
	.name     = igb_driver_name,
	.id_table = igb_pci_tbl,
	.probe    = igb_probe,
241
	.remove   = igb_remove,
242
#ifdef CONFIG_PM
243
	.driver.pm = &igb_pm_ops,
244 245
#endif
	.shutdown = igb_shutdown,
246
	.sriov_configure = igb_pci_sriov_configure,
247 248 249 250 251 252 253 254
	.err_handler = &igb_err_handler
};

MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
MODULE_LICENSE("GPL");
MODULE_VERSION(DRV_VERSION);

255 256 257 258 259
#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
static int debug = -1;
module_param(debug, int, 0);
MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");

260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
struct igb_reg_info {
	u32 ofs;
	char *name;
};

static const struct igb_reg_info igb_reg_info_tbl[] = {

	/* General Registers */
	{E1000_CTRL, "CTRL"},
	{E1000_STATUS, "STATUS"},
	{E1000_CTRL_EXT, "CTRL_EXT"},

	/* Interrupt Registers */
	{E1000_ICR, "ICR"},

	/* RX Registers */
	{E1000_RCTL, "RCTL"},
	{E1000_RDLEN(0), "RDLEN"},
	{E1000_RDH(0), "RDH"},
	{E1000_RDT(0), "RDT"},
	{E1000_RXDCTL(0), "RXDCTL"},
	{E1000_RDBAL(0), "RDBAL"},
	{E1000_RDBAH(0), "RDBAH"},

	/* TX Registers */
	{E1000_TCTL, "TCTL"},
	{E1000_TDBAL(0), "TDBAL"},
	{E1000_TDBAH(0), "TDBAH"},
	{E1000_TDLEN(0), "TDLEN"},
	{E1000_TDH(0), "TDH"},
	{E1000_TDT(0), "TDT"},
	{E1000_TXDCTL(0), "TXDCTL"},
	{E1000_TDFH, "TDFH"},
	{E1000_TDFT, "TDFT"},
	{E1000_TDFHS, "TDFHS"},
	{E1000_TDFPC, "TDFPC"},

	/* List Terminator */
	{}
};

301
/* igb_regdump - register printout routine */
302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357
static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
{
	int n = 0;
	char rname[16];
	u32 regs[8];

	switch (reginfo->ofs) {
	case E1000_RDLEN(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDLEN(n));
		break;
	case E1000_RDH(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDH(n));
		break;
	case E1000_RDT(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDT(n));
		break;
	case E1000_RXDCTL(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RXDCTL(n));
		break;
	case E1000_RDBAL(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDBAL(n));
		break;
	case E1000_RDBAH(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDBAH(n));
		break;
	case E1000_TDBAL(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_RDBAL(n));
		break;
	case E1000_TDBAH(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TDBAH(n));
		break;
	case E1000_TDLEN(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TDLEN(n));
		break;
	case E1000_TDH(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TDH(n));
		break;
	case E1000_TDT(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TDT(n));
		break;
	case E1000_TXDCTL(0):
		for (n = 0; n < 4; n++)
			regs[n] = rd32(E1000_TXDCTL(n));
		break;
	default:
358
		pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
359 360 361 362
		return;
	}

	snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
363 364
	pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
		regs[2], regs[3]);
365 366
}

367
/* igb_dump - Print registers, Tx-rings and Rx-rings */
368 369 370 371 372 373 374 375 376 377 378
static void igb_dump(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	struct e1000_hw *hw = &adapter->hw;
	struct igb_reg_info *reginfo;
	struct igb_ring *tx_ring;
	union e1000_adv_tx_desc *tx_desc;
	struct my_u0 { u64 a; u64 b; } *u0;
	struct igb_ring *rx_ring;
	union e1000_adv_rx_desc *rx_desc;
	u32 staterr;
379
	u16 i, n;
380 381 382 383 384 385 386

	if (!netif_msg_hw(adapter))
		return;

	/* Print netdevice Info */
	if (netdev) {
		dev_info(&adapter->pdev->dev, "Net device Info\n");
387 388 389 390
		pr_info("Device Name     state            trans_start      "
			"last_rx\n");
		pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
			netdev->state, netdev->trans_start, netdev->last_rx);
391 392 393 394
	}

	/* Print Registers */
	dev_info(&adapter->pdev->dev, "Register Dump\n");
395
	pr_info(" Register Name   Value\n");
396 397 398 399 400 401 402 403 404 405
	for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
	     reginfo->name; reginfo++) {
		igb_regdump(hw, reginfo);
	}

	/* Print TX Ring Summary */
	if (!netdev || !netif_running(netdev))
		goto exit;

	dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
406
	pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
407
	for (n = 0; n < adapter->num_tx_queues; n++) {
408
		struct igb_tx_buffer *buffer_info;
409
		tx_ring = adapter->tx_ring[n];
410
		buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
411 412
		pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
			n, tx_ring->next_to_use, tx_ring->next_to_clean,
413 414
			(u64)dma_unmap_addr(buffer_info, dma),
			dma_unmap_len(buffer_info, len),
415 416
			buffer_info->next_to_watch,
			(u64)buffer_info->time_stamp);
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437
	}

	/* Print TX Rings */
	if (!netif_msg_tx_done(adapter))
		goto rx_ring_summary;

	dev_info(&adapter->pdev->dev, "TX Rings Dump\n");

	/* Transmit Descriptor Formats
	 *
	 * Advanced Transmit Descriptor
	 *   +--------------------------------------------------------------+
	 * 0 |         Buffer Address [63:0]                                |
	 *   +--------------------------------------------------------------+
	 * 8 | PAYLEN  | PORTS  |CC|IDX | STA | DCMD  |DTYP|MAC|RSV| DTALEN |
	 *   +--------------------------------------------------------------+
	 *   63      46 45    40 39 38 36 35 32 31   24             15       0
	 */

	for (n = 0; n < adapter->num_tx_queues; n++) {
		tx_ring = adapter->tx_ring[n];
438 439 440 441 442 443
		pr_info("------------------------------------\n");
		pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
		pr_info("------------------------------------\n");
		pr_info("T [desc]     [address 63:0  ] [PlPOCIStDDM Ln] "
			"[bi->dma       ] leng  ntw timestamp        "
			"bi->skb\n");
444 445

		for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
446
			const char *next_desc;
447
			struct igb_tx_buffer *buffer_info;
448
			tx_desc = IGB_TX_DESC(tx_ring, i);
449
			buffer_info = &tx_ring->tx_buffer_info[i];
450
			u0 = (struct my_u0 *)tx_desc;
451 452 453 454 455 456 457 458 459 460 461 462
			if (i == tx_ring->next_to_use &&
			    i == tx_ring->next_to_clean)
				next_desc = " NTC/U";
			else if (i == tx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == tx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

			pr_info("T [0x%03X]    %016llX %016llX %016llX"
				" %04X  %p %016llX %p%s\n", i,
463 464
				le64_to_cpu(u0->a),
				le64_to_cpu(u0->b),
465 466
				(u64)dma_unmap_addr(buffer_info, dma),
				dma_unmap_len(buffer_info, len),
467 468
				buffer_info->next_to_watch,
				(u64)buffer_info->time_stamp,
469
				buffer_info->skb, next_desc);
470

471
			if (netif_msg_pktdata(adapter) && buffer_info->skb)
472 473
				print_hex_dump(KERN_INFO, "",
					DUMP_PREFIX_ADDRESS,
474
					16, 1, buffer_info->skb->data,
475 476
					dma_unmap_len(buffer_info, len),
					true);
477 478 479 480 481 482
		}
	}

	/* Print RX Rings Summary */
rx_ring_summary:
	dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
483
	pr_info("Queue [NTU] [NTC]\n");
484 485
	for (n = 0; n < adapter->num_rx_queues; n++) {
		rx_ring = adapter->rx_ring[n];
486 487
		pr_info(" %5d %5X %5X\n",
			n, rx_ring->next_to_use, rx_ring->next_to_clean);
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
	}

	/* Print RX Rings */
	if (!netif_msg_rx_status(adapter))
		goto exit;

	dev_info(&adapter->pdev->dev, "RX Rings Dump\n");

	/* Advanced Receive Descriptor (Read) Format
	 *    63                                           1        0
	 *    +-----------------------------------------------------+
	 *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
	 *    +----------------------------------------------+------+
	 *  8 |       Header Buffer Address [63:1]           |  DD  |
	 *    +-----------------------------------------------------+
	 *
	 *
	 * Advanced Receive Descriptor (Write-Back) Format
	 *
	 *   63       48 47    32 31  30      21 20 17 16   4 3     0
	 *   +------------------------------------------------------+
	 * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
	 *   | Checksum   Ident  |   |           |    | Type | Type |
	 *   +------------------------------------------------------+
	 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
	 *   +------------------------------------------------------+
	 *   63       48 47    32 31            20 19               0
	 */

	for (n = 0; n < adapter->num_rx_queues; n++) {
		rx_ring = adapter->rx_ring[n];
519 520 521 522 523 524 525
		pr_info("------------------------------------\n");
		pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
		pr_info("------------------------------------\n");
		pr_info("R  [desc]      [ PktBuf     A0] [  HeadBuf   DD] "
			"[bi->dma       ] [bi->skb] <-- Adv Rx Read format\n");
		pr_info("RWB[desc]      [PcsmIpSHl PtRs] [vl er S cks ln] -----"
			"----------- [bi->skb] <-- Adv Rx Write-Back format\n");
526 527

		for (i = 0; i < rx_ring->count; i++) {
528
			const char *next_desc;
529 530
			struct igb_rx_buffer *buffer_info;
			buffer_info = &rx_ring->rx_buffer_info[i];
531
			rx_desc = IGB_RX_DESC(rx_ring, i);
532 533
			u0 = (struct my_u0 *)rx_desc;
			staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
534 535 536 537 538 539 540 541

			if (i == rx_ring->next_to_use)
				next_desc = " NTU";
			else if (i == rx_ring->next_to_clean)
				next_desc = " NTC";
			else
				next_desc = "";

542 543
			if (staterr & E1000_RXD_STAT_DD) {
				/* Descriptor Done */
544 545
				pr_info("%s[0x%03X]     %016llX %016llX ---------------- %s\n",
					"RWB", i,
546 547
					le64_to_cpu(u0->a),
					le64_to_cpu(u0->b),
548
					next_desc);
549
			} else {
550 551
				pr_info("%s[0x%03X]     %016llX %016llX %016llX %s\n",
					"R  ", i,
552 553 554
					le64_to_cpu(u0->a),
					le64_to_cpu(u0->b),
					(u64)buffer_info->dma,
555
					next_desc);
556

557
				if (netif_msg_pktdata(adapter) &&
558
				    buffer_info->dma && buffer_info->page) {
559 560 561
					print_hex_dump(KERN_INFO, "",
					  DUMP_PREFIX_ADDRESS,
					  16, 1,
562 563
					  page_address(buffer_info->page) +
						      buffer_info->page_offset,
564
					  IGB_RX_BUFSZ, true);
565 566 567 568 569 570 571 572 573
				}
			}
		}
	}

exit:
	return;
}

574 575
/**
 *  igb_get_i2c_data - Reads the I2C SDA data bit
576 577 578 579
 *  @hw: pointer to hardware structure
 *  @i2cctl: Current value of I2CCTL register
 *
 *  Returns the I2C data bit value
580
 **/
581 582 583 584 585 586 587 588 589
static int igb_get_i2c_data(void *data)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	struct e1000_hw *hw = &adapter->hw;
	s32 i2cctl = rd32(E1000_I2CPARAMS);

	return ((i2cctl & E1000_I2C_DATA_IN) != 0);
}

590 591
/**
 *  igb_set_i2c_data - Sets the I2C data bit
592 593 594 595
 *  @data: pointer to hardware structure
 *  @state: I2C data value (0 or 1) to set
 *
 *  Sets the I2C data bit
596
 **/
597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614
static void igb_set_i2c_data(void *data, int state)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	struct e1000_hw *hw = &adapter->hw;
	s32 i2cctl = rd32(E1000_I2CPARAMS);

	if (state)
		i2cctl |= E1000_I2C_DATA_OUT;
	else
		i2cctl &= ~E1000_I2C_DATA_OUT;

	i2cctl &= ~E1000_I2C_DATA_OE_N;
	i2cctl |= E1000_I2C_CLK_OE_N;
	wr32(E1000_I2CPARAMS, i2cctl);
	wrfl();

}

615 616
/**
 *  igb_set_i2c_clk - Sets the I2C SCL clock
617 618 619 620
 *  @data: pointer to hardware structure
 *  @state: state to set clock
 *
 *  Sets the I2C clock line to state
621
 **/
622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638
static void igb_set_i2c_clk(void *data, int state)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	struct e1000_hw *hw = &adapter->hw;
	s32 i2cctl = rd32(E1000_I2CPARAMS);

	if (state) {
		i2cctl |= E1000_I2C_CLK_OUT;
		i2cctl &= ~E1000_I2C_CLK_OE_N;
	} else {
		i2cctl &= ~E1000_I2C_CLK_OUT;
		i2cctl &= ~E1000_I2C_CLK_OE_N;
	}
	wr32(E1000_I2CPARAMS, i2cctl);
	wrfl();
}

639 640
/**
 *  igb_get_i2c_clk - Gets the I2C SCL clock state
641 642 643
 *  @data: pointer to hardware structure
 *
 *  Gets the I2C clock state
644
 **/
645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662
static int igb_get_i2c_clk(void *data)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	struct e1000_hw *hw = &adapter->hw;
	s32 i2cctl = rd32(E1000_I2CPARAMS);

	return ((i2cctl & E1000_I2C_CLK_IN) != 0);
}

static const struct i2c_algo_bit_data igb_i2c_algo = {
	.setsda		= igb_set_i2c_data,
	.setscl		= igb_set_i2c_clk,
	.getsda		= igb_get_i2c_data,
	.getscl		= igb_get_i2c_clk,
	.udelay		= 5,
	.timeout	= 20,
};

663
/**
664 665 666 667
 *  igb_get_hw_dev - return device
 *  @hw: pointer to hardware structure
 *
 *  used by hardware layer to print debugging information
668
 **/
669
struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
670 671
{
	struct igb_adapter *adapter = hw->back;
672
	return adapter->netdev;
673
}
Patrick Ohly's avatar
Patrick Ohly committed
674

675
/**
676
 *  igb_init_module - Driver Registration Routine
677
 *
678 679
 *  igb_init_module is the first routine called when the driver is
 *  loaded. All it does is register with the PCI subsystem.
680 681 682 683
 **/
static int __init igb_init_module(void)
{
	int ret;
684
	pr_info("%s - version %s\n",
685 686
	       igb_driver_string, igb_driver_version);

687
	pr_info("%s\n", igb_copyright);
688

689
#ifdef CONFIG_IGB_DCA
Jeb Cramer's avatar
Jeb Cramer committed
690 691
	dca_register_notify(&dca_notifier);
#endif
692
	ret = pci_register_driver(&igb_driver);
693 694 695 696 697 698
	return ret;
}

module_init(igb_init_module);

/**
699
 *  igb_exit_module - Driver Exit Cleanup Routine
700
 *
701 702
 *  igb_exit_module is called just before the driver is removed
 *  from memory.
703 704 705
 **/
static void __exit igb_exit_module(void)
{
706
#ifdef CONFIG_IGB_DCA
Jeb Cramer's avatar
Jeb Cramer committed
707 708
	dca_unregister_notify(&dca_notifier);
#endif
709 710 711 712 713
	pci_unregister_driver(&igb_driver);
}

module_exit(igb_exit_module);

714 715
#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
/**
716 717
 *  igb_cache_ring_register - Descriptor ring to register mapping
 *  @adapter: board private structure to initialize
718
 *
719 720
 *  Once we know the feature-set enabled for the device, we'll cache
 *  the register offset the descriptor ring is assigned to.
721 722 723
 **/
static void igb_cache_ring_register(struct igb_adapter *adapter)
{
724
	int i = 0, j = 0;
725
	u32 rbase_offset = adapter->vfs_allocated_count;
726 727 728 729 730 731 732 733

	switch (adapter->hw.mac.type) {
	case e1000_82576:
		/* The queues are allocated for virtualization such that VF 0
		 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
		 * In order to avoid collision we start at the first free queue
		 * and continue consuming queues in the same sequence
		 */
734
		if (adapter->vfs_allocated_count) {
735
			for (; i < adapter->rss_queues; i++)
736
				adapter->rx_ring[i]->reg_idx = rbase_offset +
737
							       Q_IDX_82576(i);
738
		}
739
	case e1000_82575:
740
	case e1000_82580:
741
	case e1000_i350:
742
	case e1000_i354:
743 744
	case e1000_i210:
	case e1000_i211:
745
	default:
746
		for (; i < adapter->num_rx_queues; i++)
747
			adapter->rx_ring[i]->reg_idx = rbase_offset + i;
748
		for (; j < adapter->num_tx_queues; j++)
749
			adapter->tx_ring[j]->reg_idx = rbase_offset + j;
750 751 752 753
		break;
	}
}

754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
/**
 *  igb_write_ivar - configure ivar for given MSI-X vector
 *  @hw: pointer to the HW structure
 *  @msix_vector: vector number we are allocating to a given ring
 *  @index: row index of IVAR register to write within IVAR table
 *  @offset: column offset of in IVAR, should be multiple of 8
 *
 *  This function is intended to handle the writing of the IVAR register
 *  for adapters 82576 and newer.  The IVAR table consists of 2 columns,
 *  each containing an cause allocation for an Rx and Tx ring, and a
 *  variable number of rows depending on the number of queues supported.
 **/
static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
			   int index, int offset)
{
	u32 ivar = array_rd32(E1000_IVAR0, index);

	/* clear any bits that are currently set */
	ivar &= ~((u32)0xFF << offset);

	/* write vector and valid bit */
	ivar |= (msix_vector | E1000_IVAR_VALID) << offset;

	array_wr32(E1000_IVAR0, index, ivar);
}

780
#define IGB_N0_QUEUE -1
781
static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
782
{
783
	struct igb_adapter *adapter = q_vector->adapter;
784
	struct e1000_hw *hw = &adapter->hw;
785 786
	int rx_queue = IGB_N0_QUEUE;
	int tx_queue = IGB_N0_QUEUE;
787
	u32 msixbm = 0;
788

789 790 791 792
	if (q_vector->rx.ring)
		rx_queue = q_vector->rx.ring->reg_idx;
	if (q_vector->tx.ring)
		tx_queue = q_vector->tx.ring->reg_idx;
Alexander Duyck's avatar
Alexander Duyck committed
793 794 795

	switch (hw->mac.type) {
	case e1000_82575:
796
		/* The 82575 assigns vectors using a bitmask, which matches the
797 798 799 800
		 * bitmask for the EICR/EIMS/EIMC registers.  To assign one
		 * or more queues to a vector, we write the appropriate bits
		 * into the MSIXBM register for that vector.
		 */
801
		if (rx_queue > IGB_N0_QUEUE)
802
			msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
803
		if (tx_queue > IGB_N0_QUEUE)
804
			msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
805 806
		if (!adapter->msix_entries && msix_vector == 0)
			msixbm |= E1000_EIMS_OTHER;
807
		array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
808
		q_vector->eims_value = msixbm;
Alexander Duyck's avatar
Alexander Duyck committed
809 810
		break;
	case e1000_82576:
811
		/* 82576 uses a table that essentially consists of 2 columns
812 813 814 815 816 817 818 819 820 821 822 823
		 * with 8 rows.  The ordering is column-major so we use the
		 * lower 3 bits as the row index, and the 4th bit as the
		 * column offset.
		 */
		if (rx_queue > IGB_N0_QUEUE)
			igb_write_ivar(hw, msix_vector,
				       rx_queue & 0x7,
				       (rx_queue & 0x8) << 1);
		if (tx_queue > IGB_N0_QUEUE)
			igb_write_ivar(hw, msix_vector,
				       tx_queue & 0x7,
				       ((tx_queue & 0x8) << 1) + 8);
824
		q_vector->eims_value = 1 << msix_vector;
Alexander Duyck's avatar
Alexander Duyck committed
825
		break;
826
	case e1000_82580:
827
	case e1000_i350:
828
	case e1000_i354:
829 830
	case e1000_i210:
	case e1000_i211:
831
		/* On 82580 and newer adapters the scheme is similar to 82576
832 833 834 835 836 837 838 839 840 841 842 843 844
		 * however instead of ordering column-major we have things
		 * ordered row-major.  So we traverse the table by using
		 * bit 0 as the column offset, and the remaining bits as the
		 * row index.
		 */
		if (rx_queue > IGB_N0_QUEUE)
			igb_write_ivar(hw, msix_vector,
				       rx_queue >> 1,
				       (rx_queue & 0x1) << 4);
		if (tx_queue > IGB_N0_QUEUE)
			igb_write_ivar(hw, msix_vector,
				       tx_queue >> 1,
				       ((tx_queue & 0x1) << 4) + 8);
845 846
		q_vector->eims_value = 1 << msix_vector;
		break;
Alexander Duyck's avatar
Alexander Duyck committed
847 848 849 850
	default:
		BUG();
		break;
	}
851 852 853 854 855 856

	/* add q_vector eims value to global eims_enable_mask */
	adapter->eims_enable_mask |= q_vector->eims_value;

	/* configure q_vector to set itr on first interrupt */
	q_vector->set_itr = 1;
857 858 859
}

/**
860 861
 *  igb_configure_msix - Configure MSI-X hardware
 *  @adapter: board private structure to initialize
862
 *
863 864
 *  igb_configure_msix sets up the hardware to properly
 *  generate MSI-X interrupts.
865 866 867 868 869 870 871 872 873 874
 **/
static void igb_configure_msix(struct igb_adapter *adapter)
{
	u32 tmp;
	int i, vector = 0;
	struct e1000_hw *hw = &adapter->hw;

	adapter->eims_enable_mask = 0;

	/* set vector for other causes, i.e. link changes */
Alexander Duyck's avatar
Alexander Duyck committed
875 876
	switch (hw->mac.type) {
	case e1000_82575:
877 878 879 880 881 882 883 884 885
		tmp = rd32(E1000_CTRL_EXT);
		/* enable MSI-X PBA support*/
		tmp |= E1000_CTRL_EXT_PBA_CLR;

		/* Auto-Mask interrupts upon ICR read. */
		tmp |= E1000_CTRL_EXT_EIAME;
		tmp |= E1000_CTRL_EXT_IRCA;

		wr32(E1000_CTRL_EXT, tmp);
886 887

		/* enable msix_other interrupt */
888
		array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
889
		adapter->eims_other = E1000_EIMS_OTHER;
890

Alexander Duyck's avatar
Alexander Duyck committed
891 892 893
		break;

	case e1000_82576:
894
	case e1000_82580:
895
	case e1000_i350:
896
	case e1000_i354:
897 898
	case e1000_i210:
	case e1000_i211:
899
		/* Turn on MSI-X capability first, or our settings
900 901
		 * won't stick.  And it will take days to debug.
		 */
902
		wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
903 904
		     E1000_GPIE_PBA | E1000_GPIE_EIAME |
		     E1000_GPIE_NSICR);
905 906 907

		/* enable msix_other interrupt */
		adapter->eims_other = 1 << vector;
Alexander Duyck's avatar
Alexander Duyck committed
908 909
		tmp = (vector++ | E1000_IVAR_VALID) << 8;

910
		wr32(E1000_IVAR_MISC, tmp);
Alexander Duyck's avatar
Alexander Duyck committed
911 912 913 914 915
		break;
	default:
		/* do nothing, since nothing else supports MSI-X */
		break;
	} /* switch (hw->mac.type) */
916 917 918

	adapter->eims_enable_mask |= adapter->eims_other;

919 920
	for (i = 0; i < adapter->num_q_vectors; i++)
		igb_assign_vector(adapter->q_vector[i], vector++);
921

922 923 924 925
	wrfl();
}

/**
926 927
 *  igb_request_msix - Initialize MSI-X interrupts
 *  @adapter: board private structure to initialize
928
 *
929 930
 *  igb_request_msix allocates MSI-X vectors and requests interrupts from the
 *  kernel.
931 932 933 934
 **/
static int igb_request_msix(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
935
	struct e1000_hw *hw = &adapter->hw;
936
	int i, err = 0, vector = 0, free_vector = 0;
937

938
	err = request_irq(adapter->msix_entries[vector].vector,
939
			  igb_msix_other, 0, netdev->name, adapter);
940
	if (err)
941
		goto err_out;
942 943 944 945

	for (i = 0; i < adapter->num_q_vectors; i++) {
		struct igb_q_vector *q_vector = adapter->q_vector[i];

946 947
		vector++;

948 949
		q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);

950
		if (q_vector->rx.ring && q_vector->tx.ring)
951
			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
952 953
				q_vector->rx.ring->queue_index);
		else if (q_vector->tx.ring)
954
			sprintf(q_vector->name, "%s-tx-%u", netdev->name,
955 956
				q_vector->tx.ring->queue_index);
		else if (q_vector->rx.ring)
957
			sprintf(q_vector->name, "%s-rx-%u", netdev->name,
958
				q_vector->rx.ring->queue_index);
959
		else
960 961
			sprintf(q_vector->name, "%s-unused", netdev->name);

962
		err = request_irq(adapter->msix_entries[vector].vector,
963 964
				  igb_msix_ring, 0, q_vector->name,
				  q_vector);
965
		if (err)
966
			goto err_free;
967 968 969 970
	}

	igb_configure_msix(adapter);
	return 0;
971 972 973 974 975 976 977 978 979 980 981

err_free:
	/* free already assigned IRQs */
	free_irq(adapter->msix_entries[free_vector++].vector, adapter);

	vector--;
	for (i = 0; i < vector; i++) {
		free_irq(adapter->msix_entries[free_vector++].vector,
			 adapter->q_vector[i]);
	}
err_out:
982 983 984 985 986 987 988 989 990
	return err;
}

static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
{
	if (adapter->msix_entries) {
		pci_disable_msix(adapter->pdev);
		kfree(adapter->msix_entries);
		adapter->msix_entries = NULL;
991
	} else if (adapter->flags & IGB_FLAG_HAS_MSI) {
992
		pci_disable_msi(adapter->pdev);
993
	}
994 995
}

996
/**
997 998 999
 *  igb_free_q_vector - Free memory allocated for specific interrupt vector
 *  @adapter: board private structure to initialize
 *  @v_idx: Index of vector to be freed
1000
 *
1001 1002 1003
 *  This function frees the memory allocated to the q_vector.  In addition if
 *  NAPI is enabled it will delete any references to the NAPI struct prior
 *  to freeing the q_vector.
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017
 **/
static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
{
	struct igb_q_vector *q_vector = adapter->q_vector[v_idx];

	if (q_vector->tx.ring)
		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;

	if (q_vector->rx.ring)
		adapter->tx_ring[q_vector->rx.ring->queue_index] = NULL;

	adapter->q_vector[v_idx] = NULL;
	netif_napi_del(&q_vector->napi);

1018
	/* igb_get_stats64() might access the rings on this vector,
1019 1020 1021 1022 1023
	 * we must wait a grace period before freeing it.
	 */
	kfree_rcu(q_vector, rcu);
}

1024
/**
1025 1026
 *  igb_free_q_vectors - Free memory allocated for interrupt vectors
 *  @adapter: board private structure to initialize
1027
 *
1028 1029 1030
 *  This function frees the memory allocated to the q_vectors.  In addition if
 *  NAPI is enabled it will delete any references to the NAPI struct prior
 *  to freeing the q_vector.
1031 1032 1033
 **/
static void igb_free_q_vectors(struct igb_adapter *adapter)
{
1034 1035 1036 1037
	int v_idx = adapter->num_q_vectors;

	adapter->num_tx_queues = 0;
	adapter->num_rx_queues = 0;
1038
	adapter->num_q_vectors = 0;
1039 1040 1041

	while (v_idx--)
		igb_free_q_vector(adapter, v_idx);
1042 1043 1044
}

/**
1045 1046
 *  igb_clear_interrupt_scheme - reset the device to a state of no interrupts
 *  @adapter: board private structure to initialize
1047
 *
1048 1049
 *  This function resets the device so that it has 0 Rx queues, Tx queues, and
 *  MSI-X interrupts allocated.
1050 1051 1052 1053 1054 1055
 */
static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
{
	igb_free_q_vectors(adapter);
	igb_reset_interrupt_capability(adapter);
}
1056 1057

/**
1058 1059 1060
 *  igb_set_interrupt_capability - set MSI or MSI-X if supported
 *  @adapter: board private structure to initialize
 *  @msix: boolean value of MSIX capability
1061
 *
1062 1063
 *  Attempt to configure interrupts using the best available
 *  capabilities of the hardware and kernel.
1064
 **/
1065
static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
1066 1067 1068 1069
{
	int err;
	int numvecs, i;

1070 1071 1072
	if (!msix)
		goto msi_only;

1073
	/* Number of supported queues. */
1074
	adapter->num_rx_queues = adapter->rss_queues;
1075 1076 1077 1078
	if (adapter->vfs_allocated_count)
		adapter->num_tx_queues = 1;
	else
		adapter->num_tx_queues = adapter->rss_queues;
1079

1080
	/* start with one vector for every Rx queue */
1081 1082
	numvecs = adapter->num_rx_queues;

1083
	/* if Tx handler is separate add 1 for every Tx queue */
1084 1085
	if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
		numvecs += adapter->num_tx_queues;
1086 1087 1088 1089 1090 1091

	/* store the number of vectors reserved for queues */
	adapter->num_q_vectors = numvecs;

	/* add 1 vector for link status interrupts */
	numvecs++;
1092 1093
	adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
					GFP_KERNEL);
1094

1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
	if (!adapter->msix_entries)
		goto msi_only;

	for (i = 0; i < numvecs; i++)
		adapter->msix_entries[i].entry = i;

	err = pci_enable_msix(adapter->pdev,
			      adapter->msix_entries,
			      numvecs);
	if (err == 0)
1105
		return;
1106 1107 1108 1109 1110

	igb_reset_interrupt_capability(adapter);

	/* If we can't do MSI-X, try MSI */
msi_only:
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
#ifdef CONFIG_PCI_IOV
	/* disable SR-IOV for non MSI-X configurations */
	if (adapter->vf_data) {
		struct e1000_hw *hw = &adapter->hw;
		/* disable iov and allow time for transactions to clear */
		pci_disable_sriov(adapter->pdev);
		msleep(500);

		kfree(adapter->vf_data);
		adapter->vf_data = NULL;
		wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1122
		wrfl();
1123 1124 1125 1126
		msleep(100);
		dev_info(&adapter->pdev->dev, "IOV Disabled\n");
	}
#endif
1127
	adapter->vfs_allocated_count = 0;
1128
	adapter->rss_queues = 1;
1129
	adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1130
	adapter->num_rx_queues = 1;
1131
	adapter->num_tx_queues = 1;
1132
	adapter->num_q_vectors = 1;
1133
	if (!pci_enable_msi(adapter->pdev))
1134
		adapter->flags |= IGB_FLAG_HAS_MSI;
1135 1136
}

1137 1138 1139 1140 1141 1142 1143
static void igb_add_ring(struct igb_ring *ring,
			 struct igb_ring_container *head)
{
	head->ring = ring;
	head->count++;
}

1144
/**
1145 1146 1147 1148 1149 1150 1151 1152
 *  igb_alloc_q_vector - Allocate memory for a single interrupt vector
 *  @adapter: board private structure to initialize
 *  @v_count: q_vectors allocated on adapter, used for ring interleaving
 *  @v_idx: index of vector in adapter struct
 *  @txr_count: total number of Tx rings to allocate
 *  @txr_idx: index of first Tx ring to allocate
 *  @rxr_count: total number of Rx rings to allocate
 *  @rxr_idx: index of first Rx ring to allocate
1153
 *
1154
 *  We allocate one q_vector.  If allocation fails we return -ENOMEM.
1155
 **/
1156 1157 1158 1159
static int igb_alloc_q_vector(struct igb_adapter *adapter,
			      int v_count, int v_idx,
			      int txr_count, int txr_idx,
			      int rxr_count, int rxr_idx)
1160 1161
{
	struct igb_q_vector *q_vector;
1162 1163
	struct igb_ring *ring;
	int ring_count, size;
1164

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
	/* igb only supports 1 Tx and/or 1 Rx queue per vector */
	if (txr_count > 1 || rxr_count > 1)
		return -ENOMEM;

	ring_count = txr_count + rxr_count;
	size = sizeof(struct igb_q_vector) +
	       (sizeof(struct igb_ring) * ring_count);

	/* allocate q_vector and rings */
	q_vector = kzalloc(size, GFP_KERNEL);
	if (!q_vector)
		return -ENOMEM;

	/* initialize NAPI */
	netif_napi_add(adapter->netdev, &q_vector->napi,
		       igb_poll, 64);

	/* tie q_vector and adapter together */
	adapter->q_vector[v_idx] = q_vector;
	q_vector->adapter = adapter;

	/* initialize work limits */
	q_vector->tx.work_limit = adapter->tx_work_limit;

	/* initialize ITR configuration */
	q_vector->itr_register = adapter->hw.hw_addr + E1000_EITR(0);
	q_vector->itr_val = IGB_START_ITR;

	/* initialize pointer to rings */
	ring = q_vector->ring;

1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
	/* intialize ITR */
	if (rxr_count) {
		/* rx or rx/tx vector */
		if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
			q_vector->itr_val = adapter->rx_itr_setting;
	} else {
		/* tx only vector */
		if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
			q_vector->itr_val = adapter->tx_itr_setting;
	}

1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
	if (txr_count) {
		/* assign generic ring traits */
		ring->dev = &adapter->pdev->dev;
		ring->netdev = adapter->netdev;

		/* configure backlink on ring */
		ring->q_vector = q_vector;

		/* update q_vector Tx values */
		igb_add_ring(ring, &q_vector->tx);

		/* For 82575, context index must be unique per ring. */
		if (adapter->hw.mac.type == e1000_82575)
			set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);

		/* apply Tx specific ring traits */
		ring->count = adapter->tx_ring_count;
		ring->queue_index = txr_idx;

		/* assign ring to adapter */
		adapter->tx_ring[txr_idx] = ring;

		/* push pointer to next ring */
		ring++;
1231
	}
1232

1233 1234 1235 1236
	if (rxr_count) {
		/* assign generic ring traits */
		ring->dev = &adapter->pdev->dev;
		ring->netdev = adapter->netdev;
1237

1238 1239
		/* configure backlink on ring */
		ring->q_vector = q_vector;
1240

1241 1242
		/* update q_vector Rx values */
		igb_add_ring(ring, &q_vector->rx);
1243

1244 1245 1246
		/* set flag indicating ring supports SCTP checksum offload */
		if (adapter->hw.mac.type >= e1000_82576)
			set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
1247

1248 1249
		/*
		 * On i350, i354, i210, and i211, loopback VLAN packets
1250
		 * have the tag byte-swapped.
1251
		 */
1252 1253
		if (adapter->hw.mac.type >= e1000_i350)
			set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
1254

1255 1256 1257 1258 1259 1260 1261 1262 1263
		/* apply Rx specific ring traits */
		ring->count = adapter->rx_ring_count;
		ring->queue_index = rxr_idx;

		/* assign ring to adapter */
		adapter->rx_ring[rxr_idx] = ring;
	}

	return 0;
1264 1265
}

1266

1267
/**
1268 1269
 *  igb_alloc_q_vectors - Allocate memory for interrupt vectors
 *  @adapter: board private structure to initialize
1270
 *
1271 1272
 *  We allocate one q_vector per queue interrupt.  If allocation fails we
 *  return -ENOMEM.
1273
 **/
1274
static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1275
{
1276 1277 1278 1279 1280
	int q_vectors = adapter->num_q_vectors;
	int rxr_remaining = adapter->num_rx_queues;
	int txr_remaining = adapter->num_tx_queues;
	int rxr_idx = 0, txr_idx = 0, v_idx = 0;
	int err;
1281

1282 1283 1284 1285
	if (q_vectors >= (rxr_remaining + txr_remaining)) {
		for (; rxr_remaining; v_idx++) {
			err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
						 0, 0, 1, rxr_idx);
1286

1287 1288 1289 1290 1291 1292
			if (err)
				goto err_out;

			/* update counts and index */
			rxr_remaining--;
			rxr_idx++;
1293 1294
		}
	}
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311

	for (; v_idx < q_vectors; v_idx++) {
		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
		err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
					 tqpv, txr_idx, rqpv, rxr_idx);

		if (err)
			goto err_out;

		/* update counts and index */
		rxr_remaining -= rqpv;
		txr_remaining -= tqpv;
		rxr_idx++;
		txr_idx++;
	}

1312
	return 0;
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322

err_out:
	adapter->num_tx_queues = 0;
	adapter->num_rx_queues = 0;
	adapter->num_q_vectors = 0;

	while (v_idx--)
		igb_free_q_vector(adapter, v_idx);

	return -ENOMEM;
1323 1324 1325
}

/**
1326 1327 1328
 *  igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
 *  @adapter: board private structure to initialize
 *  @msix: boolean value of MSIX capability
1329
 *
1330
 *  This function initializes the interrupts and allocates all of the queues.
1331
 **/
1332
static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
1333 1334 1335 1336
{
	struct pci_dev *pdev = adapter->pdev;
	int err;

1337
	igb_set_interrupt_capability(adapter, msix);
1338 1339 1340 1341 1342 1343 1344

	err = igb_alloc_q_vectors(adapter);
	if (err) {
		dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
		goto err_alloc_q_vectors;
	}

1345
	igb_cache_ring_register(adapter);
1346 1347

	return 0;
1348

1349 1350 1351 1352 1353
err_alloc_q_vectors:
	igb_reset_interrupt_capability(adapter);
	return err;
}

1354
/**
1355 1356
 *  igb_request_irq - initialize interrupts
 *  @adapter: board private structure to initialize
1357
 *
1358 1359
 *  Attempts to configure interrupts using the best available
 *  capabilities of the hardware and kernel.
1360 1361 1362 1363
 **/
static int igb_request_irq(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
1364
	struct pci_dev *pdev = adapter->pdev;
1365 1366 1367 1368
	int err = 0;

	if (adapter->msix_entries) {
		err = igb_request_msix(adapter);
1369
		if (!err)
1370 1371
			goto request_done;
		/* fall back to MSI */
1372 1373
		igb_free_all_tx_resources(adapter);
		igb_free_all_rx_resources(adapter);
1374

1375
		igb_clear_interrupt_scheme(adapter);
1376 1377
		err = igb_init_interrupt_scheme(adapter, false);
		if (err)
1378
			goto request_done;
1379

1380 1381
		igb_setup_all_tx_resources(adapter);
		igb_setup_all_rx_resources(adapter);
1382
		igb_configure(adapter);
1383
	}
1384

1385 1386
	igb_assign_vector(adapter->q_vector[0], 0);

1387
	if (adapter->flags & IGB_FLAG_HAS_MSI) {
1388
		err = request_irq(pdev->irq, igb_intr_msi, 0,
1389
				  netdev->name, adapter);
1390 1391
		if (!err)
			goto request_done;
1392

1393 1394
		/* fall back to legacy interrupts */
		igb_reset_interrupt_capability(adapter);
1395
		adapter->flags &= ~IGB_FLAG_HAS_MSI;
1396 1397
	}

1398
	err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1399
			  netdev->name, adapter);
1400

1401
	if (err)
1402
		dev_err(&pdev->dev, "Error %d getting interrupt\n",
1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413
			err);

request_done:
	return err;
}

static void igb_free_irq(struct igb_adapter *adapter)
{
	if (adapter->msix_entries) {
		int vector = 0, i;

1414
		free_irq(adapter->msix_entries[vector++].vector, adapter);
1415

1416
		for (i = 0; i < adapter->num_q_vectors; i++)
1417
			free_irq(adapter->msix_entries[vector++].vector,
1418
				 adapter->q_vector[i]);
1419 1420
	} else {
		free_irq(adapter->pdev->irq, adapter);
1421 1422 1423 1424
	}
}

/**
1425 1426
 *  igb_irq_disable - Mask off interrupt generation on the NIC
 *  @adapter: board private structure
1427 1428 1429 1430 1431
 **/
static void igb_irq_disable(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;

1432
	/* we need to be careful when disabling interrupts.  The VFs are also
1433 1434 1435
	 * mapped into these registers and so clearing the bits can cause
	 * issues on the VF drivers so we only need to clear what we set
	 */
1436
	if (adapter->msix_entries) {
1437 1438 1439 1440 1441
		u32 regval = rd32(E1000_EIAM);
		wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
		wr32(E1000_EIMC, adapter->eims_enable_mask);
		regval = rd32(E1000_EIAC);
		wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1442
	}
1443 1444

	wr32(E1000_IAM, 0);
1445 1446
	wr32(E1000_IMC, ~0);
	wrfl();
1447 1448 1449 1450 1451 1452 1453
	if (adapter->msix_entries) {
		int i;
		for (i = 0; i < adapter->num_q_vectors; i++)
			synchronize_irq(adapter->msix_entries[i].vector);
	} else {
		synchronize_irq(adapter->pdev->irq);
	}
1454 1455 1456
}

/**
1457 1458
 *  igb_irq_enable - Enable default interrupt generation settings
 *  @adapter: board private structure
1459 1460 1461 1462 1463 1464
 **/
static void igb_irq_enable(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;

	if (adapter->msix_entries) {
1465
		u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1466 1467 1468 1469
		u32 regval = rd32(E1000_EIAC);
		wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
		regval = rd32(E1000_EIAM);
		wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1470
		wr32(E1000_EIMS, adapter->eims_enable_mask);
1471
		if (adapter->vfs_allocated_count) {
1472
			wr32(E1000_MBVFIMR, 0xFF);
1473 1474 1475
			ims |= E1000_IMS_VMMB;
		}
		wr32(E1000_IMS, ims);
1476
	} else {
1477 1478 1479 1480
		wr32(E1000_IMS, IMS_ENABLE_MASK |
				E1000_IMS_DRSTA);
		wr32(E1000_IAM, IMS_ENABLE_MASK |
				E1000_IMS_DRSTA);
1481
	}
1482 1483 1484 1485
}

static void igb_update_mng_vlan(struct igb_adapter *adapter)
{
1486
	struct e1000_hw *hw = &adapter->hw;
1487 1488
	u16 vid = adapter->hw.mng_cookie.vlan_id;
	u16 old_vid = adapter->mng_vlan_id;
1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499

	if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
		/* add VID to filter table */
		igb_vfta_set(hw, vid, true);
		adapter->mng_vlan_id = vid;
	} else {
		adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
	}

	if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
	    (vid != old_vid) &&
Jiri Pirko's avatar
Jiri Pirko committed
1500
	    !test_bit(old_vid, adapter->active_vlans)) {
1501 1502
		/* remove VID from filter table */
		igb_vfta_set(hw, old_vid, false);
1503 1504 1505 1506
	}
}

/**
1507 1508
 *  igb_release_hw_control - release control of the h/w to f/w
 *  @adapter: address of board private structure
1509
 *
1510 1511 1512
 *  igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 *  For ASF and Pass Through versions of f/w this means that the
 *  driver is no longer loaded.
1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525
 **/
static void igb_release_hw_control(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 ctrl_ext;

	/* Let firmware take over control of h/w */
	ctrl_ext = rd32(E1000_CTRL_EXT);
	wr32(E1000_CTRL_EXT,
			ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
}

/**
1526 1527
 *  igb_get_hw_control - get control of the h/w from f/w
 *  @adapter: address of board private structure
1528
 *
1529 1530 1531
 *  igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 *  For ASF and Pass Through versions of f/w this means that
 *  the driver is loaded.
1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
 **/
static void igb_get_hw_control(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 ctrl_ext;

	/* Let firmware know the driver has taken over */
	ctrl_ext = rd32(E1000_CTRL_EXT);
	wr32(E1000_CTRL_EXT,
			ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
}

/**
1545 1546
 *  igb_configure - configure the hardware for RX and TX
 *  @adapter: private board structure
1547 1548 1549 1550 1551 1552 1553
 **/
static void igb_configure(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
	int i;

	igb_get_hw_control(adapter);
1554
	igb_set_rx_mode(netdev);
1555 1556 1557

	igb_restore_vlan(adapter);

1558
	igb_setup_tctl(adapter);
1559
	igb_setup_mrqc(adapter);
1560
	igb_setup_rctl(adapter);
1561 1562

	igb_configure_tx(adapter);
1563
	igb_configure_rx(adapter);
1564 1565 1566

	igb_rx_fifo_flush_82575(&adapter->hw);

1567
	/* call igb_desc_unused which always leaves
1568
	 * at least 1 descriptor unused to make sure
1569 1570
	 * next_to_use != next_to_clean
	 */
1571
	for (i = 0; i < adapter->num_rx_queues; i++) {
1572
		struct igb_ring *ring = adapter->rx_ring[i];
1573
		igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1574 1575 1576
	}
}

1577
/**
1578 1579
 *  igb_power_up_link - Power up the phy/serdes link
 *  @adapter: address of board private structure
1580 1581 1582
 **/
void igb_power_up_link(struct igb_adapter *adapter)
{
1583 1584
	igb_reset_phy(&adapter->hw);

1585 1586 1587 1588 1589 1590 1591
	if (adapter->hw.phy.media_type == e1000_media_type_copper)
		igb_power_up_phy_copper(&adapter->hw);
	else
		igb_power_up_serdes_link_82575(&adapter->hw);
}

/**
1592 1593
 *  igb_power_down_link - Power down the phy/serdes link
 *  @adapter: address of board private structure
1594 1595 1596 1597 1598 1599 1600 1601
 */
static void igb_power_down_link(struct igb_adapter *adapter)
{
	if (adapter->hw.phy.media_type == e1000_media_type_copper)
		igb_power_down_phy_copper_82575(&adapter->hw);
	else
		igb_shutdown_serdes_link_82575(&adapter->hw);
}
1602 1603

/**
1604 1605
 *  igb_up - Open the interface and prepare it to handle traffic
 *  @adapter: board private structure
1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
 **/
int igb_up(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	int i;

	/* hardware has been reset, we need to reload some things */
	igb_configure(adapter);

	clear_bit(__IGB_DOWN, &adapter->state);

1617 1618 1619
	for (i = 0; i < adapter->num_q_vectors; i++)
		napi_enable(&(adapter->q_vector[i]->napi));

1620
	if (adapter->msix_entries)
1621
		igb_configure_msix(adapter);
1622 1623
	else
		igb_assign_vector(adapter->q_vector[0], 0);
1624 1625 1626 1627 1628

	/* Clear any pending interrupts. */
	rd32(E1000_ICR);
	igb_irq_enable(adapter);

1629 1630 1631 1632 1633 1634 1635
	/* notify VFs that reset has been completed */
	if (adapter->vfs_allocated_count) {
		u32 reg_data = rd32(E1000_CTRL_EXT);
		reg_data |= E1000_CTRL_EXT_PFRSTD;
		wr32(E1000_CTRL_EXT, reg_data);
	}

1636 1637
	netif_tx_start_all_queues(adapter->netdev);

1638 1639 1640 1641
	/* start the watchdog. */
	hw->mac.get_link_status = 1;
	schedule_work(&adapter->watchdog_task);

1642 1643 1644 1645 1646 1647
	return 0;
}

void igb_down(struct igb_adapter *adapter)
{
	struct net_device *netdev = adapter->netdev;
1648
	struct e1000_hw *hw = &adapter->hw;
1649 1650 1651 1652
	u32 tctl, rctl;
	int i;

	/* signal that we're down so the interrupt handler does not
1653 1654
	 * reschedule our watchdog timer
	 */
1655 1656 1657 1658 1659 1660 1661
	set_bit(__IGB_DOWN, &adapter->state);

	/* disable receives in the hardware */
	rctl = rd32(E1000_RCTL);
	wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
	/* flush and sleep below */

1662
	netif_tx_stop_all_queues(netdev);
1663 1664 1665 1666 1667 1668 1669 1670 1671

	/* disable transmits in the hardware */
	tctl = rd32(E1000_TCTL);
	tctl &= ~E1000_TCTL_EN;
	wr32(E1000_TCTL, tctl);
	/* flush both disables and wait for them to finish */
	wrfl();
	msleep(10);

1672 1673
	igb_irq_disable(adapter);

1674 1675
	adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;

1676 1677
	for (i = 0; i < adapter->num_q_vectors; i++) {
		napi_synchronize(&(adapter->q_vector[i]->napi));
1678
		napi_disable(&(adapter->q_vector[i]->napi));
1679
	}
1680 1681 1682 1683 1684 1685


	del_timer_sync(&adapter->watchdog_timer);
	del_timer_sync(&adapter->phy_info_timer);

	netif_carrier_off(netdev);
1686 1687

	/* record the stats before reset*/
Eric Dumazet's avatar
Eric Dumazet committed
1688 1689 1690
	spin_lock(&adapter->stats64_lock);
	igb_update_stats(adapter, &adapter->stats64);
	spin_unlock(&adapter->stats64_lock);
1691

1692 1693 1694
	adapter->link_speed = 0;
	adapter->link_duplex = 0;

1695 1696
	if (!pci_channel_offline(adapter->pdev))
		igb_reset(adapter);
1697 1698
	igb_clean_all_tx_rings(adapter);
	igb_clean_all_rx_rings(adapter);
1699 1700 1701 1702 1703
#ifdef CONFIG_IGB_DCA

	/* since we reset the hardware DCA settings were cleared */
	igb_setup_dca(adapter);
#endif
1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
}

void igb_reinit_locked(struct igb_adapter *adapter)
{
	WARN_ON(in_interrupt());
	while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
		msleep(1);
	igb_down(adapter);
	igb_up(adapter);
	clear_bit(__IGB_RESETTING, &adapter->state);
}

void igb_reset(struct igb_adapter *adapter)
{
1718
	struct pci_dev *pdev = adapter->pdev;
1719
	struct e1000_hw *hw = &adapter->hw;
Alexander Duyck's avatar
Alexander Duyck committed
1720 1721
	struct e1000_mac_info *mac = &hw->mac;
	struct e1000_fc_info *fc = &hw->fc;
1722
	u32 pba = 0, tx_space, min_tx_space, min_rx_space, hwm;
1723 1724 1725 1726

	/* Repartition Pba for greater than 9k mtu
	 * To take effect CTRL.RST is required.
	 */
1727
	switch (mac->type) {
1728
	case e1000_i350:
1729
	case e1000_i354:
1730 1731 1732 1733
	case e1000_82580:
		pba = rd32(E1000_RXPBS);
		pba = igb_rxpbs_adjust_82580(pba);
		break;
1734
	case e1000_82576:
1735 1736
		pba = rd32(E1000_RXPBS);
		pba &= E1000_RXPBS_SIZE_MASK_82576;
1737 1738
		break;
	case e1000_82575:
1739 1740
	case e1000_i210:
	case e1000_i211:
1741 1742 1743
	default:
		pba = E1000_PBA_34K;
		break;
Alexander Duyck's avatar
Alexander Duyck committed
1744
	}
1745

Alexander Duyck's avatar
Alexander Duyck committed
1746 1747
	if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
	    (mac->type < e1000_82576)) {
1748 1749 1750 1751 1752 1753 1754 1755
		/* adjust PBA for jumbo frames */
		wr32(E1000_PBA, pba);

		/* To maintain wire speed transmits, the Tx FIFO should be
		 * large enough to accommodate two full transmit packets,
		 * rounded up to the next 1KB and expressed in KB.  Likewise,
		 * the Rx FIFO should be large enough to accommodate at least
		 * one full receive packet and is similarly rounded up and
1756 1757
		 * expressed in KB.
		 */
1758 1759 1760 1761 1762
		pba = rd32(E1000_PBA);
		/* upper 16 bits has Tx packet buffer allocation size in KB */
		tx_space = pba >> 16;
		/* lower 16 bits has Rx packet buffer allocation size in KB */
		pba &= 0xffff;
1763 1764 1765
		/* the Tx fifo also stores 16 bytes of information about the Tx
		 * but don't include ethernet FCS because hardware appends it
		 */
1766
		min_tx_space = (adapter->max_frame_size +
1767
				sizeof(union e1000_adv_tx_desc) -
1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
				ETH_FCS_LEN) * 2;
		min_tx_space = ALIGN(min_tx_space, 1024);
		min_tx_space >>= 10;
		/* software strips receive CRC, so leave room for it */
		min_rx_space = adapter->max_frame_size;
		min_rx_space = ALIGN(min_rx_space, 1024);
		min_rx_space >>= 10;

		/* If current Tx allocation is less than the min Tx FIFO size,
		 * and the min Tx FIFO size is less than the current Rx FIFO
1778 1779
		 * allocation, take space away from current Rx allocation
		 */
1780 1781 1782 1783
		if (tx_space < min_tx_space &&
		    ((min_tx_space - tx_space) < pba)) {
			pba = pba - (min_tx_space - tx_space);

1784 1785 1786
			/* if short on Rx space, Rx wins and must trump Tx
			 * adjustment
			 */
1787 1788 1789
			if (pba < min_rx_space)
				pba = min_rx_space;
		}
Alexander Duyck's avatar
Alexander Duyck committed
1790
		wr32(E1000_PBA, pba);
1791 1792 1793 1794 1795 1796 1797
	}

	/* flow control settings */
	/* The high water mark must be low enough to fit one full frame
	 * (or the size used for early receive) above it in the Rx FIFO.
	 * Set it to the lower of:
	 * - 90% of the Rx FIFO size, or
1798 1799
	 * - the full Rx FIFO size minus one full frame
	 */
1800
	hwm = min(((pba << 10) * 9 / 10),
Alexander Duyck's avatar
Alexander Duyck committed
1801
			((pba << 10) - 2 * adapter->max_frame_size));
1802

1803
	fc->high_water = hwm & 0xFFFFFFF0;	/* 16-byte granularity */
1804
	fc->low_water = fc->high_water - 16;
1805 1806
	fc->pause_time = 0xFFFF;
	fc->send_xon = 1;
1807
	fc->current_mode = fc->requested_mode;
1808

1809 1810 1811 1812
	/* disable receive for all VFs and wait one second */
	if (adapter->vfs_allocated_count) {
		int i;
		for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1813
			adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1814 1815

		/* ping all the active vfs to let them know we are going down */
1816
		igb_ping_all_vfs(adapter);
1817 1818 1819 1820 1821 1822

		/* disable transmits and receives */
		wr32(E1000_VFRE, 0);
		wr32(E1000_VFTE, 0);
	}

1823
	/* Allow time for pending master requests to run */
1824
	hw->mac.ops.reset_hw(hw);
1825 1826
	wr32(E1000_WUC, 0);

1827
	if (hw->mac.ops.init_hw(hw))
1828
		dev_err(&pdev->dev, "Hardware Error\n");
1829

1830
	/* Flow control settings reset on hardware reset, so guarantee flow
1831 1832 1833 1834 1835
	 * control is off when forcing speed.
	 */
	if (!hw->mac.autoneg)
		igb_force_mac_fc(hw);

1836
	igb_init_dmac(adapter, pba);
1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848
#ifdef CONFIG_IGB_HWMON
	/* Re-initialize the thermal sensor on i350 devices. */
	if (!test_bit(__IGB_DOWN, &adapter->state)) {
		if (mac->type == e1000_i350 && hw->bus.func == 0) {
			/* If present, re-initialize the external thermal sensor
			 * interface.
			 */
			if (adapter->ets)
				mac->ops.init_thermal_sensor_thresh(hw);
		}
	}
#endif
1849 1850 1851
	if (!netif_running(adapter->netdev))
		igb_power_down_link(adapter);

1852 1853 1854 1855 1856
	igb_update_mng_vlan(adapter);

	/* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
	wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);

1857 1858 1859
	/* Re-enable PTP, where applicable. */
	igb_ptp_reset(adapter);

1860
	igb_get_phy_info(hw);
1861 1862
}

1863 1864
static netdev_features_t igb_fix_features(struct net_device *netdev,
	netdev_features_t features)
Jiri Pirko's avatar
Jiri Pirko committed
1865
{
1866 1867
	/* Since there is no support for separate Rx/Tx vlan accel
	 * enable/disable make sure Tx flag is always in same state as Rx.
Jiri Pirko's avatar
Jiri Pirko committed
1868
	 */
1869 1870
	if (features & NETIF_F_HW_VLAN_CTAG_RX)
		features |= NETIF_F_HW_VLAN_CTAG_TX;
Jiri Pirko's avatar
Jiri Pirko committed
1871
	else
1872
		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
Jiri Pirko's avatar
Jiri Pirko committed
1873 1874 1875 1876

	return features;
}

1877 1878
static int igb_set_features(struct net_device *netdev,
	netdev_features_t features)
1879
{
1880
	netdev_features_t changed = netdev->features ^ features;
1881
	struct igb_adapter *adapter = netdev_priv(netdev);
1882

1883
	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
Jiri Pirko's avatar
Jiri Pirko committed
1884 1885
		igb_vlan_mode(netdev, features);

1886 1887 1888 1889 1890 1891 1892 1893 1894 1895
	if (!(changed & NETIF_F_RXALL))
		return 0;

	netdev->features = features;

	if (netif_running(netdev))
		igb_reinit_locked(adapter);
	else
		igb_reset(adapter);

1896 1897 1898
	return 0;
}

1899
static const struct net_device_ops igb_netdev_ops = {
1900
	.ndo_open		= igb_open,
1901
	.ndo_stop		= igb_close,
1902
	.ndo_start_xmit		= igb_xmit_frame,
Eric Dumazet's avatar
Eric Dumazet committed
1903
	.ndo_get_stats64	= igb_get_stats64,
1904
	.ndo_set_rx_mode	= igb_set_rx_mode,
1905 1906 1907 1908 1909 1910 1911
	.ndo_set_mac_address	= igb_set_mac,
	.ndo_change_mtu		= igb_change_mtu,
	.ndo_do_ioctl		= igb_ioctl,
	.ndo_tx_timeout		= igb_tx_timeout,
	.ndo_validate_addr	= eth_validate_addr,
	.ndo_vlan_rx_add_vid	= igb_vlan_rx_add_vid,
	.ndo_vlan_rx_kill_vid	= igb_vlan_rx_kill_vid,
1912 1913 1914
	.ndo_set_vf_mac		= igb_ndo_set_vf_mac,
	.ndo_set_vf_vlan	= igb_ndo_set_vf_vlan,
	.ndo_set_vf_tx_rate	= igb_ndo_set_vf_bw,
1915
	.ndo_set_vf_spoofchk	= igb_ndo_set_vf_spoofchk,
1916
	.ndo_get_vf_config	= igb_ndo_get_vf_config,
1917 1918 1919
#ifdef CONFIG_NET_POLL_CONTROLLER
	.ndo_poll_controller	= igb_netpoll,
#endif
Jiri Pirko's avatar
Jiri Pirko committed
1920 1921
	.ndo_fix_features	= igb_fix_features,
	.ndo_set_features	= igb_set_features,
1922 1923
};

1924 1925 1926 1927 1928 1929 1930
/**
 * igb_set_fw_version - Configure version string for ethtool
 * @adapter: adapter struct
 **/
void igb_set_fw_version(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
1931 1932 1933 1934 1935
	struct e1000_fw_version fw;

	igb_get_fw_version(hw, &fw);

	switch (hw->mac.type) {
1936
	case e1000_i210:
1937
	case e1000_i211:
1938 1939 1940 1941 1942 1943 1944 1945 1946
		if (!(igb_get_flash_presence_i210(hw))) {
			snprintf(adapter->fw_version,
				 sizeof(adapter->fw_version),
				 "%2d.%2d-%d",
				 fw.invm_major, fw.invm_minor,
				 fw.invm_img_type);
			break;
		}
		/* fall through */
1947 1948 1949 1950 1951 1952 1953 1954 1955
	default:
		/* if option is rom valid, display its version too */
		if (fw.or_valid) {
			snprintf(adapter->fw_version,
				 sizeof(adapter->fw_version),
				 "%d.%d, 0x%08x, %d.%d.%d",
				 fw.eep_major, fw.eep_minor, fw.etrack_id,
				 fw.or_major, fw.or_build, fw.or_patch);
		/* no option rom */
1956
		} else if (fw.etrack_id != 0X0000) {
1957
			snprintf(adapter->fw_version,
1958 1959 1960 1961 1962 1963 1964 1965
			    sizeof(adapter->fw_version),
			    "%d.%d, 0x%08x",
			    fw.eep_major, fw.eep_minor, fw.etrack_id);
		} else {
		snprintf(adapter->fw_version,
		    sizeof(adapter->fw_version),
		    "%d.%d.%d",
		    fw.eep_major, fw.eep_minor, fw.eep_build);
1966 1967
		}
		break;
1968 1969 1970 1971
	}
	return;
}

1972 1973
/**
 *  igb_init_i2c - Init I2C interface
1974
 *  @adapter: pointer to adapter structure
1975
 **/
1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998
static s32 igb_init_i2c(struct igb_adapter *adapter)
{
	s32 status = E1000_SUCCESS;

	/* I2C interface supported on i350 devices */
	if (adapter->hw.mac.type != e1000_i350)
		return E1000_SUCCESS;

	/* Initialize the i2c bus which is controlled by the registers.
	 * This bus will use the i2c_algo_bit structue that implements
	 * the protocol through toggling of the 4 bits in the register.
	 */
	adapter->i2c_adap.owner = THIS_MODULE;
	adapter->i2c_algo = igb_i2c_algo;
	adapter->i2c_algo.data = adapter;
	adapter->i2c_adap.algo_data = &adapter->i2c_algo;
	adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
	strlcpy(adapter->i2c_adap.name, "igb BB",
		sizeof(adapter->i2c_adap.name));
	status = i2c_bit_add_bus(&adapter->i2c_adap);
	return status;
}

1999
/**
2000 2001 2002
 *  igb_probe - Device Initialization Routine
 *  @pdev: PCI device information struct
 *  @ent: entry in igb_pci_tbl
2003
 *
2004
 *  Returns 0 on success, negative on failure
2005
 *
2006 2007 2008
 *  igb_probe initializes an adapter identified by a pci_dev structure.
 *  The OS initialization, configuring of the adapter private structure,
 *  and a hardware reset occur.
2009
 **/
2010
static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2011 2012 2013 2014
{
	struct net_device *netdev;
	struct igb_adapter *adapter;
	struct e1000_hw *hw;
2015
	u16 eeprom_data = 0;
2016
	s32 ret_val;
2017
	static int global_quad_port_a; /* global quad port a indication */
2018 2019
	const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
	unsigned long mmio_start, mmio_len;
2020
	int err, pci_using_dac;
2021
	u8 part_str[E1000_PBANUM_LENGTH];
2022

2023 2024 2025 2026 2027
	/* Catch broken hardware that put the wrong VF device ID in
	 * the PCIe SR-IOV capability.
	 */
	if (pdev->is_virtfn) {
		WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
2028
			pci_name(pdev), pdev->vendor, pdev->device);
2029 2030 2031
		return -EINVAL;
	}

2032
	err = pci_enable_device_mem(pdev);
2033 2034 2035 2036
	if (err)
		return err;

	pci_using_dac = 0;
2037
	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2038
	if (!err) {
2039
		pci_using_dac = 1;
2040
	} else {
2041
		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2042
		if (err) {
2043 2044 2045
			dev_err(&pdev->dev,
				"No usable DMA configuration, aborting\n");
			goto err_dma;
2046 2047 2048
		}
	}

2049
	err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
2050 2051
					   IORESOURCE_MEM),
					   igb_driver_name);
2052 2053 2054
	if (err)
		goto err_pci_reg;

2055
	pci_enable_pcie_error_reporting(pdev);
2056

2057
	pci_set_master(pdev);
2058
	pci_save_state(pdev);
2059 2060

	err = -ENOMEM;
2061
	netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
2062
				   IGB_MAX_TX_QUEUES);
2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073
	if (!netdev)
		goto err_alloc_etherdev;

	SET_NETDEV_DEV(netdev, &pdev->dev);

	pci_set_drvdata(pdev, netdev);
	adapter = netdev_priv(netdev);
	adapter->netdev = netdev;
	adapter->pdev = pdev;
	hw = &adapter->hw;
	hw->back = adapter;
2074
	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2075 2076 2077 2078 2079

	mmio_start = pci_resource_start(pdev, 0);
	mmio_len = pci_resource_len(pdev, 0);

	err = -EIO;
2080 2081
	hw->hw_addr = ioremap(mmio_start, mmio_len);
	if (!hw->hw_addr)
2082 2083
		goto err_ioremap;

2084
	netdev->netdev_ops = &igb_netdev_ops;
2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106
	igb_set_ethtool_ops(netdev);
	netdev->watchdog_timeo = 5 * HZ;

	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);

	netdev->mem_start = mmio_start;
	netdev->mem_end = mmio_start + mmio_len;

	/* PCI config space info */
	hw->vendor_id = pdev->vendor;
	hw->device_id = pdev->device;
	hw->revision_id = pdev->revision;
	hw->subsystem_vendor_id = pdev->subsystem_vendor;
	hw->subsystem_device_id = pdev->subsystem_device;

	/* Copy the default MAC, PHY and NVM function pointers */
	memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
	memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
	memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
	/* Initialize skew-specific constants */
	err = ei->get_invariants(hw);
	if (err)
2107
		goto err_sw_init;
2108

2109
	/* setup the private structure */
2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
	err = igb_sw_init(adapter);
	if (err)
		goto err_sw_init;

	igb_get_bus_info_pcie(hw);

	hw->phy.autoneg_wait_to_complete = false;

	/* Copper options */
	if (hw->phy.media_type == e1000_media_type_copper) {
		hw->phy.mdix = AUTO_ALL_MODES;
		hw->phy.disable_polarity_correction = false;
		hw->phy.ms_type = e1000_ms_hw_default;
	}

	if (igb_check_reset_block(hw))
		dev_info(&pdev->dev,
			"PHY reset is blocked due to SOL/IDER session.\n");

2129
	/* features is initialized to 0 in allocation, it might have bits
2130 2131 2132 2133 2134 2135 2136 2137 2138 2139
	 * set by igb_sw_init so we should use an or instead of an
	 * assignment.
	 */
	netdev->features |= NETIF_F_SG |
			    NETIF_F_IP_CSUM |
			    NETIF_F_IPV6_CSUM |
			    NETIF_F_TSO |
			    NETIF_F_TSO6 |
			    NETIF_F_RXHASH |
			    NETIF_F_RXCSUM |
2140 2141
			    NETIF_F_HW_VLAN_CTAG_RX |
			    NETIF_F_HW_VLAN_CTAG_TX;
2142 2143 2144

	/* copy netdev features into list of user selectable features */
	netdev->hw_features |= netdev->features;
2145
	netdev->hw_features |= NETIF_F_RXALL;
2146 2147

	/* set this bit last since it cannot be part of hw_features */
2148
	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2149 2150 2151 2152 2153 2154

	netdev->vlan_features |= NETIF_F_TSO |
				 NETIF_F_TSO6 |
				 NETIF_F_IP_CSUM |
				 NETIF_F_IPV6_CSUM |
				 NETIF_F_SG;
2155

2156 2157
	netdev->priv_flags |= IFF_SUPP_NOFCS;

2158
	if (pci_using_dac) {
2159
		netdev->features |= NETIF_F_HIGHDMA;
2160 2161
		netdev->vlan_features |= NETIF_F_HIGHDMA;
	}
2162

2163 2164
	if (hw->mac.type >= e1000_82576) {
		netdev->hw_features |= NETIF_F_SCTP_CSUM;
2165
		netdev->features |= NETIF_F_SCTP_CSUM;
2166
	}
2167

2168 2169
	netdev->priv_flags |= IFF_UNICAST_FLT;

2170
	adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
2171 2172

	/* before reading the NVM, reset the controller to put the device in a
2173 2174
	 * known good starting state
	 */
2175 2176
	hw->mac.ops.reset_hw(hw);

2177 2178
	/* make sure the NVM is good , i211/i210 parts can have special NVM
	 * that doesn't contain a checksum
2179
	 */
2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192
	switch (hw->mac.type) {
	case e1000_i210:
	case e1000_i211:
		if (igb_get_flash_presence_i210(hw)) {
			if (hw->nvm.ops.validate(hw) < 0) {
				dev_err(&pdev->dev,
					"The NVM Checksum Is Not Valid\n");
				err = -EIO;
				goto err_eeprom;
			}
		}
		break;
	default:
2193 2194 2195 2196 2197
		if (hw->nvm.ops.validate(hw) < 0) {
			dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
			err = -EIO;
			goto err_eeprom;
		}
2198
		break;
2199 2200 2201 2202 2203 2204 2205 2206
	}

	/* copy the MAC address out of the NVM */
	if (hw->mac.ops.read_mac_addr(hw))
		dev_err(&pdev->dev, "NVM Read Error\n");

	memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);

2207
	if (!is_valid_ether_addr(netdev->dev_addr)) {
2208 2209 2210 2211 2212
		dev_err(&pdev->dev, "Invalid MAC Address\n");
		err = -EIO;
		goto err_eeprom;
	}

2213 2214 2215
	/* get firmware version for ethtool -i */
	igb_set_fw_version(adapter);

2216
	setup_timer(&adapter->watchdog_timer, igb_watchdog,
2217
		    (unsigned long) adapter);
2218
	setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
2219
		    (unsigned long) adapter);
2220 2221 2222 2223

	INIT_WORK(&adapter->reset_task, igb_reset_task);
	INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);

2224
	/* Initialize link properties that are user-changeable */
2225 2226 2227 2228
	adapter->fc_autoneg = true;
	hw->mac.autoneg = true;
	hw->phy.autoneg_advertised = 0x2f;

2229 2230
	hw->fc.requested_mode = e1000_fc_default;
	hw->fc.current_mode = e1000_fc_default;
2231 2232 2233

	igb_validate_mdi_setting(hw);

2234
	/* By default, support wake on port A */
2235
	if (hw->bus.func == 0)
2236 2237 2238 2239
		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;

	/* Check the NVM for wake support on non-port A ports */
	if (hw->mac.type >= e1000_82580)
2240
		hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2241 2242
				 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
				 &eeprom_data);
2243 2244
	else if (hw->bus.func == 1)
		hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2245

2246 2247
	if (eeprom_data & IGB_EEPROM_APME)
		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
2248 2249 2250

	/* now that we have the eeprom settings, apply the special cases where
	 * the eeprom may be wrong or the board simply won't support wake on
2251 2252
	 * lan on a particular port
	 */
2253 2254
	switch (pdev->device) {
	case E1000_DEV_ID_82575GB_QUAD_COPPER:
2255
		adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2256 2257
		break;
	case E1000_DEV_ID_82575EB_FIBER_SERDES:
Alexander Duyck's avatar
Alexander Duyck committed
2258 2259
	case E1000_DEV_ID_82576_FIBER:
	case E1000_DEV_ID_82576_SERDES:
2260
		/* Wake events only supported on port A for dual fiber
2261 2262
		 * regardless of eeprom setting
		 */
2263
		if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2264
			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2265
		break;
2266
	case E1000_DEV_ID_82576_QUAD_COPPER:
2267
	case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2268 2269
		/* if quad port adapter, disable WoL on all but port A */
		if (global_quad_port_a != 0)
2270
			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2271 2272 2273 2274 2275 2276
		else
			adapter->flags |= IGB_FLAG_QUAD_PORT_A;
		/* Reset for multiple quad port adapters */
		if (++global_quad_port_a == 4)
			global_quad_port_a = 0;
		break;
2277 2278 2279 2280
	default:
		/* If the device can't wake, don't set software support */
		if (!device_can_wakeup(&adapter->pdev->dev))
			adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
2281 2282 2283
	}

	/* initialize the wol settings based on the eeprom settings */
2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295
	if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
		adapter->wol |= E1000_WUFC_MAG;

	/* Some vendors want WoL disabled by default, but still supported */
	if ((hw->mac.type == e1000_i350) &&
	    (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
		adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
		adapter->wol = 0;
	}

	device_set_wakeup_enable(&adapter->pdev->dev,
				 adapter->flags & IGB_FLAG_WOL_SUPPORTED);
2296 2297 2298 2299

	/* reset the hardware with the new settings */
	igb_reset(adapter);

2300 2301 2302 2303 2304 2305 2306
	/* Init the I2C interface */
	err = igb_init_i2c(adapter);
	if (err) {
		dev_err(&pdev->dev, "failed to init i2c interface\n");
		goto err_eeprom;
	}

2307 2308 2309 2310 2311 2312 2313 2314 2315
	/* let the f/w know that the h/w is now under the control of the
	 * driver. */
	igb_get_hw_control(adapter);

	strcpy(netdev->name, "eth%d");
	err = register_netdev(netdev);
	if (err)
		goto err_register;

2316 2317 2318
	/* carrier off reporting is important to ethtool even BEFORE open */
	netif_carrier_off(netdev);

2319
#ifdef CONFIG_IGB_DCA
2320
	if (dca_add_requester(&pdev->dev) == 0) {
2321
		adapter->flags |= IGB_FLAG_DCA_ENABLED;
Jeb Cramer's avatar
Jeb Cramer committed
2322 2323 2324 2325
		dev_info(&pdev->dev, "DCA enabled\n");
		igb_setup_dca(adapter);
	}

Patrick Ohly's avatar
Patrick Ohly committed
2326
#endif
2327 2328 2329 2330
#ifdef CONFIG_IGB_HWMON
	/* Initialize the thermal sensor on i350 devices. */
	if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
		u16 ets_word;
2331

2332
		/* Read the NVM to determine if this i350 device supports an
2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346
		 * external thermal sensor.
		 */
		hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word);
		if (ets_word != 0x0000 && ets_word != 0xFFFF)
			adapter->ets = true;
		else
			adapter->ets = false;
		if (igb_sysfs_init(adapter))
			dev_err(&pdev->dev,
				"failed to allocate sysfs resources\n");
	} else {
		adapter->ets = false;
	}
#endif
Anders Berggren's avatar
Anders Berggren committed
2347
	/* do hw tstamp init after resetting */
2348
	igb_ptp_init(adapter);
Anders Berggren's avatar
Anders Berggren committed
2349

2350
	dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364
	/* print bus type/speed/width info, not applicable to i354 */
	if (hw->mac.type != e1000_i354) {
		dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
			 netdev->name,
			 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
			  (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
			   "unknown"),
			 ((hw->bus.width == e1000_bus_width_pcie_x4) ?
			  "Width x4" :
			  (hw->bus.width == e1000_bus_width_pcie_x2) ?
			  "Width x2" :
			  (hw->bus.width == e1000_bus_width_pcie_x1) ?
			  "Width x1" : "unknown"), netdev->dev_addr);
	}
2365

2366 2367 2368 2369 2370 2371 2372 2373
	if ((hw->mac.type >= e1000_i210 ||
	     igb_get_flash_presence_i210(hw))) {
		ret_val = igb_read_part_string(hw, part_str,
					       E1000_PBANUM_LENGTH);
	} else {
		ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND;
	}

2374 2375 2376
	if (ret_val)
		strcpy(part_str, "Unknown");
	dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2377 2378 2379
	dev_info(&pdev->dev,
		"Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
		adapter->msix_entries ? "MSI-X" :
2380
		(adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2381
		adapter->num_rx_queues, adapter->num_tx_queues);
2382 2383
	switch (hw->mac.type) {
	case e1000_i350:
2384 2385
	case e1000_i210:
	case e1000_i211:
2386 2387
		igb_set_eee_i350(hw);
		break;
2388 2389 2390 2391 2392 2393 2394
	case e1000_i354:
		if (hw->phy.media_type == e1000_media_type_copper) {
			if ((rd32(E1000_CTRL_EXT) &
			    E1000_CTRL_EXT_LINK_MODE_SGMII))
				igb_set_eee_i354(hw);
		}
		break;
2395 2396 2397
	default:
		break;
	}
2398 2399

	pm_runtime_put_noidle(&pdev->dev);
2400 2401 2402 2403
	return 0;

err_register:
	igb_release_hw_control(adapter);
2404
	memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
2405 2406
err_eeprom:
	if (!igb_check_reset_block(hw))
2407
		igb_reset_phy(hw);
2408 2409 2410 2411

	if (hw->flash_address)
		iounmap(hw->flash_address);
err_sw_init:
2412
	igb_clear_interrupt_scheme(adapter);
2413 2414 2415 2416
	iounmap(hw->hw_addr);
err_ioremap:
	free_netdev(netdev);
err_alloc_etherdev:
2417
	pci_release_selected_regions(pdev,
2418
				     pci_select_bars(pdev, IORESOURCE_MEM));
2419 2420 2421 2422 2423 2424
err_pci_reg:
err_dma:
	pci_disable_device(pdev);
	return err;
}

2425 2426 2427 2428 2429 2430 2431 2432 2433 2434
#ifdef CONFIG_PCI_IOV
static int  igb_disable_sriov(struct pci_dev *pdev)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;

	/* reclaim resources allocated to VFs */
	if (adapter->vf_data) {
		/* disable iov and allow time for transactions to clear */
2435
		if (pci_vfs_assigned(pdev)) {
2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466
			dev_warn(&pdev->dev,
				 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
			return -EPERM;
		} else {
			pci_disable_sriov(pdev);
			msleep(500);
		}

		kfree(adapter->vf_data);
		adapter->vf_data = NULL;
		adapter->vfs_allocated_count = 0;
		wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
		wrfl();
		msleep(100);
		dev_info(&pdev->dev, "IOV Disabled\n");

		/* Re-enable DMA Coalescing flag since IOV is turned off */
		adapter->flags |= IGB_FLAG_DMAC;
	}

	return 0;
}

static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	int old_vfs = pci_num_vf(pdev);
	int err = 0;
	int i;

2467 2468 2469 2470 2471
	if (!adapter->msix_entries) {
		err = -EPERM;
		goto out;
	}

2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522
	if (!num_vfs)
		goto out;
	else if (old_vfs && old_vfs == num_vfs)
		goto out;
	else if (old_vfs && old_vfs != num_vfs)
		err = igb_disable_sriov(pdev);

	if (err)
		goto out;

	if (num_vfs > 7) {
		err = -EPERM;
		goto out;
	}

	adapter->vfs_allocated_count = num_vfs;

	adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
				sizeof(struct vf_data_storage), GFP_KERNEL);

	/* if allocation failed then we do not support SR-IOV */
	if (!adapter->vf_data) {
		adapter->vfs_allocated_count = 0;
		dev_err(&pdev->dev,
			"Unable to allocate memory for VF Data Storage\n");
		err = -ENOMEM;
		goto out;
	}

	err = pci_enable_sriov(pdev, adapter->vfs_allocated_count);
	if (err)
		goto err_out;

	dev_info(&pdev->dev, "%d VFs allocated\n",
		 adapter->vfs_allocated_count);
	for (i = 0; i < adapter->vfs_allocated_count; i++)
		igb_vf_configure(adapter, i);

	/* DMA Coalescing is not supported in IOV mode. */
	adapter->flags &= ~IGB_FLAG_DMAC;
	goto out;

err_out:
	kfree(adapter->vf_data);
	adapter->vf_data = NULL;
	adapter->vfs_allocated_count = 0;
out:
	return err;
}

#endif
2523
/**
2524 2525
 *  igb_remove_i2c - Cleanup  I2C interface
 *  @adapter: pointer to adapter structure
2526
 **/
2527 2528 2529 2530 2531 2532
static void igb_remove_i2c(struct igb_adapter *adapter)
{
	/* free the adapter bus structure */
	i2c_del_adapter(&adapter->i2c_adap);
}

2533
/**
2534 2535
 *  igb_remove - Device Removal Routine
 *  @pdev: PCI device information struct
2536
 *
2537 2538 2539 2540
 *  igb_remove is called by the PCI subsystem to alert the driver
 *  that it should release a PCI device.  The could be caused by a
 *  Hot-Plug event, or because the driver is going to be removed from
 *  memory.
2541
 **/
2542
static void igb_remove(struct pci_dev *pdev)
2543 2544 2545
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
Jeb Cramer's avatar
Jeb Cramer committed
2546
	struct e1000_hw *hw = &adapter->hw;
2547

2548
	pm_runtime_get_noresume(&pdev->dev);
2549 2550 2551
#ifdef CONFIG_IGB_HWMON
	igb_sysfs_exit(adapter);
#endif
2552
	igb_remove_i2c(adapter);
2553
	igb_ptp_stop(adapter);
2554
	/* The watchdog timer may be rescheduled, so explicitly
2555 2556
	 * disable watchdog from being rescheduled.
	 */
2557 2558 2559 2560
	set_bit(__IGB_DOWN, &adapter->state);
	del_timer_sync(&adapter->watchdog_timer);
	del_timer_sync(&adapter->phy_info_timer);

2561 2562
	cancel_work_sync(&adapter->reset_task);
	cancel_work_sync(&adapter->watchdog_task);
2563

2564
#ifdef CONFIG_IGB_DCA
2565
	if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
Jeb Cramer's avatar
Jeb Cramer committed
2566 2567
		dev_info(&pdev->dev, "DCA disabled\n");
		dca_remove_requester(&pdev->dev);
2568
		adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
Alexander Duyck's avatar
Alexander Duyck committed
2569
		wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
Jeb Cramer's avatar
Jeb Cramer committed
2570 2571 2572
	}
#endif

2573
	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
2574 2575
	 * would have already happened in close and is redundant.
	 */
2576 2577 2578 2579
	igb_release_hw_control(adapter);

	unregister_netdev(netdev);

2580
	igb_clear_interrupt_scheme(adapter);
2581

2582
#ifdef CONFIG_PCI_IOV
2583
	igb_disable_sriov(pdev);
2584
#endif
2585

2586 2587 2588
	iounmap(hw->hw_addr);
	if (hw->flash_address)
		iounmap(hw->flash_address);
2589
	pci_release_selected_regions(pdev,
2590
				     pci_select_bars(pdev, IORESOURCE_MEM));
2591

2592
	kfree(adapter->shadow_vfta);
2593 2594
	free_netdev(netdev);

2595
	pci_disable_pcie_error_reporting(pdev);
2596

2597 2598 2599
	pci_disable_device(pdev);
}

2600
/**
2601 2602
 *  igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
 *  @adapter: board private structure to initialize
2603
 *
2604 2605 2606 2607
 *  This function initializes the vf specific data storage and then attempts to
 *  allocate the VFs.  The reason for ordering it this way is because it is much
 *  mor expensive time wise to disable SR-IOV than it is to allocate and free
 *  the memory for the VFs.
2608
 **/
2609
static void igb_probe_vfs(struct igb_adapter *adapter)
2610 2611 2612
{
#ifdef CONFIG_PCI_IOV
	struct pci_dev *pdev = adapter->pdev;
2613
	struct e1000_hw *hw = &adapter->hw;
2614

2615 2616 2617 2618
	/* Virtualization features not supported on i210 family. */
	if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
		return;

2619
	pci_sriov_set_totalvfs(pdev, 7);
2620
	igb_enable_sriov(pdev, max_vfs);
2621

2622 2623 2624
#endif /* CONFIG_PCI_IOV */
}

2625
static void igb_init_queue_configuration(struct igb_adapter *adapter)
2626 2627
{
	struct e1000_hw *hw = &adapter->hw;
2628
	u32 max_rss_queues;
2629

2630
	/* Determine the maximum number of RSS queues supported. */
2631
	switch (hw->mac.type) {
2632 2633 2634 2635
	case e1000_i211:
		max_rss_queues = IGB_MAX_RX_QUEUES_I211;
		break;
	case e1000_82575:
2636
	case e1000_i210:
2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652
		max_rss_queues = IGB_MAX_RX_QUEUES_82575;
		break;
	case e1000_i350:
		/* I350 cannot do RSS and SR-IOV at the same time */
		if (!!adapter->vfs_allocated_count) {
			max_rss_queues = 1;
			break;
		}
		/* fall through */
	case e1000_82576:
		if (!!adapter->vfs_allocated_count) {
			max_rss_queues = 2;
			break;
		}
		/* fall through */
	case e1000_82580:
2653
	case e1000_i354:
2654 2655
	default:
		max_rss_queues = IGB_MAX_RX_QUEUES;
2656
		break;
2657 2658 2659 2660 2661 2662 2663
	}

	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());

	/* Determine if we need to pair queues. */
	switch (hw->mac.type) {
	case e1000_82575:
2664
	case e1000_i211:
2665
		/* Device supports enough interrupts without queue pairing. */
2666
		break;
2667
	case e1000_82576:
2668
		/* If VFs are going to be allocated with RSS queues then we
2669 2670 2671 2672 2673 2674 2675 2676 2677
		 * should pair the queues in order to conserve interrupts due
		 * to limited supply.
		 */
		if ((adapter->rss_queues > 1) &&
		    (adapter->vfs_allocated_count > 6))
			adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
		/* fall through */
	case e1000_82580:
	case e1000_i350:
2678
	case e1000_i354:
2679
	case e1000_i210:
2680
	default:
2681
		/* If rss_queues > half of max_rss_queues, pair the queues in
2682 2683 2684 2685
		 * order to conserve interrupts due to limited supply.
		 */
		if (adapter->rss_queues > (max_rss_queues / 2))
			adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2686 2687
		break;
	}
2688 2689 2690
}

/**
2691 2692
 *  igb_sw_init - Initialize general software structures (struct igb_adapter)
 *  @adapter: board private structure to initialize
2693
 *
2694 2695 2696
 *  igb_sw_init initializes the Adapter private data structure.
 *  Fields are initialized based on PCI device information and
 *  OS network device settings (MTU size).
2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728
 **/
static int igb_sw_init(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	struct net_device *netdev = adapter->netdev;
	struct pci_dev *pdev = adapter->pdev;

	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);

	/* set default ring sizes */
	adapter->tx_ring_count = IGB_DEFAULT_TXD;
	adapter->rx_ring_count = IGB_DEFAULT_RXD;

	/* set default ITR values */
	adapter->rx_itr_setting = IGB_DEFAULT_ITR;
	adapter->tx_itr_setting = IGB_DEFAULT_ITR;

	/* set default work limits */
	adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;

	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
				  VLAN_HLEN;
	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;

	spin_lock_init(&adapter->stats64_lock);
#ifdef CONFIG_PCI_IOV
	switch (hw->mac.type) {
	case e1000_82576:
	case e1000_i350:
		if (max_vfs > 7) {
			dev_warn(&pdev->dev,
				 "Maximum of 7 VFs per PF, using max\n");
2729
			max_vfs = adapter->vfs_allocated_count = 7;
2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741
		} else
			adapter->vfs_allocated_count = max_vfs;
		if (adapter->vfs_allocated_count)
			dev_warn(&pdev->dev,
				 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
		break;
	default:
		break;
	}
#endif /* CONFIG_PCI_IOV */

	igb_init_queue_configuration(adapter);
2742

2743
	/* Setup and initialize a copy of the hw vlan table array */
2744 2745
	adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32),
				       GFP_ATOMIC);
2746

2747
	/* This call may decrease the number of queues */
2748
	if (igb_init_interrupt_scheme(adapter, true)) {
2749 2750 2751 2752
		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
		return -ENOMEM;
	}

2753 2754
	igb_probe_vfs(adapter);

2755 2756 2757
	/* Explicitly disable IRQ since the NIC can be in any state. */
	igb_irq_disable(adapter);

2758
	if (hw->mac.type >= e1000_i350)
2759 2760
		adapter->flags &= ~IGB_FLAG_DMAC;

2761 2762 2763 2764 2765
	set_bit(__IGB_DOWN, &adapter->state);
	return 0;
}

/**
2766 2767
 *  igb_open - Called when a network interface is made active
 *  @netdev: network interface device structure
2768
 *
2769
 *  Returns 0 on success, negative value on failure
2770
 *
2771 2772 2773 2774 2775
 *  The open entry point is called when a network interface is made
 *  active by the system (IFF_UP).  At this point all resources needed
 *  for transmit and receive operations are allocated, the interrupt
 *  handler is registered with the OS, the watchdog timer is started,
 *  and the stack is notified that the interface is ready.
2776
 **/
2777
static int __igb_open(struct net_device *netdev, bool resuming)
2778 2779 2780
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
2781
	struct pci_dev *pdev = adapter->pdev;
2782 2783 2784 2785
	int err;
	int i;

	/* disallow open during test */
2786 2787
	if (test_bit(__IGB_TESTING, &adapter->state)) {
		WARN_ON(resuming);
2788
		return -EBUSY;
2789 2790 2791 2792
	}

	if (!resuming)
		pm_runtime_get_sync(&pdev->dev);
2793

2794 2795
	netif_carrier_off(netdev);

2796 2797 2798 2799 2800 2801 2802 2803 2804 2805
	/* allocate transmit descriptors */
	err = igb_setup_all_tx_resources(adapter);
	if (err)
		goto err_setup_tx;

	/* allocate receive descriptors */
	err = igb_setup_all_rx_resources(adapter);
	if (err)
		goto err_setup_rx;

2806
	igb_power_up_link(adapter);
2807 2808 2809 2810

	/* before we allocate an interrupt, we must be ready to handle it.
	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
	 * as soon as we call pci_request_irq, so we have to setup our
2811 2812
	 * clean_rx handler before we do so.
	 */
2813 2814 2815 2816 2817 2818
	igb_configure(adapter);

	err = igb_request_irq(adapter);
	if (err)
		goto err_req_irq;

2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829
	/* Notify the stack of the actual queue counts. */
	err = netif_set_real_num_tx_queues(adapter->netdev,
					   adapter->num_tx_queues);
	if (err)
		goto err_set_queues;

	err = netif_set_real_num_rx_queues(adapter->netdev,
					   adapter->num_rx_queues);
	if (err)
		goto err_set_queues;

2830 2831 2832
	/* From here on the code is the same as igb_up() */
	clear_bit(__IGB_DOWN, &adapter->state);

2833 2834
	for (i = 0; i < adapter->num_q_vectors; i++)
		napi_enable(&(adapter->q_vector[i]->napi));
2835 2836 2837

	/* Clear any pending interrupts. */
	rd32(E1000_ICR);
2838 2839 2840

	igb_irq_enable(adapter);

2841 2842 2843 2844 2845 2846 2847
	/* notify VFs that reset has been completed */
	if (adapter->vfs_allocated_count) {
		u32 reg_data = rd32(E1000_CTRL_EXT);
		reg_data |= E1000_CTRL_EXT_PFRSTD;
		wr32(E1000_CTRL_EXT, reg_data);
	}

2848 2849
	netif_tx_start_all_queues(netdev);

2850 2851 2852
	if (!resuming)
		pm_runtime_put(&pdev->dev);

2853 2854 2855
	/* start the watchdog. */
	hw->mac.get_link_status = 1;
	schedule_work(&adapter->watchdog_task);
2856 2857 2858

	return 0;

2859 2860
err_set_queues:
	igb_free_irq(adapter);
2861 2862
err_req_irq:
	igb_release_hw_control(adapter);
2863
	igb_power_down_link(adapter);
2864 2865 2866 2867 2868
	igb_free_all_rx_resources(adapter);
err_setup_rx:
	igb_free_all_tx_resources(adapter);
err_setup_tx:
	igb_reset(adapter);
2869 2870
	if (!resuming)
		pm_runtime_put(&pdev->dev);
2871 2872 2873 2874

	return err;
}

2875 2876 2877 2878 2879
static int igb_open(struct net_device *netdev)
{
	return __igb_open(netdev, false);
}

2880
/**
2881 2882
 *  igb_close - Disables a network interface
 *  @netdev: network interface device structure
2883
 *
2884
 *  Returns 0, this is not allowed to fail
2885
 *
2886 2887 2888 2889
 *  The close entry point is called when an interface is de-activated
 *  by the OS.  The hardware is still under the driver's control, but
 *  needs to be disabled.  A global MAC reset is issued to stop the
 *  hardware, and all transmit and receive resources are freed.
2890
 **/
2891
static int __igb_close(struct net_device *netdev, bool suspending)
2892 2893
{
	struct igb_adapter *adapter = netdev_priv(netdev);
2894
	struct pci_dev *pdev = adapter->pdev;
2895 2896 2897

	WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));

2898 2899 2900 2901
	if (!suspending)
		pm_runtime_get_sync(&pdev->dev);

	igb_down(adapter);
2902 2903 2904 2905 2906
	igb_free_irq(adapter);

	igb_free_all_tx_resources(adapter);
	igb_free_all_rx_resources(adapter);

2907 2908
	if (!suspending)
		pm_runtime_put_sync(&pdev->dev);
2909 2910 2911
	return 0;
}

2912 2913 2914 2915 2916
static int igb_close(struct net_device *netdev)
{
	return __igb_close(netdev, false);
}

2917
/**
2918 2919
 *  igb_setup_tx_resources - allocate Tx resources (Descriptors)
 *  @tx_ring: tx descriptor ring (for a specific queue) to setup
2920
 *
2921
 *  Return 0 on success, negative on failure
2922
 **/
2923
int igb_setup_tx_resources(struct igb_ring *tx_ring)
2924
{
2925
	struct device *dev = tx_ring->dev;
2926 2927
	int size;

2928
	size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2929 2930

	tx_ring->tx_buffer_info = vzalloc(size);
2931
	if (!tx_ring->tx_buffer_info)
2932 2933 2934
		goto err;

	/* round up to nearest 4K */
2935
	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2936 2937
	tx_ring->size = ALIGN(tx_ring->size, 4096);

2938 2939
	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
					   &tx_ring->dma, GFP_KERNEL);
2940 2941 2942 2943 2944
	if (!tx_ring->desc)
		goto err;

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
2945

2946 2947 2948
	return 0;

err:
2949
	vfree(tx_ring->tx_buffer_info);
2950 2951
	tx_ring->tx_buffer_info = NULL;
	dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
2952 2953 2954 2955
	return -ENOMEM;
}

/**
2956 2957 2958
 *  igb_setup_all_tx_resources - wrapper to allocate Tx resources
 *				 (Descriptors) for all queues
 *  @adapter: board private structure
2959
 *
2960
 *  Return 0 on success, negative on failure
2961 2962 2963
 **/
static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
{
2964
	struct pci_dev *pdev = adapter->pdev;
2965 2966 2967
	int i, err = 0;

	for (i = 0; i < adapter->num_tx_queues; i++) {
2968
		err = igb_setup_tx_resources(adapter->tx_ring[i]);
2969
		if (err) {
2970
			dev_err(&pdev->dev,
2971 2972
				"Allocation for Tx Queue %u failed\n", i);
			for (i--; i >= 0; i--)
2973
				igb_free_tx_resources(adapter->tx_ring[i]);
2974 2975 2976 2977 2978 2979 2980 2981
			break;
		}
	}

	return err;
}

/**
2982 2983
 *  igb_setup_tctl - configure the transmit control registers
 *  @adapter: Board private structure
2984
 **/
2985
void igb_setup_tctl(struct igb_adapter *adapter)
2986 2987 2988 2989
{
	struct e1000_hw *hw = &adapter->hw;
	u32 tctl;

2990 2991
	/* disable queue 0 which is enabled by default on 82575 and 82576 */
	wr32(E1000_TXDCTL(0), 0);
2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006

	/* Program the Transmit Control Register */
	tctl = rd32(E1000_TCTL);
	tctl &= ~E1000_TCTL_CT;
	tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
		(E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);

	igb_config_collision_dist(hw);

	/* Enable transmits */
	tctl |= E1000_TCTL_EN;

	wr32(E1000_TCTL, tctl);
}

3007
/**
3008 3009 3010
 *  igb_configure_tx_ring - Configure transmit ring after Reset
 *  @adapter: board private structure
 *  @ring: tx ring to configure
3011
 *
3012
 *  Configure a transmit ring after a reset.
3013
 **/
3014 3015
void igb_configure_tx_ring(struct igb_adapter *adapter,
                           struct igb_ring *ring)
3016 3017
{
	struct e1000_hw *hw = &adapter->hw;
3018
	u32 txdctl = 0;
3019 3020 3021 3022
	u64 tdba = ring->dma;
	int reg_idx = ring->reg_idx;

	/* disable the queue */
3023
	wr32(E1000_TXDCTL(reg_idx), 0);
3024 3025 3026 3027
	wrfl();
	mdelay(10);

	wr32(E1000_TDLEN(reg_idx),
3028
	     ring->count * sizeof(union e1000_adv_tx_desc));
3029
	wr32(E1000_TDBAL(reg_idx),
3030
	     tdba & 0x00000000ffffffffULL);
3031 3032
	wr32(E1000_TDBAH(reg_idx), tdba >> 32);

3033
	ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
3034
	wr32(E1000_TDH(reg_idx), 0);
3035
	writel(0, ring->tail);
3036 3037 3038 3039 3040 3041 3042 3043 3044 3045

	txdctl |= IGB_TX_PTHRESH;
	txdctl |= IGB_TX_HTHRESH << 8;
	txdctl |= IGB_TX_WTHRESH << 16;

	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
	wr32(E1000_TXDCTL(reg_idx), txdctl);
}

/**
3046 3047
 *  igb_configure_tx - Configure transmit Unit after Reset
 *  @adapter: board private structure
3048
 *
3049
 *  Configure the Tx unit of the MAC after a reset.
3050 3051 3052 3053 3054 3055
 **/
static void igb_configure_tx(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_tx_queues; i++)
3056
		igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
3057 3058
}

3059
/**
3060 3061
 *  igb_setup_rx_resources - allocate Rx resources (Descriptors)
 *  @rx_ring: Rx descriptor ring (for a specific queue) to setup
3062
 *
3063
 *  Returns 0 on success, negative on failure
3064
 **/
3065
int igb_setup_rx_resources(struct igb_ring *rx_ring)
3066
{
3067
	struct device *dev = rx_ring->dev;
3068
	int size;
3069

3070
	size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3071 3072

	rx_ring->rx_buffer_info = vzalloc(size);
3073
	if (!rx_ring->rx_buffer_info)
3074 3075 3076
		goto err;

	/* Round up to nearest 4K */
3077
	rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
3078 3079
	rx_ring->size = ALIGN(rx_ring->size, 4096);

3080 3081
	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
					   &rx_ring->dma, GFP_KERNEL);
3082 3083 3084
	if (!rx_ring->desc)
		goto err;

3085
	rx_ring->next_to_alloc = 0;
3086 3087 3088 3089 3090 3091
	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;

	return 0;

err:
3092 3093
	vfree(rx_ring->rx_buffer_info);
	rx_ring->rx_buffer_info = NULL;
3094
	dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
3095 3096 3097 3098
	return -ENOMEM;
}

/**
3099 3100 3101
 *  igb_setup_all_rx_resources - wrapper to allocate Rx resources
 *				 (Descriptors) for all queues
 *  @adapter: board private structure
3102
 *
3103
 *  Return 0 on success, negative on failure
3104 3105 3106
 **/
static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
{
3107
	struct pci_dev *pdev = adapter->pdev;
3108 3109 3110
	int i, err = 0;

	for (i = 0; i < adapter->num_rx_queues; i++) {
3111
		err = igb_setup_rx_resources(adapter->rx_ring[i]);
3112
		if (err) {
3113
			dev_err(&pdev->dev,
3114 3115
				"Allocation for Rx Queue %u failed\n", i);
			for (i--; i >= 0; i--)
3116
				igb_free_rx_resources(adapter->rx_ring[i]);
3117 3118 3119 3120 3121 3122 3123
			break;
		}
	}

	return err;
}

3124
/**
3125 3126
 *  igb_setup_mrqc - configure the multiple receive queue control registers
 *  @adapter: Board private structure
3127 3128 3129 3130 3131
 **/
static void igb_setup_mrqc(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 mrqc, rxcsum;
3132
	u32 j, num_rx_queues;
3133 3134 3135 3136
	static const u32 rsskey[10] = { 0xDA565A6D, 0xC20E5B25, 0x3D256741,
					0xB08FA343, 0xCB2BCAD0, 0xB4307BAE,
					0xA32DCB77, 0x0CF23080, 0x3BB7426A,
					0xFA01ACBE };
3137 3138

	/* Fill out hash function seeds */
3139 3140
	for (j = 0; j < 10; j++)
		wr32(E1000_RSSRK(j), rsskey[j]);
3141

3142
	num_rx_queues = adapter->rss_queues;
3143

3144 3145 3146
	switch (hw->mac.type) {
	case e1000_82576:
		/* 82576 supports 2 RSS queues for SR-IOV */
3147
		if (adapter->vfs_allocated_count)
3148
			num_rx_queues = 2;
3149 3150 3151
		break;
	default:
		break;
3152 3153
	}

3154 3155 3156 3157
	if (adapter->rss_indir_tbl_init != num_rx_queues) {
		for (j = 0; j < IGB_RETA_SIZE; j++)
			adapter->rss_indir_tbl[j] = (j * num_rx_queues) / IGB_RETA_SIZE;
		adapter->rss_indir_tbl_init = num_rx_queues;
3158
	}
3159
	igb_write_rss_indir_tbl(adapter);
3160

3161
	/* Disable raw packet checksumming so that RSS hash is placed in
3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173
	 * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
	 * offloads as they are enabled by default
	 */
	rxcsum = rd32(E1000_RXCSUM);
	rxcsum |= E1000_RXCSUM_PCSD;

	if (adapter->hw.mac.type >= e1000_82576)
		/* Enable Receive Checksum Offload for SCTP */
		rxcsum |= E1000_RXCSUM_CRCOFL;

	/* Don't need to set TUOFL or IPOFL, they default to 1 */
	wr32(E1000_RXCSUM, rxcsum);
3174

3175 3176 3177
	/* Generate RSS hash based on packet types, TCP/UDP
	 * port numbers and/or IPv4/v6 src and dst addresses
	 */
3178 3179 3180 3181 3182
	mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
	       E1000_MRQC_RSS_FIELD_IPV4_TCP |
	       E1000_MRQC_RSS_FIELD_IPV6 |
	       E1000_MRQC_RSS_FIELD_IPV6_TCP |
	       E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
3183

3184 3185 3186 3187 3188
	if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
		mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
	if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
		mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;

3189 3190
	/* If VMDq is enabled then we set the appropriate mode for that, else
	 * we default to RSS so that an RSS hash is calculated per packet even
3191 3192
	 * if we are only using one queue
	 */
3193 3194 3195 3196 3197 3198 3199 3200 3201 3202
	if (adapter->vfs_allocated_count) {
		if (hw->mac.type > e1000_82575) {
			/* Set the default pool for the PF's first queue */
			u32 vtctl = rd32(E1000_VT_CTL);
			vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
				   E1000_VT_CTL_DISABLE_DEF_POOL);
			vtctl |= adapter->vfs_allocated_count <<
				E1000_VT_CTL_DEFAULT_POOL_SHIFT;
			wr32(E1000_VT_CTL, vtctl);
		}
3203
		if (adapter->rss_queues > 1)
3204
			mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
3205
		else
3206
			mrqc |= E1000_MRQC_ENABLE_VMDQ;
3207
	} else {
3208 3209
		if (hw->mac.type != e1000_i211)
			mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
3210 3211 3212 3213 3214 3215
	}
	igb_vmm_control(adapter);

	wr32(E1000_MRQC, mrqc);
}

3216
/**
3217 3218
 *  igb_setup_rctl - configure the receive control registers
 *  @adapter: Board private structure
3219
 **/
3220
void igb_setup_rctl(struct igb_adapter *adapter)
3221 3222 3223 3224 3225 3226 3227
{
	struct e1000_hw *hw = &adapter->hw;
	u32 rctl;

	rctl = rd32(E1000_RCTL);

	rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
3228
	rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
3229

3230
	rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
3231
		(hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
3232

3233
	/* enable stripping of CRC. It's unlikely this will break BMC
3234 3235
	 * redirection as it did with e1000. Newer features require
	 * that the HW strips the CRC.
3236
	 */
3237
	rctl |= E1000_RCTL_SECRC;
3238

3239
	/* disable store bad packets and clear size bits. */
3240
	rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
3241

Alexander Duyck's avatar
Alexander Duyck committed
3242 3243
	/* enable LPE to prevent packets larger than max_frame_size */
	rctl |= E1000_RCTL_LPE;
3244

3245 3246
	/* disable queue 0 to prevent tail write w/o re-config */
	wr32(E1000_RXDCTL(0), 0);
3247

3248 3249 3250 3251 3252 3253 3254 3255 3256
	/* Attention!!!  For SR-IOV PF driver operations you must enable
	 * queue drop for all VF and PF queues to prevent head of line blocking
	 * if an un-trusted VF does not provide descriptors to hardware.
	 */
	if (adapter->vfs_allocated_count) {
		/* set all queue drop enable bits */
		wr32(E1000_QDE, ALL_QUEUES);
	}

3257 3258 3259
	/* This is useful for sniffing bad packets. */
	if (adapter->netdev->features & NETIF_F_RXALL) {
		/* UPE and MPE will be handled by normal PROMISC logic
3260 3261
		 * in e1000e_set_rx_mode
		 */
3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273
		rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
			 E1000_RCTL_BAM | /* RX All Bcast Pkts */
			 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */

		rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
			  E1000_RCTL_DPF | /* Allow filtered pause */
			  E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
		/* Do not mess with E1000_CTRL_VME, it affects transmit as well,
		 * and that breaks VLANs.
		 */
	}

3274 3275 3276
	wr32(E1000_RCTL, rctl);
}

3277 3278 3279 3280 3281 3282 3283
static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
                                   int vfn)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 vmolr;

	/* if it isn't the PF check to see if VFs are enabled and
3284 3285
	 * increase the size to support vlan tags
	 */
3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297
	if (vfn < adapter->vfs_allocated_count &&
	    adapter->vf_data[vfn].vlans_enabled)
		size += VLAN_TAG_SIZE;

	vmolr = rd32(E1000_VMOLR(vfn));
	vmolr &= ~E1000_VMOLR_RLPML_MASK;
	vmolr |= size | E1000_VMOLR_LPE;
	wr32(E1000_VMOLR(vfn), vmolr);

	return 0;
}

3298
/**
3299 3300
 *  igb_rlpml_set - set maximum receive packet size
 *  @adapter: board private structure
3301
 *
3302
 *  Configure maximum receivable packet size.
3303 3304 3305
 **/
static void igb_rlpml_set(struct igb_adapter *adapter)
{
3306
	u32 max_frame_size = adapter->max_frame_size;
3307 3308 3309 3310 3311
	struct e1000_hw *hw = &adapter->hw;
	u16 pf_id = adapter->vfs_allocated_count;

	if (pf_id) {
		igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3312
		/* If we're in VMDQ or SR-IOV mode, then set global RLPML
3313 3314 3315 3316 3317
		 * to our max jumbo frame size, in case we need to enable
		 * jumbo frames on one of the rings later.
		 * This will not pass over-length frames into the default
		 * queue because it's gated by the VMOLR.RLPML.
		 */
3318
		max_frame_size = MAX_JUMBO_FRAME_SIZE;
3319 3320 3321 3322 3323
	}

	wr32(E1000_RLPML, max_frame_size);
}

3324 3325
static inline void igb_set_vmolr(struct igb_adapter *adapter,
				 int vfn, bool aupe)
3326 3327 3328 3329
{
	struct e1000_hw *hw = &adapter->hw;
	u32 vmolr;

3330
	/* This register exists only on 82576 and newer so if we are older then
3331 3332 3333 3334 3335 3336
	 * we should exit and do nothing
	 */
	if (hw->mac.type < e1000_82576)
		return;

	vmolr = rd32(E1000_VMOLR(vfn));
3337
	vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3338
	if (aupe)
3339
		vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3340 3341
	else
		vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3342 3343 3344 3345

	/* clear all bits that might not be set */
	vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);

3346
	if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3347
		vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3348
	/* for VMDq only allow the VFs and pool 0 to accept broadcast and
3349 3350 3351
	 * multicast packets
	 */
	if (vfn <= adapter->vfs_allocated_count)
3352
		vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3353 3354 3355 3356

	wr32(E1000_VMOLR(vfn), vmolr);
}

3357
/**
3358 3359 3360
 *  igb_configure_rx_ring - Configure a receive ring after Reset
 *  @adapter: board private structure
 *  @ring: receive ring to be configured
3361
 *
3362
 *  Configure the Rx unit of the MAC after a reset.
3363
 **/
3364
void igb_configure_rx_ring(struct igb_adapter *adapter,
3365
			   struct igb_ring *ring)
3366 3367 3368 3369
{
	struct e1000_hw *hw = &adapter->hw;
	u64 rdba = ring->dma;
	int reg_idx = ring->reg_idx;
3370
	u32 srrctl = 0, rxdctl = 0;
3371 3372

	/* disable the queue */
3373
	wr32(E1000_RXDCTL(reg_idx), 0);
3374 3375 3376 3377 3378 3379

	/* Set DMA base address registers */
	wr32(E1000_RDBAL(reg_idx),
	     rdba & 0x00000000ffffffffULL);
	wr32(E1000_RDBAH(reg_idx), rdba >> 32);
	wr32(E1000_RDLEN(reg_idx),
3380
	     ring->count * sizeof(union e1000_adv_rx_desc));
3381 3382

	/* initialize head and tail */
3383
	ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3384
	wr32(E1000_RDH(reg_idx), 0);
3385
	writel(0, ring->tail);
3386

3387
	/* set descriptor configuration */
3388
	srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3389
	srrctl |= IGB_RX_BUFSZ >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3390
	srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3391
	if (hw->mac.type >= e1000_82580)
3392
		srrctl |= E1000_SRRCTL_TIMESTAMP;
3393 3394 3395
	/* Only set Drop Enable if we are supporting multiple queues */
	if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
		srrctl |= E1000_SRRCTL_DROP_EN;
3396 3397 3398

	wr32(E1000_SRRCTL(reg_idx), srrctl);

3399
	/* set filtering for VMDQ pools */
3400
	igb_set_vmolr(adapter, reg_idx & 0x7, true);
3401

3402 3403 3404
	rxdctl |= IGB_RX_PTHRESH;
	rxdctl |= IGB_RX_HTHRESH << 8;
	rxdctl |= IGB_RX_WTHRESH << 16;
3405 3406 3407

	/* enable receive descriptor fetching */
	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3408 3409 3410
	wr32(E1000_RXDCTL(reg_idx), rxdctl);
}

3411
/**
3412 3413
 *  igb_configure_rx - Configure receive Unit after Reset
 *  @adapter: board private structure
3414
 *
3415
 *  Configure the Rx unit of the MAC after a reset.
3416 3417 3418
 **/
static void igb_configure_rx(struct igb_adapter *adapter)
{
3419
	int i;
3420

3421 3422 3423
	/* set UTA to appropriate mode */
	igb_set_uta(adapter);

3424 3425
	/* set the correct pool for the PF default MAC address in entry 0 */
	igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3426
			 adapter->vfs_allocated_count);
3427

3428
	/* Setup the HW Rx Head and Tail Descriptor Pointers and
3429 3430
	 * the Base and Length of the Rx Descriptor Ring
	 */
3431 3432
	for (i = 0; i < adapter->num_rx_queues; i++)
		igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3433 3434 3435
}

/**
3436 3437
 *  igb_free_tx_resources - Free Tx Resources per Queue
 *  @tx_ring: Tx descriptor ring for a specific queue
3438
 *
3439
 *  Free all transmit software resources
3440
 **/
3441
void igb_free_tx_resources(struct igb_ring *tx_ring)
3442
{
3443
	igb_clean_tx_ring(tx_ring);
3444

3445 3446
	vfree(tx_ring->tx_buffer_info);
	tx_ring->tx_buffer_info = NULL;
3447

3448 3449 3450 3451
	/* if not set, then don't free */
	if (!tx_ring->desc)
		return;

3452 3453
	dma_free_coherent(tx_ring->dev, tx_ring->size,
			  tx_ring->desc, tx_ring->dma);
3454 3455 3456 3457 3458

	tx_ring->desc = NULL;
}

/**
3459 3460
 *  igb_free_all_tx_resources - Free Tx Resources for All Queues
 *  @adapter: board private structure
3461
 *
3462
 *  Free all transmit software resources
3463 3464 3465 3466 3467 3468
 **/
static void igb_free_all_tx_resources(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_tx_queues; i++)
3469
		igb_free_tx_resources(adapter->tx_ring[i]);
3470 3471
}

3472 3473 3474 3475 3476
void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
				    struct igb_tx_buffer *tx_buffer)
{
	if (tx_buffer->skb) {
		dev_kfree_skb_any(tx_buffer->skb);
3477
		if (dma_unmap_len(tx_buffer, len))
3478
			dma_unmap_single(ring->dev,
3479 3480
					 dma_unmap_addr(tx_buffer, dma),
					 dma_unmap_len(tx_buffer, len),
3481
					 DMA_TO_DEVICE);
3482
	} else if (dma_unmap_len(tx_buffer, len)) {
3483
		dma_unmap_page(ring->dev,
3484 3485
			       dma_unmap_addr(tx_buffer, dma),
			       dma_unmap_len(tx_buffer, len),
3486 3487 3488 3489
			       DMA_TO_DEVICE);
	}
	tx_buffer->next_to_watch = NULL;
	tx_buffer->skb = NULL;
3490
	dma_unmap_len_set(tx_buffer, len, 0);
3491
	/* buffer_info must be completely set up in the transmit path */
3492 3493 3494
}

/**
3495 3496
 *  igb_clean_tx_ring - Free Tx Buffers
 *  @tx_ring: ring to be cleaned
3497
 **/
3498
static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3499
{
3500
	struct igb_tx_buffer *buffer_info;
3501
	unsigned long size;
3502
	u16 i;
3503

3504
	if (!tx_ring->tx_buffer_info)
3505 3506 3507 3508
		return;
	/* Free all the Tx ring sk_buffs */

	for (i = 0; i < tx_ring->count; i++) {
3509
		buffer_info = &tx_ring->tx_buffer_info[i];
3510
		igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3511 3512
	}

3513 3514
	netdev_tx_reset_queue(txring_txq(tx_ring));

3515 3516
	size = sizeof(struct igb_tx_buffer) * tx_ring->count;
	memset(tx_ring->tx_buffer_info, 0, size);
3517 3518 3519 3520 3521 3522 3523 3524 3525

	/* Zero out the descriptor ring */
	memset(tx_ring->desc, 0, tx_ring->size);

	tx_ring->next_to_use = 0;
	tx_ring->next_to_clean = 0;
}

/**
3526 3527
 *  igb_clean_all_tx_rings - Free Tx Buffers for all queues
 *  @adapter: board private structure
3528 3529 3530 3531 3532 3533
 **/
static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_tx_queues; i++)
3534
		igb_clean_tx_ring(adapter->tx_ring[i]);
3535 3536 3537
}

/**
3538 3539
 *  igb_free_rx_resources - Free Rx Resources
 *  @rx_ring: ring to clean the resources from
3540
 *
3541
 *  Free all receive software resources
3542
 **/
3543
void igb_free_rx_resources(struct igb_ring *rx_ring)
3544
{
3545
	igb_clean_rx_ring(rx_ring);
3546

3547 3548
	vfree(rx_ring->rx_buffer_info);
	rx_ring->rx_buffer_info = NULL;
3549

3550 3551 3552 3553
	/* if not set, then don't free */
	if (!rx_ring->desc)
		return;

3554 3555
	dma_free_coherent(rx_ring->dev, rx_ring->size,
			  rx_ring->desc, rx_ring->dma);
3556 3557 3558 3559 3560

	rx_ring->desc = NULL;
}

/**
3561 3562
 *  igb_free_all_rx_resources - Free Rx Resources for All Queues
 *  @adapter: board private structure
3563
 *
3564
 *  Free all receive software resources
3565 3566 3567 3568 3569 3570
 **/
static void igb_free_all_rx_resources(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_rx_queues; i++)
3571
		igb_free_rx_resources(adapter->rx_ring[i]);
3572 3573 3574
}

/**
3575 3576
 *  igb_clean_rx_ring - Free Rx Buffers per Queue
 *  @rx_ring: ring to free buffers from
3577
 **/
3578
static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3579 3580
{
	unsigned long size;
3581
	u16 i;
3582

3583 3584 3585 3586
	if (rx_ring->skb)
		dev_kfree_skb(rx_ring->skb);
	rx_ring->skb = NULL;

3587
	if (!rx_ring->rx_buffer_info)
3588
		return;
3589

3590 3591
	/* Free all the Rx ring sk_buffs */
	for (i = 0; i < rx_ring->count; i++) {
3592
		struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3593

3594 3595 3596 3597 3598 3599 3600 3601 3602
		if (!buffer_info->page)
			continue;

		dma_unmap_page(rx_ring->dev,
			       buffer_info->dma,
			       PAGE_SIZE,
			       DMA_FROM_DEVICE);
		__free_page(buffer_info->page);

3603
		buffer_info->page = NULL;
3604 3605
	}

3606 3607
	size = sizeof(struct igb_rx_buffer) * rx_ring->count;
	memset(rx_ring->rx_buffer_info, 0, size);
3608 3609 3610 3611

	/* Zero out the descriptor ring */
	memset(rx_ring->desc, 0, rx_ring->size);

3612
	rx_ring->next_to_alloc = 0;
3613 3614 3615 3616 3617
	rx_ring->next_to_clean = 0;
	rx_ring->next_to_use = 0;
}

/**
3618 3619
 *  igb_clean_all_rx_rings - Free Rx Buffers for all queues
 *  @adapter: board private structure
3620 3621 3622 3623 3624 3625
 **/
static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
{
	int i;

	for (i = 0; i < adapter->num_rx_queues; i++)
3626
		igb_clean_rx_ring(adapter->rx_ring[i]);
3627 3628 3629
}

/**
3630 3631 3632
 *  igb_set_mac - Change the Ethernet Address of the NIC
 *  @netdev: network interface device structure
 *  @p: pointer to an address structure
3633
 *
3634
 *  Returns 0 on success, negative on failure
3635 3636 3637 3638
 **/
static int igb_set_mac(struct net_device *netdev, void *p)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
3639
	struct e1000_hw *hw = &adapter->hw;
3640 3641 3642 3643 3644 3645
	struct sockaddr *addr = p;

	if (!is_valid_ether_addr(addr->sa_data))
		return -EADDRNOTAVAIL;

	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3646
	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3647

3648 3649
	/* set the correct pool for the new PF MAC address in entry 0 */
	igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3650
			 adapter->vfs_allocated_count);
3651

3652 3653 3654 3655
	return 0;
}

/**
3656 3657
 *  igb_write_mc_addr_list - write multicast addresses to MTA
 *  @netdev: network interface device structure
3658
 *
3659 3660 3661 3662
 *  Writes multicast address list to the MTA hash table.
 *  Returns: -ENOMEM on failure
 *           0 on no addresses written
 *           X on writing X addresses to MTA
3663
 **/
3664
static int igb_write_mc_addr_list(struct net_device *netdev)
3665 3666 3667
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
3668
	struct netdev_hw_addr *ha;
3669
	u8  *mta_list;
3670 3671
	int i;

3672
	if (netdev_mc_empty(netdev)) {
3673 3674 3675 3676 3677
		/* nothing to program, so clear mc list */
		igb_update_mc_addr_list(hw, NULL, 0);
		igb_restore_vf_multicasts(adapter);
		return 0;
	}
3678

3679
	mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3680 3681
	if (!mta_list)
		return -ENOMEM;
3682

3683
	/* The shared function expects a packed array of only addresses. */
3684
	i = 0;
3685 3686
	netdev_for_each_mc_addr(ha, netdev)
		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3687 3688 3689 3690

	igb_update_mc_addr_list(hw, mta_list, i);
	kfree(mta_list);

3691
	return netdev_mc_count(netdev);
3692 3693 3694
}

/**
3695 3696
 *  igb_write_uc_addr_list - write unicast addresses to RAR table
 *  @netdev: network interface device structure
3697
 *
3698 3699 3700 3701
 *  Writes unicast address list to the RAR table.
 *  Returns: -ENOMEM on failure/insufficient address space
 *           0 on no addresses written
 *           X on writing X addresses to the RAR table
3702 3703 3704 3705 3706 3707 3708 3709 3710 3711
 **/
static int igb_write_uc_addr_list(struct net_device *netdev)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	unsigned int vfn = adapter->vfs_allocated_count;
	unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
	int count = 0;

	/* return ENOMEM indicating insufficient memory for addresses */
3712
	if (netdev_uc_count(netdev) > rar_entries)
3713
		return -ENOMEM;
3714

3715
	if (!netdev_uc_empty(netdev) && rar_entries) {
3716
		struct netdev_hw_addr *ha;
3717 3718

		netdev_for_each_uc_addr(ha, netdev) {
3719 3720
			if (!rar_entries)
				break;
3721
			igb_rar_set_qsel(adapter, ha->addr,
3722 3723
					 rar_entries--,
					 vfn);
3724
			count++;
3725 3726 3727 3728 3729 3730 3731 3732 3733
		}
	}
	/* write the addresses in reverse order to avoid write combining */
	for (; rar_entries > 0 ; rar_entries--) {
		wr32(E1000_RAH(rar_entries), 0);
		wr32(E1000_RAL(rar_entries), 0);
	}
	wrfl();

3734 3735 3736 3737
	return count;
}

/**
3738 3739
 *  igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
 *  @netdev: network interface device structure
3740
 *
3741 3742 3743 3744
 *  The set_rx_mode entry point is called whenever the unicast or multicast
 *  address lists or the network interface flags are updated.  This routine is
 *  responsible for configuring the hardware for proper unicast, multicast,
 *  promiscuous mode, and all-multi behavior.
3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760
 **/
static void igb_set_rx_mode(struct net_device *netdev)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	unsigned int vfn = adapter->vfs_allocated_count;
	u32 rctl, vmolr = 0;
	int count;

	/* Check for Promiscuous and All Multicast modes */
	rctl = rd32(E1000_RCTL);

	/* clear the effected bits */
	rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);

	if (netdev->flags & IFF_PROMISC) {
3761
		/* retain VLAN HW filtering if in VT mode */
3762
		if (adapter->vfs_allocated_count)
3763
			rctl |= E1000_RCTL_VFE;
3764 3765 3766 3767 3768 3769 3770
		rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
		vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
	} else {
		if (netdev->flags & IFF_ALLMULTI) {
			rctl |= E1000_RCTL_MPE;
			vmolr |= E1000_VMOLR_MPME;
		} else {
3771
			/* Write addresses to the MTA, if the attempt fails
Lucas De Marchi's avatar
Lucas De Marchi committed
3772
			 * then we should just turn on promiscuous mode so
3773 3774 3775 3776 3777 3778 3779 3780 3781 3782
			 * that we can at least receive multicast traffic
			 */
			count = igb_write_mc_addr_list(netdev);
			if (count < 0) {
				rctl |= E1000_RCTL_MPE;
				vmolr |= E1000_VMOLR_MPME;
			} else if (count) {
				vmolr |= E1000_VMOLR_ROMPE;
			}
		}
3783
		/* Write addresses to available RAR registers, if there is not
3784
		 * sufficient space to store all the addresses then enable
Lucas De Marchi's avatar
Lucas De Marchi committed
3785
		 * unicast promiscuous mode
3786 3787 3788 3789 3790 3791 3792
		 */
		count = igb_write_uc_addr_list(netdev);
		if (count < 0) {
			rctl |= E1000_RCTL_UPE;
			vmolr |= E1000_VMOLR_ROPE;
		}
		rctl |= E1000_RCTL_VFE;
3793
	}
3794
	wr32(E1000_RCTL, rctl);
3795

3796
	/* In order to support SR-IOV and eventually VMDq it is necessary to set
3797 3798 3799 3800
	 * the VMOLR to enable the appropriate modes.  Without this workaround
	 * we will have issues with VLAN tag stripping not being done for frames
	 * that are only arriving because we are the default pool
	 */
3801
	if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
3802
		return;
3803

3804
	vmolr |= rd32(E1000_VMOLR(vfn)) &
3805
		 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3806
	wr32(E1000_VMOLR(vfn), vmolr);
3807
	igb_restore_vf_multicasts(adapter);
3808 3809
}

3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848
static void igb_check_wvbr(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 wvbr = 0;

	switch (hw->mac.type) {
	case e1000_82576:
	case e1000_i350:
		if (!(wvbr = rd32(E1000_WVBR)))
			return;
		break;
	default:
		break;
	}

	adapter->wvbr |= wvbr;
}

#define IGB_STAGGERED_QUEUE_OFFSET 8

static void igb_spoof_check(struct igb_adapter *adapter)
{
	int j;

	if (!adapter->wvbr)
		return;

	for(j = 0; j < adapter->vfs_allocated_count; j++) {
		if (adapter->wvbr & (1 << j) ||
		    adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
			dev_warn(&adapter->pdev->dev,
				"Spoof event(s) detected on VF %d\n", j);
			adapter->wvbr &=
				~((1 << j) |
				  (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
		}
	}
}

3849
/* Need to wait a few seconds after link up to get diagnostic information from
3850 3851
 * the phy
 */
3852 3853 3854
static void igb_update_phy_info(unsigned long data)
{
	struct igb_adapter *adapter = (struct igb_adapter *) data;
3855
	igb_get_phy_info(&adapter->hw);
3856 3857
}

3858
/**
3859 3860
 *  igb_has_link - check shared code for link and determine up/down
 *  @adapter: pointer to driver private info
3861
 **/
3862
bool igb_has_link(struct igb_adapter *adapter)
3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873
{
	struct e1000_hw *hw = &adapter->hw;
	bool link_active = false;

	/* get_link_status is set on LSC (link status) interrupt or
	 * rx sequence error interrupt.  get_link_status will stay
	 * false until the e1000_check_for_link establishes link
	 * for copper adapters ONLY
	 */
	switch (hw->phy.media_type) {
	case e1000_media_type_copper:
3874 3875
		if (!hw->mac.get_link_status)
			return true;
3876
	case e1000_media_type_internal_serdes:
3877 3878
		hw->mac.ops.check_for_link(hw);
		link_active = !hw->mac.get_link_status;
3879 3880 3881 3882 3883 3884
		break;
	default:
	case e1000_media_type_unknown:
		break;
	}

3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895
	if (((hw->mac.type == e1000_i210) ||
	     (hw->mac.type == e1000_i211)) &&
	     (hw->phy.id == I210_I_PHY_ID)) {
		if (!netif_carrier_ok(adapter->netdev)) {
			adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
		} else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
			adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
			adapter->link_check_timeout = jiffies;
		}
	}

3896 3897 3898
	return link_active;
}

3899 3900 3901 3902 3903
static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
{
	bool ret = false;
	u32 ctrl_ext, thstat;

3904
	/* check for thermal sensor event on i350 copper only */
3905 3906 3907 3908 3909
	if (hw->mac.type == e1000_i350) {
		thstat = rd32(E1000_THSTAT);
		ctrl_ext = rd32(E1000_CTRL_EXT);

		if ((hw->phy.media_type == e1000_media_type_copper) &&
3910
		    !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII))
3911 3912 3913 3914 3915 3916
			ret = !!(thstat & event);
	}

	return ret;
}

3917
/**
3918 3919
 *  igb_watchdog - Timer Call-back
 *  @data: pointer to adapter cast into an unsigned long
3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930
 **/
static void igb_watchdog(unsigned long data)
{
	struct igb_adapter *adapter = (struct igb_adapter *)data;
	/* Do the rest outside of interrupt context */
	schedule_work(&adapter->watchdog_task);
}

static void igb_watchdog_task(struct work_struct *work)
{
	struct igb_adapter *adapter = container_of(work,
3931 3932
						   struct igb_adapter,
						   watchdog_task);
3933
	struct e1000_hw *hw = &adapter->hw;
3934
	struct e1000_phy_info *phy = &hw->phy;
3935
	struct net_device *netdev = adapter->netdev;
3936
	u32 link;
3937
	int i;
3938

3939
	link = igb_has_link(adapter);
3940 3941 3942 3943 3944 3945 3946 3947

	if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
		if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
			adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
		else
			link = false;
	}

3948
	if (link) {
3949 3950 3951
		/* Cancel scheduled suspend requests. */
		pm_runtime_resume(netdev->dev.parent);

3952 3953
		if (!netif_carrier_ok(netdev)) {
			u32 ctrl;
3954
			hw->mac.ops.get_speed_and_duplex(hw,
3955 3956
							 &adapter->link_speed,
							 &adapter->link_duplex);
3957 3958

			ctrl = rd32(E1000_CTRL);
3959
			/* Links status message must follow this format */
3960 3961
			printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
			       "Duplex, Flow Control: %s\n",
3962 3963 3964
			       netdev->name,
			       adapter->link_speed,
			       adapter->link_duplex == FULL_DUPLEX ?
3965 3966 3967 3968 3969
			       "Full" : "Half",
			       (ctrl & E1000_CTRL_TFCE) &&
			       (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
			       (ctrl & E1000_CTRL_RFCE) ?  "RX" :
			       (ctrl & E1000_CTRL_TFCE) ?  "TX" : "None");
3970

3971 3972 3973 3974 3975
			/* check if SmartSpeed worked */
			igb_check_downshift(hw);
			if (phy->speed_downgraded)
				netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");

3976
			/* check for thermal sensor event */
3977 3978 3979 3980 3981
			if (igb_thermal_sensor_event(hw,
			    E1000_THSTAT_LINK_THROTTLE)) {
				netdev_info(netdev, "The network adapter link "
					    "speed was downshifted because it "
					    "overheated\n");
3982
			}
3983

3984
			/* adjust timeout factor according to speed/duplex */
3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996
			adapter->tx_timeout_factor = 1;
			switch (adapter->link_speed) {
			case SPEED_10:
				adapter->tx_timeout_factor = 14;
				break;
			case SPEED_100:
				/* maybe add some timeout factor ? */
				break;
			}

			netif_carrier_on(netdev);

3997
			igb_ping_all_vfs(adapter);
3998
			igb_check_vf_rate_limit(adapter);
3999

4000
			/* link state has changed, schedule phy info update */
4001 4002 4003 4004 4005 4006 4007 4008
			if (!test_bit(__IGB_DOWN, &adapter->state))
				mod_timer(&adapter->phy_info_timer,
					  round_jiffies(jiffies + 2 * HZ));
		}
	} else {
		if (netif_carrier_ok(netdev)) {
			adapter->link_speed = 0;
			adapter->link_duplex = 0;
4009 4010

			/* check for thermal sensor event */
4011 4012 4013 4014
			if (igb_thermal_sensor_event(hw,
			    E1000_THSTAT_PWR_DOWN)) {
				netdev_err(netdev, "The network adapter was "
					   "stopped because it overheated\n");
4015
			}
4016

4017 4018 4019
			/* Links status message must follow this format */
			printk(KERN_INFO "igb: %s NIC Link is Down\n",
			       netdev->name);
4020
			netif_carrier_off(netdev);
4021

4022 4023
			igb_ping_all_vfs(adapter);

4024
			/* link state has changed, schedule phy info update */
4025 4026 4027
			if (!test_bit(__IGB_DOWN, &adapter->state))
				mod_timer(&adapter->phy_info_timer,
					  round_jiffies(jiffies + 2 * HZ));
4028 4029 4030

			pm_schedule_suspend(netdev->dev.parent,
					    MSEC_PER_SEC * 5);
4031 4032 4033
		}
	}

Eric Dumazet's avatar
Eric Dumazet committed
4034 4035 4036
	spin_lock(&adapter->stats64_lock);
	igb_update_stats(adapter, &adapter->stats64);
	spin_unlock(&adapter->stats64_lock);
4037

4038
	for (i = 0; i < adapter->num_tx_queues; i++) {
4039
		struct igb_ring *tx_ring = adapter->tx_ring[i];
4040
		if (!netif_carrier_ok(netdev)) {
4041 4042 4043
			/* We've lost link, so the controller stops DMA,
			 * but we've got queued Tx work that's never going
			 * to get done, so reset controller to flush Tx.
4044 4045
			 * (Do the reset outside of interrupt context).
			 */
4046 4047 4048 4049 4050 4051
			if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
				adapter->tx_timeout_count++;
				schedule_work(&adapter->reset_task);
				/* return immediately since reset is imminent */
				return;
			}
4052 4053
		}

4054
		/* Force detection of hung controller every watchdog period */
4055
		set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
4056
	}
4057

4058
	/* Cause software interrupt to ensure Rx ring is cleaned */
4059
	if (adapter->msix_entries) {
4060
		u32 eics = 0;
4061 4062
		for (i = 0; i < adapter->num_q_vectors; i++)
			eics |= adapter->q_vector[i]->eims_value;
4063 4064 4065 4066
		wr32(E1000_EICS, eics);
	} else {
		wr32(E1000_ICS, E1000_ICS_RXDMT0);
	}
4067

4068
	igb_spoof_check(adapter);
4069
	igb_ptp_rx_hang(adapter);
4070

4071
	/* Reset the timer */
4072 4073 4074 4075 4076 4077 4078 4079
	if (!test_bit(__IGB_DOWN, &adapter->state)) {
		if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
			mod_timer(&adapter->watchdog_timer,
				  round_jiffies(jiffies +  HZ));
		else
			mod_timer(&adapter->watchdog_timer,
				  round_jiffies(jiffies + 2 * HZ));
	}
4080 4081 4082 4083 4084 4085 4086 4087 4088
}

enum latency_range {
	lowest_latency = 0,
	low_latency = 1,
	bulk_latency = 2,
	latency_invalid = 255
};

4089
/**
4090 4091
 *  igb_update_ring_itr - update the dynamic ITR value based on packet size
 *  @q_vector: pointer to q_vector
4092
 *
4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103
 *  Stores a new ITR value based on strictly on packet size.  This
 *  algorithm is less sophisticated than that used in igb_update_itr,
 *  due to the difficulty of synchronizing statistics across multiple
 *  receive rings.  The divisors and thresholds used by this function
 *  were determined based on theoretical maximum wire speed and testing
 *  data, in order to minimize response time while increasing bulk
 *  throughput.
 *  This functionality is controlled by the InterruptThrottleRate module
 *  parameter (see igb_param.c)
 *  NOTE:  This function is called only when operating in a multiqueue
 *         receive environment.
4104
 **/
4105
static void igb_update_ring_itr(struct igb_q_vector *q_vector)
4106
{
4107
	int new_val = q_vector->itr_val;
4108
	int avg_wire_size = 0;
4109
	struct igb_adapter *adapter = q_vector->adapter;
Eric Dumazet's avatar
Eric Dumazet committed
4110
	unsigned int packets;
4111

4112 4113 4114 4115
	/* For non-gigabit speeds, just fix the interrupt rate at 4000
	 * ints/sec - ITR timer value of 120 ticks.
	 */
	if (adapter->link_speed != SPEED_1000) {
4116
		new_val = IGB_4K_ITR;
4117
		goto set_itr_val;
4118
	}
4119

4120 4121 4122
	packets = q_vector->rx.total_packets;
	if (packets)
		avg_wire_size = q_vector->rx.total_bytes / packets;
4123

4124 4125 4126 4127
	packets = q_vector->tx.total_packets;
	if (packets)
		avg_wire_size = max_t(u32, avg_wire_size,
				      q_vector->tx.total_bytes / packets);
4128 4129 4130 4131

	/* if avg_wire_size isn't set no work was done */
	if (!avg_wire_size)
		goto clear_counts;
4132

4133 4134 4135 4136 4137
	/* Add 24 bytes to size to account for CRC, preamble, and gap */
	avg_wire_size += 24;

	/* Don't starve jumbo frames */
	avg_wire_size = min(avg_wire_size, 3000);
4138

4139 4140 4141 4142 4143
	/* Give a little boost to mid-size frames */
	if ((avg_wire_size > 300) && (avg_wire_size < 1200))
		new_val = avg_wire_size / 3;
	else
		new_val = avg_wire_size / 2;
4144

4145 4146 4147 4148 4149
	/* conservative mode (itr 3) eliminates the lowest_latency setting */
	if (new_val < IGB_20K_ITR &&
	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
		new_val = IGB_20K_ITR;
4150

4151
set_itr_val:
4152 4153 4154
	if (new_val != q_vector->itr_val) {
		q_vector->itr_val = new_val;
		q_vector->set_itr = 1;
4155
	}
4156
clear_counts:
4157 4158 4159 4160
	q_vector->rx.total_bytes = 0;
	q_vector->rx.total_packets = 0;
	q_vector->tx.total_bytes = 0;
	q_vector->tx.total_packets = 0;
4161 4162 4163
}

/**
4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178
 *  igb_update_itr - update the dynamic ITR value based on statistics
 *  @q_vector: pointer to q_vector
 *  @ring_container: ring info to update the itr for
 *
 *  Stores a new ITR value based on packets and byte
 *  counts during the last interrupt.  The advantage of per interrupt
 *  computation is faster updates and more accurate ITR for the current
 *  traffic pattern.  Constants in this function were computed
 *  based on theoretical maximum wire speed and thresholds were set based
 *  on testing data as well as attempting to minimize response time
 *  while increasing bulk throughput.
 *  this functionality is controlled by the InterruptThrottleRate module
 *  parameter (see igb_param.c)
 *  NOTE:  These calculations are only valid when operating in a single-
 *         queue environment.
4179
 **/
4180 4181
static void igb_update_itr(struct igb_q_vector *q_vector,
			   struct igb_ring_container *ring_container)
4182
{
4183 4184 4185
	unsigned int packets = ring_container->total_packets;
	unsigned int bytes = ring_container->total_bytes;
	u8 itrval = ring_container->itr;
4186

4187
	/* no packets, exit with status unchanged */
4188
	if (packets == 0)
4189
		return;
4190

4191
	switch (itrval) {
4192 4193 4194
	case lowest_latency:
		/* handle TSO and jumbo frames */
		if (bytes/packets > 8000)
4195
			itrval = bulk_latency;
4196
		else if ((packets < 5) && (bytes > 512))
4197
			itrval = low_latency;
4198 4199 4200 4201 4202
		break;
	case low_latency:  /* 50 usec aka 20000 ints/s */
		if (bytes > 10000) {
			/* this if handles the TSO accounting */
			if (bytes/packets > 8000) {
4203
				itrval = bulk_latency;
4204
			} else if ((packets < 10) || ((bytes/packets) > 1200)) {
4205
				itrval = bulk_latency;
4206
			} else if ((packets > 35)) {
4207
				itrval = lowest_latency;
4208 4209
			}
		} else if (bytes/packets > 2000) {
4210
			itrval = bulk_latency;
4211
		} else if (packets <= 2 && bytes < 512) {
4212
			itrval = lowest_latency;
4213 4214 4215 4216 4217
		}
		break;
	case bulk_latency: /* 250 usec aka 4000 ints/s */
		if (bytes > 25000) {
			if (packets > 35)
4218
				itrval = low_latency;
4219
		} else if (bytes < 1500) {
4220
			itrval = low_latency;
4221 4222 4223 4224
		}
		break;
	}

4225 4226 4227 4228 4229 4230
	/* clear work counters since we have the values we need */
	ring_container->total_bytes = 0;
	ring_container->total_packets = 0;

	/* write updated itr to ring container */
	ring_container->itr = itrval;
4231 4232
}

4233
static void igb_set_itr(struct igb_q_vector *q_vector)
4234
{
4235
	struct igb_adapter *adapter = q_vector->adapter;
4236
	u32 new_itr = q_vector->itr_val;
4237
	u8 current_itr = 0;
4238 4239 4240 4241

	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
	if (adapter->link_speed != SPEED_1000) {
		current_itr = 0;
4242
		new_itr = IGB_4K_ITR;
4243 4244 4245
		goto set_itr_now;
	}

4246 4247
	igb_update_itr(q_vector, &q_vector->tx);
	igb_update_itr(q_vector, &q_vector->rx);
4248

4249
	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4250

4251
	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4252 4253 4254
	if (current_itr == lowest_latency &&
	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
	     (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4255 4256
		current_itr = low_latency;

4257 4258 4259
	switch (current_itr) {
	/* counts and packets in update_itr are dependent on these numbers */
	case lowest_latency:
4260
		new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
4261 4262
		break;
	case low_latency:
4263
		new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
4264 4265
		break;
	case bulk_latency:
4266
		new_itr = IGB_4K_ITR;  /* 4,000 ints/sec */
4267 4268 4269 4270 4271 4272
		break;
	default:
		break;
	}

set_itr_now:
4273
	if (new_itr != q_vector->itr_val) {
4274 4275
		/* this attempts to bias the interrupt rate towards Bulk
		 * by adding intermediate steps when interrupt rate is
4276 4277
		 * increasing
		 */
4278
		new_itr = new_itr > q_vector->itr_val ?
4279 4280 4281
			  max((new_itr * q_vector->itr_val) /
			  (new_itr + (q_vector->itr_val >> 2)),
			  new_itr) : new_itr;
4282 4283 4284 4285 4286 4287
		/* Don't write the value here; it resets the adapter's
		 * internal timer, and causes us to delay far longer than
		 * we should between interrupts.  Instead, we write the ITR
		 * value at the beginning of the next interrupt so the timing
		 * ends up being correct.
		 */
4288 4289
		q_vector->itr_val = new_itr;
		q_vector->set_itr = 1;
4290 4291 4292
	}
}

4293 4294
static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
			    u32 type_tucmd, u32 mss_l4len_idx)
4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307
{
	struct e1000_adv_tx_context_desc *context_desc;
	u16 i = tx_ring->next_to_use;

	context_desc = IGB_TX_CTXTDESC(tx_ring, i);

	i++;
	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;

	/* set bits to identify this as an advanced context descriptor */
	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;

	/* For 82575, context index must be unique per ring. */
4308
	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4309 4310 4311 4312 4313 4314 4315 4316
		mss_l4len_idx |= tx_ring->reg_idx << 4;

	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
	context_desc->seqnum_seed	= 0;
	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
}

4317 4318 4319
static int igb_tso(struct igb_ring *tx_ring,
		   struct igb_tx_buffer *first,
		   u8 *hdr_len)
4320
{
4321
	struct sk_buff *skb = first->skb;
4322 4323 4324
	u32 vlan_macip_lens, type_tucmd;
	u32 mss_l4len_idx, l4len;

4325 4326 4327
	if (skb->ip_summed != CHECKSUM_PARTIAL)
		return 0;

4328 4329
	if (!skb_is_gso(skb))
		return 0;
4330 4331

	if (skb_header_cloned(skb)) {
4332
		int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4333 4334 4335 4336
		if (err)
			return err;
	}

4337 4338
	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
	type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4339

4340
	if (first->protocol == __constant_htons(ETH_P_IP)) {
4341 4342 4343 4344 4345 4346 4347
		struct iphdr *iph = ip_hdr(skb);
		iph->tot_len = 0;
		iph->check = 0;
		tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
							 iph->daddr, 0,
							 IPPROTO_TCP,
							 0);
4348
		type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4349 4350 4351
		first->tx_flags |= IGB_TX_FLAGS_TSO |
				   IGB_TX_FLAGS_CSUM |
				   IGB_TX_FLAGS_IPV4;
4352
	} else if (skb_is_gso_v6(skb)) {
4353 4354 4355 4356
		ipv6_hdr(skb)->payload_len = 0;
		tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
						       &ipv6_hdr(skb)->daddr,
						       0, IPPROTO_TCP, 0);
4357 4358
		first->tx_flags |= IGB_TX_FLAGS_TSO |
				   IGB_TX_FLAGS_CSUM;
4359 4360
	}

4361
	/* compute header lengths */
4362 4363
	l4len = tcp_hdrlen(skb);
	*hdr_len = skb_transport_offset(skb) + l4len;
4364

4365 4366 4367 4368
	/* update gso size and bytecount with header size */
	first->gso_segs = skb_shinfo(skb)->gso_segs;
	first->bytecount += (first->gso_segs - 1) * *hdr_len;

4369
	/* MSS L4LEN IDX */
4370 4371
	mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4372

4373 4374 4375
	/* VLAN MACLEN IPLEN */
	vlan_macip_lens = skb_network_header_len(skb);
	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4376
	vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4377

4378
	igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4379

4380
	return 1;
4381 4382
}

4383
static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4384
{
4385
	struct sk_buff *skb = first->skb;
4386 4387 4388
	u32 vlan_macip_lens = 0;
	u32 mss_l4len_idx = 0;
	u32 type_tucmd = 0;
4389

4390
	if (skb->ip_summed != CHECKSUM_PARTIAL) {
4391 4392
		if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
			return;
4393 4394
	} else {
		u8 l4_hdr = 0;
4395
		switch (first->protocol) {
4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407
		case __constant_htons(ETH_P_IP):
			vlan_macip_lens |= skb_network_header_len(skb);
			type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
			l4_hdr = ip_hdr(skb)->protocol;
			break;
		case __constant_htons(ETH_P_IPV6):
			vlan_macip_lens |= skb_network_header_len(skb);
			l4_hdr = ipv6_hdr(skb)->nexthdr;
			break;
		default:
			if (unlikely(net_ratelimit())) {
				dev_warn(tx_ring->dev,
4408 4409
					 "partial checksum but proto=%x!\n",
					 first->protocol);
4410
			}
4411 4412
			break;
		}
4413

4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431
		switch (l4_hdr) {
		case IPPROTO_TCP:
			type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
			mss_l4len_idx = tcp_hdrlen(skb) <<
					E1000_ADVTXD_L4LEN_SHIFT;
			break;
		case IPPROTO_SCTP:
			type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
			mss_l4len_idx = sizeof(struct sctphdr) <<
					E1000_ADVTXD_L4LEN_SHIFT;
			break;
		case IPPROTO_UDP:
			mss_l4len_idx = sizeof(struct udphdr) <<
					E1000_ADVTXD_L4LEN_SHIFT;
			break;
		default:
			if (unlikely(net_ratelimit())) {
				dev_warn(tx_ring->dev,
4432 4433
					 "partial checksum but l4 proto=%x!\n",
					 l4_hdr);
4434
			}
4435
			break;
4436
		}
4437 4438 4439

		/* update TX checksum flag */
		first->tx_flags |= IGB_TX_FLAGS_CSUM;
4440
	}
4441

4442
	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4443
	vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4444

4445
	igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4446 4447
}

4448 4449 4450 4451 4452 4453
#define IGB_SET_FLAG(_input, _flag, _result) \
	((_flag <= _result) ? \
	 ((u32)(_input & _flag) * (_result / _flag)) : \
	 ((u32)(_input & _flag) / (_flag / _result)))

static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
4454 4455
{
	/* set type for advanced descriptor with frame checksum insertion */
4456 4457 4458
	u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
		       E1000_ADVTXD_DCMD_DEXT |
		       E1000_ADVTXD_DCMD_IFCS;
4459 4460

	/* set HW vlan bit if vlan is present */
4461 4462 4463 4464 4465 4466
	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
				 (E1000_ADVTXD_DCMD_VLE));

	/* set segmentation bits for TSO */
	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
				 (E1000_ADVTXD_DCMD_TSE));
4467 4468

	/* set timestamp bit if present */
4469 4470
	cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
				 (E1000_ADVTXD_MAC_TSTAMP));
4471

4472 4473
	/* insert frame checksum */
	cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS);
4474 4475 4476 4477

	return cmd_type;
}

4478 4479 4480
static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
				 union e1000_adv_tx_desc *tx_desc,
				 u32 tx_flags, unsigned int paylen)
4481 4482 4483
{
	u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;

4484 4485
	/* 82575 requires a unique index per ring */
	if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4486 4487 4488
		olinfo_status |= tx_ring->reg_idx << 4;

	/* insert L4 checksum */
4489 4490 4491
	olinfo_status |= IGB_SET_FLAG(tx_flags,
				      IGB_TX_FLAGS_CSUM,
				      (E1000_TXD_POPTS_TXSM << 8));
4492

4493 4494 4495 4496
	/* insert IPv4 checksum */
	olinfo_status |= IGB_SET_FLAG(tx_flags,
				      IGB_TX_FLAGS_IPV4,
				      (E1000_TXD_POPTS_IXSM << 8));
4497

4498
	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4499 4500
}

4501 4502
static void igb_tx_map(struct igb_ring *tx_ring,
		       struct igb_tx_buffer *first,
4503
		       const u8 hdr_len)
4504
{
4505
	struct sk_buff *skb = first->skb;
4506
	struct igb_tx_buffer *tx_buffer;
4507
	union e1000_adv_tx_desc *tx_desc;
4508
	struct skb_frag_struct *frag;
4509
	dma_addr_t dma;
4510
	unsigned int data_len, size;
4511
	u32 tx_flags = first->tx_flags;
4512
	u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
4513 4514 4515 4516
	u16 i = tx_ring->next_to_use;

	tx_desc = IGB_TX_DESC(tx_ring, i);

4517 4518 4519 4520
	igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);

	size = skb_headlen(skb);
	data_len = skb->data_len;
4521 4522

	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4523

4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534
	tx_buffer = first;

	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
		if (dma_mapping_error(tx_ring->dev, dma))
			goto dma_error;

		/* record length, and DMA address */
		dma_unmap_len_set(tx_buffer, len, size);
		dma_unmap_addr_set(tx_buffer, dma, dma);

		tx_desc->read.buffer_addr = cpu_to_le64(dma);
4535 4536 4537

		while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
			tx_desc->read.cmd_type_len =
4538
				cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
4539 4540 4541 4542 4543 4544 4545

			i++;
			tx_desc++;
			if (i == tx_ring->count) {
				tx_desc = IGB_TX_DESC(tx_ring, 0);
				i = 0;
			}
4546
			tx_desc->read.olinfo_status = 0;
4547 4548 4549 4550 4551 4552 4553 4554 4555

			dma += IGB_MAX_DATA_PER_TXD;
			size -= IGB_MAX_DATA_PER_TXD;

			tx_desc->read.buffer_addr = cpu_to_le64(dma);
		}

		if (likely(!data_len))
			break;
4556

4557
		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
4558

4559
		i++;
4560 4561 4562
		tx_desc++;
		if (i == tx_ring->count) {
			tx_desc = IGB_TX_DESC(tx_ring, 0);
4563
			i = 0;
4564
		}
4565
		tx_desc->read.olinfo_status = 0;
4566

4567
		size = skb_frag_size(frag);
4568 4569 4570
		data_len -= size;

		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4571
				       size, DMA_TO_DEVICE);
4572

4573
		tx_buffer = &tx_ring->tx_buffer_info[i];
4574 4575
	}

4576
	/* write last descriptor with RS and EOP bits */
4577 4578
	cmd_type |= size | IGB_TXD_DCMD;
	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
4579

4580 4581
	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);

4582 4583 4584
	/* set the timestamp */
	first->time_stamp = jiffies;

4585
	/* Force memory writes to complete before letting h/w know there
4586 4587 4588 4589 4590 4591 4592 4593
	 * are new descriptors to fetch.  (Only applicable for weak-ordered
	 * memory model archs, such as IA-64).
	 *
	 * We also need this memory barrier to make certain all of the
	 * status bits have been updated before next_to_watch is written.
	 */
	wmb();

4594
	/* set next_to_watch value indicating a packet is present */
4595
	first->next_to_watch = tx_desc;
4596

4597 4598 4599
	i++;
	if (i == tx_ring->count)
		i = 0;
4600

4601
	tx_ring->next_to_use = i;
4602

4603
	writel(i, tx_ring->tail);
4604

4605
	/* we need this if more than one processor can write to our tail
4606 4607
	 * at a time, it synchronizes IO on IA64/Altix systems
	 */
4608 4609 4610 4611 4612 4613 4614 4615 4616
	mmiowb();

	return;

dma_error:
	dev_err(tx_ring->dev, "TX DMA map failed\n");

	/* clear dma mappings for failed tx_buffer_info map */
	for (;;) {
4617 4618 4619
		tx_buffer = &tx_ring->tx_buffer_info[i];
		igb_unmap_and_free_tx_resource(tx_ring, tx_buffer);
		if (tx_buffer == first)
4620
			break;
4621 4622
		if (i == 0)
			i = tx_ring->count;
4623 4624 4625
		i--;
	}

4626 4627 4628
	tx_ring->next_to_use = i;
}

4629
static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4630
{
4631 4632
	struct net_device *netdev = tx_ring->netdev;

4633 4634
	netif_stop_subqueue(netdev, tx_ring->queue_index);

4635 4636
	/* Herbert's original patch had:
	 *  smp_mb__after_netif_stop_queue();
4637 4638
	 * but since that doesn't exist yet, just open code it.
	 */
4639 4640 4641
	smp_mb();

	/* We need to check again in a case another CPU has just
4642 4643
	 * made room available.
	 */
4644
	if (igb_desc_unused(tx_ring) < size)
4645 4646 4647
		return -EBUSY;

	/* A reprieve! */
4648
	netif_wake_subqueue(netdev, tx_ring->queue_index);
Eric Dumazet's avatar
Eric Dumazet committed
4649 4650 4651 4652 4653

	u64_stats_update_begin(&tx_ring->tx_syncp2);
	tx_ring->tx_stats.restart_queue2++;
	u64_stats_update_end(&tx_ring->tx_syncp2);

4654 4655 4656
	return 0;
}

4657
static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4658
{
4659
	if (igb_desc_unused(tx_ring) >= size)
4660
		return 0;
4661
	return __igb_maybe_stop_tx(tx_ring, size);
4662 4663
}

4664 4665
netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
				struct igb_ring *tx_ring)
4666
{
4667
	struct igb_tx_buffer *first;
4668
	int tso;
Nick Nunley's avatar
Nick Nunley committed
4669
	u32 tx_flags = 0;
4670
	u16 count = TXD_USE_COUNT(skb_headlen(skb));
4671
	__be16 protocol = vlan_get_protocol(skb);
Nick Nunley's avatar
Nick Nunley committed
4672
	u8 hdr_len = 0;
4673

4674 4675
	/* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
	 *       + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
4676 4677
	 *       + 2 desc gap to keep tail from touching head,
	 *       + 1 desc for context descriptor,
4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688
	 * otherwise try next time
	 */
	if (NETDEV_FRAG_PAGE_MAX_SIZE > IGB_MAX_DATA_PER_TXD) {
		unsigned short f;
		for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
			count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
	} else {
		count += skb_shinfo(skb)->nr_frags;
	}

	if (igb_maybe_stop_tx(tx_ring, count + 3)) {
4689 4690 4691
		/* this is a hard error */
		return NETDEV_TX_BUSY;
	}
4692

4693 4694 4695 4696 4697 4698
	/* record the location of the first descriptor for this packet */
	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
	first->skb = skb;
	first->bytecount = skb->len;
	first->gso_segs = 1;

4699 4700
	skb_tx_timestamp(skb);

4701 4702
	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
		struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
4703

4704 4705 4706 4707 4708 4709 4710 4711 4712
		if (!(adapter->ptp_tx_skb)) {
			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
			tx_flags |= IGB_TX_FLAGS_TSTAMP;

			adapter->ptp_tx_skb = skb_get(skb);
			adapter->ptp_tx_start = jiffies;
			if (adapter->hw.mac.type == e1000_82576)
				schedule_work(&adapter->ptp_tx_work);
		}
4713
	}
4714

4715
	if (vlan_tx_tag_present(skb)) {
4716 4717 4718 4719
		tx_flags |= IGB_TX_FLAGS_VLAN;
		tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
	}

4720 4721 4722
	/* record initial flags and protocol */
	first->tx_flags = tx_flags;
	first->protocol = protocol;
4723

4724 4725
	tso = igb_tso(tx_ring, first, &hdr_len);
	if (tso < 0)
4726
		goto out_drop;
4727 4728
	else if (!tso)
		igb_tx_csum(tx_ring, first);
4729

4730
	igb_tx_map(tx_ring, first, hdr_len);
4731 4732

	/* Make sure there is space in the ring for the next send. */
4733
	igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
4734

4735
	return NETDEV_TX_OK;
4736 4737

out_drop:
4738 4739
	igb_unmap_and_free_tx_resource(tx_ring, first);

4740
	return NETDEV_TX_OK;
4741 4742
}

4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753
static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
						    struct sk_buff *skb)
{
	unsigned int r_idx = skb->queue_mapping;

	if (r_idx >= adapter->num_tx_queues)
		r_idx = r_idx % adapter->num_tx_queues;

	return adapter->tx_ring[r_idx];
}

4754 4755
static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
				  struct net_device *netdev)
4756 4757
{
	struct igb_adapter *adapter = netdev_priv(netdev);
4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768

	if (test_bit(__IGB_DOWN, &adapter->state)) {
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
	}

	if (skb->len <= 0) {
		dev_kfree_skb_any(skb);
		return NETDEV_TX_OK;
	}

4769
	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
4770 4771
	 * in order to meet this minimum size requirement.
	 */
4772 4773
	if (unlikely(skb->len < 17)) {
		if (skb_pad(skb, 17 - skb->len))
4774 4775
			return NETDEV_TX_OK;
		skb->len = 17;
4776
		skb_set_tail_pointer(skb, 17);
4777
	}
4778

4779
	return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4780 4781 4782
}

/**
4783 4784
 *  igb_tx_timeout - Respond to a Tx Hang
 *  @netdev: network interface device structure
4785 4786 4787 4788 4789 4790 4791 4792
 **/
static void igb_tx_timeout(struct net_device *netdev)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;

	/* Do the reset outside of interrupt context */
	adapter->tx_timeout_count++;
4793

4794
	if (hw->mac.type >= e1000_82580)
4795 4796
		hw->dev_spec._82575.global_device_reset = true;

4797
	schedule_work(&adapter->reset_task);
4798 4799
	wr32(E1000_EICS,
	     (adapter->eims_enable_mask & ~adapter->eims_other));
4800 4801 4802 4803 4804 4805 4806
}

static void igb_reset_task(struct work_struct *work)
{
	struct igb_adapter *adapter;
	adapter = container_of(work, struct igb_adapter, reset_task);

4807 4808
	igb_dump(adapter);
	netdev_err(adapter->netdev, "Reset adapter\n");
4809 4810 4811 4812
	igb_reinit_locked(adapter);
}

/**
4813 4814 4815
 *  igb_get_stats64 - Get System Network Statistics
 *  @netdev: network interface device structure
 *  @stats: rtnl_link_stats64 pointer
4816
 **/
Eric Dumazet's avatar
Eric Dumazet committed
4817
static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4818
						struct rtnl_link_stats64 *stats)
4819
{
Eric Dumazet's avatar
Eric Dumazet committed
4820 4821 4822 4823 4824 4825 4826 4827
	struct igb_adapter *adapter = netdev_priv(netdev);

	spin_lock(&adapter->stats64_lock);
	igb_update_stats(adapter, &adapter->stats64);
	memcpy(stats, &adapter->stats64, sizeof(*stats));
	spin_unlock(&adapter->stats64_lock);

	return stats;
4828 4829 4830
}

/**
4831 4832 4833
 *  igb_change_mtu - Change the Maximum Transfer Unit
 *  @netdev: network interface device structure
 *  @new_mtu: new value for maximum frame size
4834
 *
4835
 *  Returns 0 on success, negative on failure
4836 4837 4838 4839
 **/
static int igb_change_mtu(struct net_device *netdev, int new_mtu)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
4840
	struct pci_dev *pdev = adapter->pdev;
4841
	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4842

4843
	if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4844
		dev_err(&pdev->dev, "Invalid MTU setting\n");
4845 4846 4847
		return -EINVAL;
	}

4848
#define MAX_STD_JUMBO_FRAME_SIZE 9238
4849
	if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4850
		dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4851 4852 4853
		return -EINVAL;
	}

4854 4855 4856 4857
	/* adjust max frame to be at least the size of a standard frame */
	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
		max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;

4858 4859
	while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
		msleep(1);
4860

4861 4862
	/* igb_down has a dependency on max_frame_size */
	adapter->max_frame_size = max_frame;
4863

4864 4865
	if (netif_running(netdev))
		igb_down(adapter);
4866

4867
	dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881
		 netdev->mtu, new_mtu);
	netdev->mtu = new_mtu;

	if (netif_running(netdev))
		igb_up(adapter);
	else
		igb_reset(adapter);

	clear_bit(__IGB_RESETTING, &adapter->state);

	return 0;
}

/**
4882 4883
 *  igb_update_stats - Update the board statistics counters
 *  @adapter: board private structure
4884
 **/
Eric Dumazet's avatar
Eric Dumazet committed
4885 4886
void igb_update_stats(struct igb_adapter *adapter,
		      struct rtnl_link_stats64 *net_stats)
4887 4888 4889
{
	struct e1000_hw *hw = &adapter->hw;
	struct pci_dev *pdev = adapter->pdev;
4890
	u32 reg, mpc;
4891
	u16 phy_tmp;
4892 4893
	int i;
	u64 bytes, packets;
Eric Dumazet's avatar
Eric Dumazet committed
4894 4895
	unsigned int start;
	u64 _bytes, _packets;
4896 4897 4898

#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF

4899
	/* Prevent stats update while adapter is being reset, or if the pci
4900 4901 4902 4903 4904 4905 4906
	 * connection is down.
	 */
	if (adapter->link_speed == 0)
		return;
	if (pci_channel_offline(pdev))
		return;

4907 4908
	bytes = 0;
	packets = 0;
4909 4910

	rcu_read_lock();
4911
	for (i = 0; i < adapter->num_rx_queues; i++) {
4912
		u32 rqdpc = rd32(E1000_RQDPC(i));
4913
		struct igb_ring *ring = adapter->rx_ring[i];
Eric Dumazet's avatar
Eric Dumazet committed
4914

4915 4916 4917 4918
		if (rqdpc) {
			ring->rx_stats.drops += rqdpc;
			net_stats->rx_fifo_errors += rqdpc;
		}
Eric Dumazet's avatar
Eric Dumazet committed
4919 4920 4921 4922 4923 4924 4925 4926

		do {
			start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
			_bytes = ring->rx_stats.bytes;
			_packets = ring->rx_stats.packets;
		} while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
		bytes += _bytes;
		packets += _packets;
4927 4928
	}

4929 4930
	net_stats->rx_bytes = bytes;
	net_stats->rx_packets = packets;
4931 4932 4933 4934

	bytes = 0;
	packets = 0;
	for (i = 0; i < adapter->num_tx_queues; i++) {
4935
		struct igb_ring *ring = adapter->tx_ring[i];
Eric Dumazet's avatar
Eric Dumazet committed
4936 4937 4938 4939 4940 4941 4942
		do {
			start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
			_bytes = ring->tx_stats.bytes;
			_packets = ring->tx_stats.packets;
		} while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
		bytes += _bytes;
		packets += _packets;
4943
	}
4944 4945
	net_stats->tx_bytes = bytes;
	net_stats->tx_packets = packets;
4946
	rcu_read_unlock();
4947 4948

	/* read stats registers */
4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965
	adapter->stats.crcerrs += rd32(E1000_CRCERRS);
	adapter->stats.gprc += rd32(E1000_GPRC);
	adapter->stats.gorc += rd32(E1000_GORCL);
	rd32(E1000_GORCH); /* clear GORCL */
	adapter->stats.bprc += rd32(E1000_BPRC);
	adapter->stats.mprc += rd32(E1000_MPRC);
	adapter->stats.roc += rd32(E1000_ROC);

	adapter->stats.prc64 += rd32(E1000_PRC64);
	adapter->stats.prc127 += rd32(E1000_PRC127);
	adapter->stats.prc255 += rd32(E1000_PRC255);
	adapter->stats.prc511 += rd32(E1000_PRC511);
	adapter->stats.prc1023 += rd32(E1000_PRC1023);
	adapter->stats.prc1522 += rd32(E1000_PRC1522);
	adapter->stats.symerrs += rd32(E1000_SYMERRS);
	adapter->stats.sec += rd32(E1000_SEC);

4966 4967 4968
	mpc = rd32(E1000_MPC);
	adapter->stats.mpc += mpc;
	net_stats->rx_fifo_errors += mpc;
4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982
	adapter->stats.scc += rd32(E1000_SCC);
	adapter->stats.ecol += rd32(E1000_ECOL);
	adapter->stats.mcc += rd32(E1000_MCC);
	adapter->stats.latecol += rd32(E1000_LATECOL);
	adapter->stats.dc += rd32(E1000_DC);
	adapter->stats.rlec += rd32(E1000_RLEC);
	adapter->stats.xonrxc += rd32(E1000_XONRXC);
	adapter->stats.xontxc += rd32(E1000_XONTXC);
	adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
	adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
	adapter->stats.fcruc += rd32(E1000_FCRUC);
	adapter->stats.gptc += rd32(E1000_GPTC);
	adapter->stats.gotc += rd32(E1000_GOTCL);
	rd32(E1000_GOTCH); /* clear GOTCL */
4983
	adapter->stats.rnbc += rd32(E1000_RNBC);
4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000
	adapter->stats.ruc += rd32(E1000_RUC);
	adapter->stats.rfc += rd32(E1000_RFC);
	adapter->stats.rjc += rd32(E1000_RJC);
	adapter->stats.tor += rd32(E1000_TORH);
	adapter->stats.tot += rd32(E1000_TOTH);
	adapter->stats.tpr += rd32(E1000_TPR);

	adapter->stats.ptc64 += rd32(E1000_PTC64);
	adapter->stats.ptc127 += rd32(E1000_PTC127);
	adapter->stats.ptc255 += rd32(E1000_PTC255);
	adapter->stats.ptc511 += rd32(E1000_PTC511);
	adapter->stats.ptc1023 += rd32(E1000_PTC1023);
	adapter->stats.ptc1522 += rd32(E1000_PTC1522);

	adapter->stats.mptc += rd32(E1000_MPTC);
	adapter->stats.bptc += rd32(E1000_BPTC);

5001 5002
	adapter->stats.tpt += rd32(E1000_TPT);
	adapter->stats.colc += rd32(E1000_COLC);
5003 5004

	adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
5005 5006 5007 5008
	/* read internal phy specific stats */
	reg = rd32(E1000_CTRL_EXT);
	if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
		adapter->stats.rxerrc += rd32(E1000_RXERRC);
5009 5010 5011 5012 5013

		/* this stat has invalid values on i210/i211 */
		if ((hw->mac.type != e1000_i210) &&
		    (hw->mac.type != e1000_i211))
			adapter->stats.tncrs += rd32(E1000_TNCRS);
5014 5015
	}

5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029
	adapter->stats.tsctc += rd32(E1000_TSCTC);
	adapter->stats.tsctfc += rd32(E1000_TSCTFC);

	adapter->stats.iac += rd32(E1000_IAC);
	adapter->stats.icrxoc += rd32(E1000_ICRXOC);
	adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
	adapter->stats.icrxatc += rd32(E1000_ICRXATC);
	adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
	adapter->stats.ictxatc += rd32(E1000_ICTXATC);
	adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
	adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
	adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);

	/* Fill out the OS statistics structure */
5030 5031
	net_stats->multicast = adapter->stats.mprc;
	net_stats->collisions = adapter->stats.colc;
5032 5033 5034 5035

	/* Rx Errors */

	/* RLEC on some newer hardware can be incorrect so build
5036 5037
	 * our own version based on RUC and ROC
	 */
5038
	net_stats->rx_errors = adapter->stats.rxerrc +
5039 5040 5041
		adapter->stats.crcerrs + adapter->stats.algnerrc +
		adapter->stats.ruc + adapter->stats.roc +
		adapter->stats.cexterr;
5042 5043 5044 5045 5046
	net_stats->rx_length_errors = adapter->stats.ruc +
				      adapter->stats.roc;
	net_stats->rx_crc_errors = adapter->stats.crcerrs;
	net_stats->rx_frame_errors = adapter->stats.algnerrc;
	net_stats->rx_missed_errors = adapter->stats.mpc;
5047 5048

	/* Tx Errors */
5049 5050 5051 5052 5053
	net_stats->tx_errors = adapter->stats.ecol +
			       adapter->stats.latecol;
	net_stats->tx_aborted_errors = adapter->stats.ecol;
	net_stats->tx_window_errors = adapter->stats.latecol;
	net_stats->tx_carrier_errors = adapter->stats.tncrs;
5054 5055 5056 5057 5058 5059

	/* Tx Dropped needs to be maintained elsewhere */

	/* Phy Stats */
	if (hw->phy.media_type == e1000_media_type_copper) {
		if ((adapter->link_speed == SPEED_1000) &&
5060
		   (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
5061 5062 5063 5064 5065 5066 5067 5068 5069
			phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
			adapter->phy_stats.idle_errors += phy_tmp;
		}
	}

	/* Management Stats */
	adapter->stats.mgptc += rd32(E1000_MGTPTC);
	adapter->stats.mgprc += rd32(E1000_MGTPRC);
	adapter->stats.mgpdc += rd32(E1000_MGTPDC);
5070 5071 5072 5073 5074 5075 5076 5077 5078

	/* OS2BMC Stats */
	reg = rd32(E1000_MANC);
	if (reg & E1000_MANC_EN_BMC2OS) {
		adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
		adapter->stats.o2bspc += rd32(E1000_O2BSPC);
		adapter->stats.b2ospc += rd32(E1000_B2OSPC);
		adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
	}
5079 5080 5081 5082
}

static irqreturn_t igb_msix_other(int irq, void *data)
{
5083
	struct igb_adapter *adapter = data;
5084
	struct e1000_hw *hw = &adapter->hw;
5085 5086
	u32 icr = rd32(E1000_ICR);
	/* reading ICR causes bit 31 of EICR to be cleared */
5087

5088 5089 5090
	if (icr & E1000_ICR_DRSTA)
		schedule_work(&adapter->reset_task);

5091
	if (icr & E1000_ICR_DOUTSYNC) {
5092 5093
		/* HW is reporting DMA is out of sync */
		adapter->stats.doosync++;
5094 5095
		/* The DMA Out of Sync is also indication of a spoof event
		 * in IOV mode. Check the Wrong VM Behavior register to
5096 5097
		 * see if it is really a spoof event.
		 */
5098
		igb_check_wvbr(adapter);
5099
	}
5100

5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111
	/* Check for a mailbox event */
	if (icr & E1000_ICR_VMMB)
		igb_msg_task(adapter);

	if (icr & E1000_ICR_LSC) {
		hw->mac.get_link_status = 1;
		/* guard against interrupt when we're going down */
		if (!test_bit(__IGB_DOWN, &adapter->state))
			mod_timer(&adapter->watchdog_timer, jiffies + 1);
	}

5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122
	if (icr & E1000_ICR_TS) {
		u32 tsicr = rd32(E1000_TSICR);

		if (tsicr & E1000_TSICR_TXTS) {
			/* acknowledge the interrupt */
			wr32(E1000_TSICR, E1000_TSICR_TXTS);
			/* retrieve hardware timestamp */
			schedule_work(&adapter->ptp_tx_work);
		}
	}

5123
	wr32(E1000_EIMS, adapter->eims_other);
5124 5125 5126 5127

	return IRQ_HANDLED;
}

5128
static void igb_write_itr(struct igb_q_vector *q_vector)
5129
{
5130
	struct igb_adapter *adapter = q_vector->adapter;
5131
	u32 itr_val = q_vector->itr_val & 0x7FFC;
5132

5133 5134
	if (!q_vector->set_itr)
		return;
5135

5136 5137
	if (!itr_val)
		itr_val = 0x4;
5138

5139 5140
	if (adapter->hw.mac.type == e1000_82575)
		itr_val |= itr_val << 16;
5141
	else
5142
		itr_val |= E1000_EITR_CNT_IGNR;
5143

5144 5145
	writel(itr_val, q_vector->itr_register);
	q_vector->set_itr = 0;
5146 5147
}

5148
static irqreturn_t igb_msix_ring(int irq, void *data)
5149
{
5150
	struct igb_q_vector *q_vector = data;
5151

5152 5153
	/* Write the ITR value calculated from the previous interrupt. */
	igb_write_itr(q_vector);
5154

5155
	napi_schedule(&q_vector->napi);
5156

5157
	return IRQ_HANDLED;
Jeb Cramer's avatar
Jeb Cramer committed
5158 5159
}

5160
#ifdef CONFIG_IGB_DCA
5161 5162 5163 5164 5165 5166 5167 5168 5169 5170
static void igb_update_tx_dca(struct igb_adapter *adapter,
			      struct igb_ring *tx_ring,
			      int cpu)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);

	if (hw->mac.type != e1000_82575)
		txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT;

5171
	/* We can enable relaxed ordering for reads, but not writes when
5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191
	 * DCA is enabled.  This is due to a known issue in some chipsets
	 * which will cause the DCA tag to be cleared.
	 */
	txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
		  E1000_DCA_TXCTRL_DATA_RRO_EN |
		  E1000_DCA_TXCTRL_DESC_DCA_EN;

	wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
}

static void igb_update_rx_dca(struct igb_adapter *adapter,
			      struct igb_ring *rx_ring,
			      int cpu)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);

	if (hw->mac.type != e1000_82575)
		rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT;

5192
	/* We can enable relaxed ordering for reads, but not writes when
5193 5194 5195 5196 5197 5198 5199 5200 5201
	 * DCA is enabled.  This is due to a known issue in some chipsets
	 * which will cause the DCA tag to be cleared.
	 */
	rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
		  E1000_DCA_RXCTRL_DESC_DCA_EN;

	wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
}

5202
static void igb_update_dca(struct igb_q_vector *q_vector)
Jeb Cramer's avatar
Jeb Cramer committed
5203
{
5204
	struct igb_adapter *adapter = q_vector->adapter;
Jeb Cramer's avatar
Jeb Cramer committed
5205 5206
	int cpu = get_cpu();

5207 5208 5209
	if (q_vector->cpu == cpu)
		goto out_no_update;

5210 5211 5212 5213 5214 5215
	if (q_vector->tx.ring)
		igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);

	if (q_vector->rx.ring)
		igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);

5216 5217
	q_vector->cpu = cpu;
out_no_update:
Jeb Cramer's avatar
Jeb Cramer committed
5218 5219 5220 5221 5222
	put_cpu();
}

static void igb_setup_dca(struct igb_adapter *adapter)
{
5223
	struct e1000_hw *hw = &adapter->hw;
Jeb Cramer's avatar
Jeb Cramer committed
5224 5225
	int i;

5226
	if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
Jeb Cramer's avatar
Jeb Cramer committed
5227 5228
		return;

5229 5230 5231
	/* Always use CB2 mode, difference is masked in the CB driver. */
	wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);

5232
	for (i = 0; i < adapter->num_q_vectors; i++) {
5233 5234
		adapter->q_vector[i]->cpu = -1;
		igb_update_dca(adapter->q_vector[i]);
Jeb Cramer's avatar
Jeb Cramer committed
5235 5236 5237 5238 5239 5240 5241
	}
}

static int __igb_notify_dca(struct device *dev, void *data)
{
	struct net_device *netdev = dev_get_drvdata(dev);
	struct igb_adapter *adapter = netdev_priv(netdev);
5242
	struct pci_dev *pdev = adapter->pdev;
Jeb Cramer's avatar
Jeb Cramer committed
5243 5244 5245 5246 5247 5248
	struct e1000_hw *hw = &adapter->hw;
	unsigned long event = *(unsigned long *)data;

	switch (event) {
	case DCA_PROVIDER_ADD:
		/* if already enabled, don't do it again */
5249
		if (adapter->flags & IGB_FLAG_DCA_ENABLED)
Jeb Cramer's avatar
Jeb Cramer committed
5250 5251
			break;
		if (dca_add_requester(dev) == 0) {
5252
			adapter->flags |= IGB_FLAG_DCA_ENABLED;
5253
			dev_info(&pdev->dev, "DCA enabled\n");
Jeb Cramer's avatar
Jeb Cramer committed
5254 5255 5256 5257 5258
			igb_setup_dca(adapter);
			break;
		}
		/* Fall Through since DCA is disabled. */
	case DCA_PROVIDER_REMOVE:
5259
		if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
Jeb Cramer's avatar
Jeb Cramer committed
5260
			/* without this a class_device is left
5261 5262
			 * hanging around in the sysfs model
			 */
Jeb Cramer's avatar
Jeb Cramer committed
5263
			dca_remove_requester(dev);
5264
			dev_info(&pdev->dev, "DCA disabled\n");
5265
			adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
Alexander Duyck's avatar
Alexander Duyck committed
5266
			wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
Jeb Cramer's avatar
Jeb Cramer committed
5267 5268 5269
		}
		break;
	}
5270

Jeb Cramer's avatar
Jeb Cramer committed
5271
	return 0;
5272 5273
}

Jeb Cramer's avatar
Jeb Cramer committed
5274
static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
5275
			  void *p)
Jeb Cramer's avatar
Jeb Cramer committed
5276 5277 5278 5279
{
	int ret_val;

	ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
5280
					 __igb_notify_dca);
Jeb Cramer's avatar
Jeb Cramer committed
5281 5282 5283

	return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
}
5284
#endif /* CONFIG_IGB_DCA */
5285

5286 5287 5288 5289 5290
#ifdef CONFIG_PCI_IOV
static int igb_vf_configure(struct igb_adapter *adapter, int vf)
{
	unsigned char mac_addr[ETH_ALEN];

5291
	eth_zero_addr(mac_addr);
5292 5293
	igb_set_vf_mac(adapter, vf, mac_addr);

5294 5295 5296
	/* By default spoof check is enabled for all VFs */
	adapter->vf_data[vf].spoofchk_enabled = true;

5297
	return 0;
5298 5299 5300
}

#endif
5301 5302 5303 5304 5305 5306 5307 5308
static void igb_ping_all_vfs(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 ping;
	int i;

	for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
		ping = E1000_PF_CONTROL_MSG;
5309
		if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5310 5311 5312 5313 5314
			ping |= E1000_VT_MSGTYPE_CTS;
		igb_write_mbx(hw, &ping, 1, i);
	}
}

5315 5316 5317 5318 5319 5320
static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 vmolr = rd32(E1000_VMOLR(vf));
	struct vf_data_storage *vf_data = &adapter->vf_data[vf];

5321
	vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5322
			    IGB_VF_FLAG_MULTI_PROMISC);
5323 5324 5325 5326
	vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);

	if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
		vmolr |= E1000_VMOLR_MPME;
5327
		vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5328 5329
		*msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
	} else {
5330
		/* if we have hashes and we are clearing a multicast promisc
5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352
		 * flag we need to write the hashes to the MTA as this step
		 * was previously skipped
		 */
		if (vf_data->num_vf_mc_hashes > 30) {
			vmolr |= E1000_VMOLR_MPME;
		} else if (vf_data->num_vf_mc_hashes) {
			int j;
			vmolr |= E1000_VMOLR_ROMPE;
			for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
				igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
		}
	}

	wr32(E1000_VMOLR(vf), vmolr);

	/* there are flags left unprocessed, likely not supported */
	if (*msgbuf & E1000_VT_MSGINFO_MASK)
		return -EINVAL;

	return 0;
}

5353 5354 5355 5356 5357 5358 5359 5360
static int igb_set_vf_multicasts(struct igb_adapter *adapter,
				  u32 *msgbuf, u32 vf)
{
	int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
	u16 *hash_list = (u16 *)&msgbuf[1];
	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
	int i;

5361
	/* salt away the number of multicast addresses assigned
5362 5363 5364 5365 5366
	 * to this VF for later use to restore when the PF multi cast
	 * list changes
	 */
	vf_data->num_vf_mc_hashes = n;

5367 5368 5369 5370 5371
	/* only up to 30 hash values supported */
	if (n > 30)
		n = 30;

	/* store the hashes for later use */
5372
	for (i = 0; i < n; i++)
5373
		vf_data->vf_mc_hashes[i] = hash_list[i];
5374 5375

	/* Flush and reset the mta with the new values */
5376
	igb_set_rx_mode(adapter->netdev);
5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387

	return 0;
}

static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	struct vf_data_storage *vf_data;
	int i, j;

	for (i = 0; i < adapter->vfs_allocated_count; i++) {
5388 5389 5390
		u32 vmolr = rd32(E1000_VMOLR(i));
		vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);

5391
		vf_data = &adapter->vf_data[i];
5392 5393 5394 5395 5396 5397 5398 5399 5400 5401

		if ((vf_data->num_vf_mc_hashes > 30) ||
		    (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
			vmolr |= E1000_VMOLR_MPME;
		} else if (vf_data->num_vf_mc_hashes) {
			vmolr |= E1000_VMOLR_ROMPE;
			for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
				igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
		}
		wr32(E1000_VMOLR(i), vmolr);
5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429
	}
}

static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 pool_mask, reg, vid;
	int i;

	pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);

	/* Find the vlan filter for this id */
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
		reg = rd32(E1000_VLVF(i));

		/* remove the vf from the pool */
		reg &= ~pool_mask;

		/* if pool is empty then remove entry from vfta */
		if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
		    (reg & E1000_VLVF_VLANID_ENABLE)) {
			reg = 0;
			vid = reg & E1000_VLVF_VLANID_MASK;
			igb_vfta_set(hw, vid, false);
		}

		wr32(E1000_VLVF(i), reg);
	}
5430 5431

	adapter->vf_data[vf].vlans_enabled = 0;
5432 5433 5434 5435 5436 5437 5438
}

static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 reg, i;

5439 5440 5441 5442 5443
	/* The vlvf table only exists on 82576 hardware and newer */
	if (hw->mac.type < e1000_82576)
		return -1;

	/* we only need to do this if VMDq is enabled */
5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472
	if (!adapter->vfs_allocated_count)
		return -1;

	/* Find the vlan filter for this id */
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
		reg = rd32(E1000_VLVF(i));
		if ((reg & E1000_VLVF_VLANID_ENABLE) &&
		    vid == (reg & E1000_VLVF_VLANID_MASK))
			break;
	}

	if (add) {
		if (i == E1000_VLVF_ARRAY_SIZE) {
			/* Did not find a matching VLAN ID entry that was
			 * enabled.  Search for a free filter entry, i.e.
			 * one without the enable bit set
			 */
			for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
				reg = rd32(E1000_VLVF(i));
				if (!(reg & E1000_VLVF_VLANID_ENABLE))
					break;
			}
		}
		if (i < E1000_VLVF_ARRAY_SIZE) {
			/* Found an enabled/available entry */
			reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);

			/* if !enabled we need to set this up in vfta */
			if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5473 5474
				/* add VID to filter table */
				igb_vfta_set(hw, vid, true);
5475 5476
				reg |= E1000_VLVF_VLANID_ENABLE;
			}
Alexander Duyck's avatar
Alexander Duyck committed
5477 5478
			reg &= ~E1000_VLVF_VLANID_MASK;
			reg |= vid;
5479
			wr32(E1000_VLVF(i), reg);
5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494

			/* do not modify RLPML for PF devices */
			if (vf >= adapter->vfs_allocated_count)
				return 0;

			if (!adapter->vf_data[vf].vlans_enabled) {
				u32 size;
				reg = rd32(E1000_VMOLR(vf));
				size = reg & E1000_VMOLR_RLPML_MASK;
				size += 4;
				reg &= ~E1000_VMOLR_RLPML_MASK;
				reg |= size;
				wr32(E1000_VMOLR(vf), reg);
			}

5495
			adapter->vf_data[vf].vlans_enabled++;
5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506
		}
	} else {
		if (i < E1000_VLVF_ARRAY_SIZE) {
			/* remove vf from the pool */
			reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
			/* if pool is empty then remove entry from vfta */
			if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
				reg = 0;
				igb_vfta_set(hw, vid, false);
			}
			wr32(E1000_VLVF(i), reg);
5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521

			/* do not modify RLPML for PF devices */
			if (vf >= adapter->vfs_allocated_count)
				return 0;

			adapter->vf_data[vf].vlans_enabled--;
			if (!adapter->vf_data[vf].vlans_enabled) {
				u32 size;
				reg = rd32(E1000_VMOLR(vf));
				size = reg & E1000_VMOLR_RLPML_MASK;
				size -= 4;
				reg &= ~E1000_VMOLR_RLPML_MASK;
				reg |= size;
				wr32(E1000_VMOLR(vf), reg);
			}
5522 5523
		}
	}
5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556
	return 0;
}

static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;

	if (vid)
		wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
	else
		wr32(E1000_VMVIR(vf), 0);
}

static int igb_ndo_set_vf_vlan(struct net_device *netdev,
			       int vf, u16 vlan, u8 qos)
{
	int err = 0;
	struct igb_adapter *adapter = netdev_priv(netdev);

	if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
		return -EINVAL;
	if (vlan || qos) {
		err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
		if (err)
			goto out;
		igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
		igb_set_vmolr(adapter, vf, !vlan);
		adapter->vf_data[vf].pf_vlan = vlan;
		adapter->vf_data[vf].pf_qos = qos;
		dev_info(&adapter->pdev->dev,
			 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
		if (test_bit(__IGB_DOWN, &adapter->state)) {
			dev_warn(&adapter->pdev->dev,
5557
				 "The VF VLAN has been set, but the PF device is not up.\n");
5558
			dev_warn(&adapter->pdev->dev,
5559
				 "Bring the PF device up before attempting to use the VF device.\n");
5560 5561 5562
		}
	} else {
		igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5563
			     false, vf);
5564 5565 5566 5567
		igb_set_vmvir(adapter, vlan, vf);
		igb_set_vmolr(adapter, vf, true);
		adapter->vf_data[vf].pf_vlan = 0;
		adapter->vf_data[vf].pf_qos = 0;
5568
	}
5569
out:
5570
	return err;
5571 5572
}

5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592
static int igb_find_vlvf_entry(struct igb_adapter *adapter, int vid)
{
	struct e1000_hw *hw = &adapter->hw;
	int i;
	u32 reg;

	/* Find the vlan filter for this id */
	for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
		reg = rd32(E1000_VLVF(i));
		if ((reg & E1000_VLVF_VLANID_ENABLE) &&
		    vid == (reg & E1000_VLVF_VLANID_MASK))
			break;
	}

	if (i >= E1000_VLVF_ARRAY_SIZE)
		i = -1;

	return i;
}

5593 5594
static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
{
5595
	struct e1000_hw *hw = &adapter->hw;
5596 5597
	int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
	int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5598
	int err = 0;
5599

5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641
	/* If in promiscuous mode we need to make sure the PF also has
	 * the VLAN filter set.
	 */
	if (add && (adapter->netdev->flags & IFF_PROMISC))
		err = igb_vlvf_set(adapter, vid, add,
				   adapter->vfs_allocated_count);
	if (err)
		goto out;

	err = igb_vlvf_set(adapter, vid, add, vf);

	if (err)
		goto out;

	/* Go through all the checks to see if the VLAN filter should
	 * be wiped completely.
	 */
	if (!add && (adapter->netdev->flags & IFF_PROMISC)) {
		u32 vlvf, bits;

		int regndx = igb_find_vlvf_entry(adapter, vid);
		if (regndx < 0)
			goto out;
		/* See if any other pools are set for this VLAN filter
		 * entry other than the PF.
		 */
		vlvf = bits = rd32(E1000_VLVF(regndx));
		bits &= 1 << (E1000_VLVF_POOLSEL_SHIFT +
			      adapter->vfs_allocated_count);
		/* If the filter was removed then ensure PF pool bit
		 * is cleared if the PF only added itself to the pool
		 * because the PF is in promiscuous mode.
		 */
		if ((vlvf & VLAN_VID_MASK) == vid &&
		    !test_bit(vid, adapter->active_vlans) &&
		    !bits)
			igb_vlvf_set(adapter, vid, add,
				     adapter->vfs_allocated_count);
	}

out:
	return err;
5642 5643
}

5644
static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5645
{
5646 5647
	/* clear flags - except flag that indicates PF has set the MAC */
	adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5648
	adapter->vf_data[vf].last_nack = jiffies;
5649 5650

	/* reset offloads to defaults */
5651
	igb_set_vmolr(adapter, vf, true);
5652 5653 5654

	/* reset vlans for device */
	igb_clear_vf_vfta(adapter, vf);
5655 5656 5657 5658 5659 5660
	if (adapter->vf_data[vf].pf_vlan)
		igb_ndo_set_vf_vlan(adapter->netdev, vf,
				    adapter->vf_data[vf].pf_vlan,
				    adapter->vf_data[vf].pf_qos);
	else
		igb_clear_vf_vfta(adapter, vf);
5661 5662 5663 5664 5665

	/* reset multicast table array for vf */
	adapter->vf_data[vf].num_vf_mc_hashes = 0;

	/* Flush and reset the mta with the new values */
5666
	igb_set_rx_mode(adapter->netdev);
5667 5668
}

5669 5670 5671 5672
static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
{
	unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;

5673
	/* clear mac address as we were hotplug removed/added */
5674
	if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5675
		eth_zero_addr(vf_mac);
5676 5677 5678 5679 5680 5681

	/* process remaining reset events */
	igb_vf_reset(adapter, vf);
}

static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5682 5683 5684
{
	struct e1000_hw *hw = &adapter->hw;
	unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5685
	int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5686 5687 5688 5689
	u32 reg, msgbuf[3];
	u8 *addr = (u8 *)(&msgbuf[1]);

	/* process all the same items cleared in a function level reset */
5690
	igb_vf_reset(adapter, vf);
5691 5692

	/* set vf mac address */
5693
	igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5694 5695 5696 5697 5698 5699 5700

	/* enable transmit and receive for vf */
	reg = rd32(E1000_VFTE);
	wr32(E1000_VFTE, reg | (1 << vf));
	reg = rd32(E1000_VFRE);
	wr32(E1000_VFRE, reg | (1 << vf));

5701
	adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5702 5703 5704 5705 5706 5707 5708 5709 5710

	/* reply to reset with ack and vf mac address */
	msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
	memcpy(addr, vf_mac, 6);
	igb_write_mbx(hw, msgbuf, 3, vf);
}

static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
{
5711
	/* The VF MAC Address is stored in a packed array of bytes
Greg Rose's avatar
Greg Rose committed
5712 5713
	 * starting at the second 32 bit word of the msg array
	 */
5714 5715
	unsigned char *addr = (char *)&msg[1];
	int err = -1;
5716

5717 5718
	if (is_valid_ether_addr(addr))
		err = igb_set_vf_mac(adapter, vf, addr);
5719

5720
	return err;
5721 5722 5723 5724 5725
}

static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
{
	struct e1000_hw *hw = &adapter->hw;
5726
	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5727 5728 5729
	u32 msg = E1000_VT_MSGTYPE_NACK;

	/* if device isn't clear to send it shouldn't be reading either */
5730 5731
	if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
	    time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5732
		igb_write_mbx(hw, &msg, 1, vf);
5733
		vf_data->last_nack = jiffies;
5734 5735 5736
	}
}

5737
static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5738
{
5739 5740
	struct pci_dev *pdev = adapter->pdev;
	u32 msgbuf[E1000_VFMAILBOX_SIZE];
5741
	struct e1000_hw *hw = &adapter->hw;
5742
	struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5743 5744
	s32 retval;

5745
	retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5746

5747 5748
	if (retval) {
		/* if receive failed revoke VF CTS stats and restart init */
5749
		dev_err(&pdev->dev, "Error receiving message from VF\n");
5750 5751 5752 5753 5754
		vf_data->flags &= ~IGB_VF_FLAG_CTS;
		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
			return;
		goto out;
	}
5755 5756 5757

	/* this is a message we already processed, do nothing */
	if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5758
		return;
5759

5760
	/* until the vf completes a reset it should not be
5761 5762 5763 5764
	 * allowed to start any configuration.
	 */
	if (msgbuf[0] == E1000_VF_RESET) {
		igb_vf_reset_msg(adapter, vf);
5765
		return;
5766 5767
	}

5768
	if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5769 5770 5771 5772
		if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
			return;
		retval = -1;
		goto out;
5773 5774 5775 5776
	}

	switch ((msgbuf[0] & 0xFFFF)) {
	case E1000_VF_SET_MAC_ADDR:
5777 5778 5779 5780 5781
		retval = -EINVAL;
		if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
			retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
		else
			dev_warn(&pdev->dev,
5782 5783
				 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
				 vf);
5784
		break;
5785 5786 5787
	case E1000_VF_SET_PROMISC:
		retval = igb_set_vf_promisc(adapter, msgbuf, vf);
		break;
5788 5789 5790 5791 5792 5793 5794
	case E1000_VF_SET_MULTICAST:
		retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
		break;
	case E1000_VF_SET_LPE:
		retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
		break;
	case E1000_VF_SET_VLAN:
5795 5796 5797
		retval = -1;
		if (vf_data->pf_vlan)
			dev_warn(&pdev->dev,
5798 5799
				 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
				 vf);
5800 5801
		else
			retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5802 5803
		break;
	default:
5804
		dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5805 5806 5807 5808
		retval = -1;
		break;
	}

5809 5810
	msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
out:
5811 5812 5813 5814 5815 5816 5817
	/* notify the VF of the results of what it sent us */
	if (retval)
		msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
	else
		msgbuf[0] |= E1000_VT_MSGTYPE_ACK;

	igb_write_mbx(hw, msgbuf, 1, vf);
5818
}
5819

5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837
static void igb_msg_task(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 vf;

	for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
		/* process any reset requests */
		if (!igb_check_for_rst(hw, vf))
			igb_vf_reset_event(adapter, vf);

		/* process any messages pending */
		if (!igb_check_for_msg(hw, vf))
			igb_rcv_msg_from_vf(adapter, vf);

		/* process any acks */
		if (!igb_check_for_ack(hw, vf))
			igb_rcv_ack_from_vf(adapter, vf);
	}
5838 5839
}

5840 5841 5842 5843 5844 5845 5846
/**
 *  igb_set_uta - Set unicast filter table address
 *  @adapter: board private structure
 *
 *  The unicast table address is a register array of 32-bit registers.
 *  The table is meant to be used in a way similar to how the MTA is used
 *  however due to certain limitations in the hardware it is necessary to
Lucas De Marchi's avatar
Lucas De Marchi committed
5847 5848
 *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
 *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866
 **/
static void igb_set_uta(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
	int i;

	/* The UTA table only exists on 82576 hardware and newer */
	if (hw->mac.type < e1000_82576)
		return;

	/* we only need to do this if VMDq is enabled */
	if (!adapter->vfs_allocated_count)
		return;

	for (i = 0; i < hw->mac.uta_reg_count; i++)
		array_wr32(E1000_UTA, i, ~0);
}

5867
/**
5868 5869 5870
 *  igb_intr_msi - Interrupt Handler
 *  @irq: interrupt number
 *  @data: pointer to a network interface device structure
5871 5872 5873
 **/
static irqreturn_t igb_intr_msi(int irq, void *data)
{
5874 5875
	struct igb_adapter *adapter = data;
	struct igb_q_vector *q_vector = adapter->q_vector[0];
5876 5877 5878 5879
	struct e1000_hw *hw = &adapter->hw;
	/* read ICR disables interrupts using IAM */
	u32 icr = rd32(E1000_ICR);

5880
	igb_write_itr(q_vector);
5881

5882 5883 5884
	if (icr & E1000_ICR_DRSTA)
		schedule_work(&adapter->reset_task);

5885
	if (icr & E1000_ICR_DOUTSYNC) {
5886 5887 5888 5889
		/* HW is reporting DMA is out of sync */
		adapter->stats.doosync++;
	}

5890 5891 5892 5893 5894 5895
	if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
		hw->mac.get_link_status = 1;
		if (!test_bit(__IGB_DOWN, &adapter->state))
			mod_timer(&adapter->watchdog_timer, jiffies + 1);
	}

5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906
	if (icr & E1000_ICR_TS) {
		u32 tsicr = rd32(E1000_TSICR);

		if (tsicr & E1000_TSICR_TXTS) {
			/* acknowledge the interrupt */
			wr32(E1000_TSICR, E1000_TSICR_TXTS);
			/* retrieve hardware timestamp */
			schedule_work(&adapter->ptp_tx_work);
		}
	}

5907
	napi_schedule(&q_vector->napi);
5908 5909 5910 5911 5912

	return IRQ_HANDLED;
}

/**
5913 5914 5915
 *  igb_intr - Legacy Interrupt Handler
 *  @irq: interrupt number
 *  @data: pointer to a network interface device structure
5916 5917 5918
 **/
static irqreturn_t igb_intr(int irq, void *data)
{
5919 5920
	struct igb_adapter *adapter = data;
	struct igb_q_vector *q_vector = adapter->q_vector[0];
5921 5922
	struct e1000_hw *hw = &adapter->hw;
	/* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5923 5924
	 * need for the IMC write
	 */
5925 5926 5927
	u32 icr = rd32(E1000_ICR);

	/* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5928 5929
	 * not set, then the adapter didn't send an interrupt
	 */
5930 5931 5932
	if (!(icr & E1000_ICR_INT_ASSERTED))
		return IRQ_NONE;

5933 5934
	igb_write_itr(q_vector);

5935 5936 5937
	if (icr & E1000_ICR_DRSTA)
		schedule_work(&adapter->reset_task);

5938
	if (icr & E1000_ICR_DOUTSYNC) {
5939 5940 5941 5942
		/* HW is reporting DMA is out of sync */
		adapter->stats.doosync++;
	}

5943 5944 5945 5946 5947 5948 5949
	if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
		hw->mac.get_link_status = 1;
		/* guard against interrupt when we're going down */
		if (!test_bit(__IGB_DOWN, &adapter->state))
			mod_timer(&adapter->watchdog_timer, jiffies + 1);
	}

5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960
	if (icr & E1000_ICR_TS) {
		u32 tsicr = rd32(E1000_TSICR);

		if (tsicr & E1000_TSICR_TXTS) {
			/* acknowledge the interrupt */
			wr32(E1000_TSICR, E1000_TSICR_TXTS);
			/* retrieve hardware timestamp */
			schedule_work(&adapter->ptp_tx_work);
		}
	}

5961
	napi_schedule(&q_vector->napi);
5962 5963 5964 5965

	return IRQ_HANDLED;
}

5966
static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5967
{
5968
	struct igb_adapter *adapter = q_vector->adapter;
5969
	struct e1000_hw *hw = &adapter->hw;
5970

5971 5972 5973 5974
	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
		if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
			igb_set_itr(q_vector);
5975
		else
5976
			igb_update_ring_itr(q_vector);
5977 5978
	}

5979 5980
	if (!test_bit(__IGB_DOWN, &adapter->state)) {
		if (adapter->msix_entries)
5981
			wr32(E1000_EIMS, q_vector->eims_value);
5982 5983 5984
		else
			igb_irq_enable(adapter);
	}
5985 5986
}

5987
/**
5988 5989 5990
 *  igb_poll - NAPI Rx polling callback
 *  @napi: napi polling structure
 *  @budget: count of how many packets we should handle
5991 5992
 **/
static int igb_poll(struct napi_struct *napi, int budget)
5993
{
5994
	struct igb_q_vector *q_vector = container_of(napi,
5995 5996
						     struct igb_q_vector,
						     napi);
5997
	bool clean_complete = true;
5998

5999
#ifdef CONFIG_IGB_DCA
6000 6001
	if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
		igb_update_dca(q_vector);
Jeb Cramer's avatar
Jeb Cramer committed
6002
#endif
6003
	if (q_vector->tx.ring)
6004
		clean_complete = igb_clean_tx_irq(q_vector);
6005

6006
	if (q_vector->rx.ring)
6007
		clean_complete &= igb_clean_rx_irq(q_vector, budget);
6008

6009 6010 6011
	/* If all work not completed, return budget and keep polling */
	if (!clean_complete)
		return budget;
6012

6013
	/* If not enough Rx work done, exit the polling mode */
6014 6015
	napi_complete(napi);
	igb_ring_irq_enable(q_vector);
6016

6017
	return 0;
6018
}
Al Viro's avatar
Al Viro committed
6019

6020
/**
6021 6022
 *  igb_clean_tx_irq - Reclaim resources after transmit completes
 *  @q_vector: pointer to q_vector containing needed info
6023
 *
6024
 *  returns true if ring is completely cleaned
6025
 **/
6026
static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
6027
{
6028
	struct igb_adapter *adapter = q_vector->adapter;
6029
	struct igb_ring *tx_ring = q_vector->tx.ring;
6030
	struct igb_tx_buffer *tx_buffer;
6031
	union e1000_adv_tx_desc *tx_desc;
6032
	unsigned int total_bytes = 0, total_packets = 0;
6033
	unsigned int budget = q_vector->tx.work_limit;
6034
	unsigned int i = tx_ring->next_to_clean;
6035

6036 6037
	if (test_bit(__IGB_DOWN, &adapter->state))
		return true;
6038

6039
	tx_buffer = &tx_ring->tx_buffer_info[i];
6040
	tx_desc = IGB_TX_DESC(tx_ring, i);
6041
	i -= tx_ring->count;
6042

6043 6044
	do {
		union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
6045 6046 6047 6048

		/* if next_to_watch is not set then there is no work pending */
		if (!eop_desc)
			break;
6049

6050
		/* prevent any other reads prior to eop_desc */
6051
		read_barrier_depends();
6052

6053 6054 6055 6056
		/* if DD is not set pending work has not been completed */
		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
			break;

6057 6058
		/* clear next_to_watch to prevent false hangs */
		tx_buffer->next_to_watch = NULL;
6059

6060 6061 6062
		/* update the statistics for this packet */
		total_bytes += tx_buffer->bytecount;
		total_packets += tx_buffer->gso_segs;
6063

6064 6065
		/* free the skb */
		dev_kfree_skb_any(tx_buffer->skb);
6066

6067 6068
		/* unmap skb header data */
		dma_unmap_single(tx_ring->dev,
6069 6070
				 dma_unmap_addr(tx_buffer, dma),
				 dma_unmap_len(tx_buffer, len),
6071 6072
				 DMA_TO_DEVICE);

6073 6074 6075 6076
		/* clear tx_buffer data */
		tx_buffer->skb = NULL;
		dma_unmap_len_set(tx_buffer, len, 0);

6077 6078
		/* clear last DMA location and unmap remaining buffers */
		while (tx_desc != eop_desc) {
6079 6080
			tx_buffer++;
			tx_desc++;
6081
			i++;
6082 6083
			if (unlikely(!i)) {
				i -= tx_ring->count;
6084
				tx_buffer = tx_ring->tx_buffer_info;
6085 6086
				tx_desc = IGB_TX_DESC(tx_ring, 0);
			}
6087 6088

			/* unmap any remaining paged data */
6089
			if (dma_unmap_len(tx_buffer, len)) {
6090
				dma_unmap_page(tx_ring->dev,
6091 6092
					       dma_unmap_addr(tx_buffer, dma),
					       dma_unmap_len(tx_buffer, len),
6093
					       DMA_TO_DEVICE);
6094
				dma_unmap_len_set(tx_buffer, len, 0);
6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106
			}
		}

		/* move us one more past the eop_desc for start of next pkt */
		tx_buffer++;
		tx_desc++;
		i++;
		if (unlikely(!i)) {
			i -= tx_ring->count;
			tx_buffer = tx_ring->tx_buffer_info;
			tx_desc = IGB_TX_DESC(tx_ring, 0);
		}
6107 6108 6109 6110 6111 6112 6113

		/* issue prefetch for next Tx descriptor */
		prefetch(tx_desc);

		/* update budget accounting */
		budget--;
	} while (likely(budget));
6114

6115 6116
	netdev_tx_completed_queue(txring_txq(tx_ring),
				  total_packets, total_bytes);
6117
	i += tx_ring->count;
6118
	tx_ring->next_to_clean = i;
6119 6120 6121 6122
	u64_stats_update_begin(&tx_ring->tx_syncp);
	tx_ring->tx_stats.bytes += total_bytes;
	tx_ring->tx_stats.packets += total_packets;
	u64_stats_update_end(&tx_ring->tx_syncp);
6123 6124
	q_vector->tx.total_bytes += total_bytes;
	q_vector->tx.total_packets += total_packets;
6125

6126
	if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
6127
		struct e1000_hw *hw = &adapter->hw;
Eric Dumazet's avatar
Eric Dumazet committed
6128

6129
		/* Detect a transmit hang in hardware, this serializes the
6130 6131
		 * check with the clearing of time_stamp and movement of i
		 */
6132
		clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
6133
		if (tx_buffer->next_to_watch &&
6134
		    time_after(jiffies, tx_buffer->time_stamp +
6135 6136
			       (adapter->tx_timeout_factor * HZ)) &&
		    !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
6137 6138

			/* detected Tx unit hang */
6139
			dev_err(tx_ring->dev,
6140
				"Detected Tx Unit Hang\n"
Alexander Duyck's avatar
Alexander Duyck committed
6141
				"  Tx Queue             <%d>\n"
6142 6143 6144 6145 6146 6147
				"  TDH                  <%x>\n"
				"  TDT                  <%x>\n"
				"  next_to_use          <%x>\n"
				"  next_to_clean        <%x>\n"
				"buffer_info[next_to_clean]\n"
				"  time_stamp           <%lx>\n"
6148
				"  next_to_watch        <%p>\n"
6149 6150
				"  jiffies              <%lx>\n"
				"  desc.status          <%x>\n",
Alexander Duyck's avatar
Alexander Duyck committed
6151
				tx_ring->queue_index,
6152
				rd32(E1000_TDH(tx_ring->reg_idx)),
6153
				readl(tx_ring->tail),
6154 6155
				tx_ring->next_to_use,
				tx_ring->next_to_clean,
6156
				tx_buffer->time_stamp,
6157
				tx_buffer->next_to_watch,
6158
				jiffies,
6159
				tx_buffer->next_to_watch->wb.status);
6160 6161 6162 6163 6164
			netif_stop_subqueue(tx_ring->netdev,
					    tx_ring->queue_index);

			/* we are about to reset, no point in enabling stuff */
			return true;
6165 6166
		}
	}
6167

6168
#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
6169
	if (unlikely(total_packets &&
6170 6171
	    netif_carrier_ok(tx_ring->netdev) &&
	    igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188
		/* Make sure that anybody stopping the queue after this
		 * sees the new next_to_clean.
		 */
		smp_mb();
		if (__netif_subqueue_stopped(tx_ring->netdev,
					     tx_ring->queue_index) &&
		    !(test_bit(__IGB_DOWN, &adapter->state))) {
			netif_wake_subqueue(tx_ring->netdev,
					    tx_ring->queue_index);

			u64_stats_update_begin(&tx_ring->tx_syncp);
			tx_ring->tx_stats.restart_queue++;
			u64_stats_update_end(&tx_ring->tx_syncp);
		}
	}

	return !!budget;
6189 6190
}

6191
/**
6192 6193 6194
 *  igb_reuse_rx_page - page flip buffer and store it back on the ring
 *  @rx_ring: rx descriptor ring to store buffers on
 *  @old_buff: donor buffer to have page reused
6195
 *
6196
 *  Synchronizes page for reuse by the adapter
6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215
 **/
static void igb_reuse_rx_page(struct igb_ring *rx_ring,
			      struct igb_rx_buffer *old_buff)
{
	struct igb_rx_buffer *new_buff;
	u16 nta = rx_ring->next_to_alloc;

	new_buff = &rx_ring->rx_buffer_info[nta];

	/* update, and store next to alloc */
	nta++;
	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;

	/* transfer page from old buffer to new buffer */
	memcpy(new_buff, old_buff, sizeof(struct igb_rx_buffer));

	/* sync the buffer for use by the device */
	dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma,
					 old_buff->page_offset,
6216
					 IGB_RX_BUFSZ,
6217 6218 6219
					 DMA_FROM_DEVICE);
}

6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254
static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
				  struct page *page,
				  unsigned int truesize)
{
	/* avoid re-using remote pages */
	if (unlikely(page_to_nid(page) != numa_node_id()))
		return false;

#if (PAGE_SIZE < 8192)
	/* if we are only owner of page we can reuse it */
	if (unlikely(page_count(page) != 1))
		return false;

	/* flip page offset to other buffer */
	rx_buffer->page_offset ^= IGB_RX_BUFSZ;

	/* since we are the only owner of the page and we need to
	 * increment it, just set the value to 2 in order to avoid
	 * an unnecessary locked operation
	 */
	atomic_set(&page->_count, 2);
#else
	/* move offset up to the next cache line */
	rx_buffer->page_offset += truesize;

	if (rx_buffer->page_offset > (PAGE_SIZE - IGB_RX_BUFSZ))
		return false;

	/* bump ref count on page before it is given to the stack */
	get_page(page);
#endif

	return true;
}

6255
/**
6256 6257 6258 6259 6260
 *  igb_add_rx_frag - Add contents of Rx buffer to sk_buff
 *  @rx_ring: rx descriptor ring to transact packets on
 *  @rx_buffer: buffer containing page to add
 *  @rx_desc: descriptor containing length of buffer written by hardware
 *  @skb: sk_buff to place the data into
6261
 *
6262 6263 6264 6265
 *  This function will add the data contained in rx_buffer->page to the skb.
 *  This is done either through a direct copy if the data in the buffer is
 *  less than the skb header size, otherwise it will just attach the page as
 *  a frag to the skb.
6266
 *
6267 6268
 *  The function will then update the page offset if necessary and return
 *  true if the buffer can be reused by the adapter.
6269 6270 6271 6272 6273 6274 6275 6276
 **/
static bool igb_add_rx_frag(struct igb_ring *rx_ring,
			    struct igb_rx_buffer *rx_buffer,
			    union e1000_adv_rx_desc *rx_desc,
			    struct sk_buff *skb)
{
	struct page *page = rx_buffer->page;
	unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
6277 6278 6279 6280 6281
#if (PAGE_SIZE < 8192)
	unsigned int truesize = IGB_RX_BUFSZ;
#else
	unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
#endif
6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303

	if ((size <= IGB_RX_HDR_LEN) && !skb_is_nonlinear(skb)) {
		unsigned char *va = page_address(page) + rx_buffer->page_offset;

		if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
			igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);
			va += IGB_TS_HDR_LEN;
			size -= IGB_TS_HDR_LEN;
		}

		memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));

		/* we can reuse buffer as-is, just make sure it is local */
		if (likely(page_to_nid(page) == numa_node_id()))
			return true;

		/* this page cannot be reused so discard it */
		put_page(page);
		return false;
	}

	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
6304
			rx_buffer->page_offset, size, truesize);
6305

6306 6307
	return igb_can_reuse_rx_page(rx_buffer, page, truesize);
}
6308

6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338
static struct sk_buff *igb_fetch_rx_buffer(struct igb_ring *rx_ring,
					   union e1000_adv_rx_desc *rx_desc,
					   struct sk_buff *skb)
{
	struct igb_rx_buffer *rx_buffer;
	struct page *page;

	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];

	page = rx_buffer->page;
	prefetchw(page);

	if (likely(!skb)) {
		void *page_addr = page_address(page) +
				  rx_buffer->page_offset;

		/* prefetch first cache line of first page */
		prefetch(page_addr);
#if L1_CACHE_BYTES < 128
		prefetch(page_addr + L1_CACHE_BYTES);
#endif

		/* allocate a skb to store the frags */
		skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
						IGB_RX_HDR_LEN);
		if (unlikely(!skb)) {
			rx_ring->rx_stats.alloc_failed++;
			return NULL;
		}

6339
		/* we will be copying header into skb->data in
6340 6341 6342 6343 6344 6345 6346 6347 6348 6349
		 * pskb_may_pull so it is in our interest to prefetch
		 * it now to avoid a possible cache miss
		 */
		prefetchw(skb->data);
	}

	/* we are reusing so sync this buffer for CPU use */
	dma_sync_single_range_for_cpu(rx_ring->dev,
				      rx_buffer->dma,
				      rx_buffer->page_offset,
6350
				      IGB_RX_BUFSZ,
6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368
				      DMA_FROM_DEVICE);

	/* pull page into skb */
	if (igb_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
		/* hand second half of page back to the ring */
		igb_reuse_rx_page(rx_ring, rx_buffer);
	} else {
		/* we are not reusing the buffer so unmap it */
		dma_unmap_page(rx_ring->dev, rx_buffer->dma,
			       PAGE_SIZE, DMA_FROM_DEVICE);
	}

	/* clear contents of rx_buffer */
	rx_buffer->page = NULL;

	return skb;
}

6369
static inline void igb_rx_checksum(struct igb_ring *ring,
6370 6371
				   union e1000_adv_rx_desc *rx_desc,
				   struct sk_buff *skb)
6372
{
6373
	skb_checksum_none_assert(skb);
6374

6375
	/* Ignore Checksum bit is set */
6376
	if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
6377 6378 6379 6380
		return;

	/* Rx checksum disabled via ethtool */
	if (!(ring->netdev->features & NETIF_F_RXCSUM))
6381
		return;
6382

6383
	/* TCP/UDP checksum error bit is set */
6384 6385 6386
	if (igb_test_staterr(rx_desc,
			     E1000_RXDEXT_STATERR_TCPE |
			     E1000_RXDEXT_STATERR_IPE)) {
6387
		/* work around errata with sctp packets where the TCPE aka
6388 6389 6390
		 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
		 * packets, (aka let the stack check the crc32c)
		 */
6391 6392
		if (!((skb->len == 60) &&
		      test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
Eric Dumazet's avatar
Eric Dumazet committed
6393
			u64_stats_update_begin(&ring->rx_syncp);
6394
			ring->rx_stats.csum_err++;
Eric Dumazet's avatar
Eric Dumazet committed
6395 6396
			u64_stats_update_end(&ring->rx_syncp);
		}
6397 6398 6399 6400
		/* let the stack verify checksum errors */
		return;
	}
	/* It must be a TCP or UDP packet with a valid checksum */
6401 6402
	if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
				      E1000_RXD_STAT_UDPCS))
6403 6404
		skb->ip_summed = CHECKSUM_UNNECESSARY;

6405 6406
	dev_dbg(ring->dev, "cksum success: bits %08X\n",
		le32_to_cpu(rx_desc->wb.upper.status_error));
6407 6408
}

6409 6410 6411 6412 6413 6414 6415 6416
static inline void igb_rx_hash(struct igb_ring *ring,
			       union e1000_adv_rx_desc *rx_desc,
			       struct sk_buff *skb)
{
	if (ring->netdev->features & NETIF_F_RXHASH)
		skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
}

6417
/**
6418 6419 6420 6421
 *  igb_is_non_eop - process handling of non-EOP buffers
 *  @rx_ring: Rx ring being processed
 *  @rx_desc: Rx descriptor for current buffer
 *  @skb: current socket buffer containing buffer in progress
6422
 *
6423 6424 6425 6426
 *  This function updates next to clean.  If the buffer is an EOP buffer
 *  this function exits returning false, otherwise it will place the
 *  sk_buff in the next buffer to be chained and return true indicating
 *  that this is in fact a non-EOP buffer.
6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444
 **/
static bool igb_is_non_eop(struct igb_ring *rx_ring,
			   union e1000_adv_rx_desc *rx_desc)
{
	u32 ntc = rx_ring->next_to_clean + 1;

	/* fetch, update, and store next to clean */
	ntc = (ntc < rx_ring->count) ? ntc : 0;
	rx_ring->next_to_clean = ntc;

	prefetch(IGB_RX_DESC(rx_ring, ntc));

	if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
		return false;

	return true;
}

6445
/**
6446 6447 6448
 *  igb_get_headlen - determine size of header for LRO/GRO
 *  @data: pointer to the start of the headers
 *  @max_len: total length of section to find headers in
6449
 *
6450 6451 6452 6453 6454
 *  This function is meant to determine the length of headers that will
 *  be recognized by hardware for LRO, and GRO offloads.  The main
 *  motivation of doing this is to only perform one pull for IPv4 TCP
 *  packets so that we can do basic things like calculating the gso_size
 *  based on the average data per packet.
6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503
 **/
static unsigned int igb_get_headlen(unsigned char *data,
				    unsigned int max_len)
{
	union {
		unsigned char *network;
		/* l2 headers */
		struct ethhdr *eth;
		struct vlan_hdr *vlan;
		/* l3 headers */
		struct iphdr *ipv4;
		struct ipv6hdr *ipv6;
	} hdr;
	__be16 protocol;
	u8 nexthdr = 0;	/* default to not TCP */
	u8 hlen;

	/* this should never happen, but better safe than sorry */
	if (max_len < ETH_HLEN)
		return max_len;

	/* initialize network frame pointer */
	hdr.network = data;

	/* set first protocol and move network header forward */
	protocol = hdr.eth->h_proto;
	hdr.network += ETH_HLEN;

	/* handle any vlan tag if present */
	if (protocol == __constant_htons(ETH_P_8021Q)) {
		if ((hdr.network - data) > (max_len - VLAN_HLEN))
			return max_len;

		protocol = hdr.vlan->h_vlan_encapsulated_proto;
		hdr.network += VLAN_HLEN;
	}

	/* handle L3 protocols */
	if (protocol == __constant_htons(ETH_P_IP)) {
		if ((hdr.network - data) > (max_len - sizeof(struct iphdr)))
			return max_len;

		/* access ihl as a u8 to avoid unaligned access on ia64 */
		hlen = (hdr.network[0] & 0x0F) << 2;

		/* verify hlen meets minimum size requirements */
		if (hlen < sizeof(struct iphdr))
			return hdr.network - data;

6504
		/* record next protocol if header is present */
6505
		if (!(hdr.ipv4->frag_off & htons(IP_OFFSET)))
6506
			nexthdr = hdr.ipv4->protocol;
6507 6508 6509 6510 6511 6512
	} else if (protocol == __constant_htons(ETH_P_IPV6)) {
		if ((hdr.network - data) > (max_len - sizeof(struct ipv6hdr)))
			return max_len;

		/* record next protocol */
		nexthdr = hdr.ipv6->nexthdr;
6513
		hlen = sizeof(struct ipv6hdr);
6514 6515 6516 6517
	} else {
		return hdr.network - data;
	}

6518 6519 6520
	/* relocate pointer to start of L4 header */
	hdr.network += hlen;

6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540
	/* finally sort out TCP */
	if (nexthdr == IPPROTO_TCP) {
		if ((hdr.network - data) > (max_len - sizeof(struct tcphdr)))
			return max_len;

		/* access doff as a u8 to avoid unaligned access on ia64 */
		hlen = (hdr.network[12] & 0xF0) >> 2;

		/* verify hlen meets minimum size requirements */
		if (hlen < sizeof(struct tcphdr))
			return hdr.network - data;

		hdr.network += hlen;
	} else if (nexthdr == IPPROTO_UDP) {
		if ((hdr.network - data) > (max_len - sizeof(struct udphdr)))
			return max_len;

		hdr.network += sizeof(struct udphdr);
	}

6541
	/* If everything has gone correctly hdr.network should be the
6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552
	 * data section of the packet and will be the end of the header.
	 * If not then it probably represents the end of the last recognized
	 * header.
	 */
	if ((hdr.network - data) < max_len)
		return hdr.network - data;
	else
		return max_len;
}

/**
6553 6554 6555 6556
 *  igb_pull_tail - igb specific version of skb_pull_tail
 *  @rx_ring: rx descriptor ring packet is being transacted on
 *  @rx_desc: pointer to the EOP Rx descriptor
 *  @skb: pointer to current skb being adjusted
6557
 *
6558 6559 6560 6561 6562 6563
 *  This function is an igb specific version of __pskb_pull_tail.  The
 *  main difference between this version and the original function is that
 *  this function can make several assumptions about the state of things
 *  that allow for significant optimizations versus the standard function.
 *  As a result we can do things like drop a frag and maintain an accurate
 *  truesize for the skb.
6564 6565 6566 6567
 */
static void igb_pull_tail(struct igb_ring *rx_ring,
			  union e1000_adv_rx_desc *rx_desc,
			  struct sk_buff *skb)
6568
{
6569 6570 6571 6572
	struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
	unsigned char *va;
	unsigned int pull_len;

6573
	/* it is valid to use page_address instead of kmap since we are
6574 6575
	 * working with pages allocated out of the lomem pool per
	 * alloc_page(GFP_ATOMIC)
6576
	 */
6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592
	va = skb_frag_address(frag);

	if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
		/* retrieve timestamp from buffer */
		igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);

		/* update pointers to remove timestamp header */
		skb_frag_size_sub(frag, IGB_TS_HDR_LEN);
		frag->page_offset += IGB_TS_HDR_LEN;
		skb->data_len -= IGB_TS_HDR_LEN;
		skb->len -= IGB_TS_HDR_LEN;

		/* move va to start of packet data */
		va += IGB_TS_HDR_LEN;
	}

6593
	/* we need the header to contain the greater of either ETH_HLEN or
6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608
	 * 60 bytes if the skb->len is less than 60 for skb_pad.
	 */
	pull_len = igb_get_headlen(va, IGB_RX_HDR_LEN);

	/* align pull length to size of long to optimize memcpy performance */
	skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));

	/* update all of the pointers */
	skb_frag_size_sub(frag, pull_len);
	frag->page_offset += pull_len;
	skb->data_len -= pull_len;
	skb->tail += pull_len;
}

/**
6609 6610 6611 6612
 *  igb_cleanup_headers - Correct corrupted or empty headers
 *  @rx_ring: rx descriptor ring packet is being transacted on
 *  @rx_desc: pointer to the EOP Rx descriptor
 *  @skb: pointer to current skb being fixed
6613
 *
6614 6615
 *  Address the case where we are pulling data in on pages only
 *  and as such no data is present in the skb header.
6616
 *
6617 6618
 *  In addition if skb is not at least 60 bytes we need to pad it so that
 *  it is large enough to qualify as a valid Ethernet frame.
6619
 *
6620
 *  Returns true if an error was encountered and skb was freed.
6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648
 **/
static bool igb_cleanup_headers(struct igb_ring *rx_ring,
				union e1000_adv_rx_desc *rx_desc,
				struct sk_buff *skb)
{
	if (unlikely((igb_test_staterr(rx_desc,
				       E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
		struct net_device *netdev = rx_ring->netdev;
		if (!(netdev->features & NETIF_F_RXALL)) {
			dev_kfree_skb_any(skb);
			return true;
		}
	}

	/* place header in linear portion of buffer */
	if (skb_is_nonlinear(skb))
		igb_pull_tail(rx_ring, rx_desc, skb);

	/* if skb_pad returns an error the skb was freed */
	if (unlikely(skb->len < 60)) {
		int pad_len = 60 - skb->len;

		if (skb_pad(skb, pad_len))
			return true;
		__skb_put(skb, pad_len);
	}

	return false;
6649 6650
}

6651
/**
6652 6653 6654 6655
 *  igb_process_skb_fields - Populate skb header fields from Rx descriptor
 *  @rx_ring: rx descriptor ring packet is being transacted on
 *  @rx_desc: pointer to the EOP Rx descriptor
 *  @skb: pointer to current skb being populated
6656
 *
6657 6658 6659
 *  This function checks the ring, descriptor, and packet information in
 *  order to populate the hash, checksum, VLAN, timestamp, protocol, and
 *  other fields within the skb.
6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670
 **/
static void igb_process_skb_fields(struct igb_ring *rx_ring,
				   union e1000_adv_rx_desc *rx_desc,
				   struct sk_buff *skb)
{
	struct net_device *dev = rx_ring->netdev;

	igb_rx_hash(rx_ring, rx_desc, skb);

	igb_rx_checksum(rx_ring, rx_desc, skb);

6671
	igb_ptp_rx_hwtstamp(rx_ring, rx_desc, skb);
6672

6673
	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
6674 6675 6676 6677 6678 6679 6680 6681
	    igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
		u16 vid;
		if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
		    test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
			vid = be16_to_cpu(rx_desc->wb.upper.vlan);
		else
			vid = le16_to_cpu(rx_desc->wb.upper.vlan);

6682
		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
6683 6684 6685 6686 6687 6688 6689
	}

	skb_record_rx_queue(skb, rx_ring->queue_index);

	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
}

6690
static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
6691
{
6692
	struct igb_ring *rx_ring = q_vector->rx.ring;
6693
	struct sk_buff *skb = rx_ring->skb;
6694
	unsigned int total_bytes = 0, total_packets = 0;
6695
	u16 cleaned_count = igb_desc_unused(rx_ring);
6696

6697 6698
	do {
		union e1000_adv_rx_desc *rx_desc;
6699

6700 6701 6702 6703 6704
		/* return some buffers to hardware, one at a time is too slow */
		if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
			igb_alloc_rx_buffers(rx_ring, cleaned_count);
			cleaned_count = 0;
		}
6705

6706
		rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
6707

6708 6709
		if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_DD))
			break;
6710

6711 6712 6713 6714 6715 6716
		/* This memory barrier is needed to keep us from reading
		 * any other fields out of the rx_desc until we know the
		 * RXD_STAT_DD bit is set
		 */
		rmb();

6717
		/* retrieve a buffer from the ring */
6718
		skb = igb_fetch_rx_buffer(rx_ring, rx_desc, skb);
6719

6720 6721 6722
		/* exit if we failed to retrieve a buffer */
		if (!skb)
			break;
6723

6724
		cleaned_count++;
6725

6726 6727 6728
		/* fetch next buffer in frame if non-eop */
		if (igb_is_non_eop(rx_ring, rx_desc))
			continue;
6729 6730 6731 6732 6733

		/* verify the packet layout is correct */
		if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
			skb = NULL;
			continue;
6734 6735
		}

6736
		/* probably a little skewed due to removing CRC */
6737 6738
		total_bytes += skb->len;

6739 6740
		/* populate checksum, timestamp, VLAN, and protocol */
		igb_process_skb_fields(rx_ring, rx_desc, skb);
6741

Jiri Pirko's avatar
Jiri Pirko committed
6742
		napi_gro_receive(&q_vector->napi, skb);
6743

6744 6745 6746
		/* reset skb pointer */
		skb = NULL;

6747 6748 6749
		/* update budget accounting */
		total_packets++;
	} while (likely(total_packets < budget));
6750

6751 6752 6753
	/* place incomplete frames back on ring for completion */
	rx_ring->skb = skb;

Eric Dumazet's avatar
Eric Dumazet committed
6754
	u64_stats_update_begin(&rx_ring->rx_syncp);
6755 6756
	rx_ring->rx_stats.packets += total_packets;
	rx_ring->rx_stats.bytes += total_bytes;
Eric Dumazet's avatar
Eric Dumazet committed
6757
	u64_stats_update_end(&rx_ring->rx_syncp);
6758 6759
	q_vector->rx.total_packets += total_packets;
	q_vector->rx.total_bytes += total_bytes;
6760 6761

	if (cleaned_count)
6762
		igb_alloc_rx_buffers(rx_ring, cleaned_count);
6763

6764
	return (total_packets < budget);
6765 6766
}

6767
static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
6768
				  struct igb_rx_buffer *bi)
6769 6770
{
	struct page *page = bi->page;
6771
	dma_addr_t dma;
6772

6773 6774
	/* since we are recycling buffers we should seldom need to alloc */
	if (likely(page))
6775 6776
		return true;

6777 6778 6779 6780 6781
	/* alloc new page for storage */
	page = __skb_alloc_page(GFP_ATOMIC | __GFP_COLD, NULL);
	if (unlikely(!page)) {
		rx_ring->rx_stats.alloc_failed++;
		return false;
6782 6783
	}

6784 6785
	/* map page for use */
	dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
6786

6787
	/* if mapping failed free memory back to system since
6788 6789
	 * there isn't much point in holding memory we can't use
	 */
6790
	if (dma_mapping_error(rx_ring->dev, dma)) {
6791 6792
		__free_page(page);

6793 6794 6795 6796
		rx_ring->rx_stats.alloc_failed++;
		return false;
	}

6797
	bi->dma = dma;
6798 6799
	bi->page = page;
	bi->page_offset = 0;
6800

6801 6802 6803
	return true;
}

6804
/**
6805 6806
 *  igb_alloc_rx_buffers - Replace used receive buffers; packet split
 *  @adapter: address of board private structure
6807
 **/
6808
void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
6809 6810
{
	union e1000_adv_rx_desc *rx_desc;
6811
	struct igb_rx_buffer *bi;
6812
	u16 i = rx_ring->next_to_use;
6813

6814 6815 6816 6817
	/* nothing to do */
	if (!cleaned_count)
		return;

6818
	rx_desc = IGB_RX_DESC(rx_ring, i);
6819
	bi = &rx_ring->rx_buffer_info[i];
6820
	i -= rx_ring->count;
6821

6822
	do {
6823
		if (!igb_alloc_mapped_page(rx_ring, bi))
6824
			break;
6825

6826
		/* Refresh the desc even if buffer_addrs didn't change
6827 6828
		 * because each write-back erases this info.
		 */
6829
		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
6830

6831 6832
		rx_desc++;
		bi++;
6833
		i++;
6834
		if (unlikely(!i)) {
6835
			rx_desc = IGB_RX_DESC(rx_ring, 0);
6836
			bi = rx_ring->rx_buffer_info;
6837 6838 6839 6840 6841
			i -= rx_ring->count;
		}

		/* clear the hdr_addr for the next_to_use descriptor */
		rx_desc->read.hdr_addr = 0;
6842 6843 6844

		cleaned_count--;
	} while (cleaned_count);
6845

6846 6847
	i += rx_ring->count;

6848
	if (rx_ring->next_to_use != i) {
6849
		/* record the next descriptor to use */
6850 6851
		rx_ring->next_to_use = i;

6852 6853 6854
		/* update next to alloc since we have filled the ring */
		rx_ring->next_to_alloc = i;

6855
		/* Force memory writes to complete before letting h/w
6856 6857
		 * know there are new descriptors to fetch.  (Only
		 * applicable for weak-ordered memory model archs,
6858 6859
		 * such as IA-64).
		 */
6860
		wmb();
6861
		writel(i, rx_ring->tail);
6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883
	}
}

/**
 * igb_mii_ioctl -
 * @netdev:
 * @ifreq:
 * @cmd:
 **/
static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct mii_ioctl_data *data = if_mii(ifr);

	if (adapter->hw.phy.media_type != e1000_media_type_copper)
		return -EOPNOTSUPP;

	switch (cmd) {
	case SIOCGMIIPHY:
		data->phy_id = adapter->hw.phy.addr;
		break;
	case SIOCGMIIREG:
6884 6885
		if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
		                     &data->val_out))
6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907
			return -EIO;
		break;
	case SIOCSMIIREG:
	default:
		return -EOPNOTSUPP;
	}
	return 0;
}

/**
 * igb_ioctl -
 * @netdev:
 * @ifreq:
 * @cmd:
 **/
static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
{
	switch (cmd) {
	case SIOCGMIIPHY:
	case SIOCGMIIREG:
	case SIOCSMIIREG:
		return igb_mii_ioctl(netdev, ifr, cmd);
6908
	case SIOCSHWTSTAMP:
6909
		return igb_ptp_hwtstamp_ioctl(netdev, ifr, cmd);
6910 6911 6912 6913 6914
	default:
		return -EOPNOTSUPP;
	}
}

6915 6916 6917 6918
s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
{
	struct igb_adapter *adapter = hw->back;

6919
	if (pcie_capability_read_word(adapter->pdev, reg, value))
6920 6921 6922 6923 6924 6925 6926 6927 6928
		return -E1000_ERR_CONFIG;

	return 0;
}

s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
{
	struct igb_adapter *adapter = hw->back;

6929
	if (pcie_capability_write_word(adapter->pdev, reg, *value))
6930 6931 6932 6933 6934
		return -E1000_ERR_CONFIG;

	return 0;
}

6935
static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
6936 6937 6938 6939
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	u32 ctrl, rctl;
6940
	bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
6941

6942
	if (enable) {
6943 6944 6945 6946 6947
		/* enable VLAN tag insert/strip */
		ctrl = rd32(E1000_CTRL);
		ctrl |= E1000_CTRL_VME;
		wr32(E1000_CTRL, ctrl);

6948
		/* Disable CFI check */
6949 6950 6951 6952 6953 6954 6955 6956 6957 6958
		rctl = rd32(E1000_RCTL);
		rctl &= ~E1000_RCTL_CFIEN;
		wr32(E1000_RCTL, rctl);
	} else {
		/* disable VLAN tag insert/strip */
		ctrl = rd32(E1000_CTRL);
		ctrl &= ~E1000_CTRL_VME;
		wr32(E1000_CTRL, ctrl);
	}

6959
	igb_rlpml_set(adapter);
6960 6961
}

6962 6963
static int igb_vlan_rx_add_vid(struct net_device *netdev,
			       __be16 proto, u16 vid)
6964 6965 6966
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
6967
	int pf_id = adapter->vfs_allocated_count;
6968

6969 6970
	/* attempt to add filter to vlvf array */
	igb_vlvf_set(adapter, vid, true, pf_id);
6971

6972 6973
	/* add the filter since PF can receive vlans w/o entry in vlvf */
	igb_vfta_set(hw, vid, true);
Jiri Pirko's avatar
Jiri Pirko committed
6974 6975

	set_bit(vid, adapter->active_vlans);
6976 6977

	return 0;
6978 6979
}

6980 6981
static int igb_vlan_rx_kill_vid(struct net_device *netdev,
				__be16 proto, u16 vid)
6982 6983 6984
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
6985
	int pf_id = adapter->vfs_allocated_count;
6986
	s32 err;
6987

6988 6989
	/* remove vlan from VLVF table array */
	err = igb_vlvf_set(adapter, vid, false, pf_id);
6990

6991 6992
	/* if vid was not present in VLVF just remove it from table */
	if (err)
6993
		igb_vfta_set(hw, vid, false);
Jiri Pirko's avatar
Jiri Pirko committed
6994 6995

	clear_bit(vid, adapter->active_vlans);
6996 6997

	return 0;
6998 6999 7000 7001
}

static void igb_restore_vlan(struct igb_adapter *adapter)
{
Jiri Pirko's avatar
Jiri Pirko committed
7002
	u16 vid;
7003

7004 7005
	igb_vlan_mode(adapter->netdev, adapter->netdev->features);

Jiri Pirko's avatar
Jiri Pirko committed
7006
	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
7007
		igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
7008 7009
}

7010
int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
7011
{
7012
	struct pci_dev *pdev = adapter->pdev;
7013 7014 7015 7016
	struct e1000_mac_info *mac = &adapter->hw.mac;

	mac->autoneg = 0;

7017
	/* Make sure dplx is at most 1 bit and lsb of speed is not set
7018 7019
	 * for the switch() below to work
	 */
7020 7021 7022
	if ((spd & 1) || (dplx & ~1))
		goto err_inval;

7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035
	/* Fiber NIC's only allow 1000 gbps Full duplex
	 * and 100Mbps Full duplex for 100baseFx sfp
	 */
	if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
		switch (spd + dplx) {
		case SPEED_10 + DUPLEX_HALF:
		case SPEED_10 + DUPLEX_FULL:
		case SPEED_100 + DUPLEX_HALF:
			goto err_inval;
		default:
			break;
		}
	}
7036

7037
	switch (spd + dplx) {
7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055
	case SPEED_10 + DUPLEX_HALF:
		mac->forced_speed_duplex = ADVERTISE_10_HALF;
		break;
	case SPEED_10 + DUPLEX_FULL:
		mac->forced_speed_duplex = ADVERTISE_10_FULL;
		break;
	case SPEED_100 + DUPLEX_HALF:
		mac->forced_speed_duplex = ADVERTISE_100_HALF;
		break;
	case SPEED_100 + DUPLEX_FULL:
		mac->forced_speed_duplex = ADVERTISE_100_FULL;
		break;
	case SPEED_1000 + DUPLEX_FULL:
		mac->autoneg = 1;
		adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
		break;
	case SPEED_1000 + DUPLEX_HALF: /* not supported */
	default:
7056
		goto err_inval;
7057
	}
7058 7059 7060 7061

	/* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
	adapter->hw.phy.mdix = AUTO_ALL_MODES;

7062
	return 0;
7063 7064 7065 7066

err_inval:
	dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
	return -EINVAL;
7067 7068
}

7069 7070
static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
			  bool runtime)
7071 7072 7073 7074
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
Alexander Duyck's avatar
Alexander Duyck committed
7075
	u32 ctrl, rctl, status;
7076
	u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
7077 7078 7079 7080 7081 7082
#ifdef CONFIG_PM
	int retval = 0;
#endif

	netif_device_detach(netdev);

Alexander Duyck's avatar
Alexander Duyck committed
7083
	if (netif_running(netdev))
7084
		__igb_close(netdev, true);
Alexander Duyck's avatar
Alexander Duyck committed
7085

7086
	igb_clear_interrupt_scheme(adapter);
7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099

#ifdef CONFIG_PM
	retval = pci_save_state(pdev);
	if (retval)
		return retval;
#endif

	status = rd32(E1000_STATUS);
	if (status & E1000_STATUS_LU)
		wufc &= ~E1000_WUFC_LNKC;

	if (wufc) {
		igb_setup_rctl(adapter);
7100
		igb_set_rx_mode(netdev);
7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117

		/* turn on all-multi mode if wake on multicast is enabled */
		if (wufc & E1000_WUFC_MC) {
			rctl = rd32(E1000_RCTL);
			rctl |= E1000_RCTL_MPE;
			wr32(E1000_RCTL, rctl);
		}

		ctrl = rd32(E1000_CTRL);
		/* advertise wake from D3Cold */
		#define E1000_CTRL_ADVD3WUC 0x00100000
		/* phy power management enable */
		#define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
		ctrl |= E1000_CTRL_ADVD3WUC;
		wr32(E1000_CTRL, ctrl);

		/* Allow time for pending master requests to run */
7118
		igb_disable_pcie_master(hw);
7119 7120 7121 7122 7123 7124 7125 7126

		wr32(E1000_WUC, E1000_WUC_PME_EN);
		wr32(E1000_WUFC, wufc);
	} else {
		wr32(E1000_WUC, 0);
		wr32(E1000_WUFC, 0);
	}

7127 7128
	*enable_wake = wufc || adapter->en_mng_pt;
	if (!*enable_wake)
7129 7130 7131
		igb_power_down_link(adapter);
	else
		igb_power_up_link(adapter);
7132 7133

	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
7134 7135
	 * would have already happened in close and is redundant.
	 */
7136 7137 7138 7139 7140 7141 7142 7143
	igb_release_hw_control(adapter);

	pci_disable_device(pdev);

	return 0;
}

#ifdef CONFIG_PM
7144
#ifdef CONFIG_PM_SLEEP
7145
static int igb_suspend(struct device *dev)
7146 7147 7148
{
	int retval;
	bool wake;
7149
	struct pci_dev *pdev = to_pci_dev(dev);
7150

7151
	retval = __igb_shutdown(pdev, &wake, 0);
7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163
	if (retval)
		return retval;

	if (wake) {
		pci_prepare_to_sleep(pdev);
	} else {
		pci_wake_from_d3(pdev, false);
		pci_set_power_state(pdev, PCI_D3hot);
	}

	return 0;
}
7164
#endif /* CONFIG_PM_SLEEP */
7165

7166
static int igb_resume(struct device *dev)
7167
{
7168
	struct pci_dev *pdev = to_pci_dev(dev);
7169 7170 7171 7172 7173 7174 7175
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	u32 err;

	pci_set_power_state(pdev, PCI_D0);
	pci_restore_state(pdev);
7176
	pci_save_state(pdev);
Taku Izumi's avatar
Taku Izumi committed
7177

7178
	err = pci_enable_device_mem(pdev);
7179 7180 7181 7182 7183 7184 7185 7186 7187 7188
	if (err) {
		dev_err(&pdev->dev,
			"igb: Cannot enable PCI device from suspend\n");
		return err;
	}
	pci_set_master(pdev);

	pci_enable_wake(pdev, PCI_D3hot, 0);
	pci_enable_wake(pdev, PCI_D3cold, 0);

7189
	if (igb_init_interrupt_scheme(adapter, true)) {
Alexander Duyck's avatar
Alexander Duyck committed
7190 7191
		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
		return -ENOMEM;
7192 7193 7194
	}

	igb_reset(adapter);
7195 7196

	/* let the f/w know that the h/w is now under the control of the
7197 7198
	 * driver.
	 */
7199 7200
	igb_get_hw_control(adapter);

7201 7202
	wr32(E1000_WUS, ~0);

7203
	if (netdev->flags & IFF_UP) {
7204
		rtnl_lock();
7205
		err = __igb_open(netdev, true);
7206
		rtnl_unlock();
Alexander Duyck's avatar
Alexander Duyck committed
7207 7208 7209
		if (err)
			return err;
	}
7210 7211

	netif_device_attach(netdev);
7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243
	return 0;
}

#ifdef CONFIG_PM_RUNTIME
static int igb_runtime_idle(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);

	if (!igb_has_link(adapter))
		pm_schedule_suspend(dev, MSEC_PER_SEC * 5);

	return -EBUSY;
}

static int igb_runtime_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	int retval;
	bool wake;

	retval = __igb_shutdown(pdev, &wake, 1);
	if (retval)
		return retval;

	if (wake) {
		pci_prepare_to_sleep(pdev);
	} else {
		pci_wake_from_d3(pdev, false);
		pci_set_power_state(pdev, PCI_D3hot);
	}
7244 7245 7246

	return 0;
}
7247 7248 7249 7250 7251 7252

static int igb_runtime_resume(struct device *dev)
{
	return igb_resume(dev);
}
#endif /* CONFIG_PM_RUNTIME */
7253 7254 7255 7256
#endif

static void igb_shutdown(struct pci_dev *pdev)
{
7257 7258
	bool wake;

7259
	__igb_shutdown(pdev, &wake, 0);
7260 7261 7262 7263 7264

	if (system_state == SYSTEM_POWER_OFF) {
		pci_wake_from_d3(pdev, wake);
		pci_set_power_state(pdev, PCI_D3hot);
	}
7265 7266
}

7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332
#ifdef CONFIG_PCI_IOV
static int igb_sriov_reinit(struct pci_dev *dev)
{
	struct net_device *netdev = pci_get_drvdata(dev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct pci_dev *pdev = adapter->pdev;

	rtnl_lock();

	if (netif_running(netdev))
		igb_close(netdev);

	igb_clear_interrupt_scheme(adapter);

	igb_init_queue_configuration(adapter);

	if (igb_init_interrupt_scheme(adapter, true)) {
		dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
		return -ENOMEM;
	}

	if (netif_running(netdev))
		igb_open(netdev);

	rtnl_unlock();

	return 0;
}

static int igb_pci_disable_sriov(struct pci_dev *dev)
{
	int err = igb_disable_sriov(dev);

	if (!err)
		err = igb_sriov_reinit(dev);

	return err;
}

static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs)
{
	int err = igb_enable_sriov(dev, num_vfs);

	if (err)
		goto out;

	err = igb_sriov_reinit(dev);
	if (!err)
		return num_vfs;

out:
	return err;
}

#endif
static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
{
#ifdef CONFIG_PCI_IOV
	if (num_vfs == 0)
		return igb_pci_disable_sriov(dev);
	else
		return igb_pci_enable_sriov(dev, num_vfs);
#endif
	return 0;
}

7333
#ifdef CONFIG_NET_POLL_CONTROLLER
7334
/* Polling 'interrupt' - used by things like netconsole to send skbs
7335 7336 7337 7338 7339 7340
 * without having to re-enable interrupts. It's not called while
 * the interrupt routine is executing.
 */
static void igb_netpoll(struct net_device *netdev)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
7341
	struct e1000_hw *hw = &adapter->hw;
7342
	struct igb_q_vector *q_vector;
7343 7344
	int i;

7345
	for (i = 0; i < adapter->num_q_vectors; i++) {
7346 7347 7348 7349 7350
		q_vector = adapter->q_vector[i];
		if (adapter->msix_entries)
			wr32(E1000_EIMC, q_vector->eims_value);
		else
			igb_irq_disable(adapter);
7351
		napi_schedule(&q_vector->napi);
7352
	}
7353 7354 7355 7356
}
#endif /* CONFIG_NET_POLL_CONTROLLER */

/**
7357 7358 7359
 *  igb_io_error_detected - called when PCI error is detected
 *  @pdev: Pointer to PCI device
 *  @state: The current pci connection state
7360
 *
7361 7362 7363
 *  This function is called after a PCI bus error affecting
 *  this device has been detected.
 **/
7364 7365 7366 7367 7368 7369 7370 7371
static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
					      pci_channel_state_t state)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);

	netif_device_detach(netdev);

7372 7373 7374
	if (state == pci_channel_io_perm_failure)
		return PCI_ERS_RESULT_DISCONNECT;

7375 7376 7377 7378 7379 7380 7381 7382 7383
	if (netif_running(netdev))
		igb_down(adapter);
	pci_disable_device(pdev);

	/* Request a slot slot reset. */
	return PCI_ERS_RESULT_NEED_RESET;
}

/**
7384 7385
 *  igb_io_slot_reset - called after the pci bus has been reset.
 *  @pdev: Pointer to PCI device
7386
 *
7387 7388 7389
 *  Restart the card from scratch, as if from a cold-boot. Implementation
 *  resembles the first-half of the igb_resume routine.
 **/
7390 7391 7392 7393 7394
static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
7395
	pci_ers_result_t result;
Taku Izumi's avatar
Taku Izumi committed
7396
	int err;
7397

7398
	if (pci_enable_device_mem(pdev)) {
7399 7400
		dev_err(&pdev->dev,
			"Cannot re-enable PCI device after reset.\n");
7401 7402 7403 7404
		result = PCI_ERS_RESULT_DISCONNECT;
	} else {
		pci_set_master(pdev);
		pci_restore_state(pdev);
7405
		pci_save_state(pdev);
7406

7407 7408
		pci_enable_wake(pdev, PCI_D3hot, 0);
		pci_enable_wake(pdev, PCI_D3cold, 0);
7409

7410 7411 7412 7413
		igb_reset(adapter);
		wr32(E1000_WUS, ~0);
		result = PCI_ERS_RESULT_RECOVERED;
	}
7414

7415 7416
	err = pci_cleanup_aer_uncorrect_error_status(pdev);
	if (err) {
7417 7418 7419
		dev_err(&pdev->dev,
			"pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
			err);
7420 7421
		/* non-fatal, continue */
	}
7422 7423

	return result;
7424 7425 7426
}

/**
7427 7428
 *  igb_io_resume - called when traffic can start flowing again.
 *  @pdev: Pointer to PCI device
7429
 *
7430 7431 7432
 *  This callback is called when the error recovery driver tells us that
 *  its OK to resume normal operation. Implementation resembles the
 *  second-half of the igb_resume routine.
7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448
 */
static void igb_io_resume(struct pci_dev *pdev)
{
	struct net_device *netdev = pci_get_drvdata(pdev);
	struct igb_adapter *adapter = netdev_priv(netdev);

	if (netif_running(netdev)) {
		if (igb_up(adapter)) {
			dev_err(&pdev->dev, "igb_up failed after reset\n");
			return;
		}
	}

	netif_device_attach(netdev);

	/* let the f/w know that the h/w is now under the control of the
7449 7450
	 * driver.
	 */
7451 7452 7453
	igb_get_hw_control(adapter);
}

7454
static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
7455
			     u8 qsel)
7456 7457 7458 7459 7460 7461 7462 7463
{
	u32 rar_low, rar_high;
	struct e1000_hw *hw = &adapter->hw;

	/* HW expects these in little endian so we reverse the byte order
	 * from network order (big endian) to little endian
	 */
	rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
7464
		   ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480
	rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));

	/* Indicate to hardware the Address is Valid. */
	rar_high |= E1000_RAH_AV;

	if (hw->mac.type == e1000_82575)
		rar_high |= E1000_RAH_POOL_1 * qsel;
	else
		rar_high |= E1000_RAH_POOL_1 << qsel;

	wr32(E1000_RAL(index), rar_low);
	wrfl();
	wr32(E1000_RAH(index), rar_high);
	wrfl();
}

7481
static int igb_set_vf_mac(struct igb_adapter *adapter,
7482
			  int vf, unsigned char *mac_addr)
7483 7484
{
	struct e1000_hw *hw = &adapter->hw;
7485
	/* VF MAC addresses start at end of receive addresses and moves
7486 7487
	 * towards the first, as a result a collision should not be possible
	 */
7488
	int rar_entry = hw->mac.rar_entry_count - (vf + 1);
7489

7490
	memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
7491

7492
	igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
7493 7494 7495 7496

	return 0;
}

7497 7498 7499 7500 7501 7502 7503
static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
		return -EINVAL;
	adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
	dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
7504 7505
	dev_info(&adapter->pdev->dev,
		 "Reload the VF driver to make this change effective.");
7506
	if (test_bit(__IGB_DOWN, &adapter->state)) {
7507 7508 7509 7510
		dev_warn(&adapter->pdev->dev,
			 "The VF MAC address has been set, but the PF device is not up.\n");
		dev_warn(&adapter->pdev->dev,
			 "Bring the PF device up before attempting to use the VF device.\n");
7511 7512 7513 7514
	}
	return igb_set_vf_mac(adapter, vf, mac);
}

7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536
static int igb_link_mbps(int internal_link_speed)
{
	switch (internal_link_speed) {
	case SPEED_100:
		return 100;
	case SPEED_1000:
		return 1000;
	default:
		return 0;
	}
}

static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
				  int link_speed)
{
	int rf_dec, rf_int;
	u32 bcnrc_val;

	if (tx_rate != 0) {
		/* Calculate the rate factor values to set */
		rf_int = link_speed / tx_rate;
		rf_dec = (link_speed - (rf_int * tx_rate));
7537 7538
		rf_dec = (rf_dec * (1 << E1000_RTTBCNRC_RF_INT_SHIFT)) /
			 tx_rate;
7539 7540

		bcnrc_val = E1000_RTTBCNRC_RS_ENA;
7541 7542
		bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) &
			      E1000_RTTBCNRC_RF_INT_MASK);
7543 7544 7545 7546 7547 7548
		bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
	} else {
		bcnrc_val = 0;
	}

	wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
7549
	/* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
Lior Levy's avatar
Lior Levy committed
7550 7551 7552
	 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
	 */
	wr32(E1000_RTTBCNRM, 0x14);
7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570
	wr32(E1000_RTTBCNRC, bcnrc_val);
}

static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
{
	int actual_link_speed, i;
	bool reset_rate = false;

	/* VF TX rate limit was not set or not supported */
	if ((adapter->vf_rate_link_speed == 0) ||
	    (adapter->hw.mac.type != e1000_82576))
		return;

	actual_link_speed = igb_link_mbps(adapter->link_speed);
	if (actual_link_speed != adapter->vf_rate_link_speed) {
		reset_rate = true;
		adapter->vf_rate_link_speed = 0;
		dev_info(&adapter->pdev->dev,
7571
			 "Link speed has been changed. VF Transmit rate is disabled\n");
7572 7573 7574 7575 7576 7577 7578
	}

	for (i = 0; i < adapter->vfs_allocated_count; i++) {
		if (reset_rate)
			adapter->vf_data[i].tx_rate = 0;

		igb_set_vf_rate_limit(&adapter->hw, i,
7579 7580
				      adapter->vf_data[i].tx_rate,
				      actual_link_speed);
7581 7582 7583
	}
}

7584 7585
static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
{
7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	int actual_link_speed;

	if (hw->mac.type != e1000_82576)
		return -EOPNOTSUPP;

	actual_link_speed = igb_link_mbps(adapter->link_speed);
	if ((vf >= adapter->vfs_allocated_count) ||
	    (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
	    (tx_rate < 0) || (tx_rate > actual_link_speed))
		return -EINVAL;

	adapter->vf_rate_link_speed = actual_link_speed;
	adapter->vf_data[vf].tx_rate = (u16)tx_rate;
	igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);

	return 0;
7604 7605
}

7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632
static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
				   bool setting)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	struct e1000_hw *hw = &adapter->hw;
	u32 reg_val, reg_offset;

	if (!adapter->vfs_allocated_count)
		return -EOPNOTSUPP;

	if (vf >= adapter->vfs_allocated_count)
		return -EINVAL;

	reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
	reg_val = rd32(reg_offset);
	if (setting)
		reg_val |= ((1 << vf) |
			    (1 << (vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
	else
		reg_val &= ~((1 << vf) |
			     (1 << (vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT)));
	wr32(reg_offset, reg_val);

	adapter->vf_data[vf].spoofchk_enabled = setting;
	return E1000_SUCCESS;
}

7633 7634 7635 7636 7637 7638 7639 7640
static int igb_ndo_get_vf_config(struct net_device *netdev,
				 int vf, struct ifla_vf_info *ivi)
{
	struct igb_adapter *adapter = netdev_priv(netdev);
	if (vf >= adapter->vfs_allocated_count)
		return -EINVAL;
	ivi->vf = vf;
	memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
7641
	ivi->tx_rate = adapter->vf_data[vf].tx_rate;
7642 7643
	ivi->vlan = adapter->vf_data[vf].pf_vlan;
	ivi->qos = adapter->vf_data[vf].pf_qos;
7644
	ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
7645 7646 7647
	return 0;
}

7648 7649 7650
static void igb_vmm_control(struct igb_adapter *adapter)
{
	struct e1000_hw *hw = &adapter->hw;
7651
	u32 reg;
7652

7653 7654
	switch (hw->mac.type) {
	case e1000_82575:
7655 7656
	case e1000_i210:
	case e1000_i211:
7657
	case e1000_i354:
7658 7659
	default:
		/* replication is not supported for 82575 */
7660
		return;
7661 7662 7663 7664 7665 7666 7667 7668 7669 7670
	case e1000_82576:
		/* notify HW that the MAC is adding vlan tags */
		reg = rd32(E1000_DTXCTL);
		reg |= E1000_DTXCTL_VLAN_ADDED;
		wr32(E1000_DTXCTL, reg);
	case e1000_82580:
		/* enable replication vlan tag stripping */
		reg = rd32(E1000_RPLOLR);
		reg |= E1000_RPLOLR_STRVLAN;
		wr32(E1000_RPLOLR, reg);
7671 7672
	case e1000_i350:
		/* none of the above registers are supported by i350 */
7673 7674
		break;
	}
7675

7676 7677 7678
	if (adapter->vfs_allocated_count) {
		igb_vmdq_set_loopback_pf(hw, true);
		igb_vmdq_set_replication_pf(hw, true);
7679
		igb_vmdq_set_anti_spoofing_pf(hw, true,
7680
					      adapter->vfs_allocated_count);
7681 7682 7683 7684
	} else {
		igb_vmdq_set_loopback_pf(hw, false);
		igb_vmdq_set_replication_pf(hw, false);
	}
7685 7686
}

7687 7688 7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699
static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
{
	struct e1000_hw *hw = &adapter->hw;
	u32 dmac_thr;
	u16 hwm;

	if (hw->mac.type > e1000_82580) {
		if (adapter->flags & IGB_FLAG_DMAC) {
			u32 reg;

			/* force threshold to 0. */
			wr32(E1000_DMCTXTH, 0);

7700
			/* DMA Coalescing high water mark needs to be greater
7701 7702
			 * than the Rx threshold. Set hwm to PBA - max frame
			 * size in 16B units, capping it at PBA - 6KB.
7703
			 */
7704 7705 7706 7707 7708 7709 7710 7711 7712
			hwm = 64 * pba - adapter->max_frame_size / 16;
			if (hwm < 64 * (pba - 6))
				hwm = 64 * (pba - 6);
			reg = rd32(E1000_FCRTC);
			reg &= ~E1000_FCRTC_RTH_COAL_MASK;
			reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
				& E1000_FCRTC_RTH_COAL_MASK);
			wr32(E1000_FCRTC, reg);

7713
			/* Set the DMA Coalescing Rx threshold to PBA - 2 * max
7714 7715 7716 7717 7718
			 * frame size, capping it at PBA - 10KB.
			 */
			dmac_thr = pba - adapter->max_frame_size / 512;
			if (dmac_thr < pba - 10)
				dmac_thr = pba - 10;
7719 7720 7721 7722 7723 7724 7725 7726 7727 7728
			reg = rd32(E1000_DMACR);
			reg &= ~E1000_DMACR_DMACTHR_MASK;
			reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
				& E1000_DMACR_DMACTHR_MASK);

			/* transition to L0x or L1 if available..*/
			reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);

			/* watchdog timer= +-1000 usec in 32usec intervals */
			reg |= (1000 >> 5);
7729 7730

			/* Disable BMC-to-OS Watchdog Enable */
7731 7732 7733
			if (hw->mac.type != e1000_i354)
				reg &= ~E1000_DMACR_DC_BMC2OSW_EN;

7734 7735
			wr32(E1000_DMACR, reg);

7736
			/* no lower threshold to disable
7737 7738 7739 7740 7741 7742 7743 7744
			 * coalescing(smart fifb)-UTRESH=0
			 */
			wr32(E1000_DMCRTRH, 0);

			reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);

			wr32(E1000_DMCTLX, reg);

7745
			/* free space in tx packet buffer to wake from
7746 7747 7748 7749 7750
			 * DMA coal
			 */
			wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
			     (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);

7751
			/* make low power state decision controlled
7752 7753 7754 7755 7756 7757 7758 7759 7760 7761 7762 7763 7764
			 * by DMA coal
			 */
			reg = rd32(E1000_PCIEMISC);
			reg &= ~E1000_PCIEMISC_LX_DECISION;
			wr32(E1000_PCIEMISC, reg);
		} /* endif adapter->dmac is not disabled */
	} else if (hw->mac.type == e1000_82580) {
		u32 reg = rd32(E1000_PCIEMISC);
		wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
		wr32(E1000_DMACR, 0);
	}
}

7765 7766
/**
 *  igb_read_i2c_byte - Reads 8 bit word over I2C
7767 7768 7769 7770 7771 7772 7773
 *  @hw: pointer to hardware structure
 *  @byte_offset: byte offset to read
 *  @dev_addr: device address
 *  @data: value read
 *
 *  Performs byte read operation over I2C interface at
 *  a specified device address.
7774
 **/
7775
s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
7776
		      u8 dev_addr, u8 *data)
7777 7778
{
	struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
7779
	struct i2c_client *this_client = adapter->i2c_client;
7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802
	s32 status;
	u16 swfw_mask = 0;

	if (!this_client)
		return E1000_ERR_I2C;

	swfw_mask = E1000_SWFW_PHY0_SM;

	if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask)
	    != E1000_SUCCESS)
		return E1000_ERR_SWFW_SYNC;

	status = i2c_smbus_read_byte_data(this_client, byte_offset);
	hw->mac.ops.release_swfw_sync(hw, swfw_mask);

	if (status < 0)
		return E1000_ERR_I2C;
	else {
		*data = status;
		return E1000_SUCCESS;
	}
}

7803 7804
/**
 *  igb_write_i2c_byte - Writes 8 bit word over I2C
7805 7806 7807 7808 7809 7810 7811
 *  @hw: pointer to hardware structure
 *  @byte_offset: byte offset to write
 *  @dev_addr: device address
 *  @data: value to write
 *
 *  Performs byte write operation over I2C interface at
 *  a specified device address.
7812
 **/
7813
s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
7814
		       u8 dev_addr, u8 data)
7815 7816
{
	struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
7817
	struct i2c_client *this_client = adapter->i2c_client;
7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834
	s32 status;
	u16 swfw_mask = E1000_SWFW_PHY0_SM;

	if (!this_client)
		return E1000_ERR_I2C;

	if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask) != E1000_SUCCESS)
		return E1000_ERR_SWFW_SYNC;
	status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
	hw->mac.ops.release_swfw_sync(hw, swfw_mask);

	if (status)
		return E1000_ERR_I2C;
	else
		return E1000_SUCCESS;

}
7835
/* igb_main.c */