fm10k: Add support for SR-IOV to driver

This patch combines the recently added VF messaging and configuration
functionality with the interfaces provided by the kernel to allow for
configuration and management of SR-IOV.
Signed-off-by: Alexander Duyck <alexander.h.duyck@intel.com>
Signed-off-by: Jeff Kirsher <jeffrey.t.kirsher@intel.com>

drivers/net/ethernet/intel/fm10k/Makefile

@@ -29,4 +29,4 @@ obj-$(CONFIG_FM10K) += fm10k.o
 
 fm10k-objs := fm10k_main.o fm10k_common.o fm10k_pci.o \
 	      fm10k_netdev.o fm10k_ethtool.o fm10k_pf.o fm10k_vf.o \
-	      fm10k_mbx.o fm10k_tlv.o
+	      fm10k_mbx.o fm10k_iov.o fm10k_tlv.o

drivers/net/ethernet/intel/fm10k/fm10k.h

@@ -218,6 +218,13 @@ struct fm10k_ring_feature {
 	u16 offset;	/* offset to start of feature */
 };
 
+struct fm10k_iov_data {
+	unsigned int		num_vfs;
+	unsigned int		next_vf_mbx;
+	struct rcu_head		rcu;
+	struct fm10k_vf_info	vf_info[0];
+};
+
 #define fm10k_vxlan_port_for_each(vp, intfc) \
 	list_for_each_entry(vp, &(intfc)->vxlan_port, list)
 struct fm10k_vxlan_port {
@@ -277,6 +284,9 @@ struct fm10k_intfc {
 	int num_q_vectors;	/* current number of q_vectors for device */
 	struct fm10k_ring_feature ring_feature[RING_F_ARRAY_SIZE];
 
+	/* SR-IOV information management structure */
+	struct fm10k_iov_data *iov_data;
+
 	struct fm10k_hw_stats stats;
 	struct fm10k_hw hw;
 	u32 __iomem *uc_addr;
@@ -441,4 +451,20 @@ int fm10k_close(struct net_device *netdev);
 
 /* Ethtool */
 void fm10k_set_ethtool_ops(struct net_device *dev);
+
+/* IOV */
+s32 fm10k_iov_event(struct fm10k_intfc *interface);
+s32 fm10k_iov_mbx(struct fm10k_intfc *interface);
+void fm10k_iov_suspend(struct pci_dev *pdev);
+int fm10k_iov_resume(struct pci_dev *pdev);
+void fm10k_iov_disable(struct pci_dev *pdev);
+int fm10k_iov_configure(struct pci_dev *pdev, int num_vfs);
+s32 fm10k_iov_update_pvid(struct fm10k_intfc *interface, u16 glort, u16 pvid);
+int fm10k_ndo_set_vf_mac(struct net_device *netdev, int vf_idx, u8 *mac);
+int fm10k_ndo_set_vf_vlan(struct net_device *netdev,
+			  int vf_idx, u16 vid, u8 qos);
+int fm10k_ndo_set_vf_bw(struct net_device *netdev, int vf_idx, int rate,
+			int unused);
+int fm10k_ndo_get_vf_config(struct net_device *netdev,
+			    int vf_idx, struct ifla_vf_info *ivi);
 #endif /* _FM10K_H_ */

drivers/net/ethernet/intel/fm10k/fm10k_netdev.c

@@ -368,7 +368,21 @@ static void fm10k_request_glort_range(struct fm10k_intfc *interface)
 	if (hw->mac.dglort_map == FM10K_DGLORTMAP_NONE)
 		return;
 
-	interface->glort_count = mask + 1;
+	/* we support 3 possible GLORT configurations.
+	 * 1: VFs consume all but the last 1
+	 * 2: VFs and PF split glorts with possible gap between
+	 * 3: VFs allocated first 64, all others belong to PF
+	 */
+	if (mask <= hw->iov.total_vfs) {
+		interface->glort_count = 1;
+		interface->glort += mask;
+	} else if (mask < 64) {
+		interface->glort_count = (mask + 1) / 2;
+		interface->glort += interface->glort_count;
+	} else {
+		interface->glort_count = mask - 63;
+		interface->glort += 64;
+	}
 }
 
 /**
@@ -1325,6 +1339,10 @@ static const struct net_device_ops fm10k_netdev_ops = {
 	.ndo_set_rx_mode	= fm10k_set_rx_mode,
 	.ndo_get_stats64	= fm10k_get_stats64,
 	.ndo_setup_tc		= fm10k_setup_tc,
+	.ndo_set_vf_mac		= fm10k_ndo_set_vf_mac,
+	.ndo_set_vf_vlan	= fm10k_ndo_set_vf_vlan,
+	.ndo_set_vf_rate	= fm10k_ndo_set_vf_bw,
+	.ndo_get_vf_config	= fm10k_ndo_get_vf_config,
 	.ndo_add_vxlan_port	= fm10k_add_vxlan_port,
 	.ndo_del_vxlan_port	= fm10k_del_vxlan_port,
 	.ndo_dfwd_add_station	= fm10k_dfwd_add_station,

drivers/net/ethernet/intel/fm10k/fm10k_pci.c

@@ -152,6 +152,8 @@ static void fm10k_reinit(struct fm10k_intfc *interface)
 
 	rtnl_lock();
 
+	fm10k_iov_suspend(interface->pdev);
+
 	if (netif_running(netdev))
 		fm10k_close(netdev);
 
@@ -171,6 +173,8 @@ static void fm10k_reinit(struct fm10k_intfc *interface)
 	if (netif_running(netdev))
 		fm10k_open(netdev);
 
+	fm10k_iov_resume(interface->pdev);
+
 	rtnl_unlock();
 
 	clear_bit(__FM10K_RESETTING, &interface->state);
@@ -260,6 +264,9 @@ static void fm10k_mbx_subtask(struct fm10k_intfc *interface)
 {
 	/* process upstream mailbox and update device state */
 	fm10k_watchdog_update_host_state(interface);
+
+	/* process downstream mailboxes */
+	fm10k_iov_mbx(interface);
 }
 
 /**
@@ -975,6 +982,7 @@ static irqreturn_t fm10k_msix_mbx_pf(int irq, void *data)
 	/* service mailboxes */
 	if (fm10k_mbx_trylock(interface)) {
 		mbx->ops.process(hw, mbx);
+		fm10k_iov_event(interface);
 		fm10k_mbx_unlock(interface);
 	}
 
@@ -1159,6 +1167,11 @@ static s32 fm10k_update_pvid(struct fm10k_hw *hw, u32 **results,
 
 	interface = container_of(hw, struct fm10k_intfc, hw);
 
+	/* check to see if this belongs to one of the VFs */
+	err = fm10k_iov_update_pvid(interface, glort, pvid);
+	if (!err)
+		return 0;
+
 	/* we need to reset if default VLAN was just updated */
 	if (pvid != hw->mac.default_vid)
 		interface->flags |= FM10K_FLAG_RESET_REQUESTED;
@@ -1477,6 +1490,10 @@ static int fm10k_sw_init(struct fm10k_intfc *interface,
 	memcpy(&hw->mac.ops, fi->mac_ops, sizeof(hw->mac.ops));
 	hw->mac.type = fi->mac;
 
+	/* Setup IOV handlers */
+	if (fi->iov_ops)
+		memcpy(&hw->iov.ops, fi->iov_ops, sizeof(hw->iov.ops));
+
 	/* Set common capability flags and settings */
 	rss = min_t(int, FM10K_MAX_RSS_INDICES, num_online_cpus());
 	interface->ring_feature[RING_F_RSS].limit = rss;
@@ -1509,6 +1526,9 @@ static int fm10k_sw_init(struct fm10k_intfc *interface,
 	/* initialize hardware statistics */
 	hw->mac.ops.update_hw_stats(hw, &interface->stats);
 
+	/* Set upper limit on IOV VFs that can be allocated */
+	pci_sriov_set_totalvfs(pdev, hw->iov.total_vfs);
+
 	/* Start with random Ethernet address */
 	eth_random_addr(hw->mac.addr);
 
@@ -1708,6 +1728,9 @@ static int fm10k_probe(struct pci_dev *pdev,
 	/* print warning for non-optimal configurations */
 	fm10k_slot_warn(interface);
 
+	/* enable SR-IOV after registering netdev to enforce PF/VF ordering */
+	fm10k_iov_configure(pdev, 0);
+
 	/* clear the service task disable bit to allow service task to start */
 	clear_bit(__FM10K_SERVICE_DISABLE, &interface->state);
 
@@ -1751,6 +1774,9 @@ static void fm10k_remove(struct pci_dev *pdev)
 	if (netdev->reg_state == NETREG_REGISTERED)
 		unregister_netdev(netdev);
 
+	/* release VFs */
+	fm10k_iov_disable(pdev);
+
 	/* disable mailbox interrupt */
 	fm10k_mbx_free_irq(interface);
 
@@ -1827,6 +1853,9 @@ static int fm10k_resume(struct pci_dev *pdev)
 	if (err)
 		return err;
 
+	/* restore SR-IOV interface */
+	fm10k_iov_resume(pdev);
+
 	netif_device_attach(netdev);
 
 	return 0;
@@ -1848,6 +1877,8 @@ static int fm10k_suspend(struct pci_dev *pdev, pm_message_t state)
 
 	netif_device_detach(netdev);
 
+	fm10k_iov_suspend(pdev);
+
 	rtnl_lock();
 
 	if (netif_running(netdev))
@@ -1989,6 +2020,7 @@ static struct pci_driver fm10k_driver = {
 	.suspend		= fm10k_suspend,
 	.resume			= fm10k_resume,
 #endif
+	.sriov_configure	= fm10k_iov_configure,
 	.err_handler		= &fm10k_err_handler
 };