Commit ddf30f7f authored by Anirudh Venkataramanan's avatar Anirudh Venkataramanan Committed by Jeff Kirsher

ice: Add handler to configure SR-IOV

This patch implements parts of ice_sriov_configure and VF reset flow.

To create virtual functions (VFs), the user sets a value in num_vfs
through sysfs. This results in the kernel calling the handler for
.sriov_configure which is ice_sriov_configure.

VF setup first starts with a VF reset, followed by allocation of the VF
VSI using ice_vf_vsi_setup. Once the VF setup is complete a state bit
ICE_VF_STATE_INIT is set in the vf->states bitmap to indicate that
the VF is ready to go.

Also for VF reset to go into effect, it's necessary to issue a disable
queue command (ice_aqc_opc_dis_txqs). So this patch updates multiple
functions in the disable queue flow to take additional parameters that
distinguish if queues are being disabled due to VF reset.
Signed-off-by: default avatarAnirudh Venkataramanan <anirudh.venkataramanan@intel.com>
Tested-by: default avatarAndrew Bowers <andrewx.bowers@intel.com>
Signed-off-by: default avatarJeff Kirsher <jeffrey.t.kirsher@intel.com>
parent 75d2b253
......@@ -16,3 +16,4 @@ ice-y := ice_main.o \
ice_lib.o \
ice_txrx.o \
ice_ethtool.o
ice-$(CONFIG_PCI_IOV) += ice_virtchnl_pf.o
......@@ -28,6 +28,7 @@
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/if_bridge.h>
#include <linux/avf/virtchnl.h>
#include <net/ipv6.h>
#include "ice_devids.h"
#include "ice_type.h"
......@@ -35,6 +36,7 @@
#include "ice_switch.h"
#include "ice_common.h"
#include "ice_sched.h"
#include "ice_virtchnl_pf.h"
extern const char ice_drv_ver[];
#define ICE_BAR0 0
......@@ -65,6 +67,12 @@ extern const char ice_drv_ver[];
#define ICE_INVAL_Q_INDEX 0xffff
#define ICE_INVAL_VFID 256
#define ICE_MAX_VF_COUNT 256
#define ICE_MAX_QS_PER_VF 256
#define ICE_MIN_QS_PER_VF 1
#define ICE_DFLT_QS_PER_VF 4
#define ICE_MAX_INTR_PER_VF 65
#define ICE_MIN_INTR_PER_VF (ICE_MIN_QS_PER_VF + 1)
#define ICE_DFLT_INTR_PER_VF (ICE_DFLT_QS_PER_VF + 1)
#define ICE_VSIQF_HKEY_ARRAY_SIZE ((VSIQF_HKEY_MAX_INDEX + 1) * 4)
......@@ -135,10 +143,20 @@ enum ice_state {
__ICE_EMPR_RECV, /* set by OICR handler */
__ICE_SUSPENDED, /* set on module remove path */
__ICE_RESET_FAILED, /* set by reset/rebuild */
/* When checking for the PF to be in a nominal operating state, the
* bits that are grouped at the beginning of the list need to be
* checked. Bits occurring before __ICE_STATE_NOMINAL_CHECK_BITS will
* be checked. If you need to add a bit into consideration for nominal
* operating state, it must be added before
* __ICE_STATE_NOMINAL_CHECK_BITS. Do not move this entry's position
* without appropriate consideration.
*/
__ICE_STATE_NOMINAL_CHECK_BITS,
__ICE_ADMINQ_EVENT_PENDING,
__ICE_MAILBOXQ_EVENT_PENDING,
__ICE_MDD_EVENT_PENDING,
__ICE_FLTR_OVERFLOW_PROMISC,
__ICE_VF_DIS,
__ICE_CFG_BUSY,
__ICE_SERVICE_SCHED,
__ICE_SERVICE_DIS,
......@@ -243,6 +261,7 @@ enum ice_pf_flags {
ICE_FLAG_MSIX_ENA,
ICE_FLAG_FLTR_SYNC,
ICE_FLAG_RSS_ENA,
ICE_FLAG_SRIOV_ENA,
ICE_FLAG_SRIOV_CAPABLE,
ICE_PF_FLAGS_NBITS /* must be last */
};
......@@ -259,7 +278,12 @@ struct ice_pf {
struct ice_vsi **vsi; /* VSIs created by the driver */
struct ice_sw *first_sw; /* first switch created by firmware */
/* Virtchnl/SR-IOV config info */
struct ice_vf *vf;
int num_alloc_vfs; /* actual number of VFs allocated */
u16 num_vfs_supported; /* num VFs supported for this PF */
u16 num_vf_qps; /* num queue pairs per VF */
u16 num_vf_msix; /* num vectors per VF */
DECLARE_BITMAP(state, __ICE_STATE_NBITS);
DECLARE_BITMAP(avail_txqs, ICE_MAX_TXQS);
DECLARE_BITMAP(avail_rxqs, ICE_MAX_RXQS);
......
......@@ -2287,6 +2287,8 @@ ice_aq_add_lan_txq(struct ice_hw *hw, u8 num_qgrps,
* @num_qgrps: number of groups in the list
* @qg_list: the list of groups to disable
* @buf_size: the total size of the qg_list buffer in bytes
* @rst_src: if called due to reset, specifies the RST source
* @vmvf_num: the relative VM or VF number that is undergoing the reset
* @cd: pointer to command details structure or NULL
*
* Disable LAN Tx queue (0x0C31)
......@@ -2294,6 +2296,7 @@ ice_aq_add_lan_txq(struct ice_hw *hw, u8 num_qgrps,
static enum ice_status
ice_aq_dis_lan_txq(struct ice_hw *hw, u8 num_qgrps,
struct ice_aqc_dis_txq_item *qg_list, u16 buf_size,
enum ice_disq_rst_src rst_src, u16 vmvf_num,
struct ice_sq_cd *cd)
{
struct ice_aqc_dis_txqs *cmd;
......@@ -2303,14 +2306,45 @@ ice_aq_dis_lan_txq(struct ice_hw *hw, u8 num_qgrps,
cmd = &desc.params.dis_txqs;
ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_dis_txqs);
if (!qg_list)
/* qg_list can be NULL only in VM/VF reset flow */
if (!qg_list && !rst_src)
return ICE_ERR_PARAM;
if (num_qgrps > ICE_LAN_TXQ_MAX_QGRPS)
return ICE_ERR_PARAM;
desc.flags |= cpu_to_le16(ICE_AQ_FLAG_RD);
cmd->num_entries = num_qgrps;
cmd->vmvf_and_timeout = cpu_to_le16((5 << ICE_AQC_Q_DIS_TIMEOUT_S) &
ICE_AQC_Q_DIS_TIMEOUT_M);
switch (rst_src) {
case ICE_VM_RESET:
cmd->cmd_type = ICE_AQC_Q_DIS_CMD_VM_RESET;
cmd->vmvf_and_timeout |=
cpu_to_le16(vmvf_num & ICE_AQC_Q_DIS_VMVF_NUM_M);
break;
case ICE_VF_RESET:
cmd->cmd_type = ICE_AQC_Q_DIS_CMD_VF_RESET;
/* In this case, FW expects vmvf_num to be absolute VF id */
cmd->vmvf_and_timeout |=
cpu_to_le16((vmvf_num + hw->func_caps.vf_base_id) &
ICE_AQC_Q_DIS_VMVF_NUM_M);
break;
case ICE_NO_RESET:
default:
break;
}
/* If no queue group info, we are in a reset flow. Issue the AQ */
if (!qg_list)
goto do_aq;
/* set RD bit to indicate that command buffer is provided by the driver
* and it needs to be read by the firmware
*/
desc.flags |= cpu_to_le16(ICE_AQ_FLAG_RD);
for (i = 0; i < num_qgrps; ++i) {
/* Calculate the size taken up by the queue IDs in this group */
sz += qg_list[i].num_qs * sizeof(qg_list[i].q_id);
......@@ -2326,6 +2360,7 @@ ice_aq_dis_lan_txq(struct ice_hw *hw, u8 num_qgrps,
if (buf_size != sz)
return ICE_ERR_PARAM;
do_aq:
return ice_aq_send_cmd(hw, &desc, qg_list, buf_size, cd);
}
......@@ -2632,13 +2667,16 @@ ice_ena_vsi_txq(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u8 num_qgrps,
* @num_queues: number of queues
* @q_ids: pointer to the q_id array
* @q_teids: pointer to queue node teids
* @rst_src: if called due to reset, specifies the RST source
* @vmvf_num: the relative VM or VF number that is undergoing the reset
* @cd: pointer to command details structure or NULL
*
* This function removes queues and their corresponding nodes in SW DB
*/
enum ice_status
ice_dis_vsi_txq(struct ice_port_info *pi, u8 num_queues, u16 *q_ids,
u32 *q_teids, struct ice_sq_cd *cd)
u32 *q_teids, enum ice_disq_rst_src rst_src, u16 vmvf_num,
struct ice_sq_cd *cd)
{
enum ice_status status = ICE_ERR_DOES_NOT_EXIST;
struct ice_aqc_dis_txq_item qg_list;
......@@ -2647,6 +2685,15 @@ ice_dis_vsi_txq(struct ice_port_info *pi, u8 num_queues, u16 *q_ids,
if (!pi || pi->port_state != ICE_SCHED_PORT_STATE_READY)
return ICE_ERR_CFG;
/* if queue is disabled already yet the disable queue command has to be
* sent to complete the VF reset, then call ice_aq_dis_lan_txq without
* any queue information
*/
if (!num_queues && rst_src)
return ice_aq_dis_lan_txq(pi->hw, 0, NULL, 0, rst_src, vmvf_num,
NULL);
mutex_lock(&pi->sched_lock);
for (i = 0; i < num_queues; i++) {
......@@ -2659,7 +2706,8 @@ ice_dis_vsi_txq(struct ice_port_info *pi, u8 num_queues, u16 *q_ids,
qg_list.num_qs = 1;
qg_list.q_id[0] = cpu_to_le16(q_ids[i]);
status = ice_aq_dis_lan_txq(pi->hw, 1, &qg_list,
sizeof(qg_list), cd);
sizeof(qg_list), rst_src, vmvf_num,
cd);
if (status)
break;
......
......@@ -7,6 +7,7 @@
#include "ice.h"
#include "ice_type.h"
#include "ice_switch.h"
#include <linux/avf/virtchnl.h>
void ice_debug_cq(struct ice_hw *hw, u32 mask, void *desc, void *buf,
u16 buf_len);
......@@ -89,7 +90,8 @@ ice_aq_set_event_mask(struct ice_hw *hw, u8 port_num, u16 mask,
struct ice_sq_cd *cd);
enum ice_status
ice_dis_vsi_txq(struct ice_port_info *pi, u8 num_queues, u16 *q_ids,
u32 *q_teids, struct ice_sq_cd *cmd_details);
u32 *q_teids, enum ice_disq_rst_src rst_src, u16 vmvf_num,
struct ice_sq_cd *cmd_details);
enum ice_status
ice_cfg_vsi_lan(struct ice_port_info *pi, u16 vsi_handle, u8 tc_bitmap,
u16 *max_lanqs);
......
......@@ -90,10 +90,16 @@
#define GLGEN_RTRIG_CORER_M BIT(0)
#define GLGEN_RTRIG_GLOBR_M BIT(1)
#define GLGEN_STAT 0x000B612C
#define GLGEN_VFLRSTAT(_i) (0x00093A04 + ((_i) * 4))
#define PFGEN_CTRL 0x00091000
#define PFGEN_CTRL_PFSWR_M BIT(0)
#define PFGEN_STATE 0x00088000
#define PRTGEN_STATUS 0x000B8100
#define VFGEN_RSTAT(_VF) (0x00074000 + ((_VF) * 4))
#define VPGEN_VFRSTAT(_VF) (0x00090800 + ((_VF) * 4))
#define VPGEN_VFRSTAT_VFRD_M BIT(0)
#define VPGEN_VFRTRIG(_VF) (0x00090000 + ((_VF) * 4))
#define VPGEN_VFRTRIG_VFSWR_M BIT(0)
#define PFHMC_ERRORDATA 0x00520500
#define PFHMC_ERRORINFO 0x00520400
#define GLINT_DYN_CTL(_INT) (0x00160000 + ((_INT) * 4))
......@@ -106,6 +112,13 @@
#define GLINT_ITR(_i, _INT) (0x00154000 + ((_i) * 8192 + (_INT) * 4))
#define GLINT_RATE(_INT) (0x0015A000 + ((_INT) * 4))
#define GLINT_RATE_INTRL_ENA_M BIT(6)
#define GLINT_VECT2FUNC(_INT) (0x00162000 + ((_INT) * 4))
#define GLINT_VECT2FUNC_VF_NUM_S 0
#define GLINT_VECT2FUNC_VF_NUM_M ICE_M(0xFF, 0)
#define GLINT_VECT2FUNC_PF_NUM_S 12
#define GLINT_VECT2FUNC_PF_NUM_M ICE_M(0x7, 12)
#define GLINT_VECT2FUNC_IS_PF_S 16
#define GLINT_VECT2FUNC_IS_PF_M BIT(16)
#define PFINT_FW_CTL 0x0016C800
#define PFINT_FW_CTL_MSIX_INDX_M ICE_M(0x7FF, 0)
#define PFINT_FW_CTL_ITR_INDX_S 11
......@@ -137,6 +150,12 @@
#define QINT_TQCTL_MSIX_INDX_S 0
#define QINT_TQCTL_ITR_INDX_S 11
#define QINT_TQCTL_CAUSE_ENA_M BIT(30)
#define VPINT_ALLOC(_VF) (0x001D1000 + ((_VF) * 4))
#define VPINT_ALLOC_FIRST_S 0
#define VPINT_ALLOC_FIRST_M ICE_M(0x7FF, 0)
#define VPINT_ALLOC_LAST_S 12
#define VPINT_ALLOC_LAST_M ICE_M(0x7FF, 12)
#define VPINT_ALLOC_VALID_M BIT(31)
#define QRX_CONTEXT(_i, _QRX) (0x00280000 + ((_i) * 8192 + (_QRX) * 4))
#define QRX_CTRL(_QRX) (0x00120000 + ((_QRX) * 4))
#define QRX_CTRL_MAX_INDEX 2047
......@@ -149,6 +168,20 @@
#define QRX_TAIL_MAX_INDEX 2047
#define QRX_TAIL_TAIL_S 0
#define QRX_TAIL_TAIL_M ICE_M(0x1FFF, 0)
#define VPLAN_RX_QBASE(_VF) (0x00072000 + ((_VF) * 4))
#define VPLAN_RX_QBASE_VFFIRSTQ_S 0
#define VPLAN_RX_QBASE_VFFIRSTQ_M ICE_M(0x7FF, 0)
#define VPLAN_RX_QBASE_VFNUMQ_S 16
#define VPLAN_RX_QBASE_VFNUMQ_M ICE_M(0xFF, 16)
#define VPLAN_RXQ_MAPENA(_VF) (0x00073000 + ((_VF) * 4))
#define VPLAN_RXQ_MAPENA_RX_ENA_M BIT(0)
#define VPLAN_TX_QBASE(_VF) (0x001D1800 + ((_VF) * 4))
#define VPLAN_TX_QBASE_VFFIRSTQ_S 0
#define VPLAN_TX_QBASE_VFFIRSTQ_M ICE_M(0x3FFF, 0)
#define VPLAN_TX_QBASE_VFNUMQ_S 16
#define VPLAN_TX_QBASE_VFNUMQ_M ICE_M(0xFF, 16)
#define VPLAN_TXQ_MAPENA(_VF) (0x00073800 + ((_VF) * 4))
#define VPLAN_TXQ_MAPENA_TX_ENA_M BIT(0)
#define GL_MDET_RX 0x00294C00
#define GL_MDET_RX_QNUM_S 0
#define GL_MDET_RX_QNUM_M ICE_M(0x7FFF, 0)
......@@ -196,6 +229,9 @@
#define PF_FUNC_RID 0x0009E880
#define PF_FUNC_RID_FUNC_NUM_S 0
#define PF_FUNC_RID_FUNC_NUM_M ICE_M(0x7, 0)
#define PF_PCI_CIAA 0x0009E580
#define PF_PCI_CIAA_VF_NUM_S 12
#define PF_PCI_CIAD 0x0009E500
#define GL_PWR_MODE_CTL 0x000B820C
#define GL_PWR_MODE_CTL_CAR_MAX_BW_S 30
#define GL_PWR_MODE_CTL_CAR_MAX_BW_M ICE_M(0x3, 30)
......@@ -276,5 +312,7 @@
#define GLV_UPTCH(_i) (0x0030A004 + ((_i) * 8))
#define GLV_UPTCL(_i) (0x0030A000 + ((_i) * 8))
#define VSIQF_HKEY_MAX_INDEX 12
#define VFINT_DYN_CTLN(_i) (0x00003800 + ((_i) * 4))
#define VFINT_DYN_CTLN_CLEARPBA_M BIT(1)
#endif /* _ICE_HW_AUTOGEN_H_ */
......@@ -1784,8 +1784,11 @@ int ice_vsi_stop_rx_rings(struct ice_vsi *vsi)
/**
* ice_vsi_stop_tx_rings - Disable Tx rings
* @vsi: the VSI being configured
* @rst_src: reset source
* @rel_vmvf_num: Relative id of VF/VM
*/
int ice_vsi_stop_tx_rings(struct ice_vsi *vsi)
int ice_vsi_stop_tx_rings(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src,
u16 rel_vmvf_num)
{
struct ice_pf *pf = vsi->back;
struct ice_hw *hw = &pf->hw;
......@@ -1837,7 +1840,7 @@ int ice_vsi_stop_tx_rings(struct ice_vsi *vsi)
GLINT_DYN_CTL_SWINT_TRIG_M | GLINT_DYN_CTL_INTENA_MSK_M);
}
status = ice_dis_vsi_txq(vsi->port_info, vsi->num_txq, q_ids, q_teids,
NULL);
rst_src, rel_vmvf_num, NULL);
/* if the disable queue command was exercised during an active reset
* flow, ICE_ERR_RESET_ONGOING is returned. This is not an error as
* the reset operation disables queues at the hardware level anyway.
......
......@@ -31,7 +31,8 @@ int ice_vsi_start_rx_rings(struct ice_vsi *vsi);
int ice_vsi_stop_rx_rings(struct ice_vsi *vsi);
int ice_vsi_stop_tx_rings(struct ice_vsi *vsi);
int ice_vsi_stop_tx_rings(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src,
u16 rel_vmvf_num);
int ice_cfg_vlan_pruning(struct ice_vsi *vsi, bool ena);
......
......@@ -2185,6 +2185,8 @@ static void ice_remove(struct pci_dev *pdev)
set_bit(__ICE_DOWN, pf->state);
ice_service_task_stop(pf);
if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags))
ice_free_vfs(pf);
ice_vsi_release_all(pf);
ice_free_irq_msix_misc(pf);
ice_for_each_vsi(pf, i) {
......@@ -2220,6 +2222,7 @@ static struct pci_driver ice_driver = {
.id_table = ice_pci_tbl,
.probe = ice_probe,
.remove = ice_remove,
.sriov_configure = ice_sriov_configure,
};
/**
......@@ -2955,7 +2958,7 @@ int ice_down(struct ice_vsi *vsi)
}
ice_vsi_dis_irq(vsi);
tx_err = ice_vsi_stop_tx_rings(vsi);
tx_err = ice_vsi_stop_tx_rings(vsi, ICE_NO_RESET, 0);
if (tx_err)
netdev_err(vsi->netdev,
"Failed stop Tx rings, VSI %d error %d\n",
......@@ -3357,6 +3360,7 @@ static void ice_rebuild(struct ice_pf *pf)
goto err_vsi_rebuild;
}
ice_reset_all_vfs(pf, true);
/* if we get here, reset flow is successful */
clear_bit(__ICE_RESET_FAILED, pf->state);
return;
......
......@@ -104,6 +104,15 @@ struct ice_link_status {
u8 module_type[ICE_MODULE_TYPE_TOTAL_BYTE];
};
/* Different reset sources for which a disable queue AQ call has to be made in
* order to clean the TX scheduler as a part of the reset
*/
enum ice_disq_rst_src {
ICE_NO_RESET = 0,
ICE_VM_RESET,
ICE_VF_RESET,
};
/* PHY info such as phy_type, etc... */
struct ice_phy_info {
struct ice_link_status link_info;
......@@ -130,6 +139,7 @@ struct ice_hw_common_caps {
/* Virtualization support */
u8 sr_iov_1_1; /* SR-IOV enabled */
/* RSS related capabilities */
u16 rss_table_size; /* 512 for PFs and 64 for VFs */
u8 rss_table_entry_width; /* RSS Entry width in bits */
......
This diff is collapsed.
/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (c) 2018, Intel Corporation. */
#ifndef _ICE_VIRTCHNL_PF_H_
#define _ICE_VIRTCHNL_PF_H_
#include "ice.h"
/* Static VF transaction/status register def */
#define VF_DEVICE_STATUS 0xAA
#define VF_TRANS_PENDING_M 0x20
/* Specific VF states */
enum ice_vf_states {
ICE_VF_STATE_INIT = 0,
ICE_VF_STATE_ACTIVE,
ICE_VF_STATE_ENA,
ICE_VF_STATE_DIS,
ICE_VF_STATE_MC_PROMISC,
ICE_VF_STATE_UC_PROMISC,
/* state to indicate if PF needs to do vector assignment for VF.
* This needs to be set during first time VF initialization or later
* when VF asks for more Vectors through virtchnl OP.
*/
ICE_VF_STATE_CFG_INTR,
ICE_VF_STATES_NBITS
};
/* VF capabilities */
enum ice_virtchnl_cap {
ICE_VIRTCHNL_VF_CAP_L2 = 0,
ICE_VIRTCHNL_VF_CAP_PRIVILEGE,
};
/* VF information structure */
struct ice_vf {
struct ice_pf *pf;
s16 vf_id; /* VF id in the PF space */
int first_vector_idx; /* first vector index of this VF */
struct ice_sw *vf_sw_id; /* switch id the VF VSIs connect to */
struct virtchnl_ether_addr dflt_lan_addr;
u16 port_vlan_id;
u8 trusted;
u16 lan_vsi_idx; /* index into PF struct */
u16 lan_vsi_num; /* ID as used by firmware */
unsigned long vf_caps; /* vf's adv. capabilities */
DECLARE_BITMAP(vf_states, ICE_VF_STATES_NBITS); /* VF runtime states */
u8 spoofchk;
u16 num_mac;
u16 num_vlan;
};
#ifdef CONFIG_PCI_IOV
int ice_sriov_configure(struct pci_dev *pdev, int num_vfs);
void ice_free_vfs(struct ice_pf *pf);
bool ice_reset_all_vfs(struct ice_pf *pf, bool is_vflr);
#else /* CONFIG_PCI_IOV */
#define ice_free_vfs(pf) do {} while (0)
static inline bool
ice_reset_all_vfs(struct ice_pf __always_unused *pf,
bool __always_unused is_vflr)
{
return true;
}
static inline int
ice_sriov_configure(struct pci_dev __always_unused *pdev,
int __always_unused num_vfs)
{
return -EOPNOTSUPP;
}
#endif /* CONFIG_PCI_IOV */
#endif /* _ICE_VIRTCHNL_PF_H_ */
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment